Files
CenaForge/clipper/tests/test_expressions.py
T
MateuusandClaude Opus 5.5 44f78bd483 Clipador: mapa de expressões do streamer (expressões novas, pose e corpo)
- Expressões novas dos blendshapes que já vinham do MediaPipe: gargalhada, alegria
  (no mapa), medo, espanto, nojo, tristeza, tensão e bocejo (que deixou de contar
  como grito); raiva também por olhos semicerrados + lábios apertados. Canais novos
  opcionais na série (brow_inner, mouth_frown, squint, press, stretch, upper_lip,
  pucker), z relativo ao neutro do próprio streamer só com os segundos que têm o campo.
- Pose da cabeça pela matriz de transformação facial (yaw/pitch/roll) e balanço/aceno
  por zigue-zague; YuNet nos quadros sem rosto dá a caixa da cabeça (streamer de perfil).
- Corpo mesmo sem expressão: balançando, acenando, agitado, chegou perto e saiu do
  quadro, sem contar rastreio perdido como movimento.
- API /face com expr/body por segundo e lista do corpo; linha do juiz com o corpo;
  painel com os rótulos novos e o "Mapa de expressões" no modal do corte.
- Calibrado nas sessões 7, 8, 11 e 15 (doc/clipador/EXPRESSOES.md); intensidade dos
  picos inalterada (canais novos só rotulam). Testes em tests/test_expressions.py.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 04:16:25 -03:00

310 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Mapa de expressões (reaction.py + face.py): expressões novas, pose/corpo e compatibilidade com a série antiga.
Séries sintéticas no formato da live real (doc/clipador/EXPRESSOES.md): neutro com ruído pequeno e eventos injetados.
"""
import json
import math
import random
import numpy as np
import pytest
from app import face as face_mod
from app.reaction import (
EXTRA_SECOND_FIELDS, FaceSample, FaceSecond, ReactionParams, body_moments, channel_z, expression_map, label,
per_second, reaction_moments, seconds_from_json, seconds_to_json, summarize, yawn_seconds, zigzag_legs,
)
from tests.test_face import CAM, make_service, set_cam
from tests.test_sources_and_api import auth, env, seed_session # noqa: F401 - fixture reaproveitada
FPS = 6
def calm(n: int = 300, seed: int = 1, head: bool = True) -> list[FaceSecond]:
"""Neutro como o do skipnho (sessão 8): sorriso baixo, olhos semicerrados ~0,45, lábios um pouco apertados."""
rng = random.Random(seed)
out = []
for t in range(n):
s = FaceSecond(t, 1.0, 0.5, 0.4, 0.3, smile=rng.uniform(0.02, 0.22), jaw=rng.uniform(0.0, 0.02),
surprise=rng.uniform(0.0, 0.004), frown=rng.uniform(0.01, 0.07), brow=rng.uniform(0.02, 0.1),
blink=0.5, motion=rng.uniform(0.02, 0.15))
s.brow_inner = rng.uniform(0.02, 0.1)
s.mouth_frown = rng.uniform(0.0, 0.01)
s.squint = rng.uniform(0.4, 0.5)
s.press = rng.uniform(0.05, 0.12)
s.stretch = rng.uniform(0.0, 0.03)
s.upper_lip = rng.uniform(0.0, 0.02)
s.pucker = rng.uniform(0.0, 0.05)
s.yaw, s.pitch, s.roll = rng.uniform(-3, 3), 12 + rng.uniform(-2, 2), rng.uniform(-2, 2)
s.shake, s.nod = float(rng.choice([0, 0, 1])), float(rng.choice([0, 0, 1]))
if head:
s.head, s.head_motion = 1.0, rng.uniform(0.05, 0.3)
out.append(s)
return out
def expr_at(seconds: list[FaceSecond], t: int) -> str | None:
return expression_map(seconds)[t]["expr"]
# --- expressões novas -------------------------------------------------------------------------------------------
@pytest.mark.parametrize("name,changes", [
("medo", {"brow_inner": 0.7, "brow": 0.7, "stretch": 0.4}),
("tristeza", {"mouth_frown": 0.35, "brow_inner": 0.3}),
("nojo", {"upper_lip": 0.6}),
("espanto", {"pucker": 0.7, "brow": 0.6}),
("tenso", {"press": 0.5, "squint": 0.75}),
("raiva", {"press": 0.5, "squint": 0.75, "frown": 0.15}),
])
def test_each_new_expression_fires_on_its_pattern(name, changes):
seconds = calm()
for key, value in changes.items():
setattr(seconds[200], key, value)
assert expr_at(seconds, 200) == name
# Os vizinhos continuam neutros.
assert expr_at(seconds, 190) == "neutro" and expr_at(seconds, 210) == "neutro"
def test_neutral_has_no_new_expression_nor_body():
emap = expression_map(calm(600, seed=4))
assert {e["expr"] for e in emap} <= {"neutro"}
assert all(e["body"] is None for e in emap)
def test_upper_lip_with_big_smile_is_laughing_not_disgust():
# Sessão 8: todo "nojo" da 1ª rodada era risada de dentes à mostra (sorriso 0,9 já normal dele, lábio de cima 0,4–0,8).
seconds = calm()
for s in seconds:
s.smile = 0.9 + (s.t % 3) * 0.02
seconds[200].upper_lip = 0.6
assert expr_at(seconds, 200) == "risada"
def test_laugh_with_head_thrown_back_is_gargalhada_and_small_smile_is_alegria():
seconds = calm()
seconds[200].smile, seconds[200].pitch = 0.9, -15.0 # 27° acima do neutro dele (+12°)
seconds[250].smile = 0.9
seconds[280].smile = 0.33 # sorriso acima do neutro, mas sem chegar a pico
emap = expression_map(seconds)
assert emap[200]["expr"] == "gargalhada" and emap[250]["expr"] == "risada" and emap[280]["expr"] == "alegria"
def test_yawn_is_not_a_scream_and_never_a_peak():
# Sessão 8: os 5 trechos de jawOpen ≥ 0,6 por 3 s eram bocejos e eram os maiores "gritos" (z 12–16).
seconds = calm()
for t in range(150, 153):
seconds[t].jaw, seconds[t].squint = 0.75, 0.7
seconds[250].jaw = 0.75 # boca escancarada por 1 s: grito
zs = channel_z(seconds)
assert yawn_seconds(seconds, zs, ReactionParams()) == {150, 151, 152}
assert expr_at(seconds, 151) == "bocejo"
assert [(m.peak_s, m.label) for m in reaction_moments(seconds)] == [(250, "grito")]
def test_new_channels_only_label_unless_weighted():
# Peso 0 (padrão): tristeza aparece no mapa, mas não vira pico; com peso, vira.
seconds = calm()
seconds[200].mouth_frown, seconds[200].brow_inner = 0.4, 0.3
assert reaction_moments(seconds) == []
weighted = ReactionParams(extra_weights={"mouth_frown": 1.0})
assert [(m.peak_s, m.label) for m in reaction_moments(seconds, weighted)] == [(200, "tristeza")]
def test_label_without_z_uses_raw_thresholds_for_new_channels():
second = calm(1)[0]
second.upper_lip = 0.6
assert label(second) == "nojo"
# --- corpo / cabeça pela pose -----------------------------------------------------------------------------------
def posed(duration: int, yaw=lambda t: 0.0, pitch=lambda t: 12.0, size=lambda t: 0.3, cx=lambda t: 0.5,
present=lambda t: True, head=lambda t: False) -> list[FaceSample]:
rng = random.Random(7)
out = []
for i in range(duration * FPS):
t = i / FPS
if present(t):
blend = {"mouthSmileLeft": 0.05, "mouthSmileRight": 0.05, "jawOpen": 0.01}
pose = (yaw(t) + rng.uniform(-1, 1), pitch(t) + rng.uniform(-1, 1), rng.uniform(-1, 1))
out.append(FaceSample(t, True, cx(t) + rng.uniform(-0.003, 0.003), 0.4, size(t), blend, pose))
elif head(t):
out.append(FaceSample(t, False, cx(t), 0.4, size(t), head=True))
else:
out.append(FaceSample(t, False))
return out
def window(lo: float, hi: float, fn, rest=0.0):
return lambda t: fn(t) if lo <= t < hi else rest
def test_zigzag_counts_back_and_forth():
assert zigzag_legs([0, 1, -1, 2, 0], 8) == []
assert len(zigzag_legs([0, 20, 0], 8)) == 2 # olhou para o lado e voltou
assert len(zigzag_legs([0, 15, -15, 15, -15], 8)) == 4
def test_head_shake_and_nod_from_pose():
shake = window(100, 103, lambda t: 15 * math.sin(2 * math.pi * 1.2 * t))
nod = window(200, 203, lambda t: 12 + 10 * math.sin(2 * math.pi * 1.2 * t), 12.0)
glance = window(250, 251.5, lambda t: 30 * math.sin(math.pi * (t - 250) / 1.5)) # vira e volta uma vez
seconds = per_second(posed(300, yaw=lambda t: shake(t) + glance(t), pitch=nod), 300)
labels = [(b.label, b.start_s) for b in body_moments(seconds)]
assert any(name == "balançando" and 99 <= s <= 101 for name, s in labels)
assert any(name == "acenando" and 199 <= s <= 202 for name, s in labels)
assert not any(name == "balançando" and 248 <= s <= 252 for name, s in labels)
assert seconds[50].yaw is not None and abs(seconds[50].pitch - 12) < 2
def test_lost_face_is_not_agitation_but_real_movement_is():
# Rastreio piscando: rosto some e volta noutro lugar a cada meio segundo (o "pico" de movimento da live real).
flicker = lambda t: not (100 <= t < 110 and int(t * 2) % 2 == 1) # noqa: E731
jump = window(100, 110, lambda t: 0.3 * (int(t * 2) % 4 == 0), 0.0)
moving = window(200, 206, lambda t: 0.12 * math.sin(2 * math.pi * 2.5 * t))
seconds = per_second(posed(300, present=flicker, cx=lambda t: 0.5 + jump(t) + moving(t)), 300)
agitated = [b for b in body_moments(seconds) if b.label == "agitado"]
assert agitated and all(199 <= b.start_s and b.end_s <= 206 for b in agitated)
def test_leaving_the_frame_and_leaning_in():
gone = lambda t: not (150 <= t < 170) # noqa: E731
seconds = per_second(posed(300, present=gone, size=window(80, 83, lambda t: 0.42, 0.3)), 300)
labels = {(b.label, b.start_s) for b in body_moments(seconds)}
assert ("saiu do quadro", 150) in labels
assert any(name == "chegou perto" and 80 <= s <= 82 for name, s in labels)
def test_head_box_keeps_the_body_when_the_face_is_in_profile():
# Sessão 15 (tfue de perfil): MediaPipe em 10 % dos quadros, YuNet em 100 %. A caixa da cabeça não é "rosto".
in_profile = lambda t: t < 30 # noqa: E731
out = lambda t: not (150 <= t < 160) # noqa: E731
samples = posed(300, present=in_profile, head=out)
seconds = per_second(samples, 300)
assert seconds[100].present == 0.0 and seconds[100].smile is None and seconds[100].head == 1.0
assert seconds[100].hsize == pytest.approx(0.3) and seconds[155].head == 0.0
assert ("saiu do quadro", 150) in {(b.label, b.start_s) for b in body_moments(seconds)}
text = summarize(seconds, 140, 170)
assert text.startswith("rosto sem expressão legível") and "saiu do quadro em 10 s" in text
# --- compatibilidade --------------------------------------------------------------------------------------------
def test_old_series_without_new_fields_still_works():
seconds = calm()
seconds[200].smile = 0.9
data = seconds_to_json(seconds)
for d in data:
for name in EXTRA_SECOND_FIELDS:
d.pop(name, None)
old = seconds_from_json(json.loads(json.dumps(data)))
assert old[0].brow_inner is None and old[0].head is None
assert [m.label for m in reaction_moments(old)] == ["risada"]
assert body_moments(old) == [] # sem `head` não há "saiu do quadro"; sem pose, nem balanço
assert {e["expr"] for e in expression_map(old)} <= {"neutro", "risada"}
def test_service_updated_mid_session_does_not_fake_new_expressions():
# Primeira metade gravada antes (sem os campos novos), segunda com eles: o neutro do canal novo usa só a segunda.
seconds = calm(400)
for s in seconds[:200]:
for name in EXTRA_SECOND_FIELDS:
setattr(s, name, None)
emap = expression_map(seconds)
assert {e["expr"] for e in emap} <= {"neutro"}
assert body_moments(seconds) == []
def test_new_fields_roundtrip_compact():
s = calm(1)[0]
data = s.to_json()
assert set(EXTRA_SECOND_FIELDS) - {"hsize"} <= set(data)
assert data["yaw"] == round(s.yaw, 1) and len(str(data["squint"]).split(".")[-1]) <= 3
back = FaceSecond.from_json(json.loads(json.dumps(data)))
assert back.hsize is None and abs(back.squint - s.squint) < 1e-3
# --- face.py ----------------------------------------------------------------------------------------------------
def test_pose_from_matrix():
a = math.radians(20)
rx = [[1, 0, 0, 0], [0, math.cos(a), -math.sin(a), 0], [0, math.sin(a), math.cos(a), 0], [0, 0, 0, 1]]
yaw, pitch, roll = face_mod.pose_from_matrix(rx)
assert pitch == pytest.approx(20) and abs(yaw) < 1e-6 and abs(roll) < 1e-6
ry = [[math.cos(a), 0, math.sin(a), 0], [0, 1, 0, 0], [-math.sin(a), 0, math.cos(a), 0], [0, 0, 0, 1]]
assert face_mod.pose_from_matrix(ry)[0] == pytest.approx(20)
class FakeDetector:
def __init__(self, result=None, error=None):
self.result, self.error, self.calls = result or [], error, 0
def detect(self, bgr):
self.calls += 1
assert bgr.shape == (40, 80, 3)
if self.error:
raise self.error
return self.result
def tracker_with(detector):
tracker = object.__new__(face_mod.MediaPipeTracker) # sem carregar o MediaPipe: só o caminho da caixa da cabeça
tracker._np, tracker._detector = np, detector
return tracker
def test_tracker_head_box_when_mediapipe_loses_the_face():
rgb = np.zeros((40, 80, 3), np.uint8)
sample = tracker_with(FakeDetector([(10, 4, 20, 20, 0.7), (40, 10, 16, 20, 0.9)]))._head_only(1.5, rgb)
assert sample.present is False and sample.head is True and sample.blend == {}
assert sample.cx == pytest.approx(0.6) and sample.cy == pytest.approx(0.5) and sample.size == pytest.approx(0.5)
assert tracker_with(FakeDetector([]))._head_only(1.5, rgb) == FaceSample(1.5, False)
assert tracker_with(FakeDetector(error=RuntimeError("cv2")))._head_only(1.5, rgb) == FaceSample(1.5, False)
assert tracker_with(None)._head_only(1.5, rgb) == FaceSample(1.5, False)
def test_to_frame_converts_head_size():
second = FaceSecond(0, 0.0, hsize=0.5)
assert face_mod.to_frame([second], CAM)[0].hsize == pytest.approx(0.25)
class PosedTracker:
"""Rosto com pose: sorriso alto em `laughs`; balanço da cabeça em `shakes`."""
name = "fake"
def __init__(self, laughs=(), shakes=()):
self.laughs, self.shakes = set(laughs), set(shakes)
def process(self, t, rgb):
smile = 0.9 if int(t) in self.laughs else 0.05 + 0.05 * ((int(t * 6) % 3) / 2)
yaw = 15 * math.sin(2 * math.pi * 1.2 * t) if int(t) in self.shakes else 0.0
return FaceSample(t, True, 0.5, 0.4, 0.3, {"mouthSmileLeft": smile, "mouthSmileRight": smile},
(yaw, 12.0, 0.0))
def close(self):
pass
def test_face_api_has_the_expression_map_and_body(env): # noqa: F811
client, engine = env
service = make_service(engine, tracker_factory=lambda _b=None: PosedTracker(range(300, 306), range(310, 313)))
sid = seed_session(engine)
set_cam(engine, sid)
while service.step():
pass
body = client.get(f"/v1/sessions/{sid}/face?from=290&to=320", headers=auth()).json()
by_t = {s["t"]: s for s in body["seconds"]}
assert by_t[302]["expr"] == "risada" and by_t[295]["expr"] == "neutro"
assert by_t[311]["body"] == "balançando" and "body" not in by_t[295]
assert "yaw" in by_t[295] and by_t[295]["head"] == 1.0
assert [b["label"] for b in body["body"]] == ["balançando"]
line = service.summary(engine.db.get("SELECT * FROM sessions WHERE id = ?", sid), 295, 315)
assert "risada" in line and "balançando a cabeça em 3" in line