Clipador: mapa de expressões do streamer (expressões novas, pose e corpo)

- Expressões novas dos blendshapes que já vinham do MediaPipe: gargalhada, alegria
  (no mapa), medo, espanto, nojo, tristeza, tensão e bocejo (que deixou de contar
  como grito); raiva também por olhos semicerrados + lábios apertados. Canais novos
  opcionais na série (brow_inner, mouth_frown, squint, press, stretch, upper_lip,
  pucker), z relativo ao neutro do próprio streamer só com os segundos que têm o campo.
- Pose da cabeça pela matriz de transformação facial (yaw/pitch/roll) e balanço/aceno
  por zigue-zague; YuNet nos quadros sem rosto dá a caixa da cabeça (streamer de perfil).
- Corpo mesmo sem expressão: balançando, acenando, agitado, chegou perto e saiu do
  quadro, sem contar rastreio perdido como movimento.
- API /face com expr/body por segundo e lista do corpo; linha do juiz com o corpo;
  painel com os rótulos novos e o "Mapa de expressões" no modal do corte.
- Calibrado nas sessões 7, 8, 11 e 15 (doc/clipador/EXPRESSOES.md); intensidade dos
  picos inalterada (canais novos só rotulam). Testes em tests/test_expressions.py.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mateuus
2026-11-18 04:16:25 -03:00
co-authored by Claude Opus 5.5
parent ad36f0faf4
commit 44f78bd483
7 changed files with 1207 additions and 36 deletions
+309
View File
@@ -0,0 +1,309 @@
"""Mapa de expressões (reaction.py + face.py): expressões novas, pose/corpo e compatibilidade com a série antiga.
Séries sintéticas no formato da live real (doc/clipador/EXPRESSOES.md): neutro com ruído pequeno e eventos injetados.
"""
import json
import math
import random
import numpy as np
import pytest
from app import face as face_mod
from app.reaction import (
EXTRA_SECOND_FIELDS, FaceSample, FaceSecond, ReactionParams, body_moments, channel_z, expression_map, label,
per_second, reaction_moments, seconds_from_json, seconds_to_json, summarize, yawn_seconds, zigzag_legs,
)
from tests.test_face import CAM, make_service, set_cam
from tests.test_sources_and_api import auth, env, seed_session # noqa: F401 - fixture reaproveitada
FPS = 6
def calm(n: int = 300, seed: int = 1, head: bool = True) -> list[FaceSecond]:
"""Neutro como o do skipnho (sessão 8): sorriso baixo, olhos semicerrados ~0,45, lábios um pouco apertados."""
rng = random.Random(seed)
out = []
for t in range(n):
s = FaceSecond(t, 1.0, 0.5, 0.4, 0.3, smile=rng.uniform(0.02, 0.22), jaw=rng.uniform(0.0, 0.02),
surprise=rng.uniform(0.0, 0.004), frown=rng.uniform(0.01, 0.07), brow=rng.uniform(0.02, 0.1),
blink=0.5, motion=rng.uniform(0.02, 0.15))
s.brow_inner = rng.uniform(0.02, 0.1)
s.mouth_frown = rng.uniform(0.0, 0.01)
s.squint = rng.uniform(0.4, 0.5)
s.press = rng.uniform(0.05, 0.12)
s.stretch = rng.uniform(0.0, 0.03)
s.upper_lip = rng.uniform(0.0, 0.02)
s.pucker = rng.uniform(0.0, 0.05)
s.yaw, s.pitch, s.roll = rng.uniform(-3, 3), 12 + rng.uniform(-2, 2), rng.uniform(-2, 2)
s.shake, s.nod = float(rng.choice([0, 0, 1])), float(rng.choice([0, 0, 1]))
if head:
s.head, s.head_motion = 1.0, rng.uniform(0.05, 0.3)
out.append(s)
return out
def expr_at(seconds: list[FaceSecond], t: int) -> str | None:
return expression_map(seconds)[t]["expr"]
# --- expressões novas -------------------------------------------------------------------------------------------
@pytest.mark.parametrize("name,changes", [
("medo", {"brow_inner": 0.7, "brow": 0.7, "stretch": 0.4}),
("tristeza", {"mouth_frown": 0.35, "brow_inner": 0.3}),
("nojo", {"upper_lip": 0.6}),
("espanto", {"pucker": 0.7, "brow": 0.6}),
("tenso", {"press": 0.5, "squint": 0.75}),
("raiva", {"press": 0.5, "squint": 0.75, "frown": 0.15}),
])
def test_each_new_expression_fires_on_its_pattern(name, changes):
seconds = calm()
for key, value in changes.items():
setattr(seconds[200], key, value)
assert expr_at(seconds, 200) == name
# Os vizinhos continuam neutros.
assert expr_at(seconds, 190) == "neutro" and expr_at(seconds, 210) == "neutro"
def test_neutral_has_no_new_expression_nor_body():
emap = expression_map(calm(600, seed=4))
assert {e["expr"] for e in emap} <= {"neutro"}
assert all(e["body"] is None for e in emap)
def test_upper_lip_with_big_smile_is_laughing_not_disgust():
# Sessão 8: todo "nojo" da 1ª rodada era risada de dentes à mostra (sorriso 0,9 já normal dele, lábio de cima 0,4–0,8).
seconds = calm()
for s in seconds:
s.smile = 0.9 + (s.t % 3) * 0.02
seconds[200].upper_lip = 0.6
assert expr_at(seconds, 200) == "risada"
def test_laugh_with_head_thrown_back_is_gargalhada_and_small_smile_is_alegria():
seconds = calm()
seconds[200].smile, seconds[200].pitch = 0.9, -15.0 # 27° acima do neutro dele (+12°)
seconds[250].smile = 0.9
seconds[280].smile = 0.33 # sorriso acima do neutro, mas sem chegar a pico
emap = expression_map(seconds)
assert emap[200]["expr"] == "gargalhada" and emap[250]["expr"] == "risada" and emap[280]["expr"] == "alegria"
def test_yawn_is_not_a_scream_and_never_a_peak():
# Sessão 8: os 5 trechos de jawOpen ≥ 0,6 por 3 s eram bocejos e eram os maiores "gritos" (z 12–16).
seconds = calm()
for t in range(150, 153):
seconds[t].jaw, seconds[t].squint = 0.75, 0.7
seconds[250].jaw = 0.75 # boca escancarada por 1 s: grito
zs = channel_z(seconds)
assert yawn_seconds(seconds, zs, ReactionParams()) == {150, 151, 152}
assert expr_at(seconds, 151) == "bocejo"
assert [(m.peak_s, m.label) for m in reaction_moments(seconds)] == [(250, "grito")]
def test_new_channels_only_label_unless_weighted():
# Peso 0 (padrão): tristeza aparece no mapa, mas não vira pico; com peso, vira.
seconds = calm()
seconds[200].mouth_frown, seconds[200].brow_inner = 0.4, 0.3
assert reaction_moments(seconds) == []
weighted = ReactionParams(extra_weights={"mouth_frown": 1.0})
assert [(m.peak_s, m.label) for m in reaction_moments(seconds, weighted)] == [(200, "tristeza")]
def test_label_without_z_uses_raw_thresholds_for_new_channels():
second = calm(1)[0]
second.upper_lip = 0.6
assert label(second) == "nojo"
# --- corpo / cabeça pela pose -----------------------------------------------------------------------------------
def posed(duration: int, yaw=lambda t: 0.0, pitch=lambda t: 12.0, size=lambda t: 0.3, cx=lambda t: 0.5,
present=lambda t: True, head=lambda t: False) -> list[FaceSample]:
rng = random.Random(7)
out = []
for i in range(duration * FPS):
t = i / FPS
if present(t):
blend = {"mouthSmileLeft": 0.05, "mouthSmileRight": 0.05, "jawOpen": 0.01}
pose = (yaw(t) + rng.uniform(-1, 1), pitch(t) + rng.uniform(-1, 1), rng.uniform(-1, 1))
out.append(FaceSample(t, True, cx(t) + rng.uniform(-0.003, 0.003), 0.4, size(t), blend, pose))
elif head(t):
out.append(FaceSample(t, False, cx(t), 0.4, size(t), head=True))
else:
out.append(FaceSample(t, False))
return out
def window(lo: float, hi: float, fn, rest=0.0):
return lambda t: fn(t) if lo <= t < hi else rest
def test_zigzag_counts_back_and_forth():
assert zigzag_legs([0, 1, -1, 2, 0], 8) == []
assert len(zigzag_legs([0, 20, 0], 8)) == 2 # olhou para o lado e voltou
assert len(zigzag_legs([0, 15, -15, 15, -15], 8)) == 4
def test_head_shake_and_nod_from_pose():
shake = window(100, 103, lambda t: 15 * math.sin(2 * math.pi * 1.2 * t))
nod = window(200, 203, lambda t: 12 + 10 * math.sin(2 * math.pi * 1.2 * t), 12.0)
glance = window(250, 251.5, lambda t: 30 * math.sin(math.pi * (t - 250) / 1.5)) # vira e volta uma vez
seconds = per_second(posed(300, yaw=lambda t: shake(t) + glance(t), pitch=nod), 300)
labels = [(b.label, b.start_s) for b in body_moments(seconds)]
assert any(name == "balançando" and 99 <= s <= 101 for name, s in labels)
assert any(name == "acenando" and 199 <= s <= 202 for name, s in labels)
assert not any(name == "balançando" and 248 <= s <= 252 for name, s in labels)
assert seconds[50].yaw is not None and abs(seconds[50].pitch - 12) < 2
def test_lost_face_is_not_agitation_but_real_movement_is():
# Rastreio piscando: rosto some e volta noutro lugar a cada meio segundo (o "pico" de movimento da live real).
flicker = lambda t: not (100 <= t < 110 and int(t * 2) % 2 == 1) # noqa: E731
jump = window(100, 110, lambda t: 0.3 * (int(t * 2) % 4 == 0), 0.0)
moving = window(200, 206, lambda t: 0.12 * math.sin(2 * math.pi * 2.5 * t))
seconds = per_second(posed(300, present=flicker, cx=lambda t: 0.5 + jump(t) + moving(t)), 300)
agitated = [b for b in body_moments(seconds) if b.label == "agitado"]
assert agitated and all(199 <= b.start_s and b.end_s <= 206 for b in agitated)
def test_leaving_the_frame_and_leaning_in():
gone = lambda t: not (150 <= t < 170) # noqa: E731
seconds = per_second(posed(300, present=gone, size=window(80, 83, lambda t: 0.42, 0.3)), 300)
labels = {(b.label, b.start_s) for b in body_moments(seconds)}
assert ("saiu do quadro", 150) in labels
assert any(name == "chegou perto" and 80 <= s <= 82 for name, s in labels)
def test_head_box_keeps_the_body_when_the_face_is_in_profile():
# Sessão 15 (tfue de perfil): MediaPipe em 10 % dos quadros, YuNet em 100 %. A caixa da cabeça não é "rosto".
in_profile = lambda t: t < 30 # noqa: E731
out = lambda t: not (150 <= t < 160) # noqa: E731
samples = posed(300, present=in_profile, head=out)
seconds = per_second(samples, 300)
assert seconds[100].present == 0.0 and seconds[100].smile is None and seconds[100].head == 1.0
assert seconds[100].hsize == pytest.approx(0.3) and seconds[155].head == 0.0
assert ("saiu do quadro", 150) in {(b.label, b.start_s) for b in body_moments(seconds)}
text = summarize(seconds, 140, 170)
assert text.startswith("rosto sem expressão legível") and "saiu do quadro em 10 s" in text
# --- compatibilidade --------------------------------------------------------------------------------------------
def test_old_series_without_new_fields_still_works():
seconds = calm()
seconds[200].smile = 0.9
data = seconds_to_json(seconds)
for d in data:
for name in EXTRA_SECOND_FIELDS:
d.pop(name, None)
old = seconds_from_json(json.loads(json.dumps(data)))
assert old[0].brow_inner is None and old[0].head is None
assert [m.label for m in reaction_moments(old)] == ["risada"]
assert body_moments(old) == [] # sem `head` não há "saiu do quadro"; sem pose, nem balanço
assert {e["expr"] for e in expression_map(old)} <= {"neutro", "risada"}
def test_service_updated_mid_session_does_not_fake_new_expressions():
# Primeira metade gravada antes (sem os campos novos), segunda com eles: o neutro do canal novo usa só a segunda.
seconds = calm(400)
for s in seconds[:200]:
for name in EXTRA_SECOND_FIELDS:
setattr(s, name, None)
emap = expression_map(seconds)
assert {e["expr"] for e in emap} <= {"neutro"}
assert body_moments(seconds) == []
def test_new_fields_roundtrip_compact():
s = calm(1)[0]
data = s.to_json()
assert set(EXTRA_SECOND_FIELDS) - {"hsize"} <= set(data)
assert data["yaw"] == round(s.yaw, 1) and len(str(data["squint"]).split(".")[-1]) <= 3
back = FaceSecond.from_json(json.loads(json.dumps(data)))
assert back.hsize is None and abs(back.squint - s.squint) < 1e-3
# --- face.py ----------------------------------------------------------------------------------------------------
def test_pose_from_matrix():
a = math.radians(20)
rx = [[1, 0, 0, 0], [0, math.cos(a), -math.sin(a), 0], [0, math.sin(a), math.cos(a), 0], [0, 0, 0, 1]]
yaw, pitch, roll = face_mod.pose_from_matrix(rx)
assert pitch == pytest.approx(20) and abs(yaw) < 1e-6 and abs(roll) < 1e-6
ry = [[math.cos(a), 0, math.sin(a), 0], [0, 1, 0, 0], [-math.sin(a), 0, math.cos(a), 0], [0, 0, 0, 1]]
assert face_mod.pose_from_matrix(ry)[0] == pytest.approx(20)
class FakeDetector:
def __init__(self, result=None, error=None):
self.result, self.error, self.calls = result or [], error, 0
def detect(self, bgr):
self.calls += 1
assert bgr.shape == (40, 80, 3)
if self.error:
raise self.error
return self.result
def tracker_with(detector):
tracker = object.__new__(face_mod.MediaPipeTracker) # sem carregar o MediaPipe: só o caminho da caixa da cabeça
tracker._np, tracker._detector = np, detector
return tracker
def test_tracker_head_box_when_mediapipe_loses_the_face():
rgb = np.zeros((40, 80, 3), np.uint8)
sample = tracker_with(FakeDetector([(10, 4, 20, 20, 0.7), (40, 10, 16, 20, 0.9)]))._head_only(1.5, rgb)
assert sample.present is False and sample.head is True and sample.blend == {}
assert sample.cx == pytest.approx(0.6) and sample.cy == pytest.approx(0.5) and sample.size == pytest.approx(0.5)
assert tracker_with(FakeDetector([]))._head_only(1.5, rgb) == FaceSample(1.5, False)
assert tracker_with(FakeDetector(error=RuntimeError("cv2")))._head_only(1.5, rgb) == FaceSample(1.5, False)
assert tracker_with(None)._head_only(1.5, rgb) == FaceSample(1.5, False)
def test_to_frame_converts_head_size():
second = FaceSecond(0, 0.0, hsize=0.5)
assert face_mod.to_frame([second], CAM)[0].hsize == pytest.approx(0.25)
class PosedTracker:
"""Rosto com pose: sorriso alto em `laughs`; balanço da cabeça em `shakes`."""
name = "fake"
def __init__(self, laughs=(), shakes=()):
self.laughs, self.shakes = set(laughs), set(shakes)
def process(self, t, rgb):
smile = 0.9 if int(t) in self.laughs else 0.05 + 0.05 * ((int(t * 6) % 3) / 2)
yaw = 15 * math.sin(2 * math.pi * 1.2 * t) if int(t) in self.shakes else 0.0
return FaceSample(t, True, 0.5, 0.4, 0.3, {"mouthSmileLeft": smile, "mouthSmileRight": smile},
(yaw, 12.0, 0.0))
def close(self):
pass
def test_face_api_has_the_expression_map_and_body(env): # noqa: F811
client, engine = env
service = make_service(engine, tracker_factory=lambda _b=None: PosedTracker(range(300, 306), range(310, 313)))
sid = seed_session(engine)
set_cam(engine, sid)
while service.step():
pass
body = client.get(f"/v1/sessions/{sid}/face?from=290&to=320", headers=auth()).json()
by_t = {s["t"]: s for s in body["seconds"]}
assert by_t[302]["expr"] == "risada" and by_t[295]["expr"] == "neutro"
assert by_t[311]["body"] == "balançando" and "body" not in by_t[295]
assert "yaw" in by_t[295] and by_t[295]["head"] == 1.0
assert [b["label"] for b in body["body"]] == ["balançando"]
line = service.summary(engine.db.get("SELECT * FROM sessions WHERE id = ?", sid), 295, 315)
assert "risada" in line and "balançando a cabeça em 3" in line