- Expressões novas dos blendshapes que já vinham do MediaPipe: gargalhada, alegria (no mapa), medo, espanto, nojo, tristeza, tensão e bocejo (que deixou de contar como grito); raiva também por olhos semicerrados + lábios apertados. Canais novos opcionais na série (brow_inner, mouth_frown, squint, press, stretch, upper_lip, pucker), z relativo ao neutro do próprio streamer só com os segundos que têm o campo. - Pose da cabeça pela matriz de transformação facial (yaw/pitch/roll) e balanço/aceno por zigue-zague; YuNet nos quadros sem rosto dá a caixa da cabeça (streamer de perfil). - Corpo mesmo sem expressão: balançando, acenando, agitado, chegou perto e saiu do quadro, sem contar rastreio perdido como movimento. - API /face com expr/body por segundo e lista do corpo; linha do juiz com o corpo; painel com os rótulos novos e o "Mapa de expressões" no modal do corte. - Calibrado nas sessões 7, 8, 11 e 15 (doc/clipador/EXPRESSOES.md); intensidade dos picos inalterada (canais novos só rotulam). Testes em tests/test_expressions.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
310 lines
14 KiB
Python
310 lines
14 KiB
Python
"""Mapa de expressões (reaction.py + face.py): expressões novas, pose/corpo e compatibilidade com a série antiga.
|
||
|
||
Séries sintéticas no formato da live real (doc/clipador/EXPRESSOES.md): neutro com ruído pequeno e eventos injetados.
|
||
"""
|
||
|
||
import json
|
||
import math
|
||
import random
|
||
|
||
import numpy as np
|
||
import pytest
|
||
|
||
from app import face as face_mod
|
||
from app.reaction import (
|
||
EXTRA_SECOND_FIELDS, FaceSample, FaceSecond, ReactionParams, body_moments, channel_z, expression_map, label,
|
||
per_second, reaction_moments, seconds_from_json, seconds_to_json, summarize, yawn_seconds, zigzag_legs,
|
||
)
|
||
from tests.test_face import CAM, make_service, set_cam
|
||
from tests.test_sources_and_api import auth, env, seed_session # noqa: F401 - fixture reaproveitada
|
||
|
||
FPS = 6
|
||
|
||
|
||
def calm(n: int = 300, seed: int = 1, head: bool = True) -> list[FaceSecond]:
|
||
"""Neutro como o do skipnho (sessão 8): sorriso baixo, olhos semicerrados ~0,45, lábios um pouco apertados."""
|
||
rng = random.Random(seed)
|
||
out = []
|
||
for t in range(n):
|
||
s = FaceSecond(t, 1.0, 0.5, 0.4, 0.3, smile=rng.uniform(0.02, 0.22), jaw=rng.uniform(0.0, 0.02),
|
||
surprise=rng.uniform(0.0, 0.004), frown=rng.uniform(0.01, 0.07), brow=rng.uniform(0.02, 0.1),
|
||
blink=0.5, motion=rng.uniform(0.02, 0.15))
|
||
s.brow_inner = rng.uniform(0.02, 0.1)
|
||
s.mouth_frown = rng.uniform(0.0, 0.01)
|
||
s.squint = rng.uniform(0.4, 0.5)
|
||
s.press = rng.uniform(0.05, 0.12)
|
||
s.stretch = rng.uniform(0.0, 0.03)
|
||
s.upper_lip = rng.uniform(0.0, 0.02)
|
||
s.pucker = rng.uniform(0.0, 0.05)
|
||
s.yaw, s.pitch, s.roll = rng.uniform(-3, 3), 12 + rng.uniform(-2, 2), rng.uniform(-2, 2)
|
||
s.shake, s.nod = float(rng.choice([0, 0, 1])), float(rng.choice([0, 0, 1]))
|
||
if head:
|
||
s.head, s.head_motion = 1.0, rng.uniform(0.05, 0.3)
|
||
out.append(s)
|
||
return out
|
||
|
||
|
||
def expr_at(seconds: list[FaceSecond], t: int) -> str | None:
|
||
return expression_map(seconds)[t]["expr"]
|
||
|
||
|
||
# --- expressões novas -------------------------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.parametrize("name,changes", [
|
||
("medo", {"brow_inner": 0.7, "brow": 0.7, "stretch": 0.4}),
|
||
("tristeza", {"mouth_frown": 0.35, "brow_inner": 0.3}),
|
||
("nojo", {"upper_lip": 0.6}),
|
||
("espanto", {"pucker": 0.7, "brow": 0.6}),
|
||
("tenso", {"press": 0.5, "squint": 0.75}),
|
||
("raiva", {"press": 0.5, "squint": 0.75, "frown": 0.15}),
|
||
])
|
||
def test_each_new_expression_fires_on_its_pattern(name, changes):
|
||
seconds = calm()
|
||
for key, value in changes.items():
|
||
setattr(seconds[200], key, value)
|
||
assert expr_at(seconds, 200) == name
|
||
# Os vizinhos continuam neutros.
|
||
assert expr_at(seconds, 190) == "neutro" and expr_at(seconds, 210) == "neutro"
|
||
|
||
|
||
def test_neutral_has_no_new_expression_nor_body():
|
||
emap = expression_map(calm(600, seed=4))
|
||
assert {e["expr"] for e in emap} <= {"neutro"}
|
||
assert all(e["body"] is None for e in emap)
|
||
|
||
|
||
def test_upper_lip_with_big_smile_is_laughing_not_disgust():
|
||
# Sessão 8: todo "nojo" da 1ª rodada era risada de dentes à mostra (sorriso 0,9 já normal dele, lábio de cima 0,4–0,8).
|
||
seconds = calm()
|
||
for s in seconds:
|
||
s.smile = 0.9 + (s.t % 3) * 0.02
|
||
seconds[200].upper_lip = 0.6
|
||
assert expr_at(seconds, 200) == "risada"
|
||
|
||
|
||
def test_laugh_with_head_thrown_back_is_gargalhada_and_small_smile_is_alegria():
|
||
seconds = calm()
|
||
seconds[200].smile, seconds[200].pitch = 0.9, -15.0 # 27° acima do neutro dele (+12°)
|
||
seconds[250].smile = 0.9
|
||
seconds[280].smile = 0.33 # sorriso acima do neutro, mas sem chegar a pico
|
||
emap = expression_map(seconds)
|
||
assert emap[200]["expr"] == "gargalhada" and emap[250]["expr"] == "risada" and emap[280]["expr"] == "alegria"
|
||
|
||
|
||
def test_yawn_is_not_a_scream_and_never_a_peak():
|
||
# Sessão 8: os 5 trechos de jawOpen ≥ 0,6 por 3 s eram bocejos e eram os maiores "gritos" (z 12–16).
|
||
seconds = calm()
|
||
for t in range(150, 153):
|
||
seconds[t].jaw, seconds[t].squint = 0.75, 0.7
|
||
seconds[250].jaw = 0.75 # boca escancarada por 1 s: grito
|
||
zs = channel_z(seconds)
|
||
assert yawn_seconds(seconds, zs, ReactionParams()) == {150, 151, 152}
|
||
assert expr_at(seconds, 151) == "bocejo"
|
||
assert [(m.peak_s, m.label) for m in reaction_moments(seconds)] == [(250, "grito")]
|
||
|
||
|
||
def test_new_channels_only_label_unless_weighted():
|
||
# Peso 0 (padrão): tristeza aparece no mapa, mas não vira pico; com peso, vira.
|
||
seconds = calm()
|
||
seconds[200].mouth_frown, seconds[200].brow_inner = 0.4, 0.3
|
||
assert reaction_moments(seconds) == []
|
||
weighted = ReactionParams(extra_weights={"mouth_frown": 1.0})
|
||
assert [(m.peak_s, m.label) for m in reaction_moments(seconds, weighted)] == [(200, "tristeza")]
|
||
|
||
|
||
def test_label_without_z_uses_raw_thresholds_for_new_channels():
|
||
second = calm(1)[0]
|
||
second.upper_lip = 0.6
|
||
assert label(second) == "nojo"
|
||
|
||
|
||
# --- corpo / cabeça pela pose -----------------------------------------------------------------------------------
|
||
|
||
|
||
def posed(duration: int, yaw=lambda t: 0.0, pitch=lambda t: 12.0, size=lambda t: 0.3, cx=lambda t: 0.5,
|
||
present=lambda t: True, head=lambda t: False) -> list[FaceSample]:
|
||
rng = random.Random(7)
|
||
out = []
|
||
for i in range(duration * FPS):
|
||
t = i / FPS
|
||
if present(t):
|
||
blend = {"mouthSmileLeft": 0.05, "mouthSmileRight": 0.05, "jawOpen": 0.01}
|
||
pose = (yaw(t) + rng.uniform(-1, 1), pitch(t) + rng.uniform(-1, 1), rng.uniform(-1, 1))
|
||
out.append(FaceSample(t, True, cx(t) + rng.uniform(-0.003, 0.003), 0.4, size(t), blend, pose))
|
||
elif head(t):
|
||
out.append(FaceSample(t, False, cx(t), 0.4, size(t), head=True))
|
||
else:
|
||
out.append(FaceSample(t, False))
|
||
return out
|
||
|
||
|
||
def window(lo: float, hi: float, fn, rest=0.0):
|
||
return lambda t: fn(t) if lo <= t < hi else rest
|
||
|
||
|
||
def test_zigzag_counts_back_and_forth():
|
||
assert zigzag_legs([0, 1, -1, 2, 0], 8) == []
|
||
assert len(zigzag_legs([0, 20, 0], 8)) == 2 # olhou para o lado e voltou
|
||
assert len(zigzag_legs([0, 15, -15, 15, -15], 8)) == 4
|
||
|
||
|
||
def test_head_shake_and_nod_from_pose():
|
||
shake = window(100, 103, lambda t: 15 * math.sin(2 * math.pi * 1.2 * t))
|
||
nod = window(200, 203, lambda t: 12 + 10 * math.sin(2 * math.pi * 1.2 * t), 12.0)
|
||
glance = window(250, 251.5, lambda t: 30 * math.sin(math.pi * (t - 250) / 1.5)) # vira e volta uma vez
|
||
seconds = per_second(posed(300, yaw=lambda t: shake(t) + glance(t), pitch=nod), 300)
|
||
labels = [(b.label, b.start_s) for b in body_moments(seconds)]
|
||
assert any(name == "balançando" and 99 <= s <= 101 for name, s in labels)
|
||
assert any(name == "acenando" and 199 <= s <= 202 for name, s in labels)
|
||
assert not any(name == "balançando" and 248 <= s <= 252 for name, s in labels)
|
||
assert seconds[50].yaw is not None and abs(seconds[50].pitch - 12) < 2
|
||
|
||
|
||
def test_lost_face_is_not_agitation_but_real_movement_is():
|
||
# Rastreio piscando: rosto some e volta noutro lugar a cada meio segundo (o "pico" de movimento da live real).
|
||
flicker = lambda t: not (100 <= t < 110 and int(t * 2) % 2 == 1) # noqa: E731
|
||
jump = window(100, 110, lambda t: 0.3 * (int(t * 2) % 4 == 0), 0.0)
|
||
moving = window(200, 206, lambda t: 0.12 * math.sin(2 * math.pi * 2.5 * t))
|
||
seconds = per_second(posed(300, present=flicker, cx=lambda t: 0.5 + jump(t) + moving(t)), 300)
|
||
agitated = [b for b in body_moments(seconds) if b.label == "agitado"]
|
||
assert agitated and all(199 <= b.start_s and b.end_s <= 206 for b in agitated)
|
||
|
||
|
||
def test_leaving_the_frame_and_leaning_in():
|
||
gone = lambda t: not (150 <= t < 170) # noqa: E731
|
||
seconds = per_second(posed(300, present=gone, size=window(80, 83, lambda t: 0.42, 0.3)), 300)
|
||
labels = {(b.label, b.start_s) for b in body_moments(seconds)}
|
||
assert ("saiu do quadro", 150) in labels
|
||
assert any(name == "chegou perto" and 80 <= s <= 82 for name, s in labels)
|
||
|
||
|
||
def test_head_box_keeps_the_body_when_the_face_is_in_profile():
|
||
# Sessão 15 (tfue de perfil): MediaPipe em 10 % dos quadros, YuNet em 100 %. A caixa da cabeça não é "rosto".
|
||
in_profile = lambda t: t < 30 # noqa: E731
|
||
out = lambda t: not (150 <= t < 160) # noqa: E731
|
||
samples = posed(300, present=in_profile, head=out)
|
||
seconds = per_second(samples, 300)
|
||
assert seconds[100].present == 0.0 and seconds[100].smile is None and seconds[100].head == 1.0
|
||
assert seconds[100].hsize == pytest.approx(0.3) and seconds[155].head == 0.0
|
||
assert ("saiu do quadro", 150) in {(b.label, b.start_s) for b in body_moments(seconds)}
|
||
text = summarize(seconds, 140, 170)
|
||
assert text.startswith("rosto sem expressão legível") and "saiu do quadro em 10 s" in text
|
||
|
||
|
||
# --- compatibilidade --------------------------------------------------------------------------------------------
|
||
|
||
|
||
def test_old_series_without_new_fields_still_works():
|
||
seconds = calm()
|
||
seconds[200].smile = 0.9
|
||
data = seconds_to_json(seconds)
|
||
for d in data:
|
||
for name in EXTRA_SECOND_FIELDS:
|
||
d.pop(name, None)
|
||
old = seconds_from_json(json.loads(json.dumps(data)))
|
||
assert old[0].brow_inner is None and old[0].head is None
|
||
assert [m.label for m in reaction_moments(old)] == ["risada"]
|
||
assert body_moments(old) == [] # sem `head` não há "saiu do quadro"; sem pose, nem balanço
|
||
assert {e["expr"] for e in expression_map(old)} <= {"neutro", "risada"}
|
||
|
||
|
||
def test_service_updated_mid_session_does_not_fake_new_expressions():
|
||
# Primeira metade gravada antes (sem os campos novos), segunda com eles: o neutro do canal novo usa só a segunda.
|
||
seconds = calm(400)
|
||
for s in seconds[:200]:
|
||
for name in EXTRA_SECOND_FIELDS:
|
||
setattr(s, name, None)
|
||
emap = expression_map(seconds)
|
||
assert {e["expr"] for e in emap} <= {"neutro"}
|
||
assert body_moments(seconds) == []
|
||
|
||
|
||
def test_new_fields_roundtrip_compact():
|
||
s = calm(1)[0]
|
||
data = s.to_json()
|
||
assert set(EXTRA_SECOND_FIELDS) - {"hsize"} <= set(data)
|
||
assert data["yaw"] == round(s.yaw, 1) and len(str(data["squint"]).split(".")[-1]) <= 3
|
||
back = FaceSecond.from_json(json.loads(json.dumps(data)))
|
||
assert back.hsize is None and abs(back.squint - s.squint) < 1e-3
|
||
|
||
|
||
# --- face.py ----------------------------------------------------------------------------------------------------
|
||
|
||
|
||
def test_pose_from_matrix():
|
||
a = math.radians(20)
|
||
rx = [[1, 0, 0, 0], [0, math.cos(a), -math.sin(a), 0], [0, math.sin(a), math.cos(a), 0], [0, 0, 0, 1]]
|
||
yaw, pitch, roll = face_mod.pose_from_matrix(rx)
|
||
assert pitch == pytest.approx(20) and abs(yaw) < 1e-6 and abs(roll) < 1e-6
|
||
ry = [[math.cos(a), 0, math.sin(a), 0], [0, 1, 0, 0], [-math.sin(a), 0, math.cos(a), 0], [0, 0, 0, 1]]
|
||
assert face_mod.pose_from_matrix(ry)[0] == pytest.approx(20)
|
||
|
||
|
||
class FakeDetector:
|
||
def __init__(self, result=None, error=None):
|
||
self.result, self.error, self.calls = result or [], error, 0
|
||
|
||
def detect(self, bgr):
|
||
self.calls += 1
|
||
assert bgr.shape == (40, 80, 3)
|
||
if self.error:
|
||
raise self.error
|
||
return self.result
|
||
|
||
|
||
def tracker_with(detector):
|
||
tracker = object.__new__(face_mod.MediaPipeTracker) # sem carregar o MediaPipe: só o caminho da caixa da cabeça
|
||
tracker._np, tracker._detector = np, detector
|
||
return tracker
|
||
|
||
|
||
def test_tracker_head_box_when_mediapipe_loses_the_face():
|
||
rgb = np.zeros((40, 80, 3), np.uint8)
|
||
sample = tracker_with(FakeDetector([(10, 4, 20, 20, 0.7), (40, 10, 16, 20, 0.9)]))._head_only(1.5, rgb)
|
||
assert sample.present is False and sample.head is True and sample.blend == {}
|
||
assert sample.cx == pytest.approx(0.6) and sample.cy == pytest.approx(0.5) and sample.size == pytest.approx(0.5)
|
||
assert tracker_with(FakeDetector([]))._head_only(1.5, rgb) == FaceSample(1.5, False)
|
||
assert tracker_with(FakeDetector(error=RuntimeError("cv2")))._head_only(1.5, rgb) == FaceSample(1.5, False)
|
||
assert tracker_with(None)._head_only(1.5, rgb) == FaceSample(1.5, False)
|
||
|
||
|
||
def test_to_frame_converts_head_size():
|
||
second = FaceSecond(0, 0.0, hsize=0.5)
|
||
assert face_mod.to_frame([second], CAM)[0].hsize == pytest.approx(0.25)
|
||
|
||
|
||
class PosedTracker:
|
||
"""Rosto com pose: sorriso alto em `laughs`; balanço da cabeça em `shakes`."""
|
||
|
||
name = "fake"
|
||
|
||
def __init__(self, laughs=(), shakes=()):
|
||
self.laughs, self.shakes = set(laughs), set(shakes)
|
||
|
||
def process(self, t, rgb):
|
||
smile = 0.9 if int(t) in self.laughs else 0.05 + 0.05 * ((int(t * 6) % 3) / 2)
|
||
yaw = 15 * math.sin(2 * math.pi * 1.2 * t) if int(t) in self.shakes else 0.0
|
||
return FaceSample(t, True, 0.5, 0.4, 0.3, {"mouthSmileLeft": smile, "mouthSmileRight": smile},
|
||
(yaw, 12.0, 0.0))
|
||
|
||
def close(self):
|
||
pass
|
||
|
||
|
||
def test_face_api_has_the_expression_map_and_body(env): # noqa: F811
|
||
client, engine = env
|
||
service = make_service(engine, tracker_factory=lambda _b=None: PosedTracker(range(300, 306), range(310, 313)))
|
||
sid = seed_session(engine)
|
||
set_cam(engine, sid)
|
||
while service.step():
|
||
pass
|
||
body = client.get(f"/v1/sessions/{sid}/face?from=290&to=320", headers=auth()).json()
|
||
by_t = {s["t"]: s for s in body["seconds"]}
|
||
assert by_t[302]["expr"] == "risada" and by_t[295]["expr"] == "neutro"
|
||
assert by_t[311]["body"] == "balançando" and "body" not in by_t[295]
|
||
assert "yaw" in by_t[295] and by_t[295]["head"] == 1.0
|
||
assert [b["label"] for b in body["body"]] == ["balançando"]
|
||
line = service.summary(engine.db.get("SELECT * FROM sessions WHERE id = ?", sid), 295, 315)
|
||
assert "risada" in line and "balançando a cabeça em 3" in line
|