Files
MateuusandClaude Opus 5.5 cb8450399d Clipador: Equipe de IA (papéis com IA própria), juiz de contexto e linha do tempo JSON
- roles.py: varredura, analista, editor e juiz, cada um com provedor, modelo, raciocínio
  (Codex), liga/desliga (o editor não desliga) e instruções do administrador somadas às
  padrão; gravado em settings `judge.roles`, com migração automática da configuração antiga
  (provedor/modelo único vira o de todos; o modelo do analista é respeitado). A reserva vale
  por papel.
- Juiz de contexto (só texto): vê só o que vai no corte (+20 s antes/15 s depois marcados
  out=1), escreve o que um espectador frio entende e responde sim/não; context_judge.decide
  decide pelo código: aprova, ajusta o trecho e confere uma vez de novo, frase de contexto
  na tela (vira o texto de abertura do projeto) ou descarta com motivo em português
  (forçável). Corte manual só recebe aviso.
- timeline.py: linha do tempo JSON curta (fala com falante, chat, rosto, volume e a frase
  de tela que a varredura/analista escrevem ao ver os quadros, em scene_notes); mosaico 4x2
  (media.mosaic) no lugar dos quadros soltos; perfil do jogo enxuto. O formato antigo fica
  atrás da opção timelineJson.
- Medição (F0): ai_calls por chamada (papel, IA, tamanho do prompt/imagens, tokens quando a
  CLI informa: codex --json, claude e gemini), avaliação do dono (owner_label: bom/ruim/sem
  contexto) e GET /v1/judge/report (taxa de aprovação, decisões do juiz, concordância).
- Banco v17 (v16 fica reservada às expressões, como marcador sem efeito).
- Testes: test_roles.py e test_context_judge.py; os testes bloqueiam qualquer CLI de IA real.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 04:36:25 -03:00

391 lines
20 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Quem fala cada palavra (doc/clipador/FALANTES.md): atribuição por voz + boca/volume, legenda em duas linhas sem
misturar falantes, tradução com `speaker`, juiz com a fala marcada e compatibilidade com `words_json` antigo."""
import json
import re
import numpy as np
import pytest
from app import editor, editor_render, layout, subtitles
from app import speakers as spk
from app.face import FaceService
from app.judge import _transcript
from app.scoring import Word
from app.speakers import STREAMER, SpeakerService, VoiceChunk
from .test_editor import validate
from .test_english_source import EnglishJudge, _english_session
from .test_face import CAM, fake_reader, set_cam
from .test_face import FakeTracker
from .test_render_and_duplicates import add_clip
from .test_sources_and_api import env, seed_session # noqa: F401 - fixture reaproveitada
A, B, C = np.eye(256, dtype=np.float32)[0], np.eye(256, dtype=np.float32)[1], np.eye(256, dtype=np.float32)[2]
def _noisy(vec, k):
"""Vetor de voz com um pouco de ruído (a mesma voz nunca dá o mesmo vetor duas vezes)."""
rng = np.random.default_rng(k)
v = vec + rng.normal(0, 0.02, 256).astype(np.float32)
return v / np.linalg.norm(v)
def _conversation(n=12, length=2.0, db_streamer=-28.0, db_other=-36.0):
"""Trechos alternados: streamer (voz A) e colega (voz B), `n` de cada."""
out = []
for i in range(n):
out.append(VoiceChunk(i * 6.0, i * 6.0 + length, db_streamer, _noisy(A, i)))
out.append(VoiceChunk(i * 6.0 + 3.0, i * 6.0 + 3.0 + length, db_other, _noisy(B, 100 + i)))
return out
# ---------------------------------------------------------------- dados / compatibilidade
def test_words_json_old_and_new_format():
old = json.dumps([[1.0, 1.4, "olha"], [1.5, 1.9, "isso"]])
words = spk.parse_words(old)
assert words == [Word(1.0, 1.4, "olha"), Word(1.5, 1.9, "isso")] and words[0].speaker == ""
new = spk.parse_words(json.dumps([[1.0, 1.4, "olha", "streamer"], [2.0, 2.3, "bro", "other1"], [3, 3.1, "x", ""]]))
assert [w.speaker for w in new] == ["streamer", "other1", ""]
# Sem falante, o formato gravado continua o antigo (3 elementos); com falante, 4.
assert json.loads(spk.dump_words(words)) == [[1.0, 1.4, "olha"], [1.5, 1.9, "isso"]]
assert json.loads(spk.dump_words(new))[1] == [2.0, 2.3, "bro", "other1"]
assert spk.parse_words(None) == [] and spk.parse_words("[[1, 2]]") == []
def test_speech_chunks_split_on_pause_and_length():
words = [Word(0.0, 0.4, "a"), Word(0.5, 0.9, "b"), Word(1.5, 1.8, "c"), Word(1.85, 2.0, "d")]
assert spk.speech_chunks(words) == [(0.0, 0.9), (1.5, 2.0)]
long = [Word(i * 0.3, i * 0.3 + 0.28, "w") for i in range(20)] # 6 s de fala corrida
chunks = spk.speech_chunks(long)
assert len(chunks) == 2 and all(b - a <= spk.CHUNK_MAX_S for a, b in chunks)
# Fim de frase fecha o trecho (a vez costuma trocar ali, mesmo sem pausa); frase curtinha não.
turn = [Word(0.0, 0.3, "bring"), Word(0.32, 0.6, "two."), Word(0.62, 0.8, "Ok."), Word(0.82, 1.0, "yes")]
assert spk.speech_chunks(turn) == [(0.0, 0.6), (0.62, 1.0)]
# ---------------------------------------------------------------- atribuição (vetores sintéticos)
def test_two_voices_streamer_by_loudness():
model = spk.build_model(_conversation())
assert model.active and len(model.voices) == 2 and model.reason.startswith("volume")
labels = [c.label for c in model.chunks]
assert labels == ["streamer", "other1"] * 12
def test_mouth_series_picks_the_streamer_even_when_quieter():
# Colega mais ALTO que o streamer (volume enganaria), mas a boca do streamer mexe nos trechos dele.
chunks = _conversation(db_streamer=-36.0, db_other=-29.0)
mine = {round(c.start, 1) for c in chunks if c.vec[0] > 0.5}
def mouth_of(a, b):
rng = np.random.default_rng(int(a * 10))
return float(0.30 + rng.normal(0, 0.02)) if round(a, 1) in mine else float(0.12 + rng.normal(0, 0.02))
model = spk.build_model(chunks, mouth_of)
# Boca e volume discordam → ninguém vira "outro" (seguro: fica tudo como do streamer).
assert not model.active and "apontam vozes diferentes" in model.reason
assert {c.label for c in model.chunks} == {STREAMER}
# Sem volume (sem prova do áudio), a boca decide sozinha.
for c in chunks:
c.db = None
model = spk.build_model(chunks, mouth_of)
assert model.active and model.reason.startswith("boca")
assert [c.label for c in model.chunks] == ["streamer", "other1"] * 12
def test_no_proof_or_single_voice_keeps_everything_as_streamer():
flat = _conversation(db_streamer=-30.0, db_other=-31.0) # 1 dB de diferença e sem câmera
model = spk.build_model(flat)
assert not model.active and {c.label for c in model.chunks} == {STREAMER}
solo = [VoiceChunk(i * 3.0, i * 3.0 + 2.0, -30.0, _noisy(A, i)) for i in range(20)]
model = spk.build_model(solo)
assert len(model.voices) == 1 and not model.active
words = [Word(0.2, 0.5, "oi")]
assert spk.label_words(words, model.chunks) == words # nada muda
def test_three_voices_are_numbered_and_short_chunks_follow_the_neighbor():
chunks = _conversation()
chunks += [VoiceChunk(200 + i * 4.0, 202 + i * 4.0, -37.0, _noisy(C, 300 + i)) for i in range(10)]
chunks.append(VoiceChunk(5.2, 5.5, -36.0, None)) # curto (sem vetor), logo depois do colega (3–5 s)
model = spk.build_model(chunks)
assert [v.label for v in model.voices] == ["streamer", "other1", "other2"]
words = [Word(0.1, 0.5, "yo"), Word(3.2, 3.6, "bro"), Word(5.25, 5.45, "ok"), Word(201.0, 201.4, "hey"),
Word(150.0, 150.3, "longe")]
got = [w.speaker for w in spk.label_words(words, model.chunks)]
assert got == ["streamer", "other1", "other1", "other2", "streamer"]
# Palavra que já veio com falante (4º elemento) não é trocada.
assert spk.label_words([Word(3.2, 3.6, "bro", "streamer")], model.chunks)[0].speaker == "streamer"
def test_transfer_keeps_speaker_on_retranscription():
old = [Word(1.0, 1.5, "a", "streamer"), Word(3.0, 3.5, "b", "other1")]
new = spk.transfer([Word(1.1, 1.4, "A"), Word(3.1, 3.3, "B")], old)
assert [w.speaker for w in new] == ["streamer", "other1"]
# ---------------------------------------------------------------- serviço (banco + embedder falso)
class FakeEmbedder:
"""Voz pelo volume do trecho: alto = voz A (streamer), baixo = voz B (colega)."""
def embed(self, samples):
if len(samples) < 8000:
return None
loud = float(np.sqrt(np.mean(samples.astype(np.float64) ** 2)))
return _noisy(A if loud > 4000 else B, int(loud)).tolist()
def _block_audio(start_s):
"""30 s: streamer (amplitude 8000) em [i·6, i·6+2] e colega (2000) em [i·6+3, i·6+5]; palavras nos mesmos tempos."""
sr = 16000
pcm = np.zeros(30 * sr, np.int16)
words = []
rng = np.random.default_rng(int(start_s))
for i in range(5):
for off, amp, text in ((0.0, 8000, "me"), (3.0, 2000, "you")):
a = i * 6.0 + off
pcm[int(a * sr): int((a + 2.0) * sr)] = (rng.standard_normal(int(2.0 * sr)) * amp).astype(np.int16)
words += [Word(start_s + a + k * 0.4, start_s + a + k * 0.4 + 0.35, text) for k in range(5)]
return pcm.tobytes(), words
def test_service_indexes_blocks_and_labels_the_timeline(env): # noqa: F811
_client, engine = env
engine.speakers = SpeakerService(engine, embedder_factory=FakeEmbedder)
sid = seed_session(engine, duration=120)
assert engine.speakers.view(sid)["active"] is False # nada indexado ainda
for seg in engine.db.all("SELECT * FROM segments WHERE session_id = ? ORDER BY idx", sid):
pcm, words = _block_audio(seg["start_s"])
assert engine.speakers.index_block(seg, pcm, words) == 10
engine.db.run("UPDATE segments SET words_json = ? WHERE session_id = ? AND idx = ?",
spk.dump_words(words), sid, seg["idx"])
view = engine.speakers.view(sid)
assert view["active"] and view["streamer"] == "streamer" and len(view["voices"]) == 2
words = engine.load_timeline(sid)["words"]
assert {w.speaker for w in words if w.word == "me"} == {"streamer"}
assert {w.speaker for w in words if w.word == "you"} == {"other1"}
# O mesmo vale para as palavras que o editor lê.
between = engine.words_between(sid, 0, 30)
assert {w.speaker for w in between if w.word == "you"} == {"other1"}
# Desligado (opção `speakers`), tudo volta a ser como antes.
engine.intel.opts["speakers"] = False
assert {w.speaker for w in engine.load_timeline(sid)["words"]} == {""}
def test_face_block_stores_mouth_series(env): # noqa: F811
_client, engine = env
class Meter:
def feed(self, rgb):
return 0.2
service = FaceService(engine, store=object(), tracker_factory=lambda _b=None: FakeTracker(), detector_factory=None,
reader=fake_reader, grabber=None, sizer=lambda _f: (1280, 720), mouth_factory=Meter)
engine.face = service
sid = seed_session(engine, duration=60)
set_cam(engine, sid, CAM)
assert service.step() is True
data = json.loads(engine.db.get("SELECT data_json FROM face_track WHERE session_id = ? AND idx = 0", sid)["data_json"])
assert len(data["mouth"]) == 180 and data["mouth"][0] == 0.2
# Sem medidor (store falso, sem YuNet): o bloco sai igual ao de antes, sem `mouth`.
service._mouth_factory = lambda: (_ for _ in ()).throw(RuntimeError("sem yunet"))
assert service.step() is True
data = json.loads(engine.db.get("SELECT data_json FROM face_track WHERE session_id = ? AND idx = 1", sid)["data_json"])
assert "mouth" not in data and data["frames"] == 180
# ---------------------------------------------------------------- legenda
def _dialogue(ass: str) -> list[tuple[float, float, str, str]]:
out = []
for line in ass.splitlines():
if line.startswith("Dialogue: 0,"):
parts = line.split(",", 9)
h, m, s = parts[1].split(":")
a = int(h) * 3600 + int(m) * 60 + float(s)
h, m, s = parts[2].split(":")
b = int(h) * 3600 + int(m) * 60 + float(s)
out.append((a, b, parts[3], parts[9]))
return out
def _ypos(text: str) -> int:
return int(re.search(r"\\pos\(\d+,(\d+)\)", text).group(1))
TALK = [Word(1.0, 1.3, "olha"), Word(1.35, 1.6, "isso"), Word(1.62, 1.9, "mano"),
Word(1.7, 2.0, "for", "other1"), Word(2.05, 2.4, "real", "other1"), Word(2.5, 2.9, "bro", "other1"),
Word(3.0, 3.3, "caiu"), Word(3.35, 3.7, "aqui")]
def test_caption_blocks_never_mix_speakers_and_overlap_only_across_lanes():
blocks = subtitles.caption_blocks(TALK, 3)
for group, _end in blocks:
assert len({w.speaker or STREAMER for w in group}) == 1
lanes = {"streamer": [], "other": []}
for group, end in blocks:
lanes[subtitles.lane(group[0].speaker)].append((group[0].start, end))
for spans in lanes.values(): # dentro da faixa, nunca dois blocos ao mesmo tempo
assert all(a1 <= a2 and e1 <= a2 + 1e-9 for (a1, e1), (a2, _) in zip(spans, spans[1:]))
# As duas faixas se sobrepõem (os dois falando juntos em 1,7–1,9 s).
(s_a, s_e), (o_a, o_e) = lanes["streamer"][0], lanes["other"][0]
assert o_a < s_e
@pytest.mark.parametrize("aspect", ["vertical", "horizontal"])
def test_build_ass_puts_the_other_speaker_below_in_another_color(aspect):
cfg = layout.merge({"format": {"aspect": aspect}, "caption": {"otherColor": "#00FF00"}})
ass = subtitles.build_ass(TALK, 0.0, 6.0, "", cfg)
assert "Style: CaptionOther," in ass and "&H0000FF00&" in ass
events = _dialogue(ass)
mine = [e for e in events if e[2] == "Caption"]
other = [e for e in events if e[2] == "CaptionOther"]
assert mine and other
y, oy = _ypos(mine[0][3]), _ypos(other[0][3])
assert y == layout.caption_y(cfg) and oy > y # embaixo
assert all("FOR" in e[3] or "REAL" in e[3] or "BRO" in e[3] for e in other)
assert not any("OLHA" in e[3] for e in other) and not any("BRO" in e[3] for e in mine)
# Os dois na tela ao mesmo tempo (1,7–1,9 s), em alturas diferentes.
assert any(a < 1.85 < b for a, b, _, _ in mine) and any(a < 1.85 < b for a, b, _, _ in other)
# Só o streamer: o ASS é exatamente o de antes (sem o estilo novo).
solo = [Word(w.start, w.end, w.word) for w in TALK if not w.speaker]
assert "CaptionOther" not in subtitles.build_ass(solo, 0.0, 6.0, "", cfg)
def _project(tmp_path, words, **over):
p = {
"schema": 1, "source": {"sessionId": 1, "lo": 0, "hi": 60}, "settings": {},
"cuts": [{"start": 0, "end": 20, "mode": None}],
"captions": {"enabled": True, "preset": "pop", "style": {}, "emphColor": "#22E36B",
"words": [editor.word_item(w) for w in words]},
"overlays": [], "progress": {"enabled": False, "color": "#FF7A1A", "height": 12},
"audio": {"volume": 1, "narration": [], "music": None, "sfx": []},
"cover": {"enabled": False, "t": 1, "text": ""},
}
p.update(over)
return validate(p, tmp_path)
@pytest.mark.parametrize("preset", editor.CAPTION_PRESETS)
def test_project_ass_two_lines_for_every_preset(tmp_path, preset):
p = _project(tmp_path, TALK)
assert p["captions"]["words"][3]["sp"] == "other1" and "sp" not in p["captions"]["words"][0]
p["captions"]["preset"] = preset
cfg = editor.effective_settings(layout.merge(), p)
events = _dialogue(editor_render.build_project_ass(p, cfg, ""))
mine = [e for e in events if e[2] in ("Caption", "Karaoke")]
other = [e for e in events if e[2] in ("CaptionOther", "KaraokeOther")]
assert mine and other
assert min(_ypos(e[3]) for e in other) > max(_ypos(e[3]) for e in mine)
for group in (mine, other): # na mesma linha, nunca dois eventos ao mesmo tempo
spans = sorted((a, b) for a, b, _, _ in group)
assert all(b1 <= a2 + 0.011 for (_, b1), (a2, _) in zip(spans, spans[1:]))
def test_hook_dodge_moves_both_lines_and_translation_follows_speaker(tmp_path):
hook = {"id": "hook", "kind": "hook", "text": "NAO ACREDITO", "start": 0, "end": 3, "y": 0.74, "size": 70,
"color": "#FFFFFF", "box": "#E11D48", "anim": "pop"}
p = _project(tmp_path, TALK, overlays=[hook])
cfg = editor.effective_settings(layout.merge({"dev": {"translation": True}}), p)
lines = [{"s": 1.0, "e": 1.9, "text": "olha isso mano", "pt": "olha isso", "speaker": "streamer"},
{"s": 1.7, "e": 2.9, "text": "for real bro", "pt": "sério mano", "speaker": "other1"}]
events = _dialogue(editor_render.build_project_ass(p, cfg, "", translation=lines))
cap_y = layout.caption_y(cfg)
mine = [e for e in events if e[2] == "Caption"]
other = [e for e in events if e[2] == "CaptionOther"]
trans = {e[3].split("}")[-1]: _ypos(e[3]) for e in events if e[2] == "Trans"}
# Gancho em cima da legenda nos 3 primeiros segundos: as duas linhas desviam juntas, a do colega embaixo.
y, oy = _ypos(mine[0][3]), _ypos(other[0][3])
assert y != cap_y and oy > y
hook_top = int(1920 * 0.74) - int(70 * 0.9)
hook_bottom = int(1920 * 0.74) + int(70 * 0.9)
assert y - 84 >= hook_bottom or oy + 60 <= hook_top
# Tradução de cada um embaixo da linha de quem fala, entre as duas a do streamer.
assert y < trans["olha isso"] < oy < trans["sério mano"]
def test_old_project_without_speaker_renders_like_before(tmp_path):
words = [Word(w.start, w.end, w.word) for w in TALK]
p = _project(tmp_path, words)
ass = editor_render.build_project_ass(p, editor.effective_settings(layout.merge(), p), "")
assert "CaptionOther" not in ass
assert {_ypos(e[3]) for e in _dialogue(ass)} == {layout.caption_y(layout.merge())}
def test_layout_validates_other_color_and_size():
assert layout.DEFAULTS["caption"]["otherColor"] == "#5CE1FF" and layout.DEFAULTS["caption"]["otherSize"] is None
ok = layout.validate({"caption": {"otherColor": "#ff00aa", "otherSize": 60}})["caption"]
assert ok == {"otherColor": "#FF00AA", "otherSize": 60}
assert layout.validate({"caption": {"otherSize": None}})["caption"]["otherSize"] is None
with pytest.raises(layout.LayoutError):
layout.validate({"caption": {"otherColor": "azul"}})
with pytest.raises(layout.LayoutError):
layout.validate({"caption": {"otherSize": 10}})
cap = layout.merge()["caption"]
assert subtitles.other_size(cap) == round(84 * 0.85)
assert subtitles.other_size({**cap, "otherSize": 50}) == 50
def test_project_rejects_bad_speaker(tmp_path):
p = _project(tmp_path, TALK)
p["captions"]["words"][0]["sp"] = "ninguém; drop"
with pytest.raises(editor.EditError):
validate(p, tmp_path)
# ---------------------------------------------------------------- tradução e juízes
def test_phrases_never_mix_speakers():
got = subtitles.phrases(TALK, with_speaker=True)
assert got == [(1.0, 1.9, "olha isso mano", "streamer"), (1.7, 2.9, "for real bro", "other1"),
(3.0, 3.7, "caiu aqui", "streamer")]
assert subtitles.phrases(TALK)[0] == (1.0, 1.9, "olha isso mano") # formato antigo continua
def test_translation_lines_get_speaker(env): # noqa: F811
client, engine = env
engine.judge = EnglishJudge()
sid = _english_session(engine)
words = [[281.0, 281.3, "No", "streamer"], [281.3, 281.6, "way", "streamer"], [281.6, 282.0, "bro.", "streamer"],
[282.1, 282.5, "Told", "other1"], [282.5, 283.0, "you.", "other1"]]
engine.db.run("UPDATE segments SET words_json = ? WHERE session_id = ? AND idx = 9", json.dumps(words), sid)
add_clip(engine, sid, 1, "candidate", 285, 270, 340)
engine.judge_clip(engine.db.get("SELECT * FROM clips WHERE id = 1"))
lines = json.loads(engine.db.get("SELECT translation_json FROM clips WHERE id = 1")["translation_json"])["lines"]
assert [(x["text"], x["speaker"]) for x in lines] == [("No way bro.", "streamer"), ("Told you.", "other1")]
# O juiz viu quem fala.
prompt = engine.judge.seen["judge"][1]
# Linha do tempo JSON (EQUIPE-IA.md): cada entrada é de um falante só; a de outra pessoa vem marcada.
assert '"sp":"No way bro."' in prompt and '"sp":"[other1] Told you."' in prompt and "Never attribute to" in prompt
# Tradução antiga (sem `speaker`) é refeita na próxima vez.
engine.db.run("UPDATE clips SET translation_json = ? WHERE id = 1",
json.dumps({"startS": 0, "endS": 1e6, "lines": [{"s": 1, "e": 2, "text": "x", "pt": "y"}]}))
before = len(engine.judge.translated)
engine.translate_clip(1)
assert len(engine.judge.translated) == before + 1
def test_judge_transcript_tags_only_when_someone_else_speaks():
plain = [Word(10.0, 10.3, "olha"), Word(10.4, 10.8, "isso")]
assert _transcript(plain, 0, 60, "pt", "Tfue") == "[10s] olha isso"
mixed = plain + [Word(11.0, 11.3, "bro", "other1"), Word(11.4, 11.8, "nice", "other1"), Word(12.0, 12.2, "ok")]
out = _transcript(mixed, 0, 60, "en", "Tfue").splitlines()
assert out[0].startswith("Each line is tagged") and "[Tfue]" in out[0]
assert out[1:] == ["[10s] [Tfue] olha isso", "[11s] [other1] bro nice", "[12s] [Tfue] ok"]
def test_voice_model_failure_waits_before_retrying(env): # noqa: F811
_client, engine = env
calls = []
def broken():
calls.append(1)
raise RuntimeError("sem rede")
engine.speakers = SpeakerService(engine, embedder_factory=broken)
sid = seed_session(engine, duration=60)
seg = engine.db.get("SELECT * FROM segments WHERE session_id = ? AND idx = 0", sid)
pcm, words = _block_audio(0.0)
for _ in range(3):
with pytest.raises(RuntimeError):
engine.speakers.index_block(seg, pcm, words)
assert len(calls) == 1 # não baixa o modelo a cada bloco
assert engine.load_timeline(sid)["words"][0].speaker == "" # sem voz indexada: como antes