"""Quem fala cada palavra (doc/clipador/FALANTES.md): atribuição por voz + boca/volume, legenda em duas linhas sem misturar falantes, tradução com `speaker`, juiz com a fala marcada e compatibilidade com `words_json` antigo.""" import json import re import numpy as np import pytest from app import editor, editor_render, layout, subtitles from app import speakers as spk from app.face import FaceService from app.judge import _transcript from app.scoring import Word from app.speakers import STREAMER, SpeakerService, VoiceChunk from .test_editor import validate from .test_english_source import EnglishJudge, _english_session from .test_face import CAM, fake_reader, set_cam from .test_face import FakeTracker from .test_render_and_duplicates import add_clip from .test_sources_and_api import env, seed_session # noqa: F401 - fixture reaproveitada A, B, C = np.eye(256, dtype=np.float32)[0], np.eye(256, dtype=np.float32)[1], np.eye(256, dtype=np.float32)[2] def _noisy(vec, k): """Vetor de voz com um pouco de ruído (a mesma voz nunca dá o mesmo vetor duas vezes).""" rng = np.random.default_rng(k) v = vec + rng.normal(0, 0.02, 256).astype(np.float32) return v / np.linalg.norm(v) def _conversation(n=12, length=2.0, db_streamer=-28.0, db_other=-36.0): """Trechos alternados: streamer (voz A) e colega (voz B), `n` de cada.""" out = [] for i in range(n): out.append(VoiceChunk(i * 6.0, i * 6.0 + length, db_streamer, _noisy(A, i))) out.append(VoiceChunk(i * 6.0 + 3.0, i * 6.0 + 3.0 + length, db_other, _noisy(B, 100 + i))) return out # ---------------------------------------------------------------- dados / compatibilidade def test_words_json_old_and_new_format(): old = json.dumps([[1.0, 1.4, "olha"], [1.5, 1.9, "isso"]]) words = spk.parse_words(old) assert words == [Word(1.0, 1.4, "olha"), Word(1.5, 1.9, "isso")] and words[0].speaker == "" new = spk.parse_words(json.dumps([[1.0, 1.4, "olha", "streamer"], [2.0, 2.3, "bro", "other1"], [3, 3.1, "x", ""]])) assert [w.speaker for w in new] == ["streamer", "other1", ""] # Sem falante, o formato gravado continua o antigo (3 elementos); com falante, 4. assert json.loads(spk.dump_words(words)) == [[1.0, 1.4, "olha"], [1.5, 1.9, "isso"]] assert json.loads(spk.dump_words(new))[1] == [2.0, 2.3, "bro", "other1"] assert spk.parse_words(None) == [] and spk.parse_words("[[1, 2]]") == [] def test_speech_chunks_split_on_pause_and_length(): words = [Word(0.0, 0.4, "a"), Word(0.5, 0.9, "b"), Word(1.5, 1.8, "c"), Word(1.85, 2.0, "d")] assert spk.speech_chunks(words) == [(0.0, 0.9), (1.5, 2.0)] long = [Word(i * 0.3, i * 0.3 + 0.28, "w") for i in range(20)] # 6 s de fala corrida chunks = spk.speech_chunks(long) assert len(chunks) == 2 and all(b - a <= spk.CHUNK_MAX_S for a, b in chunks) # Fim de frase fecha o trecho (a vez costuma trocar ali, mesmo sem pausa); frase curtinha não. turn = [Word(0.0, 0.3, "bring"), Word(0.32, 0.6, "two."), Word(0.62, 0.8, "Ok."), Word(0.82, 1.0, "yes")] assert spk.speech_chunks(turn) == [(0.0, 0.6), (0.62, 1.0)] # ---------------------------------------------------------------- atribuição (vetores sintéticos) def test_two_voices_streamer_by_loudness(): model = spk.build_model(_conversation()) assert model.active and len(model.voices) == 2 and model.reason.startswith("volume") labels = [c.label for c in model.chunks] assert labels == ["streamer", "other1"] * 12 def test_mouth_series_picks_the_streamer_even_when_quieter(): # Colega mais ALTO que o streamer (volume enganaria), mas a boca do streamer mexe nos trechos dele. chunks = _conversation(db_streamer=-36.0, db_other=-29.0) mine = {round(c.start, 1) for c in chunks if c.vec[0] > 0.5} def mouth_of(a, b): rng = np.random.default_rng(int(a * 10)) return float(0.30 + rng.normal(0, 0.02)) if round(a, 1) in mine else float(0.12 + rng.normal(0, 0.02)) model = spk.build_model(chunks, mouth_of) # Boca e volume discordam → ninguém vira "outro" (seguro: fica tudo como do streamer). assert not model.active and "apontam vozes diferentes" in model.reason assert {c.label for c in model.chunks} == {STREAMER} # Sem volume (sem prova do áudio), a boca decide sozinha. for c in chunks: c.db = None model = spk.build_model(chunks, mouth_of) assert model.active and model.reason.startswith("boca") assert [c.label for c in model.chunks] == ["streamer", "other1"] * 12 def test_no_proof_or_single_voice_keeps_everything_as_streamer(): flat = _conversation(db_streamer=-30.0, db_other=-31.0) # 1 dB de diferença e sem câmera model = spk.build_model(flat) assert not model.active and {c.label for c in model.chunks} == {STREAMER} solo = [VoiceChunk(i * 3.0, i * 3.0 + 2.0, -30.0, _noisy(A, i)) for i in range(20)] model = spk.build_model(solo) assert len(model.voices) == 1 and not model.active words = [Word(0.2, 0.5, "oi")] assert spk.label_words(words, model.chunks) == words # nada muda def test_three_voices_are_numbered_and_short_chunks_follow_the_neighbor(): chunks = _conversation() chunks += [VoiceChunk(200 + i * 4.0, 202 + i * 4.0, -37.0, _noisy(C, 300 + i)) for i in range(10)] chunks.append(VoiceChunk(5.2, 5.5, -36.0, None)) # curto (sem vetor), logo depois do colega (3–5 s) model = spk.build_model(chunks) assert [v.label for v in model.voices] == ["streamer", "other1", "other2"] words = [Word(0.1, 0.5, "yo"), Word(3.2, 3.6, "bro"), Word(5.25, 5.45, "ok"), Word(201.0, 201.4, "hey"), Word(150.0, 150.3, "longe")] got = [w.speaker for w in spk.label_words(words, model.chunks)] assert got == ["streamer", "other1", "other1", "other2", "streamer"] # Palavra que já veio com falante (4º elemento) não é trocada. assert spk.label_words([Word(3.2, 3.6, "bro", "streamer")], model.chunks)[0].speaker == "streamer" def test_transfer_keeps_speaker_on_retranscription(): old = [Word(1.0, 1.5, "a", "streamer"), Word(3.0, 3.5, "b", "other1")] new = spk.transfer([Word(1.1, 1.4, "A"), Word(3.1, 3.3, "B")], old) assert [w.speaker for w in new] == ["streamer", "other1"] # ---------------------------------------------------------------- serviço (banco + embedder falso) class FakeEmbedder: """Voz pelo volume do trecho: alto = voz A (streamer), baixo = voz B (colega).""" def embed(self, samples): if len(samples) < 8000: return None loud = float(np.sqrt(np.mean(samples.astype(np.float64) ** 2))) return _noisy(A if loud > 4000 else B, int(loud)).tolist() def _block_audio(start_s): """30 s: streamer (amplitude 8000) em [i·6, i·6+2] e colega (2000) em [i·6+3, i·6+5]; palavras nos mesmos tempos.""" sr = 16000 pcm = np.zeros(30 * sr, np.int16) words = [] rng = np.random.default_rng(int(start_s)) for i in range(5): for off, amp, text in ((0.0, 8000, "me"), (3.0, 2000, "you")): a = i * 6.0 + off pcm[int(a * sr): int((a + 2.0) * sr)] = (rng.standard_normal(int(2.0 * sr)) * amp).astype(np.int16) words += [Word(start_s + a + k * 0.4, start_s + a + k * 0.4 + 0.35, text) for k in range(5)] return pcm.tobytes(), words def test_service_indexes_blocks_and_labels_the_timeline(env): # noqa: F811 _client, engine = env engine.speakers = SpeakerService(engine, embedder_factory=FakeEmbedder) sid = seed_session(engine, duration=120) assert engine.speakers.view(sid)["active"] is False # nada indexado ainda for seg in engine.db.all("SELECT * FROM segments WHERE session_id = ? ORDER BY idx", sid): pcm, words = _block_audio(seg["start_s"]) assert engine.speakers.index_block(seg, pcm, words) == 10 engine.db.run("UPDATE segments SET words_json = ? WHERE session_id = ? AND idx = ?", spk.dump_words(words), sid, seg["idx"]) view = engine.speakers.view(sid) assert view["active"] and view["streamer"] == "streamer" and len(view["voices"]) == 2 words = engine.load_timeline(sid)["words"] assert {w.speaker for w in words if w.word == "me"} == {"streamer"} assert {w.speaker for w in words if w.word == "you"} == {"other1"} # O mesmo vale para as palavras que o editor lê. between = engine.words_between(sid, 0, 30) assert {w.speaker for w in between if w.word == "you"} == {"other1"} # Desligado (opção `speakers`), tudo volta a ser como antes. engine.intel.opts["speakers"] = False assert {w.speaker for w in engine.load_timeline(sid)["words"]} == {""} def test_face_block_stores_mouth_series(env): # noqa: F811 _client, engine = env class Meter: def feed(self, rgb): return 0.2 service = FaceService(engine, store=object(), tracker_factory=lambda _b=None: FakeTracker(), detector_factory=None, reader=fake_reader, grabber=None, sizer=lambda _f: (1280, 720), mouth_factory=Meter) engine.face = service sid = seed_session(engine, duration=60) set_cam(engine, sid, CAM) assert service.step() is True data = json.loads(engine.db.get("SELECT data_json FROM face_track WHERE session_id = ? AND idx = 0", sid)["data_json"]) assert len(data["mouth"]) == 180 and data["mouth"][0] == 0.2 # Sem medidor (store falso, sem YuNet): o bloco sai igual ao de antes, sem `mouth`. service._mouth_factory = lambda: (_ for _ in ()).throw(RuntimeError("sem yunet")) assert service.step() is True data = json.loads(engine.db.get("SELECT data_json FROM face_track WHERE session_id = ? AND idx = 1", sid)["data_json"]) assert "mouth" not in data and data["frames"] == 180 # ---------------------------------------------------------------- legenda def _dialogue(ass: str) -> list[tuple[float, float, str, str]]: out = [] for line in ass.splitlines(): if line.startswith("Dialogue: 0,"): parts = line.split(",", 9) h, m, s = parts[1].split(":") a = int(h) * 3600 + int(m) * 60 + float(s) h, m, s = parts[2].split(":") b = int(h) * 3600 + int(m) * 60 + float(s) out.append((a, b, parts[3], parts[9])) return out def _ypos(text: str) -> int: return int(re.search(r"\\pos\(\d+,(\d+)\)", text).group(1)) TALK = [Word(1.0, 1.3, "olha"), Word(1.35, 1.6, "isso"), Word(1.62, 1.9, "mano"), Word(1.7, 2.0, "for", "other1"), Word(2.05, 2.4, "real", "other1"), Word(2.5, 2.9, "bro", "other1"), Word(3.0, 3.3, "caiu"), Word(3.35, 3.7, "aqui")] def test_caption_blocks_never_mix_speakers_and_overlap_only_across_lanes(): blocks = subtitles.caption_blocks(TALK, 3) for group, _end in blocks: assert len({w.speaker or STREAMER for w in group}) == 1 lanes = {"streamer": [], "other": []} for group, end in blocks: lanes[subtitles.lane(group[0].speaker)].append((group[0].start, end)) for spans in lanes.values(): # dentro da faixa, nunca dois blocos ao mesmo tempo assert all(a1 <= a2 and e1 <= a2 + 1e-9 for (a1, e1), (a2, _) in zip(spans, spans[1:])) # As duas faixas se sobrepõem (os dois falando juntos em 1,7–1,9 s). (s_a, s_e), (o_a, o_e) = lanes["streamer"][0], lanes["other"][0] assert o_a < s_e @pytest.mark.parametrize("aspect", ["vertical", "horizontal"]) def test_build_ass_puts_the_other_speaker_below_in_another_color(aspect): cfg = layout.merge({"format": {"aspect": aspect}, "caption": {"otherColor": "#00FF00"}}) ass = subtitles.build_ass(TALK, 0.0, 6.0, "", cfg) assert "Style: CaptionOther," in ass and "&H0000FF00&" in ass events = _dialogue(ass) mine = [e for e in events if e[2] == "Caption"] other = [e for e in events if e[2] == "CaptionOther"] assert mine and other y, oy = _ypos(mine[0][3]), _ypos(other[0][3]) assert y == layout.caption_y(cfg) and oy > y # embaixo assert all("FOR" in e[3] or "REAL" in e[3] or "BRO" in e[3] for e in other) assert not any("OLHA" in e[3] for e in other) and not any("BRO" in e[3] for e in mine) # Os dois na tela ao mesmo tempo (1,7–1,9 s), em alturas diferentes. assert any(a < 1.85 < b for a, b, _, _ in mine) and any(a < 1.85 < b for a, b, _, _ in other) # Só o streamer: o ASS é exatamente o de antes (sem o estilo novo). solo = [Word(w.start, w.end, w.word) for w in TALK if not w.speaker] assert "CaptionOther" not in subtitles.build_ass(solo, 0.0, 6.0, "", cfg) def _project(tmp_path, words, **over): p = { "schema": 1, "source": {"sessionId": 1, "lo": 0, "hi": 60}, "settings": {}, "cuts": [{"start": 0, "end": 20, "mode": None}], "captions": {"enabled": True, "preset": "pop", "style": {}, "emphColor": "#22E36B", "words": [editor.word_item(w) for w in words]}, "overlays": [], "progress": {"enabled": False, "color": "#FF7A1A", "height": 12}, "audio": {"volume": 1, "narration": [], "music": None, "sfx": []}, "cover": {"enabled": False, "t": 1, "text": ""}, } p.update(over) return validate(p, tmp_path) @pytest.mark.parametrize("preset", editor.CAPTION_PRESETS) def test_project_ass_two_lines_for_every_preset(tmp_path, preset): p = _project(tmp_path, TALK) assert p["captions"]["words"][3]["sp"] == "other1" and "sp" not in p["captions"]["words"][0] p["captions"]["preset"] = preset cfg = editor.effective_settings(layout.merge(), p) events = _dialogue(editor_render.build_project_ass(p, cfg, "")) mine = [e for e in events if e[2] in ("Caption", "Karaoke")] other = [e for e in events if e[2] in ("CaptionOther", "KaraokeOther")] assert mine and other assert min(_ypos(e[3]) for e in other) > max(_ypos(e[3]) for e in mine) for group in (mine, other): # na mesma linha, nunca dois eventos ao mesmo tempo spans = sorted((a, b) for a, b, _, _ in group) assert all(b1 <= a2 + 0.011 for (_, b1), (a2, _) in zip(spans, spans[1:])) def test_hook_dodge_moves_both_lines_and_translation_follows_speaker(tmp_path): hook = {"id": "hook", "kind": "hook", "text": "NAO ACREDITO", "start": 0, "end": 3, "y": 0.74, "size": 70, "color": "#FFFFFF", "box": "#E11D48", "anim": "pop"} p = _project(tmp_path, TALK, overlays=[hook]) cfg = editor.effective_settings(layout.merge({"dev": {"translation": True}}), p) lines = [{"s": 1.0, "e": 1.9, "text": "olha isso mano", "pt": "olha isso", "speaker": "streamer"}, {"s": 1.7, "e": 2.9, "text": "for real bro", "pt": "sério mano", "speaker": "other1"}] events = _dialogue(editor_render.build_project_ass(p, cfg, "", translation=lines)) cap_y = layout.caption_y(cfg) mine = [e for e in events if e[2] == "Caption"] other = [e for e in events if e[2] == "CaptionOther"] trans = {e[3].split("}")[-1]: _ypos(e[3]) for e in events if e[2] == "Trans"} # Gancho em cima da legenda nos 3 primeiros segundos: as duas linhas desviam juntas, a do colega embaixo. y, oy = _ypos(mine[0][3]), _ypos(other[0][3]) assert y != cap_y and oy > y hook_top = int(1920 * 0.74) - int(70 * 0.9) hook_bottom = int(1920 * 0.74) + int(70 * 0.9) assert y - 84 >= hook_bottom or oy + 60 <= hook_top # Tradução de cada um embaixo da linha de quem fala, entre as duas a do streamer. assert y < trans["olha isso"] < oy < trans["sério mano"] def test_old_project_without_speaker_renders_like_before(tmp_path): words = [Word(w.start, w.end, w.word) for w in TALK] p = _project(tmp_path, words) ass = editor_render.build_project_ass(p, editor.effective_settings(layout.merge(), p), "") assert "CaptionOther" not in ass assert {_ypos(e[3]) for e in _dialogue(ass)} == {layout.caption_y(layout.merge())} def test_layout_validates_other_color_and_size(): assert layout.DEFAULTS["caption"]["otherColor"] == "#5CE1FF" and layout.DEFAULTS["caption"]["otherSize"] is None ok = layout.validate({"caption": {"otherColor": "#ff00aa", "otherSize": 60}})["caption"] assert ok == {"otherColor": "#FF00AA", "otherSize": 60} assert layout.validate({"caption": {"otherSize": None}})["caption"]["otherSize"] is None with pytest.raises(layout.LayoutError): layout.validate({"caption": {"otherColor": "azul"}}) with pytest.raises(layout.LayoutError): layout.validate({"caption": {"otherSize": 10}}) cap = layout.merge()["caption"] assert subtitles.other_size(cap) == round(84 * 0.85) assert subtitles.other_size({**cap, "otherSize": 50}) == 50 def test_project_rejects_bad_speaker(tmp_path): p = _project(tmp_path, TALK) p["captions"]["words"][0]["sp"] = "ninguém; drop" with pytest.raises(editor.EditError): validate(p, tmp_path) # ---------------------------------------------------------------- tradução e juízes def test_phrases_never_mix_speakers(): got = subtitles.phrases(TALK, with_speaker=True) assert got == [(1.0, 1.9, "olha isso mano", "streamer"), (1.7, 2.9, "for real bro", "other1"), (3.0, 3.7, "caiu aqui", "streamer")] assert subtitles.phrases(TALK)[0] == (1.0, 1.9, "olha isso mano") # formato antigo continua def test_translation_lines_get_speaker(env): # noqa: F811 client, engine = env engine.judge = EnglishJudge() sid = _english_session(engine) words = [[281.0, 281.3, "No", "streamer"], [281.3, 281.6, "way", "streamer"], [281.6, 282.0, "bro.", "streamer"], [282.1, 282.5, "Told", "other1"], [282.5, 283.0, "you.", "other1"]] engine.db.run("UPDATE segments SET words_json = ? WHERE session_id = ? AND idx = 9", json.dumps(words), sid) add_clip(engine, sid, 1, "candidate", 285, 270, 340) engine.judge_clip(engine.db.get("SELECT * FROM clips WHERE id = 1")) lines = json.loads(engine.db.get("SELECT translation_json FROM clips WHERE id = 1")["translation_json"])["lines"] assert [(x["text"], x["speaker"]) for x in lines] == [("No way bro.", "streamer"), ("Told you.", "other1")] # O juiz viu quem fala. prompt = engine.judge.seen["judge"][1] assert "] No way bro." in prompt and "[other1] Told you." in prompt and "Never attribute to" in prompt # Tradução antiga (sem `speaker`) é refeita na próxima vez. engine.db.run("UPDATE clips SET translation_json = ? WHERE id = 1", json.dumps({"startS": 0, "endS": 1e6, "lines": [{"s": 1, "e": 2, "text": "x", "pt": "y"}]})) before = len(engine.judge.translated) engine.translate_clip(1) assert len(engine.judge.translated) == before + 1 def test_judge_transcript_tags_only_when_someone_else_speaks(): plain = [Word(10.0, 10.3, "olha"), Word(10.4, 10.8, "isso")] assert _transcript(plain, 0, 60, "pt", "Tfue") == "[10s] olha isso" mixed = plain + [Word(11.0, 11.3, "bro", "other1"), Word(11.4, 11.8, "nice", "other1"), Word(12.0, 12.2, "ok")] out = _transcript(mixed, 0, 60, "en", "Tfue").splitlines() assert out[0].startswith("Each line is tagged") and "[Tfue]" in out[0] assert out[1:] == ["[10s] [Tfue] olha isso", "[11s] [other1] bro nice", "[12s] [Tfue] ok"] def test_voice_model_failure_waits_before_retrying(env): # noqa: F811 _client, engine = env calls = [] def broken(): calls.append(1) raise RuntimeError("sem rede") engine.speakers = SpeakerService(engine, embedder_factory=broken) sid = seed_session(engine, duration=60) seg = engine.db.get("SELECT * FROM segments WHERE session_id = ? AND idx = 0", sid) pcm, words = _block_audio(0.0) for _ in range(3): with pytest.raises(RuntimeError): engine.speakers.index_block(seg, pcm, words) assert len(calls) == 1 # não baixa o modelo a cada bloco assert engine.load_timeline(sid)["words"][0].speaker == "" # sem voz indexada: como antes