Integração: falantes + dublagem + vídeo final

- editor_api: importa dub e speakers juntos.
- voice_chunks (falantes) vira a migração v15, depois da v14 do vídeo final (antes era criada fora das migrações).
- dub.voice_for: qualquer voz que não é o streamer (other1, other2…) usa a segunda voz da dublagem.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mateuus
2026-11-18 03:59:44 -03:00
co-authored by Claude Opus 5.5
24 changed files with 1671 additions and 129 deletions
+8
View File
@@ -327,3 +327,11 @@ def test_preview_and_queue_render_dub_the_same_way(api, monkeypatch):
assert clip["status"] == "ready", clip["error"]
# Fila: a 1ª fala veio do cache da prévia; a do "other" saiu com a segunda voz.
assert synth.calls == [(layout.DUB_VOICE, "Não acredito."), (layout.DUB_VOICE_OTHER, "Mano.")]
def test_numbered_other_speakers_use_the_second_voice():
"""Integração com FALANTES.md: a separação entrega `other1`, `other2`…; todas usam a segunda voz."""
from app.dub import voice_for
opts = {"voice": "a", "voiceOther": "b"}
assert voice_for({"speaker": "other1"}, opts) == "b" and voice_for({"speaker": "other2"}, opts) == "b"
assert voice_for({"speaker": "streamer"}, opts) == "a" and voice_for({}, opts) == "a"
+389
View File
@@ -0,0 +1,389 @@
"""Quem fala cada palavra (doc/clipador/FALANTES.md): atribuição por voz + boca/volume, legenda em duas linhas sem
misturar falantes, tradução com `speaker`, juiz com a fala marcada e compatibilidade com `words_json` antigo."""
import json
import re
import numpy as np
import pytest
from app import editor, editor_render, layout, subtitles
from app import speakers as spk
from app.face import FaceService
from app.judge import _transcript
from app.scoring import Word
from app.speakers import STREAMER, SpeakerService, VoiceChunk
from .test_editor import validate
from .test_english_source import EnglishJudge, _english_session
from .test_face import CAM, fake_reader, set_cam
from .test_face import FakeTracker
from .test_render_and_duplicates import add_clip
from .test_sources_and_api import env, seed_session # noqa: F401 - fixture reaproveitada
A, B, C = np.eye(256, dtype=np.float32)[0], np.eye(256, dtype=np.float32)[1], np.eye(256, dtype=np.float32)[2]
def _noisy(vec, k):
"""Vetor de voz com um pouco de ruído (a mesma voz nunca dá o mesmo vetor duas vezes)."""
rng = np.random.default_rng(k)
v = vec + rng.normal(0, 0.02, 256).astype(np.float32)
return v / np.linalg.norm(v)
def _conversation(n=12, length=2.0, db_streamer=-28.0, db_other=-36.0):
"""Trechos alternados: streamer (voz A) e colega (voz B), `n` de cada."""
out = []
for i in range(n):
out.append(VoiceChunk(i * 6.0, i * 6.0 + length, db_streamer, _noisy(A, i)))
out.append(VoiceChunk(i * 6.0 + 3.0, i * 6.0 + 3.0 + length, db_other, _noisy(B, 100 + i)))
return out
# ---------------------------------------------------------------- dados / compatibilidade
def test_words_json_old_and_new_format():
old = json.dumps([[1.0, 1.4, "olha"], [1.5, 1.9, "isso"]])
words = spk.parse_words(old)
assert words == [Word(1.0, 1.4, "olha"), Word(1.5, 1.9, "isso")] and words[0].speaker == ""
new = spk.parse_words(json.dumps([[1.0, 1.4, "olha", "streamer"], [2.0, 2.3, "bro", "other1"], [3, 3.1, "x", ""]]))
assert [w.speaker for w in new] == ["streamer", "other1", ""]
# Sem falante, o formato gravado continua o antigo (3 elementos); com falante, 4.
assert json.loads(spk.dump_words(words)) == [[1.0, 1.4, "olha"], [1.5, 1.9, "isso"]]
assert json.loads(spk.dump_words(new))[1] == [2.0, 2.3, "bro", "other1"]
assert spk.parse_words(None) == [] and spk.parse_words("[[1, 2]]") == []
def test_speech_chunks_split_on_pause_and_length():
words = [Word(0.0, 0.4, "a"), Word(0.5, 0.9, "b"), Word(1.5, 1.8, "c"), Word(1.85, 2.0, "d")]
assert spk.speech_chunks(words) == [(0.0, 0.9), (1.5, 2.0)]
long = [Word(i * 0.3, i * 0.3 + 0.28, "w") for i in range(20)] # 6 s de fala corrida
chunks = spk.speech_chunks(long)
assert len(chunks) == 2 and all(b - a <= spk.CHUNK_MAX_S for a, b in chunks)
# Fim de frase fecha o trecho (a vez costuma trocar ali, mesmo sem pausa); frase curtinha não.
turn = [Word(0.0, 0.3, "bring"), Word(0.32, 0.6, "two."), Word(0.62, 0.8, "Ok."), Word(0.82, 1.0, "yes")]
assert spk.speech_chunks(turn) == [(0.0, 0.6), (0.62, 1.0)]
# ---------------------------------------------------------------- atribuição (vetores sintéticos)
def test_two_voices_streamer_by_loudness():
model = spk.build_model(_conversation())
assert model.active and len(model.voices) == 2 and model.reason.startswith("volume")
labels = [c.label for c in model.chunks]
assert labels == ["streamer", "other1"] * 12
def test_mouth_series_picks_the_streamer_even_when_quieter():
# Colega mais ALTO que o streamer (volume enganaria), mas a boca do streamer mexe nos trechos dele.
chunks = _conversation(db_streamer=-36.0, db_other=-29.0)
mine = {round(c.start, 1) for c in chunks if c.vec[0] > 0.5}
def mouth_of(a, b):
rng = np.random.default_rng(int(a * 10))
return float(0.30 + rng.normal(0, 0.02)) if round(a, 1) in mine else float(0.12 + rng.normal(0, 0.02))
model = spk.build_model(chunks, mouth_of)
# Boca e volume discordam → ninguém vira "outro" (seguro: fica tudo como do streamer).
assert not model.active and "apontam vozes diferentes" in model.reason
assert {c.label for c in model.chunks} == {STREAMER}
# Sem volume (sem prova do áudio), a boca decide sozinha.
for c in chunks:
c.db = None
model = spk.build_model(chunks, mouth_of)
assert model.active and model.reason.startswith("boca")
assert [c.label for c in model.chunks] == ["streamer", "other1"] * 12
def test_no_proof_or_single_voice_keeps_everything_as_streamer():
flat = _conversation(db_streamer=-30.0, db_other=-31.0) # 1 dB de diferença e sem câmera
model = spk.build_model(flat)
assert not model.active and {c.label for c in model.chunks} == {STREAMER}
solo = [VoiceChunk(i * 3.0, i * 3.0 + 2.0, -30.0, _noisy(A, i)) for i in range(20)]
model = spk.build_model(solo)
assert len(model.voices) == 1 and not model.active
words = [Word(0.2, 0.5, "oi")]
assert spk.label_words(words, model.chunks) == words # nada muda
def test_three_voices_are_numbered_and_short_chunks_follow_the_neighbor():
chunks = _conversation()
chunks += [VoiceChunk(200 + i * 4.0, 202 + i * 4.0, -37.0, _noisy(C, 300 + i)) for i in range(10)]
chunks.append(VoiceChunk(5.2, 5.5, -36.0, None)) # curto (sem vetor), logo depois do colega (3–5 s)
model = spk.build_model(chunks)
assert [v.label for v in model.voices] == ["streamer", "other1", "other2"]
words = [Word(0.1, 0.5, "yo"), Word(3.2, 3.6, "bro"), Word(5.25, 5.45, "ok"), Word(201.0, 201.4, "hey"),
Word(150.0, 150.3, "longe")]
got = [w.speaker for w in spk.label_words(words, model.chunks)]
assert got == ["streamer", "other1", "other1", "other2", "streamer"]
# Palavra que já veio com falante (4º elemento) não é trocada.
assert spk.label_words([Word(3.2, 3.6, "bro", "streamer")], model.chunks)[0].speaker == "streamer"
def test_transfer_keeps_speaker_on_retranscription():
old = [Word(1.0, 1.5, "a", "streamer"), Word(3.0, 3.5, "b", "other1")]
new = spk.transfer([Word(1.1, 1.4, "A"), Word(3.1, 3.3, "B")], old)
assert [w.speaker for w in new] == ["streamer", "other1"]
# ---------------------------------------------------------------- serviço (banco + embedder falso)
class FakeEmbedder:
"""Voz pelo volume do trecho: alto = voz A (streamer), baixo = voz B (colega)."""
def embed(self, samples):
if len(samples) < 8000:
return None
loud = float(np.sqrt(np.mean(samples.astype(np.float64) ** 2)))
return _noisy(A if loud > 4000 else B, int(loud)).tolist()
def _block_audio(start_s):
"""30 s: streamer (amplitude 8000) em [i·6, i·6+2] e colega (2000) em [i·6+3, i·6+5]; palavras nos mesmos tempos."""
sr = 16000
pcm = np.zeros(30 * sr, np.int16)
words = []
rng = np.random.default_rng(int(start_s))
for i in range(5):
for off, amp, text in ((0.0, 8000, "me"), (3.0, 2000, "you")):
a = i * 6.0 + off
pcm[int(a * sr): int((a + 2.0) * sr)] = (rng.standard_normal(int(2.0 * sr)) * amp).astype(np.int16)
words += [Word(start_s + a + k * 0.4, start_s + a + k * 0.4 + 0.35, text) for k in range(5)]
return pcm.tobytes(), words
def test_service_indexes_blocks_and_labels_the_timeline(env): # noqa: F811
_client, engine = env
engine.speakers = SpeakerService(engine, embedder_factory=FakeEmbedder)
sid = seed_session(engine, duration=120)
assert engine.speakers.view(sid)["active"] is False # nada indexado ainda
for seg in engine.db.all("SELECT * FROM segments WHERE session_id = ? ORDER BY idx", sid):
pcm, words = _block_audio(seg["start_s"])
assert engine.speakers.index_block(seg, pcm, words) == 10
engine.db.run("UPDATE segments SET words_json = ? WHERE session_id = ? AND idx = ?",
spk.dump_words(words), sid, seg["idx"])
view = engine.speakers.view(sid)
assert view["active"] and view["streamer"] == "streamer" and len(view["voices"]) == 2
words = engine.load_timeline(sid)["words"]
assert {w.speaker for w in words if w.word == "me"} == {"streamer"}
assert {w.speaker for w in words if w.word == "you"} == {"other1"}
# O mesmo vale para as palavras que o editor lê.
between = engine.words_between(sid, 0, 30)
assert {w.speaker for w in between if w.word == "you"} == {"other1"}
# Desligado (opção `speakers`), tudo volta a ser como antes.
engine.intel.opts["speakers"] = False
assert {w.speaker for w in engine.load_timeline(sid)["words"]} == {""}
def test_face_block_stores_mouth_series(env): # noqa: F811
_client, engine = env
class Meter:
def feed(self, rgb):
return 0.2
service = FaceService(engine, store=object(), tracker_factory=lambda _b=None: FakeTracker(), detector_factory=None,
reader=fake_reader, grabber=None, sizer=lambda _f: (1280, 720), mouth_factory=Meter)
engine.face = service
sid = seed_session(engine, duration=60)
set_cam(engine, sid, CAM)
assert service.step() is True
data = json.loads(engine.db.get("SELECT data_json FROM face_track WHERE session_id = ? AND idx = 0", sid)["data_json"])
assert len(data["mouth"]) == 180 and data["mouth"][0] == 0.2
# Sem medidor (store falso, sem YuNet): o bloco sai igual ao de antes, sem `mouth`.
service._mouth_factory = lambda: (_ for _ in ()).throw(RuntimeError("sem yunet"))
assert service.step() is True
data = json.loads(engine.db.get("SELECT data_json FROM face_track WHERE session_id = ? AND idx = 1", sid)["data_json"])
assert "mouth" not in data and data["frames"] == 180
# ---------------------------------------------------------------- legenda
def _dialogue(ass: str) -> list[tuple[float, float, str, str]]:
out = []
for line in ass.splitlines():
if line.startswith("Dialogue: 0,"):
parts = line.split(",", 9)
h, m, s = parts[1].split(":")
a = int(h) * 3600 + int(m) * 60 + float(s)
h, m, s = parts[2].split(":")
b = int(h) * 3600 + int(m) * 60 + float(s)
out.append((a, b, parts[3], parts[9]))
return out
def _ypos(text: str) -> int:
return int(re.search(r"\\pos\(\d+,(\d+)\)", text).group(1))
TALK = [Word(1.0, 1.3, "olha"), Word(1.35, 1.6, "isso"), Word(1.62, 1.9, "mano"),
Word(1.7, 2.0, "for", "other1"), Word(2.05, 2.4, "real", "other1"), Word(2.5, 2.9, "bro", "other1"),
Word(3.0, 3.3, "caiu"), Word(3.35, 3.7, "aqui")]
def test_caption_blocks_never_mix_speakers_and_overlap_only_across_lanes():
blocks = subtitles.caption_blocks(TALK, 3)
for group, _end in blocks:
assert len({w.speaker or STREAMER for w in group}) == 1
lanes = {"streamer": [], "other": []}
for group, end in blocks:
lanes[subtitles.lane(group[0].speaker)].append((group[0].start, end))
for spans in lanes.values(): # dentro da faixa, nunca dois blocos ao mesmo tempo
assert all(a1 <= a2 and e1 <= a2 + 1e-9 for (a1, e1), (a2, _) in zip(spans, spans[1:]))
# As duas faixas se sobrepõem (os dois falando juntos em 1,7–1,9 s).
(s_a, s_e), (o_a, o_e) = lanes["streamer"][0], lanes["other"][0]
assert o_a < s_e
@pytest.mark.parametrize("aspect", ["vertical", "horizontal"])
def test_build_ass_puts_the_other_speaker_below_in_another_color(aspect):
cfg = layout.merge({"format": {"aspect": aspect}, "caption": {"otherColor": "#00FF00"}})
ass = subtitles.build_ass(TALK, 0.0, 6.0, "", cfg)
assert "Style: CaptionOther," in ass and "&H0000FF00&" in ass
events = _dialogue(ass)
mine = [e for e in events if e[2] == "Caption"]
other = [e for e in events if e[2] == "CaptionOther"]
assert mine and other
y, oy = _ypos(mine[0][3]), _ypos(other[0][3])
assert y == layout.caption_y(cfg) and oy > y # embaixo
assert all("FOR" in e[3] or "REAL" in e[3] or "BRO" in e[3] for e in other)
assert not any("OLHA" in e[3] for e in other) and not any("BRO" in e[3] for e in mine)
# Os dois na tela ao mesmo tempo (1,7–1,9 s), em alturas diferentes.
assert any(a < 1.85 < b for a, b, _, _ in mine) and any(a < 1.85 < b for a, b, _, _ in other)
# Só o streamer: o ASS é exatamente o de antes (sem o estilo novo).
solo = [Word(w.start, w.end, w.word) for w in TALK if not w.speaker]
assert "CaptionOther" not in subtitles.build_ass(solo, 0.0, 6.0, "", cfg)
def _project(tmp_path, words, **over):
p = {
"schema": 1, "source": {"sessionId": 1, "lo": 0, "hi": 60}, "settings": {},
"cuts": [{"start": 0, "end": 20, "mode": None}],
"captions": {"enabled": True, "preset": "pop", "style": {}, "emphColor": "#22E36B",
"words": [editor.word_item(w) for w in words]},
"overlays": [], "progress": {"enabled": False, "color": "#FF7A1A", "height": 12},
"audio": {"volume": 1, "narration": [], "music": None, "sfx": []},
"cover": {"enabled": False, "t": 1, "text": ""},
}
p.update(over)
return validate(p, tmp_path)
@pytest.mark.parametrize("preset", editor.CAPTION_PRESETS)
def test_project_ass_two_lines_for_every_preset(tmp_path, preset):
p = _project(tmp_path, TALK)
assert p["captions"]["words"][3]["sp"] == "other1" and "sp" not in p["captions"]["words"][0]
p["captions"]["preset"] = preset
cfg = editor.effective_settings(layout.merge(), p)
events = _dialogue(editor_render.build_project_ass(p, cfg, ""))
mine = [e for e in events if e[2] in ("Caption", "Karaoke")]
other = [e for e in events if e[2] in ("CaptionOther", "KaraokeOther")]
assert mine and other
assert min(_ypos(e[3]) for e in other) > max(_ypos(e[3]) for e in mine)
for group in (mine, other): # na mesma linha, nunca dois eventos ao mesmo tempo
spans = sorted((a, b) for a, b, _, _ in group)
assert all(b1 <= a2 + 0.011 for (_, b1), (a2, _) in zip(spans, spans[1:]))
def test_hook_dodge_moves_both_lines_and_translation_follows_speaker(tmp_path):
hook = {"id": "hook", "kind": "hook", "text": "NAO ACREDITO", "start": 0, "end": 3, "y": 0.74, "size": 70,
"color": "#FFFFFF", "box": "#E11D48", "anim": "pop"}
p = _project(tmp_path, TALK, overlays=[hook])
cfg = editor.effective_settings(layout.merge({"dev": {"translation": True}}), p)
lines = [{"s": 1.0, "e": 1.9, "text": "olha isso mano", "pt": "olha isso", "speaker": "streamer"},
{"s": 1.7, "e": 2.9, "text": "for real bro", "pt": "sério mano", "speaker": "other1"}]
events = _dialogue(editor_render.build_project_ass(p, cfg, "", translation=lines))
cap_y = layout.caption_y(cfg)
mine = [e for e in events if e[2] == "Caption"]
other = [e for e in events if e[2] == "CaptionOther"]
trans = {e[3].split("}")[-1]: _ypos(e[3]) for e in events if e[2] == "Trans"}
# Gancho em cima da legenda nos 3 primeiros segundos: as duas linhas desviam juntas, a do colega embaixo.
y, oy = _ypos(mine[0][3]), _ypos(other[0][3])
assert y != cap_y and oy > y
hook_top = int(1920 * 0.74) - int(70 * 0.9)
hook_bottom = int(1920 * 0.74) + int(70 * 0.9)
assert y - 84 >= hook_bottom or oy + 60 <= hook_top
# Tradução de cada um embaixo da linha de quem fala, entre as duas a do streamer.
assert y < trans["olha isso"] < oy < trans["sério mano"]
def test_old_project_without_speaker_renders_like_before(tmp_path):
words = [Word(w.start, w.end, w.word) for w in TALK]
p = _project(tmp_path, words)
ass = editor_render.build_project_ass(p, editor.effective_settings(layout.merge(), p), "")
assert "CaptionOther" not in ass
assert {_ypos(e[3]) for e in _dialogue(ass)} == {layout.caption_y(layout.merge())}
def test_layout_validates_other_color_and_size():
assert layout.DEFAULTS["caption"]["otherColor"] == "#5CE1FF" and layout.DEFAULTS["caption"]["otherSize"] is None
ok = layout.validate({"caption": {"otherColor": "#ff00aa", "otherSize": 60}})["caption"]
assert ok == {"otherColor": "#FF00AA", "otherSize": 60}
assert layout.validate({"caption": {"otherSize": None}})["caption"]["otherSize"] is None
with pytest.raises(layout.LayoutError):
layout.validate({"caption": {"otherColor": "azul"}})
with pytest.raises(layout.LayoutError):
layout.validate({"caption": {"otherSize": 10}})
cap = layout.merge()["caption"]
assert subtitles.other_size(cap) == round(84 * 0.85)
assert subtitles.other_size({**cap, "otherSize": 50}) == 50
def test_project_rejects_bad_speaker(tmp_path):
p = _project(tmp_path, TALK)
p["captions"]["words"][0]["sp"] = "ninguém; drop"
with pytest.raises(editor.EditError):
validate(p, tmp_path)
# ---------------------------------------------------------------- tradução e juízes
def test_phrases_never_mix_speakers():
got = subtitles.phrases(TALK, with_speaker=True)
assert got == [(1.0, 1.9, "olha isso mano", "streamer"), (1.7, 2.9, "for real bro", "other1"),
(3.0, 3.7, "caiu aqui", "streamer")]
assert subtitles.phrases(TALK)[0] == (1.0, 1.9, "olha isso mano") # formato antigo continua
def test_translation_lines_get_speaker(env): # noqa: F811
client, engine = env
engine.judge = EnglishJudge()
sid = _english_session(engine)
words = [[281.0, 281.3, "No", "streamer"], [281.3, 281.6, "way", "streamer"], [281.6, 282.0, "bro.", "streamer"],
[282.1, 282.5, "Told", "other1"], [282.5, 283.0, "you.", "other1"]]
engine.db.run("UPDATE segments SET words_json = ? WHERE session_id = ? AND idx = 9", json.dumps(words), sid)
add_clip(engine, sid, 1, "candidate", 285, 270, 340)
engine.judge_clip(engine.db.get("SELECT * FROM clips WHERE id = 1"))
lines = json.loads(engine.db.get("SELECT translation_json FROM clips WHERE id = 1")["translation_json"])["lines"]
assert [(x["text"], x["speaker"]) for x in lines] == [("No way bro.", "streamer"), ("Told you.", "other1")]
# O juiz viu quem fala.
prompt = engine.judge.seen["judge"][1]
assert "] No way bro." in prompt and "[other1] Told you." in prompt and "Never attribute to" in prompt
# Tradução antiga (sem `speaker`) é refeita na próxima vez.
engine.db.run("UPDATE clips SET translation_json = ? WHERE id = 1",
json.dumps({"startS": 0, "endS": 1e6, "lines": [{"s": 1, "e": 2, "text": "x", "pt": "y"}]}))
before = len(engine.judge.translated)
engine.translate_clip(1)
assert len(engine.judge.translated) == before + 1
def test_judge_transcript_tags_only_when_someone_else_speaks():
plain = [Word(10.0, 10.3, "olha"), Word(10.4, 10.8, "isso")]
assert _transcript(plain, 0, 60, "pt", "Tfue") == "[10s] olha isso"
mixed = plain + [Word(11.0, 11.3, "bro", "other1"), Word(11.4, 11.8, "nice", "other1"), Word(12.0, 12.2, "ok")]
out = _transcript(mixed, 0, 60, "en", "Tfue").splitlines()
assert out[0].startswith("Each line is tagged") and "[Tfue]" in out[0]
assert out[1:] == ["[10s] [Tfue] olha isso", "[11s] [other1] bro nice", "[12s] [Tfue] ok"]
def test_voice_model_failure_waits_before_retrying(env): # noqa: F811
_client, engine = env
calls = []
def broken():
calls.append(1)
raise RuntimeError("sem rede")
engine.speakers = SpeakerService(engine, embedder_factory=broken)
sid = seed_session(engine, duration=60)
seg = engine.db.get("SELECT * FROM segments WHERE session_id = ? AND idx = 0", sid)
pcm, words = _block_audio(0.0)
for _ in range(3):
with pytest.raises(RuntimeError):
engine.speakers.index_block(seg, pcm, words)
assert len(calls) == 1 # não baixa o modelo a cada bloco
assert engine.load_timeline(sid)["words"][0].speaker == "" # sem voz indexada: como antes