- editor_api: importa dub e speakers juntos. - voice_chunks (falantes) vira a migração v15, depois da v14 do vídeo final (antes era criada fora das migrações). - dub.voice_for: qualquer voz que não é o streamer (other1, other2…) usa a segunda voz da dublagem. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
338 lines
18 KiB
Python
338 lines
18 KiB
Python
"""Dublagem em português (camada de desenvolvimento, doc/clipador/DUBLAGEM.md) — sempre com TTS falso."""
|
||
import io
|
||
import math
|
||
import shutil
|
||
import struct
|
||
import wave
|
||
from array import array
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from app import dub, editor, editor_render, layout, media
|
||
|
||
FFMPEG = shutil.which("ffmpeg")
|
||
CUTS = [{"start": 10.0, "end": 20.0, "mode": None}, {"start": 25.0, "end": 40.0, "mode": None}] # 25 s de saída
|
||
|
||
|
||
def fake_wav(seconds: float, rate: int = 24000, freq: float = 440.0) -> bytes:
|
||
"""WAV mono 16 bits com um seno (o "TTS" dos testes)."""
|
||
n = int(seconds * rate)
|
||
buf = io.BytesIO()
|
||
with wave.open(buf, "wb") as w:
|
||
w.setnchannels(1)
|
||
w.setsampwidth(2)
|
||
w.setframerate(rate)
|
||
w.writeframes(b"".join(struct.pack("<h", int(12000 * math.sin(2 * math.pi * freq * i / rate))) for i in range(n)))
|
||
return buf.getvalue()
|
||
|
||
|
||
class FakeSynth:
|
||
"""Duração da fala proporcional ao texto (0,08 s por letra); registra cada chamada."""
|
||
|
||
def __init__(self, per_char: float = 0.08, fail_voices: tuple[str, ...] = ()):
|
||
self.per_char, self.fail_voices, self.calls = per_char, fail_voices, []
|
||
|
||
def __call__(self, text: str, voice: str) -> bytes:
|
||
self.calls.append((voice, text))
|
||
if voice in self.fail_voices:
|
||
raise dub.DubError(f"voz {voice} respondeu 502")
|
||
return fake_wav(max(0.2, len(text) * self.per_char))
|
||
|
||
|
||
def opts(**over):
|
||
return {**dub.options(layout.merge({"dev": {"dub": True}})), **over}
|
||
|
||
|
||
# ---------------------------------------------------------------- plano no tempo de saída
|
||
def test_lines_go_to_output_time_and_skip_removed_parts():
|
||
lines = [
|
||
{"s": 11.0, "e": 12.0, "text": "a", "pt": "primeira"},
|
||
{"s": 21.0, "e": 23.0, "text": "b", "pt": "no trecho removido"}, # 20–25 foi cortado
|
||
{"s": 26.0, "e": 27.0, "text": "c", "pt": " depois do corte "},
|
||
{"s": 30.0, "e": 31.0, "text": "d", "pt": ""}, # sem tradução: não dubla
|
||
]
|
||
plan = dub.plan_lines(lines, CUTS, opts())
|
||
assert [(p.at, p.text) for p in plan] == [(1.0, "primeira"), (11.0, "depois do corte")]
|
||
assert all(p.voice == layout.DUB_VOICE for p in plan)
|
||
|
||
|
||
def test_speaker_other_uses_second_voice_and_missing_falls_back():
|
||
lines = [{"s": 11, "e": 12, "pt": "eu"}, {"s": 13, "e": 14, "pt": "outro", "speaker": "other"},
|
||
{"s": 15, "e": 16, "pt": "eu de novo", "speaker": "streamer"}, {"s": 17, "e": 18, "pt": "?", "speaker": "x"}]
|
||
plan = dub.plan_lines(lines, CUTS, opts(voice="pt-br-antonio", voiceOther="pt-br-cadu"))
|
||
assert [p.voice for p in plan] == ["pt-br-antonio", "pt-br-cadu", "pt-br-antonio", "pt-br-antonio"]
|
||
assert plan[1].speaker == "other"
|
||
# Sem `dubVoiceOther` na configuração salva: a padrão do Clipador; vazio = a mesma voz do streamer.
|
||
assert dub.options(layout.merge({}))["voiceOther"] == layout.DUB_VOICE_OTHER
|
||
assert dub.options({"dev": {"dubVoice": "pt-br-faber", "dubVoiceOther": ""}})["voiceOther"] == "pt-br-faber"
|
||
|
||
|
||
# ---------------------------------------------------------------- encaixe
|
||
def _line(at: float) -> dub.DubLine:
|
||
return dub.DubLine(at, "x", "v")
|
||
|
||
|
||
def test_fit_prefers_natural_speed_spills_over_and_never_overlaps_or_drifts():
|
||
lines = [_line(0.0), _line(2.0), _line(4.0), _line(5.0), _line(8.0), _line(9.0), _line(10.0)]
|
||
durations = [1.5, 2.6, 3.0, 0.9, 5.0, 0.5, 0.5]
|
||
cues = dub.fit(lines, durations, total=14.0)
|
||
by = {c.index: c for c in cues}
|
||
assert by[0].tempo == 1.0 and by[0].length == 1.5 and not by[0].cut # cabe: natural
|
||
assert by[1].tempo == 1.0 and by[1].at + by[1].length > 4.0 # escorre além da próxima frase, sem acelerar
|
||
assert by[2].at > 4.0 and by[2].at <= 4.0 + dub.MAX_LATE_S # começa depois da anterior, perto da original
|
||
# 3 s sem espaço nem escorrendo: acelera pouco (no máximo MAX_TEMPO) e corta com fade.
|
||
assert 1.0 < by[2].tempo <= dub.MAX_TEMPO and by[2].cut
|
||
for c in cues: # o atraso de cada fala é limitado pela PRÓPRIA frase original: não acumula
|
||
assert lines[c.index].at <= c.at <= lines[c.index].at + dub.MAX_LATE_S
|
||
assert c.tempo <= dub.MAX_TEMPO and c.at + c.length <= 14.0
|
||
ordered = sorted(cues, key=lambda c: c.at)
|
||
for a, b in zip(ordered, ordered[1:]):
|
||
assert a.at + a.length + dub.GAP_S <= b.at + 1e-6
|
||
|
||
|
||
def test_phrase_split_mid_sentence_is_joined():
|
||
lines = [{"s": 11.0, "e": 12.8, "pt": "se a gente fizer rápido deve dar"}, {"s": 12.9, "e": 13.1, "pt": "tudo certo."},
|
||
{"s": 13.2, "e": 13.6, "pt": "Beleza."}, # frase completa antes: não junta
|
||
{"s": 13.7, "e": 14.0, "pt": "Como assim"}, {"s": 14.1, "e": 14.5, "pt": "mano?", "speaker": "other"}]
|
||
plan = dub.plan_lines(lines, CUTS, opts())
|
||
assert [(p.at, p.text) for p in plan] == [(1.0, "se a gente fizer rápido deve dar tudo certo."), (3.2, "Beleza."),
|
||
(3.7, "Como assim"), (4.1, "mano?")] # falante diferente: não junta
|
||
|
||
|
||
def test_fit_skips_phrase_without_room_or_audio():
|
||
cues = dub.fit([_line(0.0), _line(0.05), _line(3.0)], [2.0, 0.0, 1.0], total=3.2)
|
||
assert [c.index for c in cues] == [0] # 1 falhou (não limita a 0); 2 com 0,2 s até o fim: não dá para entender
|
||
# Frase fora do trecho (None): não toca, mas segura a anterior no lugar dela.
|
||
cues = dub.fit([_line(0.0), _line(1.0)], [2.0, None], total=5.0)
|
||
assert [c.index for c in cues] == [0] and cues[0].at + cues[0].length <= 1.0 + dub.MAX_LATE_S
|
||
|
||
|
||
def test_duck_spans_merge_short_gaps_and_ramp_expression():
|
||
cues = [dub.Cue(1.0, 1.0, 1.0, False, 0), dub.Cue(2.3, 1.0, 1.0, False, 1), dub.Cue(6.0, 1.0, 1.0, False, 2)]
|
||
spans = dub.duck_spans(cues)
|
||
assert spans == [(1.0, 3.3), (6.0, 7.0)]
|
||
factors = dub.duck_factors(spans, 0.25)
|
||
env = {"clip": lambda x, lo, hi: max(lo, min(hi, x)), "min": min}
|
||
|
||
def gain(t: float) -> float:
|
||
value = 1.0
|
||
for f in factors:
|
||
value *= eval(f, env, {"t": t}) # noqa: S307 — expressão gerada pelo próprio teste
|
||
return value
|
||
|
||
assert gain(0.0) == 1.0 and gain(5.0) == 1.0 and gain(9.0) == 1.0 # fora das falas: normal
|
||
assert gain(2.0) == pytest.approx(0.25) and gain(6.5) == pytest.approx(0.25)
|
||
assert 0.25 < gain(1.0 - dub.RAMP_S / 2) < 1.0 # rampa antes de começar
|
||
assert dub.duck_factors(spans, 1.0) == []
|
||
|
||
|
||
def test_gain_follows_original_level():
|
||
assert dub.gain_for(None) == 0.0
|
||
assert dub.gain_for(-32.0) == pytest.approx(-32.0 + dub.LEVEL_OVER_ORIGINAL_DB - dub.TTS_LEVEL_DB) # live baixa
|
||
assert dub.gain_for(-5.0) == dub.GAIN_RANGE_DB[1] and dub.gain_for(-90.0) == dub.GAIN_RANGE_DB[0]
|
||
|
||
|
||
# ---------------------------------------------------------------- cache
|
||
def test_cache_synthesizes_once_per_voice_and_text(tmp_path):
|
||
synth = FakeSynth()
|
||
cache = dub.TtsCache(tmp_path / "dub", synth, ffmpeg=None)
|
||
f1, d1 = cache.get("pt-br-antonio", "olha isso")
|
||
f2, d2 = cache.get("pt-br-antonio", "olha isso")
|
||
assert f1 == f2 and d1 == d2 == pytest.approx(len("olha isso") * 0.08, abs=0.01)
|
||
assert len(synth.calls) == 1
|
||
cache.get("pt-br-cadu", "olha isso")
|
||
assert len(synth.calls) == 2 and len(list((tmp_path / "dub").glob("*.wav"))) == 2
|
||
# Outro processo (render da fila depois da prévia) acha o mesmo arquivo.
|
||
again = dub.TtsCache(tmp_path / "dub", FakeSynth(), ffmpeg=None)
|
||
assert again.get("pt-br-antonio", "olha isso")[0] == f1 and again.synth_calls == 0
|
||
# Limpeza: só o que está sem uso há mais de 30 dias.
|
||
import os
|
||
import time
|
||
old = cache.path("pt-br-cadu", "olha isso")
|
||
os.utime(old, (time.time() - 40 * 86400,) * 2)
|
||
assert cache.prune() == 1 and not old.exists() and f1.exists()
|
||
|
||
|
||
def test_voice_failure_falls_back_to_offline_voice(tmp_path):
|
||
synth = FakeSynth(fail_voices=("pt-br-antonio",))
|
||
dubber = dub.Dubber(dub.TtsCache(tmp_path, synth, ffmpeg=None), ffmpeg=None)
|
||
path, _ = dubber._speech(dub.DubLine(0.0, "oi", "pt-br-antonio"))
|
||
assert synth.calls[-1] == (dub.FALLBACK_VOICE, "oi") and path.is_file()
|
||
|
||
|
||
def test_engine_without_voice_service_builds_nothing(tmp_path):
|
||
synth = dub.voice_synth(None, None)
|
||
with pytest.raises(dub.DubError):
|
||
synth("oi", "pt-br-antonio")
|
||
dubber = dub.Dubber(dub.TtsCache(tmp_path, synth, ffmpeg="ffmpeg"), ffmpeg="ffmpeg")
|
||
cfg = layout.merge({"dev": {"dub": True}})
|
||
assert dubber.build([{"s": 11, "e": 12, "pt": "oi"}], CUTS, cfg, 0, 25, tmp_path, "x") is None
|
||
|
||
|
||
# ---------------------------------------------------------------- configuração
|
||
def test_dev_group_validates_new_keys():
|
||
ok = layout.validate({"dev": {"dub": True, "dubVoice": "pt-br-faber", "dubVoiceOther": "kokoro-pm-alex", "dubDuck": 0.3}})
|
||
assert ok["dev"] == {"dub": True, "dubVoice": "pt-br-faber", "dubVoiceOther": "kokoro-pm-alex", "dubDuck": 0.3}
|
||
for bad in ({"dubVoice": "../x"}, {"dubVoiceOther": ""}, {"dubDuck": 1.5}, {"dubDuck": "alto"}):
|
||
with pytest.raises(layout.LayoutError):
|
||
layout.validate({"dev": bad})
|
||
merged = layout.merge({"dev": {"dub": True, "dubDuck": 0.1}})
|
||
assert merged["dev"]["dubDuck"] == 0.1 and merged["dev"]["dubVoice"] == layout.DUB_VOICE
|
||
assert layout.dev_flags(merged, dev=False) == {"translation": False, "dub": False}
|
||
|
||
|
||
# ---------------------------------------------------------------- render: só no vídeo de revisão
|
||
class SpyDubber:
|
||
def __init__(self):
|
||
self.calls, self.level = 0, None
|
||
|
||
def build(self, translation, cuts, cfg, t0, t1, work, stem, level_db=None):
|
||
self.calls, self.level = self.calls + 1, level_db
|
||
file = work / f"{stem}.dub.wav"
|
||
file.write_bytes(fake_wav(0.5))
|
||
return dub.DubTrack(file, [(1.0, 2.0)], 0.25, 1, 0.1, 1)
|
||
|
||
|
||
def _validated(tmp_path: Path) -> dict:
|
||
data = {"schema": 1, "source": {"sessionId": 1, "lo": 0, "hi": 60}, "settings": {}, "cuts": CUTS,
|
||
"captions": {"enabled": False, "preset": "pop", "style": {}, "emphColor": "#22E36B", "words": []},
|
||
"overlays": [], "progress": {"enabled": False, "color": "#FF7A1A", "height": 12},
|
||
"audio": {"volume": 1, "narration": [], "music": None, "sfx": []}, "cover": {"enabled": False, "t": 12, "text": ""}}
|
||
return editor.validate_project(data, session_id=1, session_duration=100, repo_root=tmp_path, tts_dir=tmp_path / "tts",
|
||
assets_dir=tmp_path / "assets")
|
||
|
||
|
||
@pytest.mark.parametrize("dev_render", [True, False])
|
||
def test_render_mixes_dub_only_in_review_video(tmp_path, monkeypatch, dev_render):
|
||
seen: list[list[str]] = []
|
||
|
||
def fake_run(args, timeout, cwd=None):
|
||
seen.append(args)
|
||
(Path(cwd) / args[-1]).write_bytes(b"mp4")
|
||
return b""
|
||
|
||
monkeypatch.setattr(media, "run", fake_run)
|
||
monkeypatch.setattr(editor_render, "mean_volume", lambda *a, **k: -32.0)
|
||
(tmp_path / "seg.ts").write_bytes(b"")
|
||
spy = SpyDubber()
|
||
lines = [{"s": 11.0, "e": 12.0, "text": "No way.", "pt": "Não acredito."}]
|
||
info = editor_render.render_project(
|
||
"ffmpeg", None, [], [(tmp_path / "seg.ts", 0.0, 100.0)], _validated(tmp_path),
|
||
layout.merge({"dev": {"dub": True}}), "x", tmp_path / "out" / "clip.mp4", tmp_path,
|
||
translation=lines, dev=dev_render, dubber=spy)
|
||
graph = seen[-1][seen[-1].index("-filter_complex") + 1]
|
||
if dev_render:
|
||
assert spy.calls == 1 and info["dub"]["cues"] == 1 and spy.level == -32.0 # nivelada pelo original
|
||
assert any(a.endswith("clip.dub.wav") for a in seen[-1]) and "0.750*clip(" in graph
|
||
assert not (tmp_path / "out" / "clip.dub.wav").exists() # faixa temporária apagada
|
||
else: # vídeo final (depois de aprovar): nada da dublagem
|
||
assert spy.calls == 0 and info["dub"] is None
|
||
assert "clip(" not in graph and not any("dub" in a for a in seen[-1])
|
||
|
||
|
||
def test_render_without_dub_flag_does_not_call_dubber(tmp_path, monkeypatch):
|
||
monkeypatch.setattr(media, "run", lambda args, timeout, cwd=None: (Path(cwd) / args[-1]).write_bytes(b"x") and b"")
|
||
(tmp_path / "seg.ts").write_bytes(b"")
|
||
spy = SpyDubber()
|
||
editor_render.render_project("ffmpeg", None, [], [(tmp_path / "seg.ts", 0.0, 100.0)], _validated(tmp_path),
|
||
layout.merge({}), "x", tmp_path / "out" / "c.mp4", tmp_path,
|
||
translation=[{"s": 11, "e": 12, "pt": "oi"}], dubber=spy)
|
||
assert spy.calls == 0
|
||
|
||
|
||
# ---------------------------------------------------------------- FFmpeg de verdade (faixa)
|
||
def _samples(file: Path) -> tuple[array, int, int]:
|
||
with wave.open(str(file)) as w:
|
||
data = array("h", w.readframes(w.getnframes()))
|
||
return data, w.getframerate(), w.getnchannels()
|
||
|
||
|
||
def _peak(data: array, rate: int, channels: int, a: float, b: float) -> int:
|
||
chunk = data[int(a * rate) * channels:int(b * rate) * channels]
|
||
return max((abs(x) for x in chunk), default=0)
|
||
|
||
|
||
@pytest.mark.skipif(not FFMPEG, reason="FFmpeg não instalado")
|
||
def test_track_places_speech_at_phrase_start_and_uses_cache(tmp_path):
|
||
synth = FakeSynth(per_char=0.1)
|
||
dubber = dub.Dubber(dub.TtsCache(tmp_path / "cache", synth, FFMPEG), FFMPEG)
|
||
cfg = layout.merge({"dev": {"dub": True}})
|
||
lines = [{"s": 11.0, "e": 12.0, "pt": "frase um"}, # saída 1,0 s; ~0,8 s de fala
|
||
{"s": 22.0, "e": 23.0, "pt": "removida"}, # trecho removido: não dubla
|
||
{"s": 27.0, "e": 28.0, "pt": "frase dois"}] # saída 12,0 s
|
||
track = dubber.build(lines, CUTS, cfg, 0.0, 25.0, tmp_path, "full")
|
||
assert track and track.cues == 2 and track.synthesized == 2
|
||
data, rate, channels = _samples(track.file)
|
||
assert channels == 2 and abs(len(data) / channels / rate - 25.0) < 0.05
|
||
assert _peak(data, rate, channels, 0.0, 0.95) < 200 # silêncio antes da 1ª frase
|
||
assert _peak(data, rate, channels, 1.1, 1.6) > 3000 # fala no lugar
|
||
assert _peak(data, rate, channels, 3.0, 11.9) < 200
|
||
assert _peak(data, rate, channels, 12.1, 12.6) > 3000
|
||
assert track.spans[0][0] == pytest.approx(1.0, abs=0.01)
|
||
# Prévia de 5 s a partir de 10 s: só a frase que toca ali; tudo do cache.
|
||
preview = dubber.build(lines, CUTS, cfg, 10.0, 15.0, tmp_path, "preview")
|
||
assert preview and preview.cues == 1 and preview.synthesized == 0 and len(synth.calls) == 2
|
||
data, rate, channels = _samples(preview.file)
|
||
assert abs(len(data) / channels / rate - 5.0) < 0.05 and _peak(data, rate, channels, 2.1, 2.6) > 3000
|
||
assert preview.spans[0][0] == pytest.approx(2.0, abs=0.01)
|
||
|
||
|
||
@pytest.mark.skipif(not FFMPEG, reason="FFmpeg não instalado")
|
||
def test_real_render_with_dub(tmp_path):
|
||
import subprocess
|
||
seg = tmp_path / "seg.ts"
|
||
subprocess.run([FFMPEG, "-y", "-v", "error", "-f", "lavfi", "-i", "testsrc2=s=640x360:r=30:d=8", "-f", "lavfi", "-i",
|
||
"sine=frequency=300:d=8", "-c:v", "libx264", "-preset", "ultrafast", "-c:a", "aac", "-shortest",
|
||
"-f", "mpegts", str(seg)], check=True, capture_output=True)
|
||
project = _validated(tmp_path)
|
||
project["cuts"] = [{"start": 0.0, "end": 3.0, "mode": None, "muted": False}, {"start": 5.0, "end": 8.0, "mode": None, "muted": False}]
|
||
dubber = dub.Dubber(dub.TtsCache(tmp_path / "cache", FakeSynth(), FFMPEG), FFMPEG)
|
||
out = tmp_path / "out" / "clip.mp4"
|
||
info = editor_render.render_project(
|
||
FFMPEG, shutil.which("ffprobe"), ["-c:v", "libx264", "-preset", "ultrafast"], [(seg, 0.0, 8.0)], project,
|
||
layout.merge({"dev": {"dub": True}}), "x", out, tmp_path, translation=[{"s": 1.0, "e": 2.0, "pt": "olha só"},
|
||
{"s": 5.5, "e": 6.5, "pt": "voltou"}],
|
||
dubber=dubber)
|
||
assert out.is_file() and info["dub"]["cues"] == 2
|
||
assert media.probe_duration(shutil.which("ffprobe"), out) == pytest.approx(6.0, abs=0.25)
|
||
assert not list((tmp_path / "out").glob("*.dub.wav"))
|
||
|
||
|
||
# ---------------------------------------------------------------- rotas: prévia do editor e render da fila
|
||
from .test_editor import _lavfi_segments, api # noqa: E402,F401 — fixture do editor (sessão, blocos, corte 7)
|
||
|
||
|
||
@pytest.mark.skipif(not FFMPEG, reason="FFmpeg não instalado")
|
||
def test_preview_and_queue_render_dub_the_same_way(api, monkeypatch):
|
||
from .test_sources_and_api import KEY
|
||
client, engine, root = api
|
||
_lavfi_segments(root / "data" / "s1")
|
||
h = {"X-Service-Key": KEY}
|
||
monkeypatch.setattr(engine.intel, "settings_for", lambda session: layout.merge({"dev": {"dub": True}}))
|
||
lines = [{"s": 3.0, "e": 4.0, "text": "No way.", "pt": "Não acredito."},
|
||
{"s": 6.0, "e": 7.0, "text": "Bro.", "pt": "Mano.", "speaker": "other"}]
|
||
monkeypatch.setattr(engine, "translate_clip", lambda clip_id, lo=None, hi=None, force=False: {"lines": lines})
|
||
synth = FakeSynth()
|
||
engine._dubber = dub.Dubber(dub.TtsCache(root / "data" / "dub", synth, FFMPEG), FFMPEG)
|
||
assert dub.for_engine(engine) is engine._dubber
|
||
project = client.get("/v1/clips/7/edit", headers=h).json()["project"]
|
||
assert client.put("/v1/clips/7/edit", headers=h, json={"project": project, "rev": 0}).status_code == 200
|
||
prev = client.post("/v1/clips/7/preview-render", headers=h, json={"fromS": 0, "durationS": 3})
|
||
assert prev.status_code == 200, prev.text
|
||
assert synth.calls == [(layout.DUB_VOICE, "Não acredito.")] # a prévia só sintetiza o que toca nela
|
||
client.post("/v1/clips/7/export", headers=h)
|
||
engine.render_pending()
|
||
clip = client.get("/v1/clips/7", headers=h).json()
|
||
assert clip["status"] == "ready", clip["error"]
|
||
# Fila: a 1ª fala veio do cache da prévia; a do "other" saiu com a segunda voz.
|
||
assert synth.calls == [(layout.DUB_VOICE, "Não acredito."), (layout.DUB_VOICE_OTHER, "Mano.")]
|
||
|
||
|
||
def test_numbered_other_speakers_use_the_second_voice():
|
||
"""Integração com FALANTES.md: a separação entrega `other1`, `other2`…; todas usam a segunda voz."""
|
||
from app.dub import voice_for
|
||
opts = {"voice": "a", "voiceOther": "b"}
|
||
assert voice_for({"speaker": "other1"}, opts) == "b" and voice_for({"speaker": "other2"}, opts) == "b"
|
||
assert voice_for({"speaker": "streamer"}, opts) == "a" and voice_for({}, opts) == "a"
|