Clipador: Equipe de IA (papéis com IA própria), juiz de contexto e linha do tempo JSON
- roles.py: varredura, analista, editor e juiz, cada um com provedor, modelo, raciocínio (Codex), liga/desliga (o editor não desliga) e instruções do administrador somadas às padrão; gravado em settings `judge.roles`, com migração automática da configuração antiga (provedor/modelo único vira o de todos; o modelo do analista é respeitado). A reserva vale por papel. - Juiz de contexto (só texto): vê só o que vai no corte (+20 s antes/15 s depois marcados out=1), escreve o que um espectador frio entende e responde sim/não; context_judge.decide decide pelo código: aprova, ajusta o trecho e confere uma vez de novo, frase de contexto na tela (vira o texto de abertura do projeto) ou descarta com motivo em português (forçável). Corte manual só recebe aviso. - timeline.py: linha do tempo JSON curta (fala com falante, chat, rosto, volume e a frase de tela que a varredura/analista escrevem ao ver os quadros, em scene_notes); mosaico 4x2 (media.mosaic) no lugar dos quadros soltos; perfil do jogo enxuto. O formato antigo fica atrás da opção timelineJson. - Medição (F0): ai_calls por chamada (papel, IA, tamanho do prompt/imagens, tokens quando a CLI informa: codex --json, claude e gemini), avaliação do dono (owner_label: bom/ruim/sem contexto) e GET /v1/judge/report (taxa de aprovação, decisões do juiz, concordância). - Banco v17 (v16 fica reservada às expressões, como marcador sem efeito). - Testes: test_roles.py e test_context_judge.py; os testes bloqueiam qualquer CLI de IA real. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
5be6c09c16
commit
cb8450399d
@@ -1,8 +1,29 @@
|
||||
"""Configuração comum dos testes do Clipador."""
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
import app.ai_settings as ai_settings_module
|
||||
import app.intel as intel_module
|
||||
import app.judge as judge_module
|
||||
|
||||
|
||||
class _NoRealCli:
|
||||
"""`subprocess` do juiz nos testes: igual ao de verdade, mas `run` recusa (nenhuma CLI de IA roda e gasta a cota
|
||||
do dono — juízes falsos que não cobrem um papel, como o juiz de contexto, caem no caminho de falha da IA).
|
||||
Quem testa as CLIs (test_ai_fallback) troca `run` por um falso."""
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(subprocess, name)
|
||||
|
||||
@staticmethod
|
||||
def run(*_args, **_kwargs):
|
||||
raise OSError("CLI de IA bloqueada nos testes")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_real_ai_cli(monkeypatch):
|
||||
monkeypatch.setattr(judge_module, "subprocess", _NoRealCli())
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
"""JUIZ de contexto (doc/clipador/EQUIPE-IA.md): decisão pelo código (aprovar / ajustar e conferir uma vez / frase de
|
||||
contexto / descartar), o fluxo no motor, a linha do tempo JSON dentro do orçamento e o mosaico de quadros."""
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app import context_judge, editor_api, media, timeline
|
||||
from app import roles as roles_module
|
||||
from app.judge import Judge, Verdict
|
||||
from app.scoring import ChatMessage, Word
|
||||
|
||||
from .test_render_and_duplicates import add_clip
|
||||
from .test_sources_and_api import auth, env, seed_session # noqa: F401 - fixture reaproveitada
|
||||
|
||||
FFMPEG = shutil.which("ffmpeg")
|
||||
FFPROBE = shutil.which("ffprobe")
|
||||
|
||||
|
||||
def result(failed=(), fix=None, line="", reason="motivo"):
|
||||
return {"viewer_summary": "resumo", "confidence": 0.8, "reason": reason, "pt": {"reason": reason},
|
||||
"checks": {c: {"ok": c not in failed, "t": None, "why": ""} for c in roles_module.CHECKS},
|
||||
"fix": {"start_s": None, "end_s": None, "context_line": line, **(fix or {})}}
|
||||
|
||||
|
||||
LIMITS = {"start_s": 200.0, "end_s": 265.0, "lo": 80.0, "hi": 280.0, "min_s": 61.0, "max_s": 90.0}
|
||||
CONF = roles_module.default_roles()["judge"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- decisão pelo código
|
||||
def test_all_checks_ok_approves():
|
||||
d = context_judge.decide(result(), conf=CONF, score=8, min_score=6, **LIMITS)
|
||||
assert d.action == "approve" and d.failed == [] and d.window is None and d.score_after == 8
|
||||
|
||||
|
||||
def test_missing_setup_with_fix_adjusts_inside_the_limits():
|
||||
d = context_judge.decide(result(["setup"], {"start_s": 150}), conf=CONF, score=8, **LIMITS)
|
||||
assert d.action == "adjust" and d.failed_critical == ["setup"]
|
||||
# Recuou o início para pegar o começo; com máximo de 90 s o fim encolhe (fit_window mantém o início).
|
||||
assert d.window == (150.0, 240.0) and "começo (setup)" in d.reason_pt
|
||||
|
||||
|
||||
def test_fix_outside_the_window_is_clamped_and_no_change_means_discard():
|
||||
d = context_judge.decide(result(["payoff"], {"end_s": 999}), conf=CONF, **LIMITS)
|
||||
assert d.action == "adjust" and d.window[1] == 280.0 # nunca passa do limite
|
||||
same = context_judge.decide(result(["payoff"], {"end_s": 265.2}), conf=CONF, **LIMITS)
|
||||
assert same.action == "discard" # "ajuste" que não muda nada
|
||||
|
||||
|
||||
def test_critical_failure_without_fix_discards_with_portuguese_reason():
|
||||
d = context_judge.decide(result(["payoff"], reason="ends before the fight is resolved"), conf=CONF, **LIMITS)
|
||||
assert d.action == "discard" and d.reason_pt.startswith("Descartado pelo juiz: sem desfecho.")
|
||||
no_fix = context_judge.decide(result(["setup"], {"start_s": 150}), conf={**CONF, "allowFix": False}, **LIMITS)
|
||||
assert no_fix.action == "discard"
|
||||
|
||||
|
||||
def test_context_line_solves_standalone():
|
||||
d = context_judge.decide(result(["standalone"], line="Tfue tries to talk his way out of a robbery"), conf=CONF,
|
||||
score=8, min_score=6, **LIMITS)
|
||||
assert d.action == "approve" and d.solved_by_line == ["standalone"] and d.context_line.startswith("Tfue")
|
||||
off = context_judge.decide(result(["standalone"], line="Tfue tries"), conf={**CONF, "allowContextLine": False},
|
||||
**LIMITS)
|
||||
assert off.action == "discard"
|
||||
too_long = context_judge.decide(result(["standalone"], line=" ".join(["word"] * 14)), conf=CONF, **LIMITS)
|
||||
assert too_long.action == "discard"
|
||||
|
||||
|
||||
def test_non_critical_failures_only_lower_the_score():
|
||||
d = context_judge.decide(result(["strong_open"]), conf=CONF, score=7, min_score=6, **LIMITS)
|
||||
assert d.action == "approve" and d.score_after == 6.5
|
||||
low = context_judge.decide(result(["strong_open", "faithful", "intelligible"]), conf=CONF, score=7, min_score=6,
|
||||
**LIMITS)
|
||||
assert low.action == "discard" and low.score_after == 5.5 and "nota caiu" in low.reason_pt
|
||||
custom = context_judge.decide(result(["strong_open"]), conf={**CONF, "critical": ["strong_open"]}, **LIMITS)
|
||||
assert custom.action == "discard" # as críticas são calibráveis no painel
|
||||
|
||||
|
||||
def test_manual_clip_only_warns():
|
||||
d = context_judge.decide(result(["payoff"], {"end_s": 280}), conf=CONF, manual=True, **LIMITS)
|
||||
assert d.action == "warn" and d.window is None and d.reason_pt.startswith("Aviso do juiz")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- fluxo no motor
|
||||
class ContextFakeJudge(Judge):
|
||||
"""Analista, editor e juiz falsos (nunca chama CLI). `answers` = respostas do juiz, em ordem."""
|
||||
|
||||
def __init__(self, answers, keep=True, score=8.0):
|
||||
super().__init__("codex", "fake", Path("."))
|
||||
self.answers = list(answers)
|
||||
self.keep, self.score_value = keep, score
|
||||
self.context_prompts: list[str] = []
|
||||
self.editor_prompt = ""
|
||||
|
||||
def analyze(self, prompt, images, lang="pt"):
|
||||
return {"situation": "treta", "what_happened": "Ele tenta negociar e leva um tiro.", "screen": "", "speech": "",
|
||||
"chat_meaning": "", "confidence": 0.9, "action_start_s": 270, "reaction_end_s": 330, "hook": "",
|
||||
"lines": [{"n": 1, "v": "streamer aponta rifle para mascarado"}]}
|
||||
|
||||
def judge(self, prompt, start_s, end_s, images=None, lang="pt", **_limits):
|
||||
self.editor_prompt = prompt
|
||||
return Verdict(self.keep, self.score_value, 270.0, 340.0, "Tentou negociar e levou tiro", "c", "r", "fake")
|
||||
|
||||
def check_context(self, prompt, lang="pt"):
|
||||
self.context_prompts.append(prompt)
|
||||
answer = self.answers.pop(0)
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
return answer
|
||||
|
||||
|
||||
def run_clip(engine, judge, source="chat"):
|
||||
engine.judge = judge
|
||||
# Sem ffmpeg nos blocos falsos: um "mosaico" de um quadro só (em 275 s), que o analista descreve.
|
||||
engine.intel.visual = lambda *_a, **_k: ([Path("mosaico-falso.jpg")], [275.0], True)
|
||||
sid = seed_session(engine)
|
||||
cid = (engine.db.get("SELECT MAX(id) AS n FROM clips")["n"] or 0) + 1
|
||||
add_clip(engine, sid, cid, "candidate", 300, 280, 345, source=source)
|
||||
engine.judge_clip(engine.db.get("SELECT * FROM clips WHERE id = ?", cid))
|
||||
row = engine.db.get("SELECT * FROM clips WHERE id = ?", cid)
|
||||
return row, json.loads(row["judge_json"])
|
||||
|
||||
|
||||
def test_engine_approves_and_records_the_context(env): # noqa: F811
|
||||
_client, engine = env
|
||||
row, info = run_clip(engine, ContextFakeJudge([result(line="Tenta negociar com o assaltante")]))
|
||||
assert row["status"] == "pending" and info["context"]["decision"] == "approve"
|
||||
assert info["context"]["contextLine"] == "Tenta negociar com o assaltante"
|
||||
prompt = engine.judge.context_prompts[0]
|
||||
# O juiz vê só o que vai no corte: o trecho e os 20 s antes marcados out=1.
|
||||
assert "Corte que o espectador verá: 270s a 340s" in prompt and '"out":1' in prompt
|
||||
assert "História pretendida (analista): Ele tenta negociar e leva um tiro." in prompt
|
||||
# O que o analista viu no quadro virou o `v` da linha do tempo do editor.
|
||||
assert '"v":"streamer aponta rifle para mascarado"' in engine.judge.editor_prompt
|
||||
|
||||
|
||||
def test_engine_adjusts_once_then_approves(env): # noqa: F811
|
||||
_client, engine = env
|
||||
row, info = run_clip(engine, ContextFakeJudge([result(["setup"], {"start_s": 250}), result()]))
|
||||
assert row["status"] == "pending" and (row["start_s"], row["end_s"]) == (250.0, 340.0)
|
||||
ctx = info["context"]
|
||||
assert ctx["decision"] == "adjust" and ctx["window"] == [250.0, 340.0] and ctx["first"]["failedCritical"] == ["setup"]
|
||||
assert len(engine.judge.context_prompts) == 2 and "250s a 340s" in engine.judge.context_prompts[1]
|
||||
|
||||
|
||||
def test_engine_adjusts_then_discards_when_still_failing(env): # noqa: F811
|
||||
client, engine = env
|
||||
row, info = run_clip(engine, ContextFakeJudge([result(["payoff"], {"end_s": 355}),
|
||||
result(["payoff"], {"end_s": 360})]))
|
||||
assert row["status"] == "discarded" and info["discardedBy"] == "context"
|
||||
assert info["context"]["decision"] == "discard" and "depois do ajuste" in info["context"]["reasonPt"]
|
||||
assert len(engine.judge.context_prompts) == 2 # confere UMA vez de novo, nunca mais
|
||||
# Descartado pelo juiz fica visível e dá para forçar o render, como o descarte de hoje.
|
||||
forced = client.post("/v1/clips/1/render", headers=auth()).json()
|
||||
assert forced["status"] == "pending" and forced["judge"]["context"]["decision"] == "discard"
|
||||
|
||||
|
||||
def test_engine_discard_without_fix_and_judge_failure_keeps_editor(env): # noqa: F811
|
||||
_client, engine = env
|
||||
row, info = run_clip(engine, ContextFakeJudge([result(["standalone"])]))
|
||||
assert row["status"] == "discarded" and info["context"]["failedCritical"] == ["standalone"]
|
||||
from app.judge import JudgeError
|
||||
|
||||
row, info = run_clip(engine, ContextFakeJudge([JudgeError("cota")]))
|
||||
assert row["status"] == "pending" and info["context"] == {"error": "cota"}
|
||||
|
||||
|
||||
def test_judge_role_off_and_editor_discard_skip_the_judge(env): # noqa: F811
|
||||
_client, engine = env
|
||||
judge = ContextFakeJudge([result(["payoff"])])
|
||||
judge.roles["judge"]["enabled"] = False
|
||||
row, info = run_clip(engine, judge)
|
||||
assert row["status"] == "pending" and "context" not in info and judge.context_prompts == []
|
||||
kept_out = ContextFakeJudge([result()], keep=False)
|
||||
row, info = run_clip(engine, kept_out)
|
||||
assert row["status"] == "discarded" and kept_out.context_prompts == [] # o editor já descartou
|
||||
|
||||
|
||||
def test_manual_clip_is_never_discarded_by_the_judge(env): # noqa: F811
|
||||
_client, engine = env
|
||||
row, info = run_clip(engine, ContextFakeJudge([result(["payoff"], {"end_s": 360})]), source="manual")
|
||||
assert row["status"] == "pending" and info["context"]["decision"] == "warn"
|
||||
assert (row["start_s"], row["end_s"]) == (280.0, 345.0)
|
||||
|
||||
|
||||
def test_context_line_becomes_the_opening_text_of_the_edit(env): # noqa: F811
|
||||
_client, engine = env
|
||||
row, _info = run_clip(engine, ContextFakeJudge([result(["setup"], {"start_s": 250}),
|
||||
result(line="Ele tenta negociar com o assaltante")]))
|
||||
session = engine.db.get("SELECT * FROM sessions WHERE id = ?", row["session_id"])
|
||||
project = editor_api.auto_for(engine, row, session)
|
||||
hook = next(o for o in project["overlays"] if o["kind"] == "hook")
|
||||
assert hook["text"] == "Ele tenta negociar com o assaltante"
|
||||
# O ajuste do juiz não é desfeito pelo início da ação do analista (270 s).
|
||||
assert project["cuts"][0]["start"] <= 250.5
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- linha do tempo JSON
|
||||
def _evidence(seconds=240):
|
||||
words, chat = [], []
|
||||
for t in range(0, seconds, 2):
|
||||
words.append(Word(float(t), t + 1.5, f"palavra{t}", "other1" if 60 <= t < 64 else ""))
|
||||
for t in range(0, seconds, 3):
|
||||
chat.append(ChatMessage(float(t), "KEKW que isso" if t % 2 else "lol", emotes=("KEKW",) if t % 2 else ()))
|
||||
loudness = [-30.0] * seconds
|
||||
loudness[100] = -10.0
|
||||
return timeline.Evidence(words=words, chat=chat, loudness=loudness, face=[(101, "laugh", 6.4)],
|
||||
scenes=[(100.0, "explosão na garagem")])
|
||||
|
||||
|
||||
def test_timeline_entries_are_short_split_by_speaker_and_marked():
|
||||
ev = _evidence()
|
||||
items = timeline.entries(ev, 40, 130, clip=(60, 120))
|
||||
assert all(i["d"] <= 8 for i in items) and items[0]["t"] == 40
|
||||
assert any(i.get("sp", "").startswith("[other1]") for i in items) # troca de falante abre entrada nova
|
||||
# Fora do corte (antes e depois) sai marcado; nenhuma entrada atravessa o início ou o fim do corte.
|
||||
assert all(i.get("out") == 1 for i in items if i["t"] < 60 or i["t"] >= 120)
|
||||
assert not any("out" in i for i in items if 60 <= i["t"] < 120)
|
||||
assert {60, 120} <= {i["t"] for i in items} and all(i["t"] + i["d"] <= 60 for i in items if i["t"] < 60)
|
||||
hit = next(i for i in items if i["t"] <= 100 < i["t"] + i["d"])
|
||||
assert hit["v"] == "explosão na garagem" and hit["face"] == "laugh.6" and hit["lvl"] > 3
|
||||
assert hit["chat"].split("|")[0].isdigit() and "KEKW×" in hit["chat"]
|
||||
|
||||
|
||||
def test_timeline_fits_the_budget():
|
||||
ev = _evidence(600)
|
||||
full = timeline.build(ev, 0, 600, {"win": [0, 600]}, budget_chars=100_000)
|
||||
small = timeline.build(ev, 0, 600, {"win": [0, 600]}, budget_chars=6000)
|
||||
assert len(small) <= 6000 < len(full)
|
||||
assert all(json.loads(line) for line in small.splitlines()) # cada linha continua JSON válido
|
||||
coarse = timeline.build(ev, 0, 600, {"win": [480, 600]}, budget_chars=100_000, coarse_until=480)
|
||||
first = json.loads(coarse.splitlines()[1])
|
||||
assert first["d"] == 20 and "face" not in first # a "volta" da varredura vai resumida
|
||||
|
||||
|
||||
def test_scene_lines_are_saved_by_tile_number(env): # noqa: F811
|
||||
_client, engine = env
|
||||
sid = seed_session(engine)
|
||||
n = timeline.save_scene_lines(engine, sid, "scan", [{"n": 2, "v": " carro capotando "}, {"n": 9, "v": "x"},
|
||||
{"n": "a"}], [10.0, 40.0, 70.0])
|
||||
assert n == 1 and timeline.scene_lines(engine, sid, 0, 100) == [(40.0, "carro capotando")]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not (FFMPEG and FFPROBE), reason="FFmpeg não instalado")
|
||||
@pytest.mark.parametrize("count,size", [(8, "1536x432"), (6, "1536x432"), (3, "1152x216")])
|
||||
def test_mosaic_puts_frames_in_one_image(tmp_path, count, size):
|
||||
frames = []
|
||||
for i in range(count):
|
||||
out = tmp_path / f"f{i}.jpg"
|
||||
subprocess.run([FFMPEG, "-y", "-v", "error", "-f", "lavfi", "-i", f"color=c=0x{i * 30:02x}4080:s=768x432",
|
||||
"-frames:v", "1", str(out)], check=True)
|
||||
frames.append(out)
|
||||
target = media.mosaic(FFMPEG, frames, tmp_path / "m.jpg")
|
||||
probe = subprocess.run([FFPROBE, "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height",
|
||||
"-of", "csv=s=x:p=0", str(target)], capture_output=True, text=True, check=True)
|
||||
assert probe.stdout.strip() == size
|
||||
@@ -97,7 +97,10 @@ def test_english_session_judges_in_english_and_stores_portuguese(env): # noqa:
|
||||
add_clip(engine, sid, 1, "candidate", 285, 270, 340)
|
||||
engine.judge_clip(engine.db.get("SELECT * FROM clips WHERE id = 1"))
|
||||
assert engine.judge.seen["analyze"][0] == "en" and engine.judge.seen["judge"][0] == "en"
|
||||
assert "Transcript:" in engine.judge.seen["judge"][1] and "ANALYST's analysis" in engine.judge.seen["judge"][1]
|
||||
# Linha do tempo JSON (padrão desde a Equipe de IA, EQUIPE-IA.md): legenda das chaves e a fala dentro dela.
|
||||
editor_prompt = engine.judge.seen["judge"][1]
|
||||
assert "Timeline (JSON Lines" in editor_prompt and "ANALYST's analysis" in editor_prompt
|
||||
assert '"sp":"No way bro.' in editor_prompt
|
||||
clip = client.get("/v1/clips/1", headers=_auth()).json()
|
||||
assert clip["title"] == "He robbed the bank solo" # o vídeo/publicação: inglês
|
||||
assert clip["judge"]["pt"]["title"] == "Ele roubou o banco sozinho" # o painel: português
|
||||
@@ -115,6 +118,7 @@ def test_portuguese_session_has_no_translation(env): # noqa: F811
|
||||
engine.judge = EnglishJudge()
|
||||
sid = seed_session(engine)
|
||||
add_clip(engine, sid, 1, "candidate", 300, 280, 350)
|
||||
engine.intel.opts["timelineJson"] = False # formato antigo (transcrição + chat) continua atrás da opção
|
||||
engine.judge_clip(engine.db.get("SELECT * FROM clips WHERE id = 1"))
|
||||
assert engine.judge.seen["judge"][0] == "pt" and "Transcrição:" in engine.judge.seen["judge"][1]
|
||||
assert client.get("/v1/clips/1", headers=_auth()).json()["translation"] is None
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
"""Equipe de IA (doc/clipador/EQUIPE-IA.md): papéis com IA própria, migração da configuração antiga, instruções do
|
||||
administrador, esforço por papel, medição por chamada (F0) e o relatório. Nenhuma CLI de verdade roda aqui."""
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app import judge as judge_module
|
||||
from app import roles as roles_module
|
||||
from app.ai_settings import AiConfig
|
||||
from app.config import Settings
|
||||
from app.db import Database
|
||||
from app.judge import Judge, claude_usage, codex_usage, gemini_usage
|
||||
from app.worker import Engine
|
||||
|
||||
from .test_sources_and_api import KEY, auth, env, seed_session # noqa: F401 - fixture reaproveitada
|
||||
|
||||
ANALYSIS = {"situation": "jogada", "what_happened": "Ele roubou o banco.", "screen": "", "speech": "", "chat_meaning": "",
|
||||
"confidence": 0.9, "action_start_s": 100, "reaction_end_s": 160, "hook": "Assalto solo", "lines": []}
|
||||
VERDICT = {"keep": True, "score": 8, "start_s": 100, "end_s": 170, "title": "Virada", "caption": "c", "hashtags": [],
|
||||
"reason": "r"}
|
||||
CONTEXT_OK = {"viewer_summary": "Ele rouba o banco sozinho.", "fix": {"start_s": None, "end_s": None, "context_line": ""},
|
||||
"checks": {c: {"ok": True, "t": None, "why": ""} for c in roles_module.CHECKS}, "confidence": 0.9,
|
||||
"reason": "ok"}
|
||||
SCAN = {"screen_game": "GTA", "has_webcam": False, "summary": "s", "moments": [], "lines": []}
|
||||
|
||||
|
||||
def answer_for(schema: dict) -> dict:
|
||||
props = schema["properties"]
|
||||
if "viewer_summary" in props:
|
||||
return CONTEXT_OK
|
||||
if "situation" in props:
|
||||
return ANALYSIS
|
||||
if "moments" in props:
|
||||
return SCAN
|
||||
if list(props) == ["lines"]:
|
||||
return {"lines": ["oi"]}
|
||||
return VERDICT
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def team(tmp_path, monkeypatch):
|
||||
"""Juiz de verdade com as CLIs trocadas por um registro: cada chamada guarda provedor, modelo, esforço e sistema."""
|
||||
calls: list[dict] = []
|
||||
|
||||
def fake_call(self, provider, system, prompt, schema, images, search, timeout, model, effort=""):
|
||||
calls.append({"provider": provider, "model": model, "effort": effort, "system": system, "prompt": prompt})
|
||||
return answer_for(schema)
|
||||
|
||||
monkeypatch.setattr(Judge, "_call", fake_call)
|
||||
for name in ("codex", "claude", "gemini"):
|
||||
monkeypatch.setattr(judge_module, f"find_{name}", (lambda n=name: [n]))
|
||||
return calls
|
||||
|
||||
|
||||
def make_engine(tmp_path, saved: dict[str, str] | None = None) -> Engine:
|
||||
settings = Settings.load(env_file=tmp_path / "none.env", service_key=KEY, data_dir=tmp_path / "data",
|
||||
clips_dir=tmp_path / "clips", judge_provider="codex", voice_url=None, voice_key=None)
|
||||
db = Database(settings.data_dir / "clipper.db")
|
||||
if saved:
|
||||
db.many("INSERT INTO settings (key, value) VALUES (?, ?)", list(saved.items()))
|
||||
return Engine(settings, db)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- migração
|
||||
def test_old_single_provider_becomes_every_role_and_analyst_model_is_kept(tmp_path):
|
||||
engine = make_engine(tmp_path, {"judge_provider": "claude", "judge_model": "sonnet", "judge_analyst_model": "haiku"})
|
||||
roles = engine.judge.roles
|
||||
assert {r["provider"] for r in roles.values()} == {"claude"}
|
||||
assert roles["editor"]["model"] == "sonnet" and roles["scan"]["model"] == "sonnet" and roles["judge"]["model"] == "sonnet"
|
||||
assert roles["analyst"]["model"] == "haiku" # o "modelo do analista" que já existia é respeitado
|
||||
assert roles["judge"]["enabled"] and roles["judge"]["critical"] == ["setup", "payoff", "standalone"]
|
||||
# A migração fica gravada (explícita no banco); a próxima subida lê `judge.roles`.
|
||||
saved = json.loads(engine.db.get("SELECT value FROM settings WHERE key = 'judge.roles'")["value"])
|
||||
assert saved["analyst"]["model"] == "haiku"
|
||||
again = Engine(engine.settings, engine.db)
|
||||
assert again.judge.roles == roles
|
||||
|
||||
|
||||
def test_off_master_switch_is_kept_and_roles_still_migrate(tmp_path):
|
||||
engine = make_engine(tmp_path, {"judge_provider": "off", "judge_model": ""})
|
||||
assert not engine.judge.enabled and engine.judge.provider == "off"
|
||||
assert {r["provider"] for r in engine.judge.roles.values()} == {"codex"}
|
||||
|
||||
|
||||
def test_normalize_roles_field_by_field():
|
||||
base = roles_module.default_roles("codex", "gpt-6-luna")
|
||||
out = roles_module.normalize_roles({
|
||||
"editor": {"enabled": False, "provider": "claude"}, # editor nunca desliga; trocou de provedor → modelo padrão
|
||||
"judge": {"effort": "turbo", "critical": ["payoff", "inventado"], "penalty": 9, "extra": " x "},
|
||||
"scan": {"images": "frames", "model": "bad model;"},
|
||||
}, base)
|
||||
assert out["editor"]["enabled"] is True and out["editor"]["provider"] == "claude" and out["editor"]["model"] == ""
|
||||
assert out["judge"]["effort"] == "medium" and out["judge"]["critical"] == ["payoff"]
|
||||
assert out["judge"]["penalty"] == 0.5 and out["judge"]["extra"] == "x"
|
||||
assert out["scan"]["images"] == "frames" and out["scan"]["model"] == "gpt-6-luna"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- cada papel chama a sua IA
|
||||
def test_each_role_calls_its_own_provider_model_and_effort(tmp_path, team):
|
||||
roles = roles_module.default_roles("codex")
|
||||
roles["scan"] |= {"provider": "codex", "model": "gpt-6-luna", "effort": "low"}
|
||||
roles["analyst"] |= {"provider": "claude", "model": "haiku"}
|
||||
roles["editor"] |= {"provider": "codex", "model": ""}
|
||||
roles["judge"] |= {"provider": "claude", "model": "sonnet", "effort": ""}
|
||||
judge = Judge("codex", "", tmp_path / "j", ai=AiConfig(tmp_path / "ai"), roles=roles)
|
||||
judge.scan("p", [], lang="en")
|
||||
judge.analyze("p", [], lang="en")
|
||||
judge.judge("p", 100, 170, [], lang="en")
|
||||
judge.check_context("p", lang="en")
|
||||
got = [(c["provider"], c["model"], c["effort"]) for c in team]
|
||||
assert got == [("codex", "gpt-6-luna", "low"), ("claude", "haiku", ""), ("codex", "gpt-6-luna", ""),
|
||||
("claude", "sonnet", "")]
|
||||
# O que não é dos quatro papéis (tradução, pesquisa) usa a IA do editor.
|
||||
judge.translate_lines(["oi"])
|
||||
assert team[-1]["provider"] == "codex"
|
||||
assert judge.fallback_chain("claude")[0] == "claude" and judge.fallback_chain()[0] == "codex"
|
||||
|
||||
|
||||
def test_admin_instructions_are_added_after_the_default_text(tmp_path, team):
|
||||
roles = roles_module.default_roles("codex")
|
||||
roles["judge"]["extra"] = "Não aprove corte que termina antes do resultado da treta."
|
||||
roles["analyst"]["extra"] = "Lag conta como jogada."
|
||||
judge = Judge("codex", "", tmp_path / "j", ai=AiConfig(tmp_path / "ai"), roles=roles)
|
||||
judge.check_context("p", lang="en")
|
||||
judge.analyze("p", [], lang="pt")
|
||||
judge.judge("p", 100, 170, [], lang="pt")
|
||||
context_system, analyst_system, editor_system = (c["system"] for c in team)
|
||||
assert context_system.startswith(judge_module.CONTEXT_SYSTEM_EN) # o padrão continua inteiro, no começo
|
||||
assert context_system.endswith("Admin notes (may be written in Portuguese; when they conflict with the rules above, "
|
||||
"they win):\nNão aprove corte que termina antes do resultado da treta.")
|
||||
assert analyst_system.endswith("valem estas):\nLag conta como jogada.")
|
||||
assert "Instruções do administrador" not in editor_system # o editor não tem extra
|
||||
|
||||
|
||||
def test_codex_effort_goes_to_the_command_line(tmp_path, monkeypatch):
|
||||
seen: list[list[str]] = []
|
||||
|
||||
def run(args, input=None, capture_output=True, timeout=None, cwd=None, creationflags=0, env=None):
|
||||
seen.append(list(args))
|
||||
Path(args[args.index("-o") + 1]).write_text(json.dumps(CONTEXT_OK), encoding="utf-8")
|
||||
events = '{"type":"thread.started"}\n{"type":"turn.completed","usage":{"input_tokens":1234,"cached_input_tokens":200,"output_tokens":56}}\n'
|
||||
return subprocess.CompletedProcess(args, 0, events.encode(), b"")
|
||||
|
||||
monkeypatch.setattr(judge_module.subprocess, "run", run)
|
||||
monkeypatch.setattr(judge_module, "find_codex", lambda: ["codex"])
|
||||
monkeypatch.setattr(judge_module, "find_claude", lambda: None)
|
||||
records = []
|
||||
judge = Judge("codex", "", tmp_path / "j", ai=AiConfig(tmp_path / "ai"))
|
||||
judge.meter = records.append
|
||||
with judge.scope(clip_id=7, session_id=3, mode="timeline+mosaic"):
|
||||
judge.check_context("prompt de teste", lang="pt")
|
||||
assert 'model_reasoning_effort="medium"' in seen[0] and "--json" in seen[0] # juiz: esforço médio por padrão
|
||||
rec = records[0]
|
||||
assert rec["role"] == "judge" and rec["provider"] == "codex" and rec["ok"] is True
|
||||
assert (rec["tokensIn"], rec["tokensOut"], rec["tokensCached"]) == (1234, 56, 200)
|
||||
assert rec["clip_id"] == 7 and rec["mode"] == "timeline+mosaic" and rec["promptChars"] == len("prompt de teste")
|
||||
|
||||
|
||||
def test_usage_parsers():
|
||||
assert codex_usage("lixo\n{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":10,\"output_tokens\":2}}") == \
|
||||
{"in": 10, "out": 2, "cached": None}
|
||||
assert codex_usage("") is None
|
||||
assert claude_usage({"usage": {"input_tokens": 5, "cache_creation_input_tokens": 100, "cache_read_input_tokens": 2000,
|
||||
"output_tokens": 40}}) == {"in": 2105, "out": 40, "cached": 2000}
|
||||
assert gemini_usage({"stats": {"models": {"gemini-3.5-flash": {"tokens": {"prompt": 900, "candidates": 50,
|
||||
"thoughts": 10, "cached": 0}}}}}) == \
|
||||
{"in": 900, "out": 60, "cached": 0}
|
||||
assert gemini_usage({"response": "x"}) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- API e medição
|
||||
def test_put_roles_and_read_defaults(env): # noqa: F811
|
||||
client, engine = env
|
||||
roles = {"scan": {"enabled": False, "provider": "codex", "images": "frames"},
|
||||
"analyst": {"provider": "claude", "model": "haiku", "extra": "Olhe o HUD."},
|
||||
"editor": {"enabled": False, "provider": "codex"},
|
||||
"judge": {"enabled": True, "provider": "claude", "model": "sonnet", "critical": ["setup", "payoff"],
|
||||
"allowContextLine": False}}
|
||||
r = client.put("/v1/judge", json={"provider": "codex", "model": "", "minScore": 6, "roles": roles}, headers=auth())
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert data["roles"]["analyst"]["provider"] == "claude" and data["roles"]["analyst"]["extra"] == "Olhe o HUD."
|
||||
assert data["roles"]["editor"]["enabled"] is True # o editor não desliga
|
||||
assert data["roles"]["judge"]["critical"] == ["setup", "payoff"] and data["roles"]["judge"]["allowContextLine"] is False
|
||||
# O cartão da varredura liga/desliga a varredura de verdade.
|
||||
assert data["roles"]["scan"]["enabled"] is False and engine.intel.opts["scanEnabled"] is False
|
||||
got = client.get("/v1/judge", headers=auth()).json()
|
||||
assert set(got["roleDefaults"]) == {"scan", "analyst", "editor", "judge"}
|
||||
assert got["roleDefaults"]["judge"]["en"].startswith("You are the CONTEXT JUDGE")
|
||||
assert [c["id"] for c in got["checks"]][:3] == ["setup", "payoff", "who_what"]
|
||||
assert "roleDefaults" not in client.get("/v1/status", headers=auth()).json()["judge"] # status fica leve
|
||||
# A opção da linha do tempo (comparar com o formato antigo) também se troca pela mesma rota.
|
||||
client.put("/v1/judge", json={"provider": "codex", "model": "", "minScore": 6, "timelineJson": False}, headers=auth())
|
||||
assert engine.intel.opts["timelineJson"] is False
|
||||
|
||||
|
||||
def test_owner_label_and_report(env): # noqa: F811
|
||||
client, engine = env
|
||||
sid = seed_session(engine)
|
||||
db = engine.db
|
||||
|
||||
def clip(cid, status, decision, keep=True, label=None):
|
||||
judge = {"keep": keep, "score": 7}
|
||||
if decision:
|
||||
judge["context"] = {"decision": decision, "contextLine": "x" if decision == "approve" else ""}
|
||||
db.run("INSERT INTO clips (id, session_id, peak_s, start_s, end_s, score, source, status, judge_json, created_at,"
|
||||
" updated_at, owner_label) VALUES (?, ?, 1, 0, 60, 5, 'chat', ?, ?, ?, ?, ?)",
|
||||
cid, sid, status, json.dumps(judge), engine_now(), engine_now(), label)
|
||||
|
||||
clip(1, "approved", "approve") # juiz mantém, dono bom → concorda
|
||||
clip(2, "rejected", "approve") # juiz mantém, dono ruim → discorda
|
||||
clip(3, "discarded", "discard", label="no_context") # juiz descarta, dono "sem contexto" → concorda
|
||||
clip(4, "ready", "adjust") # dono não avaliou
|
||||
clip(5, "approved", "discard") # juiz descartou, dono forçou e aprovou → falso negativo
|
||||
# Rejeitar com "sem contexto" pela rota (o rótulo vale também para corte antigo/descartado).
|
||||
r = client.patch("/v1/clips/4", json={"label": "no_context", "labelNote": "não dá pra entender"}, headers=auth())
|
||||
assert r.json()["ownerLabel"] == "no_context" and r.json()["ownerNote"] == "não dá pra entender"
|
||||
assert r.json()["status"] == "ready" # avaliar não muda o estado
|
||||
engine.record_ai_call({"role": "judge", "purpose": "judge", "provider": "codex", "model": "gpt-6-luna", "ok": True,
|
||||
"ms": 1200, "systemChars": 2700, "promptChars": 3000, "images": 0, "imageBytes": 0,
|
||||
"tokensIn": 1500, "tokensOut": 200, "session_id": sid, "mode": "timeline+mosaic"})
|
||||
engine.record_ai_call({"role": "analyst", "purpose": "analyst", "provider": "codex", "model": "gpt-6-luna",
|
||||
"ok": False, "error": "cota", "systemChars": 3000, "promptChars": 5000, "images": 1,
|
||||
"imageBytes": 60_000, "session_id": sid})
|
||||
rep = client.get(f"/v1/judge/report?session={sid}", headers=auth()).json()
|
||||
assert rep["window"]["liveHours"] == round(600 / 3600, 2)
|
||||
judge_row = next(c for c in rep["calls"] if c["role"] == "judge")
|
||||
assert judge_row["tokensIn"] == 1500 and judge_row["withUsage"] == 1 and judge_row["mode"] == "timeline+mosaic"
|
||||
assert next(c for c in rep["calls"] if c["role"] == "analyst")["failures"] == 1
|
||||
assert rep["perHour"]["judge"]["calls"] == pytest.approx(6.0)
|
||||
assert rep["judge"]["approve"] == 2 and rep["judge"]["discard"] == 2 and rep["judge"]["adjust"] == 1
|
||||
assert rep["clips"]["approvalRate"] == round(2 / 3, 3) and rep["clips"]["noContext"] == 2
|
||||
agreement = rep["agreement"]["judge"]
|
||||
# pares: (keep, good) (keep, bad) (discard, no_context) (keep, no_context) (discard, good) → concordam 2 de 5
|
||||
assert agreement["pairs"] == 5 and agreement["agree"] == 2
|
||||
assert agreement["matrix"]["discard"]["good"] == 1 and agreement["matrix"]["keep"]["no_context"] == 1
|
||||
|
||||
|
||||
def engine_now() -> str:
|
||||
from app.db import now
|
||||
|
||||
return now()
|
||||
@@ -165,6 +165,9 @@ def test_judge_config_persists(env, tmp_path):
|
||||
assert client.put("/v1/judge", json={"provider": "gpt", "model": ""}, headers=auth()).status_code == 422
|
||||
assert client.put("/v1/judge", json={"provider": "codex", "model": "x; rm -rf"}, headers=auth()).status_code == 422
|
||||
saved = {r["key"]: r["value"] for r in engine.db.all("SELECT * FROM settings")}
|
||||
# Equipe de IA (EQUIPE-IA.md): o painel antigo (um provedor só) troca a IA de todos os papéis.
|
||||
roles = json.loads(saved.pop("judge.roles"))
|
||||
assert {r["provider"] for r in roles.values()} == {"claude"}
|
||||
assert saved == {"judge_provider": "claude", "judge_model": "haiku", "judge_min_score": "7.0", "judge_analyst_model": ""}
|
||||
|
||||
|
||||
|
||||
@@ -352,7 +352,8 @@ def test_translation_lines_get_speaker(env): # noqa: F811
|
||||
assert [(x["text"], x["speaker"]) for x in lines] == [("No way bro.", "streamer"), ("Told you.", "other1")]
|
||||
# O juiz viu quem fala.
|
||||
prompt = engine.judge.seen["judge"][1]
|
||||
assert "] No way bro." in prompt and "[other1] Told you." in prompt and "Never attribute to" in prompt
|
||||
# Linha do tempo JSON (EQUIPE-IA.md): cada entrada é de um falante só; a de outra pessoa vem marcada.
|
||||
assert '"sp":"No way bro."' in prompt and '"sp":"[other1] Told you."' in prompt and "Never attribute to" in prompt
|
||||
# Tradução antiga (sem `speaker`) é refeita na próxima vez.
|
||||
engine.db.run("UPDATE clips SET translation_json = ? WHERE id = 1",
|
||||
json.dumps({"startS": 0, "endS": 1e6, "lines": [{"s": 1, "e": 2, "text": "x", "pt": "y"}]}))
|
||||
|
||||
Reference in New Issue
Block a user