Files
MateuusandClaude Opus 5.5 cb8450399d Clipador: Equipe de IA (papéis com IA própria), juiz de contexto e linha do tempo JSON
- roles.py: varredura, analista, editor e juiz, cada um com provedor, modelo, raciocínio
  (Codex), liga/desliga (o editor não desliga) e instruções do administrador somadas às
  padrão; gravado em settings `judge.roles`, com migração automática da configuração antiga
  (provedor/modelo único vira o de todos; o modelo do analista é respeitado). A reserva vale
  por papel.
- Juiz de contexto (só texto): vê só o que vai no corte (+20 s antes/15 s depois marcados
  out=1), escreve o que um espectador frio entende e responde sim/não; context_judge.decide
  decide pelo código: aprova, ajusta o trecho e confere uma vez de novo, frase de contexto
  na tela (vira o texto de abertura do projeto) ou descarta com motivo em português
  (forçável). Corte manual só recebe aviso.
- timeline.py: linha do tempo JSON curta (fala com falante, chat, rosto, volume e a frase
  de tela que a varredura/analista escrevem ao ver os quadros, em scene_notes); mosaico 4x2
  (media.mosaic) no lugar dos quadros soltos; perfil do jogo enxuto. O formato antigo fica
  atrás da opção timelineJson.
- Medição (F0): ai_calls por chamada (papel, IA, tamanho do prompt/imagens, tokens quando a
  CLI informa: codex --json, claude e gemini), avaliação do dono (owner_label: bom/ruim/sem
  contexto) e GET /v1/judge/report (taxa de aprovação, decisões do juiz, concordância).
- Banco v17 (v16 fica reservada às expressões, como marcador sem efeito).
- Testes: test_roles.py e test_context_judge.py; os testes bloqueiam qualquer CLI de IA real.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 04:36:25 -03:00

245 lines
14 KiB
Python

"""Equipe de IA (doc/clipador/EQUIPE-IA.md): papéis com IA própria, migração da configuração antiga, instruções do
administrador, esforço por papel, medição por chamada (F0) e o relatório. Nenhuma CLI de verdade roda aqui."""
import json
import subprocess
from pathlib import Path
import pytest
from app import judge as judge_module
from app import roles as roles_module
from app.ai_settings import AiConfig
from app.config import Settings
from app.db import Database
from app.judge import Judge, claude_usage, codex_usage, gemini_usage
from app.worker import Engine
from .test_sources_and_api import KEY, auth, env, seed_session # noqa: F401 - fixture reaproveitada
ANALYSIS = {"situation": "jogada", "what_happened": "Ele roubou o banco.", "screen": "", "speech": "", "chat_meaning": "",
"confidence": 0.9, "action_start_s": 100, "reaction_end_s": 160, "hook": "Assalto solo", "lines": []}
VERDICT = {"keep": True, "score": 8, "start_s": 100, "end_s": 170, "title": "Virada", "caption": "c", "hashtags": [],
"reason": "r"}
CONTEXT_OK = {"viewer_summary": "Ele rouba o banco sozinho.", "fix": {"start_s": None, "end_s": None, "context_line": ""},
"checks": {c: {"ok": True, "t": None, "why": ""} for c in roles_module.CHECKS}, "confidence": 0.9,
"reason": "ok"}
SCAN = {"screen_game": "GTA", "has_webcam": False, "summary": "s", "moments": [], "lines": []}
def answer_for(schema: dict) -> dict:
props = schema["properties"]
if "viewer_summary" in props:
return CONTEXT_OK
if "situation" in props:
return ANALYSIS
if "moments" in props:
return SCAN
if list(props) == ["lines"]:
return {"lines": ["oi"]}
return VERDICT
@pytest.fixture
def team(tmp_path, monkeypatch):
"""Juiz de verdade com as CLIs trocadas por um registro: cada chamada guarda provedor, modelo, esforço e sistema."""
calls: list[dict] = []
def fake_call(self, provider, system, prompt, schema, images, search, timeout, model, effort=""):
calls.append({"provider": provider, "model": model, "effort": effort, "system": system, "prompt": prompt})
return answer_for(schema)
monkeypatch.setattr(Judge, "_call", fake_call)
for name in ("codex", "claude", "gemini"):
monkeypatch.setattr(judge_module, f"find_{name}", (lambda n=name: [n]))
return calls
def make_engine(tmp_path, saved: dict[str, str] | None = None) -> Engine:
settings = Settings.load(env_file=tmp_path / "none.env", service_key=KEY, data_dir=tmp_path / "data",
clips_dir=tmp_path / "clips", judge_provider="codex", voice_url=None, voice_key=None)
db = Database(settings.data_dir / "clipper.db")
if saved:
db.many("INSERT INTO settings (key, value) VALUES (?, ?)", list(saved.items()))
return Engine(settings, db)
# ---------------------------------------------------------------- migração
def test_old_single_provider_becomes_every_role_and_analyst_model_is_kept(tmp_path):
engine = make_engine(tmp_path, {"judge_provider": "claude", "judge_model": "sonnet", "judge_analyst_model": "haiku"})
roles = engine.judge.roles
assert {r["provider"] for r in roles.values()} == {"claude"}
assert roles["editor"]["model"] == "sonnet" and roles["scan"]["model"] == "sonnet" and roles["judge"]["model"] == "sonnet"
assert roles["analyst"]["model"] == "haiku" # o "modelo do analista" que já existia é respeitado
assert roles["judge"]["enabled"] and roles["judge"]["critical"] == ["setup", "payoff", "standalone"]
# A migração fica gravada (explícita no banco); a próxima subida lê `judge.roles`.
saved = json.loads(engine.db.get("SELECT value FROM settings WHERE key = 'judge.roles'")["value"])
assert saved["analyst"]["model"] == "haiku"
again = Engine(engine.settings, engine.db)
assert again.judge.roles == roles
def test_off_master_switch_is_kept_and_roles_still_migrate(tmp_path):
engine = make_engine(tmp_path, {"judge_provider": "off", "judge_model": ""})
assert not engine.judge.enabled and engine.judge.provider == "off"
assert {r["provider"] for r in engine.judge.roles.values()} == {"codex"}
def test_normalize_roles_field_by_field():
base = roles_module.default_roles("codex", "gpt-6-luna")
out = roles_module.normalize_roles({
"editor": {"enabled": False, "provider": "claude"}, # editor nunca desliga; trocou de provedor → modelo padrão
"judge": {"effort": "turbo", "critical": ["payoff", "inventado"], "penalty": 9, "extra": " x "},
"scan": {"images": "frames", "model": "bad model;"},
}, base)
assert out["editor"]["enabled"] is True and out["editor"]["provider"] == "claude" and out["editor"]["model"] == ""
assert out["judge"]["effort"] == "medium" and out["judge"]["critical"] == ["payoff"]
assert out["judge"]["penalty"] == 0.5 and out["judge"]["extra"] == "x"
assert out["scan"]["images"] == "frames" and out["scan"]["model"] == "gpt-6-luna"
# ---------------------------------------------------------------- cada papel chama a sua IA
def test_each_role_calls_its_own_provider_model_and_effort(tmp_path, team):
roles = roles_module.default_roles("codex")
roles["scan"] |= {"provider": "codex", "model": "gpt-6-luna", "effort": "low"}
roles["analyst"] |= {"provider": "claude", "model": "haiku"}
roles["editor"] |= {"provider": "codex", "model": ""}
roles["judge"] |= {"provider": "claude", "model": "sonnet", "effort": ""}
judge = Judge("codex", "", tmp_path / "j", ai=AiConfig(tmp_path / "ai"), roles=roles)
judge.scan("p", [], lang="en")
judge.analyze("p", [], lang="en")
judge.judge("p", 100, 170, [], lang="en")
judge.check_context("p", lang="en")
got = [(c["provider"], c["model"], c["effort"]) for c in team]
assert got == [("codex", "gpt-6-luna", "low"), ("claude", "haiku", ""), ("codex", "gpt-6-luna", ""),
("claude", "sonnet", "")]
# O que não é dos quatro papéis (tradução, pesquisa) usa a IA do editor.
judge.translate_lines(["oi"])
assert team[-1]["provider"] == "codex"
assert judge.fallback_chain("claude")[0] == "claude" and judge.fallback_chain()[0] == "codex"
def test_admin_instructions_are_added_after_the_default_text(tmp_path, team):
roles = roles_module.default_roles("codex")
roles["judge"]["extra"] = "Não aprove corte que termina antes do resultado da treta."
roles["analyst"]["extra"] = "Lag conta como jogada."
judge = Judge("codex", "", tmp_path / "j", ai=AiConfig(tmp_path / "ai"), roles=roles)
judge.check_context("p", lang="en")
judge.analyze("p", [], lang="pt")
judge.judge("p", 100, 170, [], lang="pt")
context_system, analyst_system, editor_system = (c["system"] for c in team)
assert context_system.startswith(judge_module.CONTEXT_SYSTEM_EN) # o padrão continua inteiro, no começo
assert context_system.endswith("Admin notes (may be written in Portuguese; when they conflict with the rules above, "
"they win):\nNão aprove corte que termina antes do resultado da treta.")
assert analyst_system.endswith("valem estas):\nLag conta como jogada.")
assert "Instruções do administrador" not in editor_system # o editor não tem extra
def test_codex_effort_goes_to_the_command_line(tmp_path, monkeypatch):
seen: list[list[str]] = []
def run(args, input=None, capture_output=True, timeout=None, cwd=None, creationflags=0, env=None):
seen.append(list(args))
Path(args[args.index("-o") + 1]).write_text(json.dumps(CONTEXT_OK), encoding="utf-8")
events = '{"type":"thread.started"}\n{"type":"turn.completed","usage":{"input_tokens":1234,"cached_input_tokens":200,"output_tokens":56}}\n'
return subprocess.CompletedProcess(args, 0, events.encode(), b"")
monkeypatch.setattr(judge_module.subprocess, "run", run)
monkeypatch.setattr(judge_module, "find_codex", lambda: ["codex"])
monkeypatch.setattr(judge_module, "find_claude", lambda: None)
records = []
judge = Judge("codex", "", tmp_path / "j", ai=AiConfig(tmp_path / "ai"))
judge.meter = records.append
with judge.scope(clip_id=7, session_id=3, mode="timeline+mosaic"):
judge.check_context("prompt de teste", lang="pt")
assert 'model_reasoning_effort="medium"' in seen[0] and "--json" in seen[0] # juiz: esforço médio por padrão
rec = records[0]
assert rec["role"] == "judge" and rec["provider"] == "codex" and rec["ok"] is True
assert (rec["tokensIn"], rec["tokensOut"], rec["tokensCached"]) == (1234, 56, 200)
assert rec["clip_id"] == 7 and rec["mode"] == "timeline+mosaic" and rec["promptChars"] == len("prompt de teste")
def test_usage_parsers():
assert codex_usage("lixo\n{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":10,\"output_tokens\":2}}") == \
{"in": 10, "out": 2, "cached": None}
assert codex_usage("") is None
assert claude_usage({"usage": {"input_tokens": 5, "cache_creation_input_tokens": 100, "cache_read_input_tokens": 2000,
"output_tokens": 40}}) == {"in": 2105, "out": 40, "cached": 2000}
assert gemini_usage({"stats": {"models": {"gemini-3.5-flash": {"tokens": {"prompt": 900, "candidates": 50,
"thoughts": 10, "cached": 0}}}}}) == \
{"in": 900, "out": 60, "cached": 0}
assert gemini_usage({"response": "x"}) is None
# ---------------------------------------------------------------- API e medição
def test_put_roles_and_read_defaults(env): # noqa: F811
client, engine = env
roles = {"scan": {"enabled": False, "provider": "codex", "images": "frames"},
"analyst": {"provider": "claude", "model": "haiku", "extra": "Olhe o HUD."},
"editor": {"enabled": False, "provider": "codex"},
"judge": {"enabled": True, "provider": "claude", "model": "sonnet", "critical": ["setup", "payoff"],
"allowContextLine": False}}
r = client.put("/v1/judge", json={"provider": "codex", "model": "", "minScore": 6, "roles": roles}, headers=auth())
assert r.status_code == 200
data = r.json()
assert data["roles"]["analyst"]["provider"] == "claude" and data["roles"]["analyst"]["extra"] == "Olhe o HUD."
assert data["roles"]["editor"]["enabled"] is True # o editor não desliga
assert data["roles"]["judge"]["critical"] == ["setup", "payoff"] and data["roles"]["judge"]["allowContextLine"] is False
# O cartão da varredura liga/desliga a varredura de verdade.
assert data["roles"]["scan"]["enabled"] is False and engine.intel.opts["scanEnabled"] is False
got = client.get("/v1/judge", headers=auth()).json()
assert set(got["roleDefaults"]) == {"scan", "analyst", "editor", "judge"}
assert got["roleDefaults"]["judge"]["en"].startswith("You are the CONTEXT JUDGE")
assert [c["id"] for c in got["checks"]][:3] == ["setup", "payoff", "who_what"]
assert "roleDefaults" not in client.get("/v1/status", headers=auth()).json()["judge"] # status fica leve
# A opção da linha do tempo (comparar com o formato antigo) também se troca pela mesma rota.
client.put("/v1/judge", json={"provider": "codex", "model": "", "minScore": 6, "timelineJson": False}, headers=auth())
assert engine.intel.opts["timelineJson"] is False
def test_owner_label_and_report(env): # noqa: F811
client, engine = env
sid = seed_session(engine)
db = engine.db
def clip(cid, status, decision, keep=True, label=None):
judge = {"keep": keep, "score": 7}
if decision:
judge["context"] = {"decision": decision, "contextLine": "x" if decision == "approve" else ""}
db.run("INSERT INTO clips (id, session_id, peak_s, start_s, end_s, score, source, status, judge_json, created_at,"
" updated_at, owner_label) VALUES (?, ?, 1, 0, 60, 5, 'chat', ?, ?, ?, ?, ?)",
cid, sid, status, json.dumps(judge), engine_now(), engine_now(), label)
clip(1, "approved", "approve") # juiz mantém, dono bom → concorda
clip(2, "rejected", "approve") # juiz mantém, dono ruim → discorda
clip(3, "discarded", "discard", label="no_context") # juiz descarta, dono "sem contexto" → concorda
clip(4, "ready", "adjust") # dono não avaliou
clip(5, "approved", "discard") # juiz descartou, dono forçou e aprovou → falso negativo
# Rejeitar com "sem contexto" pela rota (o rótulo vale também para corte antigo/descartado).
r = client.patch("/v1/clips/4", json={"label": "no_context", "labelNote": "não dá pra entender"}, headers=auth())
assert r.json()["ownerLabel"] == "no_context" and r.json()["ownerNote"] == "não dá pra entender"
assert r.json()["status"] == "ready" # avaliar não muda o estado
engine.record_ai_call({"role": "judge", "purpose": "judge", "provider": "codex", "model": "gpt-6-luna", "ok": True,
"ms": 1200, "systemChars": 2700, "promptChars": 3000, "images": 0, "imageBytes": 0,
"tokensIn": 1500, "tokensOut": 200, "session_id": sid, "mode": "timeline+mosaic"})
engine.record_ai_call({"role": "analyst", "purpose": "analyst", "provider": "codex", "model": "gpt-6-luna",
"ok": False, "error": "cota", "systemChars": 3000, "promptChars": 5000, "images": 1,
"imageBytes": 60_000, "session_id": sid})
rep = client.get(f"/v1/judge/report?session={sid}", headers=auth()).json()
assert rep["window"]["liveHours"] == round(600 / 3600, 2)
judge_row = next(c for c in rep["calls"] if c["role"] == "judge")
assert judge_row["tokensIn"] == 1500 and judge_row["withUsage"] == 1 and judge_row["mode"] == "timeline+mosaic"
assert next(c for c in rep["calls"] if c["role"] == "analyst")["failures"] == 1
assert rep["perHour"]["judge"]["calls"] == pytest.approx(6.0)
assert rep["judge"]["approve"] == 2 and rep["judge"]["discard"] == 2 and rep["judge"]["adjust"] == 1
assert rep["clips"]["approvalRate"] == round(2 / 3, 3) and rep["clips"]["noContext"] == 2
agreement = rep["agreement"]["judge"]
# pares: (keep, good) (keep, bad) (discard, no_context) (keep, no_context) (discard, good) → concordam 2 de 5
assert agreement["pairs"] == 5 and agreement["agree"] == 2
assert agreement["matrix"]["discard"]["good"] == 1 and agreement["matrix"]["keep"]["no_context"] == 1
def engine_now() -> str:
from app.db import now
return now()