"""Equipe de IA (doc/clipador/EQUIPE-IA.md): papéis com IA própria, migração da configuração antiga, instruções do administrador, esforço por papel, medição por chamada (F0) e o relatório. Nenhuma CLI de verdade roda aqui.""" import json import subprocess from pathlib import Path import pytest from app import judge as judge_module from app import roles as roles_module from app.ai_settings import AiConfig from app.config import Settings from app.db import Database from app.judge import Judge, claude_usage, codex_usage, gemini_usage from app.worker import Engine from .test_sources_and_api import KEY, auth, env, seed_session # noqa: F401 - fixture reaproveitada ANALYSIS = {"situation": "jogada", "what_happened": "Ele roubou o banco.", "screen": "", "speech": "", "chat_meaning": "", "confidence": 0.9, "action_start_s": 100, "reaction_end_s": 160, "hook": "Assalto solo", "lines": []} VERDICT = {"keep": True, "score": 8, "start_s": 100, "end_s": 170, "title": "Virada", "caption": "c", "hashtags": [], "reason": "r"} CONTEXT_OK = {"viewer_summary": "Ele rouba o banco sozinho.", "fix": {"start_s": None, "end_s": None, "context_line": ""}, "checks": {c: {"ok": True, "t": None, "why": ""} for c in roles_module.CHECKS}, "confidence": 0.9, "reason": "ok"} SCAN = {"screen_game": "GTA", "has_webcam": False, "summary": "s", "moments": [], "lines": []} def answer_for(schema: dict) -> dict: props = schema["properties"] if "viewer_summary" in props: return CONTEXT_OK if "situation" in props: return ANALYSIS if "moments" in props: return SCAN if list(props) == ["lines"]: return {"lines": ["oi"]} return VERDICT @pytest.fixture def team(tmp_path, monkeypatch): """Juiz de verdade com as CLIs trocadas por um registro: cada chamada guarda provedor, modelo, esforço e sistema.""" calls: list[dict] = [] def fake_call(self, provider, system, prompt, schema, images, search, timeout, model, effort=""): calls.append({"provider": provider, "model": model, "effort": effort, "system": system, "prompt": prompt}) return answer_for(schema) monkeypatch.setattr(Judge, "_call", fake_call) for name in ("codex", "claude", "gemini"): monkeypatch.setattr(judge_module, f"find_{name}", (lambda n=name: [n])) return calls def make_engine(tmp_path, saved: dict[str, str] | None = None) -> Engine: settings = Settings.load(env_file=tmp_path / "none.env", service_key=KEY, data_dir=tmp_path / "data", clips_dir=tmp_path / "clips", judge_provider="codex", voice_url=None, voice_key=None) db = Database(settings.data_dir / "clipper.db") if saved: db.many("INSERT INTO settings (key, value) VALUES (?, ?)", list(saved.items())) return Engine(settings, db) # ---------------------------------------------------------------- migração def test_old_single_provider_becomes_every_role_and_analyst_model_is_kept(tmp_path): engine = make_engine(tmp_path, {"judge_provider": "claude", "judge_model": "sonnet", "judge_analyst_model": "haiku"}) roles = engine.judge.roles assert {r["provider"] for r in roles.values()} == {"claude"} assert roles["editor"]["model"] == "sonnet" and roles["scan"]["model"] == "sonnet" and roles["judge"]["model"] == "sonnet" assert roles["analyst"]["model"] == "haiku" # o "modelo do analista" que já existia é respeitado assert roles["judge"]["enabled"] and roles["judge"]["critical"] == ["setup", "payoff", "standalone"] # A migração fica gravada (explícita no banco); a próxima subida lê `judge.roles`. saved = json.loads(engine.db.get("SELECT value FROM settings WHERE key = 'judge.roles'")["value"]) assert saved["analyst"]["model"] == "haiku" again = Engine(engine.settings, engine.db) assert again.judge.roles == roles def test_off_master_switch_is_kept_and_roles_still_migrate(tmp_path): engine = make_engine(tmp_path, {"judge_provider": "off", "judge_model": ""}) assert not engine.judge.enabled and engine.judge.provider == "off" assert {r["provider"] for r in engine.judge.roles.values()} == {"codex"} def test_normalize_roles_field_by_field(): base = roles_module.default_roles("codex", "gpt-6-luna") out = roles_module.normalize_roles({ "editor": {"enabled": False, "provider": "claude"}, # editor nunca desliga; trocou de provedor → modelo padrão "judge": {"effort": "turbo", "critical": ["payoff", "inventado"], "penalty": 9, "extra": " x "}, "scan": {"images": "frames", "model": "bad model;"}, }, base) assert out["editor"]["enabled"] is True and out["editor"]["provider"] == "claude" and out["editor"]["model"] == "" assert out["judge"]["effort"] == "medium" and out["judge"]["critical"] == ["payoff"] assert out["judge"]["penalty"] == 0.5 and out["judge"]["extra"] == "x" assert out["scan"]["images"] == "frames" and out["scan"]["model"] == "gpt-6-luna" # ---------------------------------------------------------------- cada papel chama a sua IA def test_each_role_calls_its_own_provider_model_and_effort(tmp_path, team): roles = roles_module.default_roles("codex") roles["scan"] |= {"provider": "codex", "model": "gpt-6-luna", "effort": "low"} roles["analyst"] |= {"provider": "claude", "model": "haiku"} roles["editor"] |= {"provider": "codex", "model": ""} roles["judge"] |= {"provider": "claude", "model": "sonnet", "effort": ""} judge = Judge("codex", "", tmp_path / "j", ai=AiConfig(tmp_path / "ai"), roles=roles) judge.scan("p", [], lang="en") judge.analyze("p", [], lang="en") judge.judge("p", 100, 170, [], lang="en") judge.check_context("p", lang="en") got = [(c["provider"], c["model"], c["effort"]) for c in team] assert got == [("codex", "gpt-6-luna", "low"), ("claude", "haiku", ""), ("codex", "gpt-6-luna", ""), ("claude", "sonnet", "")] # O que não é dos quatro papéis (tradução, pesquisa) usa a IA do editor. judge.translate_lines(["oi"]) assert team[-1]["provider"] == "codex" assert judge.fallback_chain("claude")[0] == "claude" and judge.fallback_chain()[0] == "codex" def test_admin_instructions_are_added_after_the_default_text(tmp_path, team): roles = roles_module.default_roles("codex") roles["judge"]["extra"] = "Não aprove corte que termina antes do resultado da treta." roles["analyst"]["extra"] = "Lag conta como jogada." judge = Judge("codex", "", tmp_path / "j", ai=AiConfig(tmp_path / "ai"), roles=roles) judge.check_context("p", lang="en") judge.analyze("p", [], lang="pt") judge.judge("p", 100, 170, [], lang="pt") context_system, analyst_system, editor_system = (c["system"] for c in team) assert context_system.startswith(judge_module.CONTEXT_SYSTEM_EN) # o padrão continua inteiro, no começo assert context_system.endswith("Admin notes (may be written in Portuguese; when they conflict with the rules above, " "they win):\nNão aprove corte que termina antes do resultado da treta.") assert analyst_system.endswith("valem estas):\nLag conta como jogada.") assert "Instruções do administrador" not in editor_system # o editor não tem extra def test_codex_effort_goes_to_the_command_line(tmp_path, monkeypatch): seen: list[list[str]] = [] def run(args, input=None, capture_output=True, timeout=None, cwd=None, creationflags=0, env=None): seen.append(list(args)) Path(args[args.index("-o") + 1]).write_text(json.dumps(CONTEXT_OK), encoding="utf-8") events = '{"type":"thread.started"}\n{"type":"turn.completed","usage":{"input_tokens":1234,"cached_input_tokens":200,"output_tokens":56}}\n' return subprocess.CompletedProcess(args, 0, events.encode(), b"") monkeypatch.setattr(judge_module.subprocess, "run", run) monkeypatch.setattr(judge_module, "find_codex", lambda: ["codex"]) monkeypatch.setattr(judge_module, "find_claude", lambda: None) records = [] judge = Judge("codex", "", tmp_path / "j", ai=AiConfig(tmp_path / "ai")) judge.meter = records.append with judge.scope(clip_id=7, session_id=3, mode="timeline+mosaic"): judge.check_context("prompt de teste", lang="pt") assert 'model_reasoning_effort="medium"' in seen[0] and "--json" in seen[0] # juiz: esforço médio por padrão rec = records[0] assert rec["role"] == "judge" and rec["provider"] == "codex" and rec["ok"] is True assert (rec["tokensIn"], rec["tokensOut"], rec["tokensCached"]) == (1234, 56, 200) assert rec["clip_id"] == 7 and rec["mode"] == "timeline+mosaic" and rec["promptChars"] == len("prompt de teste") def test_usage_parsers(): assert codex_usage("lixo\n{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":10,\"output_tokens\":2}}") == \ {"in": 10, "out": 2, "cached": None} assert codex_usage("") is None assert claude_usage({"usage": {"input_tokens": 5, "cache_creation_input_tokens": 100, "cache_read_input_tokens": 2000, "output_tokens": 40}}) == {"in": 2105, "out": 40, "cached": 2000} assert gemini_usage({"stats": {"models": {"gemini-3.5-flash": {"tokens": {"prompt": 900, "candidates": 50, "thoughts": 10, "cached": 0}}}}}) == \ {"in": 900, "out": 60, "cached": 0} assert gemini_usage({"response": "x"}) is None # ---------------------------------------------------------------- API e medição def test_put_roles_and_read_defaults(env): # noqa: F811 client, engine = env roles = {"scan": {"enabled": False, "provider": "codex", "images": "frames"}, "analyst": {"provider": "claude", "model": "haiku", "extra": "Olhe o HUD."}, "editor": {"enabled": False, "provider": "codex"}, "judge": {"enabled": True, "provider": "claude", "model": "sonnet", "critical": ["setup", "payoff"], "allowContextLine": False}} r = client.put("/v1/judge", json={"provider": "codex", "model": "", "minScore": 6, "roles": roles}, headers=auth()) assert r.status_code == 200 data = r.json() assert data["roles"]["analyst"]["provider"] == "claude" and data["roles"]["analyst"]["extra"] == "Olhe o HUD." assert data["roles"]["editor"]["enabled"] is True # o editor não desliga assert data["roles"]["judge"]["critical"] == ["setup", "payoff"] and data["roles"]["judge"]["allowContextLine"] is False # O cartão da varredura liga/desliga a varredura de verdade. assert data["roles"]["scan"]["enabled"] is False and engine.intel.opts["scanEnabled"] is False got = client.get("/v1/judge", headers=auth()).json() assert set(got["roleDefaults"]) == {"scan", "analyst", "editor", "judge"} assert got["roleDefaults"]["judge"]["en"].startswith("You are the CONTEXT JUDGE") assert [c["id"] for c in got["checks"]][:3] == ["setup", "payoff", "who_what"] assert "roleDefaults" not in client.get("/v1/status", headers=auth()).json()["judge"] # status fica leve # A opção da linha do tempo (comparar com o formato antigo) também se troca pela mesma rota. client.put("/v1/judge", json={"provider": "codex", "model": "", "minScore": 6, "timelineJson": False}, headers=auth()) assert engine.intel.opts["timelineJson"] is False def test_owner_label_and_report(env): # noqa: F811 client, engine = env sid = seed_session(engine) db = engine.db def clip(cid, status, decision, keep=True, label=None): judge = {"keep": keep, "score": 7} if decision: judge["context"] = {"decision": decision, "contextLine": "x" if decision == "approve" else ""} db.run("INSERT INTO clips (id, session_id, peak_s, start_s, end_s, score, source, status, judge_json, created_at," " updated_at, owner_label) VALUES (?, ?, 1, 0, 60, 5, 'chat', ?, ?, ?, ?, ?)", cid, sid, status, json.dumps(judge), engine_now(), engine_now(), label) clip(1, "approved", "approve") # juiz mantém, dono bom → concorda clip(2, "rejected", "approve") # juiz mantém, dono ruim → discorda clip(3, "discarded", "discard", label="no_context") # juiz descarta, dono "sem contexto" → concorda clip(4, "ready", "adjust") # dono não avaliou clip(5, "approved", "discard") # juiz descartou, dono forçou e aprovou → falso negativo # Rejeitar com "sem contexto" pela rota (o rótulo vale também para corte antigo/descartado). r = client.patch("/v1/clips/4", json={"label": "no_context", "labelNote": "não dá pra entender"}, headers=auth()) assert r.json()["ownerLabel"] == "no_context" and r.json()["ownerNote"] == "não dá pra entender" assert r.json()["status"] == "ready" # avaliar não muda o estado engine.record_ai_call({"role": "judge", "purpose": "judge", "provider": "codex", "model": "gpt-6-luna", "ok": True, "ms": 1200, "systemChars": 2700, "promptChars": 3000, "images": 0, "imageBytes": 0, "tokensIn": 1500, "tokensOut": 200, "session_id": sid, "mode": "timeline+mosaic"}) engine.record_ai_call({"role": "analyst", "purpose": "analyst", "provider": "codex", "model": "gpt-6-luna", "ok": False, "error": "cota", "systemChars": 3000, "promptChars": 5000, "images": 1, "imageBytes": 60_000, "session_id": sid}) rep = client.get(f"/v1/judge/report?session={sid}", headers=auth()).json() assert rep["window"]["liveHours"] == round(600 / 3600, 2) judge_row = next(c for c in rep["calls"] if c["role"] == "judge") assert judge_row["tokensIn"] == 1500 and judge_row["withUsage"] == 1 and judge_row["mode"] == "timeline+mosaic" assert next(c for c in rep["calls"] if c["role"] == "analyst")["failures"] == 1 assert rep["perHour"]["judge"]["calls"] == pytest.approx(6.0) assert rep["judge"]["approve"] == 2 and rep["judge"]["discard"] == 2 and rep["judge"]["adjust"] == 1 assert rep["clips"]["approvalRate"] == round(2 / 3, 3) and rep["clips"]["noContext"] == 2 agreement = rep["agreement"]["judge"] # pares: (keep, good) (keep, bad) (discard, no_context) (keep, no_context) (discard, good) → concordam 2 de 5 assert agreement["pairs"] == 5 and agreement["agree"] == 2 assert agreement["matrix"]["discard"]["good"] == 1 and agreement["matrix"]["keep"]["no_context"] == 1 def engine_now() -> str: from app.db import now return now()