"""Gemini no juiz, IA reserva e modo chave de API (contrato em doc/ia.md). Nenhuma CLI de verdade roda aqui: `subprocess.run` e os `find_*` do juiz são trocados por falsos.""" import json import os import subprocess from pathlib import Path import pytest from app import judge as judge_module from app.ai_settings import DEFAULT_SETTINGS, AiConfig, normalize_settings from app.judge import (DEFAULT_MODELS, VERDICT_SCHEMA, Judge, JudgeError, extract_json_object, is_availability_error, parse_gemini_output) from tests.test_sources_and_api import KEY, auth, env # noqa: F401 - fixture reaproveitada VERDICT = {"keep": True, "score": 8, "start_s": 100, "end_s": 170, "title": "Virada", "caption": "c", "hashtags": ["lag"], "reason": "r"} class ReversedCipher: """Cifrador falso: o "blob" é o JSON de trás para frente (garante que o cofre passa pelo cifrador).""" def unprotect(self, data: bytes) -> bytes: return data[::-1] _STAMPS = iter(range(1_700_000_000, 1_800_000_000, 7)) def write_settings(folder: Path, data: dict) -> None: folder.mkdir(parents=True, exist_ok=True) path = folder / "settings.json" path.write_text(json.dumps(data), encoding="utf-8") # mtime sempre diferente do anterior (o sistema de arquivos pode ter resolução grossa) stamp = next(_STAMPS) * 1_000_000_000 os.utime(path, ns=(stamp, stamp)) def write_vault(folder: Path, keys: dict) -> None: folder.mkdir(parents=True, exist_ok=True) (folder / "vault.bin").write_bytes(json.dumps(keys).encode("utf-8")[::-1]) def settings_with(**providers) -> dict: data = json.loads(json.dumps(DEFAULT_SETTINGS)) for name, conf in providers.items(): data["providers"][name].update(conf) return data class FakeCli: """Faz o papel das três CLIs. `behavior[provider]` = "ok", "content" (resposta sem JSON) ou um texto de erro (sai com código 1 e esse texto no stderr).""" def __init__(self, behavior: dict[str, str]): self.behavior = behavior self.calls: list[dict] = [] def run(self, args, input=None, capture_output=True, timeout=None, cwd=None, creationflags=0, env=None): provider = args[0] self.calls.append({"provider": provider, "args": list(args), "env": dict(env or {}), "cwd": cwd, "stdin": (input or b"").decode("utf-8"), "files": sorted(p.name for p in Path(cwd).iterdir()) if cwd else []}) mode = self.behavior.get(provider, "ok") if mode not in ("ok", "content"): return subprocess.CompletedProcess(args, 1, b"", mode.encode("utf-8")) answer = VERDICT if mode == "ok" else None if provider == "codex": out = Path(args[args.index("-o") + 1]) out.write_text(json.dumps(answer) if answer else "não sei", encoding="utf-8") return subprocess.CompletedProcess(args, 0, b"", b"") if provider == "claude": envelope = {"is_error": False, "structured_output": answer, "result": "" if answer else "não sei"} return subprocess.CompletedProcess(args, 0, json.dumps(envelope).encode(), b"") text = f"Aqui está:\n```json\n{json.dumps(answer)}\n```" if answer else "Não consegui." return subprocess.CompletedProcess(args, 0, json.dumps({"response": text, "stats": {}}).encode(), b"") @pytest.fixture def ai_dir(tmp_path): return tmp_path / "ai" @pytest.fixture def make_judge(tmp_path, ai_dir, monkeypatch): clock = {"t": 1_000_000.0} def factory(behavior=None, provider="codex", model="", installed=("codex", "claude", "gemini")): cli = FakeCli(behavior or {}) monkeypatch.setattr(judge_module.subprocess, "run", cli.run) for name in ("codex", "claude", "gemini"): monkeypatch.setattr(judge_module, f"find_{name}", (lambda n=name: [n] if n in installed else None)) ai = AiConfig(ai_dir, cipher=ReversedCipher(), environ={"PATH": "x", "ANTHROPIC_API_KEY": "global", "GOOGLE_GENAI_USE_GCA": "true"}) judge = Judge(provider, model, tmp_path / "judge", ai=ai, clock=lambda: clock["t"]) judge.cli, judge.clock = cli, clock return judge return factory # ---------------------------------------------------------------- classificador @pytest.mark.parametrize("text", [ "Error: 429 Too Many Requests", "You've hit your usage limit", "Your credit balance is too low", "API Error: 529 overloaded_error", "503 Service Unavailable", "Not logged in · Please run /login", "invalid api key", "IneligibleTierError: UNSUPPORTED_CLIENT", "connect ECONNREFUSED 127.0.0.1", "getaddrinfo ENOTFOUND api.openai.com", "network error", "a CLI passou de 240 s (tempo esgotado)", "Codex CLI não encontrado", "RESOURCE_EXHAUSTED: quota exceeded", "401 Unauthorized", "status 403", ]) def test_availability_errors(text): assert is_availability_error(text) @pytest.mark.parametrize("text", ["", "o Codex não devolveu o JSON pedido", "resposta sem os campos ['title']", "Expecting value: line 1 column 1", "schema validation failed"]) def test_content_errors_are_not_availability(text): assert not is_availability_error(text) # ---------------------------------------------------------------- settings.json e cofre def test_settings_default_when_missing_or_invalid(ai_dir): config = AiConfig(ai_dir, cipher=ReversedCipher()) assert config.settings() == DEFAULT_SETTINGS ai_dir.mkdir(parents=True) (ai_dir / "settings.json").write_text("{quebrado", encoding="utf-8") assert AiConfig(ai_dir).settings() == DEFAULT_SETTINGS assert DEFAULT_SETTINGS["providers"]["gemini"] == {"enabled": False, "authMode": "apiKey"} def test_settings_normalized_field_by_field(): data = normalize_settings({"providers": {"gemini": {"enabled": True, "authMode": "senha"}, "gpt": {}}, "order": ["gemini", "gpt", "gemini"], "fallback": {"enabled": False, "cooldownMin": -3}}) assert data["providers"]["gemini"] == {"enabled": True, "authMode": "apiKey"} # modo inválido fica no padrão assert data["order"] == ["gemini", "claude", "codex"] # desconhecido sai, faltantes vão para o fim assert data["fallback"] == {"enabled": False, "cooldownMin": 15.0} def test_settings_reread_when_mtime_changes(ai_dir): config = AiConfig(ai_dir) write_settings(ai_dir, settings_with(gemini={"enabled": True})) assert config.settings()["providers"]["gemini"]["enabled"] is True write_settings(ai_dir, settings_with(gemini={"enabled": False}, claude={"authMode": "apiKey"})) assert config.settings()["providers"]["gemini"]["enabled"] is False assert config.settings()["providers"]["claude"]["authMode"] == "apiKey" def test_env_per_auth_mode(ai_dir): environ = {"PATH": "x", "ANTHROPIC_API_KEY": "global", "CODEX_API_KEY": "global", "GEMINI_API_KEY": "global", "GOOGLE_GENAI_USE_GCA": "true"} config = AiConfig(ai_dir, cipher=ReversedCipher(), environ=environ) write_vault(ai_dir, {"claude": "sk-ant-1", "gemini": "AIza-2", "outro": "x"}) # padrão: claude/codex em oauth (a variável sai), gemini em apiKey (a chave do cofre entra, GCA sai) claude, codex, gemini = (config.provider_env(p) for p in ("claude", "codex", "gemini")) assert "ANTHROPIC_API_KEY" not in claude and claude["PATH"] == "x" assert "CODEX_API_KEY" not in codex assert gemini["GEMINI_API_KEY"] == "AIza-2" and "GOOGLE_GENAI_USE_GCA" not in gemini write_settings(ai_dir, settings_with(claude={"authMode": "apiKey"}, gemini={"authMode": "oauth"})) assert config.provider_env("claude")["ANTHROPIC_API_KEY"] == "sk-ant-1" gemini = config.provider_env("gemini") assert gemini["GOOGLE_GENAI_USE_GCA"] == "true" and "GEMINI_API_KEY" not in gemini assert environ["ANTHROPIC_API_KEY"] == "global" # o ambiente do processo não muda assert config.api_keys() == {"claude": "sk-ant-1", "gemini": "AIza-2"} def test_unreadable_vault_means_no_keys(ai_dir): ai_dir.mkdir(parents=True) (ai_dir / "vault.bin").write_bytes(b"\x00lixo") config = AiConfig(ai_dir, cipher=ReversedCipher()) assert config.api_keys() == {} and config.vault_error # ---------------------------------------------------------------- cadeia de reserva def test_chain_skips_disabled_missing_cli_no_key_and_paused(make_judge, ai_dir): judge = make_judge(provider="codex") # padrão: gemini desligado → codex (escolhido) e depois claude assert judge.fallback_chain() == ["codex", "claude"] write_settings(ai_dir, settings_with(gemini={"enabled": True})) # ligado, mas sem chave no cofre assert judge.fallback_chain() == ["codex", "claude"] write_vault(ai_dir, {"gemini": "AIza"}) assert judge.fallback_chain() == ["codex", "claude", "gemini"] judge.pause("claude", "429") assert judge.fallback_chain() == ["codex", "gemini"] judge.clock["t"] += 15 * 60 + 1 # a pausa vence assert judge.fallback_chain() == ["codex", "claude", "gemini"] write_settings(ai_dir, {**settings_with(gemini={"enabled": True}), "order": ["gemini", "codex", "claude"]}) assert judge.fallback_chain() == ["codex", "gemini", "claude"] write_settings(ai_dir, {**settings_with(), "fallback": {"enabled": False, "cooldownMin": 15}}) assert judge.fallback_chain() == ["codex"] write_settings(ai_dir, settings_with()) assert make_judge(provider="codex", installed=("claude",)).fallback_chain() == ["claude"] # codex sem CLI def test_unavailable_switches_to_next_with_its_default_model_and_pauses(make_judge): judge = make_judge({"codex": "Error: 429 Too Many Requests"}, provider="codex", model="gpt-caro") verdict = judge.judge("prompt", 100, 170, lo=0, hi=300) assert [c["provider"] for c in judge.cli.calls] == ["codex", "claude"] assert verdict.model == f"claude:{DEFAULT_MODELS['claude']}" # quem de fato respondeu, com o modelo padrão dele claude_args = judge.cli.calls[1]["args"] assert claude_args[claude_args.index("--model") + 1] == "haiku" pauses = judge.describe()["pauses"] assert [p["provider"] for p in pauses] == ["codex"] and "429" in pauses[0]["reason"] # próxima chamada já pula o codex em pausa judge.cli.calls.clear() judge.judge("prompt", 100, 170, lo=0, hi=300) assert [c["provider"] for c in judge.cli.calls] == ["claude"] def test_content_error_does_not_switch(make_judge): judge = make_judge({"codex": "content"}, provider="codex") with pytest.raises(JudgeError) as info: judge.judge("prompt", 100, 170) assert info.value.unavailable is False assert [c["provider"] for c in judge.cli.calls] == ["codex"] and judge.pauses() == {} def test_all_fail_raises_summary(make_judge): judge = make_judge({"codex": "503 unavailable", "claude": "Not logged in"}, provider="codex") with pytest.raises(JudgeError) as info: judge.judge("prompt", 100, 170) message = str(info.value) assert "codex:" in message and "claude:haiku" in message and info.value.unavailable is True assert set(judge.pauses()) == {"codex", "claude"} def test_fallback_disabled_keeps_only_primary(make_judge, ai_dir): write_settings(ai_dir, {**settings_with(), "fallback": {"enabled": False, "cooldownMin": 15}}) judge = make_judge({"codex": "429"}, provider="codex") with pytest.raises(JudgeError): judge.judge("prompt", 100, 170) assert [c["provider"] for c in judge.cli.calls] == ["codex"] and judge.pauses() == {} def test_primary_disabled_in_settings_is_skipped(make_judge, ai_dir): write_settings(ai_dir, settings_with(codex={"enabled": False})) judge = make_judge(provider="codex") assert judge.judge("prompt", 100, 170).model == "claude:haiku" assert judge.describe()["unavailableReason"] == "desligada em Configurações → IA" def test_describe_format(make_judge, ai_dir): judge = make_judge(provider="gemini", model="") write_settings(ai_dir, settings_with(gemini={"enabled": True})) write_vault(ai_dir, {"gemini": "AIza"}) judge.pause("codex", "usage limit") data = judge.describe() assert data["provider"] == "gemini" and data["model"] == "gemini-3.5-flash" and data["authMode"] == "apiKey" assert data["fallback"] == {"enabled": True, "cooldownMin": 15.0, "chain": ["gemini", "claude"]} assert data["pauses"][0]["provider"] == "codex" and data["pauses"][0]["until"].endswith("+00:00") assert "gemini" in judge.models() and judge.models()["defaults"]["gemini"] == "gemini-3.5-flash" # ---------------------------------------------------------------- Gemini def test_gemini_call_args_images_and_env(make_judge, ai_dir, tmp_path): write_settings(ai_dir, settings_with(gemini={"enabled": True})) write_vault(ai_dir, {"gemini": "AIza-chave"}) frames = [] for i in range(2): frame = tmp_path / "frames" / f"clip1-{i}.JPG" frame.parent.mkdir(exist_ok=True) frame.write_bytes(b"jpg") frames.append(frame) judge = make_judge(provider="gemini", model="gemini-2.5-pro") verdict = judge.judge("PROMPT GRANDE", 100, 170, frames) call = judge.cli.calls[0] args = call["args"] assert args[:2] == ["gemini", "-p"] and "@frame-01.jpg @frame-02.jpg" in args[2] assert args[args.index("-o") + 1] == "json" and args[args.index("--approval-mode") + 1] == "plan" assert "--skip-trust" in args and args[args.index("-m") + 1] == "gemini-2.5-pro" assert "PROMPT GRANDE" in call["stdin"] and '"start_s"' in call["stdin"] and "PROMPT GRANDE" not in args[2] assert call["files"] == ["frame-01.jpg", "frame-02.jpg"] # imagens copiadas para a pasta de trabalho assert call["env"]["GEMINI_API_KEY"] == "AIza-chave" and "GOOGLE_GENAI_USE_GCA" not in call["env"] assert verdict.model == "gemini:gemini-2.5-pro" and verdict.title == "Virada" def test_extract_json_object_tolerates_fences_and_prose(): assert extract_json_object('```json\n{"a": 1}\n```') == {"a": 1} assert extract_json_object('Claro! {"a": {"b": [1, 2]}} espero ter ajudado') == {"a": {"b": [1, 2]}} assert extract_json_object("o conjunto {x} e depois {\"ok\": true}") == {"ok": True} assert extract_json_object("sem json") is None and extract_json_object("") is None def test_parse_gemini_output(): assert parse_gemini_output(json.dumps({"response": '```json\n{"keep": true}\n```', "stats": {}})) == {"keep": True} with pytest.raises(JudgeError) as info: parse_gemini_output(json.dumps({"response": "não deu"})) assert info.value.unavailable is False with pytest.raises(JudgeError) as info: parse_gemini_output(json.dumps({"error": {"type": "Error", "message": "429 RESOURCE_EXHAUSTED quota"}})) assert info.value.unavailable is None and is_availability_error(str(info.value)) assert set(VERDICT) == set(VERDICT_SCHEMA["required"]) def test_find_gemini_runs_bundle_with_node_on_windows(tmp_path, monkeypatch): npm = tmp_path / "npm" bundle = npm / "node_modules" / "@google" / "gemini-cli" / "bundle" / "gemini.js" bundle.parent.mkdir(parents=True) bundle.write_text("//", encoding="utf-8") monkeypatch.delenv("GEMINI_BIN", raising=False) monkeypatch.setattr(judge_module.sys, "platform", "win32") which = {"gemini.cmd": str(npm / "gemini.cmd"), "node": "C:/node/node.exe"} monkeypatch.setattr(judge_module.shutil, "which", lambda name: which.get(name)) assert judge_module.find_gemini() == ["C:/node/node.exe", str(bundle)] # ---------------------------------------------------------------- API def test_api_accepts_gemini(env): # noqa: F811 client, engine = env r = client.put("/v1/judge", json={"provider": "gemini", "model": "", "minScore": 6}, headers=auth()) assert r.status_code == 200 body = r.json() assert body["provider"] == "gemini" and body["model"] == "gemini-3.5-flash" and "gemini" in body["models"] assert "chain" in body["fallback"] and isinstance(body["pauses"], list)