"""Relatório da Equipe de IA (F0 — medir antes de otimizar; doc/clipador/EQUIPE-IA.md). `GET /v1/judge/report?days=7[&session=N]`: - `calls`: por papel × IA × modelo × modo (linha do tempo/antigo, mosaico/frames): chamadas, falhas, tamanho médio do prompt, imagens, tokens (quando a CLI informa) e tempo médio; - `perHour`: chamadas e tokens por hora de live gravada no período (por papel); - `clips`: cortes do período por estado e a taxa de aprovação do dono; - `judge`: o que o juiz de contexto decidiu (aprovou, ajustou, descartou, avisou, erro, frase de contexto); - `agreement`: concordância juiz × dono (e editor × dono) nos cortes que o dono avaliou. Veredito do dono: `owner_label` (bom / ruim / sem contexto) quando marcado; senão aprovado/publicado = bom e rejeitado = ruim. Veredito do juiz: aprovou/ajustou/avisou = mantém; descartou = descarta. """ from __future__ import annotations from datetime import datetime, timedelta, timezone from .db import Database, loads KEEP_DECISIONS = ("approve", "adjust", "warn") def _since(days: float) -> str: return (datetime.now(timezone.utc) - timedelta(days=days)).isoformat(timespec="seconds") def owner_verdict(row: dict) -> str | None: """good | bad | no_context | None (o dono ainda não disse).""" if row.get("owner_label"): return row["owner_label"] if row["status"] in ("approved", "published"): return "good" if row["status"] == "rejected": return "bad" return None def judge_verdict(judge: dict) -> str | None: """keep | discard | None (juiz não rodou ou falhou).""" context = (judge or {}).get("context") if isinstance(judge, dict) else None if not isinstance(context, dict) or context.get("error"): return None decision = context.get("decision") if decision in KEEP_DECISIONS: return "keep" return "discard" if decision == "discard" else None def editor_verdict(judge: dict) -> str | None: if not isinstance(judge, dict) or "keep" not in judge: return None return "keep" if judge.get("keep") else "discard" def agreement(pairs: list[tuple[str, str]]) -> dict: """Concordância (e kappa de Cohen) entre "mantém/descarta" da IA e "bom/ruim" do dono ("sem contexto" = ruim).""" matrix = {"keep": {"good": 0, "bad": 0, "no_context": 0}, "discard": {"good": 0, "bad": 0, "no_context": 0}} for ai, owner in pairs: matrix[ai][owner] += 1 n = len(pairs) if not n: return {"pairs": 0, "agree": 0, "rate": None, "kappa": None, "matrix": matrix} agree = matrix["keep"]["good"] + matrix["discard"]["bad"] + matrix["discard"]["no_context"] p_keep = (sum(matrix["keep"].values())) / n p_good = (matrix["keep"]["good"] + matrix["discard"]["good"]) / n pe = p_keep * p_good + (1 - p_keep) * (1 - p_good) po = agree / n kappa = None if pe >= 1 else round((po - pe) / (1 - pe), 3) return {"pairs": n, "agree": agree, "rate": round(po, 3), "kappa": kappa, "matrix": matrix} def build(db: Database, days: float = 7, session_id: int | None = None) -> dict: since = _since(days) call_where, call_args = ("session_id = ?", [session_id]) if session_id else ("at >= ?", [since]) rows = db.all( "SELECT role, purpose, provider, model, COALESCE(mode, '') AS mode, COUNT(*) AS calls, SUM(ok) AS ok," " AVG(COALESCE(system_chars, 0) + COALESCE(prompt_chars, 0)) AS avg_chars, AVG(images) AS avg_images," " SUM(COALESCE(image_bytes, 0)) AS image_bytes, SUM(tokens_in) AS tokens_in, SUM(tokens_out) AS tokens_out," " SUM(tokens_cached) AS tokens_cached, SUM(CASE WHEN tokens_in IS NOT NULL THEN 1 ELSE 0 END) AS with_usage," " AVG(ms) AS avg_ms FROM ai_calls WHERE " + call_where + " GROUP BY role, purpose, provider, model, mode ORDER BY role, calls DESC", *call_args) calls = [{"role": r["role"], "purpose": r["purpose"], "provider": r["provider"], "model": r["model"], "mode": r["mode"], "calls": r["calls"], "failures": r["calls"] - (r["ok"] or 0), "avgChars": round(r["avg_chars"] or 0), "avgImages": round(r["avg_images"] or 0, 2), "imageKB": round((r["image_bytes"] or 0) / 1024), "tokensIn": r["tokens_in"], "tokensOut": r["tokens_out"], "tokensCached": r["tokens_cached"], "withUsage": r["with_usage"], "avgMs": round(r["avg_ms"] or 0)} for r in rows] session_where, session_args = ("id = ?", [session_id]) if session_id else ("started_at >= ?", [since]) live_s = db.get(f"SELECT COALESCE(SUM(duration_s), 0) AS s FROM sessions WHERE {session_where}", *session_args)["s"] hours = live_s / 3600 per_role: dict[str, dict] = {} for c in calls: item = per_role.setdefault(c["role"], {"calls": 0, "tokensIn": 0, "tokensOut": 0, "withUsage": 0, "chars": 0}) item["calls"] += c["calls"] item["tokensIn"] += c["tokensIn"] or 0 item["tokensOut"] += c["tokensOut"] or 0 item["withUsage"] += c["withUsage"] item["chars"] += c["avgChars"] * c["calls"] per_hour = {role: {"calls": round(v["calls"] / hours, 1) if hours else None, "tokensIn": round(v["tokensIn"] / hours) if hours and v["withUsage"] else None, "tokensOut": round(v["tokensOut"] / hours) if hours and v["withUsage"] else None, # Estimativa pelo tamanho do texto (~4 caracteres por token), para quando a CLI não informa. "estTokensIn": round(v["chars"] / 4 / hours) if hours else None} for role, v in per_role.items()} clip_where, clip_args = ("session_id = ?", [session_id]) if session_id else ("created_at >= ?", [since]) clips = db.all(f"SELECT id, status, judge_json, owner_label FROM clips WHERE {clip_where}", *clip_args) by_status: dict[str, int] = {} decisions = {"approve": 0, "adjust": 0, "discard": 0, "warn": 0, "error": 0, "contextLine": 0} judge_pairs: list[tuple[str, str]] = [] editor_pairs: list[tuple[str, str]] = [] labeled = 0 for row in clips: by_status[row["status"]] = by_status.get(row["status"], 0) + 1 judge = loads(row["judge_json"], None) context = judge.get("context") if isinstance(judge, dict) else None if isinstance(context, dict): if context.get("error"): decisions["error"] += 1 elif context.get("decision") in decisions: decisions[context["decision"]] += 1 if context.get("contextLine"): decisions["contextLine"] += 1 owner = owner_verdict(row) labeled += owner is not None if owner: if (jv := judge_verdict(judge)) is not None: judge_pairs.append((jv, owner)) if (ev := editor_verdict(judge)) is not None: editor_pairs.append((ev, owner)) approved = by_status.get("approved", 0) + by_status.get("published", 0) decided = approved + by_status.get("rejected", 0) no_context = sum(1 for r in clips if r.get("owner_label") == "no_context") return { "window": {"days": days, "since": since if not session_id else None, "sessionId": session_id, "liveHours": round(hours, 2)}, "calls": calls, "perHour": per_hour, "clips": {"total": len(clips), "byStatus": by_status, "approvalRate": round(approved / decided, 3) if decided else None, "labeled": labeled, "noContext": no_context}, "judge": decisions, "agreement": {"judge": agreement(judge_pairs), "editor": agreement(editor_pairs)}, }