Gerador de vídeos narrativos (Godot + FFmpeg), painel Next.js, serviço de voz, Clipador de lives e Publicador para YouTube, TikTok e Instagram. Inclui a leitura das visualizações dos posts para o Dashboard e o botão "Publicar no <rede>" nos cartões das publicações. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
563 lines
24 KiB
Python
563 lines
24 KiB
Python
"""Bancada de rastreio de rosto do Clipador (Fase 0 do doc/PLANO_RASTREIO_REACAO.md).
|
||
|
||
Compara, numa sessão já gravada:
|
||
1. YuNet no quadro inteiro para achar a câmera (12 quadros espalhados);
|
||
2. MediaPipe Face Landmarker (blendshapes) × OpenSeeFace (só os .onnx) no recorte da câmera,
|
||
em trechos de 30 s a 6 qps.
|
||
|
||
Uso (venv do Clipador):
|
||
clipper\\.venv\\Scripts\\python.exe clipper\\tools\\face_bench.py --session 3
|
||
|
||
Lê o banco só em modo leitura (o serviço do usuário pode estar rodando). Saídas em output/face-bench/.
|
||
Não é usado pelo serviço; a integração fica em clipper/app/face.py (Fase 2).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import json
|
||
import sqlite3
|
||
import statistics
|
||
import subprocess
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
|
||
import cv2
|
||
import numpy as np
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
MODELS = ROOT / "data" / "clipper" / "models"
|
||
YUNET = "face_detection_yunet_2026may.onnx"
|
||
YUNET_OLD = "face_detection_yunet_2023mar.onnx"
|
||
LANDMARKER = "face_landmarker.task"
|
||
OSF_LM = "osf_lm_model3_opt.onnx"
|
||
OSF_DET = "osf_mnv3_detection_opt.onnx"
|
||
|
||
BLENDS = (
|
||
"jawOpen", "mouthSmileLeft", "mouthSmileRight", "eyeWideLeft", "eyeWideRight",
|
||
"browInnerUp", "browDownLeft", "browDownRight",
|
||
)
|
||
FPS = 6
|
||
WINDOW_S = 30.0
|
||
CROP_SIDE = 384 # maior lado do recorte da câmera que vai para os modelos
|
||
|
||
|
||
# ---------------------------------------------------------------- vídeo
|
||
|
||
def ffmpeg_bin() -> str:
|
||
return "ffmpeg"
|
||
|
||
|
||
@dataclass
|
||
class Block:
|
||
path: Path
|
||
start: float
|
||
end: float
|
||
|
||
|
||
def load_blocks(session_dir: Path) -> list[Block]:
|
||
blocks = []
|
||
with open(session_dir / "segments.csv", newline="", encoding="utf-8") as fh:
|
||
for name, start, end in csv.reader(fh):
|
||
blocks.append(Block(session_dir / name, float(start), float(end)))
|
||
return blocks
|
||
|
||
|
||
def probe_size(path: Path) -> tuple[int, int]:
|
||
out = subprocess.run(
|
||
["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height",
|
||
"-of", "csv=p=0", str(path)],
|
||
capture_output=True, text=True, check=True,
|
||
).stdout.strip().splitlines()[0]
|
||
w, h = out.split(",")[:2]
|
||
return int(w), int(h)
|
||
|
||
|
||
def grab_frame(blocks: list[Block], t: float, size: tuple[int, int]) -> np.ndarray | None:
|
||
"""Um quadro BGR no tempo t (s da sessão)."""
|
||
block = next((b for b in blocks if b.start <= t < b.end), None)
|
||
if block is None:
|
||
return None
|
||
w, h = size
|
||
raw = subprocess.run(
|
||
[ffmpeg_bin(), "-v", "error", "-ss", f"{t - block.start:.3f}", "-i", str(block.path),
|
||
"-frames:v", "1", "-f", "rawvideo", "-pix_fmt", "bgr24", "-"],
|
||
capture_output=True, check=True,
|
||
).stdout
|
||
if len(raw) < w * h * 3:
|
||
return None
|
||
return np.frombuffer(raw[: w * h * 3], np.uint8).reshape(h, w, 3).copy()
|
||
|
||
|
||
def read_window(blocks: list[Block], t0: float, length: float, crop_px: tuple[int, int, int, int],
|
||
out_size: tuple[int, int]) -> list[tuple[float, np.ndarray]]:
|
||
"""Quadros RGB do recorte (x, y, w, h em px) a FPS qps, de t0 a t0+length, atravessando blocos."""
|
||
x, y, cw, ch = crop_px
|
||
ow, oh = out_size
|
||
frames: list[tuple[float, np.ndarray]] = []
|
||
t1 = t0 + length
|
||
for b in blocks:
|
||
a, z = max(t0, b.start), min(t1, b.end)
|
||
if z <= a:
|
||
continue
|
||
raw = subprocess.run(
|
||
[ffmpeg_bin(), "-v", "error", "-ss", f"{a - b.start:.3f}", "-t", f"{z - a:.3f}", "-i", str(b.path),
|
||
"-vf", f"fps={FPS},crop={cw}:{ch}:{x}:{y},scale={ow}:{oh}",
|
||
"-f", "rawvideo", "-pix_fmt", "rgb24", "-"],
|
||
capture_output=True, check=True,
|
||
).stdout
|
||
n = len(raw) // (ow * oh * 3)
|
||
arr = np.frombuffer(raw[: n * ow * oh * 3], np.uint8).reshape(n, oh, ow, 3)
|
||
for i in range(n):
|
||
frames.append((a + i / FPS, arr[i].copy()))
|
||
return frames
|
||
|
||
|
||
# ---------------------------------------------------------------- câmera (YuNet)
|
||
|
||
def make_yunet(name: str, size: tuple[int, int]):
|
||
return cv2.FaceDetectorYN.create(str(MODELS / name), "", size, 0.6, 0.3, 50)
|
||
|
||
|
||
def iou(a: dict, b: dict) -> float:
|
||
ax2, ay2, bx2, by2 = a["x"] + a["w"], a["y"] + a["h"], b["x"] + b["w"], b["y"] + b["h"]
|
||
iw = max(0.0, min(ax2, bx2) - max(a["x"], b["x"]))
|
||
ih = max(0.0, min(ay2, by2) - max(a["y"], b["y"]))
|
||
inter = iw * ih
|
||
union = a["w"] * a["h"] + b["w"] * b["h"] - inter
|
||
return inter / union if union > 0 else 0.0
|
||
|
||
|
||
def clamp_rect(r: dict) -> dict:
|
||
x, y = max(0.0, r["x"]), max(0.0, r["y"])
|
||
x2, y2 = min(1.0, r["x"] + r["w"]), min(1.0, r["y"] + r["h"])
|
||
return {"x": round(x, 4), "y": round(y, 4), "w": round(x2 - x, 4), "h": round(y2 - y, 4)}
|
||
|
||
|
||
def face_to_cam(face: dict) -> dict:
|
||
"""Retângulo da câmera estimado só pelo rosto (≈3,2× largura, 3,6× altura, rosto no terço de cima)."""
|
||
cx, cy = face["x"] + face["w"] / 2, face["y"] + face["h"] / 2
|
||
w, h = face["w"] * 3.2, face["h"] * 3.6
|
||
return clamp_rect({"x": cx - w / 2, "y": cy - h * 0.38, "w": w, "h": h})
|
||
|
||
|
||
def locate_cam(blocks, size, duration, ai_cam, out_dir: Path) -> dict:
|
||
fw, fh = size
|
||
times = [30 + i * (duration - 60) / 11 for i in range(12)]
|
||
det = make_yunet(YUNET, size)
|
||
# A 2023mar (entrada fixa) não está no models.lock.json; só entra na comparação se estiver na pasta.
|
||
det_old = make_yunet(YUNET_OLD, size) if (MODELS / YUNET_OLD).exists() else None
|
||
rows, thumbs, ms, ms_old = [], [], [], []
|
||
for t in times:
|
||
img = grab_frame(blocks, t, size)
|
||
if img is None:
|
||
continue
|
||
t_a = time.perf_counter()
|
||
_, faces = det.detect(img)
|
||
ms.append((time.perf_counter() - t_a) * 1000)
|
||
faces_old = faces
|
||
if det_old is not None:
|
||
t_a = time.perf_counter()
|
||
_, faces_old = det_old.detect(img)
|
||
ms_old.append((time.perf_counter() - t_a) * 1000)
|
||
found = []
|
||
for f in (faces if faces is not None else []):
|
||
found.append({"x": float(f[0]) / fw, "y": float(f[1]) / fh, "w": float(f[2]) / fw,
|
||
"h": float(f[3]) / fh, "score": round(float(f[14]), 3)})
|
||
rows.append({"t": round(t, 1), "faces": found,
|
||
"old_same": (faces_old is not None) == (faces is not None)})
|
||
thumbs.append((t, img, found))
|
||
|
||
# Agrupa rostos pela posição: entra no grupo de centroide mais próximo se estiver a menos de 1 largura
|
||
# (e 1 altura) de rosto dele. Comparar com o 1º membro (versão anterior) partia o streamer inclinado em 2.
|
||
clusters: list[list[dict]] = []
|
||
|
||
def centroid(cl):
|
||
return (statistics.mean(m["x"] + m["w"] / 2 for m in cl), statistics.mean(m["y"] + m["h"] / 2 for m in cl),
|
||
statistics.mean(m["w"] for m in cl), statistics.mean(m["h"] for m in cl))
|
||
|
||
for r in rows:
|
||
for f in r["faces"]:
|
||
c = (f["x"] + f["w"] / 2, f["y"] + f["h"] / 2)
|
||
best_cl, best_d = None, 1.0
|
||
for cl in clusters:
|
||
gx, gy, gw, gh = centroid(cl)
|
||
dx, dy = abs(c[0] - gx) / gw, abs(c[1] - gy) / gh
|
||
if dx < 1.0 and dy < 1.0 and max(dx, dy) < best_d:
|
||
best_cl, best_d = cl, max(dx, dy)
|
||
if best_cl is not None:
|
||
best_cl.append({**f, "t": r["t"]})
|
||
else:
|
||
clusters.append([{**f, "t": r["t"]}])
|
||
clusters.sort(key=lambda cl: len({m["t"] for m in cl}), reverse=True)
|
||
best = clusters[0] if clusters else []
|
||
result: dict = {
|
||
"frames": len(rows),
|
||
"framesWithFace": sum(1 for r in rows if r["faces"]),
|
||
"yunetMsPerFrame": round(statistics.mean(ms), 1) if ms else None,
|
||
"yunet2023marMsPerFrame": round(statistics.mean(ms_old), 1) if ms_old else None,
|
||
"variantsAgree": all(r["old_same"] for r in rows),
|
||
"clusters": [{"n": len({m["t"] for m in cl}),
|
||
"cx": round(statistics.median(m["x"] + m["w"] / 2 for m in cl), 3),
|
||
"cy": round(statistics.median(m["y"] + m["h"] / 2 for m in cl), 3)} for cl in clusters],
|
||
"rows": rows,
|
||
}
|
||
if best:
|
||
med = {k: statistics.median(m[k] for m in best) for k in ("x", "y", "w", "h")}
|
||
cxs = [m["x"] + m["w"] / 2 for m in best]
|
||
cys = [m["y"] + m["h"] / 2 for m in best]
|
||
cam = face_to_cam(med)
|
||
inside = clamp_rect({"x": max(med["x"], ai_cam["x"]), "y": max(med["y"], ai_cam["y"]),
|
||
"w": min(med["x"] + med["w"], ai_cam["x"] + ai_cam["w"]) - max(med["x"], ai_cam["x"]),
|
||
"h": min(med["y"] + med["h"], ai_cam["y"] + ai_cam["h"]) - max(med["y"], ai_cam["y"])})
|
||
result.update({
|
||
"presence": round(len({m["t"] for m in best}) / max(1, len(rows)), 3),
|
||
"face": {k: round(v, 4) for k, v in med.items()},
|
||
"facePx": round(med["h"] * fh),
|
||
"jitterCx": round(statistics.pstdev(cxs) / med["w"], 3),
|
||
"jitterCy": round(statistics.pstdev(cys) / med["h"], 3),
|
||
"cam": cam,
|
||
"iouWithAi": round(iou(cam, ai_cam), 3),
|
||
"faceInsideAi": round(max(0.0, inside["w"]) * max(0.0, inside["h"]) / (med["w"] * med["h"]), 3),
|
||
})
|
||
# Folha de contato: verde = YuNet, azul = câmera da IA, amarelo = câmera estimada pelo rosto.
|
||
tiles = []
|
||
for t, img, found in thumbs:
|
||
tile = img.copy()
|
||
for f in found:
|
||
cv2.rectangle(tile, (int(f["x"] * fw), int(f["y"] * fh)),
|
||
(int((f["x"] + f["w"]) * fw), int((f["y"] + f["h"]) * fh)), (0, 255, 0), 3)
|
||
for rect, color in ((ai_cam, (255, 120, 0)), (result.get("cam"), (0, 220, 255))):
|
||
if rect:
|
||
cv2.rectangle(tile, (int(rect["x"] * fw), int(rect["y"] * fh)),
|
||
(int((rect["x"] + rect["w"]) * fw), int((rect["y"] + rect["h"]) * fh)), color, 3)
|
||
tile = cv2.resize(tile, (480, 270))
|
||
cv2.putText(tile, f"t={t:.0f}s", (8, 24), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2)
|
||
tiles.append(tile)
|
||
save_grid(tiles, 4, out_dir / "cam_sheet.png")
|
||
return result
|
||
|
||
|
||
def save_grid(tiles: list[np.ndarray], cols: int, path: Path) -> None:
|
||
if not tiles:
|
||
return
|
||
h, w = tiles[0].shape[:2]
|
||
rows = (len(tiles) + cols - 1) // cols
|
||
sheet = np.zeros((rows * h, cols * w, 3), np.uint8)
|
||
for i, tile in enumerate(tiles):
|
||
r, c = divmod(i, cols)
|
||
sheet[r * h:(r + 1) * h, c * w:(c + 1) * w] = tile
|
||
cv2.imwrite(str(path), sheet)
|
||
|
||
|
||
# ---------------------------------------------------------------- MediaPipe
|
||
|
||
@dataclass
|
||
class Sample:
|
||
t: float
|
||
present: bool
|
||
ms: float
|
||
box: tuple[float, float, float, float] | None = None # x, y, w, h em px do recorte
|
||
points: np.ndarray | None = None
|
||
blend: dict = field(default_factory=dict)
|
||
|
||
|
||
def run_mediapipe(frames: list[tuple[float, np.ndarray]]) -> list[Sample]:
|
||
import mediapipe as mp
|
||
from mediapipe.tasks.python.core.base_options import BaseOptions
|
||
from mediapipe.tasks.python import vision
|
||
|
||
options = vision.FaceLandmarkerOptions(
|
||
base_options=BaseOptions(model_asset_path=str(MODELS / LANDMARKER)),
|
||
running_mode=vision.RunningMode.VIDEO,
|
||
num_faces=1,
|
||
min_face_detection_confidence=0.5,
|
||
min_face_presence_confidence=0.5,
|
||
min_tracking_confidence=0.5,
|
||
output_face_blendshapes=True,
|
||
)
|
||
out: list[Sample] = []
|
||
with vision.FaceLandmarker.create_from_options(options) as lm:
|
||
for t, rgb in frames:
|
||
h, w = rgb.shape[:2]
|
||
image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np.ascontiguousarray(rgb))
|
||
t_a = time.perf_counter()
|
||
res = lm.detect_for_video(image, int(round(t * 1000)))
|
||
ms = (time.perf_counter() - t_a) * 1000
|
||
if not res.face_landmarks:
|
||
out.append(Sample(t, False, ms))
|
||
continue
|
||
pts = np.array([(p.x * w, p.y * h) for p in res.face_landmarks[0]], np.float32)
|
||
x1, y1 = pts.min(0)
|
||
x2, y2 = pts.max(0)
|
||
blend = {c.category_name: float(c.score) for c in res.face_blendshapes[0] if c.category_name in BLENDS}
|
||
out.append(Sample(t, True, ms, (float(x1), float(y1), float(x2 - x1), float(y2 - y1)), pts, blend))
|
||
return out
|
||
|
||
|
||
# ---------------------------------------------------------------- OpenSeeFace (pré/pós mínimo do tracker.py)
|
||
|
||
class OsfTracker:
|
||
"""Reimplementação mínima de emilianavt/OpenSeeFace tracker.py (BSD-2), modelo 3, sem olhar/3D.
|
||
|
||
Detector mnv3 (224×224, saída 56×56 com raio) quando não há rosto; depois reaproveita a caixa dos
|
||
pontos do quadro anterior, como o original. Pontos no formato 66 do OpenSeeFace (≈ iBUG 68 sem 60/64).
|
||
"""
|
||
|
||
RES = 224
|
||
OUT_RES = 27
|
||
LOGIT = 16.0
|
||
|
||
def __init__(self, threads: int = 1, threshold: float = 0.6, det_threshold: float = 0.6):
|
||
import onnxruntime as ort
|
||
|
||
opts = ort.SessionOptions()
|
||
opts.inter_op_num_threads = 1
|
||
opts.intra_op_num_threads = threads
|
||
opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||
prov = ["CPUExecutionProvider"]
|
||
self.lm = ort.InferenceSession(str(MODELS / OSF_LM), sess_options=opts, providers=prov)
|
||
self.det = ort.InferenceSession(str(MODELS / OSF_DET), sess_options=opts, providers=prov)
|
||
self.lm_input = self.lm.get_inputs()[0].name
|
||
mean = np.float32([0.485, 0.456, 0.406])
|
||
std = np.float32([0.229, 0.224, 0.225])
|
||
self.mean = -(mean / std)
|
||
self.scale = 1.0 / (std * 255.0)
|
||
self.threshold = threshold
|
||
self.det_threshold = det_threshold
|
||
self.face: tuple[float, float, float, float] | None = None
|
||
self.misses = 0
|
||
|
||
def _norm(self, rgb: np.ndarray, size: int) -> np.ndarray:
|
||
im = cv2.resize(rgb.astype(np.float32), (size, size), interpolation=cv2.INTER_LINEAR)
|
||
im = im * self.scale + self.mean
|
||
return np.transpose(im[None], (0, 3, 1, 2)).astype(np.float32)
|
||
|
||
def detect(self, rgb: np.ndarray) -> tuple[float, float, float, float] | None:
|
||
out, maxpool = self.det.run([], {"input": self._norm(rgb, 224)})
|
||
out, maxpool = np.array(out), np.array(maxpool)
|
||
out[0, 0, out[0, 0] != maxpool[0, 0]] = 0
|
||
best = int(np.argmax(out[0, 0]))
|
||
y, x = divmod(best, 56)
|
||
if out[0, 0, y, x] < self.det_threshold:
|
||
return None
|
||
r = out[0, 1, y, x] * 112.0
|
||
x, y = x * 4, y * 4
|
||
sx, sy = rgb.shape[1] / 224.0, rgb.shape[0] / 224.0
|
||
return ((x - r) * sx, (y - r) * sy, 2 * r * sx, 2 * r * sy)
|
||
|
||
def landmarks(self, rgb: np.ndarray, box) -> tuple[float, np.ndarray]:
|
||
h, w = rgb.shape[:2]
|
||
x, y, bw, bh = box
|
||
x1 = int(np.clip(x - int(bw * 0.1), 0, w - 1))
|
||
y1 = int(np.clip(y - int(bh * 0.125), 0, h - 1))
|
||
x2 = int(np.clip(x + bw + int(bw * 0.1), 0, w - 1))
|
||
y2 = int(np.clip(y + bh + int(bh * 0.125), 0, h - 1))
|
||
if x2 - x1 < 4 or y2 - y1 < 4:
|
||
return 0.0, np.zeros((66, 3), np.float32)
|
||
sx, sy = (x2 - x1) / self.RES, (y2 - y1) / self.RES
|
||
tensor = self.lm.run([], {self.lm_input: self._norm(rgb[y1:y2, x1:x2], self.RES)})[0][0]
|
||
n = self.OUT_RES + 1
|
||
res = self.RES - 1
|
||
main = tensor[0:66].reshape(66, n * n)
|
||
idx = main.argmax(1)[:, None]
|
||
conf = np.take_along_axis(main, idx, 1)[:, 0]
|
||
off_y = np.take_along_axis(tensor[66:132].reshape(66, n * n), idx, 1)[:, 0]
|
||
off_x = np.take_along_axis(tensor[132:198].reshape(66, n * n), idx, 1)[:, 0]
|
||
|
||
def logit(p):
|
||
p = np.clip(p, 1e-7, 1 - 1e-7)
|
||
return np.log(p / (1 - p)) / self.LOGIT
|
||
|
||
m = idx[:, 0]
|
||
# No original a 1ª coluna é y e a 2ª é x; aqui já devolvemos (x, y, conf).
|
||
py = y1 + sy * (res * np.floor(m / n) / self.OUT_RES + res * logit(off_y))
|
||
px = x1 + sx * (res * np.floor(np.mod(m, n)) / self.OUT_RES + res * logit(off_x))
|
||
pts = np.stack([px, py, conf], 1).astype(np.float32)
|
||
pts[np.isnan(pts).any(axis=1)] = 0
|
||
return float(conf.mean()), pts
|
||
|
||
def step(self, rgb: np.ndarray) -> tuple[float, np.ndarray] | None:
|
||
box = self.face if self.face is not None else self.detect(rgb)
|
||
if box is None:
|
||
return None
|
||
conf, pts = self.landmarks(rgb, box)
|
||
if conf < self.threshold and self.face is not None:
|
||
# caixa antiga perdida: tenta o detector neste mesmo quadro
|
||
box = self.detect(rgb)
|
||
if box is None:
|
||
self.face = None
|
||
return None
|
||
conf, pts = self.landmarks(rgb, box)
|
||
if conf < self.threshold:
|
||
self.face = None
|
||
return None
|
||
x1, y1 = pts[:, :2].min(0)
|
||
x2, y2 = pts[:, :2].max(0)
|
||
self.face = (float(x1), float(y1), float(x2 - x1), float(y2 - y1))
|
||
return conf, pts
|
||
|
||
|
||
def run_osf(frames: list[tuple[float, np.ndarray]]) -> list[Sample]:
|
||
tr = OsfTracker()
|
||
out = []
|
||
for t, rgb in frames:
|
||
t_a = time.perf_counter()
|
||
got = tr.step(rgb)
|
||
ms = (time.perf_counter() - t_a) * 1000
|
||
if got is None:
|
||
out.append(Sample(t, False, ms))
|
||
continue
|
||
conf, pts = got
|
||
xy = pts[:, :2]
|
||
x1, y1 = xy.min(0)
|
||
x2, y2 = xy.max(0)
|
||
# boca aberta: lábios internos (60–62 em cima, 63–65 embaixo no layout de 66) / altura do rosto
|
||
mouth = float(abs(xy[[63, 64, 65], 1].mean() - xy[[60, 61, 62], 1].mean()) / max(1.0, y2 - y1))
|
||
out.append(Sample(t, True, ms, (float(x1), float(y1), float(x2 - x1), float(y2 - y1)), xy,
|
||
{"mouthOpen": mouth, "conf": conf}))
|
||
return out
|
||
|
||
|
||
# ---------------------------------------------------------------- métricas
|
||
|
||
def summarize(samples: list[Sample]) -> dict:
|
||
ms = sorted(s.ms for s in samples)
|
||
present = [s for s in samples if s.present]
|
||
out = {
|
||
"frames": len(samples),
|
||
"presentPct": round(100 * len(present) / max(1, len(samples)), 1),
|
||
"msMean": round(statistics.mean(ms), 2) if ms else None,
|
||
"msP95": round(ms[int(0.95 * (len(ms) - 1))], 2) if ms else None,
|
||
"facePxMedian": round(statistics.median(s.box[3] for s in present), 1) if present else None,
|
||
}
|
||
return out
|
||
|
||
|
||
def per_second(samples: list[Sample], keys) -> list[dict]:
|
||
by: dict[int, list[Sample]] = {}
|
||
for s in samples:
|
||
by.setdefault(int(s.t), []).append(s)
|
||
rows = []
|
||
for sec in sorted(by):
|
||
ss = [s for s in by[sec] if s.present]
|
||
row = {"t": sec, "present": round(len(ss) / len(by[sec]), 2)}
|
||
for k in keys:
|
||
vals = [s.blend.get(k, 0.0) for s in ss]
|
||
row[k] = round(max(vals), 3) if vals else None
|
||
rows.append(row)
|
||
return rows
|
||
|
||
|
||
def blend_stats(samples: list[Sample], keys, t_from=None, t_to=None) -> dict:
|
||
ss = [s for s in samples if s.present and (t_from is None or t_from <= s.t <= t_to)]
|
||
out = {}
|
||
for k in keys:
|
||
vals = [s.blend.get(k, 0.0) for s in ss]
|
||
out[k] = {"mean": round(statistics.mean(vals), 3), "p90": round(float(np.percentile(vals, 90)), 3),
|
||
"max": round(max(vals), 3)} if vals else None
|
||
return out
|
||
|
||
|
||
def draw_sheet(frames, mp_samples, osf_samples, path: Path, n: int = 10) -> None:
|
||
idxs = np.linspace(0, len(frames) - 1, n).astype(int)
|
||
tiles = []
|
||
for i in idxs:
|
||
t, rgb = frames[i]
|
||
img = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)
|
||
m, o = mp_samples[i], osf_samples[i]
|
||
if m.present:
|
||
for p in m.points[::3]:
|
||
cv2.circle(img, (int(p[0]), int(p[1])), 1, (0, 255, 0), -1)
|
||
x, y, w, h = m.box
|
||
cv2.rectangle(img, (int(x), int(y)), (int(x + w), int(y + h)), (0, 255, 0), 1)
|
||
if o.present:
|
||
for p in o.points:
|
||
cv2.circle(img, (int(p[0]), int(p[1])), 2, (0, 0, 255), -1)
|
||
label = f"t={t:.1f} jaw={m.blend.get('jawOpen', 0):.2f} sm={max(m.blend.get('mouthSmileLeft', 0), m.blend.get('mouthSmileRight', 0)):.2f}"
|
||
cv2.putText(img, label, (4, 14), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1)
|
||
tiles.append(img)
|
||
save_grid(tiles, 5, path)
|
||
|
||
|
||
# ---------------------------------------------------------------- principal
|
||
|
||
def main() -> None:
|
||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
ap.add_argument("--session", type=int, default=3)
|
||
ap.add_argument("--out", default=str(ROOT / "output" / "face-bench"))
|
||
args = ap.parse_args()
|
||
out_dir = Path(args.out)
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
con = sqlite3.connect(f"file:{ROOT / 'data' / 'clipper' / 'clipper.db'}?mode=ro", uri=True)
|
||
cam_json = json.loads(con.execute("select cam_json from sessions where id=?", (args.session,)).fetchone()[0])
|
||
ai_cam = cam_json["cam"]
|
||
clips = con.execute("select id, peak_s, start_s, end_s, title from clips where session_id=? order by peak_s",
|
||
(args.session,)).fetchall()
|
||
loud = {r[0]: json.loads(r[1]) for r in con.execute(
|
||
"select idx, loudness_json from segments where session_id=? and loudness_json is not null", (args.session,))}
|
||
con.close()
|
||
|
||
session_dir = ROOT / "data" / "clipper" / "sessions" / str(args.session)
|
||
blocks = load_blocks(session_dir)
|
||
duration = blocks[-1].end
|
||
size = probe_size(blocks[0].path)
|
||
print(f"sessão {args.session}: {len(blocks)} blocos, {duration:.0f} s, {size[0]}x{size[1]}")
|
||
|
||
report: dict = {"session": args.session, "frameSize": size, "aiCam": ai_cam,
|
||
"clips": [{"id": c[0], "peak": c[1], "start": c[2], "end": c[3]} for c in clips]}
|
||
cam = locate_cam(blocks, size, duration, ai_cam, out_dir)
|
||
report["camera"] = cam
|
||
print("câmera:", {k: v for k, v in cam.items() if k != "rows"})
|
||
|
||
# Trecho calmo: bloco com menor volume (p90) a ≥ 120 s de qualquer corte, longe das pontas.
|
||
peaks = [c[1] for c in clips]
|
||
calm_idx = min((i for i in loud if 2 <= i < len(blocks) - 2
|
||
and all(abs(blocks[i].start + 15 - p) >= 120 for p in peaks)),
|
||
key=lambda i: float(np.percentile(loud[i], 90)))
|
||
windows = [("calmo", blocks[calm_idx].start, None)]
|
||
for cid, peak, start, end, _title in clips:
|
||
if peak + 10 > duration: # corte novo além dos blocos lidos no início (o serviço continua gravando)
|
||
continue
|
||
windows.append((f"corte{cid}", max(0.0, peak - 20), peak))
|
||
|
||
# Recorte: câmera local (+10 % de margem); comparação extra com o retângulo da IA.
|
||
fw, fh = size
|
||
crops = {"local": cam.get("cam") or ai_cam, "ai": ai_cam}
|
||
report["windows"] = []
|
||
for name, t0, peak in windows:
|
||
for crop_name, rect in crops.items():
|
||
mx, my = rect["w"] * 0.1, rect["h"] * 0.1
|
||
r = clamp_rect({"x": rect["x"] - mx, "y": rect["y"] - my, "w": rect["w"] + 2 * mx, "h": rect["h"] + 2 * my})
|
||
px = (int(r["x"] * fw) // 2 * 2, int(r["y"] * fh) // 2 * 2, int(r["w"] * fw) // 2 * 2, int(r["h"] * fh) // 2 * 2)
|
||
k = CROP_SIDE / max(px[2], px[3])
|
||
out_size = (int(px[2] * k) // 2 * 2, int(px[3] * k) // 2 * 2)
|
||
t_a = time.perf_counter()
|
||
frames = read_window(blocks, t0, WINDOW_S, px, out_size)
|
||
decode_s = time.perf_counter() - t_a
|
||
mp_s = run_mediapipe(frames)
|
||
entry = {"window": name, "crop": crop_name, "t0": t0, "peak": peak, "cropPx": px, "inputSize": out_size,
|
||
"decodeS": round(decode_s, 2), "mediapipe": summarize(mp_s)}
|
||
if crop_name == "local":
|
||
osf_s = run_osf(frames)
|
||
entry["openseeface"] = summarize(osf_s)
|
||
entry["mpBlend"] = blend_stats(mp_s, BLENDS)
|
||
entry["osfMouth"] = blend_stats(osf_s, ("mouthOpen",))
|
||
if peak is not None:
|
||
entry["mpBlendNearPeak"] = blend_stats(mp_s, BLENDS, peak - 4, peak + 4)
|
||
entry["osfMouthNearPeak"] = blend_stats(osf_s, ("mouthOpen",), peak - 4, peak + 4)
|
||
entry["mpPerSecond"] = per_second(mp_s, BLENDS)
|
||
entry["osfPerSecond"] = per_second(osf_s, ("mouthOpen",))
|
||
draw_sheet(frames, mp_s, osf_s, out_dir / f"track_{name}.png")
|
||
report["windows"].append(entry)
|
||
print(name, crop_name, "MP", entry["mediapipe"], "OSF", entry.get("openseeface"))
|
||
|
||
(out_dir / "results.json").write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
print("resultado:", out_dir / "results.json")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|