diff options
Diffstat (limited to 'poc_eval_notebook.ipynb')
| -rw-r--r-- | poc_eval_notebook.ipynb | 436 |
1 files changed, 381 insertions, 55 deletions
diff --git a/poc_eval_notebook.ipynb b/poc_eval_notebook.ipynb index 55f337f..5865f7e 100644 --- a/poc_eval_notebook.ipynb +++ b/poc_eval_notebook.ipynb @@ -49,7 +49,7 @@ "import matplotlib.patches as patches\n", "from PIL import Image\n", "\n", - "from transformers import AutoModel, AutoProcessor\n", + "from transformers import AutoImageProcessor, AutoModel, AutoProcessor\n", "\n", "from huggingface_hub import whoami\n", "from sam3.model_builder import build_sam3_image_model, download_ckpt_from_hf\n", @@ -62,6 +62,18 @@ "SAM3_POINT_HINT_BOX_RADIUS_PX = 12\n", "SAM3_MAX_POINT_HINTS = 4\n", "SAM3_TORCH_DTYPE = torch.bfloat16 if DEVICE == \"cuda\" else torch.float32\n", + "DINO_MODEL_ID = \"facebook/dinov3-vitb16-pretrain-lvd1689m\"\n", + "SIGCLIP_FUSION_WEIGHT = 0.60\n", + "DINO_FUSION_WEIGHT = 0.40\n", + "DINO_SIM_TEMPERATURE = 0.10\n", + "LOW_CONF_THRESHOLD = 0.50\n", + "MIXED_MARGIN_THRESHOLD = 0.15\n", + "MAX_MIXED_ENTRIES = 3\n", + "MIN_MIXED_PERCENT = 10\n", + "MATERIAL_PROTOTYPE_IMAGE_PATHS = {\n", + " # \"wood\": [\"/content/prototypes/wood_01.jpg\", \"/content/prototypes/wood_02.jpg\"],\n", + " # \"steel\": [\"/content/prototypes/steel_01.jpg\"],\n", + "}\n", "SIGCLIP_MODEL_ID = \"google/siglip-base-patch16-224\"\n", "\n", "CSV_PATH = Path(\"/content/poc_eval_sheet.csv\")\n", @@ -918,10 +930,244 @@ " \"pred_material_conf_top1\": float(top_scores[0]),\n", " \"topk_labels\": top_labels,\n", " \"topk_scores\": [float(s) for s in top_scores],\n", + " \"per_material_probs\": {k: float(v) for k, v in agg_scores.items()},\n", " \"candidate_materials\": candidate_materials,\n", " }\n", "\n", "\n", + "def l2_normalize_np(vec: np.ndarray, eps: float = 1e-12):\n", + " n = float(np.linalg.norm(vec))\n", + " if n < eps:\n", + " return vec\n", + " return vec / n\n", + "\n", + "\n", + "def extract_dinov3_embedding(image: Image.Image, dino_model, dino_processor):\n", + " inputs = dino_processor(images=image, return_tensors=\"pt\")\n", + " inputs = {k: v.to(DEVICE) if hasattr(v, \"to\") else v for k, v in inputs.items()}\n", + " with torch.inference_mode():\n", + " outputs = dino_model(**inputs)\n", + " emb = outputs.pooler_output[0].detach().float().cpu().numpy()\n", + " return l2_normalize_np(emb)\n", + "\n", + "\n", + "def build_dino_prototype_bank(material_image_paths: dict, dino_model, dino_processor):\n", + " bank = {}\n", + " missing = []\n", + " for material in ALL_CANONICAL_MATERIALS:\n", + " paths = material_image_paths.get(material, [])\n", + " embs = []\n", + " for p in paths:\n", + " if not Path(p).exists():\n", + " missing.append(p)\n", + " continue\n", + " img = load_image(str(p))\n", + " embs.append(extract_dinov3_embedding(img, dino_model, dino_processor))\n", + " if embs:\n", + " bank[material] = l2_normalize_np(np.mean(np.stack(embs, axis=0), axis=0))\n", + " return bank, missing\n", + "\n", + "\n", + "def normalize_prob_dict(score_dict: dict, keys: list):\n", + " vals = np.array([float(score_dict.get(k, 0.0)) for k in keys], dtype=np.float64)\n", + " vals = np.clip(vals, a_min=0.0, a_max=None)\n", + " s = float(vals.sum())\n", + " if s <= 0:\n", + " vals = np.ones_like(vals) / max(len(vals), 1)\n", + " else:\n", + " vals = vals / s\n", + " return {k: float(v) for k, v in zip(keys, vals)}\n", + "\n", + "\n", + "def classify_material_dinov3(\n", + " image_views: list,\n", + " object_label: str | None,\n", + " dino_model,\n", + " dino_processor,\n", + " prototype_bank: dict,\n", + " top_k: int = 3,\n", + " view_weights: list | None = None,\n", + "):\n", + " candidate_materials = resolve_candidate_materials(object_label)\n", + "\n", + " if view_weights is None:\n", + " view_weights = [1.0 / len(image_views)] * len(image_views)\n", + " if len(view_weights) != len(image_views):\n", + " raise ValueError(\"view_weights length must match image_views length\")\n", + "\n", + " proto_materials = [m for m in candidate_materials if m in prototype_bank]\n", + " prototype_coverage = float(len(proto_materials) / max(len(candidate_materials), 1))\n", + "\n", + " if len(proto_materials) < 2:\n", + " probs = {m: 1.0 / len(candidate_materials) for m in candidate_materials}\n", + " ranked = sorted(probs.items(), key=lambda kv: kv[1], reverse=True)\n", + " top_items = ranked[: min(top_k, len(ranked))]\n", + " return {\n", + " \"pred_material_top1\": top_items[0][0],\n", + " \"pred_material_top3\": \"|\".join([x[0] for x in top_items]),\n", + " \"pred_material_conf_top1\": float(top_items[0][1]),\n", + " \"topk_labels\": [x[0] for x in top_items],\n", + " \"topk_scores\": [float(x[1]) for x in top_items],\n", + " \"per_material_probs\": probs,\n", + " \"candidate_materials\": candidate_materials,\n", + " \"prototype_coverage\": prototype_coverage,\n", + " \"prototypes_available\": False,\n", + " }\n", + "\n", + " agg = {m: 0.0 for m in candidate_materials}\n", + " for image, weight in zip(image_views, view_weights):\n", + " emb = extract_dinov3_embedding(image, dino_model, dino_processor)\n", + " sims = []\n", + " for m in candidate_materials:\n", + " if m in prototype_bank:\n", + " sims.append(float(np.dot(emb, prototype_bank[m])))\n", + " else:\n", + " sims.append(-1e3)\n", + " sims = np.array(sims, dtype=np.float64)\n", + " probs = np.exp((sims - np.max(sims)) / max(DINO_SIM_TEMPERATURE, 1e-6))\n", + " probs = probs / max(probs.sum(), 1e-12)\n", + " for m, p in zip(candidate_materials, probs.tolist()):\n", + " agg[m] += float(weight) * float(p)\n", + "\n", + " agg = normalize_prob_dict(agg, candidate_materials)\n", + " ranked = sorted(agg.items(), key=lambda kv: kv[1], reverse=True)\n", + " top_items = ranked[: min(top_k, len(ranked))]\n", + "\n", + " return {\n", + " \"pred_material_top1\": top_items[0][0],\n", + " \"pred_material_top3\": \"|\".join([x[0] for x in top_items]),\n", + " \"pred_material_conf_top1\": float(top_items[0][1]),\n", + " \"topk_labels\": [x[0] for x in top_items],\n", + " \"topk_scores\": [float(x[1]) for x in top_items],\n", + " \"per_material_probs\": agg,\n", + " \"candidate_materials\": candidate_materials,\n", + " \"prototype_coverage\": prototype_coverage,\n", + " \"prototypes_available\": True,\n", + " }\n", + "\n", + "\n", + "def fuse_material_predictions(sigclip_out: dict, dino_out: dict):\n", + " mats = sorted(set(sigclip_out[\"candidate_materials\"]) | set(dino_out[\"candidate_materials\"]))\n", + " p_sig = normalize_prob_dict(sigclip_out.get(\"per_material_probs\", {}), mats)\n", + " p_dino = normalize_prob_dict(dino_out.get(\"per_material_probs\", {}), mats)\n", + "\n", + " w_sig = float(SIGCLIP_FUSION_WEIGHT)\n", + " w_dino = float(DINO_FUSION_WEIGHT) if dino_out.get(\"prototypes_available\", False) else 0.0\n", + " if w_sig + w_dino <= 0:\n", + " w_sig = 1.0\n", + " w_sum = w_sig + w_dino\n", + " w_sig, w_dino = w_sig / w_sum, w_dino / w_sum\n", + "\n", + " fused = {m: w_sig * p_sig[m] + w_dino * p_dino[m] for m in mats}\n", + " fused = normalize_prob_dict(fused, mats)\n", + "\n", + " ranked = sorted(fused.items(), key=lambda kv: kv[1], reverse=True)\n", + " top3 = ranked[: min(3, len(ranked))]\n", + "\n", + " sig_top1 = max(p_sig.items(), key=lambda kv: kv[1])[0]\n", + " dino_top1 = max(p_dino.items(), key=lambda kv: kv[1])[0]\n", + "\n", + " return {\n", + " \"pred_material_top1\": ranked[0][0],\n", + " \"pred_material_top3\": \"|\".join([x[0] for x in top3]),\n", + " \"pred_material_conf_top1\": float(ranked[0][1]),\n", + " \"topk_labels\": [x[0] for x in top3],\n", + " \"topk_scores\": [float(x[1]) for x in top3],\n", + " \"per_material_probs\": fused,\n", + " \"candidate_materials\": mats,\n", + " \"weights\": {\"sigclip\": w_sig, \"dinov3\": w_dino},\n", + " \"disagreement\": {\n", + " \"top1_match\": bool(sig_top1 == dino_top1),\n", + " \"sigclip_top1\": sig_top1,\n", + " \"dinov3_top1\": dino_top1,\n", + " },\n", + " }\n", + "\n", + "\n", + "def build_mixed_materials(prob_dict: dict):\n", + " ranked = sorted(prob_dict.items(), key=lambda kv: kv[1], reverse=True)\n", + " selected = ranked[: max(2, MAX_MIXED_ENTRIES)]\n", + " vals = np.array([max(0.0, float(v)) for _, v in selected], dtype=np.float64)\n", + " vals = vals / max(vals.sum(), 1e-12)\n", + "\n", + " pairs = []\n", + " for (m, _), p in zip(selected, vals.tolist()):\n", + " pairs.append({\"material\": m, \"percentage\": int(round(100 * p))})\n", + "\n", + " pairs = [x for x in pairs if x[\"percentage\"] >= MIN_MIXED_PERCENT]\n", + " if len(pairs) < 2:\n", + " return []\n", + "\n", + " total = sum(x[\"percentage\"] for x in pairs)\n", + " if total <= 0:\n", + " return []\n", + " for x in pairs:\n", + " x[\"percentage\"] = int(round(100 * x[\"percentage\"] / total))\n", + " diff = 100 - sum(x[\"percentage\"] for x in pairs)\n", + " pairs[-1][\"percentage\"] += diff\n", + "\n", + " pairs = sorted(pairs, key=lambda x: x[\"percentage\"], reverse=True)[:MAX_MIXED_ENTRIES]\n", + " total2 = sum(x[\"percentage\"] for x in pairs)\n", + " if total2 != 100:\n", + " pairs[-1][\"percentage\"] += 100 - total2\n", + " return pairs\n", + "\n", + "\n", + "def suggest_material_decision(fused_out: dict):\n", + " probs = fused_out[\"per_material_probs\"]\n", + " ranked = sorted(probs.items(), key=lambda kv: kv[1], reverse=True)\n", + " top1_mat, top1_conf = ranked[0]\n", + " top2_conf = ranked[1][1] if len(ranked) > 1 else 0.0\n", + " margin12 = float(top1_conf - top2_conf)\n", + "\n", + " low_conf = float(top1_conf) < float(LOW_CONF_THRESHOLD)\n", + " mixed_suggestion = build_mixed_materials(probs)\n", + "\n", + " if low_conf:\n", + " suggested_mode = \"unknown\"\n", + " elif margin12 < float(MIXED_MARGIN_THRESHOLD) and len(mixed_suggestion) >= 2:\n", + " suggested_mode = \"mixed\"\n", + " else:\n", + " suggested_mode = \"single\"\n", + "\n", + " return {\n", + " \"suggested_mode\": suggested_mode,\n", + " \"suggested_single_material\": top1_mat if suggested_mode == \"single\" else None,\n", + " \"suggested_mixed_materials\": mixed_suggestion if suggested_mode == \"mixed\" else [],\n", + " \"low_confidence_flag\": bool(low_conf),\n", + " \"requires_user_confirmation\": bool(low_conf),\n", + " \"margin12\": margin12,\n", + " }\n", + "\n", + "\n", + "def validate_mixed_materials(mixed_materials):\n", + " if not isinstance(mixed_materials, list):\n", + " return False, \"mixed_materials must be a list\"\n", + " if len(mixed_materials) < 2 or len(mixed_materials) > MAX_MIXED_ENTRIES:\n", + " return False, f\"mixed_materials must have 2..{MAX_MIXED_ENTRIES} entries\"\n", + "\n", + " total = 0\n", + " for item in mixed_materials:\n", + " if not isinstance(item, dict) or \"material\" not in item or \"percentage\" not in item:\n", + " return False, \"each mixed entry must be {material, percentage}\"\n", + " pct = int(item[\"percentage\"])\n", + " if pct < MIN_MIXED_PERCENT:\n", + " return False, f\"each mixed percentage must be >= {MIN_MIXED_PERCENT}\"\n", + " total += pct\n", + " if total != 100:\n", + " return False, \"mixed percentages must sum to 100\"\n", + " return True, \"\"\n", + "\n", + "\n", + "def encode_final_material_for_csv(final_mode, final_single, final_mixed):\n", + " if final_mode == \"single\":\n", + " return str(final_single)\n", + " if final_mode == \"mixed\":\n", + " parts = [f\"{x['material']}={x['percentage']}\" for x in final_mixed]\n", + " return \"mixed:\" + \"|\".join(parts)\n", + " return \"unknown\"\n", + "\n", + "\n", "def save_passport(passport: dict, out_dir: Path) -> Path:\n", " out_dir.mkdir(parents=True, exist_ok=True)\n", " sample_id = passport[\"sample_id\"]\n", @@ -1000,6 +1246,23 @@ "sigclip_model = AutoModel.from_pretrained(SIGCLIP_MODEL_ID).to(DEVICE)\n", "sigclip_model.eval()\n", "\n", + "print(\"Loading DINOv3 model...\")\n", + "dino_processor = AutoImageProcessor.from_pretrained(DINO_MODEL_ID)\n", + "dino_model = AutoModel.from_pretrained(DINO_MODEL_ID).to(DEVICE)\n", + "dino_model.eval()\n", + "\n", + "print(\"Building DINO prototype bank...\")\n", + "dino_prototype_bank, missing_proto_paths = build_dino_prototype_bank(\n", + " MATERIAL_PROTOTYPE_IMAGE_PATHS,\n", + " dino_model=dino_model,\n", + " dino_processor=dino_processor,\n", + ")\n", + "print(f\"DINO prototypes loaded: {len(dino_prototype_bank)} materials\")\n", + "if missing_proto_paths:\n", + " print(\"Missing prototype files:\")\n", + " for p in missing_proto_paths:\n", + " print(f\"- {p}\")\n", + "\n", "print(\"Models ready.\")" ] }, @@ -1167,7 +1430,7 @@ " crop_bbox = crop_bbox_rgb(img, seg[\"bbox_xywh\"], pad=16)\n", "\n", " clf_start = pd.Timestamp.utcnow()\n", - " clf = classify_material_sigclip(\n", + " clf_sig = classify_material_sigclip(\n", " image_views=[crop_masked, crop_bbox, img],\n", " object_label=pred_object,\n", " sigclip_model=sigclip_model,\n", @@ -1175,6 +1438,29 @@ " top_k=3,\n", " view_weights=[0.5, 0.35, 0.15],\n", " )\n", + " clf_dino = classify_material_dinov3(\n", + " image_views=[crop_masked, crop_bbox],\n", + " object_label=pred_object,\n", + " dino_model=dino_model,\n", + " dino_processor=dino_processor,\n", + " prototype_bank=dino_prototype_bank,\n", + " top_k=3,\n", + " view_weights=[0.65, 0.35],\n", + " )\n", + " clf_fused = fuse_material_predictions(clf_sig, clf_dino)\n", + " decision = suggest_material_decision(clf_fused)\n", + "\n", + " clf = {\n", + " **clf_fused,\n", + " \"sigclip\": clf_sig,\n", + " \"dinov3\": clf_dino,\n", + " \"suggested_mode\": decision[\"suggested_mode\"],\n", + " \"suggested_single_material\": decision[\"suggested_single_material\"],\n", + " \"suggested_mixed_materials\": decision[\"suggested_mixed_materials\"],\n", + " \"low_confidence_flag\": decision[\"low_confidence_flag\"],\n", + " \"requires_user_confirmation\": decision[\"requires_user_confirmation\"],\n", + " \"margin12\": decision[\"margin12\"],\n", + " }\n", " clf_end = pd.Timestamp.utcnow()\n", "\n", " t1 = pd.Timestamp.utcnow()\n", @@ -1209,7 +1495,8 @@ " f\"sam={seg['sam_score']:.3f} sel={seg['selection_score']:.3f} \"\n", " f\"hint_iou={seg['selection_metrics']['hint_iou']:.3f} \"\n", " f\"pt_cov={seg['selection_metrics']['point_coverage']:.3f} \"\n", - " f\"top3={clf['pred_material_top3']}\"\n", + " f\"top3={clf['pred_material_top3']} mode={clf['suggested_mode']} \"\n", + " f\"w=({clf['weights']['sigclip']:.2f},{clf['weights']['dinov3']:.2f})\"\n", " )\n", "\n", " fig, axes = plt.subplots(1, 2, figsize=(12, 5))\n", @@ -1248,10 +1535,12 @@ "# Example override format:\n", "# USER_REVIEW_OVERRIDES = {\n", "# \"S003\": {\n", - "# \"segmentation_accepted\": 0,\n", - "# \"user_edited_material\": 1,\n", - "# \"user_final_material\": \"aluminum\",\n", - "# \"edit_reason\": \"frame dominates crop\",\n", + "# \"segmentation_accepted\": 1,\n", + "# \"final_material_mode\": \"mixed\", # single|mixed|unknown\n", + "# \"final_single_material\": None,\n", + "# \"final_mixed_materials\": [{\"material\": \"wood\", \"percentage\": 60}, {\"material\": \"steel\", \"percentage\": 40}],\n", + "# \"confirmed_by_user\": True,\n", + "# \"edit_reason\": \"edge case\",\n", "# \"status\": \"ok\",\n", "# \"error_type\": \"\",\n", "# }\n", @@ -1260,16 +1549,35 @@ "\n", "for run in RUN_ROWS:\n", " sid = run[\"sample_id\"]\n", - " pred_top1 = run[\"clf\"][\"pred_material_top1\"]\n", + " clf = run[\"clf\"]\n", " override = USER_REVIEW_OVERRIDES.get(sid, {})\n", "\n", - " user_edited_material = int(override.get(\"user_edited_material\", 0))\n", - " user_final_material = override.get(\"user_final_material\", pred_top1)\n", + " suggested_mode = clf[\"suggested_mode\"]\n", + " suggested_single = clf.get(\"suggested_single_material\")\n", + " suggested_mixed = clf.get(\"suggested_mixed_materials\", [])\n", + "\n", + " final_mode = override.get(\"final_material_mode\", suggested_mode)\n", + " final_single = override.get(\"final_single_material\", suggested_single if final_mode == \"single\" else None)\n", + " final_mixed = override.get(\"final_mixed_materials\", suggested_mixed if final_mode == \"mixed\" else [])\n", + "\n", + " requires_confirm = bool(clf.get(\"requires_user_confirmation\", False))\n", + " confirmed_by_user = bool(override.get(\"confirmed_by_user\", not requires_confirm))\n", + "\n", + " if final_mode == \"mixed\":\n", + " ok, err = validate_mixed_materials(final_mixed)\n", + " if not ok:\n", + " raise ValueError(f\"{sid}: invalid final_mixed_materials -> {err}\")\n", + "\n", + " user_edited = int(final_mode != suggested_mode)\n", "\n", " run[\"review\"] = {\n", " \"segmentation_accepted\": int(override.get(\"segmentation_accepted\", DEFAULT_SEGMENTATION_ACCEPTED)),\n", - " \"user_edited_material\": user_edited_material,\n", - " \"user_final_material\": user_final_material,\n", + " \"final_material_mode\": final_mode,\n", + " \"final_single_material\": final_single,\n", + " \"final_mixed_materials\": final_mixed,\n", + " \"requires_user_confirmation\": requires_confirm,\n", + " \"confirmed_by_user\": confirmed_by_user,\n", + " \"user_edited_material\": user_edited,\n", " \"edit_reason\": override.get(\"edit_reason\", \"\"),\n", " \"status\": override.get(\"status\", DEFAULT_STATUS),\n", " \"error_type\": override.get(\"error_type\", DEFAULT_ERROR_TYPE),\n", @@ -1278,10 +1586,11 @@ "pd.DataFrame([\n", " {\n", " \"sample_id\": r[\"sample_id\"],\n", - " \"pred_material_top1\": r[\"clf\"][\"pred_material_top1\"],\n", - " \"user_final_material\": r[\"review\"][\"user_final_material\"],\n", + " \"suggested_mode\": r[\"clf\"][\"suggested_mode\"],\n", + " \"final_mode\": r[\"review\"][\"final_material_mode\"],\n", + " \"requires_confirm\": r[\"review\"][\"requires_user_confirmation\"],\n", + " \"confirmed\": r[\"review\"][\"confirmed_by_user\"],\n", " \"user_edited_material\": r[\"review\"][\"user_edited_material\"],\n", - " \"segmentation_accepted\": r[\"review\"][\"segmentation_accepted\"],\n", " \"status\": r[\"review\"][\"status\"],\n", " }\n", " for r in RUN_ROWS\n", @@ -1300,6 +1609,26 @@ " clf = run[\"clf\"]\n", " timing = run[\"timing\"]\n", "\n", + " if review.get(\"requires_user_confirmation\", False) and not review.get(\"confirmed_by_user\", False):\n", + " raise ValueError(f\"{run['sample_id']}: low-confidence result requires explicit user confirmation before save\")\n", + "\n", + " final_mode = review.get(\"final_material_mode\", \"unknown\")\n", + " final_single = review.get(\"final_single_material\", None)\n", + " final_mixed = review.get(\"final_mixed_materials\", [])\n", + "\n", + " if final_mode == \"mixed\":\n", + " ok, err = validate_mixed_materials(final_mixed)\n", + " if not ok:\n", + " raise ValueError(f\"{run['sample_id']}: invalid mixed output -> {err}\")\n", + " elif final_mode == \"single\":\n", + " if not final_single:\n", + " raise ValueError(f\"{run['sample_id']}: final_single_material is required for single mode\")\n", + " elif final_mode == \"unknown\":\n", + " final_single = None\n", + " final_mixed = []\n", + " else:\n", + " raise ValueError(f\"{run['sample_id']}: unknown final_material_mode={final_mode}\")\n", + "\n", " passport = {\n", " \"sample_id\": run[\"sample_id\"],\n", " \"image_id\": run[\"image_id\"],\n", @@ -1316,12 +1645,34 @@ " },\n", " \"material\": {\n", " \"ground_truth\": run[\"ground_truth_material\"],\n", - " \"pred_top1\": clf[\"pred_material_top1\"],\n", - " \"pred_top3\": clf[\"pred_material_top3\"].split(\"|\"),\n", - " \"pred_conf_top1\": clf[\"pred_material_conf_top1\"],\n", - " \"candidate_materials\": clf[\"candidate_materials\"],\n", - " \"topk_scores\": clf[\"topk_scores\"],\n", - " \"user_final\": review.get(\"user_final_material\", clf[\"pred_material_top1\"]),\n", + " \"predicted_topk\": [\n", + " {\"material\": m, \"score\": float(s)}\n", + " for m, s in zip(clf[\"topk_labels\"], clf[\"topk_scores\"])\n", + " ],\n", + " \"predicted_confidence_top1\": clf[\"pred_material_conf_top1\"],\n", + " \"suggested_mode\": clf[\"suggested_mode\"],\n", + " \"suggested_single_material\": clf.get(\"suggested_single_material\"),\n", + " \"suggested_mixed_materials\": clf.get(\"suggested_mixed_materials\", []),\n", + " \"final_material_mode\": final_mode,\n", + " \"final_single_material\": final_single,\n", + " \"final_mixed_materials\": final_mixed,\n", + " \"low_confidence_flag\": bool(clf.get(\"low_confidence_flag\", False)),\n", + " \"requires_user_confirmation\": bool(review.get(\"requires_user_confirmation\", False)),\n", + " \"confirmed_by_user\": bool(review.get(\"confirmed_by_user\", False)),\n", + " \"fusion_weights\": clf.get(\"weights\", {}),\n", + " \"model_outputs\": {\n", + " \"sigclip\": {\n", + " \"topk_labels\": clf[\"sigclip\"].get(\"topk_labels\", []),\n", + " \"topk_scores\": clf[\"sigclip\"].get(\"topk_scores\", []),\n", + " },\n", + " \"dinov3\": {\n", + " \"topk_labels\": clf[\"dinov3\"].get(\"topk_labels\", []),\n", + " \"topk_scores\": clf[\"dinov3\"].get(\"topk_scores\", []),\n", + " \"prototype_coverage\": clf[\"dinov3\"].get(\"prototype_coverage\", 0.0),\n", + " \"prototypes_available\": bool(clf[\"dinov3\"].get(\"prototypes_available\", False)),\n", + " },\n", + " },\n", + " \"user_final\": encode_final_material_for_csv(final_mode, final_single, final_mixed),\n", " \"user_edited\": bool(review.get(\"user_edited_material\", 0)),\n", " \"edit_reason\": review.get(\"edit_reason\", \"\"),\n", " },\n", @@ -1340,7 +1691,7 @@ " },\n", " \"status\": review.get(\"status\", \"ok\"),\n", " \"error_type\": review.get(\"error_type\", \"\"),\n", - " \"model_name\": \"SigCLIP\",\n", + " \"model_name\": \"SigCLIP+DINOv3\",\n", " \"created_at_utc\": datetime.now(timezone.utc).isoformat(),\n", " }\n", "\n", @@ -1358,10 +1709,10 @@ " \"pred_material_top1\": clf[\"pred_material_top1\"],\n", " \"pred_material_top3\": clf[\"pred_material_top3\"],\n", " \"pred_material_conf_top1\": clf[\"pred_material_conf_top1\"],\n", - " \"model_name\": \"SigCLIP\",\n", + " \"model_name\": \"SigCLIP+DINOv3\",\n", " \"segmentation_accepted\": review.get(\"segmentation_accepted\", 1),\n", " \"user_edited_material\": review.get(\"user_edited_material\", 0),\n", - " \"user_final_material\": review.get(\"user_final_material\", clf[\"pred_material_top1\"]),\n", + " \"user_final_material\": encode_final_material_for_csv(final_mode, final_single, final_mixed),\n", " \"edit_reason\": review.get(\"edit_reason\", \"\"),\n", " \"end_to_end_time_sec\": timing[\"end_to_end_time_sec\"],\n", " \"segmentation_time_sec\": timing[\"segmentation_time_sec\"],\n", @@ -1433,39 +1784,14 @@ "metadata": {}, "outputs": [], "source": [ - "def model_slice(df_in: pd.DataFrame, model_name: str) -> pd.DataFrame:\n", - " return df_in[df_in[\"model_name\"].fillna(\"\").str.lower() == model_name.lower()]\n", - "\n", "df_eval = pd.read_csv(CSV_PATH)\n", - "sigclip_df = model_slice(df_eval, \"SigCLIP\")\n", - "dinov3_df = model_slice(df_eval, \"DINOv3\")\n", - "\n", - "if sigclip_df.empty:\n", - " print(\"No SigCLIP rows yet.\")\n", - "else:\n", - " print(\"SigCLIP metrics:\")\n", - " print(compute_metrics(sigclip_df)[\"metrics\"])\n", - "\n", - "if dinov3_df.empty:\n", - " print(\"No DINOv3 rows yet.\")\n", + "if df_eval.empty:\n", + " print(\"No evaluation rows yet.\")\n", "else:\n", - " print(\"DINOv3 metrics:\")\n", - " print(compute_metrics(dinov3_df)[\"metrics\"])\n", - "\n", - "if not sigclip_df.empty and not dinov3_df.empty:\n", - " m_sig = compute_metrics(sigclip_df)[\"metrics\"]\n", - " m_dino = compute_metrics(dinov3_df)[\"metrics\"]\n", - "\n", - " top1_gain = m_dino[\"top1_accuracy\"] - m_sig[\"top1_accuracy\"]\n", - " edit_drop = m_sig[\"manual_edit_rate\"] - m_dino[\"manual_edit_rate\"]\n", - " time_increase = (m_dino[\"median_e2e_time_sec\"] - m_sig[\"median_e2e_time_sec\"]) / max(m_sig[\"median_e2e_time_sec\"], 1e-9)\n", - "\n", - " promote = (top1_gain >= 0.05 or edit_drop >= 0.10) and time_increase <= 0.20\n", - "\n", - " print(f\"Top-1 gain (DINOv3 - SigCLIP): {top1_gain:.3f}\")\n", - " print(f\"Edit-rate drop (SigCLIP - DINOv3): {edit_drop:.3f}\")\n", - " print(f\"Median time increase: {time_increase:.3f}\")\n", - " print(f\"Promote DINOv3: {'YES' if promote else 'NO'}\")" + " for name in sorted(df_eval[\"model_name\"].dropna().unique().tolist()):\n", + " sub = df_eval[df_eval[\"model_name\"] == name]\n", + " print(f\"Model={name}\")\n", + " print(compute_metrics(sub)[\"metrics\"])" ] } ], |