new notebook.. had to update due to a version mismatch

1 files changed, 120 insertions, 46 deletions

diff --git a/poc_eval_notebook.ipynb b/poc_eval_notebook.ipynb
index 5865f7e..a1cfd46 100644
--- a/poc_eval_notebook.ipynb
+++ b/poc_eval_notebook.ipynb
@@ -2,7 +2,9 @@
   "cells": [
     {
       "cell_type": "markdown",
-      "metadata": {},
+      "metadata": {
+        "id": "CLas2h6IyUS2"
+      },
       "source": [
         "# Material Hunters POC Notebook\n",
         "\n",
@@ -15,23 +17,29 @@
         "- evaluation row append + go/no-go metrics\n",
         "\n",
         "Requires access to the gated SAM 3.1 checkpoint on Hugging Face."
-      ]
+      ],
+      "id": "CLas2h6IyUS2"
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "id": "0s7ATMAxyUS2"
+      },
       "outputs": [],
       "source": [
         "# Run once in Colab if needed.\n",
         "%pip -q install transformers accelerate pillow matplotlib pandas huggingface_hub\n",
         "%pip -q install git+https://github.com/facebookresearch/sam3.git"
-      ]
+      ],
+      "id": "0s7ATMAxyUS2"
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "id": "UdCZOxTTyUS2"
+      },
       "outputs": [],
       "source": [
         "from pathlib import Path\n",
@@ -61,7 +69,8 @@
         "SAM3_CONFIDENCE_THRESHOLD = 0.10\n",
         "SAM3_POINT_HINT_BOX_RADIUS_PX = 12\n",
         "SAM3_MAX_POINT_HINTS = 4\n",
-        "SAM3_TORCH_DTYPE = torch.bfloat16 if DEVICE == \"cuda\" else torch.float32\n",
+        "SAM3_USE_AUTOMIXED_PRECISION = False  # set True only after verifying your SAM3 runtime supports bf16 end-to-end\n",
+        "SAM3_TORCH_DTYPE = torch.bfloat16 if (DEVICE == \"cuda\" and SAM3_USE_AUTOMIXED_PRECISION) else torch.float32\n",
         "DINO_MODEL_ID = \"facebook/dinov3-vitb16-pretrain-lvd1689m\"\n",
         "SIGCLIP_FUSION_WEIGHT = 0.60\n",
         "DINO_FUSION_WEIGHT = 0.40\n",
@@ -118,12 +127,15 @@
         "    \"beam\": [\"steel\", \"wood\", \"concrete\"],\n",
         "    \"column\": [\"concrete\", \"steel\", \"brick\", \"wood\"],\n",
         "}"
-      ]
+      ],
+      "id": "UdCZOxTTyUS2"
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "id": "w79BN3wOyUS2"
+      },
       "outputs": [],
       "source": [
         "def ensure_eval_csv(csv_path: Path, template_path: Path):\n",
@@ -681,11 +693,27 @@
         "\n",
         "\n",
         "def sam3_autocast_context():\n",
-        "    if DEVICE == \"cuda\" and SAM3_TORCH_DTYPE in (torch.float16, torch.bfloat16):\n",
-        "        return torch.autocast(device_type=\"cuda\", dtype=SAM3_TORCH_DTYPE)\n",
+        "    if DEVICE == \"cuda\":\n",
+        "        if SAM3_USE_AUTOMIXED_PRECISION and SAM3_TORCH_DTYPE in (torch.float16, torch.bfloat16):\n",
+        "            return torch.autocast(device_type=\"cuda\", dtype=SAM3_TORCH_DTYPE)\n",
+        "        return torch.autocast(device_type=\"cuda\", enabled=False)\n",
         "    return nullcontext()\n",
         "\n",
         "\n",
+        "def cast_floating_tensors(obj, dtype: torch.dtype):\n",
+        "    if torch.is_tensor(obj):\n",
+        "        if obj.is_floating_point():\n",
+        "            return obj.to(dtype=dtype)\n",
+        "        return obj\n",
+        "    if isinstance(obj, dict):\n",
+        "        return {k: cast_floating_tensors(v, dtype) for k, v in obj.items()}\n",
+        "    if isinstance(obj, list):\n",
+        "        return [cast_floating_tensors(v, dtype) for v in obj]\n",
+        "    if isinstance(obj, tuple):\n",
+        "        return tuple(cast_floating_tensors(v, dtype) for v in obj)\n",
+        "    return obj\n",
+        "\n",
+        "\n",
         "def bbox_xyxy_to_norm_cxcywh(bbox_xyxy, image_width: int, image_height: int):\n",
         "    x0, y0, x1, y1 = [float(v) for v in bbox_xyxy]\n",
         "    x0 = max(0.0, min(float(image_width - 1), x0))\n",
@@ -787,24 +815,42 @@
         "def run_sam_guided_segmentation(image: Image.Image, prompt_bbox_xyxy, prompt_points_xy, sam_model, sam_processor):\n",
         "    img_w, img_h = image.size\n",
         "\n",
-        "    with torch.inference_mode(), sam3_autocast_context():\n",
-        "        state = sam_processor.set_image(image, state={})\n",
-        "\n",
-        "        box_norm = bbox_xyxy_to_norm_cxcywh(prompt_bbox_xyxy, image_width=img_w, image_height=img_h)\n",
-        "        state = sam_processor.add_geometric_prompt(box=box_norm, label=True, state=state)\n",
-        "\n",
-        "        if prompt_points_xy:\n",
-        "            hint_points = prompt_points_xy[:SAM3_MAX_POINT_HINTS]\n",
-        "            for px, py in hint_points:\n",
-        "                r = int(SAM3_POINT_HINT_BOX_RADIUS_PX)\n",
-        "                pbox = [\n",
-        "                    int(max(0, px - r)),\n",
-        "                    int(max(0, py - r)),\n",
-        "                    int(min(img_w - 1, px + r)),\n",
-        "                    int(min(img_h - 1, py + r)),\n",
-        "                ]\n",
-        "                pbox_norm = bbox_xyxy_to_norm_cxcywh(pbox, image_width=img_w, image_height=img_h)\n",
-        "                state = sam_processor.add_geometric_prompt(box=pbox_norm, label=True, state=state)\n",
+        "    def _infer_once(target_dtype: torch.dtype):\n",
+        "        with torch.inference_mode(), sam3_autocast_context():\n",
+        "            state = sam_processor.set_image(image, state={})\n",
+        "            if \"backbone_out\" in state:\n",
+        "                state[\"backbone_out\"] = cast_floating_tensors(state[\"backbone_out\"], target_dtype)\n",
+        "\n",
+        "            box_norm = bbox_xyxy_to_norm_cxcywh(prompt_bbox_xyxy, image_width=img_w, image_height=img_h)\n",
+        "            state = sam_processor.add_geometric_prompt(box=box_norm, label=True, state=state)\n",
+        "            if \"backbone_out\" in state:\n",
+        "                state[\"backbone_out\"] = cast_floating_tensors(state[\"backbone_out\"], target_dtype)\n",
+        "\n",
+        "            if prompt_points_xy:\n",
+        "                hint_points = prompt_points_xy[:SAM3_MAX_POINT_HINTS]\n",
+        "                for px, py in hint_points:\n",
+        "                    r = int(SAM3_POINT_HINT_BOX_RADIUS_PX)\n",
+        "                    pbox = [\n",
+        "                        int(max(0, px - r)),\n",
+        "                        int(max(0, py - r)),\n",
+        "                        int(min(img_w - 1, px + r)),\n",
+        "                        int(min(img_h - 1, py + r)),\n",
+        "                    ]\n",
+        "                    pbox_norm = bbox_xyxy_to_norm_cxcywh(pbox, image_width=img_w, image_height=img_h)\n",
+        "                    state = sam_processor.add_geometric_prompt(box=pbox_norm, label=True, state=state)\n",
+        "                    if \"backbone_out\" in state:\n",
+        "                        state[\"backbone_out\"] = cast_floating_tensors(state[\"backbone_out\"], target_dtype)\n",
+        "            return state\n",
+        "\n",
+        "    try:\n",
+        "        state = _infer_once(SAM3_TORCH_DTYPE)\n",
+        "    except RuntimeError as exc:\n",
+        "        if \"mat1 and mat2 must have the same dtype\" not in str(exc):\n",
+        "            raise\n",
+        "        fallback_dtype = torch.float32 if (SAM3_TORCH_DTYPE != torch.float32 or DEVICE != \"cuda\") else torch.bfloat16\n",
+        "        print(f\"SAM3 dtype mismatch detected, retrying once with dtype={fallback_dtype}\")\n",
+        "        sam_model.to(dtype=fallback_dtype)\n",
+        "        state = _infer_once(fallback_dtype)\n",
         "\n",
         "    masks_t = state.get(\"masks\", None)\n",
         "    scores_t = state.get(\"scores\", None)\n",
@@ -1184,12 +1230,15 @@
         "    df_existing = pd.read_csv(csv_path)\n",
         "    df_out = pd.concat([df_existing, df_new], ignore_index=True)\n",
         "    df_out.to_csv(csv_path, index=False)"
-      ]
+      ],
+      "id": "w79BN3wOyUS2"
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "id": "0N1GZrx-yUS2"
+      },
       "outputs": [],
       "source": [
         "ensure_eval_csv(CSV_PATH, TEMPLATE_PATH)\n",
@@ -1232,6 +1281,8 @@
         "    enable_inst_interactivity=False,\n",
         ")\n",
         "sam_model = sam_model.to(dtype=SAM3_TORCH_DTYPE)\n",
+        "if not SAM3_USE_AUTOMIXED_PRECISION:\n",
+        "    sam_model = sam_model.float()\n",
         "sam_model.eval()\n",
         "sam_processor = Sam3Processor(\n",
         "    model=sam_model,\n",
@@ -1264,12 +1315,15 @@
         "        print(f\"- {p}\")\n",
         "\n",
         "print(\"Models ready.\")"
-      ]
+      ],
+      "id": "0N1GZrx-yUS2"
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "id": "xsDMjiUSyUS2"
+      },
       "outputs": [],
       "source": [
         "# Configure one image and one-or-many objects in it.\n",
@@ -1397,12 +1451,15 @@
         "plt.title(\"Prompt preview (bbox + optional brush points)\")\n",
         "plt.axis(\"off\")\n",
         "plt.show()"
-      ]
+      ],
+      "id": "xsDMjiUSyUS2"
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "id": "-8KZh6FMyUS2"
+      },
       "outputs": [],
       "source": [
         "RUN_ROWS = []\n",
@@ -1519,12 +1576,15 @@
         "    plt.show()\n",
         "\n",
         "print(f\"Processed {len(RUN_ROWS)} object(s).\")"
-      ]
+      ],
+      "id": "-8KZh6FMyUS2"
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "id": "btKtYYGXyUS2"
+      },
       "outputs": [],
       "source": [
         "# Human review/edit step. Set defaults and optional per-sample overrides.\n",
@@ -1595,12 +1655,15 @@
         "    }\n",
         "    for r in RUN_ROWS\n",
         "])"
-      ]
+      ],
+      "id": "btKtYYGXyUS2"
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "id": "xW6cux7gyUS2"
+      },
       "outputs": [],
       "source": [
         "for run in RUN_ROWS:\n",
@@ -1727,12 +1790,15 @@
         "    print(f\"[{run['sample_id']}] saved passport + eval row\")\n",
         "\n",
         "print(f\"Updated eval CSV: {CSV_PATH}\")"
-      ]
+      ],
+      "id": "xW6cux7gyUS2"
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "id": "zDsg2JYsyUS2"
+      },
       "outputs": [],
       "source": [
         "def compute_metrics(df_in: pd.DataFrame) -> dict:\n",
@@ -1776,12 +1842,15 @@
         "for k, v in result[\"metrics\"].items():\n",
         "    print(f\"- {k}: {v}\")\n",
         "print(f\"Go/No-Go: {'GO' if result['go_no_go'] else 'NO-GO'}\")"
-      ]
+      ],
+      "id": "zDsg2JYsyUS2"
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "id": "x77TLxvUyUS2"
+      },
       "outputs": [],
       "source": [
         "df_eval = pd.read_csv(CSV_PATH)\n",
@@ -1792,20 +1861,25 @@
         "        sub = df_eval[df_eval[\"model_name\"] == name]\n",
         "        print(f\"Model={name}\")\n",
         "        print(compute_metrics(sub)[\"metrics\"])"
-      ]
+      ],
+      "id": "x77TLxvUyUS2"
     }
   ],
   "metadata": {
     "kernelspec": {
       "display_name": "Python 3",
-      "language": "python",
       "name": "python3"
     },
     "language_info": {
       "name": "python",
       "version": "3"
-    }
+    },
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4"
+    },
+    "accelerator": "GPU"
   },
   "nbformat": 4,
   "nbformat_minor": 5
-}
+}
\ No newline at end of file