diff options
| -rw-r--r-- | poc_eval_notebook.ipynb | 166 |
1 files changed, 120 insertions, 46 deletions
diff --git a/poc_eval_notebook.ipynb b/poc_eval_notebook.ipynb index 5865f7e..a1cfd46 100644 --- a/poc_eval_notebook.ipynb +++ b/poc_eval_notebook.ipynb @@ -2,7 +2,9 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "CLas2h6IyUS2" + }, "source": [ "# Material Hunters POC Notebook\n", "\n", @@ -15,23 +17,29 @@ "- evaluation row append + go/no-go metrics\n", "\n", "Requires access to the gated SAM 3.1 checkpoint on Hugging Face." - ] + ], + "id": "CLas2h6IyUS2" }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "id": "0s7ATMAxyUS2" + }, "outputs": [], "source": [ "# Run once in Colab if needed.\n", "%pip -q install transformers accelerate pillow matplotlib pandas huggingface_hub\n", "%pip -q install git+https://github.com/facebookresearch/sam3.git" - ] + ], + "id": "0s7ATMAxyUS2" }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "id": "UdCZOxTTyUS2" + }, "outputs": [], "source": [ "from pathlib import Path\n", @@ -61,7 +69,8 @@ "SAM3_CONFIDENCE_THRESHOLD = 0.10\n", "SAM3_POINT_HINT_BOX_RADIUS_PX = 12\n", "SAM3_MAX_POINT_HINTS = 4\n", - "SAM3_TORCH_DTYPE = torch.bfloat16 if DEVICE == \"cuda\" else torch.float32\n", + "SAM3_USE_AUTOMIXED_PRECISION = False # set True only after verifying your SAM3 runtime supports bf16 end-to-end\n", + "SAM3_TORCH_DTYPE = torch.bfloat16 if (DEVICE == \"cuda\" and SAM3_USE_AUTOMIXED_PRECISION) else torch.float32\n", "DINO_MODEL_ID = \"facebook/dinov3-vitb16-pretrain-lvd1689m\"\n", "SIGCLIP_FUSION_WEIGHT = 0.60\n", "DINO_FUSION_WEIGHT = 0.40\n", @@ -118,12 +127,15 @@ " \"beam\": [\"steel\", \"wood\", \"concrete\"],\n", " \"column\": [\"concrete\", \"steel\", \"brick\", \"wood\"],\n", "}" - ] + ], + "id": "UdCZOxTTyUS2" }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "id": "w79BN3wOyUS2" + }, "outputs": [], "source": [ "def ensure_eval_csv(csv_path: Path, template_path: Path):\n", @@ -681,11 +693,27 @@ "\n", "\n", "def sam3_autocast_context():\n", - " if DEVICE == \"cuda\" and SAM3_TORCH_DTYPE in (torch.float16, torch.bfloat16):\n", - " return torch.autocast(device_type=\"cuda\", dtype=SAM3_TORCH_DTYPE)\n", + " if DEVICE == \"cuda\":\n", + " if SAM3_USE_AUTOMIXED_PRECISION and SAM3_TORCH_DTYPE in (torch.float16, torch.bfloat16):\n", + " return torch.autocast(device_type=\"cuda\", dtype=SAM3_TORCH_DTYPE)\n", + " return torch.autocast(device_type=\"cuda\", enabled=False)\n", " return nullcontext()\n", "\n", "\n", + "def cast_floating_tensors(obj, dtype: torch.dtype):\n", + " if torch.is_tensor(obj):\n", + " if obj.is_floating_point():\n", + " return obj.to(dtype=dtype)\n", + " return obj\n", + " if isinstance(obj, dict):\n", + " return {k: cast_floating_tensors(v, dtype) for k, v in obj.items()}\n", + " if isinstance(obj, list):\n", + " return [cast_floating_tensors(v, dtype) for v in obj]\n", + " if isinstance(obj, tuple):\n", + " return tuple(cast_floating_tensors(v, dtype) for v in obj)\n", + " return obj\n", + "\n", + "\n", "def bbox_xyxy_to_norm_cxcywh(bbox_xyxy, image_width: int, image_height: int):\n", " x0, y0, x1, y1 = [float(v) for v in bbox_xyxy]\n", " x0 = max(0.0, min(float(image_width - 1), x0))\n", @@ -787,24 +815,42 @@ "def run_sam_guided_segmentation(image: Image.Image, prompt_bbox_xyxy, prompt_points_xy, sam_model, sam_processor):\n", " img_w, img_h = image.size\n", "\n", - " with torch.inference_mode(), sam3_autocast_context():\n", - " state = sam_processor.set_image(image, state={})\n", - "\n", - " box_norm = bbox_xyxy_to_norm_cxcywh(prompt_bbox_xyxy, image_width=img_w, image_height=img_h)\n", - " state = sam_processor.add_geometric_prompt(box=box_norm, label=True, state=state)\n", - "\n", - " if prompt_points_xy:\n", - " hint_points = prompt_points_xy[:SAM3_MAX_POINT_HINTS]\n", - " for px, py in hint_points:\n", - " r = int(SAM3_POINT_HINT_BOX_RADIUS_PX)\n", - " pbox = [\n", - " int(max(0, px - r)),\n", - " int(max(0, py - r)),\n", - " int(min(img_w - 1, px + r)),\n", - " int(min(img_h - 1, py + r)),\n", - " ]\n", - " pbox_norm = bbox_xyxy_to_norm_cxcywh(pbox, image_width=img_w, image_height=img_h)\n", - " state = sam_processor.add_geometric_prompt(box=pbox_norm, label=True, state=state)\n", + " def _infer_once(target_dtype: torch.dtype):\n", + " with torch.inference_mode(), sam3_autocast_context():\n", + " state = sam_processor.set_image(image, state={})\n", + " if \"backbone_out\" in state:\n", + " state[\"backbone_out\"] = cast_floating_tensors(state[\"backbone_out\"], target_dtype)\n", + "\n", + " box_norm = bbox_xyxy_to_norm_cxcywh(prompt_bbox_xyxy, image_width=img_w, image_height=img_h)\n", + " state = sam_processor.add_geometric_prompt(box=box_norm, label=True, state=state)\n", + " if \"backbone_out\" in state:\n", + " state[\"backbone_out\"] = cast_floating_tensors(state[\"backbone_out\"], target_dtype)\n", + "\n", + " if prompt_points_xy:\n", + " hint_points = prompt_points_xy[:SAM3_MAX_POINT_HINTS]\n", + " for px, py in hint_points:\n", + " r = int(SAM3_POINT_HINT_BOX_RADIUS_PX)\n", + " pbox = [\n", + " int(max(0, px - r)),\n", + " int(max(0, py - r)),\n", + " int(min(img_w - 1, px + r)),\n", + " int(min(img_h - 1, py + r)),\n", + " ]\n", + " pbox_norm = bbox_xyxy_to_norm_cxcywh(pbox, image_width=img_w, image_height=img_h)\n", + " state = sam_processor.add_geometric_prompt(box=pbox_norm, label=True, state=state)\n", + " if \"backbone_out\" in state:\n", + " state[\"backbone_out\"] = cast_floating_tensors(state[\"backbone_out\"], target_dtype)\n", + " return state\n", + "\n", + " try:\n", + " state = _infer_once(SAM3_TORCH_DTYPE)\n", + " except RuntimeError as exc:\n", + " if \"mat1 and mat2 must have the same dtype\" not in str(exc):\n", + " raise\n", + " fallback_dtype = torch.float32 if (SAM3_TORCH_DTYPE != torch.float32 or DEVICE != \"cuda\") else torch.bfloat16\n", + " print(f\"SAM3 dtype mismatch detected, retrying once with dtype={fallback_dtype}\")\n", + " sam_model.to(dtype=fallback_dtype)\n", + " state = _infer_once(fallback_dtype)\n", "\n", " masks_t = state.get(\"masks\", None)\n", " scores_t = state.get(\"scores\", None)\n", @@ -1184,12 +1230,15 @@ " df_existing = pd.read_csv(csv_path)\n", " df_out = pd.concat([df_existing, df_new], ignore_index=True)\n", " df_out.to_csv(csv_path, index=False)" - ] + ], + "id": "w79BN3wOyUS2" }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "id": "0N1GZrx-yUS2" + }, "outputs": [], "source": [ "ensure_eval_csv(CSV_PATH, TEMPLATE_PATH)\n", @@ -1232,6 +1281,8 @@ " enable_inst_interactivity=False,\n", ")\n", "sam_model = sam_model.to(dtype=SAM3_TORCH_DTYPE)\n", + "if not SAM3_USE_AUTOMIXED_PRECISION:\n", + " sam_model = sam_model.float()\n", "sam_model.eval()\n", "sam_processor = Sam3Processor(\n", " model=sam_model,\n", @@ -1264,12 +1315,15 @@ " print(f\"- {p}\")\n", "\n", "print(\"Models ready.\")" - ] + ], + "id": "0N1GZrx-yUS2" }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "id": "xsDMjiUSyUS2" + }, "outputs": [], "source": [ "# Configure one image and one-or-many objects in it.\n", @@ -1397,12 +1451,15 @@ "plt.title(\"Prompt preview (bbox + optional brush points)\")\n", "plt.axis(\"off\")\n", "plt.show()" - ] + ], + "id": "xsDMjiUSyUS2" }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "id": "-8KZh6FMyUS2" + }, "outputs": [], "source": [ "RUN_ROWS = []\n", @@ -1519,12 +1576,15 @@ " plt.show()\n", "\n", "print(f\"Processed {len(RUN_ROWS)} object(s).\")" - ] + ], + "id": "-8KZh6FMyUS2" }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "id": "btKtYYGXyUS2" + }, "outputs": [], "source": [ "# Human review/edit step. Set defaults and optional per-sample overrides.\n", @@ -1595,12 +1655,15 @@ " }\n", " for r in RUN_ROWS\n", "])" - ] + ], + "id": "btKtYYGXyUS2" }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "id": "xW6cux7gyUS2" + }, "outputs": [], "source": [ "for run in RUN_ROWS:\n", @@ -1727,12 +1790,15 @@ " print(f\"[{run['sample_id']}] saved passport + eval row\")\n", "\n", "print(f\"Updated eval CSV: {CSV_PATH}\")" - ] + ], + "id": "xW6cux7gyUS2" }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "id": "zDsg2JYsyUS2" + }, "outputs": [], "source": [ "def compute_metrics(df_in: pd.DataFrame) -> dict:\n", @@ -1776,12 +1842,15 @@ "for k, v in result[\"metrics\"].items():\n", " print(f\"- {k}: {v}\")\n", "print(f\"Go/No-Go: {'GO' if result['go_no_go'] else 'NO-GO'}\")" - ] + ], + "id": "zDsg2JYsyUS2" }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "id": "x77TLxvUyUS2" + }, "outputs": [], "source": [ "df_eval = pd.read_csv(CSV_PATH)\n", @@ -1792,20 +1861,25 @@ " sub = df_eval[df_eval[\"model_name\"] == name]\n", " print(f\"Model={name}\")\n", " print(compute_metrics(sub)[\"metrics\"])" - ] + ], + "id": "x77TLxvUyUS2" } ], "metadata": { "kernelspec": { "display_name": "Python 3", - "language": "python", "name": "python3" }, "language_info": { "name": "python", "version": "3" - } + }, + "colab": { + "provenance": [], + "gpuType": "T4" + }, + "accelerator": "GPU" }, "nbformat": 4, "nbformat_minor": 5 -} +}
\ No newline at end of file |