integrated finger process pipeline on remapped 2d image

src/orisys/sensor.py

@@ -132,6 +132,14 @@ class Sensor:
         self.centroid_found = False
         self.contour = np.empty(0)
         self.depth_map = np.zeros_like(self.img)
+        self.valid_mask_raw = None
+        self.valid_mask = None
+        self.valid_mask_hhd = None
+        self.mask_stats = {
+            "coverage": 0.0,
+            "flow_mean_valid": 0.0,
+            "flow_mean_invalid": 0.0,
+        }
         
         # Timing
         self.last_log_time = time.perf_counter()
@@ -165,6 +173,12 @@ class Sensor:
         self._hhd_grid_shape = None
         cfg_backend = self.__config_tracker.get("backend", None)
         self.poisson_backend = cfg_backend if cfg_backend is not None else backend
+        self.masked_poisson_solver = self.__config_tracker.get("masked_poisson_solver", "sparse")
+        self.masked_poisson_max_boundary = self.__config_tracker.get("masked_poisson_max_boundary", 96)
+        self.masked_poisson_boundary_stride = self.__config_tracker.get("masked_poisson_boundary_stride", 1)
+        self.masked_poisson_max_iters = self.__config_tracker.get("masked_poisson_max_iters", 3)
+        self.masked_poisson_tol = self.__config_tracker.get("masked_poisson_tol", 1e-4)
+        self.masked_poisson_reg = self.__config_tracker.get("masked_poisson_reg", 1e-8)
         self._use_pyav = False
         self._av_container = None
         self._av_stream = None
@@ -204,7 +218,72 @@ class Sensor:
     # Public API - Core Methods (Most Frequently Used)
     # =========================================================================
     
-    def get_img(self, frame=None, already_processed=False):
+    def _resize_mask(self, mask, size):
+        if mask is None:
+            return None
+        mask_u8 = np.asarray(mask, dtype=np.uint8)
+        resized = cv2.resize(mask_u8, size, interpolation=cv2.INTER_NEAREST)
+        return resized.astype(bool)
+
+    def _prepare_valid_mask(self, valid_mask, raw_shape, stitched_shape):
+        if valid_mask is None:
+            self.valid_mask_raw = None
+            self.valid_mask = None
+            self.valid_mask_hhd = None
+            self.mask_stats["coverage"] = 0.0
+            return
+
+        mask_raw = np.asarray(valid_mask, dtype=bool)
+        if mask_raw.shape != raw_shape:
+            mask_raw = self._resize_mask(mask_raw, (raw_shape[1], raw_shape[0]))
+
+        if self.mirror:
+            mask_raw = np.fliplr(mask_raw)
+
+        self.valid_mask_raw = mask_raw
+        self.valid_mask = self._resize_mask(mask_raw, (stitched_shape[1], stitched_shape[0]))
+        self._update_hhd_mask_cache()
+        self.mask_stats["coverage"] = float(np.mean(self.valid_mask)) if self.valid_mask.size else 0.0
+
+    def _hhd_grid_size(self):
+        if self.valid_mask is None:
+            return None
+        h, w = self.valid_mask.shape[:2]
+        k = max(1, int(self._hhd_downsample))
+        if k <= 1:
+            return (w, h)
+        return (max(1, w // k), max(1, h // k))
+
+    def _update_hhd_mask_cache(self):
+        """Precompute valid_mask at HHD resolution when the UV domain is fixed."""
+        if self.valid_mask is None:
+            self.valid_mask_hhd = None
+            return
+        ds_size = self._hhd_grid_size()
+        if ds_size is None:
+            self.valid_mask_hhd = None
+            return
+        if self.valid_mask.shape[1] == ds_size[0] and self.valid_mask.shape[0] == ds_size[1]:
+            self.valid_mask_hhd = self.valid_mask.astype(bool)
+        else:
+            self.valid_mask_hhd = self._resize_mask(self.valid_mask, ds_size)
+
+    def _compute_flow_means(self, flow):
+        if flow is None:
+            return
+        mag = np.linalg.norm(flow, axis=-1)
+        mask = self.valid_mask
+        if mask is None or mask.shape != mag.shape:
+            self.mask_stats["flow_mean_valid"] = float(mag.mean()) if mag.size else 0.0
+            self.mask_stats["flow_mean_invalid"] = 0.0
+            return
+
+        valid = mask.astype(bool)
+        invalid = ~valid
+        self.mask_stats["flow_mean_valid"] = float(mag[valid].mean()) if np.any(valid) else 0.0
+        self.mask_stats["flow_mean_invalid"] = float(mag[invalid].mean()) if np.any(invalid) else 0.0
+
+    def get_img(self, frame=None, already_processed=False, valid_mask=None, already_gray=False):
         """
         Capture and preprocess a frame.
 
@@ -238,23 +317,39 @@ class Sensor:
         # Start processing timing
         t_processing_start = time.perf_counter()
         
-        self.frame = frame.copy() if isinstance(frame, np.ndarray) else frame
+        self.frame = frame if (already_processed and isinstance(frame, np.ndarray)) else (
+            frame.copy() if isinstance(frame, np.ndarray) else frame
+        )
         if already_processed:
             img_bgr = frame
-            if img_bgr.ndim == 2:
-                img_bgr = cv2.cvtColor(img_bgr, cv2.COLOR_GRAY2BGR)
-            elif img_bgr.ndim == 3 and img_bgr.shape[2] == 1:
-                img_bgr = cv2.cvtColor(img_bgr[:, :, 0], cv2.COLOR_GRAY2BGR)
+            gray = None
+            if already_gray:
+                if frame.ndim == 2:
+                    gray = frame
+                elif frame.ndim == 3 and frame.shape[2] == 1:
+                    gray = frame[:, :, 0]
+                else:
+                    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+                img_bgr = None
+            else:
+                if img_bgr.ndim == 2:
+                    img_bgr = cv2.cvtColor(img_bgr, cv2.COLOR_GRAY2BGR)
+                elif img_bgr.ndim == 3 and img_bgr.shape[2] == 1:
+                    img_bgr = cv2.cvtColor(img_bgr[:, :, 0], cv2.COLOR_GRAY2BGR)
         else:
             img_bgr = self.frame_processor.process(frame)
+            gray = None
         t_after_extract = time.perf_counter()
 
         t_after_stitch = time.perf_counter()
 
-        self.img_size_raw = img_bgr.shape
-        self._img_bgr_raw = img_bgr
-
-        gray = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY)
+        if gray is None:
+            self.img_size_raw = img_bgr.shape
+            self._img_bgr_raw = img_bgr
+            gray = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY)
+        else:
+            self.img_size_raw = gray.shape
+            self._img_bgr_raw = None
         res = tuple(self._config_resolution_stitch)
         if self._img_gray_buf is None or self._img_gray_buf.shape != (res[1], res[0]):
             self._img_gray_buf = np.empty((res[1], res[0]), dtype=np.uint8)
@@ -265,6 +360,7 @@ class Sensor:
             cv2.flip(self._img_gray_buf, 1, dst=self._img_gray_buf)
 
         self.img = self._img_gray_buf
+        self._prepare_valid_mask(valid_mask, gray.shape, self.img.shape)
         
         t_processing_end = time.perf_counter()
         self._last_processing_time = t_processing_end - t_processing_start
@@ -281,7 +377,9 @@ class Sensor:
     def _get_img_bgr(self):
         """Produce a resized+mirrored BGR image on demand (for display)."""
         if self._img_bgr_raw is None:
-            return None
+            if self.img is None:
+                return None
+            return cv2.cvtColor(self.img, cv2.COLOR_GRAY2BGR)
         res = tuple(self._config_resolution_stitch)
         bgr = cv2.resize(self._img_bgr_raw, res, interpolation=cv2.INTER_LINEAR)
         if self.mirror:
@@ -292,8 +390,12 @@ class Sensor:
 
         # detect motion by optical flow magnitude
         mag = np.linalg.norm(self.flow, axis=-1)
-        # print(f"mag: {mag.max()}, {mag.min()}, {mag.mean()}")
-        return mag.mean() > threshold
+        if self.valid_mask is not None and self.valid_mask.shape == mag.shape:
+            active = self.valid_mask.astype(bool)
+            motion_value = mag[active].mean() if np.any(active) else 0.0
+        else:
+            motion_value = mag.mean()
+        return motion_value > threshold
 
     def compute_deformation(self, decompose=True, check_motion=True, threshold=6, tail_frames=10):
         """
@@ -352,15 +454,24 @@ class Sensor:
 
         return False
 
+    def warmup_deformation(self):
+        """Build optical-flow and masked-HHD caches using the current image."""
+        if self.img is None:
+            raise Exception("Warning: Get image first")
+        self.compute_flow()
+        self.compute_hhd()
+        return True
+
     def compute_flow(self):
         if self.img is None:
             raise Exception("Warning: Get image first")
         
-        flow = self.__tracker.track(self.img)
+        flow = self.__tracker.track(self.img, valid_mask=self.valid_mask)
         # flow = self.__tracker.track_raft(self.img)
         if hasattr(flow, 'download'):
             flow = flow.download()
         self.flow = flow
+        self._compute_flow_means(self.flow)
     
     def compute_hhd(self):
 
@@ -374,20 +485,50 @@ class Sensor:
         except (TypeError, ValueError):
             k = 1
 
+        mask_ds = None
+        if self.valid_mask is not None:
+            ds_size = self._hhd_grid_size()
+            if (
+                self.valid_mask_hhd is not None
+                and ds_size is not None
+                and self.valid_mask_hhd.shape[1] == ds_size[0]
+                and self.valid_mask_hhd.shape[0] == ds_size[1]
+            ):
+                mask_ds = self.valid_mask_hhd
+            else:
+                mask_ds = self.valid_mask.astype(bool)
+
         if k > 1:
             ds_w = max(1, orig_w // k)
             ds_h = max(1, orig_h // k)
             flow_ds = cv2.resize(flow, (ds_w, ds_h), interpolation=cv2.INTER_AREA)
+            if mask_ds is not None and (
+                mask_ds.shape[1] != ds_w or mask_ds.shape[0] != ds_h
+            ):
+                mask_ds = self._resize_mask(mask_ds, (ds_w, ds_h))
         else:
             flow_ds = flow
 
         ds_shape = flow_ds.shape[:2]
+        self.valid_mask_hhd = mask_ds
         if self.__hhd is None or self._hhd_grid_shape != ds_shape:
             spacings = (18 * k, 18 * k)
-            self.__hhd = nHHD(cuda=self.cuda, grid=ds_shape, spacings=spacings, poisson_backend=self.poisson_backend, debug=self.debug)
+            self.__hhd = nHHD(
+                cuda=self.cuda,
+                grid=ds_shape,
+                spacings=spacings,
+                poisson_backend=self.poisson_backend,
+                debug=self.debug,
+                masked_poisson_solver=self.masked_poisson_solver,
+                masked_poisson_max_boundary=self.masked_poisson_max_boundary,
+                masked_poisson_boundary_stride=self.masked_poisson_boundary_stride,
+                masked_poisson_max_iters=self.masked_poisson_max_iters,
+                masked_poisson_tol=self.masked_poisson_tol,
+                masked_poisson_reg=self.masked_poisson_reg,
+            )
             self._hhd_grid_shape = ds_shape
 
-        _, forces = self.__hhd.get_force(flow_ds)
+        _, forces = self.__hhd.get_force(flow_ds, valid_mask=mask_ds)
         self.fnormal, self.fshearx, self.fsheary, self.ftor = forces
 
         vnormal_ds = self.__hhd.get_normal()
@@ -400,6 +541,13 @@ class Sensor:
             self.vnormal = vnormal_ds
             self.vshear = vshear_ds
 
+        if self.valid_mask is not None:
+            active = self.valid_mask.astype(bool)
+            self.vnormal = self.vnormal.copy()
+            self.vshear = self.vshear.copy()
+            self.vnormal[~active] = 0
+            self.vshear[~active] = 0
+
         self.compute_deformation_valid = True
 
         return True
@@ -423,12 +571,12 @@ class Sensor:
             raise Exception("Warning: vnormal is required for contact computation")
 
         if multiple_foe:
-            result = self.__contact3d.compute_multiple_foe(vnormal_to_use)
+            result = self.__contact3d.compute_multiple_foe(vnormal_to_use, valid_mask=self.valid_mask)
         else:
             if self.cuda: 
-                result = self.__contact3d.compute_foe_gpu(vnormal_to_use)
+                result = self.__contact3d.compute_foe_gpu(vnormal_to_use, valid_mask=self.valid_mask)
             else:
-                result = self.__contact3d.compute_foe(vnormal_to_use)
+                result = self.__contact3d.compute_foe(vnormal_to_use, valid_mask=self.valid_mask)
 
         self.centroid = result['centroid']
         self.centroid_found = result['centroid_found']
@@ -436,6 +584,16 @@ class Sensor:
             self.contour = result['contour']
         self.depth_map = result['divergence_map']
 
+        if self.valid_mask is not None and self.centroid_found:
+            cx, cy = int(self.centroid[0]), int(self.centroid[1])
+            inside = (
+                0 <= cy < self.valid_mask.shape[0]
+                and 0 <= cx < self.valid_mask.shape[1]
+                and bool(self.valid_mask[cy, cx])
+            )
+            if not inside and self.verbose:
+                print("Warning: contact centroid lies outside the valid UV mask.")
+
         self.compute_contact_valid = True
         return result