"""Image block: ImageSettings and an independently runnable ImagePipeline. Stage-1 architecture (K8.1, K11.1, K11.3, task T05) separates the image side of measurement from :class:`~orisys.sensor.Sensor`: * :class:`ImageSettings` is the immutable description of one image configuration (optical-flow tracker settings, HHD grid/backend, flow EMA). * :class:`ImagePipeline` owns the tracker, the HHD solver and their scratch state. It turns a prepared grey frame into flow/HHD/field results, writes the optical-flow reference generation at the moment it computes a frame, and publishes measurement arrays as read-only copies. ``Sensor`` delegates to an ``ImagePipeline``; ``FingerRuntime`` reaches the same object through its tactile sensor, so all three published paths keep one image implementation. The class can also be used without a Sensor at all:: from orisys import configs from orisys.image_pipeline import ImagePipeline, ImageSettings, sensor_clock context = sensor.context # or build_sensor_context(...) settings = ImageSettings.from_config(configs.load_config("box")) pipeline = ImagePipeline(settings, context=context) measurement = pipeline.measure( prepared_grey_frame, identity=context.frame_identity(session_id="run-1", frame_id=0, source_frame_id=0), clock=sensor_clock(capture_time_ns, clock_id="host", basis="capture_monotonic"), ) print(measurement.force_raw, measurement.flow.shape, measurement.frame.reference_generation) Image preparation (stitch/unroll/resize/mirror) remains in the acquisition layer for now; the pipeline starts at the prepared grey frame, which is exactly what ``Sensor.compute_flow``/``compute_hhd`` have always consumed. """ from __future__ import annotations import time from collections.abc import Mapping, Sequence from dataclasses import dataclass from dataclasses import field as dataclass_field from typing import Any import cv2 import numpy as np from .hhd import nHHD from .stream_contract import ( FrameClock, FrameIdentity, FrameStamp, MeasurementFrame, TimeBasis, ) from .tracker import Tracker __all__ = [ "ImageDecomposition", "ImageMeasurement", "ImagePipeline", "ImageSettings", "sensor_clock", ] #: Arrays an independent measurement may request with ``include``. _INCLUDE_FIELDS = frozenset( {"image", "flow", "vnormal", "vshear", "field", "valid_mask"} ) def sensor_clock( capture_time_ns: int, *, clock_id: str, basis: TimeBasis = "capture_monotonic", ) -> FrameClock: """Convenience constructor for the capture clock of a prepared frame. ``basis`` is one of :data:`~orisys.stream_contract.TIME_BASES`; the value is validated by :class:`~orisys.stream_contract.FrameClock`. """ return FrameClock( capture_time_ns=int(capture_time_ns), clock_id=str(clock_id), capture_time_basis=basis, ) @dataclass(frozen=True) class ImageSettings: """One image configuration: tracker, HHD and flow-EMA settings. ``from_config`` reads the same config keys ``Sensor`` has always read. The flow EMA alpha is resolved through :class:`~orisys.force_readout.ForceReadout` so the existing filter-preset interpretation has exactly one implementation. """ tracker_config: Mapping[str, Any] cuda: bool verbose: bool hhd_downsample: int poisson_backend: str masked_poisson_solver: str masked_poisson_max_boundary: int masked_poisson_boundary_stride: int masked_poisson_max_iters: int masked_poisson_tol: float masked_poisson_reg: float flow_ema_alpha: float = 1.0 @classmethod def from_config( cls, config: Mapping[str, Any] | None, *, cuda: bool = True, backend: str = "auto", verbose: bool = False, flow_ema_alpha: float | None = None, ) -> ImageSettings: """Build settings from ``sensor.json``-shaped configuration.""" cfg: Mapping[str, Any] = config if isinstance(config, Mapping) else {} tracker = dict(cfg.get("tracker") or {}) raw_ds = tracker.get("hhd_downsample", 1) try: hhd_downsample = max(1, int(raw_ds)) if raw_ds is not None else 1 except (TypeError, ValueError): hhd_downsample = 1 if flow_ema_alpha is None: # Reuse the existing filter-preset resolution instead of a second # interpretation of the filters block (that block owns the EMA). from .force_readout import ForceReadout flow_ema_alpha = ForceReadout(cfg.get("filters")).flow_ema_alpha backend_from_config = tracker.get("backend", None) poisson_backend = str( backend_from_config if backend_from_config is not None else backend ) return cls( tracker_config=tracker, cuda=bool(cuda), verbose=bool(verbose), hhd_downsample=hhd_downsample, poisson_backend=poisson_backend, masked_poisson_solver=str(tracker.get("masked_poisson_solver", "sparse")), masked_poisson_max_boundary=int( tracker.get("masked_poisson_max_boundary", 96) ), masked_poisson_boundary_stride=int( tracker.get("masked_poisson_boundary_stride", 1) ), masked_poisson_max_iters=int(tracker.get("masked_poisson_max_iters", 3)), masked_poisson_tol=float(tracker.get("masked_poisson_tol", 1e-4)), masked_poisson_reg=float(tracker.get("masked_poisson_reg", 1e-8)), flow_ema_alpha=float(flow_ema_alpha), ) def as_dict(self) -> dict[str, Any]: """Stable identity of the image settings, for records and comparisons.""" return { "tracker": dict(self.tracker_config), "cuda": self.cuda, "hhd_downsample": self.hhd_downsample, "poisson_backend": self.poisson_backend, "masked_poisson_solver": self.masked_poisson_solver, "masked_poisson_max_boundary": self.masked_poisson_max_boundary, "masked_poisson_boundary_stride": self.masked_poisson_boundary_stride, "masked_poisson_max_iters": self.masked_poisson_max_iters, "masked_poisson_tol": self.masked_poisson_tol, "masked_poisson_reg": self.masked_poisson_reg, "flow_ema_alpha": self.flow_ema_alpha, } @dataclass(frozen=True) class ImageDecomposition: """Flow/HHD result for one prepared frame (internal state objects, not copies).""" flow: np.ndarray vnormal: np.ndarray vshear: np.ndarray force_raw: tuple[float, float, float] torque: float valid_mask_hhd: np.ndarray | None def _readonly_copy(value: Any) -> Any: """Copy arrays (recursively for the field payload) and freeze them.""" if isinstance(value, np.ndarray): out = np.array(value, copy=True) out.setflags(write=False) return out if isinstance(value, dict): return {key: _readonly_copy(item) for key, item in value.items()} if isinstance(value, (list, tuple)): return type(value)(_readonly_copy(item) for item in value) return value @dataclass(frozen=True) class ImageMeasurement: """One image measurement: T03 header plus read-only flow/HHD/field payload.""" frame: MeasurementFrame force_raw: tuple[float, float, float] torque: float flow: np.ndarray | None = None vnormal: np.ndarray | None = None vshear: np.ndarray | None = None valid_mask: np.ndarray | None = None field: dict | None = None image: np.ndarray | None = None elapsed_s: float = 0.0 diagnostics: tuple[tuple[str, str | float], ...] = dataclass_field( default_factory=tuple ) class ImagePipeline: """Own the image half of measurement: tracker, HHD, field and reference version. Thread ownership mirrors stage 0: one instance is used by one image thread. The reference version is **not counted here**: the stream that owns it (the stage-0 ``StreamPipeline`` reset protocol) adopts it through :meth:`set_reference_generation` or passes it to :meth:`measure`; the image block only stamps what it was told. :meth:`reset_image_state` rebases the tracker/flow EMA and never invents a version. The force/contact/model half stays in ``Sensor``. """ def __init__( self, settings: ImageSettings, *, context: Any = None, verbose: bool = False, flow_ema: Any = None, ) -> None: from .force_readout import ForceReadout self.settings = settings self.context = context self.verbose = bool(verbose) # One owner for the flow-EMA coefficient *and* its history: the published # ForceReadout when a Sensor binds this pipeline, otherwise a private one. # Sensor.configure(flow_ema_alpha=...) therefore keeps working on the next # frame, exactly as it did before the image block was split out. self._flow_ema = ( flow_ema if flow_ema is not None else ForceReadout({"flow_ema_alpha": settings.flow_ema_alpha}) ) self._tracker = Tracker( adaptive=True, cuda=settings.cuda, verbose=settings.verbose, config_tracker=dict(settings.tracker_config), ) self._hhd: nHHD | None = None self._hhd_grid_shape: tuple[int, int] | None = None self._hhd_ref_shape: tuple[int, int] | None = None self._valid_mask_hhd: np.ndarray | None = None self._valid_mask: np.ndarray | None = None self._flow_ds_buf: np.ndarray | None = None self._vn_buf: np.ndarray | None = None self._vs_buf: np.ndarray | None = None self._mask_f32: np.ndarray | None = None self._flow: np.ndarray | None = None self._vnormal: np.ndarray | None = None self._vshear: np.ndarray | None = None self._force_raw: tuple[float, float, float] = (0.0, 0.0, 0.0) self._torque = 0.0 self._reference_generation = 0 # ------------------------------------------------------------- read access @property def flow(self) -> np.ndarray | None: return self._flow @property def vnormal(self) -> np.ndarray | None: return self._vnormal @property def vshear(self) -> np.ndarray | None: return self._vshear @property def valid_mask_hhd(self) -> np.ndarray | None: return self._valid_mask_hhd @property def force_raw(self) -> tuple[float, float, float]: return self._force_raw @property def torque(self) -> float: return self._torque @property def reference_generation(self) -> int: """Reference version most recently adopted from the owning stream.""" return int(self._reference_generation) def set_reference_generation(self, generation: int) -> None: """Adopt the stream's reference version (called on the image thread). The stage-0 ``StreamPipeline`` claims the version and passes it here before computing the frame's flow; ``measure`` may stamp the same value explicitly for an independent experiment. This class never increments it. """ if isinstance(generation, bool) or not isinstance(generation, int): raise TypeError("reference_generation must be an integer") if generation < 0: raise ValueError("reference_generation must be >= 0") self._reference_generation = int(generation) @property def flow_ema_alpha(self) -> float: """Runtime flow-EMA coefficient (owned by the readout, not this pipeline).""" return float(self._flow_ema.flow_ema_alpha) # ----------------------------------------------------------- reset seams def reset_image_state(self) -> None: """Rebase the tracker and drop the flow-EMA history (image thread only). A reference version is not claimed here: the owning stream sets it with :meth:`set_reference_generation`; this method only resets image state. """ self._tracker.reset() self._flow_ema.reset_flow_ema() def reset_tracker(self) -> None: """Drop tracker/EMA history without touching the reference identity. This is the frame-boundary zeroing half used by a force session before a baseline window: the reference frame itself is unchanged. """ self._tracker.reset() self._flow_ema.reset_flow_ema() # ------------------------------------------------------------- flow / HHD def _apply_flow_ema(self, flow: np.ndarray | None) -> np.ndarray | None: """Temporal flow smoothing, delegated to its single owner (ForceReadout).""" return self._flow_ema.apply_flow_ema(flow) def _compute_flow_state( self, image: np.ndarray, *, valid_mask: np.ndarray | None = None ) -> np.ndarray: """Flow for one frame (the caller has already adopted the version).""" flow = self._tracker.track(image, valid_mask=valid_mask) if hasattr(flow, "download"): flow = flow.download() flow = self._apply_flow_ema(flow) self._flow = flow return flow def compute_flow( self, image: np.ndarray, *, valid_mask: np.ndarray | None = None, reference_generation: int | None = None, ) -> np.ndarray: """Compute and return this frame's flow (one call per frame).""" if image is None: raise Exception("Warning: Get image first") if reference_generation is not None: self.set_reference_generation(reference_generation) return self._compute_flow_state(image, valid_mask=valid_mask) def _resize_mask( self, mask: np.ndarray | None, size: tuple[int, int] ) -> np.ndarray | None: if mask is None: return None mask_u8 = np.asarray(mask, dtype=np.uint8) resized = cv2.resize(mask_u8, size, interpolation=cv2.INTER_NEAREST) return resized.astype(bool) def _hhd_grid_size(self) -> tuple[int, int] | None: mask = self._valid_mask if mask is None: return None h, w = mask.shape[:2] k = max(1, int(self.settings.hhd_downsample)) if k <= 1: return (w, h) return (max(1, w // k), max(1, h // k)) def decompose( self, flow: np.ndarray | None = None, *, valid_mask: np.ndarray | None = None ) -> ImageDecomposition: """Run HHD on one flow field and publish normal/shear/force results.""" flow = self._flow if flow is None else flow if flow is None: raise Exception("Warning: Compute flow first") self._valid_mask = valid_mask orig_h, orig_w = flow.shape[:2] try: k = max(1, int(self.settings.hhd_downsample)) except (TypeError, ValueError): k = 1 mask_ds = None if valid_mask is not None: ds_size = self._hhd_grid_size() if ( self._valid_mask_hhd is not None and ds_size is not None and self._valid_mask_hhd.shape[1] == ds_size[0] and self._valid_mask_hhd.shape[0] == ds_size[1] ): mask_ds = self._valid_mask_hhd else: mask_ds = np.asarray(valid_mask, dtype=bool) if k > 1: ds_w = max(1, orig_w // k) ds_h = max(1, orig_h // k) flow_ds = self._flow_ds_buf if ( flow_ds is None or flow_ds.shape != (ds_h, ds_w, 2) or flow_ds.dtype != np.float32 ): flow_ds = np.empty((ds_h, ds_w, 2), dtype=np.float32) self._flow_ds_buf = flow_ds cv2.resize(flow, (ds_w, ds_h), dst=flow_ds, interpolation=cv2.INTER_AREA) if mask_ds is not None and ( mask_ds.shape[1] != ds_w or mask_ds.shape[0] != ds_h ): mask_ds = self._resize_mask(mask_ds, (ds_w, ds_h)) else: flow_ds = flow ds_shape = flow_ds.shape[:2] ref_shape = (orig_h, orig_w) self._valid_mask_hhd = mask_ds if ( self._hhd is None or self._hhd_grid_shape != ds_shape or self._hhd_ref_shape != ref_shape ): spacings = (18 * k, 18 * k) self._hhd = nHHD( cuda=self.settings.cuda, grid=ds_shape, spacings=spacings, poisson_backend=self.settings.poisson_backend, debug=self.settings.verbose, force_reference_grid=ref_shape, masked_poisson_solver=self.settings.masked_poisson_solver, masked_poisson_max_boundary=self.settings.masked_poisson_max_boundary, masked_poisson_boundary_stride=self.settings.masked_poisson_boundary_stride, masked_poisson_max_iters=self.settings.masked_poisson_max_iters, masked_poisson_tol=self.settings.masked_poisson_tol, masked_poisson_reg=self.settings.masked_poisson_reg, ) self._hhd_grid_shape = ds_shape self._hhd_ref_shape = ref_shape _, forces = self._hhd.get_force(flow_ds, valid_mask=mask_ds) fn, fx, fy, ftor = forces self._force_raw = (float(fn), float(fx), float(fy)) self._torque = float(ftor) vnormal_ds = self._hhd.get_normal() vshear_ds = self._hhd.get_shear() if k > 1: vn = self._vn_buf vs = self._vs_buf if vn is None or vn.shape != (orig_h, orig_w, 2) or vn.dtype != np.float32: vn = np.empty((orig_h, orig_w, 2), dtype=np.float32) vs = np.empty((orig_h, orig_w, 2), dtype=np.float32) self._vn_buf = vn self._vs_buf = vs cv2.resize( vnormal_ds, (orig_w, orig_h), dst=vn, interpolation=cv2.INTER_LINEAR ) cv2.resize( vshear_ds, (orig_w, orig_h), dst=vs, interpolation=cv2.INTER_LINEAR ) self._vnormal = vn self._vshear = vs else: self._vnormal = vnormal_ds.copy() self._vshear = vshear_ds.copy() if valid_mask is not None: m3 = self._mask_f32 if m3 is None or m3.shape != (orig_h, orig_w, 1) or m3.dtype != np.float32: m3 = np.empty((orig_h, orig_w, 1), dtype=np.float32) self._mask_f32 = m3 np.copyto(m3[:, :, 0], valid_mask, casting="unsafe") np.multiply(self._vnormal, m3, out=self._vnormal) np.multiply(self._vshear, m3, out=self._vshear) return ImageDecomposition( flow=self._flow, vnormal=self._vnormal, vshear=self._vshear, force_raw=self._force_raw, torque=self._torque, valid_mask_hhd=self._valid_mask_hhd, ) def current_field(self) -> dict: """Same-frame HHD field payload (the solver returns a private copy).""" if self._hhd is None: raise RuntimeError("compute_hhd() before reading the field") return self._hhd.get_field(valid_mask=self._valid_mask_hhd) # ------------------------------------------------------------ measurement def measure( self, image: np.ndarray, *, identity: FrameIdentity, clock: FrameClock, source_dt_s: float | None = None, valid_mask: np.ndarray | None = None, include: Sequence[str] = ("flow",), want_field: bool = False, reference_generation: int | None = None, ) -> ImageMeasurement: """Measure one prepared frame once and return read-only payload + T03 header. ``reference_generation`` is the version the owning stream claimed for this frame; when omitted the last adopted version is used. The version is fixed before flow is computed: a newer version adopted while this frame is in flight does not relabel it. """ unknown = set(include) - _INCLUDE_FIELDS if unknown: raise ValueError( f"unknown include field(s) {sorted(unknown)}; " f"expected a subset of {sorted(_INCLUDE_FIELDS)}" ) started = time.perf_counter() if reference_generation is not None: self.set_reference_generation(reference_generation) generated = int(self._reference_generation) self._compute_flow_state(image, valid_mask=valid_mask) decomposition = self.decompose(self._flow, valid_mask=valid_mask) stamp = FrameStamp( sensor_id=identity.sensor_id, session_id=identity.session_id, frame_id=identity.frame_id, source_frame_id=identity.source_frame_id, capture_time_ns=clock.capture_time_ns, clock_id=clock.clock_id, capture_time_basis=clock.capture_time_basis, source_pts=clock.source_pts, acquisition_id=identity.acquisition_id, sync_group_id=identity.sync_group_id, ) frame = MeasurementFrame( stamp=stamp, source_dt_s=source_dt_s, reference_generation=generated ) include_set = set(include) payload = { "flow": _readonly_copy(decomposition.flow) if "flow" in include_set else None, "vnormal": _readonly_copy(decomposition.vnormal) if "vnormal" in include_set else None, "vshear": _readonly_copy(decomposition.vshear) if "vshear" in include_set else None, "valid_mask": _readonly_copy(decomposition.valid_mask_hhd) if "valid_mask" in include_set else None, "field": _readonly_copy(self.current_field()) if ("field" in include_set or want_field) else None, "image": _readonly_copy(image) if "image" in include_set else None, } return ImageMeasurement( frame=frame, force_raw=decomposition.force_raw, torque=decomposition.torque, flow=payload["flow"], vnormal=payload["vnormal"], vshear=payload["vshear"], valid_mask=payload["valid_mask"], field=payload["field"], image=payload["image"], elapsed_s=time.perf_counter() - started, diagnostics=(("reference_generation", generated),), )