orisys-tech/orisys-sdk
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 将图像块拆分为可独立运行的 ImagePipeline

Browse Source 
光流、HHD 和场从 Sensor 中抽到 ImageSettings/ImagePipeline，Sensor 与 FingerRuntime 改为委托，公开接口与数值不变（222 个基线数组逐位一致，含运行时 flow_ema_alpha 场景）。参考版本只有一个来源：StreamPipeline 在 flow 线程领取的版本逐帧交给图像块。对外数组为只读副本；输出带 stream-contract 的 MeasurementFrame 头。回归 357 项、lint 通过。
This commit is contained in:
ykcheungab
2026-10-21 13:35:02 +08:00
parent 474dbf5a64
commit e3b60f3d6d
6 changed files with 1306 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/orisys/finger/runtime.py
+12
View File
@@ -546,6 +546,18 @@ class FingerRuntime:
return StreamPipeline.for_finger_runtime(self, **kwargs)
@property
def image_pipeline(self):
"""The ImagePipeline the tactile sensor delegates its image block to.
Capture and UV unrolling stay in this runtime; flow/HHD/field come from
the tactile sensor's single ``ImagePipeline``, so an offline experiment
can inspect or reuse the same ``ImageSettings``/``SensorContext`` without
a running stream.
"""
raw = getattr(self.tactile, "raw_sensor", None)
return None if raw is None else getattr(raw, "image_pipeline", None)
def close(self):
if self.capture_source is not None:
self.capture_source.close()
src/orisys/image_pipeline.py
+592
View File
@@ -0,0 +1,592 @@
"""Image block: ImageSettings and an independently runnable ImagePipeline.
Stage-1 architecture (K8.1, K11.1, K11.3, task T05) separates the image side of
measurement from :class:`~orisys.sensor.Sensor`:
* :class:`ImageSettings` is the immutable description of one image
configuration (optical-flow tracker settings, HHD grid/backend, flow EMA).
* :class:`ImagePipeline` owns the tracker, the HHD solver and their scratch
state. It turns a prepared grey frame into flow/HHD/field results, writes the
optical-flow reference generation at the moment it computes a frame, and
publishes measurement arrays as read-only copies.
``Sensor`` delegates to an ``ImagePipeline``; ``FingerRuntime`` reaches the same
object through its tactile sensor, so all three published paths keep one image
implementation. The class can also be used without a Sensor at all::
from orisys import configs
from orisys.image_pipeline import ImagePipeline, ImageSettings, sensor_clock
context = sensor.context # or build_sensor_context(...)
settings = ImageSettings.from_config(configs.load_config("box"))
pipeline = ImagePipeline(settings, context=context)
measurement = pipeline.measure(
prepared_grey_frame,
identity=context.frame_identity(session_id="run-1", frame_id=0, source_frame_id=0),
clock=sensor_clock(capture_time_ns, clock_id="host", basis="capture_monotonic"),
)
print(measurement.force_raw, measurement.flow.shape, measurement.frame.reference_generation)
Image preparation (stitch/unroll/resize/mirror) remains in the acquisition layer
for now; the pipeline starts at the prepared grey frame, which is exactly what
``Sensor.compute_flow``/``compute_hhd`` have always consumed.
"""
from __future__ import annotations
import time
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from dataclasses import field as dataclass_field
from typing import Any
import cv2
import numpy as np
from .hhd import nHHD
from .stream_contract import (
FrameClock,
FrameIdentity,
FrameStamp,
MeasurementFrame,
TimeBasis,
)
from .tracker import Tracker
__all__ = [
"ImageDecomposition",
"ImageMeasurement",
"ImagePipeline",
"ImageSettings",
"sensor_clock",
]
#: Arrays an independent measurement may request with ``include``.
_INCLUDE_FIELDS = frozenset(
{"image", "flow", "vnormal", "vshear", "field", "valid_mask"}
)
def sensor_clock(
capture_time_ns: int,
*,
clock_id: str,
basis: TimeBasis = "capture_monotonic",
) -> FrameClock:
"""Convenience constructor for the capture clock of a prepared frame.
``basis`` is one of :data:`~orisys.stream_contract.TIME_BASES`; the value is
validated by :class:`~orisys.stream_contract.FrameClock`.
"""
return FrameClock(
capture_time_ns=int(capture_time_ns),
clock_id=str(clock_id),
capture_time_basis=basis,
)
@dataclass(frozen=True)
class ImageSettings:
"""One image configuration: tracker, HHD and flow-EMA settings.
``from_config`` reads the same config keys ``Sensor`` has always read. The
flow EMA alpha is resolved through :class:`~orisys.force_readout.ForceReadout`
so the existing filter-preset interpretation has exactly one implementation.
"""
tracker_config: Mapping[str, Any]
cuda: bool
verbose: bool
hhd_downsample: int
poisson_backend: str
masked_poisson_solver: str
masked_poisson_max_boundary: int
masked_poisson_boundary_stride: int
masked_poisson_max_iters: int
masked_poisson_tol: float
masked_poisson_reg: float
flow_ema_alpha: float = 1.0
@classmethod
def from_config(
cls,
config: Mapping[str, Any] | None,
*,
cuda: bool = True,
backend: str = "auto",
verbose: bool = False,
flow_ema_alpha: float | None = None,
) -> ImageSettings:
"""Build settings from ``sensor.json``-shaped configuration."""
cfg: Mapping[str, Any] = config if isinstance(config, Mapping) else {}
tracker = dict(cfg.get("tracker") or {})
raw_ds = tracker.get("hhd_downsample", 1)
try:
hhd_downsample = max(1, int(raw_ds)) if raw_ds is not None else 1
except (TypeError, ValueError):
hhd_downsample = 1
if flow_ema_alpha is None:
# Reuse the existing filter-preset resolution instead of a second
# interpretation of the filters block (that block owns the EMA).
from .force_readout import ForceReadout
flow_ema_alpha = ForceReadout(cfg.get("filters")).flow_ema_alpha
backend_from_config = tracker.get("backend", None)
poisson_backend = str(
backend_from_config if backend_from_config is not None else backend
)
return cls(
tracker_config=tracker,
cuda=bool(cuda),
verbose=bool(verbose),
hhd_downsample=hhd_downsample,
poisson_backend=poisson_backend,
masked_poisson_solver=str(tracker.get("masked_poisson_solver", "sparse")),
masked_poisson_max_boundary=int(
tracker.get("masked_poisson_max_boundary", 96)
),
masked_poisson_boundary_stride=int(
tracker.get("masked_poisson_boundary_stride", 1)
),
masked_poisson_max_iters=int(tracker.get("masked_poisson_max_iters", 3)),
masked_poisson_tol=float(tracker.get("masked_poisson_tol", 1e-4)),
masked_poisson_reg=float(tracker.get("masked_poisson_reg", 1e-8)),
flow_ema_alpha=float(flow_ema_alpha),
)
def as_dict(self) -> dict[str, Any]:
"""Stable identity of the image settings, for records and comparisons."""
return {
"tracker": dict(self.tracker_config),
"cuda": self.cuda,
"hhd_downsample": self.hhd_downsample,
"poisson_backend": self.poisson_backend,
"masked_poisson_solver": self.masked_poisson_solver,
"masked_poisson_max_boundary": self.masked_poisson_max_boundary,
"masked_poisson_boundary_stride": self.masked_poisson_boundary_stride,
"masked_poisson_max_iters": self.masked_poisson_max_iters,
"masked_poisson_tol": self.masked_poisson_tol,
"masked_poisson_reg": self.masked_poisson_reg,
"flow_ema_alpha": self.flow_ema_alpha,
}
@dataclass(frozen=True)
class ImageDecomposition:
"""Flow/HHD result for one prepared frame (internal state objects, not copies)."""
flow: np.ndarray
vnormal: np.ndarray
vshear: np.ndarray
force_raw: tuple[float, float, float]
torque: float
valid_mask_hhd: np.ndarray | None
def _readonly_copy(value: Any) -> Any:
"""Copy arrays (recursively for the field payload) and freeze them."""
if isinstance(value, np.ndarray):
out = np.array(value, copy=True)
out.setflags(write=False)
return out
if isinstance(value, dict):
return {key: _readonly_copy(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return type(value)(_readonly_copy(item) for item in value)
return value
@dataclass(frozen=True)
class ImageMeasurement:
"""One image measurement: T03 header plus read-only flow/HHD/field payload."""
frame: MeasurementFrame
force_raw: tuple[float, float, float]
torque: float
flow: np.ndarray | None = None
vnormal: np.ndarray | None = None
vshear: np.ndarray | None = None
valid_mask: np.ndarray | None = None
field: dict | None = None
image: np.ndarray | None = None
elapsed_s: float = 0.0
diagnostics: tuple[tuple[str, str | float], ...] = dataclass_field(
default_factory=tuple
)
class ImagePipeline:
"""Own the image half of measurement: tracker, HHD, field and reference version.
Thread ownership mirrors stage 0: one instance is used by one image thread.
The reference version is **not counted here**: the stream that owns it (the
stage-0 ``StreamPipeline`` reset protocol) adopts it through
:meth:`set_reference_generation` or passes it to :meth:`measure`; the image
block only stamps what it was told. :meth:`reset_image_state` rebases the
tracker/flow EMA and never invents a version. The force/contact/model half
stays in ``Sensor``.
"""
def __init__(
self,
settings: ImageSettings,
*,
context: Any = None,
verbose: bool = False,
flow_ema: Any = None,
) -> None:
from .force_readout import ForceReadout
self.settings = settings
self.context = context
self.verbose = bool(verbose)
# One owner for the flow-EMA coefficient *and* its history: the published
# ForceReadout when a Sensor binds this pipeline, otherwise a private one.
# Sensor.configure(flow_ema_alpha=...) therefore keeps working on the next
# frame, exactly as it did before the image block was split out.
self._flow_ema = (
flow_ema
if flow_ema is not None
else ForceReadout({"flow_ema_alpha": settings.flow_ema_alpha})
)
self._tracker = Tracker(
adaptive=True,
cuda=settings.cuda,
verbose=settings.verbose,
config_tracker=dict(settings.tracker_config),
)
self._hhd: nHHD | None = None
self._hhd_grid_shape: tuple[int, int] | None = None
self._hhd_ref_shape: tuple[int, int] | None = None
self._valid_mask_hhd: np.ndarray | None = None
self._valid_mask: np.ndarray | None = None
self._flow_ds_buf: np.ndarray | None = None
self._vn_buf: np.ndarray | None = None
self._vs_buf: np.ndarray | None = None
self._mask_f32: np.ndarray | None = None
self._flow: np.ndarray | None = None
self._vnormal: np.ndarray | None = None
self._vshear: np.ndarray | None = None
self._force_raw: tuple[float, float, float] = (0.0, 0.0, 0.0)
self._torque = 0.0
self._reference_generation = 0
# ------------------------------------------------------------- read access
@property
def flow(self) -> np.ndarray | None:
return self._flow
@property
def vnormal(self) -> np.ndarray | None:
return self._vnormal
@property
def vshear(self) -> np.ndarray | None:
return self._vshear
@property
def valid_mask_hhd(self) -> np.ndarray | None:
return self._valid_mask_hhd
@property
def force_raw(self) -> tuple[float, float, float]:
return self._force_raw
@property
def torque(self) -> float:
return self._torque
@property
def reference_generation(self) -> int:
"""Reference version most recently adopted from the owning stream."""
return int(self._reference_generation)
def set_reference_generation(self, generation: int) -> None:
"""Adopt the stream's reference version (called on the image thread).
The stage-0 ``StreamPipeline`` claims the version and passes it here
before computing the frame's flow; ``measure`` may stamp the same value
explicitly for an independent experiment. This class never increments it.
"""
if isinstance(generation, bool) or not isinstance(generation, int):
raise TypeError("reference_generation must be an integer")
if generation < 0:
raise ValueError("reference_generation must be >= 0")
self._reference_generation = int(generation)
@property
def flow_ema_alpha(self) -> float:
"""Runtime flow-EMA coefficient (owned by the readout, not this pipeline)."""
return float(self._flow_ema.flow_ema_alpha)
# ----------------------------------------------------------- reset seams
def reset_image_state(self) -> None:
"""Rebase the tracker and drop the flow-EMA history (image thread only).
A reference version is not claimed here: the owning stream sets it with
:meth:`set_reference_generation`; this method only resets image state.
"""
self._tracker.reset()
self._flow_ema.reset_flow_ema()
def reset_tracker(self) -> None:
"""Drop tracker/EMA history without touching the reference identity.
This is the frame-boundary zeroing half used by a force session before a
baseline window: the reference frame itself is unchanged.
"""
self._tracker.reset()
self._flow_ema.reset_flow_ema()
# ------------------------------------------------------------- flow / HHD
def _apply_flow_ema(self, flow: np.ndarray | None) -> np.ndarray | None:
"""Temporal flow smoothing, delegated to its single owner (ForceReadout)."""
return self._flow_ema.apply_flow_ema(flow)
def _compute_flow_state(
self, image: np.ndarray, *, valid_mask: np.ndarray | None = None
) -> np.ndarray:
"""Flow for one frame (the caller has already adopted the version)."""
flow = self._tracker.track(image, valid_mask=valid_mask)
if hasattr(flow, "download"):
flow = flow.download()
flow = self._apply_flow_ema(flow)
self._flow = flow
return flow
def compute_flow(
self,
image: np.ndarray,
*,
valid_mask: np.ndarray | None = None,
reference_generation: int | None = None,
) -> np.ndarray:
"""Compute and return this frame's flow (one call per frame)."""
if image is None:
raise Exception("Warning: Get image first")
if reference_generation is not None:
self.set_reference_generation(reference_generation)
return self._compute_flow_state(image, valid_mask=valid_mask)
def _resize_mask(
self, mask: np.ndarray | None, size: tuple[int, int]
) -> np.ndarray | None:
if mask is None:
return None
mask_u8 = np.asarray(mask, dtype=np.uint8)
resized = cv2.resize(mask_u8, size, interpolation=cv2.INTER_NEAREST)
return resized.astype(bool)
def _hhd_grid_size(self) -> tuple[int, int] | None:
mask = self._valid_mask
if mask is None:
return None
h, w = mask.shape[:2]
k = max(1, int(self.settings.hhd_downsample))
if k <= 1:
return (w, h)
return (max(1, w // k), max(1, h // k))
def decompose(
self, flow: np.ndarray | None = None, *, valid_mask: np.ndarray | None = None
) -> ImageDecomposition:
"""Run HHD on one flow field and publish normal/shear/force results."""
flow = self._flow if flow is None else flow
if flow is None:
raise Exception("Warning: Compute flow first")
self._valid_mask = valid_mask
orig_h, orig_w = flow.shape[:2]
try:
k = max(1, int(self.settings.hhd_downsample))
except (TypeError, ValueError):
k = 1
mask_ds = None
if valid_mask is not None:
ds_size = self._hhd_grid_size()
if (
self._valid_mask_hhd is not None
and ds_size is not None
and self._valid_mask_hhd.shape[1] == ds_size[0]
and self._valid_mask_hhd.shape[0] == ds_size[1]
):
mask_ds = self._valid_mask_hhd
else:
mask_ds = np.asarray(valid_mask, dtype=bool)
if k > 1:
ds_w = max(1, orig_w // k)
ds_h = max(1, orig_h // k)
flow_ds = self._flow_ds_buf
if (
flow_ds is None
or flow_ds.shape != (ds_h, ds_w, 2)
or flow_ds.dtype != np.float32
):
flow_ds = np.empty((ds_h, ds_w, 2), dtype=np.float32)
self._flow_ds_buf = flow_ds
cv2.resize(flow, (ds_w, ds_h), dst=flow_ds, interpolation=cv2.INTER_AREA)
if mask_ds is not None and (
mask_ds.shape[1] != ds_w or mask_ds.shape[0] != ds_h
):
mask_ds = self._resize_mask(mask_ds, (ds_w, ds_h))
else:
flow_ds = flow
ds_shape = flow_ds.shape[:2]
ref_shape = (orig_h, orig_w)
self._valid_mask_hhd = mask_ds
if (
self._hhd is None
or self._hhd_grid_shape != ds_shape
or self._hhd_ref_shape != ref_shape
):
spacings = (18 * k, 18 * k)
self._hhd = nHHD(
cuda=self.settings.cuda,
grid=ds_shape,
spacings=spacings,
poisson_backend=self.settings.poisson_backend,
debug=self.settings.verbose,
force_reference_grid=ref_shape,
masked_poisson_solver=self.settings.masked_poisson_solver,
masked_poisson_max_boundary=self.settings.masked_poisson_max_boundary,
masked_poisson_boundary_stride=self.settings.masked_poisson_boundary_stride,
masked_poisson_max_iters=self.settings.masked_poisson_max_iters,
masked_poisson_tol=self.settings.masked_poisson_tol,
masked_poisson_reg=self.settings.masked_poisson_reg,
)
self._hhd_grid_shape = ds_shape
self._hhd_ref_shape = ref_shape
_, forces = self._hhd.get_force(flow_ds, valid_mask=mask_ds)
fn, fx, fy, ftor = forces
self._force_raw = (float(fn), float(fx), float(fy))
self._torque = float(ftor)
vnormal_ds = self._hhd.get_normal()
vshear_ds = self._hhd.get_shear()
if k > 1:
vn = self._vn_buf
vs = self._vs_buf
if vn is None or vn.shape != (orig_h, orig_w, 2) or vn.dtype != np.float32:
vn = np.empty((orig_h, orig_w, 2), dtype=np.float32)
vs = np.empty((orig_h, orig_w, 2), dtype=np.float32)
self._vn_buf = vn
self._vs_buf = vs
cv2.resize(
vnormal_ds, (orig_w, orig_h), dst=vn, interpolation=cv2.INTER_LINEAR
)
cv2.resize(
vshear_ds, (orig_w, orig_h), dst=vs, interpolation=cv2.INTER_LINEAR
)
self._vnormal = vn
self._vshear = vs
else:
self._vnormal = vnormal_ds.copy()
self._vshear = vshear_ds.copy()
if valid_mask is not None:
m3 = self._mask_f32
if m3 is None or m3.shape != (orig_h, orig_w, 1) or m3.dtype != np.float32:
m3 = np.empty((orig_h, orig_w, 1), dtype=np.float32)
self._mask_f32 = m3
np.copyto(m3[:, :, 0], valid_mask, casting="unsafe")
np.multiply(self._vnormal, m3, out=self._vnormal)
np.multiply(self._vshear, m3, out=self._vshear)
return ImageDecomposition(
flow=self._flow,
vnormal=self._vnormal,
vshear=self._vshear,
force_raw=self._force_raw,
torque=self._torque,
valid_mask_hhd=self._valid_mask_hhd,
)
def current_field(self) -> dict:
"""Same-frame HHD field payload (the solver returns a private copy)."""
if self._hhd is None:
raise RuntimeError("compute_hhd() before reading the field")
return self._hhd.get_field(valid_mask=self._valid_mask_hhd)
# ------------------------------------------------------------ measurement
def measure(
self,
image: np.ndarray,
*,
identity: FrameIdentity,
clock: FrameClock,
source_dt_s: float | None = None,
valid_mask: np.ndarray | None = None,
include: Sequence[str] = ("flow",),
want_field: bool = False,
reference_generation: int | None = None,
) -> ImageMeasurement:
"""Measure one prepared frame once and return read-only payload + T03 header.
``reference_generation`` is the version the owning stream claimed for this
frame; when omitted the last adopted version is used. The version is fixed
before flow is computed: a newer version adopted while this frame is in
flight does not relabel it.
"""
unknown = set(include) - _INCLUDE_FIELDS
if unknown:
raise ValueError(
f"unknown include field(s) {sorted(unknown)}; "
f"expected a subset of {sorted(_INCLUDE_FIELDS)}"
)
started = time.perf_counter()
if reference_generation is not None:
self.set_reference_generation(reference_generation)
generated = int(self._reference_generation)
self._compute_flow_state(image, valid_mask=valid_mask)
decomposition = self.decompose(self._flow, valid_mask=valid_mask)
stamp = FrameStamp(
sensor_id=identity.sensor_id,
session_id=identity.session_id,
frame_id=identity.frame_id,
source_frame_id=identity.source_frame_id,
capture_time_ns=clock.capture_time_ns,
clock_id=clock.clock_id,
capture_time_basis=clock.capture_time_basis,
source_pts=clock.source_pts,
acquisition_id=identity.acquisition_id,
sync_group_id=identity.sync_group_id,
)
frame = MeasurementFrame(
stamp=stamp, source_dt_s=source_dt_s, reference_generation=generated
)
include_set = set(include)
payload = {
"flow": _readonly_copy(decomposition.flow)
if "flow" in include_set
else None,
"vnormal": _readonly_copy(decomposition.vnormal)
if "vnormal" in include_set
else None,
"vshear": _readonly_copy(decomposition.vshear)
if "vshear" in include_set
else None,
"valid_mask": _readonly_copy(decomposition.valid_mask_hhd)
if "valid_mask" in include_set
else None,
"field": _readonly_copy(self.current_field())
if ("field" in include_set or want_field)
else None,
"image": _readonly_copy(image) if "image" in include_set else None,
}
return ImageMeasurement(
frame=frame,
force_raw=decomposition.force_raw,
torque=decomposition.torque,
flow=payload["flow"],
vnormal=payload["vnormal"],
vshear=payload["vshear"],
valid_mask=payload["valid_mask"],
field=payload["field"],
image=payload["image"],
elapsed_s=time.perf_counter() - started,
diagnostics=(("reference_generation", generated),),
)
src/orisys/pipeline.py
+9
View File
@@ -636,6 +636,15 @@ class StreamPipeline:
t0 = time.perf_counter()
# The frame is passed explicitly: the SDK's self.img slot is shared
# with the producer, so it must not be the transport.
# One reference-version source: the version claimed by this stage is
# handed to an image block that supports adopting it, so the block
# stamps the same number it will publish on the snapshot. Sensors
# without the optional capability keep the historical call.
adopt_reference = getattr(
self.sensor, "set_stream_reference_generation", None
)
if adopt_reference is not None:
adopt_reference(reference_version)
self.sensor.compute_flow(frame)
flow_s = time.perf_counter() - t0
flow = getattr(self.sensor, "flow", None)
src/orisys/sensor.py
+129 -54
View File
@@ -28,11 +28,11 @@ from .frame_processors import create_frame_processor
from .field_calibration import FIELD_PROFILE
from .get_depth import Contact3D
from .hhd import nHHD
from .image_pipeline import ImagePipeline, ImageSettings
# from threading import Thread, Event
# from queue import Queue, Empty
from .Stitcher import Stitcher
from .tracker import Tracker
from .capture import (
CaptureConfig,
CaptureSource,
@@ -296,25 +296,38 @@ class Sensor:
else None
)
self.frame_processor = create_frame_processor(self.sensor_type, stitcher=self.stitcher)
self.__tracker = Tracker(adaptive=True, cuda=cuda, verbose=self.verbose, config_tracker=self.__config_tracker)
# The image block (tracker + HHD + flow EMA) lives in ImagePipeline. The
# attributes below stay as compatibility mirrors for code that reads
# them; ``__tracker``/``__hhd`` are only used by the direct (bare
# Sensor/test-double) fallback path below.
self.image_settings = ImageSettings.from_config(
self._config,
cuda=bool(cuda),
backend=backend,
verbose=self.verbose,
flow_ema_alpha=self._force_readout.flow_ema_alpha,
)
self.image_pipeline = ImagePipeline(
self.image_settings,
context=getattr(self, "context", None),
verbose=self.verbose,
flow_ema=self._force_readout,
)
self.__tracker = None
self.__contact3d = Contact3D(config_contact=self.__config_contact, verbose=self.verbose)
self.__hhd = None
# Integer >= 1: run HHD on flow downsampled by this factor for speed (1 = full res).
_raw_ds = self.__config_tracker.get("hhd_downsample", 1)
try:
self._hhd_downsample = max(1, int(_raw_ds)) if _raw_ds is not None else 1
except (TypeError, ValueError):
self._hhd_downsample = 1
self._hhd_downsample = self.image_settings.hhd_downsample
self._hhd_grid_shape = None
self._hhd_ref_shape = None
cfg_backend = self.__config_tracker.get("backend", None)
self.poisson_backend = cfg_backend if cfg_backend is not None else backend
self.masked_poisson_solver = self.__config_tracker.get("masked_poisson_solver", "sparse")
self.masked_poisson_max_boundary = self.__config_tracker.get("masked_poisson_max_boundary", 96)
self.masked_poisson_boundary_stride = self.__config_tracker.get("masked_poisson_boundary_stride", 1)
self.masked_poisson_max_iters = self.__config_tracker.get("masked_poisson_max_iters", 3)
self.masked_poisson_tol = self.__config_tracker.get("masked_poisson_tol", 1e-4)
self.masked_poisson_reg = self.__config_tracker.get("masked_poisson_reg", 1e-8)
self.poisson_backend = self.image_settings.poisson_backend
self.masked_poisson_solver = self.image_settings.masked_poisson_solver
self.masked_poisson_max_boundary = self.image_settings.masked_poisson_max_boundary
self.masked_poisson_boundary_stride = (
self.image_settings.masked_poisson_boundary_stride
)
self.masked_poisson_max_iters = self.image_settings.masked_poisson_max_iters
self.masked_poisson_tol = self.image_settings.masked_poisson_tol
self.masked_poisson_reg = self.image_settings.masked_poisson_reg
# Legacy PyAV capture state (unused after orisys.capture refactor).
# self._use_pyav = False
# self._av_container = None
@@ -682,7 +695,14 @@ class Sensor:
image = self.img if img is None else img
if image is None:
raise Exception("Warning: Get image first")
pipeline = getattr(self, "image_pipeline", None)
if pipeline is not None:
self.flow = pipeline.compute_flow(image, valid_mask=self.valid_mask)
self._compute_flow_means(self.flow)
return
# Bare Sensor/test-double fallback: no ImagePipeline was constructed.
flow = self.__tracker.track(image, valid_mask=self.valid_mask)
# flow = self.__tracker.track_raft(self.img)
if hasattr(flow, 'download'):
@@ -701,6 +721,45 @@ class Sensor:
"""``config``, ``measured`` or ``config (file)``."""
return self._force_readout.sample_rate_source
def _publish_force_sample(self) -> None:
"""Publish one HHD frame's raw feature to the force side (no image work)."""
if self._measurement_session:
# Session path: publish the same-frame unfiltered feature only. Model
# inference and readout filtering belong to orisys.force_session, so
# the old Newton accessors must not report a value the session did not
# produce (they would be stale or half-processed).
self.fnormal = self.fshearx = self.fsheary = float("nan")
self._force_filtered_raw = None
self._force_unfiltered = None
self._force_raw_unfiltered = self._force_raw
else:
self._force_readout.update_sample_rate()
if self._uses_calibration_model():
self._model_frame_id += 1
self.compute_contact_valid = False
self.centroid_found = False
self._force_readout.apply(self._force_raw)
self._force_filtered_raw = self._force_readout.filtered_raw
self._force_raw_unfiltered = self._force_raw
else:
self._apply_force_filter()
def _infer_hhd_model(self, field_source) -> None:
"""Run the model/contact side on this frame's already computed field."""
if self._uses_calibration_model() and not self._measurement_session:
# Spatial inference must use this HHD frame's contact, even when the
# caller did not request contact diagnostics from StreamPipeline.
# The field model needs no locator: it consumes this frame's field.
if self.calibration.input_profile == FIELD_PROFILE:
self.apply_calibration_frame(
frame_id=self._model_frame_id,
field=field_source(),
)
elif self.calibration.input_profile == "fn_delta_autocal_uv/1":
self.compute_contact()
else:
self.apply_calibration_frame(frame_id=self._model_frame_id)
def compute_hhd(self, flow=None):
"""Decompose the optical flow and update the force readout.
@@ -708,12 +767,32 @@ class Sensor:
``self.flow``. StreamPipeline passes its own per-frame copy so the flow
stage cannot overwrite ``self.flow`` mid-decomposition. Existing
callers are unaffected.
The image half (HHD solver, grid buffers, mask handling, upsampling) is
delegated to :attr:`image_pipeline` when a real Sensor was constructed;
the force half (readout/model/contact) stays here. Bare Sensor and
test-double instances keep the historical direct path because they carry
their own solver through ``_Sensor__hhd``.
"""
flow = self.flow if flow is None else flow
if flow is None:
raise Exception("Warning: Compute flow first")
pipeline = getattr(self, "image_pipeline", None)
if pipeline is not None:
decomposition = pipeline.decompose(flow, valid_mask=self.valid_mask)
self._force_raw = decomposition.force_raw
self.ftor = decomposition.torque
self.vnormal = decomposition.vnormal
self.vshear = decomposition.vshear
self.valid_mask_hhd = decomposition.valid_mask_hhd
self.compute_deformation_valid = True
self._publish_force_sample()
self._infer_hhd_model(self.current_field)
return True
# ---- direct fallback: the historical implementation, unchanged
orig_h, orig_w = flow.shape[:2]
try:
k = max(1, int(self._hhd_downsample))
@@ -780,26 +859,7 @@ class Sensor:
fn, fx, fy, ftor = forces
self._force_raw = (float(fn), float(fx), float(fy))
self.ftor = float(ftor)
if self._measurement_session:
# Session path: publish the same-frame unfiltered feature only. Model
# inference and readout filtering belong to orisys.force_session, so
# the old Newton accessors must not report a value the session did not
# produce (they would be stale or half-processed).
self.fnormal = self.fshearx = self.fsheary = float("nan")
self._force_filtered_raw = None
self._force_unfiltered = None
self._force_raw_unfiltered = self._force_raw
else:
self._force_readout.update_sample_rate()
if self._uses_calibration_model():
self._model_frame_id += 1
self.compute_contact_valid = False
self.centroid_found = False
self._force_readout.apply(self._force_raw)
self._force_filtered_raw = self._force_readout.filtered_raw
self._force_raw_unfiltered = self._force_raw
else:
self._apply_force_filter()
self._publish_force_sample()
vnormal_ds = self.__hhd.get_normal()
vshear_ds = self.__hhd.get_shear()
@@ -834,21 +894,7 @@ class Sensor:
np.multiply(self.vshear, m3, out=self.vshear)
self.compute_deformation_valid = True
if self._uses_calibration_model() and not self._measurement_session:
# Spatial inference must use this HHD frame's contact, even when the
# caller did not request contact diagnostics from StreamPipeline.
# The field model needs no locator: it consumes this frame's field.
if self.calibration.input_profile == FIELD_PROFILE:
self.apply_calibration_frame(
frame_id=self._model_frame_id,
field=self.__hhd.get_field(valid_mask=mask_ds),
)
elif self.calibration.input_profile == "fn_delta_autocal_uv/1":
self.compute_contact()
else:
self.apply_calibration_frame(frame_id=self._model_frame_id)
self._infer_hhd_model(lambda: self.__hhd.get_field(valid_mask=mask_ds))
return True
def compute_contact(self, vnormal=None):
@@ -941,14 +987,34 @@ class Sensor:
def measurement_session(self) -> bool:
return bool(self._measurement_session)
def set_stream_reference_generation(self, generation: int) -> None:
"""Adopt the driving stream's reference version for the next frame.
Stage-0 ``StreamPipeline`` claims reference versions; the image block must
stamp that same version instead of counting its own. The pipeline calls
this on its flow thread immediately before ``compute_flow``, so each
frame keeps the version it was measured with. Calling it directly outside
a stream is allowed (the value is what ``ImagePipeline.measure`` will
stamp); it never changes tracker/EMA state.
"""
pipeline = getattr(self, "image_pipeline", None)
if pipeline is not None:
pipeline.set_reference_generation(int(generation))
def reset_tracker(self) -> None:
"""Reset the optical-flow tracker at a frame boundary (session zeroing).
The narrow half of :meth:`reset`: tracker and readout memory only, never
the calibration baseline. ``ForceSession`` calls this on its processing
thread between two frames, before the first baseline frame is measured.
The tracker/EMA history belongs to the ImagePipeline when one is bound;
bare Sensor/test-double instances keep the historical attribute path.
"""
self.__tracker.reset()
pipeline = getattr(self, "image_pipeline", None)
if pipeline is not None:
pipeline.reset_tracker()
else:
self.__tracker.reset()
self._force_readout.reset()
self._force_unfiltered = None
self._force_raw_unfiltered = None
@@ -1088,6 +1154,9 @@ class Sensor:
def current_field(self) -> dict:
"""Same-frame HHD field payload of the last decomposition (a private copy)."""
pipeline = getattr(self, "image_pipeline", None)
if pipeline is not None:
return pipeline.current_field()
if self.__hhd is None:
raise RuntimeError("compute_hhd() before reading the field")
mask = self.valid_mask_hhd
@@ -1155,7 +1224,11 @@ class Sensor:
image buffers. The force half belongs to the HHD thread and is reset
separately by :meth:`reset_force_state`.
"""
self.__tracker.reset()
pipeline = getattr(self, "image_pipeline", None)
if pipeline is not None:
pipeline.reset_image_state()
else:
self.__tracker.reset()
self._force_readout.reset_flow_ema()
if self.img is not None:
self.initial_img = self.img.copy()
@@ -1288,6 +1361,8 @@ class Sensor:
flow_ema_alpha=flow_ema_alpha,
force_median_window=force_median_window,
)
# The flow-EMA coefficient/history has one owner (this readout); the image
# pipeline reads it on the next frame, so runtime configure stays live.
for readout in getattr(self, "_model_readouts", {}).values():
readout.configure(
lowpass_cutoff_hz=lowpass_cutoff_hz,