orisys-tech/orisys-sdk
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 将图像块拆分为可独立运行的 ImagePipeline

Browse Source 
光流、HHD 和场从 Sensor 中抽到 ImageSettings/ImagePipeline，Sensor 与 FingerRuntime 改为委托，公开接口与数值不变（222 个基线数组逐位一致，含运行时 flow_ema_alpha 场景）。参考版本只有一个来源：StreamPipeline 在 flow 线程领取的版本逐帧交给图像块。对外数组为只读副本；输出带 stream-contract 的 MeasurementFrame 头。回归 357 项、lint 通过。
This commit is contained in:
ykcheungab
2026-10-21 13:35:02 +08:00
parent 474dbf5a64
commit e3b60f3d6d
6 changed files with 1306 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/orisys/image_pipeline.py
+592
View File
@@ -0,0 +1,592 @@
"""Image block: ImageSettings and an independently runnable ImagePipeline.
Stage-1 architecture (K8.1, K11.1, K11.3, task T05) separates the image side of
measurement from :class:`~orisys.sensor.Sensor`:
* :class:`ImageSettings` is the immutable description of one image
configuration (optical-flow tracker settings, HHD grid/backend, flow EMA).
* :class:`ImagePipeline` owns the tracker, the HHD solver and their scratch
state. It turns a prepared grey frame into flow/HHD/field results, writes the
optical-flow reference generation at the moment it computes a frame, and
publishes measurement arrays as read-only copies.
``Sensor`` delegates to an ``ImagePipeline``; ``FingerRuntime`` reaches the same
object through its tactile sensor, so all three published paths keep one image
implementation. The class can also be used without a Sensor at all::
from orisys import configs
from orisys.image_pipeline import ImagePipeline, ImageSettings, sensor_clock
context = sensor.context # or build_sensor_context(...)
settings = ImageSettings.from_config(configs.load_config("box"))
pipeline = ImagePipeline(settings, context=context)
measurement = pipeline.measure(
prepared_grey_frame,
identity=context.frame_identity(session_id="run-1", frame_id=0, source_frame_id=0),
clock=sensor_clock(capture_time_ns, clock_id="host", basis="capture_monotonic"),
)
print(measurement.force_raw, measurement.flow.shape, measurement.frame.reference_generation)
Image preparation (stitch/unroll/resize/mirror) remains in the acquisition layer
for now; the pipeline starts at the prepared grey frame, which is exactly what
``Sensor.compute_flow``/``compute_hhd`` have always consumed.
"""
from __future__ import annotations
import time
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from dataclasses import field as dataclass_field
from typing import Any
import cv2
import numpy as np
from .hhd import nHHD
from .stream_contract import (
FrameClock,
FrameIdentity,
FrameStamp,
MeasurementFrame,
TimeBasis,
)
from .tracker import Tracker
__all__ = [
"ImageDecomposition",
"ImageMeasurement",
"ImagePipeline",
"ImageSettings",
"sensor_clock",
]
#: Arrays an independent measurement may request with ``include``.
_INCLUDE_FIELDS = frozenset(
{"image", "flow", "vnormal", "vshear", "field", "valid_mask"}
)
def sensor_clock(
capture_time_ns: int,
*,
clock_id: str,
basis: TimeBasis = "capture_monotonic",
) -> FrameClock:
"""Convenience constructor for the capture clock of a prepared frame.
``basis`` is one of :data:`~orisys.stream_contract.TIME_BASES`; the value is
validated by :class:`~orisys.stream_contract.FrameClock`.
"""
return FrameClock(
capture_time_ns=int(capture_time_ns),
clock_id=str(clock_id),
capture_time_basis=basis,
)
@dataclass(frozen=True)
class ImageSettings:
"""One image configuration: tracker, HHD and flow-EMA settings.
``from_config`` reads the same config keys ``Sensor`` has always read. The
flow EMA alpha is resolved through :class:`~orisys.force_readout.ForceReadout`
so the existing filter-preset interpretation has exactly one implementation.
"""
tracker_config: Mapping[str, Any]
cuda: bool
verbose: bool
hhd_downsample: int
poisson_backend: str
masked_poisson_solver: str
masked_poisson_max_boundary: int
masked_poisson_boundary_stride: int
masked_poisson_max_iters: int
masked_poisson_tol: float
masked_poisson_reg: float
flow_ema_alpha: float = 1.0
@classmethod
def from_config(
cls,
config: Mapping[str, Any] | None,
*,
cuda: bool = True,
backend: str = "auto",
verbose: bool = False,
flow_ema_alpha: float | None = None,
) -> ImageSettings:
"""Build settings from ``sensor.json``-shaped configuration."""
cfg: Mapping[str, Any] = config if isinstance(config, Mapping) else {}
tracker = dict(cfg.get("tracker") or {})
raw_ds = tracker.get("hhd_downsample", 1)
try:
hhd_downsample = max(1, int(raw_ds)) if raw_ds is not None else 1
except (TypeError, ValueError):
hhd_downsample = 1
if flow_ema_alpha is None:
# Reuse the existing filter-preset resolution instead of a second
# interpretation of the filters block (that block owns the EMA).
from .force_readout import ForceReadout
flow_ema_alpha = ForceReadout(cfg.get("filters")).flow_ema_alpha
backend_from_config = tracker.get("backend", None)
poisson_backend = str(
backend_from_config if backend_from_config is not None else backend
)
return cls(
tracker_config=tracker,
cuda=bool(cuda),
verbose=bool(verbose),
hhd_downsample=hhd_downsample,
poisson_backend=poisson_backend,
masked_poisson_solver=str(tracker.get("masked_poisson_solver", "sparse")),
masked_poisson_max_boundary=int(
tracker.get("masked_poisson_max_boundary", 96)
),
masked_poisson_boundary_stride=int(
tracker.get("masked_poisson_boundary_stride", 1)
),
masked_poisson_max_iters=int(tracker.get("masked_poisson_max_iters", 3)),
masked_poisson_tol=float(tracker.get("masked_poisson_tol", 1e-4)),
masked_poisson_reg=float(tracker.get("masked_poisson_reg", 1e-8)),
flow_ema_alpha=float(flow_ema_alpha),
)
def as_dict(self) -> dict[str, Any]:
"""Stable identity of the image settings, for records and comparisons."""
return {
"tracker": dict(self.tracker_config),
"cuda": self.cuda,
"hhd_downsample": self.hhd_downsample,
"poisson_backend": self.poisson_backend,
"masked_poisson_solver": self.masked_poisson_solver,
"masked_poisson_max_boundary": self.masked_poisson_max_boundary,
"masked_poisson_boundary_stride": self.masked_poisson_boundary_stride,
"masked_poisson_max_iters": self.masked_poisson_max_iters,
"masked_poisson_tol": self.masked_poisson_tol,
"masked_poisson_reg": self.masked_poisson_reg,
"flow_ema_alpha": self.flow_ema_alpha,
}
@dataclass(frozen=True)
class ImageDecomposition:
"""Flow/HHD result for one prepared frame (internal state objects, not copies)."""
flow: np.ndarray
vnormal: np.ndarray
vshear: np.ndarray
force_raw: tuple[float, float, float]
torque: float
valid_mask_hhd: np.ndarray | None
def _readonly_copy(value: Any) -> Any:
"""Copy arrays (recursively for the field payload) and freeze them."""
if isinstance(value, np.ndarray):
out = np.array(value, copy=True)
out.setflags(write=False)
return out
if isinstance(value, dict):
return {key: _readonly_copy(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return type(value)(_readonly_copy(item) for item in value)
return value
@dataclass(frozen=True)
class ImageMeasurement:
"""One image measurement: T03 header plus read-only flow/HHD/field payload."""
frame: MeasurementFrame
force_raw: tuple[float, float, float]
torque: float
flow: np.ndarray | None = None
vnormal: np.ndarray | None = None
vshear: np.ndarray | None = None
valid_mask: np.ndarray | None = None
field: dict | None = None
image: np.ndarray | None = None
elapsed_s: float = 0.0
diagnostics: tuple[tuple[str, str | float], ...] = dataclass_field(
default_factory=tuple
)
class ImagePipeline:
"""Own the image half of measurement: tracker, HHD, field and reference version.
Thread ownership mirrors stage 0: one instance is used by one image thread.
The reference version is **not counted here**: the stream that owns it (the
stage-0 ``StreamPipeline`` reset protocol) adopts it through
:meth:`set_reference_generation` or passes it to :meth:`measure`; the image
block only stamps what it was told. :meth:`reset_image_state` rebases the
tracker/flow EMA and never invents a version. The force/contact/model half
stays in ``Sensor``.
"""
def __init__(
self,
settings: ImageSettings,
*,
context: Any = None,
verbose: bool = False,
flow_ema: Any = None,
) -> None:
from .force_readout import ForceReadout
self.settings = settings
self.context = context
self.verbose = bool(verbose)
# One owner for the flow-EMA coefficient *and* its history: the published
# ForceReadout when a Sensor binds this pipeline, otherwise a private one.
# Sensor.configure(flow_ema_alpha=...) therefore keeps working on the next
# frame, exactly as it did before the image block was split out.
self._flow_ema = (
flow_ema
if flow_ema is not None
else ForceReadout({"flow_ema_alpha": settings.flow_ema_alpha})
)
self._tracker = Tracker(
adaptive=True,
cuda=settings.cuda,
verbose=settings.verbose,
config_tracker=dict(settings.tracker_config),
)
self._hhd: nHHD | None = None
self._hhd_grid_shape: tuple[int, int] | None = None
self._hhd_ref_shape: tuple[int, int] | None = None
self._valid_mask_hhd: np.ndarray | None = None
self._valid_mask: np.ndarray | None = None
self._flow_ds_buf: np.ndarray | None = None
self._vn_buf: np.ndarray | None = None
self._vs_buf: np.ndarray | None = None
self._mask_f32: np.ndarray | None = None
self._flow: np.ndarray | None = None
self._vnormal: np.ndarray | None = None
self._vshear: np.ndarray | None = None
self._force_raw: tuple[float, float, float] = (0.0, 0.0, 0.0)
self._torque = 0.0
self._reference_generation = 0
# ------------------------------------------------------------- read access
@property
def flow(self) -> np.ndarray | None:
return self._flow
@property
def vnormal(self) -> np.ndarray | None:
return self._vnormal
@property
def vshear(self) -> np.ndarray | None:
return self._vshear
@property
def valid_mask_hhd(self) -> np.ndarray | None:
return self._valid_mask_hhd
@property
def force_raw(self) -> tuple[float, float, float]:
return self._force_raw
@property
def torque(self) -> float:
return self._torque
@property
def reference_generation(self) -> int:
"""Reference version most recently adopted from the owning stream."""
return int(self._reference_generation)
def set_reference_generation(self, generation: int) -> None:
"""Adopt the stream's reference version (called on the image thread).
The stage-0 ``StreamPipeline`` claims the version and passes it here
before computing the frame's flow; ``measure`` may stamp the same value
explicitly for an independent experiment. This class never increments it.
"""
if isinstance(generation, bool) or not isinstance(generation, int):
raise TypeError("reference_generation must be an integer")
if generation < 0:
raise ValueError("reference_generation must be >= 0")
self._reference_generation = int(generation)
@property
def flow_ema_alpha(self) -> float:
"""Runtime flow-EMA coefficient (owned by the readout, not this pipeline)."""
return float(self._flow_ema.flow_ema_alpha)
# ----------------------------------------------------------- reset seams
def reset_image_state(self) -> None:
"""Rebase the tracker and drop the flow-EMA history (image thread only).
A reference version is not claimed here: the owning stream sets it with
:meth:`set_reference_generation`; this method only resets image state.
"""
self._tracker.reset()
self._flow_ema.reset_flow_ema()
def reset_tracker(self) -> None:
"""Drop tracker/EMA history without touching the reference identity.
This is the frame-boundary zeroing half used by a force session before a
baseline window: the reference frame itself is unchanged.
"""
self._tracker.reset()
self._flow_ema.reset_flow_ema()
# ------------------------------------------------------------- flow / HHD
def _apply_flow_ema(self, flow: np.ndarray | None) -> np.ndarray | None:
"""Temporal flow smoothing, delegated to its single owner (ForceReadout)."""
return self._flow_ema.apply_flow_ema(flow)
def _compute_flow_state(
self, image: np.ndarray, *, valid_mask: np.ndarray | None = None
) -> np.ndarray:
"""Flow for one frame (the caller has already adopted the version)."""
flow = self._tracker.track(image, valid_mask=valid_mask)
if hasattr(flow, "download"):
flow = flow.download()
flow = self._apply_flow_ema(flow)
self._flow = flow
return flow
def compute_flow(
self,
image: np.ndarray,
*,
valid_mask: np.ndarray | None = None,
reference_generation: int | None = None,
) -> np.ndarray:
"""Compute and return this frame's flow (one call per frame)."""
if image is None:
raise Exception("Warning: Get image first")
if reference_generation is not None:
self.set_reference_generation(reference_generation)
return self._compute_flow_state(image, valid_mask=valid_mask)
def _resize_mask(
self, mask: np.ndarray | None, size: tuple[int, int]
) -> np.ndarray | None:
if mask is None:
return None
mask_u8 = np.asarray(mask, dtype=np.uint8)
resized = cv2.resize(mask_u8, size, interpolation=cv2.INTER_NEAREST)
return resized.astype(bool)
def _hhd_grid_size(self) -> tuple[int, int] | None:
mask = self._valid_mask
if mask is None:
return None
h, w = mask.shape[:2]
k = max(1, int(self.settings.hhd_downsample))
if k <= 1:
return (w, h)
return (max(1, w // k), max(1, h // k))
def decompose(
self, flow: np.ndarray | None = None, *, valid_mask: np.ndarray | None = None
) -> ImageDecomposition:
"""Run HHD on one flow field and publish normal/shear/force results."""
flow = self._flow if flow is None else flow
if flow is None:
raise Exception("Warning: Compute flow first")
self._valid_mask = valid_mask
orig_h, orig_w = flow.shape[:2]
try:
k = max(1, int(self.settings.hhd_downsample))
except (TypeError, ValueError):
k = 1
mask_ds = None
if valid_mask is not None:
ds_size = self._hhd_grid_size()
if (
self._valid_mask_hhd is not None
and ds_size is not None
and self._valid_mask_hhd.shape[1] == ds_size[0]
and self._valid_mask_hhd.shape[0] == ds_size[1]
):
mask_ds = self._valid_mask_hhd
else:
mask_ds = np.asarray(valid_mask, dtype=bool)
if k > 1:
ds_w = max(1, orig_w // k)
ds_h = max(1, orig_h // k)
flow_ds = self._flow_ds_buf
if (
flow_ds is None
or flow_ds.shape != (ds_h, ds_w, 2)
or flow_ds.dtype != np.float32
):
flow_ds = np.empty((ds_h, ds_w, 2), dtype=np.float32)
self._flow_ds_buf = flow_ds
cv2.resize(flow, (ds_w, ds_h), dst=flow_ds, interpolation=cv2.INTER_AREA)
if mask_ds is not None and (
mask_ds.shape[1] != ds_w or mask_ds.shape[0] != ds_h
):
mask_ds = self._resize_mask(mask_ds, (ds_w, ds_h))
else:
flow_ds = flow
ds_shape = flow_ds.shape[:2]
ref_shape = (orig_h, orig_w)
self._valid_mask_hhd = mask_ds
if (
self._hhd is None
or self._hhd_grid_shape != ds_shape
or self._hhd_ref_shape != ref_shape
):
spacings = (18 * k, 18 * k)
self._hhd = nHHD(
cuda=self.settings.cuda,
grid=ds_shape,
spacings=spacings,
poisson_backend=self.settings.poisson_backend,
debug=self.settings.verbose,
force_reference_grid=ref_shape,
masked_poisson_solver=self.settings.masked_poisson_solver,
masked_poisson_max_boundary=self.settings.masked_poisson_max_boundary,
masked_poisson_boundary_stride=self.settings.masked_poisson_boundary_stride,
masked_poisson_max_iters=self.settings.masked_poisson_max_iters,
masked_poisson_tol=self.settings.masked_poisson_tol,
masked_poisson_reg=self.settings.masked_poisson_reg,
)
self._hhd_grid_shape = ds_shape
self._hhd_ref_shape = ref_shape
_, forces = self._hhd.get_force(flow_ds, valid_mask=mask_ds)
fn, fx, fy, ftor = forces
self._force_raw = (float(fn), float(fx), float(fy))
self._torque = float(ftor)
vnormal_ds = self._hhd.get_normal()
vshear_ds = self._hhd.get_shear()
if k > 1:
vn = self._vn_buf
vs = self._vs_buf
if vn is None or vn.shape != (orig_h, orig_w, 2) or vn.dtype != np.float32:
vn = np.empty((orig_h, orig_w, 2), dtype=np.float32)
vs = np.empty((orig_h, orig_w, 2), dtype=np.float32)
self._vn_buf = vn
self._vs_buf = vs
cv2.resize(
vnormal_ds, (orig_w, orig_h), dst=vn, interpolation=cv2.INTER_LINEAR
)
cv2.resize(
vshear_ds, (orig_w, orig_h), dst=vs, interpolation=cv2.INTER_LINEAR
)
self._vnormal = vn
self._vshear = vs
else:
self._vnormal = vnormal_ds.copy()
self._vshear = vshear_ds.copy()
if valid_mask is not None:
m3 = self._mask_f32
if m3 is None or m3.shape != (orig_h, orig_w, 1) or m3.dtype != np.float32:
m3 = np.empty((orig_h, orig_w, 1), dtype=np.float32)
self._mask_f32 = m3
np.copyto(m3[:, :, 0], valid_mask, casting="unsafe")
np.multiply(self._vnormal, m3, out=self._vnormal)
np.multiply(self._vshear, m3, out=self._vshear)
return ImageDecomposition(
flow=self._flow,
vnormal=self._vnormal,
vshear=self._vshear,
force_raw=self._force_raw,
torque=self._torque,
valid_mask_hhd=self._valid_mask_hhd,
)
def current_field(self) -> dict:
"""Same-frame HHD field payload (the solver returns a private copy)."""
if self._hhd is None:
raise RuntimeError("compute_hhd() before reading the field")
return self._hhd.get_field(valid_mask=self._valid_mask_hhd)
# ------------------------------------------------------------ measurement
def measure(
self,
image: np.ndarray,
*,
identity: FrameIdentity,
clock: FrameClock,
source_dt_s: float | None = None,
valid_mask: np.ndarray | None = None,
include: Sequence[str] = ("flow",),
want_field: bool = False,
reference_generation: int | None = None,
) -> ImageMeasurement:
"""Measure one prepared frame once and return read-only payload + T03 header.
``reference_generation`` is the version the owning stream claimed for this
frame; when omitted the last adopted version is used. The version is fixed
before flow is computed: a newer version adopted while this frame is in
flight does not relabel it.
"""
unknown = set(include) - _INCLUDE_FIELDS
if unknown:
raise ValueError(
f"unknown include field(s) {sorted(unknown)}; "
f"expected a subset of {sorted(_INCLUDE_FIELDS)}"
)
started = time.perf_counter()
if reference_generation is not None:
self.set_reference_generation(reference_generation)
generated = int(self._reference_generation)
self._compute_flow_state(image, valid_mask=valid_mask)
decomposition = self.decompose(self._flow, valid_mask=valid_mask)
stamp = FrameStamp(
sensor_id=identity.sensor_id,
session_id=identity.session_id,
frame_id=identity.frame_id,
source_frame_id=identity.source_frame_id,
capture_time_ns=clock.capture_time_ns,
clock_id=clock.clock_id,
capture_time_basis=clock.capture_time_basis,
source_pts=clock.source_pts,
acquisition_id=identity.acquisition_id,
sync_group_id=identity.sync_group_id,
)
frame = MeasurementFrame(
stamp=stamp, source_dt_s=source_dt_s, reference_generation=generated
)
include_set = set(include)
payload = {
"flow": _readonly_copy(decomposition.flow)
if "flow" in include_set
else None,
"vnormal": _readonly_copy(decomposition.vnormal)
if "vnormal" in include_set
else None,
"vshear": _readonly_copy(decomposition.vshear)
if "vshear" in include_set
else None,
"valid_mask": _readonly_copy(decomposition.valid_mask_hhd)
if "valid_mask" in include_set
else None,
"field": _readonly_copy(self.current_field())
if ("field" in include_set or want_field)
else None,
"image": _readonly_copy(image) if "image" in include_set else None,
}
return ImageMeasurement(
frame=frame,
force_raw=decomposition.force_raw,
torque=decomposition.torque,
flow=payload["flow"],
vnormal=payload["vnormal"],
vshear=payload["vshear"],
valid_mask=payload["valid_mask"],
field=payload["field"],
image=payload["image"],
elapsed_s=time.perf_counter() - started,
diagnostics=(("reference_generation", generated),),
)