orisys-tech/orisys-app
3
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

接入 Autocal 实时相机及帧时间显示

Browse Source 
视频和相机共用 Finger worker，并保留采集时间、源帧编号及媒体时间。通过界面、采集链和真实视频验证。
This commit is contained in:
ykcheungab
2026-10-11 12:50:52 +08:00
parent 12c2fb1c4b
commit 32dcecb270
13 changed files with 280 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
shared/pyproject.toml
+1 -1
View File
@@ -17,7 +17,7 @@ dependencies = [
"numpy>=1.24.4",
"scipy>=1.10.1",
"pyqtgraph>=0.13",
"orisys>=0.5",
"orisys>=0.5.4",
]
[project.optional-dependencies]
shared/src/orisys_app_shared/acquisition/finger_worker.py
+91 -56
View File
@@ -7,6 +7,7 @@ import threading
import time
from collections import deque
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
import numpy as np
@@ -71,6 +72,38 @@ class _RuntimeCapture:
return getattr(self._runtime, "capture_source", None)
@dataclass(frozen=True)
class _FingerFrameMeta:
raw_frame: np.ndarray
source_seq: int | None
mono_ns: int
utc_ns: int
media_ms: float | None
stamped: bool
def _read_source_with_stamp(runtime):
"""Take the frame and its capture stamp through one runtime read."""
read_stamped = getattr(runtime, "read_source_stamped", None)
if callable(read_stamped):
captured = read_stamped()
if captured is None:
return None
return (
captured.image,
captured.sequence,
captured.host_mono_ns,
captured.host_utc_ns,
captured.media_time_ms,
True,
)
# Older SDK runtimes still work, but their read has no frame-bound stamp.
ok, frame = runtime.read_source()
if not ok or frame is None:
return None
return (frame, None, time.perf_counter_ns(), time.time_ns(), None, False)
class _FingerProducer:
"""The ``StreamPipeline`` produce callable for a ``FingerRuntime``.
@@ -84,18 +117,15 @@ class _FingerProducer:
each frame as many times as it could poll. Only a source with a running
background reader has the sequence counter that says so; a file decodes a
new frame per call and is read once per call.
* **The pre-processing frame.** ``runtime.process_frame`` hands the unrolled
frame to the tactile sensor and drops the source frame; the record channel's
pre-roll history wants the source frame, so it is copied here while it is
still this frame's, and handed on by sequence.
* **The source frame and capture stamp.** They travel with the unrolled image
through the pipeline queues, so a pause or backlog cannot make a snapshot
pick up another frame's provenance.
* **The camera's own rate.** The consumer reads ``capture_fps`` from this
producer, the same way the box path reads it from ``_FrameProducer``: the
source's delivered-frame counter, anchored to the wall clock of the wait
that observed it.
"""
CACHE = 8
def __init__(self, runtime, *, is_file: bool, frame_period: float = 0.0) -> None:
self.runtime = runtime
self.is_file = bool(is_file)
@@ -105,8 +135,6 @@ class _FingerProducer:
self.produced = 0
self.capture_source = _RuntimeCapture(runtime)
self.capture_seq = _capture_sequence(self.capture_source)
self._cache: deque = deque(maxlen=self.CACHE)
self._times: deque = deque()
self._last_read_at: float | None = None
#: ``(clock, seconds, delivered frames)`` samples behind ``capture_fps``.
self._rate_samples: deque = deque()
@@ -118,20 +146,32 @@ class _FingerProducer:
self._wait_for_a_new_frame()
self._pace()
t0 = time.perf_counter()
ok, source_frame = self.runtime.read_source()
if not ok or source_frame is None:
captured = _read_source_with_stamp(self.runtime)
if captured is None:
return None
source_frame, source_seq, mono_ns, utc_ns, media_ms, stamped = captured
if not self.is_file and source_seq is not None:
# A newer frame can arrive between the sequence wait and this read.
# Follow the frame actually taken, so it is not published twice.
self.capture_seq = source_seq
unrolled = self.runtime.process_frame(source_frame, color=False)
if unrolled is None:
return None
self._last_read_at = t0
self.produced += 1
self._times.append((self.produced, t0))
cutoff = t0 - FPS_WINDOW_S
while self._times and self._times[0][1] < cutoff:
self._times.popleft()
self._cache.append((self.produced, np.array(source_frame, copy=True)))
return unrolled
from orisys.pipeline import ProducedFrame
return ProducedFrame(
unrolled,
_FingerFrameMeta(
np.array(source_frame, copy=True),
source_seq,
mono_ns,
utc_ns,
media_ms,
stamped,
),
)
def _pace(self) -> None:
if not self.is_file or self.frame_period <= 0.0 or self._last_read_at is None:
@@ -160,27 +200,6 @@ class _FingerProducer:
return
time.sleep(0)
def raw_frame(self, seq: int) -> np.ndarray | None:
for number, frame in reversed(self._cache):
if number == seq:
return frame
return None
def collect_perf(self, seq: int) -> float | None:
self._trim()
for number, stamp in reversed(self._times):
if number == seq:
return stamp
return None
def _trim(self) -> None:
"""Drop collection times older than the frame-rate window."""
if not self._times:
return
cutoff = self._times[-1][1] - FPS_WINDOW_S
while self._times and self._times[0][1] < cutoff:
self._times.popleft()
def capture_window(self, previous: tuple[float, int] | None) -> tuple[float, int]:
"""``(seconds, delivered frames)`` over the interval since ``previous``.
@@ -590,7 +609,7 @@ class FingerAcquisitionWorker(RecordSinkMixin):
try:
runtime.close()
except Exception:
pass
logger.exception("Could not close finger runtime for session %s", session)
def _close_all_runtimes(self) -> None:
with self._lock:
@@ -718,10 +737,8 @@ class FingerAcquisitionWorker(RecordSinkMixin):
The pipeline is built the way ``runtime.pipeline()`` builds it — a
``StreamPipeline`` over the runtime's tactile sensor — but with this
worker's own produce callable in place of ``grab_and_process``. The callable
is what reads the source, so it is where the two things that must happen on
the producer thread live: one published frame per *delivered* frame (a live
source hands the same frame back until its reader produces another), and the
copy of the pre-unroll source frame the record channel's history wants. See
is what reads the source, so it is where one published frame per *delivered*
frame is enforced and where the source frame and capture stamp are bound. See
:class:`_FingerProducer`.
"""
import queue as queue_module
@@ -816,7 +833,6 @@ class FingerAcquisitionWorker(RecordSinkMixin):
live_capture=live_capture,
snap=snap,
frame_id=frame_id,
frame_period=frame_period,
is_file=is_file,
frame_times=frame_times,
mono_t0=mono_t0,
@@ -940,7 +956,6 @@ class FingerAcquisitionWorker(RecordSinkMixin):
live_capture: bool,
snap,
frame_id: int,
frame_period: float,
is_file: bool,
frame_times: list[float],
mono_t0: float,
@@ -949,9 +964,13 @@ class FingerAcquisitionWorker(RecordSinkMixin):
compute_error: str,
):
"""One :class:`SdkPacket` from one finished finger snapshot."""
produced_seq = frame_id + 1
t_collect = producer.collect_perf(produced_seq) or time.perf_counter()
mono = frame_id * frame_period if is_file else max(0.0, t_collect - mono_t0)
meta = snap.source_metadata
if not isinstance(meta, _FingerFrameMeta):
raise RuntimeError(f"capture metadata missing for frame {frame_id}")
source_seq, mono_ns, utc_ns = meta.source_seq, meta.mono_ns, meta.utc_ns
media_ms, stamped = meta.media_ms, meta.stamped
t_collect = mono_ns / 1e9
mono = max(0.0, mono_ns / 1e9 - mono_t0)
frame_times.append(t_collect)
cutoff = t_collect - FPS_WINDOW_S
del frame_times[
@@ -969,8 +988,8 @@ class FingerAcquisitionWorker(RecordSinkMixin):
taken = _capture_dropped(producer.capture_source)
if taken is not None:
capture_dropped = int(taken)
elif producer.produced > produced_seq:
capture_dropped = int(producer.produced - produced_seq)
elif producer.produced > snap.seq + 1:
capture_dropped = int(producer.produced - snap.seq - 1)
with self._lock:
self._measured_capture_fps = capture_fps
raw_sensor = getattr(tactile, "raw_sensor", None)
@@ -980,10 +999,18 @@ class FingerAcquisitionWorker(RecordSinkMixin):
session_id=session,
frame_id=frame_id,
mono_t=mono,
wall_t=utc_ns / 1e9,
capture_mono_ns=mono_ns if stamped else None,
capture_utc_ns=utc_ns if stamped else None,
source_frame_id=source_seq,
time_basis=("host_replay" if is_file else "host_camera")
if stamped
else "host_fallback",
media_time_ms=media_ms,
fps=fps,
capture_fps=capture_fps,
capture_dropped=capture_dropped,
raw_frame=producer.raw_frame(produced_seq),
raw_frame=meta.raw_frame,
valid_mask=runtime.valid_mask,
contact_analysed=True,
contact_found=bool(getattr(raw_sensor, "centroid_found", False)),
@@ -1061,12 +1088,13 @@ class FingerAcquisitionWorker(RecordSinkMixin):
break
t0 = time.perf_counter()
ok, source_frame = runtime.read_source()
if not ok or source_frame is None:
captured = _read_source_with_stamp(runtime)
if captured is None:
if saw_frame and is_file and none_streak_start is None:
none_streak_start = time.perf_counter()
time.sleep(0.005)
continue
source_frame, source_seq, mono_ns, utc_ns, media_ms, stamped = captured
if self._pause.is_set():
# Paused: the source read above proves the camera is alive, so refresh the
@@ -1091,7 +1119,7 @@ class FingerAcquisitionWorker(RecordSinkMixin):
compute_error = ""
# The frame's acquisition time, so an event that happens on this
# frame is dated when it happened rather than when it was queued.
frame_mono = max(0.0, time.perf_counter() - self._mono_t0)
frame_mono = max(0.0, mono_ns / 1e9 - self._mono_t0)
try:
self._consume_pending_calibration(tactile, frame_id, frame_mono)
except Exception as exc: # noqa: BLE001
@@ -1166,9 +1194,16 @@ class FingerAcquisitionWorker(RecordSinkMixin):
packet = SdkPacket(
raw_frame=raw_frame,
processed_frame=_ensure_bgr(unrolled),
timestamp=time.time(),
wall_t=time.time(),
mono_t=max(0.0, t1 - self._mono_t0),
timestamp=utc_ns / 1e9,
wall_t=utc_ns / 1e9,
mono_t=frame_mono,
capture_mono_ns=mono_ns if stamped else None,
capture_utc_ns=utc_ns if stamped else None,
source_frame_id=source_seq,
time_basis=("host_replay" if is_file else "host_camera")
if stamped
else "host_fallback",
media_time_ms=media_ms,
fps=fps,
process_ms=(t1 - t0) * 1000.0,
normal_force=forces["normal_force"],
shared/src/orisys_app_shared/acquisition/models.py
+18
View File
@@ -84,6 +84,14 @@ class SdkPacket:
mono_t: float | None = None
#: Wall clock at hand-off, for metadata only. ``None`` means unmeasured.
wall_t: float | None = None
#: Host clocks bound to the source image at read completion. ``None`` for
#: older packets and producers without a stamped capture interface.
capture_mono_ns: int | None = None
capture_utc_ns: int | None = None
source_frame_id: int | None = None
#: ``host_camera``, ``host_replay`` or ``host_fallback`` for finger input.
time_basis: str = ""
media_time_ms: float | None = None
#: False when the force readout failed. Invalid packets carry NaN forces and
#: must never be averaged into calibration data or written as zeros.
valid: bool = True
@@ -163,6 +171,11 @@ def from_snapshot(
frame_id: int | None = None,
mono_t: float | None = None,
wall_t: float | None = None,
capture_mono_ns: int | None = None,
capture_utc_ns: int | None = None,
source_frame_id: int | None = None,
time_basis: str = "",
media_time_ms: float | None = None,
fps: float | None = None,
process_ms: float | None = None,
capture_fps: float = 0.0,
@@ -223,6 +236,11 @@ def from_snapshot(
timestamp=wall,
wall_t=wall,
mono_t=float(mono_t),
capture_mono_ns=capture_mono_ns,
capture_utc_ns=capture_utc_ns,
source_frame_id=source_frame_id,
time_basis=time_basis,
media_time_ms=media_time_ms,
fps=float(snapshot.fps if fps is None else fps),
process_ms=float(
(snapshot.produce_ms + snapshot.flow_ms + snapshot.hhd_ms)
shared/tests/test_stream_pipeline_acquisition.py
+19
View File
@@ -176,6 +176,7 @@ class _Runtime:
"""A stand-in for ``FingerRuntime`` with the API the worker uses."""
def __init__(self, *args, **kwargs) -> None:
self._source_seq = 0
self.tactile = types.SimpleNamespace(
raw_sensor=_Sensor(),
reset=lambda: None,
@@ -192,6 +193,16 @@ class _Runtime:
def read_source(self):
return True, np.zeros((8, 8, 3), dtype=np.uint8)
def read_source_stamped(self):
self._source_seq += 1
return types.SimpleNamespace(
image=np.zeros((8, 8, 3), dtype=np.uint8),
sequence=self._source_seq,
host_mono_ns=time.perf_counter_ns(),
host_utc_ns=time.time_ns(),
media_time_ms=self._source_seq * 50.0,
)
def process_frame(self, frame, *, color=False):
return np.zeros((8, 8), dtype=np.uint8)
@@ -276,6 +287,14 @@ def test_finger_video_and_camera_use_the_same_packet_path(source, monkeypatch):
assert packet.processed_frame.shape == (8, 8, 3)
assert packet.session_id == 1
assert packet.frame_id == 0
assert packet.source_frame_id == 1
assert packet.capture_mono_ns is not None
assert packet.capture_utc_ns is not None
assert packet.timestamp == pytest.approx(packet.capture_utc_ns / 1e9)
assert packet.time_basis == (
"host_replay" if isinstance(source, str) else "host_camera"
)
assert packet.media_time_ms == 50.0
finally:
worker.stop(wait=True)
_restore(saved)