From 32dcecb2704b700305d499f91e0f74cea48d21d1 Mon Sep 17 00:00:00 2001 From: ykcheungab Date: Sun, 11 Oct 2026 12:50:52 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8E=A5=E5=85=A5=20Autocal=20=E5=AE=9E?= =?UTF-8?q?=E6=97=B6=E7=9B=B8=E6=9C=BA=E5=8F=8A=E5=B8=A7=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 视频和相机共用 Finger worker，并保留采集时间、源帧编号及媒体时间。通过界面、采集链和真实视频验证。 --- apps/autocal/config/locales/en_US.json | 24 ++- apps/autocal/config/locales/zh_CN.json | 24 ++- apps/autocal/config/preferences.default.json | 2 + .../docs/calibration-data-format.zh.md | 4 +- apps/autocal/docs/implementation-plan.zh.md | 10 +- .../autocal/src/orisys_autocal/preferences.py | 2 + .../orisys_autocal/ui/finger_source_panel.py | 91 +++++++++-- apps/autocal/src/orisys_autocal/ui/window.py | 4 + .../autocal/tests/test_finger_source_panel.py | 24 +++ shared/pyproject.toml | 2 +- .../acquisition/finger_worker.py | 147 +++++++++++------- .../orisys_app_shared/acquisition/models.py | 18 +++ .../tests/test_stream_pipeline_acquisition.py | 19 +++ 13 files changed, 280 insertions(+), 91 deletions(-) diff --git a/apps/autocal/config/locales/en_US.json b/apps/autocal/config/locales/en_US.json index 1b6ecf5..b5aed23 100644 --- a/apps/autocal/config/locales/en_US.json +++ b/apps/autocal/config/locales/en_US.json @@ -233,24 +233,34 @@ "reserved_points": "The calibration point table and capture controls will appear here: platform coordinates, reference load, sampling statistics and the resulting sensitivity." }, "source": { + "kind": "Input source", + "kind_video": "Video file", + "kind_camera": "Live camera", "video": "Finger video", + "camera_index": "Camera index", "config": "Sensor config", "browse": "Browse…", - "start": "Play video", + "start": "Start input", "stop": "Stop", - "idle": "Choose a video and sensor.json", + "idle": "Choose an input source and sensor.json", "no_frame": "Waiting for processed frames", + "preview_only": "Use this page to check the input image. Calibration trial recording is not connected yet.", "choose_video": "Choose Finger video", "choose_config": "Choose sensor.json", "video_filter": "Video files (*.mp4 *.avi *.mov *.mkv);;All files (*)", - "missing_input": "Choose an existing video file and sensor.json", - "opening": "Opening video…", - "running": "Processing video", + "missing_input": "Choose an existing sensor.json and, for video mode, a video file", + "opening": "Opening input…", + "running": "Processing frames", "stopping": "Stopping…", "stopped": "Stopped", "finished": "Video finished", - "error": "Video processing failed: {detail}", - "frame": "Frame {frame} · acquired at {time:.3f}" + "disconnected": "Camera disconnected", + "failed": "Acquisition failed; check the input source", + "error": "Acquisition or processing failed: {detail}", + "basis_replay": "decoded now", + "basis_camera": "camera received", + "basis_unknown": "read time (source unverified)", + "frame": "Frame {frame} · source {source_frame} · {basis} {time} · dropped {dropped}" }, "finger": { "title": "Finger calibration-point debug", diff --git a/apps/autocal/config/locales/zh_CN.json b/apps/autocal/config/locales/zh_CN.json index bdd26f6..d15ee04 100644 --- a/apps/autocal/config/locales/zh_CN.json +++ b/apps/autocal/config/locales/zh_CN.json @@ -233,24 +233,34 @@ "reserved_points": "标定点列表与采集控制将在此显示：平台坐标、参考载荷、采样统计与灵敏度结果。" }, "source": { + "kind": "输入来源", + "kind_video": "视频文件", + "kind_camera": "实时相机", "video": "Finger 视频", + "camera_index": "相机编号", "config": "传感器配置", "browse": "选择…", - "start": "开始播放", + "start": "启动输入", "stop": "停止", - "idle": "请选择视频和 sensor.json", + "idle": "请选择输入来源和 sensor.json", "no_frame": "等待处理后的画面", + "preview_only": "此页用于检查输入画面；标定事件记录尚未接通。", "choose_video": "选择 Finger 视频", "choose_config": "选择 sensor.json", "video_filter": "视频文件 (*.mp4 *.avi *.mov *.mkv);;所有文件 (*)", - "missing_input": "请选择存在的视频文件和 sensor.json", - "opening": "正在打开视频…", - "running": "正在处理视频", + "missing_input": "请选择存在的 sensor.json；视频模式还需选择视频文件", + "opening": "正在打开输入…", + "running": "正在处理画面", "stopping": "正在停止…", "stopped": "已停止", "finished": "视频播放完毕", - "error": "视频处理失败：{detail}", - "frame": "帧 {frame} · 采集时间 {time:.3f}" + "disconnected": "相机连接中断", + "failed": "采集失败，请检查输入来源", + "error": "采集或处理失败：{detail}", + "basis_replay": "本次解码", + "basis_camera": "相机接收", + "basis_unknown": "读取时间（来源未确认）", + "frame": "帧 {frame} · 源帧 {source_frame} · {basis} {time} · 丢帧 {dropped}" }, "finger": { "title": "手指标定点调试", diff --git a/apps/autocal/config/preferences.default.json b/apps/autocal/config/preferences.default.json index 5836f10..196146b 100644 --- a/apps/autocal/config/preferences.default.json +++ b/apps/autocal/config/preferences.default.json @@ -18,6 +18,8 @@ "show_diagnostics": false, "finger_video_path": "", "finger_config_ref": "", + "finger_source_kind": "video", + "finger_camera_index": 0, "safety_limits": { "max_travel_mm": 100.0, "max_descent_mm": 10.0, diff --git a/apps/autocal/docs/calibration-data-format.zh.md b/apps/autocal/docs/calibration-data-format.zh.md index 3f6cab4..8138275 100644 --- a/apps/autocal/docs/calibration-data-format.zh.md +++ b/apps/autocal/docs/calibration-data-format.zh.md @@ -11,7 +11,7 @@ | 压头 | 直径 6 mm、球面半径 3 mm 的半球压头 | | 参考 | 砝码或 50 N／2 N·m FT sensor，首版先接通一种 | | 图像 | 一个完整测试事件一个连续视频，档位间不切段 | -| 调试输入 | 先用已有视频接通 finger 采集链；实时相机随后复用同一个 worker 和 `SdkPacket` 消费路径 | +| 调试输入 | 已有视频与实时相机编号共用 finger worker 和 `SdkPacket`；相机实机尚未验收 | | 交付 | 单个 session 能重新打开、追溯参考载荷，计算并显示不同测点的灵敏度 | [LF606 规格书](references/lf606-d40-h18-4x-spec.pdf)量程表包含 50 N／2 N·m 档位，但图签为 200 N／8 N·m；实机以铭牌及标定证书核对。 @@ -79,7 +79,7 @@ 实时采集以电脑收到**该帧**时的单调时间为主时间。采集线程需把 `t_host_mono_ns` 与帧序号、图像原子配对；记录器再保存 UTC 对照，处理结束时间不能代替采集时间。FT／ECAT 样本也在电脑收到时打同源时间戳，按时间关联，并记录可测得的延迟与失同步；低延迟不等于零延迟。砝码的人工到档时刻只定义有效区间，稳定窗口仍需检查。 -旧视频没有原始逐帧电脑时间时，重放取得的是本次解码时间，`time_basis=host_replay`；它可验证处理链，不能还原原实验与 FT 的同步。若视频附有原始帧时间则保留 `time_basis=original_recorded` 和原时钟身份。现有 finger worker 对视频按配置帧率推算 `mono_t`，且已检查的 DD02 视频约 20 fps、配置为 30 fps；实施时须停止把该推算值当采集时刻。manifest 保留来源路径、哈希、配置、媒体帧率及各时间基准。 +旧视频没有原始逐帧电脑时间时，重放取得的是本次解码时间，`time_basis=host_replay`；它可验证处理链，不能还原原实验与 FT 的同步。若视频附有原始帧时间则保留 `time_basis=original_recorded` 和原时钟身份。finger worker 已停止按配置帧率推算 `mono_t`，并在包中保留主机采集时间、源帧编号及可用的媒体时间；正式记录器仍须将这些字段写入 `frames.csv`。manifest 保留来源路径、哈希、配置、媒体帧率及各时间基准。 事件边界如下。`events.csv` 每行至少保存 `event_id`、`trial_id`、可为空的 `step_id`、`event_type`、`t_host_mono_ns`、`clock_id` 和 `reason`。操作员的确认时间和设备反馈时间分别标明来源；不得把按钮时刻冒充设备实际到档。所有流用 `trial_id` 与同源单调时间关联。 diff --git a/apps/autocal/docs/implementation-plan.zh.md b/apps/autocal/docs/implementation-plan.zh.md index 40357ae..60def03 100644 --- a/apps/autocal/docs/implementation-plan.zh.md +++ b/apps/autocal/docs/implementation-plan.zh.md @@ -1,6 +1,6 @@ # Autocal finger 标定实施计划 -状态：2026-10-11 修订。用途：方案。阶段一的交付改为操作员手动控制完成整条标定流程；现有 P0 只验证了视频进入 finger 采集链，尚未实现会话记录或灵敏度页面。操作与文件字段以[数据格式草稿](calibration-data-format.zh.md)为准。 +状态：2026-10-11 修订。用途：方案。阶段一的交付为操作员手动控制完成整条标定流程；M1 已接通视频与相机编号输入，并把主机采集时间绑定到 finger 帧。相机实机尚未验收，会话记录及灵敏度页面尚未实现。操作与文件字段以[数据格式草稿](calibration-data-format.zh.md)为准。 ## 阶段一目标 @@ -18,7 +18,7 @@ 实时相机在电脑收到新帧时记录高精度单调时钟，时间戳须与**该帧身份**一起传递；`frames.csv` 记录采集时间，不用处理完成时间或配置帧率推算时间。另存 UTC 墙钟供外部追溯，时长和同进程流对齐用单调时钟。ECAT／FT 采样在电脑收到对应样本时使用相同时间基准，记录来源、采样序号、有效性及可测得的通信延迟。低延迟是待验证条件，不能省略时间戳或默认各流无偏移。 -`orisys-sdk` 的 `CaptureSource` 现在只在状态中保存 `last_frame_time`，不能保证后续读取时该值仍属于取到的帧；shared finger worker 的视频 `mono_t` 还按配置帧率推算。阶段一需在采集层把时间与帧序号原子配对，再传到 shared worker 和记录器，避免让 Autocal UI 修补时间。文件视频的电脑时间是**本次解码时间**，只用于调试运行过程；若输入没有原始逐帧采集时间，不能与原实验的 FT／运动流做物理同步。原视频的媒体时间与重放时间分别保存；重新录制的 trial 视频保留本次帧索引和时间。 +`orisys-sdk` 0.5.4 的 `CaptureSource.read_raw_stamped`／`read_latest_stamped` 现将图像、源帧编号、主机单调与 UTC 时钟一并返回；shared finger worker 将这些字段写入 `SdkPacket`，并区分 `host_camera` 与 `host_replay`。自定义运行时若没有带时间的读取接口则仅标 `host_fallback`，不能把该时间当成已验证的采集时刻。文件视频的主机时间是**本次解码时间**；媒体时间另存，不能凭配置帧率伪造原实验时间，也不能与原实验的 FT／运动流做物理同步。相机输入只经过模拟验证，仍需实机检查驱动延迟、丢帧与时钟对应。 ## 复用边界与实施顺序 @@ -31,7 +31,9 @@ | M5 集成验收 | 合成会话、旧消费者回归、视频重放、真实设备手动试验及操作指南核对 | 主代理集成 | 6–10 | | 合计 | 可手动完成一次多点标定并复核结果 | 见下文 | 44–66 | -先实现 M1 的视频输入和带时间的数据包，再以同一包格式接相机；M2 记录器与 M3 手动 UI 可在接口固定后并行。M4 可先用合成 session 开发，但阶段一只有在真实记录上重新打开并计算逐测点结果才算完成。Studio 的 `SessionController`、widget 和 `RecordingCoordinator` 保留在 Studio；Autocal 直接复用 shared `FingerAcquisitionWorker`、`SdkPacket` 和 sink，不跨应用导入私有模块。只有出现两端实际共用的逻辑时才移到 shared；SDK 不依赖 UI。 +M1 的视频与相机输入现共用带时间的数据包；M2 记录器与 M3 手动 UI 可在接口固定后并行。M4 可先用合成 session 开发，但阶段一只有在真实记录上重新打开并计算逐测点结果才算完成。Studio 的 `SessionController`、widget 和 `RecordingCoordinator` 保留在 Studio；Autocal 直接复用 shared `FingerAcquisitionWorker`、`SdkPacket` 和 sink，不跨应用导入私有模块。只有出现两端实际共用的逻辑时才移到 shared；SDK 不依赖 UI。 + +界面复核：当前“手指调试”用于选点，“测试序列”用于计划，“传感器”只检查输入，三个页面之间尚无 trial 状态贯通；全局“运行”按钮也尚未启动记录。M3 应让操作员按“选点→设档→开始记录→逐档确认→卸载→结束”前进，并在操作期间持续看到选中点、输入状态和记录状态。现阶段传感器页明确提示仅供预览，避免误以为已保存标定数据。 主代理负责接口、M3 和集成；一名子代理负责 M1/M2，另一名负责 M4 与合成会话验证。建议总计主代理加两名子代理，约 4–6 个工程工作日并留设备联调时间；单人约 6–9 日。估算不包括设备、FT 协议或硬件等待。当前计划不启动子代理。 @@ -45,4 +47,4 @@ | 恢复与故障 | 全新进程重开会话可复核结果；中断、写盘失败及无效参考不显示成有效系数 | | 硬件 | 实际运动只在确认设备、行程／载荷边界及停机条件后执行，软件离线通过不等于实机通过 | -阶段二再增加自动测点遍历或自动加载：它们复用阶段一的 trial、事件、采集和分析接口，只替换人工动作及到档确认来源。FT／ECAT 自动参考接入可按设备可用性单独排期；测量延迟与钟偏通过实测确认。剪切、复杂空间模型和实时部署不属于阶段一。 \ No newline at end of file +阶段二再增加自动测点遍历或自动加载：它们复用阶段一的 trial、事件、采集和分析接口，只替换人工动作及到档确认来源。FT／ECAT 自动参考接入可按设备可用性单独排期；测量延迟与钟偏通过实测确认。剪切、复杂空间模型和实时部署不属于阶段一。 diff --git a/apps/autocal/src/orisys_autocal/preferences.py b/apps/autocal/src/orisys_autocal/preferences.py index 4dee22c..47776d0 100644 --- a/apps/autocal/src/orisys_autocal/preferences.py +++ b/apps/autocal/src/orisys_autocal/preferences.py @@ -211,6 +211,8 @@ def default_preferences() -> dict[str, Any]: "finger_bundle_path": DEFAULT_FINGER_BUNDLE, "finger_video_path": "", "finger_config_ref": "", + "finger_source_kind": "video", + "finger_camera_index": 0, #: The UV-space texture for the flat picker. Preferred over the derived #: path when it exists; empty falls back to the layout map. "finger_texture_path": DEFAULT_FINGER_TEXTURE, diff --git a/apps/autocal/src/orisys_autocal/ui/finger_source_panel.py b/apps/autocal/src/orisys_autocal/ui/finger_source_panel.py index cd2b166..6881143 100644 --- a/apps/autocal/src/orisys_autocal/ui/finger_source_panel.py +++ b/apps/autocal/src/orisys_autocal/ui/finger_source_panel.py @@ -1,7 +1,8 @@ -"""Finger video input and preview for the AutoCal sensor workspace.""" +"""Finger video or live camera input for the AutoCal sensor workspace.""" from __future__ import annotations +from datetime import datetime from pathlib import Path import numpy as np @@ -11,18 +12,20 @@ from PySide6.QtCore import Qt, QTimer, Signal from PySide6.QtGui import QImage, QPixmap from PySide6.QtWidgets import ( QFileDialog, + QComboBox, QGridLayout, QHBoxLayout, QLabel, QLineEdit, QPushButton, + QSpinBox, QVBoxLayout, QWidget, ) class FingerSourcePanel(QWidget): - """Run the shared finger processing path from a video file. + """Run the shared finger processing path from a video or camera. Worker callbacks originate on its acquisition thread; the GUI timer alone reads packets and changes widgets. The same worker also accepts a camera ID. @@ -35,6 +38,9 @@ class FingerSourcePanel(QWidget): video_path: str = "", config_ref: str = "", parent: QWidget | None = None, + *, + source_kind: str = "video", + camera_index: int = 0, ) -> None: super().__init__(parent) self.worker = FingerAcquisitionWorker(on_error=self.errorReported.emit) @@ -46,20 +52,41 @@ class FingerSourcePanel(QWidget): layout = QVBoxLayout(self) fields = QGridLayout() + self.source_label = QLabel(self) + self.source_kind = QComboBox(self) + self.source_kind.setObjectName("fingerSourceKind") + self.source_kind.setMaximumWidth(220) + self.source_kind.addItem("", "video") + self.source_kind.addItem("", "camera") + self.source_kind.setCurrentIndex(1 if source_kind == "camera" else 0) self.video_label = QLabel(self) self.video_path = QLineEdit(video_path, self) self.video_path.setObjectName("fingerVideoPath") self.video_browse = QPushButton(self) + self.camera_label = QLabel(self) + self.camera_index = QSpinBox(self) + self.camera_index.setObjectName("fingerCameraIndex") + self.camera_index.setRange(0, 31) + try: + remembered_camera = int(camera_index) + except (TypeError, ValueError): + remembered_camera = 0 + self.camera_index.setValue(max(0, min(31, remembered_camera))) + self.camera_index.setMaximumWidth(100) self.config_label = QLabel(self) self.config_ref = QLineEdit(config_ref, self) self.config_ref.setObjectName("fingerConfigRef") self.config_browse = QPushButton(self) - fields.addWidget(self.video_label, 0, 0) - fields.addWidget(self.video_path, 0, 1) - fields.addWidget(self.video_browse, 0, 2) - fields.addWidget(self.config_label, 1, 0) - fields.addWidget(self.config_ref, 1, 1) - fields.addWidget(self.config_browse, 1, 2) + fields.addWidget(self.source_label, 0, 0) + fields.addWidget(self.source_kind, 0, 1) + fields.addWidget(self.video_label, 1, 0) + fields.addWidget(self.video_path, 1, 1) + fields.addWidget(self.video_browse, 1, 2) + fields.addWidget(self.camera_label, 2, 0) + fields.addWidget(self.camera_index, 2, 1) + fields.addWidget(self.config_label, 3, 0) + fields.addWidget(self.config_ref, 3, 1) + fields.addWidget(self.config_browse, 3, 2) layout.addLayout(fields) controls = QHBoxLayout() @@ -74,6 +101,9 @@ class FingerSourcePanel(QWidget): controls.addWidget(self.stop_button) controls.addWidget(self.status_label, 1) layout.addLayout(controls) + self.preview_hint = QLabel(self) + self.preview_hint.setWordWrap(True) + layout.addWidget(self.preview_hint) self.preview = QLabel(self) self.preview.setObjectName("fingerVideoPreview") @@ -91,15 +121,22 @@ class FingerSourcePanel(QWidget): self.config_browse.clicked.connect(self._choose_config) self.start_button.clicked.connect(self.start) self.stop_button.clicked.connect(self.stop) + self.source_kind.currentIndexChanged.connect(self._on_source_changed) + self._on_source_changed() self.retranslate() def retranslate(self) -> None: + self.source_label.setText(t("source.kind")) + self.source_kind.setItemText(0, t("source.kind_video")) + self.source_kind.setItemText(1, t("source.kind_camera")) self.video_label.setText(t("source.video")) + self.camera_label.setText(t("source.camera_index")) self.config_label.setText(t("source.config")) self.video_browse.setText(t("source.browse")) self.config_browse.setText(t("source.browse")) self.start_button.setText(t("source.start")) self.stop_button.setText(t("source.stop")) + self.preview_hint.setText(t("source.preview_only")) self._set_status(self._status_key) if self._last_image is None: self.preview.setText(t("source.no_frame")) @@ -121,12 +158,21 @@ class FingerSourcePanel(QWidget): if path: self.config_ref.setText(path) + def _on_source_changed(self) -> None: + video = self.source_kind.currentData() == "video" + for widget in (self.video_label, self.video_path, self.video_browse): + widget.setVisible(video) + self.camera_label.setVisible(not video) + self.camera_index.setVisible(not video) + def start(self) -> None: video = Path(self.video_path.text().strip()) config = Path(self.config_ref.text().strip()) - if not video.is_file() or not config.is_file(): + is_video = self.source_kind.currentData() == "video" + if not config.is_file() or (is_video and not video.is_file()): self._show_error(t("source.missing_input")) return + source = str(video) if is_video else self.camera_index.value() self._error = "" self._last_frame_id = None self._last_image = None @@ -134,7 +180,7 @@ class FingerSourcePanel(QWidget): self.preview.setText(t("source.no_frame")) self.frame_label.clear() try: - self.worker.start(str(video), config_ref=str(config)) + self.worker.start(source, config_ref=str(config)) except Exception as exc: # noqa: BLE001 - show source/config errors to operator self._show_error(str(exc)) return @@ -155,6 +201,8 @@ class FingerSourcePanel(QWidget): def _set_running_controls(self, running: bool) -> None: self.start_button.setEnabled(not running) self.stop_button.setEnabled(running) + self.source_kind.setEnabled(not running) + self.camera_index.setEnabled(not running) for widget in ( self.video_path, self.video_browse, @@ -181,9 +229,21 @@ class FingerSourcePanel(QWidget): if identity != self._last_frame_id: self._last_frame_id = identity self._show_frame(packet.processed_frame) + acquired = datetime.fromtimestamp(packet.timestamp).strftime( + "%H:%M:%S.%f" + )[:-3] + basis_key = { + "host_replay": "source.basis_replay", + "host_camera": "source.basis_camera", + }.get(getattr(packet, "time_basis", ""), "source.basis_unknown") + basis = t(basis_key) self.frame_label.setText( t("source.frame").format( - frame=packet.frame_id, time=packet.timestamp + frame=packet.frame_id, + source_frame=getattr(packet, "source_frame_id", None) or "–", + time=acquired, + basis=basis, + dropped=getattr(packet, "capture_dropped", 0), ) ) if not self._error: @@ -194,9 +254,12 @@ class FingerSourcePanel(QWidget): self._active = False self._set_running_controls(False) if not self._error: - self._set_status( - "source.finished" if reason == "ended" else "source.stopped" - ) + key = { + "ended": "source.finished", + "disconnected": "source.disconnected", + "error": "source.failed", + }.get(reason, "source.stopped") + self._set_status(key) def _show_frame(self, frame: np.ndarray) -> None: if frame.ndim != 3 or frame.shape[2] != 3: diff --git a/apps/autocal/src/orisys_autocal/ui/window.py b/apps/autocal/src/orisys_autocal/ui/window.py index f34894c..e77a5e9 100644 --- a/apps/autocal/src/orisys_autocal/ui/window.py +++ b/apps/autocal/src/orisys_autocal/ui/window.py @@ -292,6 +292,8 @@ class AutoCalMainWindow(QMainWindow): video_path=str(self.prefs.get("finger_video_path") or ""), config_ref=str(self.prefs.get("finger_config_ref") or ""), parent=body, + source_kind=str(self.prefs.get("finger_source_kind") or "video"), + camera_index=self.prefs.get("finger_camera_index") or 0, ) self.workspace = WorkspaceTabs( self.axis_panel, @@ -1370,6 +1372,8 @@ class AutoCalMainWindow(QMainWindow): self.prefs["finger_config_ref"] = ( self.finger_source_panel.config_ref.text().strip() ) + self.prefs["finger_source_kind"] = self.finger_source_panel.source_kind.currentData() + self.prefs["finger_camera_index"] = self.finger_source_panel.camera_index.value() try: save_preferences(self.prefs) except OSError: diff --git a/apps/autocal/tests/test_finger_source_panel.py b/apps/autocal/tests/test_finger_source_panel.py index c0071d8..632bba2 100644 --- a/apps/autocal/tests/test_finger_source_panel.py +++ b/apps/autocal/tests/test_finger_source_panel.py @@ -71,9 +71,33 @@ def test_sensor_tab_is_live_panel_and_paths_are_saved(make_window, tmp_dir): assert window.workspace.page("sensor") is window.finger_source_panel window.finger_source_panel.video_path.setText(str(tmp_dir / "input.mp4")) window.finger_source_panel.config_ref.setText(str(tmp_dir / "sensor.json")) + window.finger_source_panel.source_kind.setCurrentIndex(1) + window.finger_source_panel.camera_index.setValue(3) window._persist_prefs() assert window.prefs["finger_video_path"].endswith("input.mp4") assert window.prefs["finger_config_ref"].endswith("sensor.json") + assert window.prefs["finger_source_kind"] == "camera" + assert window.prefs["finger_camera_index"] == 3 + + +def test_live_camera_uses_integer_source_and_same_worker(qt_app, tmp_dir, monkeypatch): + from orisys_autocal.ui import finger_source_panel + + monkeypatch.setattr( + finger_source_panel, "FingerAcquisitionWorker", FakeFingerWorker + ) + config = tmp_dir / "sensor.json" + config.write_text("{}", encoding="utf-8") + panel = finger_source_panel.FingerSourcePanel( + config_ref=str(config), source_kind="camera", camera_index=3 + ) + panel.start_button.click() + assert panel.worker.started == (3, str(config)) + assert not panel.camera_index.isEnabled() + panel.stop_button.click() + panel._poll() + assert panel.camera_index.isEnabled() + panel.shutdown() def test_missing_input_is_visible_and_does_not_start_worker(qt_app, monkeypatch): diff --git a/shared/pyproject.toml b/shared/pyproject.toml index fcde39f..4413b54 100644 --- a/shared/pyproject.toml +++ b/shared/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ "numpy>=1.24.4", "scipy>=1.10.1", "pyqtgraph>=0.13", - "orisys>=0.5", + "orisys>=0.5.4", ] [project.optional-dependencies] diff --git a/shared/src/orisys_app_shared/acquisition/finger_worker.py b/shared/src/orisys_app_shared/acquisition/finger_worker.py index 5391e95..017ebde 100644 --- a/shared/src/orisys_app_shared/acquisition/finger_worker.py +++ b/shared/src/orisys_app_shared/acquisition/finger_worker.py @@ -7,6 +7,7 @@ import threading import time from collections import deque from collections.abc import Callable +from dataclasses import dataclass from pathlib import Path import numpy as np @@ -71,6 +72,38 @@ class _RuntimeCapture: return getattr(self._runtime, "capture_source", None) +@dataclass(frozen=True) +class _FingerFrameMeta: + raw_frame: np.ndarray + source_seq: int | None + mono_ns: int + utc_ns: int + media_ms: float | None + stamped: bool + + +def _read_source_with_stamp(runtime): + """Take the frame and its capture stamp through one runtime read.""" + read_stamped = getattr(runtime, "read_source_stamped", None) + if callable(read_stamped): + captured = read_stamped() + if captured is None: + return None + return ( + captured.image, + captured.sequence, + captured.host_mono_ns, + captured.host_utc_ns, + captured.media_time_ms, + True, + ) + # Older SDK runtimes still work, but their read has no frame-bound stamp. + ok, frame = runtime.read_source() + if not ok or frame is None: + return None + return (frame, None, time.perf_counter_ns(), time.time_ns(), None, False) + + class _FingerProducer: """The ``StreamPipeline`` produce callable for a ``FingerRuntime``. @@ -84,18 +117,15 @@ class _FingerProducer: each frame as many times as it could poll. Only a source with a running background reader has the sequence counter that says so; a file decodes a new frame per call and is read once per call. - * **The pre-processing frame.** ``runtime.process_frame`` hands the unrolled - frame to the tactile sensor and drops the source frame; the record channel's - pre-roll history wants the source frame, so it is copied here while it is - still this frame's, and handed on by sequence. + * **The source frame and capture stamp.** They travel with the unrolled image + through the pipeline queues, so a pause or backlog cannot make a snapshot + pick up another frame's provenance. * **The camera's own rate.** The consumer reads ``capture_fps`` from this producer, the same way the box path reads it from ``_FrameProducer``: the source's delivered-frame counter, anchored to the wall clock of the wait that observed it. """ - CACHE = 8 - def __init__(self, runtime, *, is_file: bool, frame_period: float = 0.0) -> None: self.runtime = runtime self.is_file = bool(is_file) @@ -105,8 +135,6 @@ class _FingerProducer: self.produced = 0 self.capture_source = _RuntimeCapture(runtime) self.capture_seq = _capture_sequence(self.capture_source) - self._cache: deque = deque(maxlen=self.CACHE) - self._times: deque = deque() self._last_read_at: float | None = None #: ``(clock, seconds, delivered frames)`` samples behind ``capture_fps``. self._rate_samples: deque = deque() @@ -118,20 +146,32 @@ class _FingerProducer: self._wait_for_a_new_frame() self._pace() t0 = time.perf_counter() - ok, source_frame = self.runtime.read_source() - if not ok or source_frame is None: + captured = _read_source_with_stamp(self.runtime) + if captured is None: return None + source_frame, source_seq, mono_ns, utc_ns, media_ms, stamped = captured + if not self.is_file and source_seq is not None: + # A newer frame can arrive between the sequence wait and this read. + # Follow the frame actually taken, so it is not published twice. + self.capture_seq = source_seq unrolled = self.runtime.process_frame(source_frame, color=False) if unrolled is None: return None self._last_read_at = t0 self.produced += 1 - self._times.append((self.produced, t0)) - cutoff = t0 - FPS_WINDOW_S - while self._times and self._times[0][1] < cutoff: - self._times.popleft() - self._cache.append((self.produced, np.array(source_frame, copy=True))) - return unrolled + from orisys.pipeline import ProducedFrame + + return ProducedFrame( + unrolled, + _FingerFrameMeta( + np.array(source_frame, copy=True), + source_seq, + mono_ns, + utc_ns, + media_ms, + stamped, + ), + ) def _pace(self) -> None: if not self.is_file or self.frame_period <= 0.0 or self._last_read_at is None: @@ -160,27 +200,6 @@ class _FingerProducer: return time.sleep(0) - def raw_frame(self, seq: int) -> np.ndarray | None: - for number, frame in reversed(self._cache): - if number == seq: - return frame - return None - - def collect_perf(self, seq: int) -> float | None: - self._trim() - for number, stamp in reversed(self._times): - if number == seq: - return stamp - return None - - def _trim(self) -> None: - """Drop collection times older than the frame-rate window.""" - if not self._times: - return - cutoff = self._times[-1][1] - FPS_WINDOW_S - while self._times and self._times[0][1] < cutoff: - self._times.popleft() - def capture_window(self, previous: tuple[float, int] | None) -> tuple[float, int]: """``(seconds, delivered frames)`` over the interval since ``previous``. @@ -590,7 +609,7 @@ class FingerAcquisitionWorker(RecordSinkMixin): try: runtime.close() except Exception: - pass + logger.exception("Could not close finger runtime for session %s", session) def _close_all_runtimes(self) -> None: with self._lock: @@ -718,10 +737,8 @@ class FingerAcquisitionWorker(RecordSinkMixin): The pipeline is built the way ``runtime.pipeline()`` builds it — a ``StreamPipeline`` over the runtime's tactile sensor — but with this worker's own produce callable in place of ``grab_and_process``. The callable - is what reads the source, so it is where the two things that must happen on - the producer thread live: one published frame per *delivered* frame (a live - source hands the same frame back until its reader produces another), and the - copy of the pre-unroll source frame the record channel's history wants. See + is what reads the source, so it is where one published frame per *delivered* + frame is enforced and where the source frame and capture stamp are bound. See :class:`_FingerProducer`. """ import queue as queue_module @@ -816,7 +833,6 @@ class FingerAcquisitionWorker(RecordSinkMixin): live_capture=live_capture, snap=snap, frame_id=frame_id, - frame_period=frame_period, is_file=is_file, frame_times=frame_times, mono_t0=mono_t0, @@ -940,7 +956,6 @@ class FingerAcquisitionWorker(RecordSinkMixin): live_capture: bool, snap, frame_id: int, - frame_period: float, is_file: bool, frame_times: list[float], mono_t0: float, @@ -949,9 +964,13 @@ class FingerAcquisitionWorker(RecordSinkMixin): compute_error: str, ): """One :class:`SdkPacket` from one finished finger snapshot.""" - produced_seq = frame_id + 1 - t_collect = producer.collect_perf(produced_seq) or time.perf_counter() - mono = frame_id * frame_period if is_file else max(0.0, t_collect - mono_t0) + meta = snap.source_metadata + if not isinstance(meta, _FingerFrameMeta): + raise RuntimeError(f"capture metadata missing for frame {frame_id}") + source_seq, mono_ns, utc_ns = meta.source_seq, meta.mono_ns, meta.utc_ns + media_ms, stamped = meta.media_ms, meta.stamped + t_collect = mono_ns / 1e9 + mono = max(0.0, mono_ns / 1e9 - mono_t0) frame_times.append(t_collect) cutoff = t_collect - FPS_WINDOW_S del frame_times[ @@ -969,8 +988,8 @@ class FingerAcquisitionWorker(RecordSinkMixin): taken = _capture_dropped(producer.capture_source) if taken is not None: capture_dropped = int(taken) - elif producer.produced > produced_seq: - capture_dropped = int(producer.produced - produced_seq) + elif producer.produced > snap.seq + 1: + capture_dropped = int(producer.produced - snap.seq - 1) with self._lock: self._measured_capture_fps = capture_fps raw_sensor = getattr(tactile, "raw_sensor", None) @@ -980,10 +999,18 @@ class FingerAcquisitionWorker(RecordSinkMixin): session_id=session, frame_id=frame_id, mono_t=mono, + wall_t=utc_ns / 1e9, + capture_mono_ns=mono_ns if stamped else None, + capture_utc_ns=utc_ns if stamped else None, + source_frame_id=source_seq, + time_basis=("host_replay" if is_file else "host_camera") + if stamped + else "host_fallback", + media_time_ms=media_ms, fps=fps, capture_fps=capture_fps, capture_dropped=capture_dropped, - raw_frame=producer.raw_frame(produced_seq), + raw_frame=meta.raw_frame, valid_mask=runtime.valid_mask, contact_analysed=True, contact_found=bool(getattr(raw_sensor, "centroid_found", False)), @@ -1061,12 +1088,13 @@ class FingerAcquisitionWorker(RecordSinkMixin): break t0 = time.perf_counter() - ok, source_frame = runtime.read_source() - if not ok or source_frame is None: + captured = _read_source_with_stamp(runtime) + if captured is None: if saw_frame and is_file and none_streak_start is None: none_streak_start = time.perf_counter() time.sleep(0.005) continue + source_frame, source_seq, mono_ns, utc_ns, media_ms, stamped = captured if self._pause.is_set(): # Paused: the source read above proves the camera is alive, so refresh the @@ -1091,7 +1119,7 @@ class FingerAcquisitionWorker(RecordSinkMixin): compute_error = "" # The frame's acquisition time, so an event that happens on this # frame is dated when it happened rather than when it was queued. - frame_mono = max(0.0, time.perf_counter() - self._mono_t0) + frame_mono = max(0.0, mono_ns / 1e9 - self._mono_t0) try: self._consume_pending_calibration(tactile, frame_id, frame_mono) except Exception as exc: # noqa: BLE001 @@ -1166,9 +1194,16 @@ class FingerAcquisitionWorker(RecordSinkMixin): packet = SdkPacket( raw_frame=raw_frame, processed_frame=_ensure_bgr(unrolled), - timestamp=time.time(), - wall_t=time.time(), - mono_t=max(0.0, t1 - self._mono_t0), + timestamp=utc_ns / 1e9, + wall_t=utc_ns / 1e9, + mono_t=frame_mono, + capture_mono_ns=mono_ns if stamped else None, + capture_utc_ns=utc_ns if stamped else None, + source_frame_id=source_seq, + time_basis=("host_replay" if is_file else "host_camera") + if stamped + else "host_fallback", + media_time_ms=media_ms, fps=fps, process_ms=(t1 - t0) * 1000.0, normal_force=forces["normal_force"], diff --git a/shared/src/orisys_app_shared/acquisition/models.py b/shared/src/orisys_app_shared/acquisition/models.py index 564b28b..b0db9c3 100644 --- a/shared/src/orisys_app_shared/acquisition/models.py +++ b/shared/src/orisys_app_shared/acquisition/models.py @@ -84,6 +84,14 @@ class SdkPacket: mono_t: float | None = None #: Wall clock at hand-off, for metadata only. ``None`` means unmeasured. wall_t: float | None = None + #: Host clocks bound to the source image at read completion. ``None`` for + #: older packets and producers without a stamped capture interface. + capture_mono_ns: int | None = None + capture_utc_ns: int | None = None + source_frame_id: int | None = None + #: ``host_camera``, ``host_replay`` or ``host_fallback`` for finger input. + time_basis: str = "" + media_time_ms: float | None = None #: False when the force readout failed. Invalid packets carry NaN forces and #: must never be averaged into calibration data or written as zeros. valid: bool = True @@ -163,6 +171,11 @@ def from_snapshot( frame_id: int | None = None, mono_t: float | None = None, wall_t: float | None = None, + capture_mono_ns: int | None = None, + capture_utc_ns: int | None = None, + source_frame_id: int | None = None, + time_basis: str = "", + media_time_ms: float | None = None, fps: float | None = None, process_ms: float | None = None, capture_fps: float = 0.0, @@ -223,6 +236,11 @@ def from_snapshot( timestamp=wall, wall_t=wall, mono_t=float(mono_t), + capture_mono_ns=capture_mono_ns, + capture_utc_ns=capture_utc_ns, + source_frame_id=source_frame_id, + time_basis=time_basis, + media_time_ms=media_time_ms, fps=float(snapshot.fps if fps is None else fps), process_ms=float( (snapshot.produce_ms + snapshot.flow_ms + snapshot.hhd_ms) diff --git a/shared/tests/test_stream_pipeline_acquisition.py b/shared/tests/test_stream_pipeline_acquisition.py index 18dda04..b3be318 100644 --- a/shared/tests/test_stream_pipeline_acquisition.py +++ b/shared/tests/test_stream_pipeline_acquisition.py @@ -176,6 +176,7 @@ class _Runtime: """A stand-in for ``FingerRuntime`` with the API the worker uses.""" def __init__(self, *args, **kwargs) -> None: + self._source_seq = 0 self.tactile = types.SimpleNamespace( raw_sensor=_Sensor(), reset=lambda: None, @@ -192,6 +193,16 @@ class _Runtime: def read_source(self): return True, np.zeros((8, 8, 3), dtype=np.uint8) + def read_source_stamped(self): + self._source_seq += 1 + return types.SimpleNamespace( + image=np.zeros((8, 8, 3), dtype=np.uint8), + sequence=self._source_seq, + host_mono_ns=time.perf_counter_ns(), + host_utc_ns=time.time_ns(), + media_time_ms=self._source_seq * 50.0, + ) + def process_frame(self, frame, *, color=False): return np.zeros((8, 8), dtype=np.uint8) @@ -276,6 +287,14 @@ def test_finger_video_and_camera_use_the_same_packet_path(source, monkeypatch): assert packet.processed_frame.shape == (8, 8, 3) assert packet.session_id == 1 assert packet.frame_id == 0 + assert packet.source_frame_id == 1 + assert packet.capture_mono_ns is not None + assert packet.capture_utc_ns is not None + assert packet.timestamp == pytest.approx(packet.capture_utc_ns / 1e9) + assert packet.time_basis == ( + "host_replay" if isinstance(source, str) else "host_camera" + ) + assert packet.media_time_ms == 50.0 finally: worker.stop(wait=True) _restore(saved)