From 22cc59c40294f335043870056fb42d16e57d3710 Mon Sep 17 00:00:00 2001 From: ykcheungab Date: Sun, 13 Sep 2026 14:55:53 +0800 Subject: [PATCH] Fix the phase-1/2 review findings in recording, calibration and depth replay What - record: one packet->row conversion (record/sample.py). A failed compute chain records no force value at all, a newton value is written only for an axis that really has a calibration curve, and mono_t (not the wall clock) orders history with a per-row calibration hash. - record lifecycle: explicit starting/recording/stopping/stopped/failed states owned by the writer thread; stop() reports completed=False on timeout and does not write ended_at; consecutive write failures stop acceptance immediately. - record capacity: byte budget for the video queue (512 MiB default); a full queue sheds the *frame* and parks the tiny data row, so a video overflow no longer deletes CSV rows. Lost rows leave a dropped=1 index row and are counted as dropped_gaps / unrecorded_gaps. - calibration identity: sha256 content hash instead of a path, minted from the calibration the sensor actually loaded (reported before the first frame) and confirmed by the acquisition thread; events are timestamped when they happen. - calibration diagnostics: the fit summary, residual and validation error now come from the model that export/apply would use (piecewise_linear); validation points are held out of the build; repeated reference loads are merged with an n-weighted mean and pooled std. - calibration record: every point keeps its full three-axis sample series (frame id, time, raw, newtons, valid) plus diagnostics, written to an independent calibration_session_*.json that can be reopened for inspection; save record / export coefficients / apply to device are three separate actions. - depth replay: the cursor's frame is bound to the depth result by request sequence and frame id, so a moved cursor never shows another frame's map; the SDK is stepped from its current position instead of re-reading from the start; the live chain (box deformation+contact, finger unroll+tactile) is replayed with the exact resolved config path; algorithm state is rebuilt from a reset marker or from the recorded pre-roll, and says so when it cannot be. Why The review (D:/orisys_ws/_review_phase12/review.md) showed the app writing raw counts into the newton columns, recording a stale value after a failed compute, losing CSV rows when the video queue filled, reporting success while a writer was still running, diagnosing a model nobody applies, applying a calibration before the device confirmed it, dropping the samples behind a calibration point, showing a stale depth map under a moved cursor and recomputing depth of the wrong frame. Verification - D:/orisys_ws/_review_phase12/reproduce.py: all ten acceptance lines flipped (log saved as _review_phase12/reproduce_after.log). Including 25/25 rows kept with 8 video frames shed, frames [2]->[3], depth 8.0/8.0, fit 2/2 points. - apps/studio/tools/ui_check.py: 249 checks, 0 failures (twice). - shared suites: 91 tests, 0 failures (new: test_record_semantics, test_record_lifecycle, test_calibration_model, test_calibration_record, test_depth_replay). - apps/studio/tests/test_ui_smoke.py: 23/30; the 7 remaining are harness artifacts of the sandbox (4 tmp_path PermissionError, 3 parametrized tests the shim cannot drive), unchanged from before this change. - compileall clean; locale parity zh/en + t() key coverage: 0 failures. Not in this change - P2#10 disk-based browsing of a long recording and range CSV export (next round, per your choice). - Stability-triggered calibration sampling, replay fixed colour scale, PyOpenGL three-dimensional view. - Hardware verification: no camera attached, so the SDK's real processing chain and the pre-roll's effect on a live tracker were exercised with fakes only. --- .gitignore | 1 + apps/studio/config/locales/en_US.json | 38 +- apps/studio/config/locales/zh_CN.json | 37 +- apps/studio/docs/record-channel.md | 77 +++ apps/studio/src/orisys_studio/app_window.py | 139 ++++- apps/studio/src/orisys_studio/ui/cal_panel.py | 124 +++- .../src/orisys_studio/ui/force_plots.py | 17 +- .../src/orisys_studio/ui/playback_dialog.py | 122 +++- apps/studio/tests/test_ui_smoke.py | 40 +- apps/studio/tools/ui_check.py | 62 +- .../acquisition/finger_worker.py | 56 +- .../acquisition/sdk_worker.py | 174 +++++- .../orisys_app_shared/acquisition/sinks.py | 52 ++ .../orisys_app_shared/force/calibration_io.py | 74 ++- .../force/calibration_recorder.py | 148 ++++- .../force/calibration_session.py | 240 ++++++++ shared/src/orisys_app_shared/force/session.py | 109 +++- shared/src/orisys_app_shared/record/depth.py | 341 +++++++++-- .../orisys_app_shared/record/experiment.py | 578 +++++++++++++++--- .../src/orisys_app_shared/record/playback.py | 85 +++ shared/src/orisys_app_shared/record/sample.py | 216 +++++++ shared/tests/test_calibration_model.py | 153 +++++ shared/tests/test_calibration_record.py | 143 +++++ shared/tests/test_depth_replay.py | 248 ++++++++ shared/tests/test_record_lifecycle.py | 251 ++++++++ shared/tests/test_record_semantics.py | 152 +++++ 26 files changed, 3387 insertions(+), 290 deletions(-) create mode 100644 shared/src/orisys_app_shared/force/calibration_session.py create mode 100644 shared/src/orisys_app_shared/record/sample.py create mode 100644 shared/tests/test_calibration_model.py create mode 100644 shared/tests/test_calibration_record.py create mode 100644 shared/tests/test_depth_replay.py create mode 100644 shared/tests/test_record_lifecycle.py create mode 100644 shared/tests/test_record_semantics.py diff --git a/.gitignore b/.gitignore index b4edad3..1bb7297 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ apps/studio/preference.json .pytestlib/ .pytest_cache/ shared/tests/_scratch/ +shared/tests/_scratch_*/ # IDE / OS .idea/ diff --git a/apps/studio/config/locales/en_US.json b/apps/studio/config/locales/en_US.json index cc04211..49f5287 100644 --- a/apps/studio/config/locales/en_US.json +++ b/apps/studio/config/locales/en_US.json @@ -146,12 +146,22 @@ "depth_config_missing": "Cannot recompute: the recording has no sensor config reference", "depth_config_file_missing": "Cannot recompute: the config file no longer exists at that path", "depth_no_video": "Cannot recompute: this recording has no video segment", + "depth_not_recomputed": "Depth has not been recomputed for this cursor position\n(the previous frame's map is not reused)", + "depth_recomputing": "Recomputing depth for the current cursor position…", + "depth_frame_not_recorded": "This frame was not recorded: its data row was lost to a full queue and only the index row remains", + "depth_frame_not_in_video": "This frame never reached the video, so its depth cannot be recomputed", + "depth_state_across_segment": "Replay cannot carry the algorithm state across segments: this one restarts at its first frame", + "depth_state_approximate": "No reset or pre-roll history for this segment: the algorithm state is an approximation", + "depth_warnings": "Recomputation notes: {detail}", "warnings": "Replay notes: {detail}", "warn_config_changed": "the config differs from recording time, so the recomputed depth may differ", "warn_sdk_changed": "the SDK version differs from recording time, so the recomputation may differ", "warn_config_missing": "no config hash was recorded, so recomputation cannot be verified", "warn_depth_recomputed": "depth comes from offline recomputation, not from stored data", - "warn_rows_skipped": "some unparsable CSV rows were skipped" + "warn_rows_skipped": "some unparsable CSV rows were skipped", + "warn_frames_dropped": "frames were lost to a full queue; only their index rows remain (shown as \"not recorded\")", + "warn_state_across_segment": "the algorithm state cannot be carried across a segment", + "warn_state_approximate": "the algorithm state is an approximation" }, "record": { "label": "Record", @@ -180,7 +190,23 @@ "delete_row": "Delete row", "clear_channel": "Clear channel", "clear_all": "Clear all", - "export": "Export…", "sn": "SN", + "export": "Export coefficients…", + "export_tip": "Write the SDK coefficient file only; the running device is untouched", + "apply": "Apply to device", + "apply_tip": "Push the last export to the running device (confirmed again)", + "apply_no_export": "No exported coefficients yet — export them first", + "save_record": "Save record…", + "save_record_tip": "Save every point of this session with its full three-axis samples (audit data), independently of the coefficient file", + "save_record_empty": "No calibration points to save yet", + "save_record_ok": "Saved the calibration record to {path}\n{points} point(s), {samples} three-axis samples", + "save_record_failed": "Could not save the calibration record: {error}", + "open_record": "Open record…", + "open_record_tip": "Reopen a saved calibration session to inspect every point's samples and diagnostics", + "open_record_filter": "Calibration records (*.json)", + "open_record_ok": "Loaded {path} ({points} point(s))", + "open_record_warnings": "Note: {detail}", + "open_record_failed": "Could not open the calibration record: {error}", + "sn": "SN", "sn_placeholder": "Serial number (optional)", "live_raw_label": "Live raw", "live_raw": "Live raw FN={fn} FX={fx} FY={fy}", @@ -206,8 +232,10 @@ "marked_validation": "Point marked as a validation point", "unmarked_validation": "Validation mark removed", "fit_insufficient": "A fit needs at least 2 usable points (currently {count})", - "fit_summary": "Fit {channel}: N = {gain} × raw + {offset} ({count} points, max residual {residual} N)", + "fit_summary": "Exported model {channel}: {method} ({count} points / {breakpoints} breakpoints, max residual {residual} N)", "fit_validation": "Independent validation on {count} point(s), max deviation {error} N", + "method_piecewise_linear": "piecewise linear", + "method_linear": "linear", "point_title": "Calibration point {channel} @ {load} N", "point_channel": "Loaded axis", "point_direction": "Direction", @@ -230,7 +258,9 @@ "apply_title": "Apply calibration", "apply_confirm": "Apply the calibration just exported to the current device?\n\nSensor type: {sensor_type}\nConfig: {config}\n\nChoosing No keeps the exported file and leaves the running device untouched.", "apply_skipped": "Exported; not applied to the device", - "apply_done": "Calibration applied to {sensor_type}", + "apply_pending": "Requested calibration for {sensor_type}; waiting for the device to confirm…", + "applied": "Device confirmed the calibration is in effect (version {version})", + "export_only": "Exported coefficients to {path} (not applied to the device)", "apply_no_target": "No device to apply to (run an acquisition first)" }, "status": { diff --git a/apps/studio/config/locales/zh_CN.json b/apps/studio/config/locales/zh_CN.json index 98473a2..9e9d3d0 100644 --- a/apps/studio/config/locales/zh_CN.json +++ b/apps/studio/config/locales/zh_CN.json @@ -146,12 +146,22 @@ "depth_config_missing": "无法重算：录制未记录传感器配置", "depth_config_file_missing": "无法重算：配置文件的原始路径已不存在", "depth_no_video": "无法重算：该录制没有视频分段", + "depth_not_recomputed": "当前游标位置尚未重算深度\n（不会沿用其他帧的结果）", + "depth_recomputing": "正在为当前游标位置重算深度…", + "depth_frame_not_recorded": "该帧未录制：数据行在队列溢出时丢失，只有索引记录", + "depth_frame_not_in_video": "该帧未写入视频，无法重算深度", + "depth_state_across_segment": "重算无法跨分段延续算法状态：本段从段首重放", + "depth_state_approximate": "本段缺少归零或预卷历史，算法参考状态为近似重建", + "depth_warnings": "重算提示: {detail}", "warnings": "回放提示: {detail}", "warn_config_changed": "配置文件与录制时不一致，重算出的深度可能与当时不同", "warn_sdk_changed": "SDK 版本与录制时不一致，重算结果可能不同", "warn_config_missing": "找不到配置哈希，无法校验重算一致性", "warn_depth_recomputed": "深度由离线重算得到，不是当时保存的数据", - "warn_rows_skipped": "有无法解析的 CSV 行被跳过" + "warn_rows_skipped": "有无法解析的 CSV 行被跳过", + "warn_frames_dropped": "有帧在队列溢出时丢失，仅保留索引记录（回放显示为“该帧未录制”）", + "warn_state_across_segment": "重算不能跨分段延续算法状态", + "warn_state_approximate": "算法参考状态为近似重建" }, "record": { "label": "记录", @@ -180,7 +190,22 @@ "delete_row": "删除行", "clear_channel": "清空通道", "clear_all": "全部清空", - "export": "导出…", + "export": "导出系数…", + "export_tip": "只写出 SDK 标定系数文件，不改变运行中的设备", + "apply": "应用到设备", + "apply_tip": "把最近导出的系数下发到运行中的设备（需再次确认）", + "apply_no_export": "还没有导出的系数文件，请先「导出系数」", + "save_record": "保存标定记录…", + "save_record_tip": "保存本次标定会话的全部点与三轴完整采样（审计数据），与系数文件相互独立", + "save_record_empty": "还没有标定点，无法保存标定记录", + "save_record_ok": "已保存标定记录 {path}\n{points} 个点，共 {samples} 条三轴采样", + "save_record_failed": "保存标定记录失败: {error}", + "open_record": "打开标定记录…", + "open_record_tip": "重新打开已保存的标定会话，检查每个点的完整采样与诊断状态", + "open_record_filter": "标定记录 (*.json)", + "open_record_ok": "已载入 {path}（{points} 个点）", + "open_record_warnings": "注意: {detail}", + "open_record_failed": "打开标定记录失败: {error}", "sn": "序列号", "sn_placeholder": "序列号（可选）", "live_raw_label": "实时原始值", @@ -207,8 +232,10 @@ "marked_validation": "该点已标记为验证点", "unmarked_validation": "该点已取消验证点标记", "fit_insufficient": "拟合需要至少 2 个有效点（当前 {count} 个）", - "fit_summary": "拟合 {channel}: N = {gain} × raw + {offset}（{count} 点，最大残差 {residual} N）", + "fit_summary": "导出模型 {channel}: {method}（{count} 点 / {breakpoints} 个折点，最大残差 {residual} N）", "fit_validation": "独立验证 {count} 点，最大偏差 {error} N", + "method_piecewise_linear": "分段线性", + "method_linear": "线性", "point_title": "标定点 {channel} @ {load} N", "point_channel": "加载轴", "point_direction": "方向", @@ -231,7 +258,9 @@ "apply_title": "应用标定", "apply_confirm": "将刚导出的标定应用到当前设备？\n\n设备类型: {sensor_type}\n配置: {config}\n\n选择「否」只保留导出文件，不改动运行中的设备。", "apply_skipped": "已导出，未应用到设备", - "apply_done": "标定已应用到 {sensor_type}", + "apply_pending": "已请求把标定应用到 {sensor_type}，等待设备确认…", + "applied": "设备已确认标定生效（版本 {version}）", + "export_only": "已导出系数 {path}（未应用到设备）", "apply_no_target": "没有可应用的设备（请先运行一次采集）" }, "status": { diff --git a/apps/studio/docs/record-channel.md b/apps/studio/docs/record-channel.md index 355623c..412748f 100644 --- a/apps/studio/docs/record-channel.md +++ b/apps/studio/docs/record-channel.md @@ -166,6 +166,83 @@ outputs/recordings/_session/ --- +## 7. 第一、二阶段评审 P1/P2 的整改（数据语义、终态、标定身份、深度重算） + +评审脚本 `D:/orisys_ws/_review_phase12/reproduce.py` 的十条复现条件已全部翻转，改后 +输出见同目录 `reproduce_after.log`。以下按发现逐条说明改法。 + +### 7.1 记录语义（P1-1） + +唯一转换入口 `shared/.../record/sample.py::to_record_sample`。规则： + +1. 计算链失败（`valid=False` 或 `stale_forces=True`）的帧，六个力值**全部**记为不可用。 + SDK 的 `info.FNORMAL` 在计算失败时仍返回上一帧的数值，记录它等于伪造测量值。 +2. 只有**该轴确实加载了标定曲线**才写 N 值（`calibrated_axes` 按轴判断，而不是 + `force_calibrated` 这个传感器级标志）。未标定的轴留空。 +3. `mono_t` 是排序基准，墙钟只进元数据。`SdkPacket.mono_t/wall_t` 默认 `None` 而不是 + `0.0`：默认 0 会让所有帧的时间相同。 +4. 每行 `calibration_id` 取 **packet 自带版本**（采集线程在采集瞬间盖的哈希）， + 会话级版本只作回退；一份 CSV 中途换标定也说得清。 + +### 7.2 记录终态与队列损失（P1-5、P1-6） + +- 生命周期显式化为 `starting/recording/stopping/stopped/failed`；**写线程拥有终态与最终 + 报告**，文件打开、写入、flush、close 全在异常边界内。连续写失败会立刻停止接收 + （`rejected_after_failure` 计数），`is_recording` 随之变假 —— 不会出现 failed=True 却 + 仍在录制。`starting` 也算“录制中”，否则开始录制后立刻点停止会失效。 +- `stop(wait=True, timeout=…)` 超时返回 `completed=False` 的报告，**不写 `ended_at`**、 + 不丢弃线程引用；未结束的 recorder 不允许再次 `start()`。 +- 队列按**字节预算**（`max_queue_bytes`，默认 512 MiB）而不是仅按条数：1920×1080 BGR + 排 2048 条副本约 11.9 GiB。预算用尽时**丢视频保数据行**；连条目队列都满时把数据行放进 + 独立的 `_row_backlog`（默认 8192 行），因此视频溢出不再连带丢掉 CSV。 +- 真的丢掉的数据行会在 `frames.csv` 留下 `dropped=1` 的索引行，回放据此显示“该帧未录制”， + 而不是拿上一帧冒充。统计区分 `dropped_data`、`dropped_video`、`dropped_gaps` + （已标记的缺口）与 `unrecorded_gaps`（连标记都放不下的）。 + +### 7.3 标定身份与生效确认（P1-7） + +- 标定身份是**内容哈希**（`calibration_digest`，sha256），不是路径：同一路径可以重新导出 + 不同系数。`Sensor` 打开时自带 bundle 标定，因此采集线程在**第一帧之前**就把实际加载的 + 标定哈希报出来（`on_calibration_applied(path, version, frame_id)`），事件与每行版本都 + 来自这次确认，而不是“已下发”的意图。 +- 事件在**发生时**打时间戳（`log_event(..., mono_t=…, frame_id=…)`），不再在写线程出队时 + 重新计时。 +- 面板拆成三个独立动作：**保存标定记录 / 导出系数 / 应用到设备**。导出只写文件， + 应用需要再次确认并等设备确认后才记 `calibration_applied`。 + +### 7.4 标定诊断与实验明细（P1-2、P1-8） + +- `CalibrationRecorder.fit` 用**将要导出的那个模型**（`build_export_model` + SDK 的 + `Calibration.convert`）计算残差与验证误差。分段线性模型会精确穿过它自己的标定点， + 所以训练残差恒为 0 —— 真正有意义的诊断是**留出验证点**的偏差。界面按“导出模型：分段线性 + （N 点 / M 个折点，最大残差）…”显示。 +- 验证点**不进入**构建（`CalPanel._rebuild_store` 跳过 `is_validation`），重复参考载荷按 + n 加权合并（`merge_samples`），不再“最后一个覆盖前面的”。 +- `CalibrationPoint.samples` 保存该点**全部**三轴采样（帧号、相对时间、三轴 raw、三轴 N、 + valid），连同排除/验证状态、方向、备注与实际时长一起写入独立的标定会话文件 + （`calibration_session_*.json`，`force/calibration_session.py`），并可用「打开标定记录」 + 重新载入检查。界面上的降采样 trace 只是预览，不是审计数据。 + +### 7.5 深度重算与游标同步（P1-3、P1-4） + +- **不再重开回放位置**：`DepthRecomputer` 记录当前已读到的段内帧号，请求第 2 帧再请求第 3 + 帧只多读 1 帧（旧实现每次都从头读，2 和 3 会落到 5）；往回退才重开视频。 +- 每帧都执行**实时同一条链路**：盒子 `compute_deformation + compute_contact`，手指再加上 + `FingerRuntime` 的展开与有效区域（`_BoxReplay` / `_FingerReplay`）。 +- `config_name` 用**完整解析后的配置路径**（`load_fixed_mode`），不再缩减成父目录名或 + stem —— 那可能加载同名但不同的配置，而哈希校验还在比对原路径。 +- **算法参考状态**：SDK 的跟踪器/滤波是有状态的，离线复现需要历史。两条可用依据： + 1. 操作归零（`zeroing` 事件，带帧号）会清空跟踪器，因此从该帧重放是**精确**的； + 2. 录制开始前采集线程保留 `PREROLL_FRAMES`（30 帧，且不超过 192 MiB）的预处理帧， + 录制开始时写入 `preroll_0001.mp4`，`session.json` 记 `preroll_frames` / + `preroll_segment`，回放先把它们喂给 SDK 再进入首帧。 + 都不满足时（跨分段、无归零、无预卷）仍然重算，但结果带 + `state_approximate` / `state_across_segment` 提示，**不假装与当时一致**。 +- 回放游标与深度结果按**请求序号 + 帧号**绑定：等待期间移动游标会作废旧请求，结果回来 + 只渲染属于当前帧的那一张；否则面板显示「当前游标位置尚未重算深度」，绝不复用旧图。 + +--- + ## 7. 同步回放与深度重算 **决定：不保存深度。** 回放时按录像与配置**离线重算**。 diff --git a/apps/studio/src/orisys_studio/app_window.py b/apps/studio/src/orisys_studio/app_window.py index f3df000..ea5c209 100644 --- a/apps/studio/src/orisys_studio/app_window.py +++ b/apps/studio/src/orisys_studio/app_window.py @@ -37,6 +37,7 @@ from orisys_app_shared.force.calibration_recorder import CalibrationRecorder from orisys_app_shared.force.session import SessionSample, SessionStore from orisys_app_shared.paths import recordings_dir from orisys_app_shared.record import ExperimentRecorder, RecordConfig +from orisys_app_shared.record.sample import to_record_sample from orisys_app_shared.i18n import set_locale, t from orisys_app_shared.logging import report_error from orisys_app_shared.paths import logo_path, sensor_json_path @@ -142,6 +143,10 @@ class StudioMainWindow(QMainWindow): self._record_error_count = 0 self._dropped_reported = 0 self._active_calibration_id = "" + #: Last coefficients file written by the calibration panel (not applied). + self._last_calibration_export = "" + #: Confirmed calibration changes, handed over from the acquisition thread. + self._pending_calibration_events: list[tuple[str, str, int]] = [] self._depth_range_value: tuple[float, float] | None = None self._last_depth_frame = None @@ -192,6 +197,7 @@ class StudioMainWindow(QMainWindow): self.statusBar().addPermanentWidget(self.status_chip) self.cal_panel.status_message.connect(self.statusBar().showMessage) self.cal_panel.calibration_exported.connect(self._on_calibration_exported) + self.cal_panel.apply_requested.connect(self._on_calibration_apply) self._set_phase("idle") self._build_menus() self._wire_toolbar() @@ -230,11 +236,13 @@ class StudioMainWindow(QMainWindow): self._box_worker = SdkAcquisitionWorker( on_error=self._on_worker_error, on_status=self._on_worker_status, + on_calibration_applied=self._on_calibration_applied, ) if self._finger_worker is None: self._finger_worker = FingerAcquisitionWorker( on_error=self._on_worker_error, on_status=self._on_worker_status, + on_calibration_applied=self._on_calibration_applied, ) # Both workers publish into the same record channel. for worker in (self._box_worker, self._finger_worker): @@ -278,6 +286,11 @@ class StudioMainWindow(QMainWindow): session_id=int(self.session_store.session_id), metadata=self._build_session_metadata(config), calibration_id=self._active_calibration_id, + calibration_version=self._active_calibration_id, + # The frames acquired just before this moment. The depth chain is + # stateful, so without them the first frames of the recording + # could never be recomputed faithfully. + preroll=self._take_preroll(), ) except Exception as exc: self.toolbar.set_record_checked(False) @@ -313,6 +326,23 @@ class StudioMainWindow(QMainWindow): ) ) + def _take_preroll(self) -> list: + """Frame history from the active worker, if it keeps one. + + The buffered frames are consumed (and the buffer restarted) even when no + recording follows, so a stale history can never be attached to a later + recording. + """ + worker = self._active_worker + take = getattr(worker, "take_preroll", None) + if not callable(take): + return [] + try: + return list(take()) + except Exception: # noqa: BLE001 - the history is optional, not the record + logger.exception("Could not take the pre-roll buffer") + return [] + def _build_session_metadata(self, config: RecordConfig) -> dict: from orisys_app_shared import API_VERSION, __version__ as shared_version @@ -345,6 +375,10 @@ class StudioMainWindow(QMainWindow): "calibration": { "config_ref": config_ref, "force_calibrated": bool(getattr(info, "force_calibrated", False)), + # Content hash of the coefficients the device confirmed. It is + # filled in by the acquisition thread's confirmation, so it can + # be empty here; the CSV's per-row version is authoritative. + "version": self._active_calibration_id, }, # Depth is deliberately NOT stored (decided with the team): it is a # function of the frame and the sensor configuration, so replay @@ -401,20 +435,15 @@ class StudioMainWindow(QMainWindow): The plots are allowed to drop frames; this path is not. It is also the reason calibration sampling no longer depends on the calibration panel being visible. + + The packet is converted exactly once, by the same function the CSV writer + uses, so the store and the file can never disagree about whether a value + is a measurement. """ - sample = SessionSample( - frame_id=int(getattr(packet, "frame_id", 0)), - t_rel_s=float(packet.timestamp), - fn_raw=float(packet.normal_force_raw), - fx_raw=float(packet.shear_x_raw), - fy_raw=float(packet.shear_y_raw), - fn_n=float(packet.normal_force), - fx_n=float(packet.shear_x), - fy_n=float(packet.shear_y), - valid=bool(getattr(packet, "valid", True)), - calibration_id=self._calibration_id(), - error=str(getattr(packet, "invalid_reason", "") or ""), + record = to_record_sample( + packet, calibration_version=self._active_calibration_id ) + sample = SessionSample.from_record(record) self.session_store.append(sample) self.cal_recorder.feed(sample) # Video + CSV + metadata share this one record path. @@ -422,14 +451,9 @@ class StudioMainWindow(QMainWindow): fps = float(getattr(packet, "fps", 0.0) or 0.0) if fps > 0.0: self._record_fps = fps - if not sample.valid: + if not record.valid: self._record_error_count += 1 - def _calibration_id(self) -> str: - """Cached per session — this runs for every acquired sample.""" - return self._active_calibration_id - - def _wire_toolbar(self) -> None: self.toolbar.search_clicked.connect(self._on_search_cameras) self.toolbar.preview_toggled.connect(self._on_preview_toggled) @@ -886,12 +910,25 @@ class StudioMainWindow(QMainWindow): self._persist_prefs() def _on_calibration_exported(self, path: str) -> None: - """Apply a freshly exported calibration — only on explicit confirmation. + """Note the coefficients that were written. Nothing is applied here. - The export no longer overwrites the device bundle, and it no longer - silently pushes the coefficients into both workers. The user is shown - the exact target device first, and only the matching worker is updated. + Exporting and applying are separate actions: writing a file must never + change what a running device is using. """ + self._last_calibration_export = str(path) + self.recorder.log_event("calibration_exported", str(path)) + self.statusBar().showMessage(t("cal.export_only", path=Path(path).name)) + + def _on_calibration_apply(self, path: str) -> None: + """Push freshly exported coefficients into the device — on confirmation. + + The user is shown the exact target device first, and only the matching + worker is updated. + """ + path = str(path or self._last_calibration_export) + if not path: + self.statusBar().showMessage(t("cal.apply_no_export")) + return sensor_type = self.sensor_picker.sensor_type() config_ref = self._resolve_config_ref() or "" answer = QMessageBox.question( @@ -908,12 +945,38 @@ class StudioMainWindow(QMainWindow): if worker is None or not hasattr(worker, "set_pending_calibration"): self.statusBar().showMessage(t("cal.apply_no_target")) return + # Only a request. The acquisition thread confirms once the coefficients + # are actually loaded; the event and the recorded version come from there. worker.set_pending_calibration(path) - self.recorder.log_event("calibration_applied", f"{sensor_type}: {path}") self.statusBar().showMessage( - t("cal.apply_done", sensor_type=sensor_type) + t("cal.apply_pending", sensor_type=sensor_type) ) + def _on_calibration_applied(self, path: str, version: str, frame_id: int) -> None: + """Called from the acquisition thread once new coefficients are loaded.""" + # The acquisition thread must not touch Qt, so only plain state moves here. + self._pending_calibration_events.append( + (str(path), str(version), int(frame_id)) + ) + + def _drain_calibration_events(self) -> None: + """Publish confirmed calibration changes on the UI thread.""" + while self._pending_calibration_events: + path, version, frame_id = self._pending_calibration_events.pop(0) + # The version is the calibration's content hash: a path can be + # re-exported with different coefficients, a hash cannot. + self._active_calibration_id = version or path + self.session_store.set_calibration_id(self._active_calibration_id) + self.recorder.set_calibration_version(self._active_calibration_id) + self.recorder.log_event( + "calibration_applied", + f"{path} ({self._active_calibration_id[:12] or 'unhashed'})", + frame_id=frame_id, + ) + self.statusBar().showMessage( + t("cal.applied", version=self._active_calibration_id[:12]) + ) + def _worker_for(self, sensor_type: str): self._ensure_workers() if sensor_type == "finger": @@ -1160,8 +1223,12 @@ class StudioMainWindow(QMainWindow): self._last_session_error = "" self._run_source_label = str(source) self._active_sensor_type = sensor_type - # Fresh record channel for the new session. - self._active_calibration_id = str(config_ref) + # Fresh record channel for the new session. The calibration identity is + # unknown until the device confirms which coefficients it loaded, so it + # starts empty and is filled from that confirmation — a path is not a + # version, and the same path can hold different numbers. + self._active_calibration_id = "" + self._pending_calibration_events.clear() self._record_error_count = 0 self._dropped_reported = 0 self._depth_range_value = None @@ -1171,7 +1238,7 @@ class StudioMainWindow(QMainWindow): self._stop_recording(wait=True) self._session_counter += 1 self.session_store.reset( - session_id=self._session_counter, calibration_id=str(config_ref) + session_id=self._session_counter, calibration_id="" ) self._ensure_workers() self._active_worker = ( @@ -1254,11 +1321,26 @@ class StudioMainWindow(QMainWindow): self._active_worker.reset() self.force_plots.reset() self._last_plot_packet_key = None - self.recorder.log_event("zeroing", "tracker reset") + # Recorded with the frame it happened at: a reset clears the SDK's + # tracker, so an offline depth replay can start here instead of + # guessing at the algorithm's earlier state. + self.recorder.log_event( + "zeroing", + "tracker reset", + frame_id=self._current_frame_id(), + ) self.statusBar().showMessage(t("status.reset_ok")) except Exception as exc: self.statusBar().showMessage(t("status.run_failed", error=str(exc))) + def _current_frame_id(self) -> int: + """The most recent frame id in the session store (-1 when unknown).""" + try: + frame_ids = self.session_store.data().frame_ids + except Exception: # noqa: BLE001 - an empty store is not an error here + return -1 + return int(frame_ids[-1]) if len(frame_ids) else -1 + def _on_worker_error(self, message: str) -> None: self._last_session_error = message logger.error("Acquisition error: %s", message) @@ -1360,6 +1442,7 @@ class StudioMainWindow(QMainWindow): self.cal_recorder.tick() self.cal_panel.poll_recorder() self._drain_record_alarms() + self._drain_calibration_events() self._report_dropped_samples() mode = self.arrow_panel.view_mode() diff --git a/apps/studio/src/orisys_studio/ui/cal_panel.py b/apps/studio/src/orisys_studio/ui/cal_panel.py index b081c71..7c5626b 100644 --- a/apps/studio/src/orisys_studio/ui/cal_panel.py +++ b/apps/studio/src/orisys_studio/ui/cal_panel.py @@ -9,6 +9,7 @@ from PyQt5.QtWidgets import ( QAbstractItemView, QButtonGroup, QDoubleSpinBox, + QFileDialog, QFormLayout, QFrame, QHBoxLayout, @@ -32,6 +33,11 @@ from orisys_app_shared.force.calibration_io import ( export_force_calibration_v2, meta_from_sensor_json, ) +from orisys_app_shared.force.calibration_session import ( + read_calibration_session, + session_filename, + write_calibration_session, +) from orisys_app_shared.i18n import t from orisys_app_shared.paths import calibration_export_dir from orisys_app_shared.theme import color @@ -56,8 +62,10 @@ def _mono_font() -> str: class CalPanel(QFrame): status_message = pyqtSignal(str) - #: Coefficients were written; the window asks the user before applying them. + #: Coefficients were written to disk. It does **not** mean they were applied. calibration_exported = pyqtSignal(str) + #: The operator asked to push the last export into the running device. + apply_requested = pyqtSignal(str) def __init__(self, parent=None, *, theme: str = "dark") -> None: super().__init__(parent) @@ -69,6 +77,8 @@ class CalPanel(QFrame): #: Set by :meth:`bind_recorder`. Sampling lives there, not here, so the #: lifecycle no longer depends on this panel being visible. self._recorder = None + #: Last successfully written coefficients file; applying is explicit. + self._last_export_path = "" self._live_fn = 0.0 self._live_fx = 0.0 self._live_fy = 0.0 @@ -190,14 +200,19 @@ class CalPanel(QFrame): self.btn_clear_all = QPushButton() self.btn_toggle_excluded = QPushButton() self.btn_toggle_validation = QPushButton() + self.btn_open_record = QPushButton() + self.btn_save_record = QPushButton() self.btn_export = QPushButton() - self.btn_export.setObjectName("primaryButton") + self.btn_apply = QPushButton() + self.btn_apply.setObjectName("primaryButton") for button in ( self.btn_delete, self.btn_clear_channel, self.btn_clear_all, self.btn_toggle_excluded, self.btn_toggle_validation, + self.btn_open_record, + self.btn_save_record, ): # The action row must survive a narrow three-panel layout. button.setProperty("compact", "true") @@ -207,14 +222,20 @@ class CalPanel(QFrame): self.btn_delete.clicked.connect(self._on_delete_row) self.btn_clear_channel.clicked.connect(self._on_clear_channel) self.btn_clear_all.clicked.connect(self._on_clear_all) + self.btn_open_record.clicked.connect(self._on_open_record) + self.btn_save_record.clicked.connect(self._on_save_record) self.btn_export.clicked.connect(self._on_export) + self.btn_apply.clicked.connect(self._on_apply) actions.addWidget(self.btn_toggle_excluded) actions.addWidget(self.btn_toggle_validation) actions.addWidget(self.btn_delete) actions.addWidget(self.btn_clear_channel) actions.addWidget(self.btn_clear_all) actions.addStretch(1) + actions.addWidget(self.btn_open_record) + actions.addWidget(self.btn_save_record) actions.addWidget(self.btn_export) + actions.addWidget(self.btn_apply) root.addLayout(actions) self._refresh_enabled() @@ -284,8 +305,14 @@ class CalPanel(QFrame): self.btn_delete.setText(t("cal.delete_row")) self.btn_clear_channel.setText(t("cal.clear_channel")) self.btn_clear_all.setText(t("cal.clear_all")) + self.btn_open_record.setText(t("cal.open_record")) + self.btn_save_record.setText(t("cal.save_record")) self.btn_export.setText(t("cal.export")) - self.btn_export.setText(t("cal.export")) + self.btn_apply.setText(t("cal.apply")) + self.btn_open_record.setToolTip(t("cal.open_record_tip")) + self.btn_save_record.setToolTip(t("cal.save_record_tip")) + self.btn_export.setToolTip(t("cal.export_tip")) + self.btn_apply.setToolTip(t("cal.apply_tip")) self.btn_toggle_excluded.setText(t("cal.state_exclude")) self.btn_toggle_validation.setText(t("cal.mark_validation")) self.btn_toggle_excluded.setToolTip(t("cal.state_exclude_tip")) @@ -352,6 +379,11 @@ class CalPanel(QFrame): for radio in (self.radio_fn, self.radio_fx, self.radio_fy): radio.setEnabled(not self._recording) self.btn_export.setEnabled(not self._recording) + # Saving the audit record is independent of the coefficients, and applying + # is only meaningful once coefficients exist. + self.btn_save_record.setEnabled(not self._recording and self._has_points()) + self.btn_open_record.setEnabled(not self._recording) + self.btn_apply.setEnabled(bool(self._last_export_path) and not self._recording) if self._recording: self.progress.setVisible(True) record_hint = t("cal.record_disabled") if not self._running else "" @@ -491,12 +523,14 @@ class CalPanel(QFrame): if not fit.usable: self.fit_label.setText(t("cal.fit_insufficient", count=fit.n_points)) return + # The method and the breakpoint count describe the model that export and + # the device use; the least-squares pair is only an equivalent summary. text = t( "cal.fit_summary", channel=self._channel.upper(), - gain=f"{fit.gain:g}", - offset=f"{fit.offset:g}", + method=t(f"cal.method_{fit.method}"), count=fit.n_points, + breakpoints=fit.breakpoints, residual=f"{fit.max_residual_n:.4f}", ) if fit.validation_points: @@ -547,23 +581,29 @@ class CalPanel(QFrame): ) def _rebuild_store(self) -> None: - """Rebuild the export store from the points that are still in the fit.""" + """Rebuild the export store from the points that are still in the fit. + + Validation points are held out: they exist to check the model, so letting + them into the build would both corrupt it and make the "independent" + error meaningless. Repeated reference loads are merged rather than + overwritten, so every sample that was actually taken reaches the model. + """ self._store.clear() if self._recorder is None: return for point in self._recorder.all_points(): - if point.excluded: + if point.excluded or point.is_validation: continue stats = point.loaded_stats() if not np.isfinite(stats.mean): continue - self._store.upsert( + self._store.aggregate( point.channel, CalSample( weight_n=float(point.reference_load_n), raw_mean=float(stats.mean), n=int(stats.n), - std=float(stats.std), + std=float(stats.std) if np.isfinite(stats.std) else 0.0, ), ) @@ -613,10 +653,74 @@ class CalPanel(QFrame): QMessageBox.warning(self, t("panels.cal"), t("cal.export_failed", error=str(exc))) return + self._last_export_path = str(primary) + self._refresh_enabled() msg = t("cal.export_ok", path=str(primary)) if warnings: msg += "\n" + t("cal.export_warnings", detail="; ".join(warnings)) self.status_message.emit(msg) - # Ask the window to confirm the target device before applying. + # The window records which coefficients were exported. Applying them is a + # separate button, so exporting cannot silently change a running device. self.calibration_exported.emit(str(primary)) QMessageBox.information(self, t("panels.cal"), msg) + + def _on_apply(self) -> None: + if not self._last_export_path: + self.status_message.emit(t("cal.apply_no_export")) + return + self.apply_requested.emit(self._last_export_path) + + def _on_save_record(self) -> None: + """Write the session's audit record: every point and every sample.""" + if self._recorder is None: + return + points = self._recorder.all_points() + if not points: + self.status_message.emit(t("cal.save_record_empty")) + return + meta = meta_from_sensor_json(self._config_ref) + meta["sensor_type"] = self._sensor_type + meta["sn"] = self.sn_edit.text().strip() + path = calibration_export_dir() / session_filename(self._sensor_type) + try: + written = write_calibration_session(path, points, meta=meta) + except Exception as exc: # noqa: BLE001 - reported to the operator + QMessageBox.warning( + self, t("panels.cal"), t("cal.save_record_failed", error=str(exc)) + ) + return + samples = sum(len(point.samples) for point in points) + msg = t("cal.save_record_ok", path=str(written), points=len(points), samples=samples) + self.status_message.emit(msg) + QMessageBox.information(self, t("panels.cal"), msg) + + def _on_open_record(self) -> None: + """Reopen a saved session so it can be inspected again.""" + if self._recorder is None: + return + path, _filter = QFileDialog.getOpenFileName( + self, + t("cal.open_record"), + str(calibration_export_dir()), + t("cal.open_record_filter"), + ) + if not path: + return + try: + record = read_calibration_session(path) + except Exception as exc: # noqa: BLE001 - reported to the operator + QMessageBox.warning( + self, t("panels.cal"), t("cal.open_record_failed", error=str(exc)) + ) + return + self._recorder.load_points(record.points, replace=True) + self._rebuild_store() + self._refresh_table() + self._refresh_enabled() + msg = t("cal.open_record_ok", path=record.path.name, points=len(record.points)) + if record.warnings: + msg += "\n" + t("cal.open_record_warnings", detail="; ".join(record.warnings)) + self.status_message.emit(msg) + + def _has_points(self) -> bool: + return bool(self._recorder is not None and self._recorder.all_points()) diff --git a/apps/studio/src/orisys_studio/ui/force_plots.py b/apps/studio/src/orisys_studio/ui/force_plots.py index 4adbe2a..e10df69 100644 --- a/apps/studio/src/orisys_studio/ui/force_plots.py +++ b/apps/studio/src/orisys_studio/ui/force_plots.py @@ -463,18 +463,21 @@ class ForcePlotsPanel(QFrame): if self._store is None: return t0, t1 = self._window - slice_ = self._store.visible(t0, t1, max_points=self._max_draw_points) + slice_ = self._store.visible( + t0, t1, max_points=self._max_draw_points, unit=self._unit + ) self._syncing = True try: if len(slice_) == 0: for curve in self.curves: curve.setData([], []) else: - values = slice_.value_for_unit(self._unit) + # One shared rule for every view: valid *and* finite. A stale + # number left by a failed compute step must not reappear here + # after another view already excluded it. + times, values, _mask = slice_.plottable(self._unit) for index, curve in enumerate(self.curves): - column = values[:, index] - finite = np.isfinite(column) - curve.setData(slice_.times[finite], column[finite]) + curve.setData(times, values[:, index]) for plot in self.plots: plot.setXRange(t0, t1, padding=0.0) self._apply_axis_mode() @@ -584,7 +587,9 @@ class ForcePlotsPanel(QFrame): return # The overview is a full-store read, so only rebuild it when it can have # changed (window/unit/session) rather than on every pan. - slice_ = self._store.visible(first, last, max_points=self._overview_points()) + slice_ = self._store.visible( + first, last, max_points=self._overview_points(), unit=self._unit + ) values = slice_.value_for_unit(self._unit) column = values[:, 0] finite = np.isfinite(column) diff --git a/apps/studio/src/orisys_studio/ui/playback_dialog.py b/apps/studio/src/orisys_studio/ui/playback_dialog.py index 8378773..a80433a 100644 --- a/apps/studio/src/orisys_studio/ui/playback_dialog.py +++ b/apps/studio/src/orisys_studio/ui/playback_dialog.py @@ -29,7 +29,12 @@ from PyQt5.QtWidgets import ( from orisys_app_shared.i18n import t from orisys_app_shared.paths import recordings_dir -from orisys_app_shared.record import DepthRecomputer, RecordingReader, list_recordings +from orisys_app_shared.record import ( + DepthRecomputer, + DepthResult, + RecordingReader, + list_recordings, +) from orisys_app_shared.record.playback import FORCE_COLUMNS from orisys_app_shared.theme import color @@ -63,6 +68,13 @@ class PlaybackDialog(QDialog): self._depth_thread: threading.Thread | None = None self._depth_result = None self._depth_error: BaseException | None = None + #: Which cursor position each depth answer belongs to. Recomputation is + #: asynchronous, so an answer can arrive after the cursor has moved; the + #: request it answers is remembered and compared, never assumed. + self._depth_request_seq = 0 + self._depth_pending_seq: int | None = None + self._depth_pending_position: float | None = None + self._depth_result_seq: int | None = None self._depth_poll = QTimer(self) self._depth_poll.setInterval(120) self._depth_poll.timeout.connect(self._poll_depth) @@ -172,6 +184,8 @@ class PlaybackDialog(QDialog): self._render_curves() self._refresh_warnings() self._depth_ok = False + self._depth_result = None + self._depth_result_seq = None self.set_position(0.0) def _refresh_warnings(self) -> None: @@ -197,11 +211,11 @@ class PlaybackDialog(QDialog): for curve in self.curves: curve.setData([], []) return - values = info.value_for_unit(self._unit) + # Same rule as the live charts: invalid or non-finite points are not + # drawn, whichever view is asking. + times, drawn, _mask = info.plottable(self._unit) for index, curve in enumerate(self.curves): - column = values[:, index] - finite = np.isfinite(column) - curve.setData(info.times[finite], column[finite]) + curve.setData(times, drawn[:, index]) self.plot_widget.ci.layout.setRowStretchFactor(index, 1) self.plot_widget.getItem(index, 0).setXRange( float(info.times[0]), float(info.times[-1]), padding=0.02 @@ -238,7 +252,9 @@ class PlaybackDialog(QDialog): for line in self.cursor_lines: line.setValue(self._position) self._show_video_frame() - if self._depth_ok: + if self._depth_ok or self._depth_result is not None: + # The held depth belongs to the frame it was computed for. Moving the + # cursor must not keep showing it as if it were synchronised. self._show_depth_frame() self.cursor_moved.emit(self._position) @@ -287,16 +303,26 @@ class PlaybackDialog(QDialog): return self._depth_ok = False self._depth_result = None + self._depth_result_seq = None self._depth_error = None self.depth_label.setText(t("playback.depth_running")) position = self._position + # Bind the answer to this request: a result that arrives after the cursor + # moved is discarded rather than pasted onto the new position. + self._depth_request_seq += 1 + seq = self._depth_request_seq + self._depth_pending_seq = seq + self._depth_pending_position = position def work() -> None: # Runs off the UI thread and touches no Qt object. try: - self._depth_result = self._recomputer.depth_at(position) + result = self._recomputer.depth_at(position) except BaseException as exc: # noqa: BLE001 - reported via the poll self._depth_error = exc + return + self._depth_result = result + self._depth_result_seq = seq self._depth_thread = threading.Thread( target=work, name="depth-replay", daemon=True @@ -310,19 +336,62 @@ class PlaybackDialog(QDialog): if thread is None or thread.is_alive(): return self._depth_poll.stop() + pending = self._depth_pending_seq + self._depth_pending_seq = None + self._depth_pending_position = None if self._depth_error is not None: self._depth_ok = False self.depth_label.setPixmap(QPixmap()) self.depth_label.setText( t("playback.depth_failed", detail=str(self._depth_error)) ) + self._depth_error = None return result = self._depth_result - if result is None: + if result is None or self._depth_result_seq != pending: + # A newer request (or a moved cursor) owns the pane now. + self._show_depth_frame() + return + if not self._depth_matches_cursor(result): + # The cursor moved while the SDK was reading: do not draw an answer + # for another frame, and say the pane needs recomputing. + self._show_depth_frame() return self._render_depth_result(result) + def _current_frame(self): + """The recorded frame under the cursor, or None without a frame index.""" + frame_at = getattr(self._reader, "frame_at", None) + if not callable(frame_at): + return None + try: + return frame_at(self._position) + except Exception: # noqa: BLE001 - a stub reader has no frame index + return None + + def _depth_matches_cursor(self, result) -> bool: + """True when ``result`` was computed for the frame under the cursor.""" + frame = self._current_frame() + frame_id = getattr(result, "frame_id", None) + if frame is not None and frame_id is not None and int(frame_id) >= 0: + return int(frame_id) == int(frame.frame_id) + t_rel = getattr(result, "t_rel_s", None) + if t_rel is None: + # Nothing in the result says which frame it belongs to, so it cannot + # be shown to be stale either. + return True + if frame is not None: + return abs(float(t_rel) - float(frame.t_rel_s)) < 1e-6 + return abs(float(t_rel) - self._position) < 1e-9 + def _render_depth_result(self, result) -> None: + code = str(getattr(result, "error", "") or "") + if code in ("not_recomputed", "recomputing", "stale"): + # A placeholder, not a failure: the pane describes the cursor. + self._depth_ok = False + self.depth_label.setPixmap(QPixmap()) + self.depth_label.setText(t(f"playback.depth_{code}")) + return if not result.ok: self._depth_ok = False self.depth_label.setPixmap(QPixmap()) @@ -337,6 +406,7 @@ class PlaybackDialog(QDialog): self.depth_label.setText(t("playback.depth_empty")) return self._depth_ok = True + self.depth_label.setText(self._depth_warning_text(result)) rgb = bgr[:, :, ::-1].copy() height, width, channels = rgb.shape qimage = QImage( @@ -346,12 +416,40 @@ class PlaybackDialog(QDialog): self.depth_label.size(), Qt.KeepAspectRatio, Qt.SmoothTransformation ) self.depth_label.setPixmap(pixmap) - self.depth_label.setText("") + + @staticmethod + def _depth_warning_text(result) -> str: + """Say when a replay could not reproduce the live algorithm state.""" + codes = tuple(getattr(result, "warnings", ()) or ()) + if not codes: + return "" + return t( + "playback.depth_warnings", + detail=", ".join(t(f"playback.warn_{code}") for code in codes), + ) def _show_depth_frame(self) -> None: - """Re-render the last result at the new position (no SDK work).""" - if self._depth_result is not None: - self._render_depth_result(self._depth_result) + """Draw the depth for the cursor's frame, or say it must be recomputed. + + Re-drawing the held map here is what used to show a stale depth as though + it were in sync with the cursor. + """ + result = self._depth_result + if result is not None and self._depth_matches_cursor(result): + self._render_depth_result(result) + return + if ( + self._depth_pending_seq is not None + and self._depth_pending_position == self._position + and self._depth_busy() + ): + self._render_depth_result( + DepthResult(error="recomputing", t_rel_s=self._position) + ) + return + self._render_depth_result( + DepthResult(error="not_recomputed", t_rel_s=self._position) + ) # ------------------------------------------------------------------- labels def retranslate(self) -> None: diff --git a/apps/studio/tests/test_ui_smoke.py b/apps/studio/tests/test_ui_smoke.py index f5aee07..aad4320 100644 --- a/apps/studio/tests/test_ui_smoke.py +++ b/apps/studio/tests/test_ui_smoke.py @@ -1,4 +1,4 @@ -"""Offscreen UI smoke tests: construction, layout adaptation, theme tokens.""" +"""Offscreen UI smoke tests: construction, layout adaptation, theme tokens.""" from __future__ import annotations @@ -309,6 +309,13 @@ def _add_calibration_point(window, *, load: float, raw: float, samples: int = 40 def test_calibration_diagnostics_drive_the_fit(make_window): + """The diagnostics must describe the model that would be exported. + + The export builds a piecewise-linear curve, which passes through every point + it is built from — so the training residual is zero by construction and an + off-trend point can only show up as a *validation* error. A private + least-squares residual would have described a model nobody applies. + """ window = _show(make_window(), (1400, 880)) window.cal_panel.set_running(True) _add_calibration_point(window, load=1.0, raw=100.2) @@ -319,23 +326,36 @@ def test_calibration_diagnostics_drive_the_fit(make_window): recorder = window.cal_recorder fit = recorder.fit("fn") assert fit.usable and fit.n_points == 4 - worst = fit.max_residual_n - assert worst > 0.05, "the off-trend point produced no residual" + assert fit.method == "piecewise_linear", fit.method + assert fit.breakpoints == 4, fit.breakpoints + assert fit.max_residual_n <= 1e-9, "the exported model misses its own points" assert window.cal_panel.fit_label.text() + # Hold the off-trend point (load 2.0 at raw 400) out: the exported model is + # then built without it and the misfit is measurable. window.cal_panel.table.setCurrentCell(1, 0) + window.cal_panel._on_toggle_validation() + held_out = recorder.fit("fn") + assert held_out.validation_points == 1 + assert held_out.validation_error_n > 0.05, held_out.validation_error_n + assert recorder.points[1].is_validation + assert np.isnan(recorder.points[1].residual_n), "validation point entered the fit" + + window.cal_panel._on_toggle_validation() window.cal_panel._on_toggle_excluded() - assert recorder.fit("fn").max_residual_n < worst, "exclusion did not tighten the fit" assert recorder.points[1].excluded # An excluded point leaves the export store but stays on record. assert len(window.cal_panel._store.channel("fn")) == 3 - window.cal_panel.table.setCurrentCell(2, 0) - window.cal_panel._on_toggle_validation() - held_out = recorder.fit("fn") - assert held_out.validation_points == 1 - assert recorder.points[2].is_validation - assert np.isnan(recorder.points[2].residual_n), "validation point entered the fit" + # The exported model no longer passes through the excluded point. + from orisys_app_shared.force.calibration_io import ( + build_export_model, + predict_channel, + ) + + store = window.cal_panel._store + model = build_export_model({key: store.channel(key) for key in ("fn", "fx", "fy")}) + assert abs(predict_channel(model, "fn", 400.0) - 2.0) > 0.05 def test_calibration_point_dialog_shows_three_axes(make_window): diff --git a/apps/studio/tools/ui_check.py b/apps/studio/tools/ui_check.py index 5edf48f..2a3a264 100644 --- a/apps/studio/tools/ui_check.py +++ b/apps/studio/tools/ui_check.py @@ -608,9 +608,12 @@ def main() -> int: "the fit summary is empty", ) - # Three more points, one of them off-trend, so the fit has a residual - # that excluding the outlier must reduce and enough left over to hold - # one out as a validation point. + # Four points, one of them off-trend (raw 400 where the others imply + # ~200). The exported model is piecewise-linear, so it passes through + # every point it is built from: the training residual is zero by + # construction and the off-trend point can only show up as a + # *validation* error. The summary must describe that model, not a + # private least-squares line. for load, raw in ((2.0, 400.0), (3.0, 300.6), (5.0, 500.9)): window.cal_recorder.start( channel="fn", reference_load_n=load, duration_s=1.0 @@ -621,22 +624,24 @@ def main() -> int: app.processEvents() check(window.cal_panel.table.rowCount() == 4, "calibration rows missing") - worst = window.cal_recorder.fit("fn").max_residual_n - check(worst > 0.05, f"the off-trend point produced no residual: {worst}") - window.cal_panel.table.setCurrentCell(1, 0) - window.cal_panel._on_toggle_excluded() - app.processEvents() + fit = window.cal_recorder.fit("fn") check( - window.cal_recorder.fit("fn").max_residual_n < worst, - "excluding an outlier did not tighten the fit", + fit.method == "piecewise_linear", + f"the summary is not about the exported model: {fit.method!r}", ) check( - window.cal_panel.table.item(1, 5).text() == t("cal.state_excluded"), - "the excluded row does not show its state", + fit.breakpoints == 4, + f"the exported model has {fit.breakpoints} breakpoints, not 4", + ) + check( + fit.max_residual_n <= 1e-9, + "the exported model does not reproduce its own calibration points: " + f"max residual {fit.max_residual_n}", ) - # A validation point is held out of the fit and reported separately. - window.cal_panel.table.setCurrentCell(2, 0) + # Hold the off-trend point out: the exported model is then built + # without it and the misfit becomes visible as a validation error. + window.cal_panel.table.setCurrentCell(1, 0) window.cal_panel._on_toggle_validation() app.processEvents() check( @@ -644,6 +649,35 @@ def main() -> int: in window.cal_panel.fit_label.text(), "no validation summary after marking a validation point", ) + validation = window.cal_recorder.fit("fn").validation_error_n + check( + validation > 0.05, + f"the off-trend point produced no validation error: {validation}", + ) + window.cal_panel._on_toggle_validation() + + # Excluding a point removes it from the build, so the exported model + # no longer passes through it. + window.cal_panel._on_toggle_excluded() + app.processEvents() + check( + window.cal_panel.table.item(1, 5).text() == t("cal.state_excluded"), + "the excluded row does not show its state", + ) + from orisys_app_shared.force.calibration_io import ( + build_export_model, + predict_channel, + ) + + store = window.cal_panel._store + model = build_export_model( + {key: store.channel(key) for key in ("fn", "fx", "fy")} + ) + predicted = predict_channel(model, "fn", 400.0) + check( + abs(predicted - 2.0) > 0.05, + f"excluding a point left the exported model unchanged: {predicted}", + ) window.cal_recorder.reset_session() window.cal_panel._refresh_table() diff --git a/shared/src/orisys_app_shared/acquisition/finger_worker.py b/shared/src/orisys_app_shared/acquisition/finger_worker.py index bbbd29c..dad3ef9 100644 --- a/shared/src/orisys_app_shared/acquisition/finger_worker.py +++ b/shared/src/orisys_app_shared/acquisition/finger_worker.py @@ -23,7 +23,10 @@ from orisys_app_shared.acquisition.sdk_worker import ( SdkPacket, _ensure_bgr, _is_file_source, + calibration_digest, load_fixed_mode, + loaded_calibration_version, + loaded_path, orisys_importable, read_forces_from_sensor, ) @@ -62,10 +65,13 @@ class FingerAcquisitionWorker(RecordSinkMixin): *, on_error: Optional[Callable[[str], None]] = None, on_status: Optional[Callable[[str], None]] = None, + on_calibration_applied: Optional[Callable[[str, str, int], None]] = None, ): self._init_sinks() self._on_error = on_error self._on_status = on_status + #: Called from the acquisition thread once new coefficients are loaded. + self._on_calibration_applied = on_calibration_applied self._thread: Optional[threading.Thread] = None self._stop = threading.Event() self._lock = threading.Lock() @@ -80,6 +86,8 @@ class FingerAcquisitionWorker(RecordSinkMixin): self._end_reason: Optional[str] = None self._phase: str = "idle" self._pending_calibration: Optional[str] = None + self._mono_t0: float = 0.0 + self._calibration_version: str = "" @property def is_running(self) -> bool: @@ -124,6 +132,8 @@ class FingerAcquisitionWorker(RecordSinkMixin): self._session_id += 1 session = self._session_id self._stop.clear() + # Fresh monotonic origin for this session. + self._mono_t0 = time.perf_counter() self._thread = threading.Thread( target=self._run, name=f"FingerAcquisition-{session}", @@ -172,14 +182,20 @@ class FingerAcquisitionWorker(RecordSinkMixin): with self._lock: self._pending_calibration = str(path) if path is not None else None - def _consume_pending_calibration(self, sensor) -> None: + def _consume_pending_calibration(self, sensor, frame_id: int = 0) -> None: + """Apply queued coefficients and report confirmation, not intent.""" with self._lock: path = self._pending_calibration self._pending_calibration = None if path is None or sensor is None: return - if hasattr(sensor, "set_calibration"): - sensor.set_calibration(path) + if not hasattr(sensor, "set_calibration"): + raise RuntimeError("sensor does not support set_calibration") + sensor.set_calibration(path) + version = calibration_digest(path) + self._calibration_version = version + if self._on_calibration_applied is not None: + self._on_calibration_applied(str(path), version, int(frame_id)) def _set_end_reason(self, reason: str) -> None: with self._lock: @@ -267,6 +283,16 @@ class FingerAcquisitionWorker(RecordSinkMixin): except Exception: pass self._status(f"Streaming finger {mode.label()}") + # The tactile sensor loads its own bundle calibration while opening; + # name those coefficients from the first frame, not the request path. + tactile = getattr(runtime, "tactile", None) + loaded = loaded_calibration_version(tactile) if tactile else "" + if loaded: + self._calibration_version = loaded + if self._on_calibration_applied is not None: + self._on_calibration_applied( + str(loaded_path(tactile) or ""), loaded, 0 + ) except Exception as exc: self._set_end_reason("error") if self._on_error is not None and session == self._session_id: @@ -316,11 +342,19 @@ class FingerAcquisitionWorker(RecordSinkMixin): tactile = runtime.tactile compute_error = "" try: - self._consume_pending_calibration(tactile) + self._consume_pending_calibration(tactile, frame_id) + except Exception as exc: # noqa: BLE001 + message = f"calibration rejected: {type(exc).__name__}: {exc}" + if self._on_error is not None and session == self._session_id: + self._on_error(message) + compute_error = message + try: tactile.compute_deformation(decompose=True, check_motion=False) tactile.compute_contact() except Exception as exc: # noqa: BLE001 - reported on the packet - compute_error = f"compute: {type(exc).__name__}: {exc}" + compute_error = compute_error or ( + f"compute: {type(exc).__name__}: {exc}" + ) t1 = time.perf_counter() frame_times.append(t1) cutoff = t1 - 1.0 @@ -331,10 +365,15 @@ class FingerAcquisitionWorker(RecordSinkMixin): config_ref=mode.config_ref, ) invalid_reason = forces.get("invalid_reason") or compute_error + raw_frame = _ensure_bgr(source_frame) + # The frame the tactile chain consumed, kept for depth replay. + self.note_preroll(raw_frame) packet = SdkPacket( - raw_frame=_ensure_bgr(source_frame), + raw_frame=raw_frame, processed_frame=_ensure_bgr(unrolled), timestamp=time.time(), + wall_t=time.time(), + mono_t=max(0.0, t1 - self._mono_t0), fps=fps, process_ms=(t1 - t0) * 1000.0, normal_force=forces["normal_force"], @@ -345,10 +384,15 @@ class FingerAcquisitionWorker(RecordSinkMixin): shear_y_raw=forces["shear_y_raw"], force_unit=forces["force_unit"], force_calibrated=forces["force_calibrated"], + calibrated_axes=tuple(forces.get("calibrated_axes", ())), + calibration_version=self._calibration_version, session_id=session, frame_id=frame_id, valid=bool(forces.get("valid", True)) and not compute_error, invalid_reason=invalid_reason, + # A failed chain leaves the previous frame's force fields in + # place; the app must not record them as this frame's data. + stale_forces=bool(compute_error), raw_available=bool(forces.get("raw_available", True)), flow=forces["flow"], depth=forces.get("depth"), diff --git a/shared/src/orisys_app_shared/acquisition/sdk_worker.py b/shared/src/orisys_app_shared/acquisition/sdk_worker.py index e8074b3..2908939 100644 --- a/shared/src/orisys_app_shared/acquisition/sdk_worker.py +++ b/shared/src/orisys_app_shared/acquisition/sdk_worker.py @@ -56,14 +56,31 @@ class SdkPacket: shear_y_raw: float = 0.0 force_unit: str = "raw" force_calibrated: bool = False + #: Which axes actually have a calibration curve loaded. The sensor-level + #: ``force_calibrated`` flag cannot express a partially calibrated device. + calibrated_axes: tuple[str, ...] = () session_id: int = 0 #: Monotonic per-session frame index; the join key for video, CSV and #: calibration records. frame_id: int = 0 + #: Monotonic, session-relative acquisition time. Used for ordering so a wall + #: clock that steps backwards cannot reorder history. ``None`` means the + #: producer did not measure one, and the wall clock is used instead — a + #: default of ``0.0`` would have made every frame look simultaneous. + mono_t: Optional[float] = None + #: Wall clock at hand-off, for metadata only. ``None`` means unmeasured. + wall_t: Optional[float] = None #: False when the force readout failed. Invalid packets carry NaN forces and #: must never be averaged into calibration data or written as zeros. valid: bool = True invalid_reason: str = "" + #: True when the compute chain failed for this frame. The SDK's force fields + #: then still hold the value computed for an earlier frame, so they must not + #: be recorded as this frame's measurement. + stale_forces: bool = False + #: Content hash of the calibration actually in effect when this frame was + #: acquired; changes mid-session when new coefficients are applied. + calibration_version: str = "" #: False when the sensor exposes no RAW channels; the raw fields are then #: NaN rather than a copy of the calibrated values. raw_available: bool = True @@ -227,6 +244,7 @@ def read_forces_from_sensor(sensor, *, config_ref: str | Path | None = None): "shear_x_raw": float(fx_raw), "shear_y_raw": float(fy_raw), "force_calibrated": calibrated, + "calibrated_axes": _calibrated_axes(sensor), "force_unit": "N" if calibrated else "raw", "flow": flow, "depth": depth, @@ -236,10 +254,96 @@ def read_forces_from_sensor(sensor, *, config_ref: str | Path | None = None): } +#: Maps the SDK's canonical channel keys to the app's short axis names. +_CALIBRATION_AXIS_ALIASES = { + "fn": ("fn", "normal", "normal_force"), + "fx": ("fx", "shear_x", "shearx"), + "fy": ("fy", "shear_y", "sheary"), +} + + +def _calibrated_axes(sensor) -> tuple[str, ...]: + """Which axes have a loaded calibration curve. + + The SDK stores one curve per channel, so a device may be calibrated on its + normal axis only. Reporting a single sensor-level flag would make the app + write raw counts into the newton columns for the uncalibrated axes. + """ + calibration = getattr(sensor, "calibration", None) + channels = getattr(calibration, "channels", None) + if not channels: + return () + try: + keys = {str(key).lower() for key in channels} + except TypeError: + return () + axes: list[str] = [] + for axis, aliases in _CALIBRATION_AXIS_ALIASES.items(): + if keys & set(aliases): + axes.append(axis) + return tuple(axes) + + def _is_file_source(video_source: str | int) -> bool: return isinstance(video_source, str) +def calibration_digest(path: str | Path) -> str: + """Content hash of a calibration document — the calibration's identity. + + A path is not a version: coefficients can be re-exported to the same path + with different numbers. The hash is what tells one calibration apart from + another in a recording. + """ + try: + import hashlib + import json + + # A parsed SDK document has no path of its own; hash its content. + to_dict = getattr(path, "to_dict", None) + if callable(to_dict): + return hashlib.sha256( + json.dumps(to_dict(), sort_keys=True, default=str).encode("utf-8") + ).hexdigest() + target = Path(str(path)) + if target.is_file(): + return hashlib.sha256(target.read_bytes()).hexdigest() + if not isinstance(path, (str, Path)): + return hashlib.sha256( + json.dumps(path, sort_keys=True, default=str).encode("utf-8") + ).hexdigest() + except Exception: # noqa: BLE001 - identity must never block acquisition + logger.exception("Could not hash calibration %r", path) + return "" + + +def loaded_path(sensor) -> Optional[str]: + """Where the sensor's active calibration came from, if it knows.""" + calibration = getattr(sensor, "calibration", None) + if calibration is None: + return None + source = getattr(calibration, "source_path", None) + return str(source) if source else None + + +def loaded_calibration_version(sensor) -> str: + """Content hash of the calibration a sensor actually has loaded. + + The sensor may have loaded a product-bundle file while opening, which is + exactly the set of coefficients the first frames are measured with. + """ + calibration = getattr(sensor, "calibration", None) + if calibration is None: + return "" + source = getattr(calibration, "source_path", None) + if source: + digest = calibration_digest(source) + if digest: + return digest + return calibration_digest(calibration) + + + class SdkAcquisitionWorker(RecordSinkMixin): """Background loop: Sensor.get_img + deformation + force readout.""" @@ -248,10 +352,14 @@ class SdkAcquisitionWorker(RecordSinkMixin): *, on_error: Optional[Callable[[str], None]] = None, on_status: Optional[Callable[[str], None]] = None, + on_calibration_applied: Optional[Callable[[str, str, int], None]] = None, ): self._init_sinks() self._on_error = on_error self._on_status = on_status + #: Called from the acquisition thread once new coefficients are actually + #: loaded, with ``(path, version, frame_id)``. Confirmation, not intent. + self._on_calibration_applied = on_calibration_applied self._thread: Optional[threading.Thread] = None self._stop = threading.Event() self._lock = threading.Lock() @@ -265,6 +373,11 @@ class SdkAcquisitionWorker(RecordSinkMixin): self._end_reason: Optional[str] = None self._phase: str = "idle" self._pending_calibration: Optional[str] = None + #: Monotonic origin for ``SdkPacket.mono_t``, reset per session. + self._mono_t0: float = 0.0 + #: Content hash of the calibration currently in effect. + self._calibration_version: str = "" + self._frame_id: int = 0 @property def is_running(self) -> bool: @@ -308,6 +421,9 @@ class SdkAcquisitionWorker(RecordSinkMixin): self._session_id += 1 session = self._session_id self._stop.clear() + # Fresh monotonic origin and frame counter for this session. + self._mono_t0 = time.perf_counter() + self._frame_id = 0 thread = threading.Thread( target=self._run, name=f"SdkAcquisition-{session}", @@ -358,14 +474,26 @@ class SdkAcquisitionWorker(RecordSinkMixin): with self._lock: self._pending_calibration = str(path) if path is not None else None - def _consume_pending_calibration(self, sensor) -> None: + def _consume_pending_calibration(self, sensor, frame_id: int = 0) -> None: + """Apply queued coefficients and report **confirmation**, not intent. + + ``Sensor.set_calibration`` loads with ``strict=True`` and raises on an + invalid document, so success is observable. The old code swallowed that + exception, which is why the app logged "calibration applied" before + anything had actually been loaded. + """ with self._lock: path = self._pending_calibration self._pending_calibration = None if path is None or sensor is None: return - if hasattr(sensor, "set_calibration"): - sensor.set_calibration(path) + if not hasattr(sensor, "set_calibration"): + raise RuntimeError("sensor does not support set_calibration") + sensor.set_calibration(path) + version = calibration_digest(path) + self._calibration_version = version + if self._on_calibration_applied is not None: + self._on_calibration_applied(str(path), version, int(frame_id)) def _set_end_reason(self, reason: str) -> None: with self._lock: @@ -441,6 +569,17 @@ class SdkAcquisitionWorker(RecordSinkMixin): lowpass_cutoff_hz=LOWPASS_CUTOFF_HZ, lowpass_sample_rate_hz=FORCE_SAMPLE_RATE_HZ, ) + # A sensor loads its own bundle calibration while opening, so the + # coefficients in effect from the very first frame are already known + # here. Without this the record would name the path passed on the + # command line instead of the calibration that was applied. + loaded = loaded_calibration_version(sensor) + if loaded: + self._calibration_version = loaded + if self._on_calibration_applied is not None: + self._on_calibration_applied( + str(loaded_path(sensor) or ""), loaded, 0 + ) self._status(f"Streaming {mode.label()}") except Exception as exc: self._set_end_reason("error") @@ -489,11 +628,22 @@ class SdkAcquisitionWorker(RecordSinkMixin): none_streak_start = None compute_error = "" try: - self._consume_pending_calibration(sensor) + # Applying new coefficients is reported separately from the + # compute chain: a rejected document is an operator error, + # not a numerical failure. + self._consume_pending_calibration(sensor, frame_id) + except Exception as exc: # noqa: BLE001 + message = f"calibration rejected: {type(exc).__name__}: {exc}" + if self._on_error is not None and session == self._session_id: + self._on_error(message) + compute_error = message + try: sensor.compute_deformation(decompose=True, check_motion=False) sensor.compute_contact() except Exception as exc: # noqa: BLE001 - reported on the packet - compute_error = f"compute: {type(exc).__name__}: {exc}" + compute_error = compute_error or ( + f"compute: {type(exc).__name__}: {exc}" + ) t1 = time.perf_counter() frame_times.append(t1) cutoff = t1 - 1.0 @@ -501,10 +651,17 @@ class SdkAcquisitionWorker(RecordSinkMixin): fps = float(len(frame_times)) if frame_times else 0.0 forces = read_forces_from_sensor(sensor, config_ref=mode.config_ref) invalid_reason = forces.get("invalid_reason") or compute_error + raw_frame = _ensure_bgr(raw) + # The frame the chain above actually consumed, kept as the + # history an offline depth replay needs. + self.note_preroll(raw_frame) packet = SdkPacket( - raw_frame=_ensure_bgr(raw), + raw_frame=raw_frame, processed_frame=_ensure_bgr(processed), + # Wall clock for metadata; monotonic for ordering. timestamp=time.time(), + wall_t=time.time(), + mono_t=max(0.0, t1 - self._mono_t0), fps=fps, process_ms=(t1 - t0) * 1000.0, normal_force=forces["normal_force"], @@ -515,10 +672,15 @@ class SdkAcquisitionWorker(RecordSinkMixin): shear_y_raw=forces["shear_y_raw"], force_unit=forces["force_unit"], force_calibrated=forces["force_calibrated"], + calibrated_axes=tuple(forces.get("calibrated_axes", ())), + calibration_version=self._calibration_version, session_id=session, frame_id=frame_id, valid=bool(forces.get("valid", True)) and not compute_error, invalid_reason=invalid_reason, + # A failed chain leaves the previous frame's force fields in + # place; the app must not record them as this frame's data. + stale_forces=bool(compute_error), raw_available=bool(forces.get("raw_available", True)), flow=forces["flow"], depth=forces.get("depth"), diff --git a/shared/src/orisys_app_shared/acquisition/sinks.py b/shared/src/orisys_app_shared/acquisition/sinks.py index 6fd7d24..1eef5b5 100644 --- a/shared/src/orisys_app_shared/acquisition/sinks.py +++ b/shared/src/orisys_app_shared/acquisition/sinks.py @@ -14,10 +14,18 @@ from __future__ import annotations import logging import threading +from collections import deque from typing import Any, Callable, Iterable +import numpy as np + logger = logging.getLogger(__name__) +#: How many pre-processing frames are kept before a recording starts. +PREROLL_FRAMES = 30 +#: Byte ceiling for the same buffer (30 frames of 1080p BGR is already ~190 MB). +PREROLL_BYTES = 192 * 1024 * 1024 + class RecordSinkMixin: """Mixin adding ``add_sink`` / ``remove_sink`` / ``_emit`` to a worker.""" @@ -26,6 +34,50 @@ class RecordSinkMixin: self._sinks: list[Callable[[Any], None]] = [] self._sinks_lock = threading.Lock() self._emit_failures = 0 + self._init_preroll() + + # ------------------------------------------------------------- pre-roll + def _init_preroll(self) -> None: + """Buffer of the frames captured before recording starts. + + The SDK's depth chain is stateful (optical-flow tracker plus temporal + filters), so the first frames of a recording cannot be recomputed offline + without the frames that preceded them. This is that history, bounded by + both a frame count and a byte budget. + """ + self._preroll: deque = deque() + self._preroll_lock = threading.Lock() + self._preroll_bytes = 0 + + def note_preroll(self, frame) -> None: + """Remember one pre-processing frame. Called from the acquisition thread.""" + if frame is None: + return + block = np.asarray(frame) + if block.size == 0: + return + copy = block.copy() # the sensor reuses its buffer between reads + with self._preroll_lock: + self._preroll.append(copy) + self._preroll_bytes += int(copy.nbytes) + while self._preroll and ( + len(self._preroll) > PREROLL_FRAMES + or self._preroll_bytes > PREROLL_BYTES + ): + dropped = self._preroll.popleft() + self._preroll_bytes -= int(np.asarray(dropped).nbytes) + + def take_preroll(self) -> list: + """Hand the buffered frames to a recorder and start a fresh buffer.""" + with self._preroll_lock: + frames = list(self._preroll) + self._preroll.clear() + self._preroll_bytes = 0 + return frames + + def preroll_size(self) -> int: + with self._preroll_lock: + return len(self._preroll) def add_sink(self, sink: Callable[[Any], None]) -> None: """Register a record-channel consumer of every packet.""" diff --git a/shared/src/orisys_app_shared/force/calibration_io.py b/shared/src/orisys_app_shared/force/calibration_io.py index 15a616f..6737232 100644 --- a/shared/src/orisys_app_shared/force/calibration_io.py +++ b/shared/src/orisys_app_shared/force/calibration_io.py @@ -6,11 +6,12 @@ one format. ``CalSessionStore`` remains the UI sampling table. from __future__ import annotations +import math from copy import deepcopy from dataclasses import asdict, dataclass, field from datetime import datetime from pathlib import Path -from typing import Any, Literal +from typing import Any, Literal, Mapping from orisys.calibration import ( CalSample as SdkCalSample, @@ -24,6 +25,8 @@ ChannelKey = Literal["fn", "fx", "fy"] CHANNEL_KEYS: tuple[ChannelKey, ...] = ("fn", "fx", "fy") WEIGHT_EPS = 1e-6 +#: Channel slot of each axis inside ``Calibration.convert``'s triple. +_CONVERT_SLOT: dict[str, int] = {"fn": 0, "fx": 1, "fy": 2} @dataclass @@ -42,6 +45,58 @@ class CalSample: ) +def merge_samples(first: CalSample, second: CalSample) -> CalSample: + """Combine two observations of one reference load. + + The means are n-weighted and the standard deviations are pooled about the + combined mean, so a repeated load ends up with the scatter of every sample + that went into it rather than of whichever one happened to be last. + """ + n_total = int(first.n) + int(second.n) + if n_total <= 0: + return CalSample( + weight_n=float(first.weight_n), + raw_mean=(float(first.raw_mean) + float(second.raw_mean)) / 2.0, + n=0, + std=0.0, + ) + mean = ( + float(first.raw_mean) * int(first.n) + float(second.raw_mean) * int(second.n) + ) / n_total + ss = (int(first.n) - 1) * float(first.std) ** 2 + ss += (int(second.n) - 1) * float(second.std) ** 2 + ss += int(first.n) * (float(first.raw_mean) - mean) ** 2 + ss += int(second.n) * (float(second.raw_mean) - mean) ** 2 + var = ss / (n_total - 1) if n_total > 1 else 0.0 + return CalSample( + weight_n=float(first.weight_n), + raw_mean=float(mean), + n=n_total, + std=math.sqrt(max(0.0, float(var))), + ) + + +def build_export_model(samples: Mapping[str, list[CalSample]]): + """Build exactly the model that export and application would use. + + Diagnostics must be computed on *this* curve. A separate least-squares fit + can look healthy while the piecewise curve the device actually applies is + the one with the error. + """ + payload = { + key: [sample.to_sdk() for sample in (samples.get(key) or [])] + for key in CHANNEL_KEYS + } + return sdk_build(payload, origin="diagnostics") + + +def predict_channel(cal, channel: str, raw: float) -> float: + """Apply ``cal`` to one axis exactly as the runtime conversion does.""" + triple = [0.0, 0.0, 0.0] + triple[_CONVERT_SLOT[channel]] = float(raw) + return float(cal.convert(triple[0], triple[1], triple[2])[_CONVERT_SLOT[channel]]) + + @dataclass class CalSessionStore: """In-memory calibration points for FN / FX / FY.""" @@ -75,6 +130,23 @@ class CalSessionStore: rows.sort(key=lambda s: s.weight_n) self.set_channel(key, rows) + def aggregate(self, key: ChannelKey, sample: CalSample) -> None: + """Add one observation, merging repeated loads instead of overwriting. + + The operator may sample the same reference load several times on purpose. + Letting the last sample win threw that evidence away — and with it the + scatter that tells whether a load was actually stable. + """ + rows = self.channel(key) + for i, existing in enumerate(rows): + if abs(existing.weight_n - sample.weight_n) <= WEIGHT_EPS: + rows[i] = merge_samples(existing, sample) + break + else: + rows.append(sample) + rows.sort(key=lambda s: s.weight_n) + self.set_channel(key, rows) + def remove_at(self, key: ChannelKey, index: int) -> None: rows = self.channel(key) if 0 <= index < len(rows): diff --git a/shared/src/orisys_app_shared/force/calibration_recorder.py b/shared/src/orisys_app_shared/force/calibration_recorder.py index 62dd711..a3558c0 100644 --- a/shared/src/orisys_app_shared/force/calibration_recorder.py +++ b/shared/src/orisys_app_shared/force/calibration_recorder.py @@ -31,6 +31,21 @@ import numpy as np from orisys_app_shared.force.session import NA, SessionSample, summarise +try: # pragma: no cover - the module is a plain data definition + from orisys_app_shared.force.calibration_session import SAMPLE_FIELDS +except Exception: # noqa: BLE001 - a partial install must not break sampling + SAMPLE_FIELDS = ( + "frame_id", + "t_rel_s", + "fn_raw", + "fx_raw", + "fy_raw", + "fn_n", + "fx_n", + "fy_n", + "valid", + ) + ChannelKey = Literal["fn", "fx", "fy"] CHANNEL_KEYS: tuple[ChannelKey, ...] = ("fn", "fx", "fy") @@ -72,6 +87,10 @@ class CalibrationPoint: #: Downsampled per-axis raw trace, so a row can be inspected after the fact. trace: dict[str, list[float]] = field(default_factory=dict) trace_dt_s: float = 0.0 + #: **Audit** series: every sample of the point, in + #: :data:`~orisys_app_shared.force.calibration_session.SAMPLE_FIELDS` order. + #: The trace above is a preview and must never be the only surviving copy. + samples: list[tuple] = field(default_factory=list) #: Diagnostics, all operator-controlled. Excluded points leave the fit; #: validation points are held out of it and used to test it. excluded: bool = False @@ -114,35 +133,43 @@ class CalibrationPoint: "residual_n": self.residual_n, "raw": {key: stats.as_dict() for key, stats in self.raw.items()}, "newtons": {key: stats.as_dict() for key, stats in self.newtons.items()}, + # The audit series is part of the point's identity, not a UI extra: + # without it the only surviving copy of the samples is the + # downsampled trace. + "sample_fields": list(SAMPLE_FIELDS), + "n_samples": len(self.samples), + "samples": [list(row) for row in self.samples], } @dataclass(frozen=True) class FitResult: - """Least-squares result for one channel.""" + """Diagnostics for the model that would actually be exported.""" channel: ChannelKey - gain: float = NA - offset: float = NA n_points: int = 0 max_residual_n: float = NA validation_error_n: float = NA validation_points: int = 0 + #: Method of the exported curve (``piecewise_linear`` / ``linear``). + method: str = "" + #: Breakpoints in the exported curve, for the UI to describe the model. + breakpoints: int = 0 + #: Least-squares summary shown next to the model, never used for residuals. + linear_gain: float = NA + linear_offset: float = NA @property def usable(self) -> bool: - return self.n_points >= 2 and np.isfinite(self.gain) and np.isfinite(self.offset) - - def predict(self, raw_value: float) -> float: - if not self.usable: - return NA - return float(self.gain) * float(raw_value) + float(self.offset) + return self.n_points >= 2 and bool(self.method) def as_dict(self) -> dict: return { "channel": self.channel, - "gain_n_per_raw": self.gain, - "offset_n": self.offset, + "method": self.method, + "breakpoints": self.breakpoints, + "linear_gain_n_per_raw": self.linear_gain, + "linear_offset_n": self.linear_offset, "points_in_fit": self.n_points, "max_residual_n": self.max_residual_n, "validation_error_n": self.validation_error_n, @@ -260,6 +287,7 @@ class CalibrationRecorder: return self._samples.append( ( + int(sample.frame_id), float(sample.t_rel_s), float(sample.fn_raw), float(sample.fx_raw), @@ -303,10 +331,10 @@ class CalibrationRecorder: ended_at = self._wall_clock().isoformat(timespec="seconds") actual_duration = 0.0 if samples: - actual_duration = max(0.0, samples[-1][0] - samples[0][0]) + actual_duration = max(0.0, samples[-1][1] - samples[0][1]) - block = np.asarray(samples, dtype=np.float64) if samples else np.zeros((0, 8)) - valid_mask = block[:, 7] > 0.5 if block.size else np.zeros(0, dtype=bool) + block = np.asarray(samples, dtype=np.float64) if samples else np.zeros((0, 9)) + valid_mask = block[:, 8] > 0.5 if block.size else np.zeros(0, dtype=bool) valid_block = block[valid_mask] if block.size else block def axis_stats(column: int) -> AxisStats: @@ -327,11 +355,15 @@ class CalibrationRecorder: requested_duration_s=duration, n_total=int(block.shape[0]), n_valid=int(valid_block.shape[0]), - raw={"fn": axis_stats(1), "fx": axis_stats(2), "fy": axis_stats(3)}, - newtons={"fn": axis_stats(4), "fx": axis_stats(5), "fy": axis_stats(6)}, + raw={"fn": axis_stats(2), "fx": axis_stats(3), "fy": axis_stats(4)}, + newtons={"fn": axis_stats(5), "fx": axis_stats(6), "fy": axis_stats(7)}, status=STATUS_OK if valid_block.shape[0] >= 2 else STATUS_FEW_SAMPLES, trace=_trace_from(valid_block), trace_dt_s=_trace_dt(valid_block), + # Every sample is kept, including the invalid ones: a point that was + # mostly invalid is itself a finding. ``samples`` holds the original + # tuples so ``frame_id`` stays an integer. + samples=list(samples), ) with self._lock: self.points.append(point) @@ -403,16 +435,42 @@ class CalibrationRecorder: self.points = kept return removed + def load_points(self, points, *, replace: bool = True) -> int: + """Adopt points read back from a saved calibration session. + + Reopening a session has to put the operator back exactly where they left + off, so the diagnostics (excluded, validation) travel with the points. + """ + loaded = list(points) + with self._lock: + if replace: + self.points = loaded + else: + self.points.extend(loaded) + for point in loaded: + index = self._rounds.get(point.channel, 0) + self._rounds[point.channel] = max(index, int(point.round_index) + 1) + return len(loaded) + def points_for(self, channel: ChannelKey) -> list[CalibrationPoint]: with self._lock: return [point for point in self.points if point.channel == channel] def fit(self, channel: ChannelKey) -> FitResult: - """Least-squares gain/offset from the raw mean to the reference load. + """Diagnose the model that export and application would actually use. - Excluded points are dropped, validation points are held out, and every - point gets its residual filled in so the table can show the misfit. + The residual and validation figures used to come from a private + least-squares line while the exported file carried a piecewise curve, so + the numbers on screen did not describe the calibration that would be + applied. Both now come from the exported curve. Excluded points are + dropped, validation points are held out of the build and only measured. """ + from orisys_app_shared.force.calibration_io import ( + CalSample, + build_export_model, + predict_channel, + ) + points = self.points_for(channel) usable = [ point @@ -426,14 +484,33 @@ class CalibrationRecorder: if len(usable) < 2: return FitResult(channel=channel, n_points=len(usable)) - raws = np.asarray([point.loaded_stats().mean for point in usable], dtype=float) - loads = np.asarray([point.reference_load_n for point in usable], dtype=float) - if np.allclose(raws, raws[0]): + samples: dict[str, list[CalSample]] = {key: [] for key in CHANNEL_KEYS} + for point in usable: + samples[point.channel].append( + CalSample( + weight_n=float(point.reference_load_n), + raw_mean=float(point.loaded_stats().mean), + n=int(point.loaded_stats().n), + std=float(point.loaded_stats().std) + if np.isfinite(point.loaded_stats().std) + else 0.0, + ) + ) + try: + model = build_export_model(samples) + curve = model.get(channel) + except Exception: + # Without a buildable model there is nothing honest to report. return FitResult(channel=channel, n_points=len(usable)) - gain, offset = np.polyfit(raws, loads, 1) - residuals = (gain * raws + offset) - loads - for point, residual in zip(usable, residuals): + if curve is None: + return FitResult(channel=channel, n_points=len(usable)) + + residuals = [] + for point in usable: + predicted = predict_channel(model, point.channel, point.loaded_stats().mean) + residual = predicted - float(point.reference_load_n) point.residual_n = float(residual) + residuals.append(residual) validation = [ point @@ -445,19 +522,30 @@ class CalibrationRecorder: validation_error = NA if validation: errors = [ - abs(gain * point.loaded_stats().mean + offset - point.reference_load_n) + abs( + predict_channel(model, point.channel, point.loaded_stats().mean) + - float(point.reference_load_n) + ) for point in validation ] validation_error = float(max(errors)) + raws = np.asarray([point.loaded_stats().mean for point in usable], dtype=float) + loads = np.asarray([point.reference_load_n for point in usable], dtype=float) + linear_gain = linear_offset = NA + if not np.allclose(raws, raws[0]): + linear_gain, linear_offset = (float(v) for v in np.polyfit(raws, loads, 1)) + return FitResult( channel=channel, - gain=float(gain), - offset=float(offset), n_points=len(usable), max_residual_n=float(np.max(np.abs(residuals))), validation_error_n=validation_error, validation_points=len(validation), + method=str(getattr(curve, "method", "") or ""), + breakpoints=len(getattr(curve, "points", ()) or ()), + linear_gain=linear_gain, + linear_offset=linear_offset, ) def _next_round(self, channel: ChannelKey) -> int: @@ -472,7 +560,7 @@ def _trace_from(valid_block: np.ndarray) -> dict[str, list[float]]: if valid_block.size == 0: return {} traces: dict[str, list[float]] = {} - for key, column in (("fn", 1), ("fx", 2), ("fy", 3)): + for key, column in (("fn", 2), ("fx", 3), ("fy", 4)): values = valid_block[:, column] values = values[np.isfinite(values)] if values.size == 0: @@ -498,5 +586,5 @@ def _trace_from(valid_block: np.ndarray) -> dict[str, list[float]]: def _trace_dt(valid_block: np.ndarray) -> float: if valid_block.shape[0] < 2: return 0.0 - span = float(valid_block[-1, 0] - valid_block[0, 0]) + span = float(valid_block[-1, 1] - valid_block[0, 1]) return span / max(1, valid_block.shape[0] - 1) diff --git a/shared/src/orisys_app_shared/force/calibration_session.py b/shared/src/orisys_app_shared/force/calibration_session.py new file mode 100644 index 0000000..7b27c2a --- /dev/null +++ b/shared/src/orisys_app_shared/force/calibration_session.py @@ -0,0 +1,240 @@ +"""Persist a calibration session: every point and its full three-axis samples. + +The exported SDK coefficients are the *result* of a calibration; this file is the +*evidence*. It keeps what the coefficients cannot: + +* the complete sample series of every point (frame id, relative time, all three + raw axes, all three newton axes and the validity flag), so a point can be + re-examined even though the diagnostic trace on screen is downsampled; +* the actual duration of each point next to the duration that was requested; +* which points were excluded and which were held out as validation points; +* the direction (load / unload) and note of each point. + +The file is written next to the exported coefficients and can be reopened to +inspect the session again. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, Optional + +logger = logging.getLogger(__name__) + +#: Format version of the calibration session document. +SESSION_FORMAT_VERSION = "1.0" +SESSION_REQUIRED = ("format", "format_version", "created_at", "points") +#: Columns of each sample row inside a point, in order. +SAMPLE_FIELDS = ( + "frame_id", + "t_rel_s", + "fn_raw", + "fx_raw", + "fy_raw", + "fn_n", + "fx_n", + "fy_n", + "valid", +) +POINT_REQUIRED = ("channel", "reference_load_n") + + +@dataclass +class CalibrationSessionRecord: + """One reopened calibration session.""" + + path: Path + format_version: str + created_at: str + meta: dict = field(default_factory=dict) + points: list = field(default_factory=list) + #: What could not be restored exactly (reported, never silently dropped). + warnings: list[str] = field(default_factory=list) + + @property + def ok(self) -> bool: + return bool(self.points) and not self.warnings + + +def _jsonable(value: Any) -> Any: + """NaN/Inf are not valid JSON; they are written as ``null``.""" + if isinstance(value, float): + if value != value or value in (float("inf"), float("-inf")): + return None + return value + if isinstance(value, dict): + return {key: _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + return value + + +def build_session_document( + points: Iterable, + *, + meta: Optional[dict] = None, + created_at: Optional[str] = None, +) -> dict: + """Build the document for ``points`` without touching the disk.""" + documents = [] + for point in points: + entry = _jsonable(point.as_dict()) + # The full series is the audit data; the trace stays as a quick preview. + entry["sample_fields"] = list(SAMPLE_FIELDS) + entry["samples"] = [ + [None if value != value else value for value in row] + for row in getattr(point, "samples", ()) + ] + documents.append(entry) + return { + "format": "orisys.calibration_session", + "format_version": SESSION_FORMAT_VERSION, + "created_at": created_at + or datetime.now(timezone.utc).isoformat(timespec="seconds"), + "sample_fields": list(SAMPLE_FIELDS), + "meta": _jsonable(dict(meta or {})), + "points": documents, + } + + +def session_filename(sensor_type: str, *, stamp: Optional[str] = None) -> str: + when = stamp or datetime.now().strftime("%Y%m%d_%H%M%S") + return f"calibration_session_{sensor_type or 'unknown'}_{when}.json" + + +def write_calibration_session( + path: str | Path, + points: Iterable, + *, + meta: Optional[dict] = None, +) -> Path: + """Write the session document. Returns the written path.""" + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + document = build_session_document(points, meta=meta) + target.write_text( + json.dumps(document, indent=2, ensure_ascii=False, allow_nan=False), + encoding="utf-8", + ) + return target + + +def read_calibration_session(path: str | Path) -> CalibrationSessionRecord: + """Load a session document and rebuild its points. + + A file that cannot be trusted must say so: a truncated or foreign document + raises, and a document from a newer format is loaded with a warning instead + of being interpreted as if the layout were known. + """ + target = Path(path) + data = json.loads(target.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError(f"not a calibration session document: {target}") + missing = [key for key in SESSION_REQUIRED if key not in data] + if missing: + raise ValueError( + f"calibration session {target.name} is missing {', '.join(missing)}" + ) + if str(data.get("format")) != "orisys.calibration_session": + raise ValueError(f"not a calibration session document: {target}") + + warnings: list[str] = [] + version = str(data.get("format_version") or "") + if version != SESSION_FORMAT_VERSION: + warnings.append(f"format_version {version or '?'} is not {SESSION_FORMAT_VERSION}") + + from orisys_app_shared.force.calibration_recorder import ( + CalibrationPoint, + AxisStats, + ) + + points = [] + for index, raw in enumerate(data.get("points") or []): + if not isinstance(raw, dict): + warnings.append(f"point {index}: not an object") + continue + absent = [key for key in POINT_REQUIRED if key not in raw] + if absent: + warnings.append(f"point {index}: missing {', '.join(absent)}") + continue + points.append(_point_from_dict(raw, CalibrationPoint, AxisStats)) + return CalibrationSessionRecord( + path=target, + format_version=version, + created_at=str(data.get("created_at") or ""), + meta=dict(data.get("meta") or {}), + points=points, + warnings=warnings, + ) + + +def _stats_from_dict(raw: Any, AxisStats): + if not isinstance(raw, dict): + return AxisStats() + def _number(key: str, fallback: float) -> float: + value = raw.get(key) + if value is None: + return fallback + try: + return float(value) + except (TypeError, ValueError): + return fallback + + return AxisStats( + mean=_number("mean", float("nan")), + std=_number("std", float("nan")), + n=int(raw.get("n") or 0), + ) + + +def _point_from_dict(raw: dict, CalibrationPoint, AxisStats): + point = CalibrationPoint( + channel=str(raw.get("channel") or "fn"), + reference_load_n=float(raw.get("reference_load_n") or 0.0), + direction=str(raw.get("direction") or "load"), + round_index=int(raw.get("round_index") or 0), + note=str(raw.get("note") or ""), + started_at=str(raw.get("started_at") or ""), + ended_at=str(raw.get("ended_at") or ""), + duration_s=float(raw.get("duration_s") or 0.0), + requested_duration_s=float(raw.get("requested_duration_s") or 0.0), + n_total=int(raw.get("n_total") or 0), + n_valid=int(raw.get("n_valid") or 0), + status=str(raw.get("status") or "ok"), + excluded=bool(raw.get("excluded", False)), + is_validation=bool(raw.get("is_validation", False)), + ) + point.raw = { + key: _stats_from_dict(value, AxisStats) + for key, value in (raw.get("raw") or {}).items() + } + point.newtons = { + key: _stats_from_dict(value, AxisStats) + for key, value in (raw.get("newtons") or {}).items() + } + point.trace = { + str(key): [float(value) for value in (series or [])] + for key, series in (raw.get("trace") or {}).items() + } + point.residual_n = _optional_float(raw.get("residual_n")) + rows = [] + for row in raw.get("samples") or []: + if not isinstance(row, (list, tuple)): + continue + values = [float("nan") if value is None else value for value in row] + rows.append(tuple(values)) + point.samples = rows + return point + + +def _optional_float(value: Any) -> float: + if value is None: + return float("nan") + try: + return float(value) + except (TypeError, ValueError): + return float("nan") diff --git a/shared/src/orisys_app_shared/force/session.py b/shared/src/orisys_app_shared/force/session.py index 8cab55c..f5eac44 100644 --- a/shared/src/orisys_app_shared/force/session.py +++ b/shared/src/orisys_app_shared/force/session.py @@ -64,6 +64,32 @@ class SessionSample: valid: bool = True calibration_id: str = "" error: str = "" + #: Axes that actually have a calibration curve. An axis outside this set has + #: no newton value, which the UI shows as a dash rather than as a number. + calibrated_axes: tuple[str, ...] = () + + @classmethod + def from_record(cls, record) -> "SessionSample": + """Build from the canonical :class:`RecordSample`. + + The record channel has one conversion from a packet (see + ``record.sample.to_record_sample``); this only renames the fields so the + store keeps its own shape. + """ + return cls( + frame_id=int(record.frame_id), + t_rel_s=float(record.mono_t), + fn_raw=float(record.fn_raw), + fx_raw=float(record.fx_raw), + fy_raw=float(record.fy_raw), + fn_n=float(record.fn_n), + fx_n=float(record.fx_n), + fy_n=float(record.fy_n), + valid=bool(record.valid), + calibration_id=str(record.calibration_version or ""), + error=str(record.invalid_reason or ""), + calibrated_axes=tuple(record.calibrated_axes or ()), + ) def raw_triplet(self) -> tuple[float, float, float]: return (self.fn_raw, self.fx_raw, self.fy_raw) @@ -98,6 +124,18 @@ class SessionSlice: return self.newtons return self.raw + def plottable(self, unit: str) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """``(times, values, mask)`` for drawing, under one shared rule. + + A point is drawn only when the sample is **valid** *and* the value for the + requested unit is finite. Filtering on finiteness alone let a stale + number — left behind by a failed compute step — reappear in one view + after another view had already excluded it. + """ + values = self.value_for_unit(unit) + mask = self.valid & np.isfinite(values).all(axis=1) + return self.times[mask], values[mask], mask + @dataclass(frozen=True) class SelectionStats: @@ -110,39 +148,49 @@ class SelectionStats: stds: tuple[float, float, float] = (NA, NA, NA) -def _decimate_preserving_peaks(rows: np.ndarray, max_points: int) -> np.ndarray: +def _decimate_preserving_peaks(rows: np.ndarray, max_points: int, unit: str = "raw") -> np.ndarray: """Reduce ``rows`` to about ``max_points`` while keeping every spike. - Each bucket contributes its minimum and its maximum sample (in time order), - so the envelope of the signal survives decimation instead of being averaged - flat. + Each bucket contributes the extreme sample of **every plotted axis** in the + active unit, deduplicated and put back in time order. Selecting on one axis + only (the old behaviour) erased a shear spike completely: with FN flat and a + single FX point at 999, the decimated data reported an FX maximum of 0. """ total = rows.shape[0] if max_points < 4 or total <= max_points: return rows - bucket = max(2, total // max(2, max_points // 2)) + columns = ( + (_COL_FN_N, _COL_FX_N, _COL_FY_N) + if str(unit).upper().startswith("N") + else (_COL_FN_RAW, _COL_FX_RAW, _COL_FY_RAW) + ) + bucket = max(2, total // max(2, max_points // (2 * len(columns)))) usable = (total // bucket) * bucket head = rows[:usable].reshape(-1, bucket, rows.shape[1]) tail = rows[usable:] - # Index of the extreme value on the first channel; ties go to the earlier one. - values = head[:, :, _COL_FN_RAW] - finite = np.isfinite(values) - values = np.where(finite, values, -np.inf) - low = np.argmin(values, axis=1) - high = np.argmax(values, axis=1) - first = np.minimum(low, high) - second = np.maximum(low, high) - index = np.arange(head.shape[0]) - picked = np.concatenate( - [ - head[index, first], - head[index, second], - ], - axis=0, - ) - # ``concatenate`` above groups all lows then all highs; restore time order. - order = np.argsort(picked[:, _COL_FRAME_ID], kind="stable") - picked = picked[order] + buckets = head.shape[0] + index = np.arange(buckets) + + picks: list[np.ndarray] = [] + for column in columns: + values = head[:, :, column] + finite = np.isfinite(values) + ordered = np.where(finite, values, -np.inf) + low = np.argmin(ordered, axis=1) + high = np.argmax(ordered, axis=1) + picks.append(head[index, np.minimum(low, high)]) + picks.append(head[index, np.maximum(low, high)]) + picked = np.concatenate(picks, axis=0) + + # De-duplicate (the same row is often the extreme for several axes) and + # restore time order. Sorting the *rows* by frame id is what restores the + # order; sorting the indices of a concatenation of per-axis picks does not, + # because the picks restart from the first bucket for every axis. + picked = picked[np.argsort(picked[:, _COL_FRAME_ID], kind="stable")] + keep = np.ones(picked.shape[0], dtype=bool) + if picked.shape[0] > 1: + keep[1:] = np.diff(picked[:, _COL_FRAME_ID]) != 0 + picked = picked[keep] if tail.shape[0]: picked = np.vstack([picked, tail]) return picked @@ -321,12 +369,15 @@ class SessionStore: def _block_span(self, block: np.ndarray, rows: int) -> tuple[float, float]: return (float(block[0, _COL_T]), float(block[rows - 1, _COL_T])) - def visible(self, t0: float, t1: float, max_points: int = 2000) -> SessionSlice: - """Decimated slice for drawing, keeping peaks. + def visible( + self, t0: float, t1: float, max_points: int = 2000, *, unit: str = "raw" + ) -> SessionSlice: + """Decimated slice for drawing, keeping peaks on every plotted axis. Only the blocks that overlap ``[t0, t1]`` are touched, so browsing a - long session does not copy the whole thing, and the decimation emits the - min and the max of each bucket so a spike can never be smoothed away. + long session does not copy the whole thing, and the decimation keeps the + extremes of all three axes in ``unit`` so no channel's spike is smoothed + away. """ t0 = float(t0) t1 = float(t1) @@ -336,7 +387,7 @@ class SessionStore: rows = self._packed_window(t0, t1) truncated = self._dropped > 0 if max_points and rows.shape[0] > max_points: - rows = _decimate_preserving_peaks(rows, max_points) + rows = _decimate_preserving_peaks(rows, max_points, unit) return replace(self._slice_from(rows), truncated=truncated) def _packed_window(self, t0: float, t1: float) -> np.ndarray: diff --git a/shared/src/orisys_app_shared/record/depth.py b/shared/src/orisys_app_shared/record/depth.py index 61875b8..fe4ba4d 100644 --- a/shared/src/orisys_app_shared/record/depth.py +++ b/shared/src/orisys_app_shared/record/depth.py @@ -38,12 +38,28 @@ class DepthResult: error: str = "" frame_id: int = -1 t_rel_s: float = 0.0 + #: What a replay could not reproduce exactly (never silently empty). + warnings: tuple[str, ...] = () @property def ok(self) -> bool: return self.depth is not None and not self.error +@dataclass(frozen=True) +class ReplayPlan: + """Where the SDK must start reading to reproduce one frame's state.""" + + segment: Path + #: Frame within ``segment`` to step to (1-based, as the SDK reads them). + stop_at: int + #: Frames to feed before ``stop_at`` to rebuild the algorithm's state. + preroll: Path | None = None + preroll_frames: int = 0 + warnings: tuple[str, ...] = () + error: str = "" + + class DepthRecomputer: """Recompute depth maps for one recording, on demand.""" @@ -51,7 +67,10 @@ class DepthRecomputer: self.reader = reader self._sensor = None self._segment: Optional[Path] = None + #: How many frames the open sensor has already consumed in ``_segment``. + self._position = 0 self.last_error = "" + self.last_warnings: list[str] = [] # ------------------------------------------------------------------ probing @staticmethod @@ -68,6 +87,21 @@ class DepthRecomputer: device = session.get("device") or {} return str(device.get("config_ref") or "") + def resolved_config(self) -> str: + """The exact configuration file to open, resolved as the live run did. + + Reducing the path to a directory name or a stem could silently load a + *different* file with the same name while the hash check still compared + the recorded path — so the full resolved path is what gets opened. + """ + config_ref = self.config_ref() + try: + from orisys_app_shared.acquisition.sdk_worker import load_fixed_mode + + return str(load_fixed_mode(config_ref).config_ref) + except Exception: # noqa: BLE001 - reported by the caller as a failure + return config_ref + def preflight(self) -> str: """Return an empty string when recomputation can be attempted.""" ok, detail = self.sdk_available() @@ -82,6 +116,72 @@ class DepthRecomputer: return "no_video" return "" + # --------------------------------------------------------------- replay plan + def plan_for(self, frame) -> ReplayPlan: + """Decide where replay must start so ``frame`` is reproducible. + + The SDK's depth is the output of a stateful chain (optical-flow tracker, + temporal filters), so a frame cannot be recomputed in isolation. Two + things make the state recoverable: + + * an operator **reset** clears the tracker, so replaying from the reset + frame reproduces the state exactly — no earlier history needed; + * the **pre-roll** recorded with the session holds the frames acquired + just before recording started, which is the only history a reset-free + start has. + + Anything else is replayed as faithfully as possible *and reported*: the + caller is told the state may differ instead of being shown a depth map + that pretends to be the live one. + """ + warnings: list[str] = [] + segment_path = self.reader.segment_path(frame.segment) + if segment_path is None: + return ReplayPlan( + segment=Path(), stop_at=0, error="segment_missing" + ) + + reset_frame = self.reader.last_reset_frame(frame.frame_id) + if reset_frame is not None: + origin = self.reader.frame_of(reset_frame) + if origin is not None and origin.segment == frame.segment: + # Exact: the tracker state at ``frame`` is fully determined by + # the frames since the reset in this same segment. + return ReplayPlan( + segment=segment_path, + stop_at=max(1, int(frame.segment_frame)), + warnings=(), + ) + if origin is not None: + # The reset is in an earlier segment. The SDK opens one video per + # sensor, so the state cannot be carried across the boundary; the + # replay restarts at this segment and says so. + warnings.append("state_across_segment") + + start = self.reader.segment_start_frame(frame.segment) + if start is None: + return ReplayPlan(segment=Path(), stop_at=0, error="frame_not_indexed") + + preroll = self.reader.preroll_segment() + preroll_frames = self.reader.preroll_frames() if preroll is not None else 0 + if preroll is not None and preroll_frames > 0 and frame.segment == 1: + # Recording started mid-stream: the frames before the first recorded + # one are the only history that can rebuild the tracker. + return ReplayPlan( + segment=segment_path, + stop_at=max(1, int(frame.segment_frame)), + preroll=preroll, + preroll_frames=int(preroll_frames), + warnings=tuple(warnings), + ) + if int(frame.segment_frame) > 1: + warnings.append("state_approximate") + return ReplayPlan( + segment=segment_path, + stop_at=max(1, int(frame.segment_frame)), + warnings=tuple(warnings), + ) + # ----------------------------------------------------------------- recompute def depth_at(self, t_rel_s: float) -> DepthResult: """Recompute the depth map for the frame at ``t_rel_s``.""" @@ -93,6 +193,13 @@ class DepthRecomputer: frame = self.reader.frame_at(t_rel_s) if frame is None: return DepthResult(error="no_frame", t_rel_s=float(t_rel_s)) + if frame.dropped: + # The row was lost to a full queue; the index row marks the hole. + return DepthResult( + error="frame_not_recorded", + frame_id=frame.frame_id, + t_rel_s=frame.t_rel_s, + ) if not frame.in_video: # The queue dropped this frame; say so rather than show the previous # frame as if it belonged to the cursor position. @@ -100,67 +207,131 @@ class DepthRecomputer: error="frame_not_in_video", frame_id=frame.frame_id, t_rel_s=frame.t_rel_s ) - segment = self.reader.segment_path(frame.segment) - if segment is None: + plan = self.plan_for(frame) + if plan.error: + self.last_error = plan.error return DepthResult( - error="segment_missing", frame_id=frame.frame_id, t_rel_s=frame.t_rel_s + error=plan.error, frame_id=frame.frame_id, t_rel_s=frame.t_rel_s ) try: - depth = self._depth_from_segment(segment, frame.segment_frame) + depth = self._replay(plan) except Exception as exc: # noqa: BLE001 - reported to the caller logger.exception("Depth recomputation failed") self.last_error = f"{type(exc).__name__}: {exc}" return DepthResult( error=self.last_error, frame_id=frame.frame_id, t_rel_s=frame.t_rel_s ) + self.last_warnings = list(plan.warnings) if depth is None: self.last_error = "depth_not_available" return DepthResult( - error=self.last_error, frame_id=frame.frame_id, t_rel_s=frame.t_rel_s + error=self.last_error, + frame_id=frame.frame_id, + t_rel_s=frame.t_rel_s, + warnings=plan.warnings, ) self.last_error = "" - return DepthResult(depth=depth, frame_id=frame.frame_id, t_rel_s=frame.t_rel_s) + return DepthResult( + depth=depth, + frame_id=frame.frame_id, + t_rel_s=frame.t_rel_s, + warnings=plan.warnings, + ) - def _depth_from_segment(self, segment: Path, segment_frame: int) -> Optional[np.ndarray]: - """Open the segment with the SDK and read the depth channel.""" - import orisys + def _replay(self, plan: ReplayPlan) -> Optional[np.ndarray]: + """Read the pre-roll (if any), then step to the planned frame.""" + if plan.preroll is not None and plan.preroll_frames > 0: + self._feed(plan.preroll, int(plan.preroll_frames)) + if self._segment != plan.segment or self._position > plan.stop_at: + self._reopen_segment(plan.segment) + return self._advance_to(int(plan.stop_at)) - if self._segment != segment or self._sensor is None: - self._close() - self._sensor = orisys.Sensor( - str(segment), - config_name=self._config_name(), - verbose=False, - backend="auto", - ) - self._segment = segment - sensor = self._sensor - # The SDK's video source advances frame by frame, so a seek means - # stepping. Correctness first: we never return a frame we did not read. + def _depth_from_segment( + self, segment: Path, segment_frame: int + ) -> Optional[np.ndarray]: + """Recompute the map for ``segment_frame`` of ``segment``. + + The SDK's video source only advances, so stepping costs one read per + frame — and the position is tracked, so asking for frame 2 and then + frame 3 reads two frames and then one, instead of two more. + """ target = max(1, int(segment_frame)) - last_depth = None - for _ in range(target): - image = sensor.get_img() - if image is None: - break - info = sensor.info - if not hasattr(info, "DEPTH"): - return None - last_depth = sensor.read_info(info.DEPTH) - return None if last_depth is None else np.asarray(last_depth) + if self._segment != segment or self._position > target: + self._reopen_segment(segment) + return self._advance_to(target) - def _config_name(self) -> str: - config_ref = self.config_ref() - path = Path(config_ref) - return path.parent.name if path.name == "sensor.json" else path.stem + # --------------------------------------------------------------------- video + def sensor_type(self) -> str: + device = self.reader.session().get("device") or {} + return str(device.get("sensor_type") or "box") + + def _reopen_segment(self, segment: Path) -> None: + self._close() + if self.sensor_type() == "finger": + self._sensor = _FingerReplay( + segment, self.resolved_config(), self._finger_export_dir() + ) + else: + self._sensor = _BoxReplay(segment, self.resolved_config()) + self._segment = Path(segment) + self._position = 0 + + def _finger_export_dir(self): + """The finger pipeline assets, resolved exactly as the live run did.""" + try: + from orisys_app_shared.acquisition.finger_worker import ( + resolve_finger_export_dir, + ) + + return resolve_finger_export_dir(self.resolved_config()) + except Exception: # noqa: BLE001 - the SDK default is used instead + logger.exception("Could not resolve the finger export directory") + return None + + def _feed(self, segment: Path, frames: int) -> None: + """Consume ``frames`` frames from another file to rebuild state.""" + if frames <= 0: + return + if self._sensor is None or self._segment != segment: + self._reopen_segment(segment) + for _ in range(int(frames)): + if self._read_one() is None: + break + + def _advance_to(self, target: int) -> Optional[np.ndarray]: + last_depth = None + while self._position < int(target): + depth = self._read_one() + if depth is None: + break + last_depth = depth + return last_depth + + def _read_one(self) -> Optional[np.ndarray]: + """Read one frame through the **live** processing chain. + + ``get_img``/``read_source`` alone is not the live pipeline: the SDK's + depth map is produced by ``compute_deformation`` + ``compute_contact`` + (and, for the finger, the unrolling step), so skipping them would replay + a different algorithm than the one that produced the recording. + """ + replay = self._sensor + if replay is None: + return None + depth = replay.read() + if depth is None: + return None + self._position += 1 + return depth def _close(self) -> None: - sensor, self._sensor = self._sensor, None + replay, self._sensor = self._sensor, None self._segment = None - if sensor is None: + self._position = 0 + if replay is None: return try: - sensor.disconnect() + replay.close() except Exception: # noqa: BLE001 pass @@ -172,3 +343,97 @@ class DepthRecomputer: def __exit__(self, *_exc) -> None: self.close() + + +class _BoxReplay: + """The box sensor's live chain, opened on one video file.""" + + def __init__(self, segment: Path, config_ref: str) -> None: + import orisys + + self._sensor = orisys.Sensor( + str(segment), + config_name=config_ref, + verbose=False, + backend="auto", + ) + + def read(self) -> Optional[np.ndarray]: + image = self._sensor.get_img() + if image is None: + return None + compute_deformation = getattr(self._sensor, "compute_deformation", None) + compute_contact = getattr(self._sensor, "compute_contact", None) + if callable(compute_deformation) and callable(compute_contact): + # Same calls and arguments as the acquisition loop. + compute_deformation(decompose=True, check_motion=False) + compute_contact() + else: + logger.warning( + "Sensor %r has no processing chain; replay depth is not the live algorithm", + type(self._sensor).__name__, + ) + info = self._sensor.info + if not hasattr(info, "DEPTH"): + return None + value = self._sensor.read_info(info.DEPTH) + return None if value is None else np.asarray(value) + + def close(self) -> None: + self._sensor.disconnect() + + +class _FingerReplay: + """The finger's live chain: unroll the source frame, then process the tactile.""" + + def __init__(self, segment: Path, config_ref: str, export_dir) -> None: + from orisys.finger import FingerRuntime + + from orisys_app_shared.acquisition.constants import ( + FORCE_SAMPLE_RATE_HZ, + LOWPASS_CUTOFF_HZ, + ) + + kwargs: dict = { + "config": config_ref, + "video": str(segment), + "verbose": False, + } + if export_dir is not None: + kwargs["export_dir"] = str(export_dir) + self._runtime = FingerRuntime.open(**kwargs) + self._runtime.warmup(color=False) + tactile = getattr(self._runtime, "tactile", None) + if tactile is not None and hasattr(tactile, "configure"): + try: + # The same filter settings the acquisition loop applies. + tactile.configure( + lowpass_cutoff_hz=LOWPASS_CUTOFF_HZ, + lowpass_sample_rate_hz=FORCE_SAMPLE_RATE_HZ, + ) + except Exception: # noqa: BLE001 - the defaults still work + logger.exception("Could not configure the replayed tactile sensor") + + def read(self) -> Optional[np.ndarray]: + ok, source_frame = self._runtime.read_source() + if not ok or source_frame is None: + return None + unrolled = self._runtime.process_frame(source_frame, color=False) + if unrolled is None: + return None + tactile = self._runtime.tactile + compute_deformation = getattr(tactile, "compute_deformation", None) + compute_contact = getattr(tactile, "compute_contact", None) + if callable(compute_deformation) and callable(compute_contact): + compute_deformation(decompose=True, check_motion=False) + compute_contact() + else: + logger.warning("Finger tactile sensor has no processing chain") + info = tactile.info + if not hasattr(info, "DEPTH"): + return None + value = tactile.read_info(info.DEPTH) + return None if value is None else np.asarray(value) + + def close(self) -> None: + self._runtime.close() diff --git a/shared/src/orisys_app_shared/record/experiment.py b/shared/src/orisys_app_shared/record/experiment.py index 6fea1cb..04dc743 100644 --- a/shared/src/orisys_app_shared/record/experiment.py +++ b/shared/src/orisys_app_shared/record/experiment.py @@ -31,6 +31,7 @@ import logging import queue import threading import time +from collections import deque from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path @@ -39,9 +40,21 @@ from typing import Any, Optional import cv2 import numpy as np +from orisys_app_shared.record.sample import RecordSample, to_record_sample + logger = logging.getLogger(__name__) -RECORD_FORMAT_VERSION = "1.0" +RECORD_FORMAT_VERSION = "1.1" + +# Lifecycle states. ``failed`` and ``stopped`` are terminal; a recorder that has +# not reached a terminal state must not be reused. +STATE_IDLE = "idle" +STATE_STARTING = "starting" +STATE_RECORDING = "recording" +STATE_STOPPING = "stopping" +STATE_STOPPED = "stopped" +STATE_FAILED = "failed" +TERMINAL_STATES = frozenset({STATE_STOPPED, STATE_FAILED}) #: Column order required by the review; do not reorder without a format bump. FORCES_HEADER = ( @@ -66,11 +79,16 @@ FRAMES_HEADER = ( "in_video", "dropped", ) -EVENTS_HEADER = ("at", "t_rel_s", "kind", "detail") +#: ``frame_id`` was added so an operator action can be tied to a frame position. +EVENTS_HEADER = ("at", "t_rel_s", "frame_id", "kind", "detail") MAX_CONSECUTIVE_ERRORS = 5 #: A sustained queue overflow reports at most every N drops. ALARM_EVERY_N_DROPS = 1000 +#: How many lost-row gaps may be remembered for the frame index. +GAP_BACKLOG = 4096 +#: How many data rows may be parked while the item queue is full of video. +ROW_BACKLOG = 8192 def format_value(value: float) -> str: @@ -98,7 +116,16 @@ class RecordConfig: codec: str = "mp4v" segment_seconds: float = 300.0 max_queue: int = 2048 + #: Byte budget for the video queue. The item count alone is not a bound: at + #: 1920x1080 BGR, 2048 queued frame copies would be ~11.9 GiB. + max_queue_bytes: int = 512 * 1024 * 1024 flush_interval_s: float = 1.0 + #: Frames captured just before recording starts. The depth replay runs the + #: SDK's stateful chain, so the first frames of a recording cannot be + #: reproduced without the history that preceded them. Rate-limited by + #: ``preroll_bytes`` so a large frame does not turn into a large buffer. + preroll_frames: int = 30 + preroll_bytes: int = 192 * 1024 * 1024 def as_dict(self) -> dict: return { @@ -108,6 +135,9 @@ class RecordConfig: "codec": self.codec, "segment_seconds": float(self.segment_seconds), "max_queue": int(self.max_queue), + "max_queue_bytes": int(self.max_queue_bytes), + "preroll_frames": int(self.preroll_frames), + "preroll_bytes": int(self.preroll_bytes), "lossy_note": ( "MP4 is a lossy container: this is the pre-processing camera " "image, not a lossless record of the raw sensor data." @@ -122,11 +152,16 @@ class RecordStats: segments: int = 0 dropped_data: int = 0 dropped_video: int = 0 + #: Frames recorded as a hole because their row was lost from the queue. + dropped_gaps: int = 0 + #: Lost rows that could not even be marked: the gap backlog was full. + unrecorded_gaps: int = 0 write_errors: int = 0 failed: bool = False last_error: str = "" duration_s: float = 0.0 bytes_written: int = 0 + rejected_after_failure: int = 0 def as_dict(self) -> dict: return { @@ -135,11 +170,14 @@ class RecordStats: "segments": self.segments, "dropped_data": self.dropped_data, "dropped_video": self.dropped_video, + "dropped_gaps": self.dropped_gaps, + "unrecorded_gaps": self.unrecorded_gaps, "write_errors": self.write_errors, "failed": self.failed, "last_error": self.last_error, "duration_s": self.duration_s, "bytes_written": self.bytes_written, + "rejected_after_failure": self.rejected_after_failure, } @@ -150,10 +188,17 @@ class RecordReport: directory: Optional[Path] stats: RecordStats problems: list[str] = field(default_factory=list) + #: False when the writer had not finished closing the files. + completed: bool = True @property def ok(self) -> bool: - return bool(self.directory) and not self.stats.failed and not self.problems + return ( + bool(self.directory) + and self.completed + and not self.stats.failed + and not self.problems + ) class ExperimentRecorder: @@ -162,14 +207,26 @@ class ExperimentRecorder: def __init__(self, config: Optional[RecordConfig] = None) -> None: self.config = config or RecordConfig() self._queue: queue.Queue = queue.Queue(maxsize=max(16, self.config.max_queue)) + #: Bounded side channel for frames whose data row was lost to a full + #: queue. It is tiny, so it never grows with the recording. + self._gaps: deque = deque(maxlen=GAP_BACKLOG) + #: Rows that did not fit in the item queue because it was full of video. + #: They are tiny, so a small backlog covers any realistic stall. + self._row_backlog: deque = deque(maxlen=ROW_BACKLOG) + self._queued_bytes = 0 self._thread: Optional[threading.Thread] = None self._stop = threading.Event() self._lock = threading.Lock() - self._writing = False + #: Serialises session.json writes: the UI thread writes at start and the + #: writer thread writes again once the pre-roll is on disk. + self._json_lock = threading.Lock() + #: Explicit lifecycle. A boolean could not express "the writer died". + self._state: str = STATE_IDLE self._directory: Optional[Path] = None self._stats = RecordStats() self._session_id = 0 self._calibration_id = "" + self._calibration_version = "" self._metadata: dict[str, Any] = {} self._events: list[dict] = [] self._t0: Optional[float] = None @@ -177,12 +234,51 @@ class ExperimentRecorder: self._started_at = "" self._consecutive_errors = 0 self._on_alarm = None + self._report: Optional[RecordReport] = None + self._accepting = False + #: Outcome of the pre-roll write, published for session.json. + self._preroll: list = [] + self._preroll_frames = 0 + self._preroll_segment: Optional[str] = None + #: Problems the writer found while closing, carried into the report. + self._writer_problems: list[str] = [] # ------------------------------------------------------------------ state @property - def is_recording(self) -> bool: + def state(self) -> str: with self._lock: - return self._writing + return self._state + + @property + def is_recording(self) -> bool: + """True while packets are being accepted and written. + + ``starting`` counts: the writer may still be opening its files, but the + session is live and buffering, and the operator's cancel affordance has + to be live from the moment ``start`` returns. A writer that has failed + clears this, which is what makes a dead recording visible. + """ + with self._lock: + return self._state in (STATE_STARTING, STATE_RECORDING) + + @property + def report(self) -> Optional[RecordReport]: + """The terminal report, available once the writer has finished.""" + return self._report + + def _set_state(self, state: str) -> None: + with self._lock: + self._state = state + + def _accepts_packets(self) -> bool: + """Packets are accepted from the moment ``start`` returns. + + Waiting for the writer thread to open the files would silently drop + everything acquired in between; the bounded queue buffers those rows, and + a failed writer still flips the state so acceptance stops. + """ + with self._lock: + return self._accepting and self._state in (STATE_STARTING, STATE_RECORDING) @property def directory(self) -> Optional[Path]: @@ -204,10 +300,22 @@ class ExperimentRecorder: session_id: int, metadata: dict[str, Any], calibration_id: str = "", + calibration_version: str = "", + preroll=(), ) -> Path: """Create the experiment directory and start the writer thread.""" - if self.is_recording: - raise RuntimeError("experiment recording is already running") + # A previous writer that has not reached a terminal state still owns the + # files and the statistics; reusing it would silently mix two sessions. + if self._thread is not None and self._thread.is_alive(): + raise RuntimeError( + "a previous recording writer is still running; stop() it first" + ) + with self._lock: + if self._state not in (STATE_IDLE,) and self._state not in TERMINAL_STATES: + raise RuntimeError( + f"cannot start recording from state {self._state!r}" + ) + self._state = STATE_STARTING stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") directory = Path(root) / f"{stamp}_session{session_id:04d}" directory.mkdir(parents=True, exist_ok=True) @@ -216,13 +324,24 @@ class ExperimentRecorder: self._stats = RecordStats() self._session_id = int(session_id) self._calibration_id = str(calibration_id or "") + self._calibration_version = str(calibration_version or "") self._metadata = dict(metadata) self._events = [] self._t0 = None self._last_t = None self._consecutive_errors = 0 self._started_at = datetime.now(timezone.utc).isoformat(timespec="seconds") - self._writing = True + self._report = None + self._preroll_frames = 0 + self._preroll_segment = None + self._writer_problems = [] + # Stale bookkeeping from a previous run must not leak into this one. + self._gaps.clear() + self._row_backlog.clear() + self._queued_bytes = 0 + # Accept immediately: see ``_accepts_packets``. + self._accepting = True + self._preroll = list(preroll or ()) self._stop.clear() self._drain_queue() self._thread = threading.Thread( @@ -231,17 +350,25 @@ class ExperimentRecorder: daemon=True, ) self._thread.start() + # The writer owns the transition to ``recording``: it is the one that + # knows the files opened successfully. self._write_session_json(final=False) logger.info("Recording experiment to %s", directory) return directory def stop(self, *, wait: bool = True, timeout: float = 20.0) -> RecordReport: - """Flush and close. Returns what actually happened.""" + """Flush and close. Returns what actually happened. + + A writer that does not finish inside ``timeout`` is **not** reported as + a success: the report says ``completed=False``, the thread reference is + kept (so a later call can observe it) and ``ended_at`` is not written. + """ with self._lock: - was_writing = self._writing - self._writing = False - if not was_writing: - return RecordReport(self._directory, self.stats()) + state = self._state + if state in TERMINAL_STATES and self._report is not None: + return self._report + self._state = STATE_STOPPING + self._accepting = False self._stop.set() try: self._queue.put_nowait(("stop", None)) @@ -250,17 +377,59 @@ class ExperimentRecorder: thread = self._thread if wait and thread is not None and thread.is_alive(): thread.join(timeout=timeout) + if thread is not None and thread.is_alive(): + # Still writing: do not claim the experiment was recorded. + report = RecordReport( + directory=self._directory, + stats=self.stats(), + problems=[ + "the recording writer did not stop within " + f"{timeout:g}s; the files are not closed yet" + ], + completed=False, + ) + self._report = report + return report self._thread = None - report = self._finalise() - return report + return self._report or self._finalise() - def log_event(self, kind: str, detail: str = "") -> None: - """Record a timestamped event (zeroing, calibration applied, disconnect…).""" - if not self.is_recording: + def set_calibration_version(self, version: str) -> None: + """Record which calibration is in effect from now on. + + A session can outlive the coefficients it started with, so the version is + per packet, not per session: the acquisition thread stamps every packet + with the calibration that was loaded when it was acquired. + """ + with self._lock: + self._calibration_version = str(version or "") + if self.is_recording: + self.log_event("calibration_version", self._calibration_version) + + def log_event( + self, + kind: str, + detail: str = "", + *, + mono_t: Optional[float] = None, + frame_id: int = -1, + wall_at: Optional[str] = None, + ) -> None: + """Record an event, timestamped when it actually happened. + + The old version re-derived the relative time when the writer dequeued + the event, so a congested queue shifted operator actions in the record. + The caller now supplies the moment of occurrence. + """ + # Events are accepted from the moment ``start`` returns, like packets: the + # writer thread may still be opening the files, and dropping "recording + # started" because of that would lose the first event of every session. + if not self._accepts_packets(): return event = { - "at": datetime.now(timezone.utc).isoformat(timespec="milliseconds"), - "t_rel_s": None, + "at": wall_at + or datetime.now(timezone.utc).isoformat(timespec="milliseconds"), + "mono_t": None if mono_t is None else round(float(mono_t), 6), + "frame_id": int(frame_id), "kind": str(kind), "detail": str(detail or ""), } @@ -271,33 +440,79 @@ class ExperimentRecorder: # ----------------------------------------------------------- sink interface def on_packet(self, packet) -> None: - """Called on the acquisition thread: enqueue, never do I/O here.""" - if not self.is_recording: + """Called on the acquisition thread: enqueue, never do I/O here. + + A data row is small and a frame is large, so video is what gives way. + When the byte budget is exhausted the **frame** is shed, and when even + the item queue is full the frame is shed too and the row is parked in a + small backlog. Losing the row because the video could not keep up is what + made a video overflow look like a video-only loss while CSV rows quietly + disappeared. + """ + if not self._accepts_packets(): + with self._lock: + if self._state == STATE_FAILED: + self._stats.rejected_after_failure += 1 return - timestamp = float(packet.timestamp) - if self._t0 is None: - self._t0 = timestamp - with self._lock: - self._last_t = timestamp - row = ( - int(getattr(packet, "frame_id", 0)), - timestamp, - float(packet.normal_force_raw), - float(packet.shear_x_raw), - float(packet.shear_y_raw), - float(packet.normal_force), - float(packet.shear_x), - float(packet.shear_y), - 1 if bool(getattr(packet, "valid", True)) else 0, + sample = to_record_sample( + packet, calibration_version=self._calibration_version ) + if self._t0 is None: + self._t0 = sample.mono_t + with self._lock: + self._last_t = sample.mono_t + frame = None if self.config.video or self.config.lossless_frames: - frame = self._capture_frame(packet) + candidate = self._capture_frame(packet) + size = 0 if candidate is None else int(candidate.nbytes) + with self._lock: + affordable = (self._queued_bytes + size) <= max( + 1024, int(self.config.max_queue_bytes) + ) + if candidate is None: + pass + elif affordable: + frame = candidate + with self._lock: + self._queued_bytes += size + else: + self._shed_video("video byte budget reached") try: - self._queue.put_nowait(("frame", (row, frame))) + self._queue.put_nowait(("frame", (sample, frame))) except queue.Full: - # Never block acquisition. The loss is counted, not hidden. - self._note_drop("video" if frame is not None else "data") + if frame is not None: + # The queue is full of video: the row is tiny and must survive. + frame = None + self._shed_video("record queue overflow") + with self._lock: + room = len(self._row_backlog) < self._row_backlog.maxlen + if room: + self._row_backlog.append(sample) + if not room: + # Even the row backlog is full: the hole is at least marked. + self._note_gap(sample) + + def _shed_video(self, reason: str) -> None: + with self._lock: + self._stats.dropped_video += 1 + total = self._stats.dropped_video + self._stats.dropped_data + # Rate-limited: a sustained overflow must not flood the log or the UI. + if total == 1 or total % ALARM_EVERY_N_DROPS == 0: + self._alarm( + f"{reason}; {total} frame(s) not recorded " + "(the data rows are still being written)" + ) + + def _note_gap(self, sample) -> None: + """Remember a frame whose row was lost, so the replay can show the hole.""" + with self._lock: + self._stats.dropped_data += 1 + if len(self._gaps) >= self._gaps.maxlen: + # The gap backlog is bounded, so a long overflow loses the marker. + self._stats.unrecorded_gaps += 1 + return + self._gaps.append((sample.frame_id, sample.mono_t, sample.wall_t)) @staticmethod def _capture_frame(packet): @@ -333,7 +548,6 @@ class ExperimentRecorder: # Rate-limited: a sustained overflow must not flood the log or the UI. if total == 1 or total % ALARM_EVERY_N_DROPS == 0: self._alarm(f"record queue overflow ({kind}); {total} item(s) not written") - def _alarm(self, message: str) -> None: logger.warning("Recording problem: %s", message) handler = self._on_alarm @@ -344,14 +558,38 @@ class ExperimentRecorder: logger.exception("Recording alarm handler failed") def _run(self) -> None: - writer = _SessionWriter( - directory=self._directory, - config=self.config, - session_id=self._session_id, - calibration_id=self._calibration_id, - ) + """The writer thread owns the terminal state and the final report. + + A failure anywhere — opening the files, writing a row, flushing, closing + — has to end in a state the caller can see. Opening the writer happens + inside the guard for that reason: a full disk used to kill this thread + before any statistics or alarm existed. + """ + writer: Optional[_SessionWriter] = None + failure = "" try: + writer = _SessionWriter( + directory=self._directory, + config=self.config, + session_id=self._session_id, + calibration_id=self._calibration_id, + preroll=self._preroll, + ) + # Written once the pre-roll file exists, so session.json never names + # a file that was not created. + with self._lock: + self._preroll_frames = int(writer.preroll_frames) + self._preroll_segment = writer.preroll_segment + self._preroll = [] + self._write_session_json(final=False) + with self._lock: + self._state = STATE_RECORDING + self._accepting = True while True: + # Parked rows and gaps first: a stop request must not strand the + # data this session already accepted. + self._drain_rows(writer) + self._drain_gaps(writer) if self._stop.is_set() and self._queue.empty(): break try: @@ -364,32 +602,51 @@ class ExperimentRecorder: if kind == "event": self._handle_event(writer, payload) continue - row, frame = payload - self._handle_frame(writer, row, frame) + sample, frame = payload + self._handle_frame(writer, sample, frame) writer.maybe_flush() except Exception as exc: # noqa: BLE001 - reported through stats logger.exception("Experiment writer stopped") + failure = f"{type(exc).__name__}: {exc}" with self._lock: self._stats.failed = True - self._stats.last_error = f"{type(exc).__name__}: {exc}" + self._stats.last_error = failure self._alarm(f"recording stopped: {exc}") finally: - problems = writer.close() with self._lock: - self._stats.write_errors += writer.write_errors - self._stats.segments = writer.segments - self._stats.video_frames = writer.video_frames - self._stats.bytes_written = writer.bytes_written - if problems: + self._accepting = False + problems: list[str] = [] + if writer is not None: + # A packet accepted just before the stop flag may have left its + # row or gap behind; write those before the file is closed. + self._drain_rows(writer) + self._drain_gaps(writer) + try: + problems = writer.close() + except Exception as exc: # noqa: BLE001 + problems = [f"close failed: {type(exc).__name__}: {exc}"] + with self._lock: + self._stats.write_errors += writer.write_errors + self._stats.segments = writer.segments + self._stats.video_frames = writer.video_frames + self._stats.bytes_written = writer.bytes_written + with self._lock: + if problems or failure: self._stats.failed = True if not self._stats.last_error: - self._stats.last_error = problems[0] + self._stats.last_error = problems[0] if problems else failure + # The writer's own findings (a flush that failed, a pre-roll that + # could not be stored) belong in the report: an operator reading + # only the terminal report must not have to grep the log. + self._writer_problems = list(problems) + failed = self._stats.failed + self._state = STATE_FAILED if failed else STATE_STOPPED for problem in problems: self._alarm(problem) + # Only now, with the files closed, is the report trustworthy. + self._report = self._finalise(completed=True) def _handle_event(self, writer, event: dict) -> None: - if self._t0 is not None: - event["t_rel_s"] = round(float(time.time()) - self._t0, 6) with self._lock: self._events.append(event) try: @@ -397,9 +654,12 @@ class ExperimentRecorder: except Exception as exc: # noqa: BLE001 self._record_error(exc) - def _handle_frame(self, writer, row, frame) -> None: + def _handle_frame(self, writer, sample, frame) -> None: try: - writer.write_sample(row, frame) + if frame is not None: + with self._lock: + self._queued_bytes = max(0, self._queued_bytes - int(frame.nbytes)) + writer.write_sample(sample, frame) with self._lock: self._stats.frames += 1 self._stats.video_frames = writer.video_frames @@ -408,6 +668,30 @@ class ExperimentRecorder: except Exception as exc: # noqa: BLE001 self._record_error(exc) + def _drain_rows(self, writer) -> None: + """Write rows parked while the item queue was full of video.""" + while True: + with self._lock: + if not self._row_backlog: + return + sample = self._row_backlog.popleft() + self._handle_frame(writer, sample, None) + + def _drain_gaps(self, writer) -> None: + """Write the frame-index rows for frames whose data row was lost.""" + while True: + with self._lock: + if not self._gaps: + return + frame_id, mono_t, wall_t = self._gaps.popleft() + try: + writer.write_gap(frame_id, mono_t, wall_t) + with self._lock: + self._stats.dropped_gaps += 1 + except Exception as exc: # noqa: BLE001 + self._record_error(exc) + return + def _record_error(self, exc: Exception) -> None: with self._lock: self._stats.write_errors += 1 @@ -416,12 +700,15 @@ class ExperimentRecorder: fatal = self._consecutive_errors >= MAX_CONSECUTIVE_ERRORS if fatal: self._stats.failed = True + # Stop accepting immediately: consuming packets that cannot be + # written only hides the loss behind a "recording" state. + self._accepting = False + self._state = STATE_FAILED self._alarm(f"recording write failed: {exc}") if fatal: - # Stop pretending we are recording rather than losing data quietly. self._stop.set() - def _finalise(self) -> RecordReport: + def _finalise(self, *, completed: bool = True) -> RecordReport: with self._lock: if self._t0 is not None and self._last_t is not None: self._stats.duration_s = round(float(self._last_t - self._t0), 6) @@ -432,10 +719,29 @@ class ExperimentRecorder: f"{stats.dropped_data} data and {stats.dropped_video} video frame(s) " "were dropped by the bounded queue" ) + if stats.dropped_gaps: + problems.append( + f"{stats.dropped_gaps} frame(s) were recorded as gaps (no data row)" + ) + if stats.unrecorded_gaps: + problems.append( + f"{stats.unrecorded_gaps} lost frame(s) could not be marked: " + "the gap backlog was full" + ) if stats.write_errors: problems.append(f"{stats.write_errors} write error(s); last: {stats.last_error}") - self._write_session_json(final=True) - return RecordReport(directory=self._directory, stats=stats, problems=problems) + with self._lock: + for problem in self._writer_problems: + if problem not in problems: + problems.append(problem) + # ``ended_at`` is only written once the files are actually closed. + self._write_session_json(final=completed) + return RecordReport( + directory=self._directory, + stats=stats, + problems=problems, + completed=completed, + ) def _write_session_json(self, *, final: bool) -> None: directory = self._directory @@ -445,12 +751,17 @@ class ExperimentRecorder: target = directory / "session.json" try: # Written at start too, so a crash still leaves identifiable metadata. - temporary = target.with_suffix(".json.tmp") - temporary.write_text( - json.dumps(payload, indent=2, ensure_ascii=False) + "\n", - encoding="utf-8", - ) - temporary.replace(target) + # The start-time write and the writer thread's write can overlap, so + # the temp file is touched under a lock: two threads renaming the same + # path is a PermissionError on Windows, which then looked like a + # recording failure. + with self._json_lock: + temporary = target.with_suffix(".json.tmp") + temporary.write_text( + json.dumps(payload, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + temporary.replace(target) except Exception as exc: # noqa: BLE001 with self._lock: self._stats.write_errors += 1 @@ -472,6 +783,10 @@ class ExperimentRecorder: "video": self.config.as_dict(), "events": events, "stats": stats, + # The replay needs these to know whether the algorithm state at the + # start of the recording can be rebuilt at all. + "preroll_frames": int(self._preroll_frames), + "preroll_segment": self._preroll_segment, "files": { "forces": "forces.csv", "frames": "frames.csv", @@ -487,7 +802,9 @@ class ExperimentRecorder: class _SessionWriter: """Owns the file handles. Only ever touched by the writer thread.""" - def __init__(self, *, directory, config, session_id, calibration_id) -> None: + def __init__( + self, *, directory, config, session_id, calibration_id, preroll=() + ) -> None: self.directory = Path(directory) self.config = config self.session_id = session_id @@ -496,7 +813,11 @@ class _SessionWriter: self.segments = 0 self.video_frames = 0 self.png_frames = 0 + self.gap_rows = 0 self.bytes_written = 0 + #: Frames captured before recording started, written as their own file. + self.preroll_frames = 0 + self.preroll_segment: Optional[str] = None self._last_flush = time.perf_counter() self._t0: Optional[float] = None self._segment: Optional[cv2.VideoWriter] = None @@ -513,6 +834,51 @@ class _SessionWriter: if self.config.lossless_frames: self._png_dir = self.directory / "frames" self._png_dir.mkdir(parents=True, exist_ok=True) + self._write_preroll(preroll) + + def _write_preroll(self, preroll) -> None: + """Write the frames captured just before recording started. + + They are not part of the timeline (no frame ids, no CSV rows); they are + the algorithm's history, and the replay feeds them to the SDK before the + first recorded frame so that frame can be reproduced at all. + """ + frames = [frame for frame in (preroll or ()) if frame is not None] + if not frames or not self.config.video: + if frames and not self.config.video: + self._problems.append( + "pre-roll frames were captured but the data-only mode has no " + "video to store them in; depth replay of the first frames " + "cannot reproduce the algorithm state" + ) + return + budget = max(0, int(self.config.preroll_bytes)) + kept: list = [] + used = 0 + # Keep the frames closest to the start of the recording. + for frame in reversed(frames): + size = int(np.asarray(frame).nbytes) + if kept and used + size > budget: + break + kept.append(frame) + used += size + kept.reverse() + self._frame_shape = (int(kept[0].shape[1]), int(kept[0].shape[0])) + name = "preroll_0001.mp4" + path = self.directory / name + fourcc = cv2.VideoWriter_fourcc(*self.config.codec) + writer = cv2.VideoWriter( + str(path), fourcc, max(1.0, float(self.config.fps)), self._frame_shape + ) + if not writer.isOpened(): + raise OSError(f"could not open video writer for {name}") + try: + for frame in kept: + writer.write(frame) + self.preroll_frames += 1 + finally: + writer.release() + self.preroll_segment = name if self.preroll_frames else None def _open_csv(self, name: str, header): handle = (self.directory / name).open("w", newline="", encoding="utf-8") @@ -521,31 +887,33 @@ class _SessionWriter: return (handle, writer) # ------------------------------------------------------------------ writing - def write_sample(self, row, frame) -> None: + def write_sample(self, sample: RecordSample, frame) -> None: """Write one acquired frame: a forces row, the frame index, and video. ``forces.csv`` and ``frames.csv`` share the packet's ``frame_id`` and the same ``t_rel_s`` origin, which is what lets video and data be aligned afterwards even when frames were dropped or split into segments. """ - frame_id, timestamp, fn_r, fx_r, fy_r, fn_n, fx_n, fy_n, valid = row if self._t0 is None: - self._t0 = timestamp - t_rel = round(timestamp - self._t0, 6) + self._t0 = sample.mono_t + t_rel = round(sample.mono_t - self._t0, 6) + acquired = datetime.fromtimestamp( + sample.wall_t or time.time(), timezone.utc + ).isoformat(timespec="milliseconds") self._forces[1].writerow( ( self.session_id, - frame_id, + sample.frame_id, repr(t_rel), - format_value(fn_r), - format_value(fx_r), - format_value(fy_r), - format_value(fn_n), - format_value(fx_n), - format_value(fy_n), - valid, - self.calibration_id, + format_value(sample.fn_raw), + format_value(sample.fx_raw), + format_value(sample.fy_raw), + format_value(sample.fn_n), + format_value(sample.fx_n), + format_value(sample.fy_n), + 1 if sample.valid else 0, + sample.calibration_version or self.calibration_id, ) ) @@ -560,11 +928,9 @@ class _SessionWriter: self._frames[1].writerow( ( - frame_id, + sample.frame_id, repr(t_rel), - datetime.fromtimestamp(timestamp, timezone.utc).isoformat( - timespec="milliseconds" - ), + acquired, segment, segment_frame, 1 if segment else 0, @@ -572,6 +938,24 @@ class _SessionWriter: ) ) + def write_gap(self, frame_id: int, mono_t: float, wall_t: float) -> None: + """Record a frame whose data row never reached the writer. + + The row is gone, but the *hole* is not: the replay uses this to say + "this frame was not recorded" instead of silently showing its + predecessor as if it belonged to the cursor position. + """ + if self._t0 is None: + self._t0 = mono_t + t_rel = round(mono_t - self._t0, 6) + acquired = datetime.fromtimestamp( + wall_t or time.time(), timezone.utc + ).isoformat(timespec="milliseconds") + self._frames[1].writerow( + (frame_id, repr(t_rel), acquired, 0, 0, 0, 1) + ) + self.gap_rows += 1 + def _write_video_frame(self, frame) -> tuple[int, int]: if self._frame_shape is None: self._frame_shape = (int(frame.shape[1]), int(frame.shape[0])) @@ -591,8 +975,18 @@ class _SessionWriter: raise OSError(f"could not write {target.name}") def write_event(self, event: dict) -> None: + mono_t = event.get("mono_t") + t_rel = "" + if mono_t is not None and self._t0 is not None: + t_rel = repr(round(float(mono_t) - self._t0, 6)) self._events[1].writerow( - (event.get("at", ""), event.get("t_rel_s", ""), event["kind"], event["detail"]) + ( + event.get("at", ""), + t_rel, + event.get("frame_id", -1), + event["kind"], + event["detail"], + ) ) # ------------------------------------------------------------------ segments diff --git a/shared/src/orisys_app_shared/record/playback.py b/shared/src/orisys_app_shared/record/playback.py index 0882981..9cfd224 100644 --- a/shared/src/orisys_app_shared/record/playback.py +++ b/shared/src/orisys_app_shared/record/playback.py @@ -49,6 +49,8 @@ class RecordingFrame: segment: int segment_frame: int in_video: bool + #: True when the frame's data row was lost and only this index row exists. + dropped: bool = False @dataclass @@ -62,6 +64,7 @@ class RecordingInfo: valid: np.ndarray frame_ids: np.ndarray dropped_rows: int = 0 + frames_dropped: int = 0 @property def duration_s(self) -> float: @@ -74,6 +77,17 @@ class RecordingInfo: return self.newtons return self.raw + def plottable(self, unit: str) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """``(times, values, mask)`` — the same validity rule the live charts use. + + The replay used to filter on finiteness alone, so a stale value left by a + failed compute step could reappear here after another view had already + excluded it. + """ + values = self.value_for_unit(unit) + mask = self.valid & np.isfinite(values).all(axis=1) + return self.times[mask], values[mask], mask + class RecordingReader: """Open one experiment directory and serve replay queries from it.""" @@ -83,6 +97,7 @@ class RecordingReader: if not self.directory.is_dir(): raise FileNotFoundError(f"Not a recording directory: {self.directory}") self._info: Optional[RecordingInfo] = None + self._events: Optional[list[dict]] = None # ------------------------------------------------------------------ loading def info(self) -> RecordingInfo: @@ -113,6 +128,7 @@ class RecordingReader: valid=valid, frame_ids=frame_ids, dropped_rows=dropped, + frames_dropped=sum(1 for frame in frames if frame.dropped), ) def _load_frames(self) -> list[RecordingFrame]: @@ -131,6 +147,7 @@ class RecordingReader: segment=int(row.get("segment") or 0), segment_frame=int(row.get("segment_frame") or 0), in_video=row.get("in_video") == "1", + dropped=(row.get("dropped") or "0").strip() == "1", ) ) except (KeyError, TypeError, ValueError): @@ -203,6 +220,72 @@ class RecordingReader: def segments(self) -> list[Path]: return sorted(self.directory.glob("raw_*.mp4")) + # ------------------------------------------------------------- replay plan + def events(self) -> list[dict]: + """``events.csv`` rows, cached. Operator actions, not measurements.""" + if self._events is None: + path = self.directory / "events.csv" + rows: list[dict] = [] + if path.is_file(): + with path.open(encoding="utf-8", newline="") as handle: + rows = [dict(row) for row in csv.DictReader(handle)] + self._events = rows + return self._events + + def event_frames(self, kind: str) -> list[int]: + """Frame ids of every event of ``kind`` that names a position.""" + found: list[int] = [] + for row in self.events(): + if str(row.get("kind") or "") != kind: + continue + try: + frame_id = int(row.get("frame_id") or -1) + except (TypeError, ValueError): + continue + if frame_id >= 0: + found.append(frame_id) + return found + + def last_reset_frame(self, at_or_before: int) -> Optional[int]: + """The most recent reset at or before a frame — a tracker state origin. + + ``Sensor.reset`` clears the optical-flow tracker and the force filters, so + the state at a later frame is fully determined by the frames since that + reset and does not need any earlier history. Both spellings are accepted: + the app has logged ``zeroing`` since before the replay existed. + """ + candidates = [ + frame_id + for kind in ("zeroing", "reset") + for frame_id in self.event_frames(kind) + if frame_id <= int(at_or_before) + ] + return max(candidates) if candidates else None + + def frame_of(self, frame_id: int) -> Optional[RecordingFrame]: + for frame in self.info().frames: + if frame.frame_id == int(frame_id): + return frame + return None + + def segment_start_frame(self, segment: int) -> Optional[RecordingFrame]: + """The first indexed frame that lives in ``segment``.""" + rows = [frame for frame in self.info().frames if frame.segment == int(segment)] + if not rows: + return None + return min(rows, key=lambda frame: frame.segment_frame) + + def preroll_frames(self) -> int: + return int(self.info().session.get("preroll_frames") or 0) + + def preroll_segment(self) -> Optional[Path]: + """The pre-roll video recorded just before this recording started.""" + name = str(self.info().session.get("preroll_segment") or "") + if not name: + return None + path = self.directory / name + return path if path.is_file() else None + def video_frame_at(self, t_rel_s: float): """Decode the video frame for a replay position. @@ -287,6 +370,8 @@ class RecordingReader: warnings.append("depth_recomputed") if self.info().dropped_rows: warnings.append("rows_skipped") + if self.info().frames_dropped: + warnings.append("frames_dropped") return warnings def iter_forces(self, batch: int = 4096) -> Iterator[tuple[np.ndarray, np.ndarray]]: diff --git a/shared/src/orisys_app_shared/record/sample.py b/shared/src/orisys_app_shared/record/sample.py new file mode 100644 index 0000000..26e63f8 --- /dev/null +++ b/shared/src/orisys_app_shared/record/sample.py @@ -0,0 +1,216 @@ +"""The one conversion from an acquired packet to a recorded sample. + +Everything that records or displays force data goes through +:func:`to_record_sample`. There used to be two independent conversions — one in +``ExperimentRecorder.on_packet`` (for the CSV) and one in +``StudioMainWindow._on_packet_recorded`` (for the in-memory session store) — and +they disagreed with each other and with the truth: + +* an uncalibrated sensor reports its raw counts in the *ordinary* force fields, + so those numbers were written into the ``FN_N``/``FX_N``/``FY_N`` columns; +* when the compute chain fails, the SDK's ``info.FNORMAL`` still returns the + value it last computed, so a failed frame recorded a **stale finite number** + that was indistinguishable from a fresh measurement. + +The rules are therefore: + +1. A sample whose compute chain failed is recorded with **all six force values + unavailable**. The value may belong to an earlier frame; nothing can prove + otherwise, so it is not recorded as a measurement. +2. A newton value is recorded **only for an axis that actually has a calibration + curve loaded**. An uncalibrated axis leaves its column empty. +3. A raw value is recorded only when the sensor exposes that raw channel. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Iterable + +AXES: tuple[str, ...] = ("fn", "fx", "fy") + +#: Reason recorded when a stale force value had to be discarded. +STALE_REASON = "stale forces: the compute chain failed for this frame" + + +def _finite(value: object) -> float: + """Coerce to a float, mapping anything unusable to NaN.""" + try: + number = float(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return float("nan") + return number if math.isfinite(number) else float("nan") + + +@dataclass(frozen=True) +class RecordSample: + """One acquired step, as it is recorded.""" + + frame_id: int + #: Monotonic, session-relative acquisition time. This is the ordering key + #: for the session store, the CSV and the frame index; a wall clock that + #: steps backwards must not be able to reorder history. + mono_t: float + #: Wall clock at hand-off, kept for metadata only. + wall_t: float + valid: bool + invalid_reason: str + fn_raw: float = float("nan") + fx_raw: float = float("nan") + fy_raw: float = float("nan") + fn_n: float = float("nan") + fx_n: float = float("nan") + fy_n: float = float("nan") + calibrated_axes: tuple[str, ...] = () + calibration_version: str = "" + + # ------------------------------------------------------------- accessors + def raw_triplet(self) -> tuple[float, float, float]: + return (self.fn_raw, self.fx_raw, self.fy_raw) + + def newton_triplet(self) -> tuple[float, float, float]: + return (self.fn_n, self.fx_n, self.fy_n) + + def raw_for(self, axis: str) -> float: + return {"fn": self.fn_raw, "fx": self.fx_raw, "fy": self.fy_raw}.get( + axis, float("nan") + ) + + def newton_for(self, axis: str) -> float: + return {"fn": self.fn_n, "fx": self.fx_n, "fy": self.fy_n}.get( + axis, float("nan") + ) + + def has_newton(self, axis: str) -> bool: + return math.isfinite(self.newton_for(axis)) + + def has_raw(self, axis: str) -> bool: + return math.isfinite(self.raw_for(axis)) + + def value_for_unit(self, axis: str, unit: str) -> float: + if str(unit).upper().startswith("N"): + return self.newton_for(axis) + return self.raw_for(axis) + + def as_row(self) -> tuple: + """Positional form used by the CSV writer and the session store.""" + return ( + self.frame_id, + self.mono_t, + self.fn_raw, + self.fx_raw, + self.fy_raw, + self.fn_n, + self.fx_n, + self.fy_n, + 1 if self.valid else 0, + ) + + +def calibrated_axes_of(packet) -> tuple[str, ...]: + """Which axes have a calibration curve, per axis rather than per sensor.""" + declared = tuple(str(axis) for axis in (getattr(packet, "calibrated_axes", ()) or ())) + if declared: + return tuple(axis for axis in AXES if axis in declared) + # Backwards compatible fallback for packets that only carry the sensor-level + # flag: calibrated means all three, uncalibrated means none. + if bool(getattr(packet, "force_calibrated", False)): + return AXES + return () + + +def to_record_sample(packet, *, calibration_version: str = "") -> RecordSample: + """Convert one packet into the single canonical recorded form.""" + frame_id = int(getattr(packet, "frame_id", 0)) + # ``mono_t`` is the acquisition clock and is authoritative when present; a + # packet that carries only a wall clock (a test double, or an older + # producer) still has to produce a *moving* time, never a constant one. + mono_t = _finite(getattr(packet, "mono_t", None)) + wall_t = _finite(getattr(packet, "wall_t", None)) + if not math.isfinite(mono_t): + mono_t = wall_t + if not math.isfinite(mono_t): + mono_t = _finite(getattr(packet, "timestamp", 0.0)) + if not math.isfinite(wall_t): + wall_t = _finite(getattr(packet, "timestamp", 0.0)) + if not math.isfinite(wall_t): + wall_t = mono_t + + valid = bool(getattr(packet, "valid", True)) + reason = str(getattr(packet, "invalid_reason", "") or "") + stale = bool(getattr(packet, "stale_forces", False)) + # The packet's own version wins: it was stamped by the acquisition thread at + # the moment of acquisition, so it is the only value that stays true when a + # calibration is applied in the middle of a session. + version = str( + getattr(packet, "calibration_version", "") or calibration_version or "" + ) + axes = calibrated_axes_of(packet) + + if not valid or stale: + if stale and not reason: + reason = STALE_REASON + # Rule 1: a frame with no trustworthy readout records nothing numeric. + return RecordSample( + frame_id=frame_id, + mono_t=mono_t, + wall_t=wall_t, + valid=False, + invalid_reason=reason, + calibrated_axes=axes, + calibration_version=version, + ) + + raw_available = bool(getattr(packet, "raw_available", True)) + raw = ( + _finite(getattr(packet, "normal_force_raw", None)), + _finite(getattr(packet, "shear_x_raw", None)), + _finite(getattr(packet, "shear_y_raw", None)), + ) + if not raw_available: + raw = (float("nan"), float("nan"), float("nan")) + + newton = ( + _finite(getattr(packet, "normal_force", None)), + _finite(getattr(packet, "shear_x", None)), + _finite(getattr(packet, "shear_y", None)), + ) + # Rule 2: an axis without a calibration curve has no newton value at all. + newton = tuple( + value if axis in axes else float("nan") + for axis, value in zip(AXES, newton) + ) + + return RecordSample( + frame_id=frame_id, + mono_t=mono_t, + wall_t=wall_t, + valid=True, + invalid_reason="", + fn_raw=raw[0], + fx_raw=raw[1], + fy_raw=raw[2], + fn_n=newton[0], + fx_n=newton[1], + fy_n=newton[2], + calibrated_axes=axes, + calibration_version=version, + ) + + +def axes_without_newton(sample: RecordSample) -> tuple[str, ...]: + """Axes that produced no newton value — used by the UI to explain blanks.""" + return tuple(axis for axis in AXES if not sample.has_newton(axis)) + + +def plottable_mask(values: Iterable[float], valid: Iterable[bool]) -> list[bool]: + """Shared visibility rule: a point is drawn only when valid *and* finite. + + Every drawing path uses this so that a stale-but-finite value cannot reappear + in one view after being filtered out of another. + """ + return [ + bool(is_valid) and math.isfinite(float(value)) + for value, is_valid in zip(values, valid) + ] diff --git a/shared/tests/test_calibration_model.py b/shared/tests/test_calibration_model.py new file mode 100644 index 0000000..d411fc4 --- /dev/null +++ b/shared/tests/test_calibration_model.py @@ -0,0 +1,153 @@ +"""Calibration diagnostics must describe the model that is exported. + +The review found the panel reporting a private least-squares fit while the +exported file carried a piecewise-linear curve, validation points leaking into +the build, and repeated loads being overwritten so only the last one reached the +model. +""" + +from __future__ import annotations + +import pytest + +from orisys_app_shared.force.calibration_io import ( + CalSample, + CalSessionStore, + build_export_model, + build_force_calibration_v2, + merge_samples, + predict_channel, +) +from orisys_app_shared.force.calibration_recorder import CalibrationRecorder +from orisys_app_shared.force.session import SessionSample + + +def _point(recorder, load: float, raw: float, *, channel: str = "fn"): + recorder.start(channel=channel, reference_load_n=load, duration_s=0.05) + for index in range(8): + recorder.feed( + SessionSample( + index, + index * 0.01, + fn_raw=raw, + fx_raw=raw, + fy_raw=raw, + fn_n=raw / 10.0, + fx_n=raw / 10.0, + fy_n=raw / 10.0, + ) + ) + return recorder.finish_now() + + +def test_the_diagnosed_model_is_the_exported_model(): + recorder = CalibrationRecorder() + for load, raw in ((1.0, 100.0), (2.0, 400.0), (3.0, 300.6), (5.0, 500.9)): + _point(recorder, load, raw) + + fit = recorder.fit("fn") + assert fit.usable + assert fit.method == "piecewise_linear" + assert fit.breakpoints == 4 + # A piecewise curve passes through the points it was built from. + assert fit.max_residual_n == pytest.approx(0.0, abs=1e-9) + + +def test_validation_points_are_held_out_and_measured(): + recorder = CalibrationRecorder() + points = [ + _point(recorder, 1.0, 100.0), + _point(recorder, 2.0, 400.0), + _point(recorder, 3.0, 300.6), + _point(recorder, 5.0, 500.9), + ] + recorder.set_validation(points[1], True) + fit = recorder.fit("fn") + assert fit.n_points == 3, "a validation point entered the build" + assert fit.validation_points == 1 + assert fit.validation_error_n > 0.05, "the off-trend point was not measured" + assert fit.breakpoints == 3 + assert points[1].residual_n != points[1].residual_n # NaN: not in the fit + + +def test_excluded_points_leave_the_build(): + recorder = CalibrationRecorder() + points = [_point(recorder, 1.0, 100.0), _point(recorder, 5.0, 500.0)] + outlier = _point(recorder, 3.0, 900.0) + recorder.set_excluded(outlier, True) + + fit = recorder.fit("fn") + assert fit.n_points == 2 + store = CalSessionStore() + for point in recorder.all_points(): + if point.excluded or point.is_validation: + continue + store.aggregate( + point.channel, + CalSample( + weight_n=float(point.reference_load_n), + raw_mean=float(point.loaded_stats().mean), + n=int(point.loaded_stats().n), + ), + ) + doc, _warnings = build_force_calibration_v2(store) + points_in_doc = doc["channels"]["normal"]["points"] + assert [point[0] for point in points_in_doc] == [100.0, 500.0], ( + "the excluded point reached the exported coefficients" + ) + assert points # silence linters about the unused list + + +def test_validation_points_never_reach_the_exported_coefficients(): + recorder = CalibrationRecorder() + _point(recorder, 1.0, 100.0) + _point(recorder, 5.0, 500.0) + held_out = _point(recorder, 3.0, 300.0) + recorder.set_validation(held_out, True) + + store = CalSessionStore() + for point in recorder.all_points(): + if point.excluded or point.is_validation: + continue + store.aggregate( + point.channel, + CalSample( + weight_n=float(point.reference_load_n), + raw_mean=float(point.loaded_stats().mean), + n=int(point.loaded_stats().n), + ), + ) + doc, _ = build_force_calibration_v2(store) + raws = [point[0] for point in doc["channels"]["normal"]["points"]] + assert 300.0 not in raws, "a validation point was used to build the model" + + +def test_repeated_loads_are_merged_not_overwritten(): + """Two samples of one load carry scatter the model should see.""" + first = CalSample(weight_n=2.0, raw_mean=100.0, n=10, std=1.0) + second = CalSample(weight_n=2.0, raw_mean=110.0, n=30, std=2.0) + merged = merge_samples(first, second) + assert merged.n == 40 + assert merged.raw_mean == pytest.approx((100.0 * 10 + 110.0 * 30) / 40) + assert merged.std > 2.0, "the scatter between the two samples was lost" + + store = CalSessionStore() + store.aggregate("fn", first) + store.aggregate("fn", second) + assert len(store.channel("fn")) == 1 + assert store.channel("fn")[0].n == 40 + + +def test_predict_channel_applies_the_exported_curve(): + store = CalSessionStore() + store.aggregate("fn", CalSample(weight_n=0.0, raw_mean=0.0, n=10)) + store.aggregate("fn", CalSample(weight_n=10.0, raw_mean=100.0, n=10)) + model = build_export_model({key: store.channel(key) for key in ("fn", "fx", "fy")}) + assert predict_channel(model, "fn", 50.0) == pytest.approx(5.0, abs=1e-6) + + +def test_a_channel_with_one_point_is_not_exported(): + store = CalSessionStore() + store.aggregate("fn", CalSample(weight_n=1.0, raw_mean=10.0, n=10)) + with pytest.raises(ValueError): + build_force_calibration_v2(store) diff --git a/shared/tests/test_calibration_record.py b/shared/tests/test_calibration_record.py new file mode 100644 index 0000000..2841350 --- /dev/null +++ b/shared/tests/test_calibration_record.py @@ -0,0 +1,143 @@ +"""A calibration session must survive as evidence, not just as coefficients. + +The review's P1-8: the recorder threw away the raw sample buffer and kept only +three-axis statistics plus a 400-point trace, so the full series behind a point +could never be re-examined. +""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pytest + +from orisys_app_shared.force.calibration_recorder import CalibrationRecorder +from orisys_app_shared.force.calibration_session import ( + SAMPLE_FIELDS, + build_session_document, + read_calibration_session, + write_calibration_session, +) +from orisys_app_shared.force.session import SessionSample + +SCRATCH = Path(__file__).resolve().parent / "_scratch" + + +def _root() -> Path: + shutil.rmtree(SCRATCH, ignore_errors=True) + SCRATCH.mkdir(parents=True, exist_ok=True) + return SCRATCH + + +def _recorded_point(index_offset: float = 0.0): + recorder = CalibrationRecorder() + recorder.start(channel="fn", reference_load_n=2.0, duration_s=0.05) + for index in range(6): + recorder.feed( + SessionSample( + index, + index * 1.0 / 30.0, + fn_raw=float(index) + index_offset, + fx_raw=1.0, + fy_raw=2.0, + fn_n=float(index) * 10.0, + fx_n=1.0, + fy_n=2.0, + valid=index != 5, + ) + ) + return recorder.finish_now() + + +def test_a_point_keeps_every_sample_including_invalid_ones(): + point = _recorded_point() + assert len(point.samples) == 6 + assert point.n_total == 6 + assert point.n_valid == 5 + # frame id, relative time, three raw axes, three newton axes, validity. + first = point.samples[0] + assert first[0] == 0, "the frame id of the sample was not kept" + assert len(first) == len(SAMPLE_FIELDS) + assert point.samples[-1][-1] is False + + +def test_the_serialized_point_carries_the_series(): + point = _recorded_point() + payload = point.as_dict() + assert payload["sample_fields"] == list(SAMPLE_FIELDS) + assert payload["n_samples"] == 6 + assert len(payload["samples"]) == 6 + assert payload["duration_s"] == pytest.approx(5.0 / 30.0, abs=1e-6) + + +def test_a_session_round_trips_through_disk(): + point = _recorded_point() + path = write_calibration_session( + _root() / "session.json", [point], meta={"sensor_type": "box", "sn": "SN1"} + ) + record = read_calibration_session(path) + assert record.ok + assert record.meta["sn"] == "SN1" + assert len(record.points) == 1 + restored = record.points[0] + assert restored.channel == point.channel + assert restored.reference_load_n == point.reference_load_n + assert restored.duration_s == pytest.approx(point.duration_s) + assert restored.n_total == point.n_total + assert restored.n_valid == point.n_valid + assert restored.raw["fn"] == point.raw["fn"] + assert len(restored.samples) == len(point.samples) + assert restored.samples[0][0] == 0 + + +def test_diagnostics_survive_a_round_trip(): + point = _recorded_point() + point.excluded = False + point.is_validation = True + point.note = "operator note" + point.direction = "unload" + path = write_calibration_session(_root() / "session.json", [point]) + restored = read_calibration_session(path).points[0] + assert restored.is_validation + assert not restored.excluded + assert restored.note == "operator note" + assert restored.direction == "unload" + + +def test_a_foreign_document_is_refused_rather_than_guessed(): + target = _root() / "not-a-session.json" + target.write_text(json.dumps({"hello": "world"}), encoding="utf-8") + with pytest.raises(ValueError): + read_calibration_session(target) + + +def test_a_newer_format_is_loaded_with_a_warning(): + """The layout may have changed; the operator is told instead of guessing.""" + document = build_session_document([_recorded_point()]) + document["format_version"] = "99.0" + path = _root() / "future.json" + path.write_text(json.dumps(document), encoding="utf-8") + record = read_calibration_session(path) + assert record.points, "a readable point should still be returned" + assert record.warnings and "99.0" in record.warnings[0] + + +def test_the_recorder_can_reopen_a_saved_session(): + points = [ + _recorded_point(), + _recorded_point(5.0), + _recorded_point(9.0), + ] + path = write_calibration_session(_root() / "session.json", points) + restored = read_calibration_session(path).points + + recorder = CalibrationRecorder() + assert recorder.load_points(restored, replace=True) == 3 + assert [item.channel for item in recorder.all_points()] == ["fn", "fn", "fn"] + # Diagnostics travel with the points. + recorder.set_validation(recorder.all_points()[1], True) + fit = recorder.fit("fn") + assert fit.validation_points == 1 + assert fit.n_points == 2 diff --git a/shared/tests/test_depth_replay.py b/shared/tests/test_depth_replay.py new file mode 100644 index 0000000..e94cae9 --- /dev/null +++ b/shared/tests/test_depth_replay.py @@ -0,0 +1,248 @@ +"""Offline depth replay: where to start, how far to step, which chain to run. + +P1-3 and P1-4: the pane kept showing the previous frame's depth after the cursor +moved, the recomputer re-read ``target`` frames on every call (so requesting +frames 2 and 3 landed on 5), it skipped the live processing chain, and it +reduced the recorded config path to a name that could load a different file. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import numpy as np +import pytest + +from orisys_app_shared.record.depth import DepthRecomputer + +SCRATCH = Path(__file__).resolve().parent / "_scratch" + + +def _root() -> Path: + shutil.rmtree(SCRATCH, ignore_errors=True) + SCRATCH.mkdir(parents=True, exist_ok=True) + return SCRATCH + + +class _CountingSensor: + """A sensor whose reported depth is the number of frames it has read.""" + + opened: list[dict] = [] + + def __init__(self, *args, **kwargs): + _CountingSensor.opened.append({"args": args, "kwargs": kwargs}) + self.n = 0 + self.chain_calls = 0 + self.info = SimpleNamespace(DEPTH=1) + + def get_img(self): + self.n += 1 + return np.ones((2, 2)) + + def compute_deformation(self, **_kwargs): + self.chain_calls += 1 + + def compute_contact(self): + self.chain_calls += 1 + + def read_info(self, _channel): + return np.array([self.n]) + + def disconnect(self): + return None + + +def _reader(session: dict, *, segments=("raw_0001.mp4",), frames=()): + directory = _root() + for name in segments: + (directory / name).write_bytes(b"") + return SimpleNamespace( + session=lambda: session, + segment_path=lambda segment: directory / f"raw_{segment:04d}.mp4", + segments=lambda: [directory / name for name in segments], + info=lambda: SimpleNamespace(frames=list(frames)), + frame_of=lambda frame_id: next( + (frame for frame in frames if frame.frame_id == frame_id), None + ), + frame_at=lambda t: None, + last_reset_frame=lambda frame_id: None, + segment_start_frame=lambda segment: next( + (frame for frame in frames if frame.segment == segment), None + ), + preroll_frames=lambda: 0, + preroll_segment=lambda: None, + directory=directory, + ) + + +def _session(config_ref: str) -> dict: + return {"device": {"config_ref": config_ref, "sensor_type": "box"}} + + +def _config_file() -> str: + """A config that exists, so ``preflight`` is not what a test measures.""" + path = _root() / "sensor.json" + path.write_text("{}", encoding="utf-8") + return str(path) + + +def _reader_with_config(frames): + """A reader whose recorded config really exists on disk. + + ``_reader`` recreates the scratch directory, so the config file is written + *after* the reader is built. + """ + directory = _root() + config = directory / "sensor.json" + reader = _reader(_session(str(config)), frames=frames) + config.write_text("{}", encoding="utf-8") + return reader + + +def test_stepping_continues_from_the_current_position(): + """The old code re-read ``target`` frames every call: 2 then 3 landed on 5.""" + _CountingSensor.opened.clear() + computer = DepthRecomputer(_reader(_session("/cfg/sensor.json"))) + with patch("orisys.Sensor", _CountingSensor): + first = computer._depth_from_segment(Path("raw_0001.mp4"), 2) + second = computer._depth_from_segment(Path("raw_0001.mp4"), 3) + assert first.tolist() == [2] + assert second.tolist() == [3] + # One sensor for the whole replay: reopening would reset the algorithm state. + assert len(_CountingSensor.opened) == 1 + computer.close() + + +def test_stepping_backwards_reopens_the_segment(): + _CountingSensor.opened.clear() + computer = DepthRecomputer(_reader(_session("/cfg/sensor.json"))) + with patch("orisys.Sensor", _CountingSensor): + assert computer._depth_from_segment(Path("raw_0001.mp4"), 5).tolist() == [5] + assert computer._depth_from_segment(Path("raw_0001.mp4"), 2).tolist() == [2] + assert len(_CountingSensor.opened) == 2, "seeking backwards needs a fresh source" + computer.close() + + +def test_the_live_processing_chain_runs_for_every_replayed_frame(): + """Without it the replay is a different algorithm than the recording.""" + _CountingSensor.opened.clear() + computer = DepthRecomputer(_reader(_session("/cfg/sensor.json"))) + with patch("orisys.Sensor", _CountingSensor): + computer._depth_from_segment(Path("raw_0001.mp4"), 3) + sensor = computer._sensor._sensor + assert sensor.chain_calls == 6, "one deformation + one contact per frame" + computer.close() + + +def test_the_exact_config_path_is_opened(tmp_path=None): + """A reduced name could load a different file with the same name.""" + _CountingSensor.opened.clear() + config = Path("D:/custom/box/sensor.json") + computer = DepthRecomputer(_reader(_session(str(config)))) + with patch("orisys.Sensor", _CountingSensor): + computer._depth_from_segment(Path("raw_0001.mp4"), 1) + assert _CountingSensor.opened[0]["kwargs"]["config_name"] == str(config) + computer.close() + + +def test_the_plan_starts_at_the_last_reset_in_the_same_segment(): + frames = [ + SimpleNamespace( + frame_id=index, + t_rel_s=index / 30.0, + segment=1, + segment_frame=index + 1, + in_video=True, + dropped=False, + ) + for index in range(10) + ] + reader = _reader(_session("/cfg/sensor.json"), frames=frames) + reader.last_reset_frame = lambda frame_id: 6 + computer = DepthRecomputer(reader) + plan = computer.plan_for(frames[8]) + assert plan.stop_at == 9 + assert plan.preroll is None + assert plan.warnings == () + computer.close() + + +def test_the_plan_uses_the_pre_roll_for_the_start_of_a_recording(): + frames = [ + SimpleNamespace( + frame_id=index, + t_rel_s=index / 30.0, + segment=1, + segment_frame=index + 1, + in_video=True, + dropped=False, + ) + for index in range(5) + ] + + def reader_factory(): + reader = _reader(_session("/cfg/sensor.json"), frames=frames) + reader.preroll_frames = lambda: 30 + reader.preroll_segment = lambda: reader.directory / "preroll_0001.mp4" + return reader + + reader = reader_factory() + (reader.directory / "preroll_0001.mp4").write_bytes(b"") + computer = DepthRecomputer(reader) + plan = computer.plan_for(frames[3]) + assert plan.preroll is not None + assert plan.preroll_frames == 30 + assert plan.warnings == () + computer.close() + + +def test_a_segment_without_history_is_flagged_approximate(): + """Starting mid-recording with no reset and no pre-roll cannot be exact.""" + frames = [ + SimpleNamespace( + frame_id=index, + t_rel_s=index / 30.0, + segment=2, + segment_frame=index + 1, + in_video=True, + dropped=False, + ) + for index in range(5) + ] + reader = _reader(_session("/cfg/sensor.json"), frames=frames) + computer = DepthRecomputer(reader) + plan = computer.plan_for(frames[2]) + assert "state_approximate" in plan.warnings + computer.close() + + +def test_a_dropped_frame_is_reported_not_substituted(): + """The index marks the hole; replay must say so instead of showing a neighbour.""" + frames = [ + SimpleNamespace( + frame_id=0, + t_rel_s=0.0, + segment=1, + segment_frame=1, + in_video=True, + dropped=False, + ), + SimpleNamespace( + frame_id=1, + t_rel_s=1 / 30.0, + segment=1, + segment_frame=0, + in_video=False, + dropped=True, + ), + ] + reader = _reader_with_config(frames) + reader.frame_at = lambda _t: frames[1] + computer = DepthRecomputer(reader) + result = computer.depth_at(1 / 30.0) + assert not result.ok + assert result.error == "frame_not_recorded" + computer.close() diff --git a/shared/tests/test_record_lifecycle.py b/shared/tests/test_record_lifecycle.py new file mode 100644 index 0000000..a054ce2 --- /dev/null +++ b/shared/tests/test_record_lifecycle.py @@ -0,0 +1,251 @@ +"""Recording lifecycle: terminal states, refusal to reuse a live writer, gaps. + +The review's P1-5 and P1-6 are both about a recording that looks alive while it +is not: five write errors left ``is_recording`` true, a stop timeout reported +success and wrote ``ended_at`` while the files were still open, and a full queue +threw the CSV row away while counting only the video loss. +""" + +from __future__ import annotations + +import csv +import json +import queue +import shutil +import threading +import time +from collections import deque +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import pytest + +from orisys_app_shared.acquisition.sdk_worker import SdkPacket +from orisys_app_shared.record import ExperimentRecorder, RecordConfig +from orisys_app_shared.record.experiment import STATE_FAILED, STATE_STOPPED, _SessionWriter + +SCRATCH = Path(__file__).resolve().parent / "_scratch" + + +def _packet(index: int = 0) -> SdkPacket: + frame = np.zeros((8, 8, 3), np.uint8) + return SdkPacket( + raw_frame=frame, + processed_frame=frame, + timestamp=1000.0 + index / 30.0, + fps=30.0, + process_ms=1.0, + frame_id=index, + ) + + +def _root() -> Path: + shutil.rmtree(SCRATCH, ignore_errors=True) + SCRATCH.mkdir(parents=True, exist_ok=True) + return SCRATCH + + +def _wait_for(predicate, timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.01) + return False + + +def test_repeated_write_failures_end_the_recording_state(): + recorder = ExperimentRecorder(RecordConfig(video=False)) + with patch.object(_SessionWriter, "write_sample", side_effect=OSError("disk full")): + recorder.start(root=_root(), session_id=1, metadata={}) + for index in range(5): + recorder.on_packet(_packet(index)) + assert _wait_for(lambda: recorder.stats().write_errors >= 5) + assert recorder.stats().failed + assert not recorder.is_recording, "a dead writer still claimed to be recording" + assert recorder.state == STATE_FAILED + report = recorder.stop() + assert not report.ok + + +def test_packets_after_a_fatal_error_are_refused_not_silently_eaten(): + recorder = ExperimentRecorder(RecordConfig(video=False)) + with patch.object(_SessionWriter, "write_sample", side_effect=OSError("disk full")): + recorder.start(root=_root(), session_id=1, metadata={}) + for index in range(5): + recorder.on_packet(_packet(index)) + assert _wait_for(lambda: recorder.stats().write_errors >= 5) + for index in range(10): + recorder.on_packet(_packet(index)) + assert recorder.stats().rejected_after_failure > 0 + recorder.stop() + + +def test_a_stopped_writer_is_in_a_terminal_state(): + recorder = ExperimentRecorder(RecordConfig(video=False)) + recorder.start(root=_root(), session_id=1, metadata={}) + recorder.on_packet(_packet(0)) + report = recorder.stop() + assert report.completed and report.ok + assert recorder.state == STATE_STOPPED + assert not recorder.is_recording + + +def test_a_writer_that_misses_the_deadline_is_not_reported_as_finished(): + """A stop timeout must not claim the files are closed.""" + started = threading.Event() + release = threading.Event() + original = _SessionWriter.write_sample + + def slow(self, *args): + started.set() + release.wait(5.0) + return original(self, *args) + + recorder = ExperimentRecorder(RecordConfig(video=False)) + with patch.object(_SessionWriter, "write_sample", slow): + directory = recorder.start(root=_root(), session_id=1, metadata={}) + recorder.on_packet(_packet(0)) + assert started.wait(5.0) + thread = recorder._thread + report = recorder.stop(timeout=0.01) + try: + assert not report.ok + assert not report.completed + assert thread.is_alive(), "the report claimed the writer had stopped" + meta = json.loads((Path(directory) / "session.json").read_text(encoding="utf-8")) + assert not meta["ended_at"], "ended_at was written before the files closed" + finally: + release.set() + thread.join(5.0) + # A late stop observes the writer's own report, not the timeout placeholder. + assert recorder.state == STATE_STOPPED + + +def test_a_live_writer_cannot_be_reused_for_a_new_session(): + started = threading.Event() + release = threading.Event() + original = _SessionWriter.write_sample + + def slow(self, *args): + started.set() + release.wait(5.0) + return original(self, *args) + + recorder = ExperimentRecorder(RecordConfig(video=False)) + with patch.object(_SessionWriter, "write_sample", slow): + recorder.start(root=_root(), session_id=1, metadata={}) + recorder.on_packet(_packet(0)) + assert started.wait(5.0) + with pytest.raises(RuntimeError): + recorder.start(root=_root(), session_id=2, metadata={}) + release.set() + recorder.stop() + + +def test_events_are_accepted_as_soon_as_start_returns(): + """The writer may still be opening its files; "started" must survive that.""" + recorder = ExperimentRecorder(RecordConfig(video=False)) + directory = recorder.start(root=_root(), session_id=1, metadata={}) + recorder.log_event("record_started", "mode=video_data") + recorder.on_packet(_packet(0)) + recorder.stop() + with (Path(directory) / "events.csv").open(encoding="utf-8", newline="") as handle: + kinds = [row["kind"] for row in csv.DictReader(handle)] + assert kinds == ["record_started"] + + +def test_a_full_queue_keeps_the_rows_and_sheds_the_video(): + """P1-6: losing the CSV row because the video could not keep up.""" + started = threading.Event() + release = threading.Event() + original = _SessionWriter.write_sample + + def slow(self, *args): + started.set() + release.wait(5.0) + return original(self, *args) + + recorder = ExperimentRecorder(RecordConfig(video=True, max_queue=16)) + with patch.object(_SessionWriter, "write_sample", slow): + directory = recorder.start(root=_root(), session_id=1, metadata={}) + recorder.on_packet(_packet(0)) + assert started.wait(5.0) + for index in range(1, 25): + recorder.on_packet(_packet(index)) + release.set() + report = recorder.stop() + + with (Path(directory) / "forces.csv").open(encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + assert len(rows) == 25, "data rows were lost because the video queue filled up" + assert report.stats.dropped_data == 0 + assert report.stats.dropped_video > 0, "the video loss was not reported" + with (Path(directory) / "frames.csv").open(encoding="utf-8", newline="") as handle: + frame_rows = list(csv.DictReader(handle)) + assert len(frame_rows) == len(rows) + assert [row["frame_id"] for row in frame_rows] == [row["frame_id"] for row in rows] + + +def test_a_lost_row_is_recorded_as_an_index_gap(): + """A hole in the data must be visible, not silently bridged in replay.""" + original = _SessionWriter.write_sample + calls = {"count": 0} + + def slow(self, *args): + calls["count"] += 1 + if calls["count"] <= 2: + # Stall just long enough for the queue and the tiny backlog to fill. + time.sleep(0.3) + return original(self, *args) + + recorder = ExperimentRecorder(RecordConfig(video=False, max_queue=16)) + with patch.object(_SessionWriter, "write_sample", slow): + directory = recorder.start(root=_root(), session_id=1, metadata={}) + recorder._row_backlog = deque(recorder._row_backlog, maxlen=2) + for index in range(40): + recorder.on_packet(_packet(index)) + report = recorder.stop() + + with (Path(directory) / "frames.csv").open(encoding="utf-8", newline="") as handle: + frame_rows = list(csv.DictReader(handle)) + assert frame_rows, "the frame index is empty" + assert any(row["dropped"] == "1" for row in frame_rows), ( + "a frame whose row was lost left no trace in the index" + ) + assert report.stats.dropped_data > 0, "the loss was not counted" + assert report.stats.dropped_gaps > 0, "the loss was not marked in the index" + + +def test_session_json_names_the_pre_roll_when_one_was_written(): + frames = [np.full((8, 8, 3), index, np.uint8) for index in range(3)] + recorder = ExperimentRecorder(RecordConfig(video=True, fps=5.0)) + directory = recorder.start(root=_root(), session_id=1, metadata={}, preroll=frames) + recorder.on_packet(_packet(0)) + recorder.stop() + meta = json.loads((Path(directory) / "session.json").read_text(encoding="utf-8")) + assert meta["preroll_frames"] == 3 + assert (Path(directory) / meta["preroll_segment"]).is_file() + + +def test_a_data_only_recording_says_why_it_has_no_pre_roll(): + """The first frames cannot be replayed faithfully; that must be stated.""" + frames = [np.full((8, 8, 3), index, np.uint8) for index in range(3)] + recorder = ExperimentRecorder(RecordConfig(video=False)) + directory = recorder.start(root=_root(), session_id=1, metadata={}, preroll=frames) + recorder.on_packet(_packet(0)) + report = recorder.stop() + meta = json.loads((Path(directory) / "session.json").read_text(encoding="utf-8")) + assert meta["preroll_frames"] == 0 + assert any("pre-roll" in problem for problem in report.problems) + + +def test_stopping_twice_returns_the_same_report(): + recorder = ExperimentRecorder(RecordConfig(video=False)) + recorder.start(root=_root(), session_id=1, metadata={}) + recorder.on_packet(_packet(0)) + first = recorder.stop() + second = recorder.stop() + assert first.directory == second.directory + assert second.ok diff --git a/shared/tests/test_record_semantics.py b/shared/tests/test_record_semantics.py new file mode 100644 index 0000000..b532f6a --- /dev/null +++ b/shared/tests/test_record_semantics.py @@ -0,0 +1,152 @@ +"""Recording semantics: what a row means, and what a missing value means. + +Every case here comes from a reproducible defect in the phase-1/2 review: an +uncalibrated sensor writing its raw counts into the newton columns, a failed +compute step recording the previous frame's number, and a wall clock standing in +for the acquisition clock. +""" + +from __future__ import annotations + +import csv +import shutil +from pathlib import Path + +import numpy as np +import pytest + +from orisys_app_shared.acquisition.sdk_worker import SdkPacket +from orisys_app_shared.record import ExperimentRecorder, RecordConfig +from orisys_app_shared.record.sample import STALE_REASON, to_record_sample + +SCRATCH = Path(__file__).resolve().parent / "_scratch" + +_NUMERIC = ("FN_raw", "FX_raw", "FY_raw", "FN_N", "FX_N", "FY_N") + + +def _packet(index: int = 0, **overrides) -> SdkPacket: + frame = np.zeros((8, 8, 3), np.uint8) + kwargs = dict( + raw_frame=frame, + processed_frame=frame, + timestamp=1000.0 + index / 30.0, + fps=30.0, + process_ms=1.0, + normal_force=123.0, + shear_x=2.0, + shear_y=3.0, + normal_force_raw=123.0, + shear_x_raw=2.0, + shear_y_raw=3.0, + force_unit="raw", + force_calibrated=False, + frame_id=index, + ) + kwargs.update(overrides) + return SdkPacket(**kwargs) + + +def _root() -> Path: + shutil.rmtree(SCRATCH, ignore_errors=True) + SCRATCH.mkdir(parents=True, exist_ok=True) + return SCRATCH + + +def _record(packets) -> list[dict]: + recorder = ExperimentRecorder(RecordConfig(video=False)) + directory = recorder.start(root=_root(), session_id=1, metadata={}) + for packet in packets: + recorder.on_packet(packet) + recorder.stop() + with (directory / "forces.csv").open(encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) + + +def test_uncalibrated_sensor_writes_no_newton_values(): + """The SDK reports raw counts in the ordinary force fields when uncalibrated. + + Copying those into ``FN_N`` claimed a newton measurement that does not exist. + """ + rows = _record([_packet(0)]) + assert rows[0]["FN_raw"] == "123.0" + assert rows[0]["FN_N"] == "" + assert rows[0]["FX_N"] == "" + assert rows[0]["FY_N"] == "" + + +def test_partially_calibrated_sensor_fills_only_the_calibrated_axes(): + rows = _record( + [ + _packet( + 0, + force_calibrated=True, + calibrated_axes=("fn",), + normal_force=1.5, + shear_x=2.0, + shear_y=3.0, + ) + ] + ) + assert float(rows[0]["FN_N"]) == pytest.approx(1.5) + assert rows[0]["FX_N"] == "", "an uncalibrated axis must not claim a newton value" + assert rows[0]["FY_N"] == "" + + +def test_failed_compute_records_nothing_numeric(): + """A failed chain leaves the SDK's previous value in place; it is not this frame's.""" + rows = _record([_packet(0, valid=False, invalid_reason="compute failed", stale_forces=True)]) + for column in _NUMERIC: + assert rows[0][column] == "", f"{column} recorded a value for a failed frame" + + +def test_stale_forces_are_named_even_without_a_reason(): + sample = to_record_sample(_packet(0, stale_forces=True)) + assert not sample.valid + assert sample.invalid_reason == STALE_REASON + + +def test_invalid_frames_keep_their_row_and_frame_index(): + rows = _record([_packet(0), _packet(1, valid=False, invalid_reason="read failed")]) + assert len(rows) == 2, "an invalid frame still belongs in the recording" + assert rows[1]["valid"] == "0" + + +def test_acquisition_time_is_monotonic_and_ignores_the_wall_clock(): + """Ordering must come from the acquisition clock, not from ``time.time()``. + + A packet without an explicit ``mono_t`` used to record ``0.0`` for every + frame, which made the whole recording simultaneous. + """ + recorder = ExperimentRecorder(RecordConfig(video=False)) + directory = recorder.start(root=_root(), session_id=1, metadata={}) + for index in range(5): + recorder.on_packet(_packet(index, timestamp=1000.0 + index / 30.0)) + recorder.stop() + with (directory / "forces.csv").open(encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + times = [float(row["t_rel_s"]) for row in rows] + assert times == sorted(times) + assert times[0] == pytest.approx(0.0) + assert times[-1] > times[0], "every frame was recorded at the same instant" + + +def test_a_wall_clock_that_steps_backwards_cannot_reorder_history(): + recorder = ExperimentRecorder(RecordConfig(video=False)) + directory = recorder.start(root=_root(), session_id=1, metadata={}) + # mono_t increases while the wall clock jumps back an hour. + for index in range(4): + recorder.on_packet( + _packet(index, mono_t=index / 30.0, timestamp=1000.0 - index * 3600.0) + ) + recorder.stop() + with (directory / "forces.csv").open(encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + times = [float(row["t_rel_s"]) for row in rows] + assert times == sorted(times) + + +def test_the_recorded_calibration_version_comes_from_the_packet(): + """A session can outlive the coefficients it started with.""" + packet = _packet(0, calibration_version="hash-after-apply") + sample = to_record_sample(packet, calibration_version="hash-at-start") + assert sample.calibration_version == "hash-after-apply"