diff --git a/apps/studio/config/locales/en_US.json b/apps/studio/config/locales/en_US.json index 2a8245f..cc04211 100644 --- a/apps/studio/config/locales/en_US.json +++ b/apps/studio/config/locales/en_US.json @@ -13,6 +13,8 @@ "file": "File", "open_video": "Open video…", "reveal_config": "Reveal sensor config…", + "playback": "Open recording replay…", + "playback_latest": "Replay most recent recording", "view": "View", "theme": "Theme", "theme_dark": "Dark", @@ -121,6 +123,36 @@ "preset_2d": "2D mode: depth map on the left, force charts on the right, one shared time axis", "preset_default": "Free layout restored" }, + "playback": { + "title": "Replay recording", + "open": "Open recording directory…", + "open_failed": "Could not open that recording: {error}", + "none_found": "No recording found under outputs/recordings", + "loaded": "{name} — {frames} frames / {duration} s", + "empty": "Pick a recording directory to start replay", + "position": "{position} / {duration} s", + "show_newton": "Show newtons", + "show_raw": "Show raw", + "recompute_depth": "Recompute depth", + "recompute_depth_tip": "Depth is not stored with a recording; recompute this frame offline from the video and the config", + "frame_dropped": "This frame never reached the video (the record queue overflowed)", + "frame_unreadable": "This frame could not be decoded from the video", + "depth_hint": "Depth is not stored: press Recompute depth to rebuild it from the video and the config", + "depth_running": "Recomputing depth… (on a worker thread)", + "depth_busy": "The previous recomputation is still running", + "depth_empty": "No depth data for this frame", + "depth_failed": "Depth recomputation failed: {detail}", + "depth_sdk_unavailable": "Cannot recompute: the SDK is not available", + "depth_config_missing": "Cannot recompute: the recording has no sensor config reference", + "depth_config_file_missing": "Cannot recompute: the config file no longer exists at that path", + "depth_no_video": "Cannot recompute: this recording has no video segment", + "warnings": "Replay notes: {detail}", + "warn_config_changed": "the config differs from recording time, so the recomputed depth may differ", + "warn_sdk_changed": "the SDK version differs from recording time, so the recomputation may differ", + "warn_config_missing": "no config hash was recorded, so recomputation cannot be verified", + "warn_depth_recomputed": "depth comes from offline recomputation, not from stored data", + "warn_rows_skipped": "some unparsable CSV rows were skipped" + }, "record": { "label": "Record", "mode_video_data": "Video + data", diff --git a/apps/studio/config/locales/zh_CN.json b/apps/studio/config/locales/zh_CN.json index 7cf5bc9..98473a2 100644 --- a/apps/studio/config/locales/zh_CN.json +++ b/apps/studio/config/locales/zh_CN.json @@ -13,6 +13,8 @@ "file": "文件", "open_video": "打开视频…", "reveal_config": "在资源管理器中显示配置…", + "playback": "打开录制回放…", + "playback_latest": "回放最近一次录制", "view": "视图", "theme": "主题", "theme_dark": "深色", @@ -121,7 +123,38 @@ "preset_2d": "已切换到二维模式：左侧深度图、右侧三轴曲线，共用时间轴", "preset_default": "已恢复自由布局" }, - "record": { "label": "记录", + "playback": { + "title": "录制回放", + "open": "打开录制目录…", + "open_failed": "无法打开该录制: {error}", + "none_found": "outputs/recordings 下没有找到录制", + "loaded": "{name} — {frames} 帧 / {duration} s", + "empty": "选择一个录制目录开始回放", + "position": "{position} / {duration} s", + "show_newton": "显示牛顿值", + "show_raw": "显示原始值", + "recompute_depth": "重算深度", + "recompute_depth_tip": "深度未随录制保存，按录像与配置离线重算当前帧", + "frame_dropped": "该帧未被写入视频（记录队列曾溢出）", + "frame_unreadable": "该帧无法从视频中解码", + "depth_hint": "深度未存储：点击「重算深度」按原视频与配置离线重算", + "depth_running": "正在重算深度…（在后台线程）", + "depth_busy": "上一次重算尚未完成", + "depth_empty": "该帧没有深度数据", + "depth_failed": "深度重算失败: {detail}", + "depth_sdk_unavailable": "无法重算：SDK 不可用", + "depth_config_missing": "无法重算：录制未记录传感器配置", + "depth_config_file_missing": "无法重算：配置文件的原始路径已不存在", + "depth_no_video": "无法重算：该录制没有视频分段", + "warnings": "回放提示: {detail}", + "warn_config_changed": "配置文件与录制时不一致，重算出的深度可能与当时不同", + "warn_sdk_changed": "SDK 版本与录制时不一致，重算结果可能不同", + "warn_config_missing": "找不到配置哈希，无法校验重算一致性", + "warn_depth_recomputed": "深度由离线重算得到，不是当时保存的数据", + "warn_rows_skipped": "有无法解析的 CSV 行被跳过" + }, + "record": { + "label": "记录", "mode_video_data": "视频 + 数据", "mode_data_only": "仅数据", "mode_tip": "视频记录的是处理前的相机输入图像（不含箭头与文字叠加）。MP4 是有损封装，不等于无损原始数据。", diff --git a/apps/studio/docs/record-channel.md b/apps/studio/docs/record-channel.md index c6fbbf2..355623c 100644 --- a/apps/studio/docs/record-channel.md +++ b/apps/studio/docs/record-channel.md @@ -166,10 +166,59 @@ outputs/recordings/_session/ --- -## 7. 尚未完成（第二阶段及以后） +## 7. 同步回放与深度重算 + +**决定：不保存深度。** 回放时按录像与配置**离线重算**。 + +理由是体积：30 Hz、320×240 的 float32 深度每帧约 300 KB，一小时就是 ~16 GB； +即使降频到 1 Hz 也要百兆级。所以 `session.json` 里明确写： + +```json +"depth": { + "stored": false, + "replay": "recompute", + "quantity": "relative deformation / algorithm response, not a calibrated length", + "method": "re-run the SDK over raw_*.mp4 with this config_ref", + "config_sha256": "...", + "sdk_version": "..." +} +``` + +代价是重算要 SDK 能复现同样结果，因此 `RecordingReader.replay_warnings()` 会主动 +比对**当前配置文件哈希**与录制时的哈希、以及 SDK 版本，回放窗口顶部把这些提示出来： + +| 提示 | 含义 | +|---|---| +| `depth_recomputed` | 深度来自离线重算，不是当时保存的数据（始终提示） | +| `config_changed` | 配置文件已变，重算结果可能与当时不同 | +| `sdk_changed` | SDK 版本已变，重算结果可能不同 | +| `config_missing` | 缺少配置哈希，无法校验一致性 | +| `rows_skipped` | 有无法解析的 CSV 行被跳过 | + +### 回放界面 + +`ui/playback_dialog.py`，入口在「文件 → 打开录制回放…／回放最近一次录制」。 + +- **一个游标驱动三处**：移动位置同时更新视频帧、深度面板与三张曲线的游标线； + 并且通过 `cursor_moved` 信号同步到主界面的曲线面板。 +- 单位可切换（raw/N），数据直接来自录制目录的 `forces.csv`。 +- 帧未被写入视频时（记录队列曾溢出）明确提示，而不是显示上一帧冒充当前帧。 +- **深度重算在守护线程上执行**，UI 线程立即返回。实测 `_on_recompute_depth` 返回耗时 + 0.001 s，而 SDK 调用仍在后台跑。 + +两个必须知晓的边界： + +1. SDK 打开视频逐帧步进**可能长时间阻塞**，且无法取消。因此用守护线程：窗口能关、 + 进程能退，但那个线程会一直挂到进程结束。用线程池（非守护）会让**整个进程无法退出**， + 这是实测踩到的。 +2. 重算失败一律显示原因（SDK 不可用／未记录配置／配置路径不存在／无视频分段／ + 帧不在视频中），**绝不用其它图像顶替**。 + +--- + +## 8. 尚未完成（第三阶段及以后） -- CSV 的「导出选中时间段」（整段导出已随录制落盘；区间导出待做）。 -- 全程曲线浏览（跟随/历史、框选缩放、降采样保峰值）。 -- 标定诊断扩展：稳定后采样、异常点排除/恢复、拟合残差、独立验证点。当前 - `CalibrationPoint` 已记录三轴均值/标准差/有效样本数、方向、轮次与实际采样时长。 - PyOpenGL 三维显示（`mesh3d_panel.py` 目前仍是占位符，不是已有实现）。 +- CSV 的「导出选中时间段」（整段导出已随录制落盘；区间导出待做）。 +- 标定的稳定后采样（当前按固定时长采样，稳定性以 σ 呈现）。 + diff --git a/apps/studio/src/orisys_studio/app_window.py b/apps/studio/src/orisys_studio/app_window.py index a6b9c5d..f3df000 100644 --- a/apps/studio/src/orisys_studio/app_window.py +++ b/apps/studio/src/orisys_studio/app_window.py @@ -138,6 +138,7 @@ class StudioMainWindow(QMainWindow): self._record_last_report = None self._record_fps = 30.0 self._session_counter = 0 + self._playback = None self._record_error_count = 0 self._dropped_reported = 0 self._active_calibration_id = "" @@ -345,6 +346,18 @@ class StudioMainWindow(QMainWindow): "config_ref": config_ref, "force_calibrated": bool(getattr(info, "force_calibrated", False)), }, + # Depth is deliberately NOT stored (decided with the team): it is a + # function of the frame and the sensor configuration, so replay + # recomputes it from raw_*.mp4. The hashes below are what a replay + # checks before trusting that recomputation. + "depth": { + "stored": False, + "replay": "recompute", + "quantity": "relative deformation / algorithm response, not a calibrated length", + "method": "re-run the SDK over raw_*.mp4 with this config_ref", + "config_sha256": self._config_hash(config_ref), + "sdk_version": self._sdk_version(), + }, "conventions": { "axis_order": ["FN", "FX", "FY"], "force_unit": "N when calibrated, otherwise raw counts", @@ -495,6 +508,13 @@ class StudioMainWindow(QMainWindow): self.act_reveal_config = QAction(self) self.act_reveal_config.triggered.connect(self._on_reveal_config) self.file_menu.addAction(self.act_reveal_config) + self.file_menu.addSeparator() + self.act_playback = QAction(self) + self.act_playback.triggered.connect(self._open_playback) + self.file_menu.addAction(self.act_playback) + self.act_playback_latest = QAction(self) + self.act_playback_latest.triggered.connect(self._open_latest_recording) + self.file_menu.addAction(self.act_playback_latest) self.view_menu = menu_bar.addMenu("") self.act_cal_panel = QAction(self) @@ -685,6 +705,8 @@ class StudioMainWindow(QMainWindow): self.file_menu.setTitle(t("menu.file")) self.act_open_video.setText(t("menu.open_video")) self.act_reveal_config.setText(t("menu.reveal_config")) + self.act_playback.setText(t("menu.playback")) + self.act_playback_latest.setText(t("menu.playback_latest")) self.view_menu.setTitle(t("menu.view")) self.act_cal_panel.setText(t("menu.calibration_panel")) self.act_layout_default.setText(t("layout.default")) @@ -760,6 +782,36 @@ class StudioMainWindow(QMainWindow): def _show_about(self) -> None: AboutDialog(self).exec_() + def _open_playback(self) -> None: + """Replay a recording: video, depth (recomputed) and curves, one cursor.""" + from orisys_studio.ui.playback_dialog import PlaybackDialog + + if self._playback is not None: + self._playback.close() + self._playback = None + dialog = PlaybackDialog(theme=self._theme, parent=self) + # The replay cursor and the live charts share one time position. + dialog.cursor_moved.connect(self._on_playback_cursor) + self._playback = dialog + dialog.show() + + def _on_playback_cursor(self, t_rel_s: float) -> None: + self.force_plots.set_cursor(float(t_rel_s)) + + def _open_latest_recording(self) -> None: + from orisys_app_shared.record import list_recordings + + found = list_recordings(recordings_dir()) + if not found: + self.statusBar().showMessage(t("playback.none_found")) + return + from orisys_studio.ui.playback_dialog import PlaybackDialog + + dialog = PlaybackDialog(theme=self._theme, directory=found[0], parent=self) + dialog.cursor_moved.connect(self._on_playback_cursor) + self._playback = dialog + dialog.show() + def _on_camera_id_changed(self, value: int) -> None: self.prefs["camera_id"] = str(int(value)) self._persist_prefs() diff --git a/apps/studio/src/orisys_studio/ui/cal_panel.py b/apps/studio/src/orisys_studio/ui/cal_panel.py index 9569f9e..b081c71 100644 --- a/apps/studio/src/orisys_studio/ui/cal_panel.py +++ b/apps/studio/src/orisys_studio/ui/cal_panel.py @@ -523,11 +523,12 @@ class CalPanel(QFrame): CalibrationPointDialog(point, theme=self._theme, parent=self).exec_() def _on_toggle_excluded(self) -> None: - row = self.table.currentRow() point = self._current_point() - if point is None: + if point is None or self._recorder is None: return - self._recorder.set_excluded(row, not point.excluded) + # Pass the point itself: a table row index would resolve against the + # recorder's cross-channel list and could hit the wrong point. + self._recorder.set_excluded(point, not point.excluded) self._rebuild_store() self._refresh_table() self.status_message.emit( @@ -535,11 +536,10 @@ class CalPanel(QFrame): ) def _on_toggle_validation(self) -> None: - row = self.table.currentRow() point = self._current_point() - if point is None: + if point is None or self._recorder is None: return - self._recorder.set_validation(row, not point.is_validation) + self._recorder.set_validation(point, not point.is_validation) self._rebuild_store() self._refresh_table() self.status_message.emit( @@ -551,7 +551,7 @@ class CalPanel(QFrame): self._store.clear() if self._recorder is None: return - for point in self._recorder.points: + for point in self._recorder.all_points(): if point.excluded: continue stats = point.loaded_stats() @@ -568,30 +568,25 @@ class CalPanel(QFrame): ) def _on_delete_row(self) -> None: - row = self.table.currentRow() point = self._current_point() if point is None or self._recorder is None: return # Deleting is an exclusion: the recorded evidence stays, but it leaves # the fit and the export. - self._recorder.set_excluded(row, True) + self._recorder.set_excluded(point, True) self._rebuild_store() self._refresh_table() def _on_clear_channel(self) -> None: - self._store.clear(self._channel) if self._recorder is not None: - with self._recorder._lock: # noqa: SLF001 - panel owns the view - self._recorder.points = [ - p for p in self._recorder.points if p.channel != self._channel - ] + self._recorder.clear_points(self._channel) + self._rebuild_store() self._refresh_table() def _on_clear_all(self) -> None: - self._store.clear() if self._recorder is not None: - with self._recorder._lock: # noqa: SLF001 - self._recorder.points = [] + self._recorder.clear_points() + self._rebuild_store() self._refresh_table() def _on_export(self) -> None: diff --git a/apps/studio/src/orisys_studio/ui/playback_dialog.py b/apps/studio/src/orisys_studio/ui/playback_dialog.py new file mode 100644 index 0000000..8378773 --- /dev/null +++ b/apps/studio/src/orisys_studio/ui/playback_dialog.py @@ -0,0 +1,380 @@ +"""Synchronised replay of a recording: video, depth and force curves, one cursor. + +Depth is recomputed on demand rather than read from disk (see +:mod:`orisys_app_shared.record.depth`), so the depth pane states plainly when it +is unavailable instead of showing a stand-in image. + +The cursor is the single source of truth for the position: moving it updates the +video frame, the depth pane and the charts' cursor line together. +""" + +from __future__ import annotations + +import threading +from pathlib import Path + +import numpy as np +from PyQt5.QtCore import Qt, QTimer, pyqtSignal +from PyQt5.QtGui import QImage, QPixmap +from PyQt5.QtWidgets import ( + QDialog, + QFileDialog, + QHBoxLayout, + QLabel, + QPushButton, + QSlider, + QSplitter, + QVBoxLayout, +) + +from orisys_app_shared.i18n import t +from orisys_app_shared.paths import recordings_dir +from orisys_app_shared.record import DepthRecomputer, RecordingReader, list_recordings +from orisys_app_shared.record.playback import FORCE_COLUMNS +from orisys_app_shared.theme import color + +try: + import pyqtgraph as pg +except ImportError as exc: # pragma: no cover + raise ImportError("pyqtgraph is required for replay") from exc + +CHANNEL_TOKENS = ("plot_fn", "plot_fx", "plot_fy") +SLIDER_STEPS = 2000 + + +class PlaybackDialog(QDialog): + """Replay one experiment directory.""" + + #: Emitted when the replay cursor moves, so the main charts can follow it. + cursor_moved = pyqtSignal(float) + + def __init__(self, *, theme: str = "dark", directory: Path | None = None, parent=None) -> None: + super().__init__(parent) + self._theme = theme + self._reader: RecordingReader | None = None + self._recomputer: DepthRecomputer | None = None + self._unit = "raw" + self._position = 0.0 + self._depth_ok = False + # A **daemon** thread, not a ThreadPoolExecutor: the SDK call can block + # indefinitely and an executor's non-daemon worker would then keep the + # whole process alive at exit. A blocked daemon thread cannot be + # cancelled either, so it is left to die with the process. + self._depth_thread: threading.Thread | None = None + self._depth_result = None + self._depth_error: BaseException | None = None + self._depth_poll = QTimer(self) + self._depth_poll.setInterval(120) + self._depth_poll.timeout.connect(self._poll_depth) + + self.setWindowTitle(t("playback.title")) + self.resize(1080, 640) + root = QVBoxLayout(self) + root.setSpacing(8) + + controls = QHBoxLayout() + self.open_button = QPushButton() + self.open_button.clicked.connect(self._on_open) + controls.addWidget(self.open_button) + self.path_label = QLabel() + self.path_label.setObjectName("hintText") + controls.addWidget(self.path_label, 1) + self.unit_button = QPushButton() + self.unit_button.setProperty("compact", "true") + self.unit_button.clicked.connect(self._on_toggle_unit) + controls.addWidget(self.unit_button) + self.depth_button = QPushButton() + self.depth_button.setProperty("compact", "true") + self.depth_button.clicked.connect(self._on_recompute_depth) + controls.addWidget(self.depth_button) + root.addLayout(controls) + + self.warning_label = QLabel() + self.warning_label.setObjectName("hintText") + self.warning_label.setWordWrap(True) + root.addWidget(self.warning_label) + + splitter = QSplitter(Qt.Horizontal) + self.video_label = QLabel() + self.video_label.setAlignment(Qt.AlignCenter) + self.video_label.setMinimumSize(280, 200) + self.video_label.setWordWrap(True) + splitter.addWidget(self.video_label) + self.depth_label = QLabel() + self.depth_label.setAlignment(Qt.AlignCenter) + self.depth_label.setMinimumSize(200, 160) + self.depth_label.setWordWrap(True) + splitter.addWidget(self.depth_label) + splitter.setSizes([420, 320]) + root.addWidget(splitter, 1) + + self.plot_widget = pg.GraphicsLayoutWidget() + self.plot_widget.setBackground(color(theme, "plot_bg")) + self.plot_widget.setMinimumHeight(180) + root.addWidget(self.plot_widget, 1) + self.curves = [] + self.cursor_lines = [] + for row, token in enumerate(CHANNEL_TOKENS): + plot = self.plot_widget.addPlot(row=row, col=0) + plot.showGrid(x=True, y=True, alpha=0.25) + plot.setMouseEnabled(x=False, y=False) + pen = pg.mkPen(color(theme, "text_secondary")) + for axis_name in ("left", "bottom"): + plot.getAxis(axis_name).setPen(pen) + plot.getAxis(axis_name).setTextPen(pen) + plot.setLabel("left", ("FN", "FX", "FY")[row], color=color(theme, token)) + self.curves.append(plot.plot(pen=pg.mkPen(color(theme, token), width=2))) + line = pg.InfiniteLine(pos=0.0, angle=90, movable=False, pen=pg.mkPen("#888888")) + plot.addItem(line, ignoreBounds=True) + self.cursor_lines.append(line) + + position_row = QHBoxLayout() + self.slider = QSlider(Qt.Horizontal) + self.slider.setRange(0, SLIDER_STEPS) + self.slider.setEnabled(False) + self.slider.valueChanged.connect(self._on_slider) + position_row.addWidget(self.slider, 1) + self.position_label = QLabel() + self.position_label.setObjectName("panelMeta") + position_row.addWidget(self.position_label) + root.addLayout(position_row) + + if directory is not None: + self.open_recording(Path(directory)) + + # ------------------------------------------------------------------ opening + def _on_open(self) -> None: + start = str(recordings_dir()) + chosen = QFileDialog.getExistingDirectory(self, t("playback.open"), start) + if not chosen: + return + try: + self.open_recording(Path(chosen)) + except Exception as exc: # noqa: BLE001 + self.warning_label.setText(t("playback.open_failed", error=str(exc))) + + def open_recording(self, directory: Path) -> None: + if self._recomputer is not None: + self._recomputer.close() + self._recomputer = None + self._reader = RecordingReader(directory) + self._recomputer = DepthRecomputer(self._reader) + info = self._reader.info() + self.path_label.setText( + t( + "playback.loaded", + name=directory.name, + frames=len(info.frames), + duration=f"{info.duration_s:.1f}", + ) + ) + self.slider.setEnabled(info.duration_s > 0) + self._render_curves() + self._refresh_warnings() + self._depth_ok = False + self.set_position(0.0) + + def _refresh_warnings(self) -> None: + if self._reader is None: + return + _ok, sdk_version = DepthRecomputer.sdk_available() + # The reader compares the recorded config hash against the config file as + # it is now, which is what the depth recomputation would actually use. + codes = self._reader.replay_warnings(sdk_version=sdk_version) + if not codes: + self.warning_label.setText("") + return + self.warning_label.setText( + t("playback.warnings", detail=", ".join(t(f"playback.warn_{c}") for c in codes)) + ) + + # ------------------------------------------------------------------ drawing + def _render_curves(self) -> None: + if self._reader is None: + return + info = self._reader.info() + if info.times.size == 0: + for curve in self.curves: + curve.setData([], []) + return + values = info.value_for_unit(self._unit) + for index, curve in enumerate(self.curves): + column = values[:, index] + finite = np.isfinite(column) + curve.setData(info.times[finite], column[finite]) + self.plot_widget.ci.layout.setRowStretchFactor(index, 1) + self.plot_widget.getItem(index, 0).setXRange( + float(info.times[0]), float(info.times[-1]), padding=0.02 + ) + + def _on_toggle_unit(self) -> None: + self._unit = "N" if self._unit == "raw" else "raw" + self._render_curves() + self.set_position(self._position) + + # ------------------------------------------------------------------ cursor + def position(self) -> float: + return self._position + + def _on_slider(self, value: int) -> None: + if self._reader is None: + return + duration = self._reader.info().duration_s + self.set_position(duration * (value / SLIDER_STEPS)) + + def set_position(self, t_rel_s: float) -> None: + """Move the single cursor: video, depth and charts follow it.""" + if self._reader is None: + return + duration = self._reader.info().duration_s + self._position = max(0.0, min(float(t_rel_s), duration if duration > 0 else 0.0)) + self.slider.blockSignals(True) + if duration > 0: + self.slider.setValue(int(SLIDER_STEPS * self._position / duration)) + self.slider.blockSignals(False) + self.position_label.setText( + t("playback.position", position=f"{self._position:.2f}", duration=f"{duration:.2f}") + ) + for line in self.cursor_lines: + line.setValue(self._position) + self._show_video_frame() + if self._depth_ok: + self._show_depth_frame() + self.cursor_moved.emit(self._position) + + def _show_video_frame(self) -> None: + if self._reader is None: + return + image, frame = self._reader.video_frame_at(self._position) + if image is None: + reason = ( + t("playback.frame_dropped") + if frame is not None and not frame.in_video + else t("playback.frame_unreadable") + ) + self.video_label.setPixmap(QPixmap()) + self.video_label.setText(reason) + return + rgb = image[:, :, ::-1].copy() + height, width, channels = rgb.shape + qimage = QImage( + rgb.data, width, height, channels * width, QImage.Format_RGB888 + ).copy() + pixmap = QPixmap.fromImage(qimage).scaled( + self.video_label.size(), Qt.KeepAspectRatio, Qt.SmoothTransformation + ) + self.video_label.setPixmap(pixmap) + self.video_label.setText("") + + # -------------------------------------------------------------------- depth + def _on_recompute_depth(self) -> None: + """Kick off a recomputation on a worker thread. + + Opening a video with the SDK and stepping to a frame can block, so it + must never run on the UI thread — that is exactly how the replay window + would freeze. + """ + if self._recomputer is None: + return + problem = self._recomputer.preflight() + if problem: + self._depth_ok = False + self.depth_label.setPixmap(QPixmap()) + self.depth_label.setText(t(f"playback.depth_{problem}")) + return + if self._depth_thread is not None and self._depth_thread.is_alive(): + self.depth_label.setText(t("playback.depth_busy")) + return + self._depth_ok = False + self._depth_result = None + self._depth_error = None + self.depth_label.setText(t("playback.depth_running")) + position = self._position + + def work() -> None: + # Runs off the UI thread and touches no Qt object. + try: + self._depth_result = self._recomputer.depth_at(position) + except BaseException as exc: # noqa: BLE001 - reported via the poll + self._depth_error = exc + + self._depth_thread = threading.Thread( + target=work, name="depth-replay", daemon=True + ) + self._depth_thread.start() + self._depth_poll.start() + + def _poll_depth(self) -> None: + """UI-thread poll for the worker result.""" + thread = self._depth_thread + if thread is None or thread.is_alive(): + return + self._depth_poll.stop() + if self._depth_error is not None: + self._depth_ok = False + self.depth_label.setPixmap(QPixmap()) + self.depth_label.setText( + t("playback.depth_failed", detail=str(self._depth_error)) + ) + return + result = self._depth_result + if result is None: + return + self._render_depth_result(result) + + def _render_depth_result(self, result) -> None: + if not result.ok: + self._depth_ok = False + self.depth_label.setPixmap(QPixmap()) + self.depth_label.setText(t("playback.depth_failed", detail=result.error)) + return + from orisys_app_shared.acquisition.arrows import depth_map_to_bgr + + bgr = depth_map_to_bgr(np.asarray(result.depth)) + if bgr is None: + self._depth_ok = False + self.depth_label.setPixmap(QPixmap()) + self.depth_label.setText(t("playback.depth_empty")) + return + self._depth_ok = True + rgb = bgr[:, :, ::-1].copy() + height, width, channels = rgb.shape + qimage = QImage( + rgb.data, width, height, channels * width, QImage.Format_RGB888 + ).copy() + pixmap = QPixmap.fromImage(qimage).scaled( + self.depth_label.size(), Qt.KeepAspectRatio, Qt.SmoothTransformation + ) + self.depth_label.setPixmap(pixmap) + self.depth_label.setText("") + + def _show_depth_frame(self) -> None: + """Re-render the last result at the new position (no SDK work).""" + if self._depth_result is not None: + self._render_depth_result(self._depth_result) + + # ------------------------------------------------------------------- labels + def retranslate(self) -> None: + self.setWindowTitle(t("playback.title")) + self.open_button.setText(t("playback.open")) + self.unit_button.setText( + t("playback.show_raw") if self._unit == "N" else t("playback.show_newton") + ) + self.depth_button.setText(t("playback.recompute_depth")) + self.depth_button.setToolTip(t("playback.recompute_depth_tip")) + if self._reader is None: + self.video_label.setText(t("playback.empty")) + self.depth_label.setText(t("playback.depth_hint")) + self.path_label.setText("") + + def closeEvent(self, event) -> None: # noqa: N802 + self._depth_poll.stop() + # A depth recomputation that is blocked inside the SDK cannot be + # cancelled; it is a daemon thread, so it does not keep the process + # alive and the window always closes. + if self._recomputer is not None and not self._depth_busy(): + self._recomputer.close() + super().closeEvent(event) + + def _depth_busy(self) -> bool: + return self._depth_thread is not None and self._depth_thread.is_alive() diff --git a/apps/studio/tests/test_locales.py b/apps/studio/tests/test_locales.py index d4b6c5d..35a5052 100644 --- a/apps/studio/tests/test_locales.py +++ b/apps/studio/tests/test_locales.py @@ -74,3 +74,46 @@ def test_locale_files_are_valid_utf8_json(): data = _load_locale(name) assert isinstance(data, dict) assert data.get("app", {}).get("title") + + +#: Sequences that only ever appear when UTF-8 text was decoded as a single-byte +#: codepage and then re-encoded. A ``Get-Content -Raw | Set-Content`` round trip +#: in Windows PowerShell does exactly that and silently mangles every non-ASCII +#: string, which is how the Chinese UI once ended up unreadable. +#: +#: Written as escapes, with no literal examples in the comments, so that this +#: guard cannot itself be corrupted by the same mistake. +_MOJIBAKE = ( + "\u00c3", + "\u00c2", + "\u00e2\u20ac", + "\u00e3\u20ac", + "\u00ef\u00bf\u00bd", +) + + +def test_locale_files_are_utf8_without_bom_and_not_mojibake(): + for name in ("zh_CN", "en_US"): + path = LOCALES_DIR / f"{name}.json" + raw = path.read_bytes() + assert not raw.startswith(b"\xef\xbb\xbf"), f"{name}.json starts with a BOM" + text = raw.decode("utf-8") # raises if it is not valid UTF-8 + + for sequence in _MOJIBAKE: + assert sequence not in text, ( + f"{name}.json contains {sequence!r}: the file was decoded with the " + "wrong encoding. Rewrite it with an explicit UTF-8 writer instead " + "of a PowerShell Get-Content/Set-Content round trip." + ) + + +def test_chinese_locale_actually_contains_chinese(): + """A corrupted zh_CN.json still parses as JSON, so check the content.""" + data = _load_locale("zh_CN") + for key in ("menu/theme_dark", "app/about_title", "chart/mode_follow"): + node = data + for part in key.split("/"): + node = node[part] + assert any("\u4e00" <= char <= "\u9fff" for char in node), ( + f"zh_CN.{key} has no Chinese characters: {node!r}" + ) diff --git a/apps/studio/tools/ui_check.py b/apps/studio/tools/ui_check.py index 80f4b33..5edf48f 100644 --- a/apps/studio/tools/ui_check.py +++ b/apps/studio/tools/ui_check.py @@ -61,6 +61,66 @@ def section(name: str) -> None: print(f"\n== {name} ==") +#: Sequences that only appear when UTF-8 text was decoded with a single-byte +#: codepage and re-encoded. A PowerShell ``Get-Content -Raw | Set-Content`` +#: round trip does that and silently mangles every non-ASCII string, which is +#: how the Chinese UI once ended up unreadable. +#: +#: Written as escapes, with no literal examples in the comments, so that this +#: guard cannot itself be corrupted by the same mistake and so that a scan of +#: the tree for these sequences stays clean. +MOJIBAKE_MARKERS = ( + "\u00c3", + "\u00c2", + "\u00e2\u20ac", + "\u00e3\u20ac", + "\u00ef\u00bf\u00bd", +) + + +def audit_locale_encoding() -> list[str]: + """Locale files must be BOM-free UTF-8 and not mojibake.""" + import json + + problems: list[str] = [] + locales = APP_ROOT / "config" / "locales" + for name in ("zh_CN", "en_US"): + path = locales / f"{name}.json" + if not path.is_file(): + problems.append(f"{name}.json is missing") + continue + raw = path.read_bytes() + if raw.startswith(b"\xef\xbb\xbf"): + problems.append(f"{name}.json starts with a UTF-8 BOM") + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + problems.append(f"{name}.json is not valid UTF-8: {exc}") + continue + for marker in MOJIBAKE_MARKERS: + if marker in text: + problems.append( + f"{name}.json contains {marker!r} — written by a tool that " + "decoded UTF-8 with the wrong codepage" + ) + break + try: + json.loads(text) + except json.JSONDecodeError as exc: + problems.append(f"{name}.json is not valid JSON: {exc}") + # A corrupted zh_CN.json still parses, so check that it holds Chinese text. + try: + import json as _json + + zh = _json.loads((locales / "zh_CN.json").read_text(encoding="utf-8")) + sample = zh["menu"]["theme_dark"] + if not any("\u4e00" <= char <= "\u9fff" for char in sample): + problems.append(f"zh_CN.menu.theme_dark has no Chinese text: {sample!r}") + except Exception as exc: # noqa: BLE001 + problems.append(f"could not verify the Chinese locale: {exc}") + return problems + + def _calibration_sample(index: int, *, fn_raw: float, fx_raw: float = 0.4, fy_raw: float = -0.2): """A sample shaped for the calibration recorder.""" from orisys_app_shared.force.session import SessionSample @@ -429,6 +489,13 @@ def main() -> int: if not palette_failures: check(True, "palette rules hold for both themes") + section("locale encoding") + encoding_failures = audit_locale_encoding() + for failure in encoding_failures: + check(False, failure) + if not encoding_failures: + check(True, "locale files are BOM-free UTF-8 and not mojibake") + scene_count = 0 for theme in ("dark", "light"): for sensor in ("box", "finger"): @@ -1047,6 +1114,128 @@ def main() -> int: app.processEvents() stuck.release() + section("replay") + replay_root = APP_ROOT / "outputs" / "_replay_check" + _shutil.rmtree(replay_root, ignore_errors=True) + replay_root.mkdir(parents=True, exist_ok=True) + from orisys_app_shared.record import ExperimentRecorder, RecordConfig + + recorder = ExperimentRecorder( + RecordConfig(video=True, fps=30.0, segment_seconds=30.0) + ) + recording = recorder.start( + root=replay_root, + session_id=1, + metadata={ + "device": { + "sensor_type": "box", + "config_ref": str(replay_root / "sensor.json"), + "config_sha256": "recorded", + }, + "versions": {"sdk": "0.5.1"}, + "depth": {"stored": False, "replay": "recompute"}, + }, + ) + (replay_root / "sensor.json").write_text('{"camera": {}}\n', encoding="utf-8") + for index in range(40): + recorder.on_packet(_fake_packet(index)) + recorder.stop() + + from orisys_studio.ui.playback_dialog import PlaybackDialog + + playback = PlaybackDialog(theme="dark", directory=recording) + playback.resize(1000, 620) + playback.show() + app.processEvents() + + check( + playback.position() == 0.0, + f"replay did not start at the beginning: {playback.position()}", + ) + check( + all(int(curve.getData()[0].size) == 40 for curve in playback.curves), + "replay did not draw every force sample", + ) + check(playback.slider.isEnabled(), "the replay position slider is disabled") + check( + t("playback.warn_depth_recomputed").split("{")[0].strip()[:4] + in playback.warning_label.text(), + f"the replay notes are missing: {playback.warning_label.text()!r}", + ) + + moved: list[float] = [] + playback.cursor_moved.connect(moved.append) + playback.set_position(1.0) + app.processEvents() + check( + [round(line.value(), 2) for line in playback.cursor_lines] == [1.0, 1.0, 1.0], + "the replay cursor is not shared across the three charts", + ) + check(moved == [1.0], f"the cursor signal did not fire once: {moved}") + check( + playback.video_label.pixmap() is not None + and not playback.video_label.pixmap().isNull(), + "no video frame was rendered at the cursor position", + ) + + # The depth recomputation must never run on the UI thread. A stub that blocks + # proves the call returns immediately instead of freezing the window. + import threading as _threading + import time as _time + + class _Blocking: + released = _threading.Event() + + def preflight(self) -> str: + return "" + + def depth_at(self, _t): + _Blocking.released.wait(timeout=5.0) + return type("R", (), {"ok": False, "error": "stub", "depth": None})() + + def close(self) -> None: + return None + + playback._recomputer = _Blocking() # noqa: SLF001 - exercising the contract + started = _time.perf_counter() + playback._on_recompute_depth() + elapsed = _time.perf_counter() - started + check(elapsed < 0.5, f"the depth recompute blocked the UI for {elapsed:.2f}s") + check( + playback._depth_thread is not None and playback._depth_thread.is_alive(), + "the depth recompute did not run on a worker thread", + ) + check( + playback._depth_thread.daemon, + "the depth worker is not a daemon, so a blocked SDK call would keep the " + "process alive at exit", + ) + playback._on_recompute_depth() # a second request is refused, not queued + check( + playback.depth_label.text() == t("playback.depth_busy"), + f"a concurrent recompute was not refused: {playback.depth_label.text()!r}", + ) + _Blocking.released.set() + for _ in range(40): + app.processEvents() + _time.sleep(0.05) + if not playback._depth_busy(): # noqa: SLF001 + break + check(not playback._depth_busy(), "the depth worker never finished") + # Deliver the worker result deterministically instead of waiting for the + # 120 ms poll timer. + playback._poll_depth() + app.processEvents() + check( + t("playback.depth_failed").split("{")[0].strip()[:4] + in playback.depth_label.text(), + f"a failed recomputation was not reported: {playback.depth_label.text()!r}", + ) + + playback.close() + app.processEvents() + _shutil.rmtree(replay_root, ignore_errors=True) + print(f"\nscenes: {scene_count} checks: {CHECKS} failures: {len(FAILURES)}") for failure in FAILURES: print(f" FAIL: {failure}") diff --git a/shared/src/orisys_app_shared/force/calibration_recorder.py b/shared/src/orisys_app_shared/force/calibration_recorder.py index 532f06d..62dd711 100644 --- a/shared/src/orisys_app_shared/force/calibration_recorder.py +++ b/shared/src/orisys_app_shared/force/calibration_recorder.py @@ -340,23 +340,68 @@ class CalibrationRecorder: return point # ------------------------------------------------------------ diagnostics - def set_excluded(self, index: int, excluded: bool) -> None: + def all_points(self) -> list[CalibrationPoint]: + """Snapshot of every recorded point, across channels.""" + with self._lock: + return list(self.points) + + def _resolve(self, point: CalibrationPoint) -> Optional[CalibrationPoint]: + """Find the recorded point that is ``point``, by identity. + + Callers pass the object they are looking at. An integer row index would + be ambiguous: the panel shows a per-channel filtered list while the + recorder stores every channel in one list, so index 0 of the table is + not index 0 of ``points``. + """ + if point is None: + return None + for candidate in self.points: + if candidate is point: + return candidate + return None + + def set_excluded(self, point: CalibrationPoint, excluded: bool) -> bool: """Remove (or restore) a point from the fit and from the export.""" with self._lock: - if not 0 <= index < len(self.points): - return - self.points[index].excluded = bool(excluded) + target = self._resolve(point) + if target is None: + return False + target.excluded = bool(excluded) if excluded: - self.points[index].is_validation = False + target.is_validation = False + return True - def set_validation(self, index: int, validation: bool) -> None: + def set_validation(self, point: CalibrationPoint, validation: bool) -> bool: """Hold a point out of the fit and use it to test the fit instead.""" with self._lock: - if not 0 <= index < len(self.points): - return - self.points[index].is_validation = bool(validation) + target = self._resolve(point) + if target is None: + return False + target.is_validation = bool(validation) if validation: - self.points[index].excluded = False + target.excluded = False + return True + + def remove_point(self, point: CalibrationPoint) -> bool: + """Drop a point entirely (used by "delete row").""" + with self._lock: + target = self._resolve(point) + if target is None: + return False + self.points.remove(target) + return True + + def clear_points(self, channel: Optional[ChannelKey] = None) -> int: + """Drop every point, or every point of one channel. Returns the count.""" + with self._lock: + if channel is None: + removed = len(self.points) + self.points = [] + return removed + kept = [point for point in self.points if point.channel != channel] + removed = len(self.points) - len(kept) + self.points = kept + return removed def points_for(self, channel: ChannelKey) -> list[CalibrationPoint]: with self._lock: diff --git a/shared/src/orisys_app_shared/record/__init__.py b/shared/src/orisys_app_shared/record/__init__.py index df4cba5..9bdbd9b 100644 --- a/shared/src/orisys_app_shared/record/__init__.py +++ b/shared/src/orisys_app_shared/record/__init__.py @@ -1,5 +1,6 @@ -"""Experiment recording (video + data + metadata).""" +"""Experiment recording (video + data + metadata) and replay.""" +from orisys_app_shared.record.depth import DepthRecomputer, DepthResult from orisys_app_shared.record.experiment import ( EVENTS_HEADER, FORCES_HEADER, @@ -11,15 +12,27 @@ from orisys_app_shared.record.experiment import ( RecordStats, format_value, ) +from orisys_app_shared.record.playback import ( + RecordingFrame, + RecordingInfo, + RecordingReader, + list_recordings, +) __all__ = [ "EVENTS_HEADER", "FORCES_HEADER", "FRAMES_HEADER", "RECORD_FORMAT_VERSION", + "DepthRecomputer", + "DepthResult", "ExperimentRecorder", "RecordConfig", "RecordReport", "RecordStats", + "RecordingFrame", + "RecordingInfo", + "RecordingReader", "format_value", -] + "list_recordings", +] \ No newline at end of file diff --git a/shared/src/orisys_app_shared/record/depth.py b/shared/src/orisys_app_shared/record/depth.py new file mode 100644 index 0000000..61875b8 --- /dev/null +++ b/shared/src/orisys_app_shared/record/depth.py @@ -0,0 +1,174 @@ +"""Recompute depth for a recording, offline. + +Depth is not stored with a recording (a per-frame depth stream would be several +GB per hour). It is a function of the frame and the sensor configuration, so a +replay re-runs the SDK over ``raw_*.mp4`` using the ``config_ref`` recorded in +``session.json``. + +The honest failure modes are all surfaced rather than papered over: + +* the SDK is not importable, or the recording's sensor config is gone; +* the recording predates the recorded config hash, so the recomputation is not + guaranteed to reproduce what was shown live; +* the segment cannot be decoded. + +The caller is expected to show the returned ``error`` and to mark the depth pane +as unavailable instead of substituting some other image. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import numpy as np + +from orisys_app_shared.record.playback import RecordingReader + +logger = logging.getLogger(__name__) + + +@dataclass +class DepthResult: + """Outcome of one recomputation attempt.""" + + depth: Optional[np.ndarray] = None + error: str = "" + frame_id: int = -1 + t_rel_s: float = 0.0 + + @property + def ok(self) -> bool: + return self.depth is not None and not self.error + + +class DepthRecomputer: + """Recompute depth maps for one recording, on demand.""" + + def __init__(self, reader: RecordingReader) -> None: + self.reader = reader + self._sensor = None + self._segment: Optional[Path] = None + self.last_error = "" + + # ------------------------------------------------------------------ probing + @staticmethod + def sdk_available() -> tuple[bool, str]: + try: + import orisys # noqa: F401 + + return True, str(getattr(__import__("orisys"), "__version__", "unknown")) + except Exception as exc: # noqa: BLE001 + return False, f"{type(exc).__name__}: {exc}" + + def config_ref(self) -> str: + session = self.reader.session() + device = session.get("device") or {} + return str(device.get("config_ref") or "") + + def preflight(self) -> str: + """Return an empty string when recomputation can be attempted.""" + ok, detail = self.sdk_available() + if not ok: + return f"sdk_unavailable: {detail}" + config_ref = self.config_ref() + if not config_ref: + return "config_missing" + if not Path(config_ref).is_file(): + return "config_file_missing" + if not self.reader.segments(): + return "no_video" + return "" + + # ----------------------------------------------------------------- recompute + def depth_at(self, t_rel_s: float) -> DepthResult: + """Recompute the depth map for the frame at ``t_rel_s``.""" + problem = self.preflight() + if problem: + self.last_error = problem + return DepthResult(error=problem, t_rel_s=float(t_rel_s)) + + frame = self.reader.frame_at(t_rel_s) + if frame is None: + return DepthResult(error="no_frame", t_rel_s=float(t_rel_s)) + if not frame.in_video: + # The queue dropped this frame; say so rather than show the previous + # frame as if it belonged to the cursor position. + return DepthResult( + error="frame_not_in_video", frame_id=frame.frame_id, t_rel_s=frame.t_rel_s + ) + + segment = self.reader.segment_path(frame.segment) + if segment is None: + return DepthResult( + error="segment_missing", frame_id=frame.frame_id, t_rel_s=frame.t_rel_s + ) + try: + depth = self._depth_from_segment(segment, frame.segment_frame) + except Exception as exc: # noqa: BLE001 - reported to the caller + logger.exception("Depth recomputation failed") + self.last_error = f"{type(exc).__name__}: {exc}" + return DepthResult( + error=self.last_error, frame_id=frame.frame_id, t_rel_s=frame.t_rel_s + ) + if depth is None: + self.last_error = "depth_not_available" + return DepthResult( + error=self.last_error, frame_id=frame.frame_id, t_rel_s=frame.t_rel_s + ) + self.last_error = "" + return DepthResult(depth=depth, frame_id=frame.frame_id, t_rel_s=frame.t_rel_s) + + def _depth_from_segment(self, segment: Path, segment_frame: int) -> Optional[np.ndarray]: + """Open the segment with the SDK and read the depth channel.""" + import orisys + + if self._segment != segment or self._sensor is None: + self._close() + self._sensor = orisys.Sensor( + str(segment), + config_name=self._config_name(), + verbose=False, + backend="auto", + ) + self._segment = segment + sensor = self._sensor + # The SDK's video source advances frame by frame, so a seek means + # stepping. Correctness first: we never return a frame we did not read. + target = max(1, int(segment_frame)) + last_depth = None + for _ in range(target): + image = sensor.get_img() + if image is None: + break + info = sensor.info + if not hasattr(info, "DEPTH"): + return None + last_depth = sensor.read_info(info.DEPTH) + return None if last_depth is None else np.asarray(last_depth) + + def _config_name(self) -> str: + config_ref = self.config_ref() + path = Path(config_ref) + return path.parent.name if path.name == "sensor.json" else path.stem + + def _close(self) -> None: + sensor, self._sensor = self._sensor, None + self._segment = None + if sensor is None: + return + try: + sensor.disconnect() + except Exception: # noqa: BLE001 + pass + + def close(self) -> None: + self._close() + + def __enter__(self) -> "DepthRecomputer": + return self + + def __exit__(self, *_exc) -> None: + self.close() diff --git a/shared/src/orisys_app_shared/record/playback.py b/shared/src/orisys_app_shared/record/playback.py new file mode 100644 index 0000000..0882981 --- /dev/null +++ b/shared/src/orisys_app_shared/record/playback.py @@ -0,0 +1,328 @@ +"""Read a recorded experiment back: forces, the frame index and the video. + +Replay policy for depth (decided with the team): **depth is not stored**. It is +a function of the frame and the sensor configuration, so it is recomputed from +``raw_*.mp4`` with the config recorded in ``session.json``. That keeps a +recording small at the cost of requiring the SDK to reproduce the same result, +which is why :meth:`RecordingReader.replay_warnings` checks that the config hash +and SDK version still match what was recorded. + +Nothing here touches Qt, so the reader is testable without a display. +""" + +from __future__ import annotations + +import csv +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterator, Optional + +import numpy as np + +logger = logging.getLogger(__name__) + +#: Column names as written by the recorder (index == position in forces.csv). +FORCE_COLUMNS = ( + "session_id", + "frame_id", + "t_rel_s", + "FN_raw", + "FX_raw", + "FY_raw", + "FN_N", + "FX_N", + "FY_N", + "valid", + "calibration_id", +) + + +@dataclass(frozen=True) +class RecordingFrame: + """One row of ``frames.csv`` — the mapping from a frame to the video.""" + + frame_id: int + t_rel_s: float + acquired_at: str + segment: int + segment_frame: int + in_video: bool + + +@dataclass +class RecordingInfo: + directory: Path + session: dict + frames: list[RecordingFrame] + times: np.ndarray + raw: np.ndarray + newtons: np.ndarray + valid: np.ndarray + frame_ids: np.ndarray + dropped_rows: int = 0 + + @property + def duration_s(self) -> float: + if self.times.size == 0: + return 0.0 + return float(self.times[-1]) + + def value_for_unit(self, unit: str) -> np.ndarray: + if str(unit).upper().startswith("N"): + return self.newtons + return self.raw + + +class RecordingReader: + """Open one experiment directory and serve replay queries from it.""" + + def __init__(self, directory: str | Path) -> None: + self.directory = Path(directory) + if not self.directory.is_dir(): + raise FileNotFoundError(f"Not a recording directory: {self.directory}") + self._info: Optional[RecordingInfo] = None + + # ------------------------------------------------------------------ loading + def info(self) -> RecordingInfo: + if self._info is None: + self._info = self._load() + return self._info + + def session(self) -> dict: + path = self.directory / "session.json" + if not path.is_file(): + return {} + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: # noqa: BLE001 - a broken session.json must not crash replay + logger.exception("Could not read %s", path) + return {} + + def _load(self) -> RecordingInfo: + frames = self._load_frames() + times, raw, newtons, valid, frame_ids, dropped = self._load_forces() + return RecordingInfo( + directory=self.directory, + session=self.session(), + frames=frames, + times=times, + raw=raw, + newtons=newtons, + valid=valid, + frame_ids=frame_ids, + dropped_rows=dropped, + ) + + def _load_frames(self) -> list[RecordingFrame]: + path = self.directory / "frames.csv" + if not path.is_file(): + return [] + rows: list[RecordingFrame] = [] + with path.open(encoding="utf-8", newline="") as handle: + for row in csv.DictReader(handle): + try: + rows.append( + RecordingFrame( + frame_id=int(row["frame_id"]), + t_rel_s=float(row["t_rel_s"]), + acquired_at=row.get("acquired_at", ""), + segment=int(row.get("segment") or 0), + segment_frame=int(row.get("segment_frame") or 0), + in_video=row.get("in_video") == "1", + ) + ) + except (KeyError, TypeError, ValueError): + continue + return rows + + def _load_forces(self): + path = self.directory / "forces.csv" + if not path.is_file(): + empty = np.empty(0, dtype=np.float64) + return empty, np.empty((0, 3)), np.empty((0, 3)), np.empty(0, bool), np.empty(0, int), 0 + + def parse(text: str) -> float: + text = (text or "").strip() + if not text: + # An empty cell means "not available"; it must never become 0. + return float("nan") + try: + return float(text) + except ValueError: + return float("nan") + + times: list[float] = [] + raw: list[tuple[float, float, float]] = [] + newtons: list[tuple[float, float, float]] = [] + valid: list[bool] = [] + frame_ids: list[int] = [] + dropped = 0 + with path.open(encoding="utf-8", newline="") as handle: + for row in csv.DictReader(handle): + try: + times.append(float(row["t_rel_s"])) + frame_ids.append(int(row["frame_id"])) + except (KeyError, TypeError, ValueError): + dropped += 1 + continue + raw.append( + (parse(row.get("FN_raw")), parse(row.get("FX_raw")), parse(row.get("FY_raw"))) + ) + newtons.append( + (parse(row.get("FN_N")), parse(row.get("FX_N")), parse(row.get("FY_N"))) + ) + valid.append((row.get("valid") or "0").strip() == "1") + return ( + np.asarray(times, dtype=np.float64), + np.asarray(raw, dtype=np.float64).reshape(-1, 3), + np.asarray(newtons, dtype=np.float64).reshape(-1, 3), + np.asarray(valid, dtype=bool), + np.asarray(frame_ids, dtype=np.int64), + dropped, + ) + + # ------------------------------------------------------------------ queries + def frame_at(self, t_rel_s: float) -> Optional[RecordingFrame]: + """The recorded frame at or just before ``t_rel_s``.""" + frames = self.info().frames + if not frames: + return None + times = [frame.t_rel_s for frame in frames] + index = int(np.searchsorted(np.asarray(times), float(t_rel_s), side="right")) - 1 + index = max(0, min(index, len(frames) - 1)) + return frames[index] + + def segment_path(self, segment: int) -> Optional[Path]: + if segment <= 0: + return None + path = self.directory / f"raw_{segment:04d}.mp4" + return path if path.is_file() else None + + def segments(self) -> list[Path]: + return sorted(self.directory.glob("raw_*.mp4")) + + def video_frame_at(self, t_rel_s: float): + """Decode the video frame for a replay position. + + Returns ``(frame_bgr, RecordingFrame)`` or ``(None, frame)`` when the + frame was not written to video (the queue dropped it). + """ + import cv2 + + frame = self.frame_at(t_rel_s) + if frame is None or not frame.in_video: + return None, frame + path = self.segment_path(frame.segment) + if path is None: + return None, frame + capture = cv2.VideoCapture(str(path)) + if not capture.isOpened(): + return None, frame + try: + capture.set(cv2.CAP_PROP_POS_FRAMES, max(0, frame.segment_frame - 1)) + ok, image = capture.read() + return (image if ok else None), frame + finally: + capture.release() + + def forces_window(self, t0: float, t1: float) -> tuple[np.ndarray, np.ndarray]: + """``(times, values)`` inside ``[t0, t1]`` for the raw triplet.""" + info = self.info() + if info.times.size == 0: + return info.times, info.raw + mask = (info.times >= float(t0)) & (info.times <= float(t1)) + return info.times[mask], info.raw[mask] + + # ------------------------------------------------------------------ policy + def depth_policy(self) -> dict: + return dict(self.info().session.get("depth") or {}) + + def depth_is_stored(self) -> bool: + return bool(self.depth_policy().get("stored", False)) + + def recorded_config_ref(self) -> str: + device = self.info().session.get("device") or {} + return str(device.get("config_ref") or "") + + @staticmethod + def config_sha256(config_ref: str) -> str: + """Hash of the config file currently on disk (empty when unreadable).""" + if not config_ref: + return "" + try: + import hashlib + + path = Path(str(config_ref)) + if path.is_file(): + return hashlib.sha256(path.read_bytes()).hexdigest() + except Exception: # noqa: BLE001 - a missing config is reported, not raised + return "" + return "" + + def replay_warnings(self, *, current_config_sha: str = "", sdk_version: str = "") -> list[str]: + """What could stop this recording from replaying faithfully. + + Depth is recomputed rather than stored, so a changed config or SDK means + the depth seen on replay may differ from what was shown live. That must + be stated, not discovered later. + """ + session = self.info().session + warnings: list[str] = [] + recorded_sha = str((session.get("device") or {}).get("config_sha256") or "") + if not current_config_sha: + # Compare against the config file as it is now, which is exactly what + # the recomputation would use. + current_config_sha = self.config_sha256(self.recorded_config_ref()) + if recorded_sha and current_config_sha and recorded_sha != current_config_sha: + warnings.append("config_changed") + versions = session.get("versions") or {} + recorded_sdk = str(versions.get("sdk") or "") + if recorded_sdk and sdk_version and recorded_sdk != sdk_version: + warnings.append("sdk_changed") + if recorded_sha and not current_config_sha: + warnings.append("config_missing") + if not self.depth_is_stored(): + warnings.append("depth_recomputed") + if self.info().dropped_rows: + warnings.append("rows_skipped") + return warnings + + def iter_forces(self, batch: int = 4096) -> Iterator[tuple[np.ndarray, np.ndarray]]: + """Stream the force rows, so a long recording is not read twice.""" + path = self.directory / "forces.csv" + if not path.is_file(): + return + times: list[float] = [] + values: list[tuple[float, float, float, float, float, float]] = [] + with path.open(encoding="utf-8", newline="") as handle: + for row in csv.DictReader(handle): + try: + times.append(float(row["t_rel_s"])) + except (KeyError, TypeError, ValueError): + continue + values.append( + tuple( + float(row[key]) if (row.get(key) or "").strip() else float("nan") + for key in ("FN_raw", "FX_raw", "FY_raw", "FN_N", "FX_N", "FY_N") + ) + ) + if len(times) >= batch: + yield np.asarray(times), np.asarray(values).reshape(-1, 6) + times, values = [], [] + if times: + yield np.asarray(times), np.asarray(values).reshape(-1, 6) + + +def list_recordings(root: str | Path) -> list[Path]: + """Every recording directory under ``root``, newest first.""" + base = Path(root) + if not base.is_dir(): + return [] + candidates = [ + path + for path in base.iterdir() + if path.is_dir() and (path / "forces.csv").is_file() + ] + return sorted(candidates, key=lambda path: path.name, reverse=True) diff --git a/shared/tests/test_playback.py b/shared/tests/test_playback.py new file mode 100644 index 0000000..01d9423 --- /dev/null +++ b/shared/tests/test_playback.py @@ -0,0 +1,269 @@ +"""Replay of a recording: the reader, and the depth replay policy. + +Depth is not stored with a recording (team decision): replay recomputes it from +``raw_*.mp4`` and the recorded config. These tests cover the reader contract and +that the policy's risks are reported instead of discovered later. +""" + +from __future__ import annotations + +import csv +import json +import shutil +import sys +import types +from pathlib import Path + +import numpy as np +import pytest + +from orisys_app_shared.acquisition.sdk_worker import SdkPacket +from orisys_app_shared.record import ( + ExperimentRecorder, + RecordConfig, + RecordingReader, + list_recordings, +) +from orisys_app_shared.record.depth import DepthRecomputer + +SCRATCH = Path(__file__).resolve().parent / "_scratch_playback" + + +def _packet(index: int, *, valid: bool = True) -> SdkPacket: + nan = float("nan") + frame = np.full((24, 32, 3), (index * 5) % 255, np.uint8) + return SdkPacket( + raw_frame=frame, + processed_frame=frame, + timestamp=1000.0 + index / 30.0, + fps=30.0, + process_ms=1.0, + normal_force=2.0 if valid else nan, + shear_x=0.5 if valid else nan, + shear_y=-0.5 if valid else nan, + normal_force_raw=(9.0 + index * 0.1) if valid else nan, + shear_x_raw=0.4 if valid else nan, + shear_y_raw=-0.4 if valid else nan, + force_unit="N", + force_calibrated=True, + session_id=1, + frame_id=index, + valid=valid, + invalid_reason="" if valid else "read failed", + raw_available=True, + ) + + +@pytest.fixture +def recording(): + shutil.rmtree(SCRATCH, ignore_errors=True) + SCRATCH.mkdir(parents=True, exist_ok=True) + recorder = ExperimentRecorder(RecordConfig(video=True, fps=30.0, segment_seconds=0.4)) + directory = recorder.start( + root=SCRATCH, + session_id=3, + calibration_id="/cfg/sensor.json", + metadata={ + "device": { + "sensor_type": "box", + "config_ref": str(SCRATCH / "sensor.json"), + "config_sha256": "recorded-hash", + }, + "versions": {"sdk": "0.5.1"}, + "depth": {"stored": False, "replay": "recompute"}, + }, + ) + (SCRATCH / "sensor.json").write_text('{"camera": {}}\n', encoding="utf-8") + for index in range(60): + recorder.on_packet(_packet(index)) + recorder.on_packet(_packet(90, valid=False)) + recorder.stop() + yield directory + shutil.rmtree(SCRATCH, ignore_errors=True) + + +def test_reader_reports_the_recording_shape(recording): + reader = RecordingReader(recording) + info = reader.info() + + assert len(info.frames) == 61 + assert info.times.size == 61 + assert info.duration_s == pytest.approx(3.0, abs=0.05) + assert int(info.valid.sum()) == 60, "the failed readout should be marked invalid" + + +def test_reader_keeps_unavailable_values_empty(recording): + reader = RecordingReader(recording) + info = reader.info() + assert np.isnan(info.raw[-1]).all(), "an empty CSV cell must not read back as 0" + assert np.isfinite(info.raw[0]).all() + + +def test_reader_maps_time_to_the_video_segment(recording): + reader = RecordingReader(recording) + frame = reader.frame_at(1.0) + + assert frame is not None + assert frame.frame_id == 30 + assert frame.t_rel_s == pytest.approx(1.0, abs=0.02) + assert frame.in_video + assert frame.segment >= 1 + assert reader.segment_path(frame.segment) is not None + # Before the start and past the end clamp instead of raising. + assert reader.frame_at(-5.0).frame_id == 0 + assert reader.frame_at(500.0).frame_id == 90 + + +def test_reader_decodes_the_video_frame(recording): + import cv2 + + if not hasattr(cv2, "VideoCapture"): # pragma: no cover + pytest.skip("no video backend") + reader = RecordingReader(recording) + image, frame = reader.video_frame_at(1.0) + + assert frame is not None and frame.in_video + assert image is not None, "the frame index and the video disagreed" + assert image.shape[:2] == (24, 32) + + +def test_reader_window_and_streaming_agree(recording): + reader = RecordingReader(recording) + times, values = reader.forces_window(0.5, 1.0) + assert len(times) == 16 + assert values.shape == (16, 3) + + streamed = sum(len(batch_times) for batch_times, _ in reader.iter_forces(batch=25)) + assert streamed == 61, "streaming must see every row" + + +def test_depth_policy_says_recompute_and_warns_about_drift(recording): + reader = RecordingReader(recording) + + assert reader.depth_is_stored() is False + assert reader.depth_policy().get("replay") == "recompute" + assert reader.recorded_config_ref() == str(SCRATCH / "sensor.json") + + # The config file on disk no longer matches what was recorded. + codes = reader.replay_warnings(sdk_version="0.5.1") + assert "config_changed" in codes + assert "depth_recomputed" in codes + + # Same SDK: no version warning. + assert "sdk_changed" not in codes + assert "sdk_changed" in reader.replay_warnings(sdk_version="9.9.9") + + +def test_depth_recompute_preflight_states_why_it_cannot_run(recording): + reader = RecordingReader(recording) + recomputer = DepthRecomputer(reader) + try: + problem = recomputer.preflight() + assert problem in ("", "sdk_unavailable"), problem + + # A recording with no config reference is refused, not guessed at. + missing = RecordingReader(recording) + missing._info = None # noqa: SLF001 - force a re-read below + empty = SCRATCH / "empty" + empty.mkdir(exist_ok=True) + (empty / "forces.csv").write_text( + ",".join(("session_id", "frame_id", "t_rel_s", "FN_raw", "FX_raw", "FY_raw", + "FN_N", "FX_N", "FY_N", "valid", "calibration_id")) + "\n", + encoding="utf-8", + ) + (empty / "session.json").write_text(json.dumps({"depth": {"stored": False}}), encoding="utf-8") + bare = DepthRecomputer(RecordingReader(empty)) + assert bare.preflight() in ("config_missing", "no_video", "sdk_unavailable") + finally: + recomputer.close() + + +def test_depth_result_is_failed_not_blank_when_sdk_is_missing(recording): + """A missing SDK must produce an explicit error, never a stand-in image.""" + saved = sys.modules.get("orisys") + blocker = types.ModuleType("orisys") + try: + # A module object with no Sensor/DEPTH surface: importing succeeds but + # nothing can be recomputed with it. + sys.modules["orisys"] = blocker + reader = RecordingReader(recording) + recomputer = DepthRecomputer(reader) + problem = recomputer.preflight() + assert problem in ("sdk_unavailable", ""), problem + if problem: + result = recomputer.depth_at(1.0) + assert not result.ok + assert result.depth is None + assert result.error + finally: + if saved is None: + sys.modules.pop("orisys", None) + else: + sys.modules["orisys"] = saved + + +def test_list_recordings_finds_directories_newest_first(recording): + second = SCRATCH / "zzz_later" + second.mkdir() + (second / "forces.csv").write_text("", encoding="utf-8") + + found = list_recordings(SCRATCH) + assert found[0].name == "zzz_later", "recordings should be newest first" + assert recording in found + + +if __name__ == "__main__": # pragma: no cover - manual runner + import shutil as _shutil + + failures = 0 + for name, function in sorted(globals().items()): + if not name.startswith("test_") or not callable(function): + continue + try: + if name == "test_list_recordings_finds_directories_newest_first": + _shutil.rmtree(SCRATCH, ignore_errors=True) + SCRATCH.mkdir(parents=True, exist_ok=True) + recorder = ExperimentRecorder(RecordConfig(video=False)) + directory = recorder.start( + root=SCRATCH, session_id=1, metadata={"depth": {"stored": False}} + ) + recorder.on_packet(_packet(0)) + recorder.stop() + try: + function(directory) + finally: + _shutil.rmtree(SCRATCH, ignore_errors=True) + print(f" ok {name}") + continue + _shutil.rmtree(SCRATCH, ignore_errors=True) + SCRATCH.mkdir(parents=True, exist_ok=True) + recorder = ExperimentRecorder(RecordConfig(video=True, fps=30.0, segment_seconds=0.4)) + directory = recorder.start( + root=SCRATCH, + session_id=3, + calibration_id="/cfg/sensor.json", + metadata={ + "device": { + "sensor_type": "box", + "config_ref": str(SCRATCH / "sensor.json"), + "config_sha256": "recorded-hash", + }, + "versions": {"sdk": "0.5.1"}, + "depth": {"stored": False, "replay": "recompute"}, + }, + ) + (SCRATCH / "sensor.json").write_text('{"camera": {}}\n', encoding="utf-8") + for index in range(60): + recorder.on_packet(_packet(index)) + recorder.on_packet(_packet(90, valid=False)) + recorder.stop() + try: + function(directory) + finally: + _shutil.rmtree(SCRATCH, ignore_errors=True) + print(f" ok {name}") + except Exception as exc: # noqa: BLE001 + failures += 1 + print(f" FAIL {name}: {type(exc).__name__}: {exc}") + print(f"failures: {failures}") + raise SystemExit(1 if failures else 0) diff --git a/shared/tests/test_record_channel.py b/shared/tests/test_record_channel.py index 8061078..f5d8b7b 100644 --- a/shared/tests/test_record_channel.py +++ b/shared/tests/test_record_channel.py @@ -292,6 +292,58 @@ def test_recorder_cancel_discards_the_point(): assert recorder.points == [] +def test_recorder_diagnostics_use_point_identity_not_table_rows(): + """A per-channel table row is not an index into the cross-channel list. + + The panel shows the points of one channel while the recorder stores every + channel in a single list, so an integer index resolves to the wrong point as + soon as a second channel has been recorded. + """ + recorder = _recorder() + for channel, load, raw in (("fn", 1.0, 100.0), ("fn", 2.0, 200.0), ("fx", 1.0, 7.0)): + recorder.start(channel=channel, reference_load_n=load, duration_s=1.0) + for index in range(10): + sample = _sample(index) + sample = SessionSample( + frame_id=index, + t_rel_s=index / 100.0, + fn_raw=raw if channel == "fn" else 0.4, + fx_raw=raw if channel == "fx" else 0.4, + fy_raw=-0.2, + valid=True, + ) + recorder.feed(sample) + recorder.finish_now() + + fx_point = recorder.points_for("fx")[0] + assert recorder.set_excluded(fx_point, True) is True + assert [point.excluded for point in recorder.all_points()] == [False, False, True], ( + "excluding the FX point must not touch the FN points" + ) + + # A point that was never recorded is rejected rather than silently ignored. + from orisys_app_shared.force.calibration_recorder import CalibrationPoint + + stranger = CalibrationPoint(channel="fn", reference_load_n=9.0) + assert recorder.set_excluded(stranger, True) is False + assert recorder.set_validation(stranger, True) is False + assert recorder.remove_point(stranger) is False + + +def test_clear_points_can_target_one_channel(): + recorder = _recorder() + for channel in ("fn", "fx", "fn"): + recorder.start(channel=channel, reference_load_n=1.0, duration_s=1.0) + recorder.feed(_sample(0)) + recorder.feed(_sample(1)) + recorder.finish_now() + + assert recorder.clear_points("fn") == 2 + assert [point.channel for point in recorder.all_points()] == ["fx"] + assert recorder.clear_points() == 1 + assert recorder.all_points() == [] + + def test_recorder_round_index_advances_per_channel(): recorder = _recorder() for _ in range(2):