record the full experiment: video, CSV and metadata

Phase 1 of the review asked for one directory per experiment, a force CSV,
video of the pre-processing image, and metadata that ties them together. The
record channel from the previous change already kept the whole session; this
adds the on-disk format.

What
- Add record/experiment.py: ExperimentRecorder writes one directory per
  session containing session.json, forces.csv, frames.csv, events.csv,
  raw_NNNN.mp4 (segmented) and an optional lossless PNG sequence.
- forces.csv uses the reviewed column order
  (session_id, frame_id, t_rel_s, FN_raw, FX_raw, FY_raw, FN_N, FX_N, FY_N,
  valid, calibration_id) and leaves a cell empty when a value is unavailable:
  an uncalibrated newton value or a failed readout is never written as 0.
- frames.csv is the frame-index mapping: frame_id, t_rel_s, acquisition wall
  clock, segment, segment_frame, in_video. forces.csv and frames.csv share the
  packet frame_id and the same t_rel_s origin, so video and data can be
  aligned after dropped or segmented frames.
- session.json carries format version, session id, start/end time in UTC,
  device and config (with a sha256 of the config file), app/shared/SDK
  versions, calibration reference, axis convention, filter parameters, video
  codec and the lossy-MP4 caveat, plus the event log and end-of-run statistics.
  It is written once at start so a crash still leaves identifiable metadata.
- Add a record control to the toolbar: mode (video+data / data only) and a
  start/stop toggle that stays enabled while recording.
- Log zeroing, calibration application and session end as timestamped events.

Why
- The recorder is fed from the worker sink list on the acquisition thread and
  only appends to a bounded queue, so disk I/O can never stall acquisition.
- On_packet copies the raw frame instead of holding a reference: the packet
  frame may alias a buffer the SDK reuses between frames.
- A queue overflow is counted and alarmed (rate limited), and a write or encode
  failure is counted, reported and after 5 consecutive errors stops the
  recording with failed=true rather than silently losing data.

Verification
- shared/tests/test_experiment_recorder.py: 12 cases covering the directory
  layout, the CSV contract, frame-id/time-base agreement, segment rolling,
  data-only and lossless modes, queue overflow accounting, encoder failure,
  metadata fields, crash-before-stop, and that on_packet never blocks.
- tools/ui_check.py gains an "experiment record" section (169 checks, 0
  failures overall), driving a recording through the UI and asserting the
  layout, the empty-cell rule and the event sequence.
- apps/studio/tests/test_ui_smoke.py mirrors those invariants.
- docs/record-channel.md documents the format and the remaining phase-2 work.

Not in this change: CSV export of a selected time range, full-session curve
browsing, calibration diagnostics (outlier exclusion, fit residuals) and
PyOpenGL 3D.
This commit is contained in:
ykcheungab
2026-09-13 13:14:10 +08:00
parent f501fa93f3
commit 817f830565
12 changed files with 1531 additions and 16 deletions
+88 -1
View File
@@ -2,6 +2,8 @@
from __future__ import annotations
from pathlib import Path
import pytest
import ui_check
@@ -197,7 +199,7 @@ def test_toolbar_run_button_is_labelled(make_window):
window = _show(make_window(), (1400, 880))
label = window.toolbar.btn_run.text()
assert label.strip()
assert label != "â–"
assert label != "â–Â"
window.toolbar.set_run_checked(True)
assert window.toolbar.btn_run.text() != label
@@ -429,3 +431,88 @@ def test_depth_view_reports_missing_data_and_fixes_its_scale(make_window):
)
_finish(window, stuck)
# ------------------------------------------------------------- experiment record
#: Recording scratch goes under the app's gitignored ``outputs`` tree rather than
#: the system temp directory, so the tests also run under a file sandbox that
#: only permits writes inside the workspace.
RECORD_SCRATCH = Path(__file__).resolve().parents[1] / "outputs" / "_test_record"
@pytest.fixture
def record_root():
import shutil
shutil.rmtree(RECORD_SCRATCH, ignore_errors=True)
RECORD_SCRATCH.mkdir(parents=True, exist_ok=True)
yield RECORD_SCRATCH
shutil.rmtree(RECORD_SCRATCH, ignore_errors=True)
def test_recording_writes_the_documented_experiment_layout(make_window, record_root):
import csv
import json
import orisys_studio.app_window as app_module
from orisys_app_shared.i18n import t
window, stuck = _running_window(make_window)
app_module.recordings_dir = lambda: record_root
assert window.toolbar.btn_record.isEnabled()
assert window.toolbar.btn_record.text() == t("record.start_label")
window.toolbar.btn_record.setChecked(True)
assert window.recorder.is_recording, "the recording did not start"
assert window.toolbar.btn_record.text() == t("record.stop_label")
assert not window.toolbar.record_mode_combo.isEnabled()
for index in range(20):
stuck.emit(ui_check._fake_packet(index))
window._on_run_tick()
window.toolbar.btn_record.setChecked(False)
report = window._record_last_report
assert report is not None and report.ok, getattr(report, "problems", None)
names = sorted(path.name for path in report.directory.iterdir())
assert names == ["events.csv", "forces.csv", "frames.csv", "raw_0001.mp4", "session.json"]
with (report.directory / "forces.csv").open(encoding="utf-8") as handle:
forces = list(csv.DictReader(handle))
with (report.directory / "frames.csv").open(encoding="utf-8") as handle:
frames = list(csv.DictReader(handle))
assert len(forces) == len(frames) == 20
assert [f["frame_id"] for f in frames] == [r["frame_id"] for r in forces]
assert [f["t_rel_s"] for f in frames] == [r["t_rel_s"] for r in forces]
meta = json.loads((report.directory / "session.json").read_text(encoding="utf-8"))
assert meta["ended_at"]
assert meta["format_version"]
assert meta["stats"]["frames"] == 20
_finish(window, stuck)
def test_recording_leaves_unavailable_values_empty(make_window, record_root):
import csv
import orisys_studio.app_window as app_module
window, stuck = _running_window(make_window)
app_module.recordings_dir = lambda: record_root
window.toolbar.btn_record.setChecked(True)
stuck.emit(ui_check._fake_packet(0))
window._on_run_tick()
stuck.emit(ui_check._fake_packet(1, valid=False, reason="read failed"))
window._on_run_tick()
window.toolbar.btn_record.setChecked(False)
with (window._record_last_report.directory / "forces.csv").open(encoding="utf-8") as handle:
rows = list(csv.DictReader(handle))
failed = rows[-1]
assert failed["valid"] == "0"
assert failed["FN_raw"] == "" and failed["FN_N"] == "", (
"a failed readout was persisted as a number"
)
_finish(window, stuck)