Browse a long recording from disk instead of loading it into memory

What
- record/index.py: RecordingIndex streams forces.csv once and keeps only the
  small things — one time, one validity flag, one frame id per row plus a byte
  offset every 4096 rows (~6.5 MB per 500k rows). row_range() bisects the times,
  read_rows() seeks to the nearest checkpoint and scans forward, and overview()
  folds every row into its bucket during the same pass to build a per-axis
  min/max envelope.
- read_rows(max_points, unit) decimates **while streaming**: each bucket keeps
  the extreme row of every axis in the requested unit, so a long interval never
  has to exist in memory in full and a shear spike survives a flat normal channel.
- RecordingReader: info() decimates above max_rows (default 200k) and marks
  itself decimated, with total_rows and a duration taken from the frame index
  rather than from the view; window()/window_all()/overview()/statistics() read
  the interval being asked about. list/window semantics unchanged for small
  recordings.
- force/source.py: StoreCurveSource (live memory) and RecordingCurveSource (disk
  interval + cached envelope) implement one interface — extent, window,
  envelope, statistics, duration_s — so ForcePlotsPanel asks a *source* instead
  of knowing where the data lives.
- force_plots: reads through the source (window, envelope, statistics, extent),
  which also replaces its last direct store accesses.
- playback_dialog: draws the whole-session view through window_all() in the unit
  being shown (bounded by CHART_MAX_POINTS), and says so when the view is
  decimated ("显示 N / 共 M 行") instead of letting the operator count points.
- experiment._run: a failure that lands while the files are opening can no longer
  be overwritten by the writer's own transition to "recording" (found by a flaky
  test; the state check now covers STATE_FAILED).

Why
The store releases old blocks above ~4.6 h and the replay loaded every CSV row
into Python lists before drawing anything, so neither the live browser nor the
replay could show an earlier interval of a long recording. The index makes the
disk the data source and memory a cache of what is on screen.

Verification
- shared/tests/test_history_source.py (new, 8 cases) over a 12 000-row recording:
  the index reports its rows and reads a bounded interval; a large recording is
  marked decimated with its true duration and total rows; a small one is not; an
  interval read sees the exact rows including a single peak; a decimated window
  keeps the peak; the envelope fits the whole session in ≤200 points, keeps the
  spike and is cached; the disk and memory sources agree on extent, values and
  statistics; the whole session can be read without holding it all. Shared
  suites: 146 tests, 0 failures (three consecutive runs).
- apps/studio/tools/ui_check.py: the replay section now checks the index row
  count, that an interval read spans only the interval, the envelope size and
  total_rows. 324 checks, 0 failures.
- apps/studio/tests/test_ui_smoke.py: 43 tests, 36 pass; the 7 failures are the
  unchanged sandbox/harness artifacts.
- D:/orisys_ws/_review_phase12/reproduce.py: all ten lines unchanged;
  compileall, locale and prefs parity clean.
- apps/studio/docs/record-channel.md §8 documents the index, the reader API and
  the unified source, and lists what is still hardware work.

Not in this change
- Maximum RSS / backlog / loss measurements at real resolution on the target ARM
  board (needs the hardware), and range CSV export.
This commit is contained in:
ykcheungab
2026-09-13 16:54:26 +08:00
parent 22cc59c402
commit 73592f085f
40 changed files with 6398 additions and 1070 deletions
+531 -2
View File
@@ -289,7 +289,7 @@ def test_cal_panel_records_and_lists_samples(make_window):
]
def _add_calibration_point(window, *, load: float, raw: float, samples: int = 40) -> None:
def _add_calibration_point(window, *, load: float, raw: float, samples: int = 40):
"""Drive one point through the recorder and into the panel's table."""
from orisys_app_shared.force.session import SessionSample
@@ -305,7 +305,9 @@ def _add_calibration_point(window, *, load: float, raw: float, samples: int = 40
valid=True,
)
)
window.cal_panel.on_record_finished(window.cal_recorder.finish_now())
point = window.cal_recorder.finish_now()
window.cal_panel.on_record_finished(point)
return point
def test_calibration_diagnostics_drive_the_fit(make_window):
@@ -357,6 +359,64 @@ def test_calibration_diagnostics_drive_the_fit(make_window):
model = build_export_model({key: store.channel(key) for key in ("fn", "fx", "fy")})
assert abs(predict_channel(model, "fn", 400.0) - 2.0) > 0.05
# The store, the label and the export are the same preparation: the panel
# must not derive the model twice and show numbers for one of them.
prepared = window.cal_panel._prepared
assert prepared is not None and prepared.ok
assert list(store.channel("fn")) == list(prepared.samples["fn"])
assert prepared.points_in_fit("fn") == 3
def test_the_panel_shows_the_numbers_of_the_file_it_exports(make_window, record_root):
"""Displayed diagnostics must match the written coefficients.
A repeated reference load makes this sharp: the two samples of one load must
be merged before the build, so the exported curve and the panel's residual
and validation figures all come from that single prepared model.
"""
import json
from pathlib import Path
import orisys.calibration as sdk_calibration
from orisys_app_shared.force.calibration_io import build_force_calibration_v2
window = _show(make_window(), (1400, 880))
window.cal_panel.set_running(True)
window.cal_panel.set_meta_context(sensor_type="box", config_ref=None)
_add_calibration_point(window, load=1.0, raw=100.0)
_add_calibration_point(window, load=2.0, raw=200.0)
_add_calibration_point(window, load=2.0, raw=240.0) # a deliberate repeat
_add_calibration_point(window, load=4.0, raw=400.0)
_add_calibration_point(window, load=3.0, raw=290.0)
panel = window.cal_panel
panel.table.setCurrentCell(4, 0)
panel._on_toggle_validation()
panel._rebuild_store()
panel._refresh_fit_label()
prepared = panel._prepared
assert prepared is not None and prepared.ok
assert len(prepared.samples["fn"]) == 3, "the repeated load was not merged"
# Write the document the export would write, load it back with the SDK and
# check that its predictions are the prepared ones.
document, _warnings = build_force_calibration_v2(prepared)
path = Path(record_root) / "force_calibration.json"
path.write_text(json.dumps(document), encoding="utf-8")
loaded = sdk_calibration.load(path, strict=True)
for point in window.cal_recorder.points:
raw = float(point.loaded_stats().mean)
predicted = float(loaded.get("fn").convert(raw))
assert predicted == pytest.approx(prepared.predict("fn", raw), abs=1e-9)
# ...and the label describes that same model.
label = panel.fit_label.text()
assert "分段线性" in label or "piecewise" in label, label
assert "3" in label
assert prepared.max_validation_error("fn") > 0.0
def test_calibration_point_dialog_shows_three_axes(make_window):
window = _show(make_window(), (1400, 880))
@@ -373,6 +433,137 @@ def test_calibration_point_dialog_shows_three_axes(make_window):
dialog.close()
def _write_session_file(root, *, sensor_type, sn, config_ref, points):
import json
from orisys_app_shared.force.calibration_session import build_session_document
document = build_session_document(
points,
meta={"sensor_type": sensor_type, "sn": sn, "config_ref": config_ref},
)
path = Path(root) / f"calibration_session_{sensor_type}.json"
path.write_text(json.dumps(document), encoding="utf-8")
return path
def test_opening_device_a_record_keeps_device_a_identity(make_window, record_root):
"""Device A's record must not be restamped with device B's identity."""
import json
from unittest.mock import patch as _patch
import orisys_studio.ui.cal_panel as cal_panel_module
from orisys_app_shared.force.calibration_io import build_force_calibration_v2
window = _show(make_window(), (1400, 880))
panel = window.cal_panel
# A record taken on device A, with a full audit series.
session_points = [
_add_calibration_point(window, load=1.0, raw=100.0),
_add_calibration_point(window, load=3.0, raw=300.0),
]
source = _write_session_file(
record_root,
sensor_type="box",
sn="SN-A",
config_ref="D:/lab/deviceA/sensor.json",
points=session_points,
)
window.cal_recorder.clear_points()
# ...and the live context is device B.
panel.set_meta_context(sensor_type="finger", config_ref="D:/lab/deviceB/sensor.json")
panel.sn_edit.setText("SN-B")
panel._on_open_record(str(source))
assert panel.sn_edit.text() == "SN-A", "the record's SN was overwritten"
meta = panel._export_meta()
assert meta["sensor_type"] == "box"
assert meta["sn"] == "SN-A"
assert meta["config_ref"] == "D:/lab/deviceA/sensor.json"
assert meta["source_record"].endswith(source.name)
# Saving the record again still says A.
import orisys_studio.calibration_service as cal_service_module
with _patch.object(cal_panel_module, "calibration_export_dir", lambda: Path(record_root)):
with _patch.object(
cal_service_module, "calibration_export_dir", lambda: Path(record_root)
):
with _patch.object(cal_panel_module.QMessageBox, "information", lambda *a, **k: None):
panel._on_save_record()
saved = sorted(Path(record_root).glob("calibration_session_box_*.json"))
assert saved, "the record was not saved under device A's type"
saved_meta = json.loads(saved[-1].read_text(encoding="utf-8"))["meta"]
assert saved_meta["sn"] == "SN-A"
assert saved_meta["sensor_type"] == "box"
assert saved_meta["config_ref"] == "D:/lab/deviceA/sensor.json"
# ...and so does the exported coefficient document.
document, _warnings = build_force_calibration_v2(panel._prepared)
assert document["meta"]["sensor_type"] == "box"
assert document["meta"]["sn"] == "SN-A"
assert document["meta"]["config_ref"] == "D:/lab/deviceA/sensor.json"
def test_a_reopened_point_can_be_inspected_from_its_samples(make_window, record_root):
"""A record with samples but no cached trace still opens its diagnostics."""
window = _show(make_window(), (1400, 880))
panel = window.cal_panel
points = [
_add_calibration_point(window, load=1.0, raw=100.0),
_add_calibration_point(window, load=2.0, raw=200.0),
]
import json
from orisys_app_shared.force.calibration_session import build_session_document
document = build_session_document(points, meta={"sensor_type": "box"})
for entry in document["points"]:
entry["trace"] = {} # only the audit series survived
entry["trace_dt_s"] = 0.0
source = Path(record_root) / "no_trace.json"
source.write_text(json.dumps(document), encoding="utf-8")
panel._on_open_record(str(source))
reopened = window.cal_recorder.points[0]
assert reopened.samples, "the reopened point lost its samples"
dialog = panel.point_dialog(reopened)
assert dialog is not None, "the reopened point could not be inspected"
assert len(dialog.plot_widget.ci.items) == 3
dialog.close()
def test_applying_another_devices_calibration_is_flagged(make_window, record_root):
"""The apply path states a sensor/config mismatch instead of silently applying."""
import json
from orisys_app_shared.force.calibration_io import build_export_model, write_force_calibration
from orisys_app_shared.force.calibration_io import CalSample, CalSessionStore
window, stuck = _running_window(make_window)
store = CalSessionStore()
store.aggregate("fn", CalSample(weight_n=0.0, raw_mean=0.0, n=10))
store.aggregate("fn", CalSample(weight_n=5.0, raw_mean=500.0, n=10))
model = build_export_model({key: store.channel(key) for key in ("fn", "fx", "fy")})
document = model.to_dict()
document["identity"] = {"sensor_type": "box", "sn": "SN-A"}
document["pipeline"] = {"config_ref": "D:/lab/deviceA/sensor.json"}
path = Path(record_root) / "deviceA_calibration.json"
path.write_text(json.dumps(document), encoding="utf-8")
# The window is connected to a finger device.
mismatch = window.calibration_target_mismatch(
str(path), sensor_type="finger", config_ref="D:/lab/deviceB/sensor.json"
)
assert any("sensor_type" in reason for reason in mismatch), mismatch
assert any("config_ref" in reason for reason in mismatch), mismatch
assert not window.calibration_target_mismatch(
str(path), sensor_type="box", config_ref="D:/lab/deviceA/sensor.json"
)
_finish(window, stuck)
def test_force_plots_render_demo_data(make_window):
window = _show(make_window(), (1440, 880))
for i in range(60):
@@ -502,6 +693,344 @@ def test_history_keeps_negative_fn(make_window):
_finish(window, stuck)
def test_the_live_axis_is_session_relative_and_survives_a_unit_switch(make_window):
"""The live buffer, the store and a rebuilt history share one time base.
Feeding the wall clock into the plot put the live axis ~1.7e9 s away from the
session store's, so a unit switch (which rebuilds from the store) jumped the
X axis and the chart looked like it had lost its history.
"""
wall = 1_700_000_000.0
window, stuck = _running_window(make_window)
window.toolbar.units_combo.setCurrentIndex(0) # newton
window.force_plots.reset()
for index in range(90):
stuck.emit(
ui_check._fake_packet(
index, timestamp=wall + index / 30.0, mono_t=index / 30.0
)
)
window._on_run_tick()
before = window.force_plots.history.state()
assert before is not None
end_before = float(before.times[-1])
assert end_before < 60.0, (
f"the live X axis is not session-relative: it ends at {end_before}"
)
assert abs(end_before - 89 / 30.0) < 0.5, end_before
for units_index in (1, 0, 1, 0): # raw -> N -> raw -> N
window.toolbar.units_combo.setCurrentIndex(units_index)
after = window.force_plots.history.state()
assert after is not None, "the unit switch emptied the rebuilt history"
assert abs(float(after.times[-1]) - end_before) < 1.0, (
f"the X axis jumped across a unit switch: {end_before} -> {after.times[-1]}"
)
_finish(window, stuck)
def test_a_single_calibrated_axis_is_shown_and_the_others_are_unavailable(make_window):
"""Only FN calibrated: FN in newtons, FX/FY explicitly unavailable."""
import math
from orisys_app_shared.i18n import t
window, stuck = _running_window(make_window)
window.toolbar.units_combo.setCurrentIndex(0) # newton
window.force_plots.reset()
for index in range(10):
stuck.emit(
ui_check._fake_packet(
index,
calibrated_axes=("fn",),
shear_n=float("nan"),
)
)
window._on_run_tick()
plots = window.force_plots
assert plots._available == (True, False, False), plots._available
assert plots.value_labels[0].text() != t("chart.value_unavailable")
for index in (1, 2):
assert plots.value_labels[index].text() == t("chart.value_unavailable"), (
"an axis with no curve must not show a number"
)
state = plots.history.state()
assert math.isfinite(float(state.values[:, 0].max()))
assert not any(math.isfinite(v) for v in state.values[:, 1]), (
"FX carried a value in newton mode without a curve"
)
_finish(window, stuck)
def test_a_slow_close_is_not_reported_as_a_finished_recording(make_window):
"""Stopping must not present an unclosed file as a finished experiment."""
import threading as _threading
from unittest.mock import patch as _patch
from orisys_app_shared.i18n import t
from orisys_app_shared.record.experiment import _SessionWriter
window, stuck = _running_window(make_window)
window.toolbar.btn_record.setChecked(True)
stuck.emit(ui_check._fake_packet(0))
window._on_run_tick()
started = _threading.Event()
release = _threading.Event()
original = _SessionWriter.write_sample
def slow(self, *args):
started.set()
release.wait(10.0)
return original(self, *args)
with _patch.object(_SessionWriter, "write_sample", slow):
stuck.emit(ui_check._fake_packet(1))
window._on_run_tick()
assert started.wait(5.0), "the writer never picked up the frame"
window._stop_recording(wait=False)
assert window.recorder.report is None, "the stop claimed a finished recording"
assert window._record_report_pending, "the pending report was not tracked"
assert window._record_last_report is None
assert not window.toolbar.btn_record.isChecked(), (
"the record button still looks active while the file closes"
)
assert window.statusBar().currentMessage() == t("record.closing")
release.set()
deadline = __import__("time").monotonic() + 10.0
while window.recorder.report is None and __import__("time").monotonic() < deadline:
__import__("time").sleep(0.01)
window._on_run_tick()
report = window._record_last_report
assert report is not None and report.completed, "the terminal report was never published"
assert not window._record_report_pending
_finish(window, stuck)
def test_a_failed_recording_is_consumed_even_when_it_is_not_running(make_window):
"""A recording that died on its own still has a report to publish."""
import time as _time
from unittest.mock import patch as _patch
from orisys_app_shared.record.experiment import _SessionWriter
window, stuck = _running_window(make_window)
window.toolbar.btn_record.setChecked(True)
assert window.recorder.is_recording
with _patch.object(_SessionWriter, "write_sample", side_effect=OSError("disk full")):
for index in range(8):
stuck.emit(ui_check._fake_packet(index))
window._on_run_tick()
deadline = _time.monotonic() + 3.0
while window.recorder.stats().write_errors < 5 and _time.monotonic() < deadline:
_time.sleep(0.01)
assert window.recorder.stats().failed
assert not window.recorder.is_recording
# ``is_recording`` is already False: the stop must still consume the report.
window._stop_recording()
report = window._record_last_report
assert report is not None and report.stats.failed
assert not window.toolbar.btn_record.isChecked()
assert window._session_phase == "error"
_finish(window, stuck)
def test_the_depth_request_belongs_to_the_recording_it_started_in(make_window, record_root):
"""Opening another recording must invalidate the request still running."""
import threading as _threading
import time as _time
from PyQt5.QtWidgets import QApplication
from orisys_app_shared.record.depth import DepthResult
from orisys_studio.ui.playback_dialog import PlaybackDialog
window = _show(make_window(), (1200, 700))
dialog = PlaybackDialog(theme="dark", directory=Path(record_root))
first_serial = dialog._record_serial
started = _threading.Event()
release = _threading.Event()
closed: list[str] = []
class _Blocking:
def preflight(self) -> str:
return ""
def depth_at(self, _position, **_kwargs):
started.set()
release.wait(10.0)
return DepthResult(
depth=np.ones((2, 2)), frame_id=0, t_rel_s=0.0, record_id="old"
)
def close(self) -> None:
closed.append("closed")
blocking = _Blocking()
dialog._recomputer = blocking
dialog._on_recompute_depth()
assert started.wait(5.0), "the depth request never started"
# Another recording is opened while the request is in flight.
dialog.open_recording(Path(record_root))
assert dialog._record_serial == first_serial + 1
assert dialog._recomputer is not blocking, "the processor was not replaced"
assert closed == [], "the UI thread closed a processor a worker was reading"
release.set()
deadline = _time.monotonic() + 10.0
while not closed and _time.monotonic() < deadline:
QApplication.processEvents()
_time.sleep(0.01)
assert closed == ["closed"], "the retired processor was never closed"
assert dialog._depth_thread is not None and not dialog._depth_thread.is_alive()
# The answer belongs to the previous recording, so it is dropped, not drawn.
dialog._poll_depth()
QApplication.processEvents()
assert dialog._depth_ok is False
pixmap = dialog.depth_label.pixmap()
assert pixmap is None or pixmap.isNull(), "an answer for another recording was drawn"
dialog.close()
window.close()
def test_a_busy_depth_processor_is_not_closed_by_the_ui_thread(make_window, record_root):
import threading as _threading
import time as _time
from PyQt5.QtWidgets import QApplication
from orisys_app_shared.record.depth import DepthResult
from orisys_studio.ui.playback_dialog import PlaybackDialog
window = _show(make_window(), (1200, 700))
dialog = PlaybackDialog(theme="dark", directory=Path(record_root))
started = _threading.Event()
release = _threading.Event()
closed: list[str] = []
class _Blocking:
def preflight(self) -> str:
return ""
def depth_at(self, _position, **_kwargs):
started.set()
release.wait(10.0)
return DepthResult(depth=np.ones((2, 2)), frame_id=0, t_rel_s=0.0)
def close(self) -> None:
closed.append("closed")
dialog._recomputer = _Blocking()
dialog._on_recompute_depth()
assert started.wait(5.0)
dialog.close() # closing the window must not free a processor in use
QApplication.processEvents()
assert closed == [], "the window closed a processor that was still reading"
release.set()
deadline = _time.monotonic() + 10.0
while not closed and _time.monotonic() < deadline:
QApplication.processEvents()
_time.sleep(0.01)
assert closed == ["closed"], "the owning thread never closed the processor"
window.close()
def test_a_confirmed_calibration_is_snapshotted_into_the_recording(make_window, record_root):
"""The recording keeps the coefficients themselves, not only their hash."""
import json
from pathlib import Path
import orisys_studio.app_window as app_module
from orisys_app_shared.i18n import t
window, stuck = _running_window(make_window)
app_module.recordings_dir = lambda: record_root
window.toolbar.btn_record.setChecked(True)
assert window.recorder.is_recording
coefficients = Path(record_root) / "force_calibration_box.json"
coefficients.write_text('{"unit": "N", "channels": {}}', encoding="utf-8")
version = "deadbeefcafe1234"
window._on_calibration_applied(
str(coefficients), version, 21, 0.7, int(window.session_store.session_id)
)
window._drain_calibration_events()
directory = window.recorder.directory
assert directory is not None
snapshots = sorted(path.name for path in (directory / "calibration").iterdir())
assert snapshots == [f"force_calibration_{version[:12]}.json"], snapshots
assert (directory / "calibration" / snapshots[0]).read_text(
encoding="utf-8"
) == '{"unit": "N", "channels": {}}'
assert window._active_calibration_id == version
assert window.session_store.calibration_id == version
window._stop_recording()
meta = json.loads((directory / "session.json").read_text(encoding="utf-8"))
entries = meta["calibration_snapshots"]
assert entries and entries[0]["version"] == version
assert entries[0]["file"] == f"calibration/{snapshots[0]}"
assert entries[0]["frame_id"] == 21
event = next(e for e in meta["events"] if e["kind"] == "calibration_applied")
assert event["frame_id"] == 21 and event["mono_t"] == pytest.approx(0.7)
assert snapshots[0] in event["detail"]
_finish(window, stuck)
def test_a_confirmed_absence_of_calibration_does_not_keep_the_old_version(make_window):
"""An empty version is the honest answer; it must not fall back to the path."""
from orisys_app_shared.i18n import t
window = _show(make_window(), (1400, 880))
window._active_calibration_id = "previous-hash"
window._on_calibration_applied("/tmp/whatever.json", "", 5, 0.2, 1)
window._drain_calibration_events()
assert window._active_calibration_id == "", (
"an uncalibrated device kept claiming the previous coefficients"
)
assert window.statusBar().currentMessage() == t("cal.applied_none")
def test_a_zeroing_is_recorded_only_after_the_worker_confirms_it(make_window, record_root):
import json
import orisys_studio.app_window as app_module
window, stuck = _running_window(make_window)
app_module.recordings_dir = lambda: record_root
window.toolbar.btn_record.setChecked(True)
stuck.emit(ui_check._fake_packet(3))
window._on_run_tick()
window._on_reset()
assert stuck.reset_requests == 1, "the zeroing was not requested"
# The stub confirms from the acquisition side, as a real device does.
stuck.on_reset_applied = window._on_reset_applied
window._on_reset()
window._on_run_tick()
window._stop_recording()
directory = window.recorder.directory
meta = json.loads((directory / "session.json").read_text(encoding="utf-8"))
zeroing = [e for e in meta["events"] if e["kind"] == "zeroing"]
assert len(zeroing) == 1, f"unexpected zeroing events: {zeroing}"
assert zeroing[0]["frame_id"] == 3, zeroing[0]
assert zeroing[0]["mono_t"] is not None
_finish(window, stuck)
def test_depth_view_reports_missing_data_and_fixes_its_scale(make_window):
import numpy as np