Browse a long recording from disk instead of loading it into memory
What
- record/index.py: RecordingIndex streams forces.csv once and keeps only the
small things — one time, one validity flag, one frame id per row plus a byte
offset every 4096 rows (~6.5 MB per 500k rows). row_range() bisects the times,
read_rows() seeks to the nearest checkpoint and scans forward, and overview()
folds every row into its bucket during the same pass to build a per-axis
min/max envelope.
- read_rows(max_points, unit) decimates **while streaming**: each bucket keeps
the extreme row of every axis in the requested unit, so a long interval never
has to exist in memory in full and a shear spike survives a flat normal channel.
- RecordingReader: info() decimates above max_rows (default 200k) and marks
itself decimated, with total_rows and a duration taken from the frame index
rather than from the view; window()/window_all()/overview()/statistics() read
the interval being asked about. list/window semantics unchanged for small
recordings.
- force/source.py: StoreCurveSource (live memory) and RecordingCurveSource (disk
interval + cached envelope) implement one interface — extent, window,
envelope, statistics, duration_s — so ForcePlotsPanel asks a *source* instead
of knowing where the data lives.
- force_plots: reads through the source (window, envelope, statistics, extent),
which also replaces its last direct store accesses.
- playback_dialog: draws the whole-session view through window_all() in the unit
being shown (bounded by CHART_MAX_POINTS), and says so when the view is
decimated ("显示 N / 共 M 行") instead of letting the operator count points.
- experiment._run: a failure that lands while the files are opening can no longer
be overwritten by the writer's own transition to "recording" (found by a flaky
test; the state check now covers STATE_FAILED).
Why
The store releases old blocks above ~4.6 h and the replay loaded every CSV row
into Python lists before drawing anything, so neither the live browser nor the
replay could show an earlier interval of a long recording. The index makes the
disk the data source and memory a cache of what is on screen.
Verification
- shared/tests/test_history_source.py (new, 8 cases) over a 12 000-row recording:
the index reports its rows and reads a bounded interval; a large recording is
marked decimated with its true duration and total rows; a small one is not; an
interval read sees the exact rows including a single peak; a decimated window
keeps the peak; the envelope fits the whole session in ≤200 points, keeps the
spike and is cached; the disk and memory sources agree on extent, values and
statistics; the whole session can be read without holding it all. Shared
suites: 146 tests, 0 failures (three consecutive runs).
- apps/studio/tools/ui_check.py: the replay section now checks the index row
count, that an interval read spans only the interval, the envelope size and
total_rows. 324 checks, 0 failures.
- apps/studio/tests/test_ui_smoke.py: 43 tests, 36 pass; the 7 failures are the
unchanged sandbox/harness artifacts.
- D:/orisys_ws/_review_phase12/reproduce.py: all ten lines unchanged;
compileall, locale and prefs parity clean.
- apps/studio/docs/record-channel.md §8 documents the index, the reader API and
the unified source, and lists what is still hardware work.
Not in this change
- Maximum RSS / backlog / loss measurements at real resolution on the target ARM
board (needs the hardware), and range CSV export.
This commit is contained in:
@@ -8,6 +8,9 @@ model.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from orisys_app_shared.force.calibration_io import (
|
||||
@@ -17,10 +20,19 @@ from orisys_app_shared.force.calibration_io import (
|
||||
build_force_calibration_v2,
|
||||
merge_samples,
|
||||
predict_channel,
|
||||
write_force_calibration,
|
||||
)
|
||||
from orisys_app_shared.force.calibration_recorder import CalibrationRecorder
|
||||
from orisys_app_shared.force.session import SessionSample
|
||||
|
||||
SCRATCH = Path(__file__).resolve().parent / "_scratch"
|
||||
|
||||
|
||||
def _root() -> Path:
|
||||
shutil.rmtree(SCRATCH, ignore_errors=True)
|
||||
SCRATCH.mkdir(parents=True, exist_ok=True)
|
||||
return SCRATCH
|
||||
|
||||
|
||||
def _point(recorder, load: float, raw: float, *, channel: str = "fn"):
|
||||
recorder.start(channel=channel, reference_load_n=load, duration_s=0.05)
|
||||
@@ -151,3 +163,84 @@ def test_a_channel_with_one_point_is_not_exported():
|
||||
store.aggregate("fn", CalSample(weight_n=1.0, raw_mean=10.0, n=10))
|
||||
with pytest.raises(ValueError):
|
||||
build_force_calibration_v2(store)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- one preparation
|
||||
|
||||
|
||||
def _session_with_duplicates_and_validation():
|
||||
recorder = CalibrationRecorder()
|
||||
points = [
|
||||
_point(recorder, 1.0, 100.0),
|
||||
_point(recorder, 2.0, 200.0),
|
||||
# A deliberate repeat of load 2.0 at a different raw level.
|
||||
_point(recorder, 2.0, 240.0),
|
||||
_point(recorder, 4.0, 400.0),
|
||||
_point(recorder, 3.0, 290.0),
|
||||
]
|
||||
recorder.set_validation(points[4], True)
|
||||
return recorder, points
|
||||
|
||||
|
||||
def test_duplicate_loads_are_merged_before_the_model_is_built():
|
||||
_recorder, _points = _session_with_duplicates_and_validation()
|
||||
prepared = _recorder.prepared()
|
||||
assert prepared.samples["fn"], "no training samples"
|
||||
loads = [sample.weight_n for sample in prepared.samples["fn"]]
|
||||
assert loads == sorted(set(loads)), f"duplicate loads were not merged: {loads}"
|
||||
assert len(loads) == 3
|
||||
merged = next(sample for sample in prepared.samples["fn"] if sample.weight_n == 2.0)
|
||||
assert merged.n == 16, "the repeat's samples were not combined"
|
||||
assert merged.raw_mean == pytest.approx((200.0 + 240.0) / 2.0)
|
||||
assert prepared.points_in_fit("fn") == 3
|
||||
|
||||
|
||||
def test_the_exported_file_predicts_exactly_what_the_panel_shows():
|
||||
"""Read the written document back and compare it with the diagnostics.
|
||||
|
||||
The panel's numbers must describe the file that is actually written, so the
|
||||
test loads the file with the SDK (not with the prepared object) and predicts
|
||||
every training and validation point from it.
|
||||
"""
|
||||
import orisys.calibration as sdk_calibration
|
||||
|
||||
recorder, points = _session_with_duplicates_and_validation()
|
||||
prepared = recorder.prepared(meta={"sensor_type": "box", "sn": "SN9"})
|
||||
assert prepared.ok
|
||||
|
||||
document = prepared.to_document()
|
||||
path = _root() / "force_calibration.json"
|
||||
write_force_calibration(document, path)
|
||||
from_file = sdk_calibration.load(path, strict=True)
|
||||
|
||||
for point in points:
|
||||
raw = float(point.loaded_stats().mean)
|
||||
predicted = float(from_file.get("fn").convert(raw))
|
||||
value = prepared.predict("fn", raw)
|
||||
assert predicted == pytest.approx(value, abs=1e-9)
|
||||
if point.is_validation:
|
||||
assert prepared.validation_errors[id(point)] == pytest.approx(
|
||||
abs(predicted - point.reference_load_n), abs=1e-9
|
||||
)
|
||||
else:
|
||||
assert prepared.residuals[id(point)] == pytest.approx(
|
||||
predicted - point.reference_load_n, abs=1e-9
|
||||
)
|
||||
|
||||
# The validation point is measured, never built in.
|
||||
assert prepared.validation_points("fn") == 1
|
||||
assert prepared.max_validation_error("fn") > 0.0
|
||||
exported_raws = sorted(point[0] for point in document["channels"]["normal"]["points"])
|
||||
assert exported_raws == [100.0, 220.0, 400.0], exported_raws
|
||||
|
||||
|
||||
def test_the_recorder_fit_and_the_preparation_agree():
|
||||
recorder, _points = _session_with_duplicates_and_validation()
|
||||
prepared = recorder.prepared()
|
||||
fit = recorder.fit("fn", prepared=prepared)
|
||||
assert fit.n_points == prepared.points_in_fit("fn")
|
||||
assert fit.breakpoints == prepared.breakpoints("fn")
|
||||
assert fit.max_residual_n == pytest.approx(prepared.max_residual("fn"))
|
||||
assert fit.validation_error_n == pytest.approx(prepared.max_validation_error("fn"))
|
||||
# ...and a fit that prepares on its own gets the same numbers.
|
||||
assert recorder.fit("fn").as_dict() == fit.as_dict()
|
||||
|
||||
Reference in New Issue
Block a user