ykcheungab
73592f085f
What
- record/index.py: RecordingIndex streams forces.csv once and keeps only the
small things — one time, one validity flag, one frame id per row plus a byte
offset every 4096 rows (~6.5 MB per 500k rows). row_range() bisects the times,
read_rows() seeks to the nearest checkpoint and scans forward, and overview()
folds every row into its bucket during the same pass to build a per-axis
min/max envelope.
- read_rows(max_points, unit) decimates **while streaming**: each bucket keeps
the extreme row of every axis in the requested unit, so a long interval never
has to exist in memory in full and a shear spike survives a flat normal channel.
- RecordingReader: info() decimates above max_rows (default 200k) and marks
itself decimated, with total_rows and a duration taken from the frame index
rather than from the view; window()/window_all()/overview()/statistics() read
the interval being asked about. list/window semantics unchanged for small
recordings.
- force/source.py: StoreCurveSource (live memory) and RecordingCurveSource (disk
interval + cached envelope) implement one interface — extent, window,
envelope, statistics, duration_s — so ForcePlotsPanel asks a *source* instead
of knowing where the data lives.
- force_plots: reads through the source (window, envelope, statistics, extent),
which also replaces its last direct store accesses.
- playback_dialog: draws the whole-session view through window_all() in the unit
being shown (bounded by CHART_MAX_POINTS), and says so when the view is
decimated ("显示 N / 共 M 行") instead of letting the operator count points.
- experiment._run: a failure that lands while the files are opening can no longer
be overwritten by the writer's own transition to "recording" (found by a flaky
test; the state check now covers STATE_FAILED).
Why
The store releases old blocks above ~4.6 h and the replay loaded every CSV row
into Python lists before drawing anything, so neither the live browser nor the
replay could show an earlier interval of a long recording. The index makes the
disk the data source and memory a cache of what is on screen.
Verification
- shared/tests/test_history_source.py (new, 8 cases) over a 12 000-row recording:
the index reports its rows and reads a bounded interval; a large recording is
marked decimated with its true duration and total rows; a small one is not; an
interval read sees the exact rows including a single peak; a decimated window
keeps the peak; the envelope fits the whole session in ≤200 points, keeps the
spike and is cached; the disk and memory sources agree on extent, values and
statistics; the whole session can be read without holding it all. Shared
suites: 146 tests, 0 failures (three consecutive runs).
- apps/studio/tools/ui_check.py: the replay section now checks the index row
count, that an interval read spans only the interval, the envelope size and
total_rows. 324 checks, 0 failures.
- apps/studio/tests/test_ui_smoke.py: 43 tests, 36 pass; the 7 failures are the
unchanged sandbox/harness artifacts.
- D:/orisys_ws/_review_phase12/reproduce.py: all ten lines unchanged;
compileall, locale and prefs parity clean.
- apps/studio/docs/record-channel.md §8 documents the index, the reader API and
the unified source, and lists what is still hardware work.
Not in this change
- Maximum RSS / backlog / loss measurements at real resolution on the target ARM
board (needs the hardware), and range CSV export.
281 lines
10 KiB
Python
281 lines
10 KiB
Python
"""Calibration identity and operator events as the acquisition thread sees them.
|
|
|
|
The review's P1-7: the application event was written before the device had loaded
|
|
anything, the recorded ``calibration_id`` never changed mid-session, and an
|
|
uncalibrated device could keep claiming the coefficients of an earlier session.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from orisys_app_shared.acquisition.sdk_worker import SdkAcquisitionWorker
|
|
from orisys_app_shared.record import ExperimentRecorder, RecordConfig
|
|
from orisys_app_shared.record.calibration_snapshot import (
|
|
list_calibration_snapshots,
|
|
snapshot_name,
|
|
store_calibration_snapshot,
|
|
)
|
|
|
|
SCRATCH = Path(__file__).resolve().parent / "_scratch"
|
|
|
|
|
|
def _root() -> Path:
|
|
shutil.rmtree(SCRATCH, ignore_errors=True)
|
|
SCRATCH.mkdir(parents=True, exist_ok=True)
|
|
return SCRATCH
|
|
|
|
|
|
class _Sensor:
|
|
"""A device with a swappable calibration, like the SDK's Sensor."""
|
|
|
|
def __init__(self, calibration=None, path=None):
|
|
self.calibration = calibration
|
|
self.calibration_path = path
|
|
self.applied: list[str] = []
|
|
self.resets = 0
|
|
|
|
def set_calibration(self, path):
|
|
if str(path) == "reject-me":
|
|
raise ValueError("bad document")
|
|
self.calibration = object()
|
|
self.calibration_path = Path(path)
|
|
self.applied.append(str(path))
|
|
|
|
def reset(self):
|
|
self.resets += 1
|
|
|
|
|
|
def _calibration_file(root: Path, name: str, body: str = '{"unit": "N"}') -> Path:
|
|
path = root / name
|
|
path.write_text(body, encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def _worker(**kwargs) -> SdkAcquisitionWorker:
|
|
for key in ("on_calibration_applied", "on_reset_applied"):
|
|
kwargs.setdefault(key, None)
|
|
return SdkAcquisitionWorker(**kwargs)
|
|
|
|
|
|
# ------------------------------------------------------------------ worker state
|
|
|
|
|
|
def test_a_new_session_does_not_inherit_the_previous_version():
|
|
"""An uncalibrated device must report no calibration, not the last one."""
|
|
root = _root()
|
|
worker = _worker()
|
|
worker._calibration_version = "previous-session-hash"
|
|
worker._stop.set()
|
|
worker.start("clip.mp4", config_ref=str(_calibration_file(root, "sensor.json")))
|
|
assert worker._calibration_version == "", (
|
|
"the new session kept the previous session's calibration version"
|
|
)
|
|
worker.stop(wait=True)
|
|
|
|
|
|
def test_the_applied_version_is_the_content_hash_and_carries_the_frame():
|
|
root = _root()
|
|
applied: list[tuple] = []
|
|
worker = _worker(on_calibration_applied=lambda *args: applied.append(args))
|
|
path = _calibration_file(root, "force_calibration.json")
|
|
sensor = _Sensor()
|
|
worker._session_id = 4
|
|
worker.set_pending_calibration(path)
|
|
worker._consume_pending_calibration(sensor, frame_id=17, mono_t=0.5)
|
|
|
|
assert len(applied) == 1
|
|
got_path, version, frame_id, mono_t, session = applied[0]
|
|
assert got_path == str(path)
|
|
assert version and version == worker._calibration_version
|
|
assert version != str(path), "the path was used as the version"
|
|
assert (frame_id, mono_t, session) == (17, 0.5, 4)
|
|
assert sensor.applied == [str(path)]
|
|
|
|
|
|
def test_a_rejected_calibration_keeps_the_version_the_device_still_has():
|
|
root = _root()
|
|
applied: list[tuple] = []
|
|
worker = _worker(on_calibration_applied=lambda *args: applied.append(args))
|
|
sensor = _Sensor()
|
|
worker._session_id = 1
|
|
worker._calibration_version = "loaded-earlier"
|
|
worker.set_pending_calibration("reject-me")
|
|
with pytest.raises(ValueError):
|
|
worker._consume_pending_calibration(sensor, frame_id=3, mono_t=0.1)
|
|
assert not applied, "a rejected document was reported as applied"
|
|
assert worker._calibration_version == "loaded-earlier", (
|
|
"the device still has those coefficients, so the version is still true"
|
|
)
|
|
|
|
|
|
def test_a_zeroing_is_applied_by_the_worker_and_reported_with_its_frame():
|
|
root = _root()
|
|
events: list[tuple] = []
|
|
worker = _worker(on_reset_applied=lambda *args: events.append(args))
|
|
sensor = _Sensor()
|
|
worker._session_id = 2
|
|
worker.reset() # a request, not the zeroing itself
|
|
assert sensor.resets == 0, "the zeroing ran on the caller's thread"
|
|
worker._consume_pending_reset(sensor, frame_id=42, mono_t=1.4)
|
|
assert sensor.resets == 1
|
|
assert events == [(42, 1.4, 2)]
|
|
# The request is consumed once.
|
|
worker._consume_pending_reset(sensor, frame_id=43, mono_t=1.5)
|
|
assert events == [(42, 1.4, 2)]
|
|
|
|
|
|
def test_a_started_session_drops_a_stale_zeroing_request():
|
|
root = _root()
|
|
events: list[tuple] = []
|
|
worker = _worker(on_reset_applied=lambda *args: events.append(args))
|
|
worker._stop.set()
|
|
worker.start("clip.mp4", config_ref=str(_calibration_file(root, "sensor.json")))
|
|
sensor = _Sensor()
|
|
worker._consume_pending_reset(sensor, frame_id=1, mono_t=0.0)
|
|
assert sensor.resets == 0, "a reset requested before this session fired into it"
|
|
assert not events
|
|
worker.stop(wait=True)
|
|
|
|
|
|
# ------------------------------------------------------------------- snapshots
|
|
|
|
|
|
def test_a_snapshot_is_content_addressed_and_listed():
|
|
root = _root()
|
|
directory = root / "session"
|
|
source = _calibration_file(root, "force_calibration_box.json", '{"a": 1}')
|
|
name = store_calibration_snapshot(directory, source, "abcdef1234567890")
|
|
assert name == "calibration/force_calibration_abcdef123456.json"
|
|
assert (directory / name).is_file()
|
|
assert (directory / name).read_text(encoding="utf-8") == '{"a": 1}'
|
|
# Applying the same coefficients twice stores one file.
|
|
again = store_calibration_snapshot(directory, source, "abcdef1234567890")
|
|
assert again == name
|
|
assert list_calibration_snapshots(directory) == [name]
|
|
# A different calibration cannot collide with it.
|
|
other = _calibration_file(root, "force_calibration_box.json", '{"a": 2}')
|
|
second = store_calibration_snapshot(directory, other, "ffffff000000")
|
|
assert second != name
|
|
assert len(list_calibration_snapshots(directory)) == 2
|
|
assert snapshot_name("", source).startswith("force_calibration_")
|
|
|
|
|
|
def test_the_recording_metadata_names_the_coefficients_it_ran_with():
|
|
root = _root()
|
|
recorder = ExperimentRecorder(RecordConfig(video=False))
|
|
directory = recorder.start(root=root, session_id=1, metadata={})
|
|
source = _calibration_file(root, "force_calibration_box.json", '{"a": 1}')
|
|
name = store_calibration_snapshot(directory, source, "abcdef1234567890")
|
|
recorder.note_calibration_snapshot(
|
|
version="abcdef1234567890", name=name, frame_id=12, mono_t=0.4
|
|
)
|
|
recorder.log_event(
|
|
"calibration_applied", name, mono_t=0.4, frame_id=12
|
|
)
|
|
recorder.stop()
|
|
|
|
meta = json.loads((Path(directory) / "session.json").read_text(encoding="utf-8"))
|
|
snapshots = meta["calibration_snapshots"]
|
|
assert snapshots and snapshots[0]["version"] == "abcdef1234567890"
|
|
assert snapshots[0]["file"] == name
|
|
assert snapshots[0]["frame_id"] == 12
|
|
assert snapshots[0]["mono_t"] == pytest.approx(0.4)
|
|
event = next(e for e in meta["events"] if e["kind"] == "calibration_applied")
|
|
assert event["frame_id"] == 12
|
|
assert event["mono_t"] == pytest.approx(0.4), "the event lost its acquisition time"
|
|
assert event["at"], "the event lost its wall clock"
|
|
|
|
# The CSV carries the same acquisition time in the session's relative base.
|
|
# An event that predates the first frame has no base yet, so its relative
|
|
# time is empty — the absolute (session-relative) time is in the metadata.
|
|
with (Path(directory) / "events.csv").open(encoding="utf-8", newline="") as handle:
|
|
import csv as _csv
|
|
|
|
rows = list(_csv.DictReader(handle))
|
|
applied = next(row for row in rows if row["kind"] == "calibration_applied")
|
|
assert applied["frame_id"] == "12"
|
|
assert applied["t_rel_s"] == "", "an event before the first frame invented a base"
|
|
|
|
|
|
def test_an_event_during_the_recording_gets_the_session_relative_time():
|
|
import csv as _csv
|
|
|
|
from orisys_app_shared.acquisition.sdk_worker import SdkPacket
|
|
|
|
root = _root()
|
|
recorder = ExperimentRecorder(RecordConfig(video=False))
|
|
directory = recorder.start(root=root, session_id=1, metadata={})
|
|
frame = np.zeros((4, 4, 3), np.uint8)
|
|
recorder.on_packet(
|
|
SdkPacket(
|
|
raw_frame=frame,
|
|
processed_frame=frame,
|
|
timestamp=1000.0,
|
|
mono_t=0.5,
|
|
fps=30.0,
|
|
process_ms=1.0,
|
|
frame_id=7,
|
|
)
|
|
)
|
|
recorder.log_event("zeroing", "tracker reset", mono_t=0.9, frame_id=9)
|
|
recorder.stop()
|
|
|
|
with (Path(directory) / "events.csv").open(encoding="utf-8", newline="") as handle:
|
|
rows = list(_csv.DictReader(handle))
|
|
zeroing = next(row for row in rows if row["kind"] == "zeroing")
|
|
assert float(zeroing["t_rel_s"]) == pytest.approx(0.4, abs=1e-6)
|
|
assert zeroing["frame_id"] == "9"
|
|
|
|
|
|
def test_an_event_without_a_mono_time_still_gets_a_wall_clock():
|
|
root = _root()
|
|
recorder = ExperimentRecorder(RecordConfig(video=False))
|
|
directory = recorder.start(root=root, session_id=1, metadata={})
|
|
recorder.log_event("operator_note", "no frame attached")
|
|
recorder.stop()
|
|
meta = json.loads((Path(directory) / "session.json").read_text(encoding="utf-8"))
|
|
event = next(e for e in meta["events"] if e["kind"] == "operator_note")
|
|
assert event["at"], "the event lost its wall clock"
|
|
assert event["frame_id"] == -1
|
|
|
|
|
|
def test_the_recorded_row_uses_the_packet_version_not_the_session_default():
|
|
from orisys_app_shared.record.sample import to_record_sample
|
|
|
|
class Packet:
|
|
frame_id = 3
|
|
mono_t = 0.2
|
|
wall_t = 0.2
|
|
valid = True
|
|
calibration_version = ""
|
|
calibrated_axes = ("fn",)
|
|
force_calibrated = True
|
|
raw_available = True
|
|
|
|
# An empty packet version is the device saying "no calibration": the session
|
|
# default must not be pasted in behind its back.
|
|
assert to_record_sample(Packet(), calibration_version="session-default").calibration_version == ""
|
|
|
|
class LegacyPacket(Packet):
|
|
calibration_version = None # an older producer without the field
|
|
|
|
# ...but a packet that has no version field at all still gets the default.
|
|
assert (
|
|
to_record_sample(LegacyPacket(), calibration_version="session-default").calibration_version
|
|
== "session-default"
|
|
)
|
|
|
|
class StampedPacket(Packet):
|
|
calibration_version = "packet-hash"
|
|
|
|
assert (
|
|
to_record_sample(StampedPacket(), calibration_version="session-default").calibration_version
|
|
== "packet-hash"
|
|
)
|