Browse a long recording from disk instead of loading it into memory
What
- record/index.py: RecordingIndex streams forces.csv once and keeps only the
small things — one time, one validity flag, one frame id per row plus a byte
offset every 4096 rows (~6.5 MB per 500k rows). row_range() bisects the times,
read_rows() seeks to the nearest checkpoint and scans forward, and overview()
folds every row into its bucket during the same pass to build a per-axis
min/max envelope.
- read_rows(max_points, unit) decimates **while streaming**: each bucket keeps
the extreme row of every axis in the requested unit, so a long interval never
has to exist in memory in full and a shear spike survives a flat normal channel.
- RecordingReader: info() decimates above max_rows (default 200k) and marks
itself decimated, with total_rows and a duration taken from the frame index
rather than from the view; window()/window_all()/overview()/statistics() read
the interval being asked about. list/window semantics unchanged for small
recordings.
- force/source.py: StoreCurveSource (live memory) and RecordingCurveSource (disk
interval + cached envelope) implement one interface — extent, window,
envelope, statistics, duration_s — so ForcePlotsPanel asks a *source* instead
of knowing where the data lives.
- force_plots: reads through the source (window, envelope, statistics, extent),
which also replaces its last direct store accesses.
- playback_dialog: draws the whole-session view through window_all() in the unit
being shown (bounded by CHART_MAX_POINTS), and says so when the view is
decimated ("显示 N / 共 M 行") instead of letting the operator count points.
- experiment._run: a failure that lands while the files are opening can no longer
be overwritten by the writer's own transition to "recording" (found by a flaky
test; the state check now covers STATE_FAILED).
Why
The store releases old blocks above ~4.6 h and the replay loaded every CSV row
into Python lists before drawing anything, so neither the live browser nor the
replay could show an earlier interval of a long recording. The index makes the
disk the data source and memory a cache of what is on screen.
Verification
- shared/tests/test_history_source.py (new, 8 cases) over a 12 000-row recording:
the index reports its rows and reads a bounded interval; a large recording is
marked decimated with its true duration and total rows; a small one is not; an
interval read sees the exact rows including a single peak; a decimated window
keeps the peak; the envelope fits the whole session in ≤200 points, keeps the
spike and is cached; the disk and memory sources agree on extent, values and
statistics; the whole session can be read without holding it all. Shared
suites: 146 tests, 0 failures (three consecutive runs).
- apps/studio/tools/ui_check.py: the replay section now checks the index row
count, that an interval read spans only the interval, the envelope size and
total_rows. 324 checks, 0 failures.
- apps/studio/tests/test_ui_smoke.py: 43 tests, 36 pass; the 7 failures are the
unchanged sandbox/harness artifacts.
- D:/orisys_ws/_review_phase12/reproduce.py: all ten lines unchanged;
compileall, locale and prefs parity clean.
- apps/studio/docs/record-channel.md §8 documents the index, the reader API and
the unified source, and lists what is still hardware work.
Not in this change
- Maximum RSS / backlog / loss measurements at real resolution on the target ARM
board (needs the hardware), and range CSV export.
This commit is contained in:
@@ -16,7 +16,12 @@ from unittest.mock import patch
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from orisys_app_shared.record.depth import DepthRecomputer
|
||||
from orisys_app_shared.record.depth import (
|
||||
STATE_APPROXIMATE,
|
||||
STATE_EXACT,
|
||||
STATE_PREROLL,
|
||||
DepthRecomputer,
|
||||
)
|
||||
|
||||
SCRATCH = Path(__file__).resolve().parent / "_scratch"
|
||||
|
||||
@@ -36,6 +41,12 @@ class _CountingSensor:
|
||||
_CountingSensor.opened.append({"args": args, "kwargs": kwargs})
|
||||
self.n = 0
|
||||
self.chain_calls = 0
|
||||
self.initial_img = "reference-frame"
|
||||
# The real Sensor opens its source in the constructor, so the first
|
||||
# source is part of its history too.
|
||||
self.vid_src = str(args[0]) if args else ""
|
||||
self.sources: list[str] = [self.vid_src] if self.vid_src else []
|
||||
self.resets = 0
|
||||
self.info = SimpleNamespace(DEPTH=1)
|
||||
|
||||
def get_img(self):
|
||||
@@ -48,9 +59,17 @@ class _CountingSensor:
|
||||
def compute_contact(self):
|
||||
self.chain_calls += 1
|
||||
|
||||
def reset(self):
|
||||
self.resets += 1
|
||||
|
||||
def read_info(self, _channel):
|
||||
return np.array([self.n])
|
||||
|
||||
def _open_video_source(self, _override, _box):
|
||||
self.sources.append(str(self.vid_src))
|
||||
# A real reopen restarts the file from its first frame.
|
||||
self.n = 0
|
||||
|
||||
def disconnect(self):
|
||||
return None
|
||||
|
||||
@@ -116,13 +135,18 @@ def test_stepping_continues_from_the_current_position():
|
||||
computer.close()
|
||||
|
||||
|
||||
def test_stepping_backwards_reopens_the_segment():
|
||||
def test_stepping_backwards_reopens_the_source_but_keeps_the_processor():
|
||||
_CountingSensor.opened.clear()
|
||||
computer = DepthRecomputer(_reader(_session("/cfg/sensor.json")))
|
||||
with patch("orisys.Sensor", _CountingSensor):
|
||||
assert computer._depth_from_segment(Path("raw_0001.mp4"), 5).tolist() == [5]
|
||||
# Reading an earlier frame restarts the file; the processor object (and
|
||||
# therefore the tracker) is not thrown away.
|
||||
assert computer._depth_from_segment(Path("raw_0001.mp4"), 2).tolist() == [2]
|
||||
assert len(_CountingSensor.opened) == 2, "seeking backwards needs a fresh source"
|
||||
sensor = computer._sensor._sensor
|
||||
assert len(_CountingSensor.opened) == 1, "the processor was recreated"
|
||||
assert len(sensor.sources) == 2
|
||||
assert all(source.endswith("raw_0001.mp4") for source in sensor.sources)
|
||||
computer.close()
|
||||
|
||||
|
||||
@@ -165,11 +189,44 @@ def test_the_plan_starts_at_the_last_reset_in_the_same_segment():
|
||||
computer = DepthRecomputer(reader)
|
||||
plan = computer.plan_for(frames[8])
|
||||
assert plan.stop_at == 9
|
||||
assert plan.anchor == "reset"
|
||||
assert plan.state == STATE_EXACT
|
||||
assert plan.reset_frame_id == 6
|
||||
assert len(plan.steps) == 1
|
||||
step = plan.steps[0]
|
||||
assert step.source == reader.directory / "raw_0001.mp4"
|
||||
assert step.to_frame == 9
|
||||
assert step.reset_after == 7, "the reset boundary is not re-applied by the plan"
|
||||
assert plan.frames_to_read == 9
|
||||
assert plan.preroll is None
|
||||
assert plan.warnings == ()
|
||||
computer.close()
|
||||
|
||||
|
||||
def test_a_reset_that_is_not_in_the_video_is_not_a_recovered_state():
|
||||
"""Finding the event is not the same as being able to start there."""
|
||||
frames = [
|
||||
SimpleNamespace(
|
||||
frame_id=index,
|
||||
t_rel_s=index / 30.0,
|
||||
segment=1,
|
||||
segment_frame=index + 1,
|
||||
in_video=index != 6, # the reset frame never reached the video
|
||||
dropped=False,
|
||||
)
|
||||
for index in range(10)
|
||||
]
|
||||
reader = _reader(_session("/cfg/sensor.json"), frames=frames)
|
||||
reader.last_reset_frame = lambda frame_id: 6
|
||||
computer = DepthRecomputer(reader)
|
||||
plan = computer.plan_for(frames[8])
|
||||
assert plan.anchor != "reset"
|
||||
assert plan.state == STATE_APPROXIMATE
|
||||
assert "reset_frame_not_in_video" in plan.warnings
|
||||
assert "state_approximate" in plan.warnings
|
||||
computer.close()
|
||||
|
||||
|
||||
def test_the_plan_uses_the_pre_roll_for_the_start_of_a_recording():
|
||||
frames = [
|
||||
SimpleNamespace(
|
||||
@@ -195,6 +252,8 @@ def test_the_plan_uses_the_pre_roll_for_the_start_of_a_recording():
|
||||
plan = computer.plan_for(frames[3])
|
||||
assert plan.preroll is not None
|
||||
assert plan.preroll_frames == 30
|
||||
assert plan.anchor == "preroll"
|
||||
assert plan.state == STATE_PREROLL
|
||||
assert plan.warnings == ()
|
||||
computer.close()
|
||||
|
||||
@@ -215,10 +274,127 @@ def test_a_segment_without_history_is_flagged_approximate():
|
||||
reader = _reader(_session("/cfg/sensor.json"), frames=frames)
|
||||
computer = DepthRecomputer(reader)
|
||||
plan = computer.plan_for(frames[2])
|
||||
assert plan.state == STATE_APPROXIMATE
|
||||
assert plan.anchor == "segment_start"
|
||||
assert "state_approximate" in plan.warnings
|
||||
computer.close()
|
||||
|
||||
|
||||
def test_a_reset_in_an_earlier_segment_is_carried_across_by_swapping_the_source():
|
||||
"""The processor survives the swap, so the state *can* cross a segment."""
|
||||
frames = [
|
||||
SimpleNamespace(
|
||||
frame_id=index,
|
||||
t_rel_s=index / 30.0,
|
||||
segment=1 if index < 3 else 2,
|
||||
segment_frame=index + 1 if index < 3 else index - 2,
|
||||
in_video=True,
|
||||
dropped=False,
|
||||
)
|
||||
for index in range(6)
|
||||
]
|
||||
reader = _reader(
|
||||
_session("/cfg/sensor.json"),
|
||||
segments=("raw_0001.mp4", "raw_0002.mp4"),
|
||||
frames=frames,
|
||||
)
|
||||
reader.last_reset_frame = lambda frame_id: 1
|
||||
computer = DepthRecomputer(reader)
|
||||
plan = computer.plan_for(frames[5])
|
||||
assert plan.anchor == "reset"
|
||||
assert plan.state == STATE_EXACT
|
||||
assert "state_across_segment" not in plan.warnings
|
||||
assert [step.source.name for step in plan.steps] == ["raw_0001.mp4", "raw_0002.mp4"]
|
||||
assert plan.steps[0].reset_after == 2
|
||||
assert plan.steps[1].to_frame == 3
|
||||
computer.close()
|
||||
|
||||
|
||||
def test_a_replay_that_would_read_too_far_back_is_reported_not_attempted():
|
||||
frames = [
|
||||
SimpleNamespace(
|
||||
frame_id=index,
|
||||
t_rel_s=index / 30.0,
|
||||
segment=1,
|
||||
segment_frame=index + 1,
|
||||
in_video=True,
|
||||
dropped=False,
|
||||
)
|
||||
for index in range(20)
|
||||
]
|
||||
reader = _reader(_session("/cfg/sensor.json"), frames=frames)
|
||||
reader.last_reset_frame = lambda frame_id: 1
|
||||
computer = DepthRecomputer(reader)
|
||||
with patch("orisys_app_shared.record.depth.MAX_REPLAY_FRAMES", 5):
|
||||
plan = computer.plan_for(frames[10])
|
||||
assert "state_too_far_back" in plan.warnings
|
||||
assert plan.state == STATE_APPROXIMATE
|
||||
computer.close()
|
||||
|
||||
|
||||
def test_the_same_frame_is_answered_from_the_cache():
|
||||
"""A second request for the frame just read is a cache hit, not None."""
|
||||
_CountingSensor.opened.clear()
|
||||
computer = DepthRecomputer(_reader(_session("/cfg/sensor.json")))
|
||||
with patch("orisys.Sensor", _CountingSensor):
|
||||
first = computer._depth_from_segment(Path("raw_0001.mp4"), 3)
|
||||
second = computer._depth_from_segment(Path("raw_0001.mp4"), 3)
|
||||
sensor = computer._sensor._sensor
|
||||
assert first.tolist() == [3]
|
||||
assert second is not None and second.tolist() == [3]
|
||||
assert sensor.n == 3, "the cached answer re-read the stream"
|
||||
computer.close()
|
||||
|
||||
|
||||
def test_the_processor_survives_a_pre_roll_to_segment_switch():
|
||||
"""Only the video source is replaced; the tracker state must survive."""
|
||||
_CountingSensor.opened.clear()
|
||||
reader = _reader(_session("/cfg/sensor.json"))
|
||||
preroll = reader.directory / "preroll_0001.mp4"
|
||||
preroll.write_bytes(b"")
|
||||
computer = DepthRecomputer(reader)
|
||||
with patch("orisys.Sensor", _CountingSensor):
|
||||
computer._feed(preroll, 2)
|
||||
sensor = computer._sensor._sensor
|
||||
assert sensor.n == 2, "the pre-roll frames were not fed"
|
||||
# Feed the segment: same processor, new source, position restarted.
|
||||
computer._feed(reader.directory / "raw_0001.mp4", 1)
|
||||
assert len(_CountingSensor.opened) == 1, "the processor was recreated"
|
||||
assert computer._sensor._sensor is sensor
|
||||
assert len(sensor.sources) == 2
|
||||
assert sensor.sources[0].endswith("preroll_0001.mp4")
|
||||
assert sensor.sources[1].endswith("raw_0001.mp4")
|
||||
assert sensor.initial_img == "reference-frame", "the tracker reference was reset"
|
||||
computer.close()
|
||||
|
||||
|
||||
def test_a_reset_boundary_is_re_applied_during_the_replay():
|
||||
"""The live zeroing cleared the tracker; the replay must do the same."""
|
||||
_CountingSensor.opened.clear()
|
||||
frames = [
|
||||
SimpleNamespace(
|
||||
frame_id=index,
|
||||
t_rel_s=index / 30.0,
|
||||
segment=1,
|
||||
segment_frame=index + 1,
|
||||
in_video=True,
|
||||
dropped=False,
|
||||
)
|
||||
for index in range(10)
|
||||
]
|
||||
reader = _reader_with_config(frames)
|
||||
reader.last_reset_frame = lambda frame_id: 6
|
||||
reader.frame_at = lambda _t: frames[8]
|
||||
computer = DepthRecomputer(reader)
|
||||
with patch("orisys.Sensor", _CountingSensor):
|
||||
result = computer.depth_at(frames[8].t_rel_s)
|
||||
sensor = computer._sensor._sensor
|
||||
assert result.ok, result.error
|
||||
assert sensor.resets == 1, "the reset boundary was not re-applied"
|
||||
assert result.warnings == (), result.warnings
|
||||
computer.close()
|
||||
|
||||
|
||||
def test_a_dropped_frame_is_reported_not_substituted():
|
||||
"""The index marks the hole; replay must say so instead of showing a neighbour."""
|
||||
frames = [
|
||||
|
||||
Reference in New Issue
Block a user