Store the depth channel in an analysis result and serve it to the 2D depth view

A replay could draw arrows but not depth, and the recording view said there was
nothing to draw even when a result existed. Depth is now quantised per frame
(14 B empty, 486 B at the worst frame seen), read from the cache by the same
draw path live uses, and the raw-recording note appears only when no result
exists. Format version 2; see apps/studio/docs/replay-analysis.md.

apps/studio/tests/test_replay_analysis.py

@@ -28,6 +28,19 @@ from pathlib import Path
 
 import numpy as np
 import pytest
+from orisys_app_shared.acquisition.arrows import depth_map_to_bgr, flow_colormap
+from orisys_app_shared.acquisition.image_backend import (
+    ARROW_PURE,
+    ARROW_STITCHED,
+    DEPTH,
+    DERIVED_MODES,
+    RAW_NOTE_KEY,
+    SOURCE_ANALYSIS,
+    SOURCE_REPLAY,
+    ImageFrame,
+    plan_image,
+    render_image,
+)
 from orisys_app_shared.analysis.cache import ResultCache, scan_results
 from orisys_app_shared.analysis.reader import (
     AnalysisIncompleteError,
@@ -254,10 +267,22 @@ class _ImagePanel:
     def __init__(self) -> None:
         self.kinds: list[str] = []
         self.cleared: list[str] = []
+        #: The 2D mode selector's current value. ``recorded`` is what the panel starts
+        #: on for a recording, so a stub that never sets it behaves as before.
+        self._mode = "recorded"
 
     def set_source_kind(self, kind: str) -> None:
         self.kinds.append(str(kind))
 
+    def view_mode(self) -> str:
+        return self._mode
+
+    def set_view_mode(self, mode: str) -> None:
+        self._mode = str(mode)
+
+    def set_plan(self, plan) -> None:
+        self.plan = plan
+
     def clear_preview(self, message: str) -> None:
         self.cleared.append(str(message))
 
@@ -904,3 +929,266 @@ def test_the_worker_reports_progress_and_a_cancelled_outcome():
     assert report.cancelled is True and report.complete is False
     assert report.frames_written == 2
     assert report_from(None, cancelled=True).cancelled is True
+
+
+# ------------------------------------------------------------------ the depth view
+def _analysis_frame_for(frame, *, source=SOURCE_ANALYSIS):
+    """The frame the controller hands the window's single 2D sink for a stored frame."""
+    return ImageFrame(
+        frame.base_bgr,
+        flow=frame.flow,
+        depth=frame.depth,
+        valid_mask=frame.valid_mask,
+        source=source,
+        frame_id=int(frame.frame_id),
+        t_rel_s=float(frame.t_rel_s),
+        flow_valid=True,
+        flow_frame_id=int(frame.frame_id),
+        depth_valid=bool(frame.has_depth),
+        depth_frame_id=int(frame.frame_id) if frame.has_depth else None,
+        result_available=True,
+    )
+
+
+def _drawn(frame, mode: str):
+    image = _analysis_frame_for(frame)
+    return render_image(
+        image, plan_image(image), mode=mode, theme="dark", draw_style="box"
+    )
+
+
+def test_the_depth_view_is_drawn_from_the_cache_by_the_backend_live_uses(session):
+    """The depth mode renders the stored map through the same call live renders through.
+
+    Nothing here is a second draw path: ``render_image`` in ``depth`` mode is the same
+    function a live frame reaches, over the map the reader handed back. The check that it
+    is really that map is a direct comparison with the shared converter, and the check
+    that it is not some other view is that the arrows picture differs from it.
+    """
+    frame = session.frame(FIRST_FRAME_ID + 1)
+    assert frame.has_depth, "the pass recorded no depth channel for this frame"
+    assert frame.depth.shape == (300, 280)
+    assert frame.depth.dtype == np.float32
+
+    drawn = _drawn(frame, DEPTH)
+    assert drawn is not None
+    expected = depth_map_to_bgr(
+        frame.depth, flow_colormap("dark"), value_range=None, invalid_mask=frame.valid_mask
+    )
+    assert np.array_equal(drawn, expected), "the depth mode did not draw the stored map"
+    arrows = _drawn(frame, ARROW_PURE)
+    assert arrows is not None
+    assert not np.array_equal(drawn, arrows), "the depth view drew the arrow view"
+
+
+def test_a_seek_equals_playing_forward_in_the_depth_view(session):
+    """The seek test, in the depth mode: same frame, same map, same bytes on screen.
+
+    Playing forward records each frame's rendered depth picture; a fresh session is then
+    seeked to five stored timestamps and must produce those pictures byte for byte. The
+    tolerance is zero on the frame id, on ``depth``, on the statistics that travel with
+    it and on the rendered image — because a seek is a file read of the same record.
+    """
+    ids = [int(value) for value in session.frame_ids]
+    played: dict[int, tuple] = {}
+    for frame_id in ids:
+        frame = session.frame(frame_id)
+        played[frame_id] = (
+            frame.depth,
+            frame.depth_nonzero,
+            frame.depth_min,
+            frame.depth_max,
+            _drawn(frame, DEPTH),
+        )
+    fresh = AnalysisSession(
+        AnalysisReader(_result_directory(Path(session.directory).parent)),
+        policy=session.policy,
+        display_policy=session.display_policy,
+    )
+    try:
+        nonempty = 0
+        for frame_id in (ids[0], ids[1], ids[len(ids) // 2], ids[-2], ids[-1]):
+            moment = fresh.time_of(frame_id)
+            assert moment is not None
+            seeked = fresh.frame_at(moment)
+            assert seeked is not None and seeked.frame_id == frame_id
+            depth, nonzero, low, high, picture = played[frame_id]
+            nonempty += 1 if nonzero else 0
+            assert seeked.depth_nonzero == nonzero
+            assert (seeked.depth_min, seeked.depth_max) == (low, high)
+            assert np.array_equal(np.asarray(seeked.depth), np.asarray(depth))
+            picture_now = _drawn(seeked, DEPTH)
+            if picture is None or picture_now is None:
+                assert picture is None and picture_now is None
+            else:
+                assert np.array_equal(picture_now, picture)
+        assert nonempty <= 5
+    finally:
+        fresh.reader.close()
+
+
+def test_a_mode_change_writes_no_analysis(pass_root, session):
+    """Switching the 2D mode is a read: the result directory set cannot move.
+
+    The depth map is in the result, so drawing it is a file read like the arrows are.
+    The assertion is the set of result directories before and after every mode the
+    selector offers is selected, drawn and switched away from.
+    """
+    before = {entry.directory.name for entry in scan_results(pass_root)}
+    for mode in (DEPTH, ARROW_PURE, ARROW_STITCHED, "recorded"):
+        frame = session.frame(FIRST_FRAME_ID + 1)
+        if mode != "recorded":
+            assert _drawn(frame, mode) is not None
+    assert {entry.directory.name for entry in scan_results(pass_root)} == before
+
+
+def test_the_depth_statistics_say_which_frames_are_empty(session):
+    """An empty depth view is explainable: the frame says so, and so does the result."""
+    frames = list(session.frames())
+    emptied = [item for item in frames if item.depth_is_empty]
+    carrying = [item for item in frames if item.depth_nonzero]
+    assert len(emptied) + len(carrying) == len(frames)
+    assert all(item.has_depth for item in frames), "a frame lost its depth channel"
+    assert all(item.depth_pixels == 300 * 280 for item in frames)
+    # Every observed frame on this footage falls in one of the two groups, and the empty
+    # ones are exactly zeros rather than NaNs or a shape that failed to decode.
+    for item in emptied:
+        assert not bool(np.any(np.asarray(item.depth)))
+        assert (item.depth_min, item.depth_max) == (0.0, 0.0)
+    for item in carrying:
+        assert float(np.asarray(item.depth).max()) == item.depth_max > 0.0
+
+
+# ------------------------------------------------------- the operator's one line
+def _note_for(image: ImageFrame) -> str:
+    plan = plan_image(image)
+    return "" if not plan.note_key else t(plan.note_key, fields="flow、depth")
+
+
+def test_the_nothing_to_draw_line_appears_only_when_no_result_exists():
+    """The recording's note is an absence statement, and it must be true when it is made.
+
+    Two recording frames that differ in exactly one fact: whether a stored analysis
+    result exists. Without one, the pane really can draw nothing derived, so the line
+    appears and offers the recorded view alone. With one, the arrows and the depth map
+    are two file reads away, so the line is gone — and it is gone because the modes are
+    on offer, not because a message was suppressed.
+    """
+    raw = ImageFrame(np.zeros((8, 8, 3), np.uint8), source=SOURCE_REPLAY)
+    plan = plan_image(raw)
+    assert plan.available_modes == ("recorded",)
+    assert plan.note_key == RAW_NOTE_KEY
+    assert plan.resolve(DEPTH) == "recorded", "a view that cannot be drawn was selected"
+    assert plan.has_depth() is False and plan.has_flow() is False
+
+    backed = ImageFrame(
+        np.zeros((8, 8, 3), np.uint8), source=SOURCE_REPLAY, result_available=True
+    )
+    plan = plan_image(backed)
+    assert plan.note_key == "", "the pane still says there is nothing to draw"
+    assert plan.available_modes == ("recorded", *DERIVED_MODES)
+    assert plan.default_mode == "recorded", "the raw picture stopped being the default"
+    for mode in DERIVED_MODES:
+        assert plan.accepts(mode), f"{mode} is not on offer with a result in hand"
+
+
+def test_the_nothing_to_draw_line_names_the_action_that_produces_a_result():
+    """Both locales: the line says what to do, not merely that there is nothing."""
+    from orisys_app_shared.i18n import set_locale
+
+    seen = {}
+    try:
+        for locale in ("zh_CN", "en_US"):
+            set_locale(locale)
+            text = t(RAW_NOTE_KEY)
+            seen[locale] = text
+            # The action that produces a result is named: the pass runs on open, and the
+            # toolbar action re-runs it.
+            assert "由视频重算" in text or "Recompute from the video" in text, text
+            assert (
+                "分析" in text or "analysis result" in text
+            ), f"{locale} does not say a result is what is missing: {text}"
+    finally:
+        set_locale("zh_CN")
+    assert seen["zh_CN"] != seen["en_US"]
+
+
+def test_the_empty_depth_note_is_not_the_missing_depth_note():
+    """Both locales keep the two apart: a flat map is a reading, an absent one is not."""
+    from orisys_app_shared.i18n import set_locale
+
+    try:
+        for locale in ("zh_CN", "en_US"):
+            set_locale(locale)
+            empty = t("panels.depth_empty", pixels=84000)
+            absent = t("panels.depth_unavailable")
+            assert "84000" in empty or "84,000" in empty, empty
+            assert empty != absent
+    finally:
+        set_locale("zh_CN")
+
+
+def test_a_derived_mode_is_served_from_the_result_even_on_the_recording_timeline(
+    pass_root,
+):
+    """Through the controller: the depth mode draws the stored frame, not the video.
+
+    The timeline is put on the recording (which is the state the operator is in while a
+    pass runs, and the state a chosen source leaves them in), and the mode selector is
+    set to depth. What reaches the window's single 2D sink must be the result's own frame
+    — with its depth map — and not the decoded video frame, which carries none.
+    """
+    _require_inputs()
+    reader = AnalysisReader(_result_directory(pass_root))
+    session = AnalysisSession(
+        reader,
+        display_policy=policy_for_settings(DisplaySettings()),
+        # The recording the result describes. ``available_result`` matches it against the
+        # recording that is open, which is how a previous recording's result cannot
+        # answer for this one.
+        recording=RECORDING,
+    )
+    try:
+        panel = _ImagePanel()
+        controller, host, engine, _ = _controller(image_panel=panel)
+        controller.results.open_session(session)
+        engine.use_analysis(session)
+        engine._mode = MODE_RECORDING  # the timeline reads the video; the view does not
+        engine.set_position(session.time_of(FIRST_FRAME_ID + 1))
+        host.frames.clear()
+
+        panel.set_view_mode(DEPTH)
+        controller.render_frame()
+        assert host.frames, "the depth mode drew nothing"
+        drawn = host.frames[-1]
+        assert drawn.source == SOURCE_REPLAY, "the recording timeline was not kept"
+        assert drawn.result_available is True
+        stored = session.frame(FIRST_FRAME_ID + 1)
+        assert drawn.frame_id == stored.frame_id
+        assert np.array_equal(np.asarray(drawn.depth), np.asarray(stored.depth))
+        assert drawn.depth_valid is True
+        assert plan_image(drawn).note_key == "", "the pane despaired with a result in hand"
+
+        # And the raw view still shows the recording's own picture.
+        panel.set_view_mode("recorded")
+        host.frames.clear()
+        controller.render_frame()
+        assert host.frames[-1].depth is None
+        assert host.frames[-1].source == SOURCE_REPLAY
+
+        # Now through the engine's own way back to the video: it forgets the session it
+        # was borrowing, and the *library* still holds the result for this recording. The
+        # derived mode must keep drawing from it — a result is a property of the
+        # recording, not of which timeline the cursor is on.
+        engine.use_recording()
+        assert engine.analysis() is None
+        panel.set_view_mode(DEPTH)
+        host.frames.clear()
+        controller.render_frame()
+        assert host.frames[-1].depth is not None, "the depth view followed the borrow"
+        assert np.array_equal(
+            np.asarray(host.frames[-1].depth), np.asarray(stored.depth)
+        )
+        assert plan_image(host.frames[-1]).note_key == ""
+    finally:
+        reader.close()

apps/studio/tests/test_replay_controller.py

@@ -285,6 +285,10 @@ class _Engine:
     def video_message(self) -> str:
         return "no frame"
 
+    def analysis(self):
+        """No stored result in this stub: the derived views fall back to the video."""
+        return None
+
     def set_unit(self, unit: str) -> None:
         self.units.append(str(unit))
 
@@ -411,7 +415,6 @@ def test_a_bare_video_is_opened_as_a_video_and_refused_outside_the_view(qt_app,
     bare.write_bytes(b"not really a video")
     engine = _Engine()
     controller, host, engine, parts = _controller(engine=engine)
-    toolbar = parts[0]
 
     host.view = "live"
     assert controller.load_video(bare) is False