orisys-tech/orisys-app
3
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Store the depth channel in an analysis result and serve it to the 2D depth view

Browse Source 
A replay could draw arrows but not depth, and the recording view said there was
nothing to draw even when a result existed. Depth is now quantised per frame
(14 B empty, 486 B at the worst frame seen), read from the cache by the same
draw path live uses, and the raw-recording note appears only when no result
exists. Format version 2; see apps/studio/docs/replay-analysis.md.
This commit is contained in:
ykcheungab
2026-10-04 14:11:22 +08:00
parent cf49b0836e
commit 9e20fd0277
17 changed files with 1412 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/studio/src/orisys_studio/app_window.py
+20 -5
View File
@@ -2610,11 +2610,26 @@ class StudioMainWindow(QMainWindow):
self._depth_range_value = (lo, hi)
return self._depth_range_value
def _depth_scale_text(self) -> str:
def _depth_scale_text(self, frame: ImageFrame | None = None) -> str:
"""The colour scale, plus — when the frame's depth is exactly zero — why it is flat.
A depth view of an all-zero map is a flat island in the ramp's low colour, which
looks the same as a broken view. The frame knows which it is: the pass stored the
channel, its non-zero count and its range, so "0 of 84,000 pixels" can be said
instead of leaving the operator to wonder. It is not the same statement as
``panels.depth_unavailable``, which is for a frame that carries no depth at all.
"""
span = self._depth_range()
if span is None:
return ""
return t("panels.depth_scale", low=f"{span[0]:.3f}", high=f"{span[1]:.3f}")
text = "" if span is None else t(
"panels.depth_scale", low=f"{span[0]:.3f}", high=f"{span[1]:.3f}"
)
if frame is None or frame.depth is None:
return text
array = np.asarray(frame.depth)
if array.size and not bool(np.any(array)):
empty = t("panels.depth_empty", pixels=int(array.size))
return f"{text} · {empty}" if text else empty
return text
def _latched_domain_mask(self, frame: ImageFrame):
"""The 2D domain boundary to draw, latched for the session.
@@ -2694,7 +2709,7 @@ class StudioMainWindow(QMainWindow):
self.arrow_panel.set_valid_area(frame.valid_mask is not None)
if mode == DEPTH:
self._last_depth_frame = frame.depth
self.arrow_panel.set_depth_scale(self._depth_scale_text())
self.arrow_panel.set_depth_scale(self._depth_scale_text(frame))
else:
self.arrow_panel.set_depth_scale(None)
rendered = render_image(
apps/studio/src/orisys_studio/replay_analysis.py
+24
View File
@@ -110,6 +110,26 @@ class AnalysedFrame:
base_bgr: np.ndarray
depth_nonzero: int = 0
dropped: bool = False
#: The stored depth channel of this frame, or ``None`` when the pass recorded none.
#: An all-zero map is a *measurement* — "the response was zero everywhere" — and is
#: handed over as an array; the depth view draws it and the statistics that came
#: with it say why it is flat. See ``analysis.model`` for the encoding.
depth: np.ndarray | None = None
depth_min: float = 0.0
depth_max: float = 0.0
@property
def has_depth(self) -> bool:
return self.depth is not None
@property
def depth_pixels(self) -> int:
return 0 if self.depth is None else int(self.depth.size)
@property
def depth_is_empty(self) -> bool:
"""True when this frame has a depth channel and it is exactly zero everywhere."""
return self.depth is not None and self.depth_nonzero == 0
@property
def domain_hw(self) -> tuple[int, int]:
@@ -342,6 +362,7 @@ class AnalysisSession:
samples = payload.samples(threshold=self.threshold())
flow = self._recover_flow(samples, shape_hw)
base = np.zeros((shape_hw[0], shape_hw[1], 3), dtype=np.uint8)
depth = getattr(payload, "depth", None)
return AnalysedFrame(
frame_id=int(payload.frame_id),
t_rel_s=float(payload.t_rel_s),
@@ -351,6 +372,9 @@ class AnalysisSession:
base_bgr=base,
depth_nonzero=int(getattr(payload, "depth_nonzero", 0) or 0),
dropped=bool(getattr(payload, "dropped", False)),
depth=None if depth is None else np.asarray(depth, dtype=np.float32),
depth_min=float(getattr(payload, "depth_min", 0.0) or 0.0),
depth_max=float(getattr(payload, "depth_max", 0.0) or 0.0),
)
def _recover_flow(
apps/studio/src/orisys_studio/replay_controller.py
+98 -27
View File
@@ -28,7 +28,12 @@ from pathlib import Path
from typing import Any, Protocol
import numpy as np
from orisys_app_shared.acquisition.image_backend import SOURCE_ANALYSIS, SOURCE_REPLAY, ImageFrame
from orisys_app_shared.acquisition.image_backend import (
DERIVED_MODES,
SOURCE_ANALYSIS,
SOURCE_REPLAY,
ImageFrame,
)
from orisys_app_shared.i18n import t
from orisys_app_shared.paths import recordings_dir
from PySide6.QtCore import QObject, QTimer
@@ -891,17 +896,75 @@ class ReplayController(QObject):
self._actions.recompute.blockSignals(False)
self._toolbar.set_recompute_checked(bool(value))
def available_result(self):
"""The stored result for the recording that is open, if there is one.
Deliberately not ``engine.analysis()``: that answers "which session is the
*timeline* borrowing", and it is cleared when the timeline goes back to reading
the video (``ReplayEngine.use_recording``). A result is a property of the
recording, not of the cursor's mode, and it is the question the panel's one line
has to be answered from — "is there a result?" is why the arrows and the depth
map can be drawn at all. The session the library holds is checked against the
recording that is open, so a previous recording's result cannot answer for this
one.
"""
session = self._engine.analysis()
if session is not None:
return session
held = self.results.session
if held is None:
return None
current = self.recording_directory()
stored = getattr(held, "recording", None)
if current is None or stored is None or Path(stored) != Path(current):
return None
return held
# ------------------------------------------------------------------ drawing
def _analysis_image_frame(self, frame, *, source: str) -> ImageFrame:
"""One stored frame in the shape the window's single 2D sink takes.
The *only* place a replay builds a 2D frame from a result, so the analysis
timeline and a recording whose mode selector asks for a derived view cannot
build two slightly different pictures. ``source`` says which timeline the
operator is on — ``replay`` keeps the raw view on offer while the derived modes
are drawn from the result, so choosing a derived view is not a one-way door.
"""
return ImageFrame(
frame.base_bgr,
flow=frame.flow,
depth=frame.depth,
valid_mask=frame.valid_mask,
source=source,
frame_id=int(frame.frame_id),
t_rel_s=float(frame.t_rel_s),
flow_valid=True,
flow_frame_id=int(frame.frame_id),
depth_valid=bool(frame.has_depth),
depth_frame_id=int(frame.frame_id) if frame.has_depth else None,
# The result this frame came from is the thing that makes arrows and depth
# drawable at all; the plan reads it to decide what to offer.
result_available=True,
)
def render_frame(self) -> None:
"""Draw what the 2D panel should show for the current replay cursor.
One method for both timeline modes, because it is one cursor: in analysis mode
the picture is the stored frame's own flow field (the pass's candidate
locations, on the domain grid), and in recording mode it is the decoded video
frame, which carries no flow and no depth — so the plan says so in one line and
the panel offers exactly the one view the frame can be. The window's single 2D
sink draws both through the *same* backend, so the arrows, the idle dots, the
colour range and the domain boundary behave identically in the two modes.
the picture is the stored frame's own flow field and depth map (the pass's
candidate locations, on the domain grid), and in recording mode it is the
decoded video frame. The window's single 2D sink draws both through the *same*
backend, so the arrows, the idle dots, the colour range and the domain boundary
behave identically in the two modes.
**A mode that needs derived data is served from the result, never faked.** The
arrows and the depth map are the pass's measurements; with a result in hand they
are read from it and drawn through this same path, from the cursor's own frame.
The recording's own picture answers only for the mode that *is* the recording's
picture, so no mode can silently present the raw image as an arrow plot. With no
result there is nothing to serve: the plan offers the recorded view alone, and
the panel says so — naming the action that produces a result — instead of leaving
the operator to guess why the arrow view is empty.
Nothing here writes anything: a display change reaches this method through the
engine's ``redraw()`` and rebuilds the picture from the same stored frame.
@@ -911,30 +974,34 @@ class ReplayController(QObject):
if self._recompute_running:
# The pipeline is drawing its own frames (see ``owns_image``).
return
if self.analysis_active():
frame = self._engine.analysed_frame()
if frame is None:
self._image_panel.clear_preview(self._engine.video_message())
return
self._host._show_image_frame(
ImageFrame(
frame.base_bgr,
flow=frame.flow,
valid_mask=frame.valid_mask,
source=SOURCE_ANALYSIS,
frame_id=int(frame.frame_id),
t_rel_s=float(frame.t_rel_s),
flow_valid=True,
flow_frame_id=int(frame.frame_id),
depth_valid=False,
session = self.available_result()
wanted = str(getattr(self._image_panel, "view_mode", lambda: "")() or "")
if session is not None:
if self.analysis_active():
frame = self._engine.analysed_frame()
if frame is None:
self._image_panel.clear_preview(self._engine.video_message())
return
elif wanted in DERIVED_MODES:
# The timeline reads the video, but this view is not the video: it is a
# measurement, and the result is where measurements live.
frame = session.frame_at(self._engine.position())
if frame is None:
self._image_panel.clear_preview(self._engine.video_message())
return
else:
frame = None
if frame is not None:
self._host._show_image_frame(
self._analysis_image_frame(
frame,
source=SOURCE_ANALYSIS if self.analysis_active() else SOURCE_REPLAY,
)
)
)
session = self._engine.analysis()
if session is not None:
# The same frame, in the shape the 3D pane's producer consumes: one
# cursor, two figures, one frame id.
self._host._show_analysis_frame(session.packet(frame))
return
return
image = self._engine.video_frame_bgr()
if image is None:
self._image_panel.clear_preview(self._engine.video_message())
@@ -944,6 +1011,10 @@ class ReplayController(QObject):
image,
source=SOURCE_REPLAY,
t_rel_s=float(self._engine.position()),
# A result exists, so the derived views are offered and are drawn from
# it (above). Without one there is nothing behind the picture and the
# panel says exactly that.
result_available=session is not None,
)
)