回放叠加改用展开底图,并统一 2D/3D 投递。

箭头/采集图像与 live 的 processed_frame 对齐;deliver_measurement 先画 2D 再提交同一包给 3D。已用 overlay、scene3d 与 replay 相关测试验证。
This commit is contained in:
ykcheungab
2026-10-06 21:21:37 +08:00
parent 321e52f47a
commit 829d2c1b53
10 changed files with 519 additions and 125 deletions
+109 -68
View File
@@ -17,9 +17,11 @@ from orisys_app_shared.acquisition.display_orientation import (
)
from orisys_app_shared.acquisition.image_backend import (
DEPTH,
SOURCE_ANALYSIS,
SOURCE_LIVE,
SOURCE_PREVIEW,
SOURCE_RECOMPUTE,
SOURCE_REPLAY,
DomainMaskLatch,
ImageFrame,
plan_image,
@@ -210,7 +212,6 @@ class StudioMainWindow(QMainWindow):
self.session = SessionController(
recording=self.recording,
packet_sink=self._on_packet_recorded,
packet_for_3d=self._on_packet_for_3d,
sensor_type="box",
)
self.session.tick.connect(self._on_run_tick)
@@ -555,7 +556,25 @@ class StudioMainWindow(QMainWindow):
self._show_image_frame(frame)
def show_analysis_packet(self, packet) -> None:
self._show_analysis_frame(packet)
self.deliver_measurement(packet, advance="flush", paint_2d=False)
def deliver_measurement(
self,
packet,
*,
advance: str = "tick",
arrow_policy=None,
image_source: str | None = None,
paint_2d: bool = True,
) -> None:
"""One packet to 2D then 3D: live and replay share this exit."""
self._deliver_measurement(
packet,
advance=advance,
arrow_policy=arrow_policy,
image_source=image_source,
paint_2d=paint_2d,
)
def drop_render_marker(self) -> None:
self._drop_render_marker()
@@ -1975,19 +1994,6 @@ class StudioMainWindow(QMainWindow):
return describe
def _on_packet_for_3d(self, packet) -> None:
"""Runs on the **acquisition thread**.
The pane copies what it needs into a read-only frame and offers it to a
capacity-one mailbox; it never touches the recording, the session store or the
camera. The view is the one that decides it is switched off, so this stays a
reference read and a method call.
"""
view = self.mesh3d_panel.view
if view is None:
return
view.submit_packet(packet)
def _on_scene3d_settings(self) -> None:
"""Open the 3D settings dialog — or say why there is nothing to configure."""
if not self.mesh3d_panel.open_settings():
@@ -2346,12 +2352,12 @@ class StudioMainWindow(QMainWindow):
# A new session invalidates the previous recording.
self._stop_recording(wait=True)
self.session.select_worker(sensor_type)
# Both sinks are on the worker we are about to start (``select_worker`` wires
# them, and ``add_sink`` is idempotent): the record channel and the 3D pane.
# The 3D pane has to know that a *new* session is starting: its frames carry
# this session's id, and to the coordinator an unfamiliar id is otherwise a
# stale frame from a previous session — which is how the pane ended up frozen
# on the first run's last picture for every run after it.
# The record channel is the only acquisition-thread sink; 3D is submitted on
# the GUI deliver path with 2D. The 3D pane still has to know that a *new*
# session is starting: its frames carry this session's id, and to the
# coordinator an unfamiliar id is otherwise a stale frame from a previous
# session — which is how the pane ended up frozen on the first run's last
# picture for every run after it.
self.mesh3d_panel.begin_session()
# The filters for this session, before it opens the device: the chain runs
# in the SDK, so this is the one moment the choice can be handed over.
@@ -2585,12 +2591,13 @@ class StudioMainWindow(QMainWindow):
Errors, end reasons and a stuck connect stay here — they call stop. A live
packet's charts, calibration sampling and 2D/3D paint live in the presenter
so a replay owning the image cannot be overwritten by accident.
so a replay owning the image cannot be overwritten by accident. 3D is
submitted with 2D on ``deliver_measurement``; the timer then composes.
"""
presenter = self.acquire_presenter
if presenter is not None:
presenter.tick_3d(display_paused=self._display_paused)
if self._active_worker is None:
if presenter is not None:
presenter.tick_3d(display_paused=self._display_paused)
return
# Record bookkeeping happens before anything that can return early: a
# recording that failed while the device was quiet still has to update
@@ -2635,6 +2642,8 @@ class StudioMainWindow(QMainWindow):
if packet is None or presenter is None:
return
presenter.present(packet, display_paused=self._display_paused)
# Compose after deliver_measurement submitted this tick's packet.
presenter.tick_3d(display_paused=self._display_paused)
# ------------------------------------------------------------ depth scale
def _depth_range(self) -> tuple[float, float] | None:
@@ -2708,28 +2717,58 @@ class StudioMainWindow(QMainWindow):
return self._domain_mask.update(frame.valid_mask)
def _render_run_image(self, packet) -> None:
"""Live / recompute entry into the shared 2D→3D deliver path."""
if packet is None or self._replay_owns_image():
return
flow_valid = getattr(packet, "flow_valid", None)
depth_valid = getattr(packet, "depth_valid", None)
stale = bool(getattr(packet, "stale_forces", False)) or (
flow_valid is False and depth_valid is False
)
self._show_image_frame(
ImageFrame(
getattr(packet, "processed_frame", None),
flow=getattr(packet, "flow", None),
depth=getattr(packet, "depth", None),
valid_mask=getattr(packet, "valid_mask", None),
source=SOURCE_RECOMPUTE if self._recompute_running else SOURCE_LIVE,
frame_id=int(getattr(packet, "frame_id", 0) or 0),
t_rel_s=float(getattr(packet, "mono_t", 0.0) or 0.0),
stale=stale,
flow_valid=flow_valid,
flow_frame_id=getattr(packet, "flow_frame_id", None),
depth_valid=depth_valid,
depth_frame_id=getattr(packet, "depth_frame_id", None),
self._deliver_measurement(packet, advance="tick")
def _deliver_measurement(
self,
packet,
*,
advance: str = "tick",
arrow_policy=None,
image_source: str | None = None,
paint_2d: bool = True,
) -> None:
"""Paint 2D from a packet, then submit that same packet to 3D.
Live and replay differ only in how the packet was built and whether the
3D mailbox advances on the run timer (``tick``) or must ``flush`` now.
"""
if packet is None:
return
if paint_2d:
flow_valid = getattr(packet, "flow_valid", None)
depth_valid = getattr(packet, "depth_valid", None)
stale = bool(getattr(packet, "stale_forces", False)) or (
flow_valid is False and depth_valid is False
)
source = image_source
if source is None:
source = (
SOURCE_RECOMPUTE if self._recompute_running else SOURCE_LIVE
)
self._show_image_frame(
ImageFrame(
getattr(packet, "processed_frame", None),
flow=getattr(packet, "flow", None),
depth=getattr(packet, "depth", None),
valid_mask=getattr(packet, "valid_mask", None),
source=source,
frame_id=int(getattr(packet, "frame_id", 0) or 0),
t_rel_s=float(getattr(packet, "mono_t", 0.0) or 0.0),
stale=stale,
flow_valid=flow_valid,
flow_frame_id=getattr(packet, "flow_frame_id", None),
depth_valid=depth_valid,
depth_frame_id=getattr(packet, "depth_frame_id", None),
result_available=source in (SOURCE_ANALYSIS, SOURCE_REPLAY),
arrow_policy=arrow_policy,
)
)
self._submit_measurement_3d(
packet, advance=advance, overlay_policy=arrow_policy
)
def _show_image_frame(self, frame: ImageFrame) -> None:
@@ -2934,37 +2973,39 @@ class StudioMainWindow(QMainWindow):
self._analysis_reference_override = setup.reference_frame
return True
def _show_analysis_frame(self, packet) -> None:
"""Hand one stored analysis frame to the 3D pane's producer.
def _submit_measurement_3d(
self, packet, *, advance: str = "tick", overlay_policy=None
) -> None:
"""Submit one packet to the 3D producer after 2D has been painted.
The pane snapshots a packet through its ordinary path, so a replay's figures go
through the same adapter, domain latch and mask intersection as a live run. The
producer is O(1) and Qt-free, and it is the pane that decides whether it is on.
Live acquisition advances the mailbox on the session run timer. Replay stops
that timer, so after submit the pane must be ticked here — otherwise the frame
sits pending and later submits are skipped (``has_pending_frame``). The overlay
policy is the open session's ``paint_policy``, so 3D arrows share the 2D canvas's
grid and threshold (including the box painter's 2 px floor).
Replay (``advance=\"flush\"``) applies the result's ``paint_policy`` and the
sensor-type gate, then flushes compose because the run timer is stopped.
Live (``advance=\"tick\"``) only submits; the 16 ms timer composes.
"""
session = self.replay_ctl.available_result()
provenance = getattr(getattr(session, "reader", None), "provenance", None)
result_type = str(getattr(provenance, "sensor_type", "") or "").strip()
if result_type and result_type != self.sensor_picker.sensor_type():
# The old replay stays inspectable after a sensor switch, but its
# depth grid must never be submitted to the new sensor's 3D scene.
return
if advance == "flush":
session = self.replay_ctl.available_result()
provenance = getattr(getattr(session, "reader", None), "provenance", None)
result_type = str(getattr(provenance, "sensor_type", "") or "").strip()
if result_type and result_type != self.sensor_picker.sensor_type():
# The old replay stays inspectable after a sensor switch, but its
# depth grid must never be submitted to the new sensor's 3D scene.
return
view = self.mesh3d_panel.view
if view is None:
return
self._connect_analysis_display(self.mesh3d_panel)
policy = None
replay = getattr(self, "replay_ctl", None)
if replay is not None:
policy = replay._session_policy()
view.set_overlay_policy(policy)
if advance == "flush":
self._connect_analysis_display(self.mesh3d_panel)
policy = overlay_policy
if policy is None:
replay = getattr(self, "replay_ctl", None)
if replay is not None:
policy = replay._session_policy()
view.set_overlay_policy(policy)
elif overlay_policy is not None:
view.set_overlay_policy(overlay_policy)
view.submit_packet(packet)
self._tick_analysis_3d()
if advance == "flush":
self._tick_analysis_3d()
def _tick_analysis_3d(self) -> None:
"""Compose and present the analysis frame while replay owns the image.
@@ -195,6 +195,9 @@ class AnalysisSession:
self._domain_hw: tuple[int, int] | None = None
self._cache_id: int | None = None
self._cache: AnalysedFrame | None = None
#: Session-scoped UV unroll / box stitch for overlay bases (lazy).
self._overlay = None
self._overlay_failed = False
# ------------------------------------------------------------------ identity
@property
@@ -381,8 +384,46 @@ class AnalysisSession:
yield self.frame(int(frame_id))
# ------------------------------------------------------------------ packets
def packet(self, frame: AnalysedFrame, *, t_rel_s: float | None = None) -> SdkPacket:
"""One frame in the shape the 3D pane's producer consumes.
def overlay_base(
self, video_bgr: np.ndarray, *, frame_id: int | None = None
) -> np.ndarray | None:
"""The live-shaped processed image for this camera frame, or ``None``.
Opens a session-scoped projector from the result's provenance on first use.
Failure is sticky for the session: overlay views then report capture
unavailable rather than falling back to a resized fisheye.
"""
if self._overlay_failed:
return None
if self._overlay is None:
from orisys_app_shared.analysis.overlay_base import OverlayBaseProjector
self._overlay = OverlayBaseProjector.open_from_provenance(
self.reader.provenance
)
if self._overlay is None:
self._overlay_failed = True
return None
return self._overlay.project(video_bgr, frame_id=frame_id)
def close(self) -> None:
"""Release the overlay projector; the reader is closed by :class:`ResultLibrary`."""
overlay, self._overlay = self._overlay, None
if overlay is not None:
try:
overlay.close()
except Exception: # noqa: BLE001
logger.debug("overlay projector close skipped", exc_info=True)
def packet(
self,
frame: AnalysedFrame,
*,
t_rel_s: float | None = None,
processed_frame: np.ndarray | None = None,
raw_frame: np.ndarray | None = None,
) -> SdkPacket:
"""One frame in the shape the 2D sink and 3D producer both consume.
The pane is fed through its ordinary path — it snapshots a packet — so a
replay's figures go through the same adapter, the same domain latch and the
@@ -390,14 +431,20 @@ class AnalysisSession:
else would make "the pane looks the same in every mode" an accident of two code
paths happening to agree.
``processed_frame`` is the UV-unrolled or stitched base when the overlay view
has one; without it the stored black placeholder remains (pure-colour arrows
replace that canvas anyway).
The forces are zero and ``forces_valid`` is ``False``: the result stores no
force, ``forces.csv`` is where that measurement lives, and a zero here would be
a measurement nothing made.
"""
moment = float(frame.t_rel_s if t_rel_s is None else t_rel_s)
processed = frame.base_bgr if processed_frame is None else processed_frame
raw = frame.base_bgr if raw_frame is None else raw_frame
return SdkPacket(
raw_frame=frame.base_bgr,
processed_frame=frame.base_bgr,
raw_frame=raw,
processed_frame=processed,
timestamp=moment,
mono_t=moment,
fps=0.0,
@@ -597,6 +644,10 @@ class ResultLibrary:
def close(self) -> None:
session, self._session = self._session, None
if session is not None:
try:
session.close()
except Exception: # noqa: BLE001 - releasing is best effort
logger.debug("could not close the analysis session", exc_info=True)
try:
session.reader.close()
except Exception: # noqa: BLE001 - releasing is best effort
@@ -1037,8 +1037,8 @@ class ReplayController(QObject):
picture stays in the same selector, one step away, and the recompute switch is
not involved.
The first processed view is arrows over the captured frame: the camera picture
gives immediate context. A derived selection the operator already chose is left
The first processed view is arrows over the unrolled / stitched base: the same
texture live uses. A derived selection the operator already chose is left
alone — that mode is not this method's to override.
"""
current = str(getattr(self._image_panel, "view_mode", lambda: "")() or "")
@@ -1249,32 +1249,18 @@ class ReplayController(QObject):
return held
# ------------------------------------------------------------------ drawing
def _captured_base_for(self, frame) -> np.ndarray | None:
"""The camera picture at the result's domain size, or ``None`` when unavailable.
def _processed_base_for(self, frame, session) -> np.ndarray | None:
"""The live-shaped processed image under the arrows, or ``None``.
Stored analysis frames keep a black placeholder as ``base_bgr``: the pass does
not embed the video. Overlay views need the decoded frame resized onto the
measurement grid — painting the stitch grid onto a larger camera frame would
leave the arrows in a corner (see :meth:`AnalysisSession.domain_hw`).
not embed the video. Overlay views need the same UV unroll / box stitch live
publishes as ``processed_frame`` — a plain resize of the fisheye leaves arrows
on the wrong texture.
"""
video = self._engine.video_frame_bgr()
if video is None:
if video is None or session is None:
return None
height, width = (int(frame.flow.shape[0]), int(frame.flow.shape[1]))
image = np.asarray(video)
if image.ndim == 2:
import cv2
image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)
elif image.ndim == 3 and image.shape[2] == 4:
import cv2
image = cv2.cvtColor(image, cv2.COLOR_BGRA2BGR)
if int(image.shape[0]) == height and int(image.shape[1]) == width:
return np.ascontiguousarray(image)
import cv2
return cv2.resize(image, (width, height), interpolation=cv2.INTER_AREA)
return session.overlay_base(video, frame_id=int(frame.frame_id))
def _analysis_image_frame(
self, frame, *, source: str, base_bgr: np.ndarray | None = None
@@ -1364,23 +1350,36 @@ class ReplayController(QObject):
else:
frame = None
if frame is not None:
captured = self._captured_base_for(frame)
if wanted == ARROW_STITCHED and captured is None:
# Overlay without a camera picture is a black canvas with dots — say
# so instead of implying the capture is missing from the result.
processed = self._processed_base_for(frame, session)
if wanted == ARROW_STITCHED and processed is None:
# Overlay without the live-shaped base is a black canvas with dots —
# say so instead of painting a resized fisheye under UV-domain arrows.
self._image_panel.clear_preview(t("panels.arrow_capture_unavailable"))
self._host.show_analysis_packet(session.packet(frame))
return
self._host.show_frame(
self._analysis_image_frame(
frame,
source=SOURCE_ANALYSIS if self.analysis_active() else SOURCE_REPLAY,
base_bgr=captured,
self._host.deliver_measurement(
session.packet(frame),
advance="flush",
arrow_policy=self._session_policy(),
image_source=(
SOURCE_ANALYSIS if self.analysis_active() else SOURCE_REPLAY
),
paint_2d=False,
)
return
raw = self._engine.video_frame_bgr()
packet = session.packet(
frame,
processed_frame=processed,
raw_frame=None if raw is None else np.asarray(raw),
)
# One cursor, one packet: 2D then 3D on the shared deliver path.
self._host.deliver_measurement(
packet,
advance="flush",
arrow_policy=self._session_policy(),
image_source=(
SOURCE_ANALYSIS if self.analysis_active() else SOURCE_REPLAY
),
)
# The same frame, in the shape the 3D pane's producer consumes: one
# cursor, two figures, one frame id.
self._host.show_analysis_packet(session.packet(frame))
return
image = self._engine.video_frame_bgr()
if image is None:
@@ -58,12 +58,14 @@ class SessionController(QObject):
self,
*,
recording: Any,
packet_for_3d: Callable[[Any], None],
packet_sink: Callable[[Any], None] | None = None,
sensor_type: str = "box",
packet_for_3d: Callable[[Any], None] | None = None,
) -> None:
super().__init__()
self._recording = recording
#: Legacy optional second sink. Live 3D is delivered on the GUI tick with 2D;
#: keep the parameter so older call sites that still pass a noop do not break.
self._packet_for_3d = packet_for_3d
#: The record channel's sink. It defaults to this object's own method; the
#: window may hand in its own bound method instead, because the tests and the
@@ -256,19 +258,20 @@ class SessionController(QObject):
self.wire_sinks()
def wire_sinks(self) -> None:
"""Register the record channel and the 3D pane on both workers (idempotent)."""
"""Register the record channel on both workers (idempotent).
3D shares the GUI deliver path with 2D (``deliver_measurement``), so it is
not a second acquisition-thread sink.
"""
for worker in (self._box_worker, self._finger_worker):
if worker is None:
continue
worker.add_sink(self._packet_sink)
# The 3D pane is a second sink on the same thread. It is registered even
# while the pane is off, because a sink whose cost depends on the click
# order is worse than a view that is told to do nothing (the view's own
# enabled flag).
worker.add_sink(self._packet_for_3d)
if self._packet_for_3d is not None:
worker.add_sink(self._packet_for_3d)
def detach_sinks(self) -> None:
"""Unregister both sinks. Called on the way out, before Qt tears widgets down.
"""Unregister sinks. Called on the way out, before Qt tears widgets down.
A worker that is still winding down must not keep a reference to the window
(and through it to widgets that no longer exist).
@@ -277,7 +280,8 @@ class SessionController(QObject):
if worker is None or not hasattr(worker, "remove_sink"):
continue
worker.remove_sink(self._packet_sink)
worker.remove_sink(self._packet_for_3d)
if self._packet_for_3d is not None:
worker.remove_sink(self._packet_for_3d)
def worker_for(self, sensor_type: str) -> Any:
self.ensure_workers()
@@ -49,6 +49,16 @@ class ImageSink(Protocol):
def show_analysis_packet(self, packet: Any) -> None: ...
def deliver_measurement(
self,
packet: Any,
*,
advance: str = "tick",
arrow_policy: Any = None,
image_source: str | None = None,
paint_2d: bool = True,
) -> None: ...
def drop_render_marker(self) -> None: ...
+26
View File
@@ -465,6 +465,32 @@ class _Host:
self.frames.append(frame)
def show_analysis_packet(self, packet) -> None:
self.deliver_measurement(packet, advance="flush", paint_2d=False)
def deliver_measurement(
self,
packet,
*,
advance: str = "tick",
arrow_policy=None,
image_source: str | None = None,
paint_2d: bool = True,
) -> None:
if paint_2d:
from orisys_app_shared.acquisition.image_backend import ImageFrame
self.frames.append(
ImageFrame(
getattr(packet, "processed_frame", None),
flow=getattr(packet, "flow", None),
depth=getattr(packet, "depth", None),
valid_mask=getattr(packet, "valid_mask", None),
source=image_source or "replay",
frame_id=int(getattr(packet, "frame_id", 0) or 0),
arrow_policy=arrow_policy,
result_available=True,
)
)
self.analysis_packets.append(packet)
def drop_render_marker(self) -> None:
@@ -262,6 +262,32 @@ class _Host:
self.frames.append(frame)
def show_analysis_packet(self, packet) -> None:
self.deliver_measurement(packet, advance="flush", paint_2d=False)
def deliver_measurement(
self,
packet,
*,
advance: str = "tick",
arrow_policy=None,
image_source: str | None = None,
paint_2d: bool = True,
) -> None:
if paint_2d:
from orisys_app_shared.acquisition.image_backend import ImageFrame
self.frames.append(
ImageFrame(
getattr(packet, "processed_frame", None),
flow=getattr(packet, "flow", None),
depth=getattr(packet, "depth", None),
valid_mask=getattr(packet, "valid_mask", None),
source=image_source or "replay",
frame_id=int(getattr(packet, "frame_id", 0) or 0),
arrow_policy=arrow_policy,
result_available=True,
)
)
self.analysis_packets.append(packet)
def drop_render_marker(self) -> None:
+7 -7
View File
@@ -388,8 +388,8 @@ def _run_session(make_window, record_root, *, with_3d: bool):
if with_3d:
view = _RecordingView()
window.mesh3d_panel.attach_view(view)
# With no pane the sink still runs (it is registered on the worker) and does
# nothing, which is the case the comparison below is about.
# 3D is submitted on the GUI deliver path (same tick as 2D), not as an
# acquisition-thread sink — with or without a pane attached.
window.toolbar.btn_record.setChecked(True)
for index in range(96):
stuck.emit(ui_check._fake_packet(index, mono_t=index / 30.0))
@@ -427,20 +427,20 @@ def test_the_3d_consumer_cannot_change_the_record(make_window, next_record_root)
assert "t_rel_s" in header and len(with_3d_csv.splitlines()) > 10
def test_the_3d_sink_is_registered_and_inert_when_the_pane_is_off(make_window, next_record_root):
"""A sink that has to be added and removed around a toggle is a state machine."""
def test_live_3d_is_not_an_acquisition_thread_sink(make_window, next_record_root):
"""3D shares the GUI deliver path with 2D; only the record channel is a sink."""
import orisys_studio.app_window as app_module
window, stuck = _running_window(make_window)
app_module.recordings_dir = lambda: next_record_root()
try:
assert window._on_packet_recorded in stuck.sinks()
assert window._on_packet_for_3d in stuck.sinks(), (
"the 3D sink must be registered with the worker, not attached per toggle"
assert stuck.sinks() == (window._on_packet_recorded,), (
"live 3D must not register a second acquisition-thread sink"
)
assert window.mesh3d_panel.view is None, "a box sensor must not build a view"
for index in range(10):
stuck.emit(ui_check._fake_packet(index))
window._on_run_tick()
assert window.session_store.count == 10
assert window.mesh3d_panel.view is None
finally: