diff --git a/apps/studio/src/orisys_studio/app_window.py b/apps/studio/src/orisys_studio/app_window.py index c9d1780..60bfb34 100644 --- a/apps/studio/src/orisys_studio/app_window.py +++ b/apps/studio/src/orisys_studio/app_window.py @@ -17,9 +17,11 @@ from orisys_app_shared.acquisition.display_orientation import ( ) from orisys_app_shared.acquisition.image_backend import ( DEPTH, + SOURCE_ANALYSIS, SOURCE_LIVE, SOURCE_PREVIEW, SOURCE_RECOMPUTE, + SOURCE_REPLAY, DomainMaskLatch, ImageFrame, plan_image, @@ -210,7 +212,6 @@ class StudioMainWindow(QMainWindow): self.session = SessionController( recording=self.recording, packet_sink=self._on_packet_recorded, - packet_for_3d=self._on_packet_for_3d, sensor_type="box", ) self.session.tick.connect(self._on_run_tick) @@ -555,7 +556,25 @@ class StudioMainWindow(QMainWindow): self._show_image_frame(frame) def show_analysis_packet(self, packet) -> None: - self._show_analysis_frame(packet) + self.deliver_measurement(packet, advance="flush", paint_2d=False) + + def deliver_measurement( + self, + packet, + *, + advance: str = "tick", + arrow_policy=None, + image_source: str | None = None, + paint_2d: bool = True, + ) -> None: + """One packet to 2D then 3D: live and replay share this exit.""" + self._deliver_measurement( + packet, + advance=advance, + arrow_policy=arrow_policy, + image_source=image_source, + paint_2d=paint_2d, + ) def drop_render_marker(self) -> None: self._drop_render_marker() @@ -1975,19 +1994,6 @@ class StudioMainWindow(QMainWindow): return describe - def _on_packet_for_3d(self, packet) -> None: - """Runs on the **acquisition thread**. - - The pane copies what it needs into a read-only frame and offers it to a - capacity-one mailbox; it never touches the recording, the session store or the - camera. The view is the one that decides it is switched off, so this stays a - reference read and a method call. - """ - view = self.mesh3d_panel.view - if view is None: - return - view.submit_packet(packet) - def _on_scene3d_settings(self) -> None: """Open the 3D settings dialog — or say why there is nothing to configure.""" if not self.mesh3d_panel.open_settings(): @@ -2346,12 +2352,12 @@ class StudioMainWindow(QMainWindow): # A new session invalidates the previous recording. self._stop_recording(wait=True) self.session.select_worker(sensor_type) - # Both sinks are on the worker we are about to start (``select_worker`` wires - # them, and ``add_sink`` is idempotent): the record channel and the 3D pane. - # The 3D pane has to know that a *new* session is starting: its frames carry - # this session's id, and to the coordinator an unfamiliar id is otherwise a - # stale frame from a previous session — which is how the pane ended up frozen - # on the first run's last picture for every run after it. + # The record channel is the only acquisition-thread sink; 3D is submitted on + # the GUI deliver path with 2D. The 3D pane still has to know that a *new* + # session is starting: its frames carry this session's id, and to the + # coordinator an unfamiliar id is otherwise a stale frame from a previous + # session — which is how the pane ended up frozen on the first run's last + # picture for every run after it. self.mesh3d_panel.begin_session() # The filters for this session, before it opens the device: the chain runs # in the SDK, so this is the one moment the choice can be handed over. @@ -2585,12 +2591,13 @@ class StudioMainWindow(QMainWindow): Errors, end reasons and a stuck connect stay here — they call stop. A live packet's charts, calibration sampling and 2D/3D paint live in the presenter - so a replay owning the image cannot be overwritten by accident. + so a replay owning the image cannot be overwritten by accident. 3D is + submitted with 2D on ``deliver_measurement``; the timer then composes. """ presenter = self.acquire_presenter - if presenter is not None: - presenter.tick_3d(display_paused=self._display_paused) if self._active_worker is None: + if presenter is not None: + presenter.tick_3d(display_paused=self._display_paused) return # Record bookkeeping happens before anything that can return early: a # recording that failed while the device was quiet still has to update @@ -2635,6 +2642,8 @@ class StudioMainWindow(QMainWindow): if packet is None or presenter is None: return presenter.present(packet, display_paused=self._display_paused) + # Compose after deliver_measurement submitted this tick's packet. + presenter.tick_3d(display_paused=self._display_paused) # ------------------------------------------------------------ depth scale def _depth_range(self) -> tuple[float, float] | None: @@ -2708,28 +2717,58 @@ class StudioMainWindow(QMainWindow): return self._domain_mask.update(frame.valid_mask) def _render_run_image(self, packet) -> None: + """Live / recompute entry into the shared 2D→3D deliver path.""" if packet is None or self._replay_owns_image(): return - flow_valid = getattr(packet, "flow_valid", None) - depth_valid = getattr(packet, "depth_valid", None) - stale = bool(getattr(packet, "stale_forces", False)) or ( - flow_valid is False and depth_valid is False - ) - self._show_image_frame( - ImageFrame( - getattr(packet, "processed_frame", None), - flow=getattr(packet, "flow", None), - depth=getattr(packet, "depth", None), - valid_mask=getattr(packet, "valid_mask", None), - source=SOURCE_RECOMPUTE if self._recompute_running else SOURCE_LIVE, - frame_id=int(getattr(packet, "frame_id", 0) or 0), - t_rel_s=float(getattr(packet, "mono_t", 0.0) or 0.0), - stale=stale, - flow_valid=flow_valid, - flow_frame_id=getattr(packet, "flow_frame_id", None), - depth_valid=depth_valid, - depth_frame_id=getattr(packet, "depth_frame_id", None), + self._deliver_measurement(packet, advance="tick") + + def _deliver_measurement( + self, + packet, + *, + advance: str = "tick", + arrow_policy=None, + image_source: str | None = None, + paint_2d: bool = True, + ) -> None: + """Paint 2D from a packet, then submit that same packet to 3D. + + Live and replay differ only in how the packet was built and whether the + 3D mailbox advances on the run timer (``tick``) or must ``flush`` now. + """ + if packet is None: + return + if paint_2d: + flow_valid = getattr(packet, "flow_valid", None) + depth_valid = getattr(packet, "depth_valid", None) + stale = bool(getattr(packet, "stale_forces", False)) or ( + flow_valid is False and depth_valid is False ) + source = image_source + if source is None: + source = ( + SOURCE_RECOMPUTE if self._recompute_running else SOURCE_LIVE + ) + self._show_image_frame( + ImageFrame( + getattr(packet, "processed_frame", None), + flow=getattr(packet, "flow", None), + depth=getattr(packet, "depth", None), + valid_mask=getattr(packet, "valid_mask", None), + source=source, + frame_id=int(getattr(packet, "frame_id", 0) or 0), + t_rel_s=float(getattr(packet, "mono_t", 0.0) or 0.0), + stale=stale, + flow_valid=flow_valid, + flow_frame_id=getattr(packet, "flow_frame_id", None), + depth_valid=depth_valid, + depth_frame_id=getattr(packet, "depth_frame_id", None), + result_available=source in (SOURCE_ANALYSIS, SOURCE_REPLAY), + arrow_policy=arrow_policy, + ) + ) + self._submit_measurement_3d( + packet, advance=advance, overlay_policy=arrow_policy ) def _show_image_frame(self, frame: ImageFrame) -> None: @@ -2934,37 +2973,39 @@ class StudioMainWindow(QMainWindow): self._analysis_reference_override = setup.reference_frame return True - def _show_analysis_frame(self, packet) -> None: - """Hand one stored analysis frame to the 3D pane's producer. + def _submit_measurement_3d( + self, packet, *, advance: str = "tick", overlay_policy=None + ) -> None: + """Submit one packet to the 3D producer after 2D has been painted. - The pane snapshots a packet through its ordinary path, so a replay's figures go - through the same adapter, domain latch and mask intersection as a live run. The - producer is O(1) and Qt-free, and it is the pane that decides whether it is on. - - Live acquisition advances the mailbox on the session run timer. Replay stops - that timer, so after submit the pane must be ticked here — otherwise the frame - sits pending and later submits are skipped (``has_pending_frame``). The overlay - policy is the open session's ``paint_policy``, so 3D arrows share the 2D canvas's - grid and threshold (including the box painter's 2 px floor). + Replay (``advance=\"flush\"``) applies the result's ``paint_policy`` and the + sensor-type gate, then flushes compose because the run timer is stopped. + Live (``advance=\"tick\"``) only submits; the 16 ms timer composes. """ - session = self.replay_ctl.available_result() - provenance = getattr(getattr(session, "reader", None), "provenance", None) - result_type = str(getattr(provenance, "sensor_type", "") or "").strip() - if result_type and result_type != self.sensor_picker.sensor_type(): - # The old replay stays inspectable after a sensor switch, but its - # depth grid must never be submitted to the new sensor's 3D scene. - return + if advance == "flush": + session = self.replay_ctl.available_result() + provenance = getattr(getattr(session, "reader", None), "provenance", None) + result_type = str(getattr(provenance, "sensor_type", "") or "").strip() + if result_type and result_type != self.sensor_picker.sensor_type(): + # The old replay stays inspectable after a sensor switch, but its + # depth grid must never be submitted to the new sensor's 3D scene. + return view = self.mesh3d_panel.view if view is None: return - self._connect_analysis_display(self.mesh3d_panel) - policy = None - replay = getattr(self, "replay_ctl", None) - if replay is not None: - policy = replay._session_policy() - view.set_overlay_policy(policy) + if advance == "flush": + self._connect_analysis_display(self.mesh3d_panel) + policy = overlay_policy + if policy is None: + replay = getattr(self, "replay_ctl", None) + if replay is not None: + policy = replay._session_policy() + view.set_overlay_policy(policy) + elif overlay_policy is not None: + view.set_overlay_policy(overlay_policy) view.submit_packet(packet) - self._tick_analysis_3d() + if advance == "flush": + self._tick_analysis_3d() def _tick_analysis_3d(self) -> None: """Compose and present the analysis frame while replay owns the image. diff --git a/apps/studio/src/orisys_studio/replay_analysis.py b/apps/studio/src/orisys_studio/replay_analysis.py index 45ba764..4b4786d 100644 --- a/apps/studio/src/orisys_studio/replay_analysis.py +++ b/apps/studio/src/orisys_studio/replay_analysis.py @@ -195,6 +195,9 @@ class AnalysisSession: self._domain_hw: tuple[int, int] | None = None self._cache_id: int | None = None self._cache: AnalysedFrame | None = None + #: Session-scoped UV unroll / box stitch for overlay bases (lazy). + self._overlay = None + self._overlay_failed = False # ------------------------------------------------------------------ identity @property @@ -381,8 +384,46 @@ class AnalysisSession: yield self.frame(int(frame_id)) # ------------------------------------------------------------------ packets - def packet(self, frame: AnalysedFrame, *, t_rel_s: float | None = None) -> SdkPacket: - """One frame in the shape the 3D pane's producer consumes. + def overlay_base( + self, video_bgr: np.ndarray, *, frame_id: int | None = None + ) -> np.ndarray | None: + """The live-shaped processed image for this camera frame, or ``None``. + + Opens a session-scoped projector from the result's provenance on first use. + Failure is sticky for the session: overlay views then report capture + unavailable rather than falling back to a resized fisheye. + """ + if self._overlay_failed: + return None + if self._overlay is None: + from orisys_app_shared.analysis.overlay_base import OverlayBaseProjector + + self._overlay = OverlayBaseProjector.open_from_provenance( + self.reader.provenance + ) + if self._overlay is None: + self._overlay_failed = True + return None + return self._overlay.project(video_bgr, frame_id=frame_id) + + def close(self) -> None: + """Release the overlay projector; the reader is closed by :class:`ResultLibrary`.""" + overlay, self._overlay = self._overlay, None + if overlay is not None: + try: + overlay.close() + except Exception: # noqa: BLE001 + logger.debug("overlay projector close skipped", exc_info=True) + + def packet( + self, + frame: AnalysedFrame, + *, + t_rel_s: float | None = None, + processed_frame: np.ndarray | None = None, + raw_frame: np.ndarray | None = None, + ) -> SdkPacket: + """One frame in the shape the 2D sink and 3D producer both consume. The pane is fed through its ordinary path — it snapshots a packet — so a replay's figures go through the same adapter, the same domain latch and the @@ -390,14 +431,20 @@ class AnalysisSession: else would make "the pane looks the same in every mode" an accident of two code paths happening to agree. + ``processed_frame`` is the UV-unrolled or stitched base when the overlay view + has one; without it the stored black placeholder remains (pure-colour arrows + replace that canvas anyway). + The forces are zero and ``forces_valid`` is ``False``: the result stores no force, ``forces.csv`` is where that measurement lives, and a zero here would be a measurement nothing made. """ moment = float(frame.t_rel_s if t_rel_s is None else t_rel_s) + processed = frame.base_bgr if processed_frame is None else processed_frame + raw = frame.base_bgr if raw_frame is None else raw_frame return SdkPacket( - raw_frame=frame.base_bgr, - processed_frame=frame.base_bgr, + raw_frame=raw, + processed_frame=processed, timestamp=moment, mono_t=moment, fps=0.0, @@ -597,6 +644,10 @@ class ResultLibrary: def close(self) -> None: session, self._session = self._session, None if session is not None: + try: + session.close() + except Exception: # noqa: BLE001 - releasing is best effort + logger.debug("could not close the analysis session", exc_info=True) try: session.reader.close() except Exception: # noqa: BLE001 - releasing is best effort diff --git a/apps/studio/src/orisys_studio/replay_controller.py b/apps/studio/src/orisys_studio/replay_controller.py index 71c8d1e..e65d7c3 100644 --- a/apps/studio/src/orisys_studio/replay_controller.py +++ b/apps/studio/src/orisys_studio/replay_controller.py @@ -1037,8 +1037,8 @@ class ReplayController(QObject): picture stays in the same selector, one step away, and the recompute switch is not involved. - The first processed view is arrows over the captured frame: the camera picture - gives immediate context. A derived selection the operator already chose is left + The first processed view is arrows over the unrolled / stitched base: the same + texture live uses. A derived selection the operator already chose is left alone — that mode is not this method's to override. """ current = str(getattr(self._image_panel, "view_mode", lambda: "")() or "") @@ -1249,32 +1249,18 @@ class ReplayController(QObject): return held # ------------------------------------------------------------------ drawing - def _captured_base_for(self, frame) -> np.ndarray | None: - """The camera picture at the result's domain size, or ``None`` when unavailable. + def _processed_base_for(self, frame, session) -> np.ndarray | None: + """The live-shaped processed image under the arrows, or ``None``. Stored analysis frames keep a black placeholder as ``base_bgr``: the pass does - not embed the video. Overlay views need the decoded frame resized onto the - measurement grid — painting the stitch grid onto a larger camera frame would - leave the arrows in a corner (see :meth:`AnalysisSession.domain_hw`). + not embed the video. Overlay views need the same UV unroll / box stitch live + publishes as ``processed_frame`` — a plain resize of the fisheye leaves arrows + on the wrong texture. """ video = self._engine.video_frame_bgr() - if video is None: + if video is None or session is None: return None - height, width = (int(frame.flow.shape[0]), int(frame.flow.shape[1])) - image = np.asarray(video) - if image.ndim == 2: - import cv2 - - image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) - elif image.ndim == 3 and image.shape[2] == 4: - import cv2 - - image = cv2.cvtColor(image, cv2.COLOR_BGRA2BGR) - if int(image.shape[0]) == height and int(image.shape[1]) == width: - return np.ascontiguousarray(image) - import cv2 - - return cv2.resize(image, (width, height), interpolation=cv2.INTER_AREA) + return session.overlay_base(video, frame_id=int(frame.frame_id)) def _analysis_image_frame( self, frame, *, source: str, base_bgr: np.ndarray | None = None @@ -1364,23 +1350,36 @@ class ReplayController(QObject): else: frame = None if frame is not None: - captured = self._captured_base_for(frame) - if wanted == ARROW_STITCHED and captured is None: - # Overlay without a camera picture is a black canvas with dots — say - # so instead of implying the capture is missing from the result. + processed = self._processed_base_for(frame, session) + if wanted == ARROW_STITCHED and processed is None: + # Overlay without the live-shaped base is a black canvas with dots — + # say so instead of painting a resized fisheye under UV-domain arrows. self._image_panel.clear_preview(t("panels.arrow_capture_unavailable")) - self._host.show_analysis_packet(session.packet(frame)) - return - self._host.show_frame( - self._analysis_image_frame( - frame, - source=SOURCE_ANALYSIS if self.analysis_active() else SOURCE_REPLAY, - base_bgr=captured, + self._host.deliver_measurement( + session.packet(frame), + advance="flush", + arrow_policy=self._session_policy(), + image_source=( + SOURCE_ANALYSIS if self.analysis_active() else SOURCE_REPLAY + ), + paint_2d=False, ) + return + raw = self._engine.video_frame_bgr() + packet = session.packet( + frame, + processed_frame=processed, + raw_frame=None if raw is None else np.asarray(raw), + ) + # One cursor, one packet: 2D then 3D on the shared deliver path. + self._host.deliver_measurement( + packet, + advance="flush", + arrow_policy=self._session_policy(), + image_source=( + SOURCE_ANALYSIS if self.analysis_active() else SOURCE_REPLAY + ), ) - # The same frame, in the shape the 3D pane's producer consumes: one - # cursor, two figures, one frame id. - self._host.show_analysis_packet(session.packet(frame)) return image = self._engine.video_frame_bgr() if image is None: diff --git a/apps/studio/src/orisys_studio/session_controller.py b/apps/studio/src/orisys_studio/session_controller.py index bc51551..32788f9 100644 --- a/apps/studio/src/orisys_studio/session_controller.py +++ b/apps/studio/src/orisys_studio/session_controller.py @@ -58,12 +58,14 @@ class SessionController(QObject): self, *, recording: Any, - packet_for_3d: Callable[[Any], None], packet_sink: Callable[[Any], None] | None = None, sensor_type: str = "box", + packet_for_3d: Callable[[Any], None] | None = None, ) -> None: super().__init__() self._recording = recording + #: Legacy optional second sink. Live 3D is delivered on the GUI tick with 2D; + #: keep the parameter so older call sites that still pass a noop do not break. self._packet_for_3d = packet_for_3d #: The record channel's sink. It defaults to this object's own method; the #: window may hand in its own bound method instead, because the tests and the @@ -256,19 +258,20 @@ class SessionController(QObject): self.wire_sinks() def wire_sinks(self) -> None: - """Register the record channel and the 3D pane on both workers (idempotent).""" + """Register the record channel on both workers (idempotent). + + 3D shares the GUI deliver path with 2D (``deliver_measurement``), so it is + not a second acquisition-thread sink. + """ for worker in (self._box_worker, self._finger_worker): if worker is None: continue worker.add_sink(self._packet_sink) - # The 3D pane is a second sink on the same thread. It is registered even - # while the pane is off, because a sink whose cost depends on the click - # order is worse than a view that is told to do nothing (the view's own - # enabled flag). - worker.add_sink(self._packet_for_3d) + if self._packet_for_3d is not None: + worker.add_sink(self._packet_for_3d) def detach_sinks(self) -> None: - """Unregister both sinks. Called on the way out, before Qt tears widgets down. + """Unregister sinks. Called on the way out, before Qt tears widgets down. A worker that is still winding down must not keep a reference to the window (and through it to widgets that no longer exist). @@ -277,7 +280,8 @@ class SessionController(QObject): if worker is None or not hasattr(worker, "remove_sink"): continue worker.remove_sink(self._packet_sink) - worker.remove_sink(self._packet_for_3d) + if self._packet_for_3d is not None: + worker.remove_sink(self._packet_for_3d) def worker_for(self, sensor_type: str) -> Any: self.ensure_workers() diff --git a/apps/studio/src/orisys_studio/shell_ports.py b/apps/studio/src/orisys_studio/shell_ports.py index a77325c..372c409 100644 --- a/apps/studio/src/orisys_studio/shell_ports.py +++ b/apps/studio/src/orisys_studio/shell_ports.py @@ -49,6 +49,16 @@ class ImageSink(Protocol): def show_analysis_packet(self, packet: Any) -> None: ... + def deliver_measurement( + self, + packet: Any, + *, + advance: str = "tick", + arrow_policy: Any = None, + image_source: str | None = None, + paint_2d: bool = True, + ) -> None: ... + def drop_render_marker(self) -> None: ... diff --git a/apps/studio/tests/test_replay_analysis.py b/apps/studio/tests/test_replay_analysis.py index 0a6ff37..41087a4 100644 --- a/apps/studio/tests/test_replay_analysis.py +++ b/apps/studio/tests/test_replay_analysis.py @@ -465,6 +465,32 @@ class _Host: self.frames.append(frame) def show_analysis_packet(self, packet) -> None: + self.deliver_measurement(packet, advance="flush", paint_2d=False) + + def deliver_measurement( + self, + packet, + *, + advance: str = "tick", + arrow_policy=None, + image_source: str | None = None, + paint_2d: bool = True, + ) -> None: + if paint_2d: + from orisys_app_shared.acquisition.image_backend import ImageFrame + + self.frames.append( + ImageFrame( + getattr(packet, "processed_frame", None), + flow=getattr(packet, "flow", None), + depth=getattr(packet, "depth", None), + valid_mask=getattr(packet, "valid_mask", None), + source=image_source or "replay", + frame_id=int(getattr(packet, "frame_id", 0) or 0), + arrow_policy=arrow_policy, + result_available=True, + ) + ) self.analysis_packets.append(packet) def drop_render_marker(self) -> None: diff --git a/apps/studio/tests/test_replay_controller.py b/apps/studio/tests/test_replay_controller.py index e1f6e3c..8234e1e 100644 --- a/apps/studio/tests/test_replay_controller.py +++ b/apps/studio/tests/test_replay_controller.py @@ -262,6 +262,32 @@ class _Host: self.frames.append(frame) def show_analysis_packet(self, packet) -> None: + self.deliver_measurement(packet, advance="flush", paint_2d=False) + + def deliver_measurement( + self, + packet, + *, + advance: str = "tick", + arrow_policy=None, + image_source: str | None = None, + paint_2d: bool = True, + ) -> None: + if paint_2d: + from orisys_app_shared.acquisition.image_backend import ImageFrame + + self.frames.append( + ImageFrame( + getattr(packet, "processed_frame", None), + flow=getattr(packet, "flow", None), + depth=getattr(packet, "depth", None), + valid_mask=getattr(packet, "valid_mask", None), + source=image_source or "replay", + frame_id=int(getattr(packet, "frame_id", 0) or 0), + arrow_policy=arrow_policy, + result_available=True, + ) + ) self.analysis_packets.append(packet) def drop_render_marker(self) -> None: diff --git a/apps/studio/tests/test_scene3d_view.py b/apps/studio/tests/test_scene3d_view.py index d13e574..71a052f 100644 --- a/apps/studio/tests/test_scene3d_view.py +++ b/apps/studio/tests/test_scene3d_view.py @@ -388,8 +388,8 @@ def _run_session(make_window, record_root, *, with_3d: bool): if with_3d: view = _RecordingView() window.mesh3d_panel.attach_view(view) - # With no pane the sink still runs (it is registered on the worker) and does - # nothing, which is the case the comparison below is about. + # 3D is submitted on the GUI deliver path (same tick as 2D), not as an + # acquisition-thread sink — with or without a pane attached. window.toolbar.btn_record.setChecked(True) for index in range(96): stuck.emit(ui_check._fake_packet(index, mono_t=index / 30.0)) @@ -427,20 +427,20 @@ def test_the_3d_consumer_cannot_change_the_record(make_window, next_record_root) assert "t_rel_s" in header and len(with_3d_csv.splitlines()) > 10 -def test_the_3d_sink_is_registered_and_inert_when_the_pane_is_off(make_window, next_record_root): - """A sink that has to be added and removed around a toggle is a state machine.""" +def test_live_3d_is_not_an_acquisition_thread_sink(make_window, next_record_root): + """3D shares the GUI deliver path with 2D; only the record channel is a sink.""" import orisys_studio.app_window as app_module window, stuck = _running_window(make_window) app_module.recordings_dir = lambda: next_record_root() try: - assert window._on_packet_recorded in stuck.sinks() - assert window._on_packet_for_3d in stuck.sinks(), ( - "the 3D sink must be registered with the worker, not attached per toggle" + assert stuck.sinks() == (window._on_packet_recorded,), ( + "live 3D must not register a second acquisition-thread sink" ) assert window.mesh3d_panel.view is None, "a box sensor must not build a view" for index in range(10): stuck.emit(ui_check._fake_packet(index)) + window._on_run_tick() assert window.session_store.count == 10 assert window.mesh3d_panel.view is None finally: diff --git a/shared/src/orisys_app_shared/analysis/overlay_base.py b/shared/src/orisys_app_shared/analysis/overlay_base.py new file mode 100644 index 0000000..588c5ef --- /dev/null +++ b/shared/src/orisys_app_shared/analysis/overlay_base.py @@ -0,0 +1,202 @@ +"""Session-scoped overlay base: UV unroll or box stitch without re-running flow. + +Replay stores flow and depth, not the processed image. Live paints arrows on +``SdkPacket.processed_frame`` (finger UV unroll or box stitch). This projector +rebuilds that same base from a decoded camera frame using the result's profile, +so scrubbing can match live without caching rendered pixels or reopening a video +per frame. +""" + +from __future__ import annotations + +import logging +import os +import shutil +import tempfile +import threading +from pathlib import Path +from typing import Any + +import cv2 +import numpy as np + +logger = logging.getLogger(__name__) + +__all__ = ["OverlayBaseProjector"] + +_CWD_LOCK = threading.Lock() + + +def _ensure_bgr(image: np.ndarray) -> np.ndarray: + if image.ndim == 2: + return cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) + if image.ndim == 3 and image.shape[2] == 1: + return cv2.cvtColor(image[:, :, 0], cv2.COLOR_GRAY2BGR) + if image.ndim == 3 and image.shape[2] == 4: + return cv2.cvtColor(image, cv2.COLOR_BGRA2BGR) + return np.ascontiguousarray(image) + + +class OverlayBaseProjector: + """Maps one camera frame onto the measurement domain for overlay views.""" + + def __init__( + self, + *, + sensor_type: str, + project, + close=None, + working: Path | None = None, + ) -> None: + self.sensor_type = str(sensor_type) + self._project = project + self._close = close + self._working = None if working is None else Path(working) + self._cache_id: int | None = None + self._cache: np.ndarray | None = None + + @classmethod + def open_from_provenance(cls, provenance: Any) -> OverlayBaseProjector | None: + """Build a projector from an analysis result's provenance, or ``None``.""" + kind = str(getattr(provenance, "sensor_type", "") or "").strip() + config_ref = str(getattr(provenance, "sensor_config_ref", "") or "").strip() + if kind == "finger": + return cls._open_finger(config_ref=config_ref or None) + if kind == "box": + return cls._open_box(config_ref=config_ref or None) + logger.warning("no overlay projector for sensor type %r", kind) + return None + + @classmethod + def _open_finger(cls, *, config_ref: str | None) -> OverlayBaseProjector | None: + from orisys.finger.runtime import resolve_finger_geometry + from orisys.frame_processors import FrameUnwarper + from orisys.unroll import build_or_load_domain + from orisys_app_shared.analysis.service import resolve_finger_export_dir + + if not config_ref: + logger.warning("finger overlay needs sensor_config_ref from provenance") + return None + export_dir = resolve_finger_export_dir(config_ref) + if export_dir is None: + return None + try: + geometry = resolve_finger_geometry( + export_dir=export_dir, + config=str(config_ref), + ) + mesh_path, intrinsics_path, pose_path = geometry.export_paths.as_tuple() + domain = build_or_load_domain( + mesh_path, + intrinsics_path, + pose_path, + geometry.map_cache, + geometry.source_K, + geometry.source_width, + geometry.source_height, + geometry.out_width, + geometry.out_height, + edge_expand=geometry.edge_expand, + include_surface=True, + verbose=False, + source_scale=geometry.source_scale, + ) + domain.remap.ensure_fast_maps() + unwarper = FrameUnwarper( + domain.remap, + camera_matrix=( + geometry.source_K if geometry.dist_coeffs is not None else None + ), + dist_coeffs=geometry.dist_coeffs, + source_width=geometry.source_width, + source_height=geometry.source_height, + flip_source_vertical=True, + ) + except Exception: # noqa: BLE001 - missing assets become "capture unavailable" + logger.warning("could not open finger overlay projector", exc_info=True) + return None + + def project(frame: np.ndarray) -> np.ndarray | None: + # Match live ``process_frame(..., color=False)``: grey unroll, then BGR. + prepared = unwarper.prepare_source_gray(frame) + unrolled = unwarper.flatten_prepared_gray(prepared) + if unrolled is None: + return None + return _ensure_bgr(unrolled) + + return cls(sensor_type="finger", project=project) + + @classmethod + def _open_box(cls, *, config_ref: str | None) -> OverlayBaseProjector | None: + from orisys import Sensor + + working = Path(tempfile.mkdtemp(prefix="orisys-overlay-box-")) + try: + with _CWD_LOCK: + previous = Path.cwd() + os.chdir(working) + try: + sensor = Sensor( + "external", + config_name=str(config_ref) if config_ref else "box", + verbose=False, + backend="auto", + ) + finally: + os.chdir(previous) + except Exception: # noqa: BLE001 + logger.warning("could not open box overlay projector", exc_info=True) + shutil.rmtree(working, ignore_errors=True) + return None + + def project(frame: np.ndarray) -> np.ndarray | None: + # Stitch only — no deformation / contact. Same grey domain live publishes. + processed = sensor.get_img(frame=frame) + if processed is None: + return None + return _ensure_bgr(processed) + + def close() -> None: + try: + sensor.disconnect() + except Exception: # noqa: BLE001 + logger.debug("box overlay sensor disconnect skipped", exc_info=True) + shutil.rmtree(working, ignore_errors=True) + + return cls(sensor_type="box", project=project, close=close, working=working) + + def project( + self, video_bgr: np.ndarray, *, frame_id: int | None = None + ) -> np.ndarray | None: + """Return the measurement-domain BGR base, or ``None`` when remap fails.""" + if frame_id is not None and self._cache is not None and self._cache_id == int( + frame_id + ): + return self._cache + image = np.asarray(video_bgr) + if image.ndim == 2: + image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) + elif image.ndim == 3 and image.shape[2] == 4: + image = cv2.cvtColor(image, cv2.COLOR_BGRA2BGR) + try: + projected = self._project(image) + except Exception: # noqa: BLE001 + logger.warning("overlay projection failed", exc_info=True) + return None + if projected is None: + return None + result = np.ascontiguousarray(projected) + if frame_id is not None: + self._cache_id = int(frame_id) + self._cache = result + return result + + def close(self) -> None: + self._cache = None + self._cache_id = None + if self._close is not None: + try: + self._close() + except Exception: # noqa: BLE001 + logger.debug("overlay projector close skipped", exc_info=True) + self._close = None diff --git a/shared/tests/test_overlay_base.py b/shared/tests/test_overlay_base.py new file mode 100644 index 0000000..e42b674 --- /dev/null +++ b/shared/tests/test_overlay_base.py @@ -0,0 +1,35 @@ +"""Overlay base projector: session cache and no fisheye resize fallback.""" + +from __future__ import annotations + +import numpy as np + +from orisys_app_shared.analysis.overlay_base import OverlayBaseProjector + + +def test_overlay_projector_reuses_the_same_frame_id() -> None: + calls: list[int] = [] + + def project(frame: np.ndarray) -> np.ndarray: + calls.append(1) + out = np.zeros((8, 8, 3), dtype=np.uint8) + out[..., 1] = 40 + return out + + projector = OverlayBaseProjector(sensor_type="finger", project=project) + first = projector.project(np.zeros((32, 32, 3), dtype=np.uint8), frame_id=7) + second = projector.project(np.ones((32, 32, 3), dtype=np.uint8) * 9, frame_id=7) + assert len(calls) == 1 + assert first is second + third = projector.project(np.zeros((32, 32, 3), dtype=np.uint8), frame_id=8) + assert len(calls) == 2 + assert third is not second + projector.close() + + +def test_open_from_unknown_sensor_type_returns_none() -> None: + class _Prov: + sensor_type = "unknown" + sensor_config_ref = "" + + assert OverlayBaseProjector.open_from_provenance(_Prov()) is None