diff --git a/apps/studio/config/locales/en_US.json b/apps/studio/config/locales/en_US.json index f2d64c5..b261662 100644 --- a/apps/studio/config/locales/en_US.json +++ b/apps/studio/config/locales/en_US.json @@ -149,6 +149,12 @@ "view_mode_tip": "Image display mode while running", "replay_pipeline": "Recorded video (raw camera image)", "replay_pipeline_tip": "Replay shows the recorded raw camera image with no live arrow overlay; to check the algorithm, use View → Recompute forces from the video", + "replay_raw_note": "A recording stores only the raw camera image: there is no flow or depth to draw. Use “Recompute from the video” for the algorithm's output", + "frame_missing_note": "This frame carries no {fields}: there is nothing for this view to draw", + "field_flow": "flow", + "field_depth": "depth", + "recompute_pipeline": "Recomputed from the recording (not the recorded value)", + "recompute_pipeline_tip": "This frame is what the current algorithm version computes for that footage — not the measurement that was recorded at the time.", "replay_empty": "No recording open\nPress “Open recording…” below", "arrow_empty": "No image yet\nPick a video or camera, then press Run", "arrow_empty_tip": "The deformation arrow image appears here while running or previewing", diff --git a/apps/studio/config/locales/zh_CN.json b/apps/studio/config/locales/zh_CN.json index d0841fa..9e47ff0 100644 --- a/apps/studio/config/locales/zh_CN.json +++ b/apps/studio/config/locales/zh_CN.json @@ -149,6 +149,12 @@ "view_mode_tip": "运行时的图像显示模式", "replay_pipeline": "录制视频（原始相机图像）", "replay_pipeline_tip": "回放显示的是录制下来的原始相机图像，不含实时箭头叠加；要复核算法请用「视图 → 由视频重算力值」", + "replay_raw_note": "录像只保存原始相机图像：这里没有光流/深度可画。要看算法处理结果，请用「由视频重算」", + "frame_missing_note": "本帧没有{fields}：这个视图这一帧没有数据可画", + "field_flow": "光流", + "field_depth": "深度", + "recompute_pipeline": "由录像重算（非记录值）", + "recompute_pipeline_tip": "这一帧是当前算法版本对录像重新计算得到的，不是当初记录下来的测量值。", "replay_empty": "尚未打开录制\n点击下方「打开录制…」", "arrow_empty": "尚未采集图像\n请选择视频或摄像机后点击「运行」", "arrow_empty_tip": "运行或预览后在此显示形变箭头图", diff --git a/apps/studio/src/orisys_studio/app_window.py b/apps/studio/src/orisys_studio/app_window.py index 028e2ba..6caa2bf 100644 --- a/apps/studio/src/orisys_studio/app_window.py +++ b/apps/studio/src/orisys_studio/app_window.py @@ -29,6 +29,16 @@ from PyQt5.QtWidgets import ( ) from orisys_app_shared.acquisition.constants import FORCE_SAMPLE_RATE_HZ +from orisys_app_shared.acquisition.image_backend import ( + DEPTH, + SOURCE_LIVE, + SOURCE_PREVIEW, + SOURCE_RECOMPUTE, + SOURCE_REPLAY, + ImageFrame, + plan_image, + render_image, +) from orisys_app_shared.acquisition.sdk_worker import short_capture_fallacy from orisys_app_shared.force import select_display_forces from orisys_app_shared.force.calibration_recorder import CalibrationRecorder @@ -48,7 +58,7 @@ from orisys_app_shared.theme import build_qss, clear_theme_cache, color from orisys_studio.ui import record_status as record_states from orisys_studio.ui.about_dialog import AboutDialog -from orisys_studio.ui.arrow_panel import ArrowPanel +from orisys_studio.ui.arrow_panel import ArrowPanel, bgr_to_qimage from orisys_studio.ui.brand_bar import BrandBar from orisys_studio.ui.cal_panel import CalPanel from orisys_studio.ui.force_plots import CHANNEL_LABELS, ForcePlotsPanel @@ -111,11 +121,9 @@ CONNECT_SLOW_HINT_S = 8.0 PLAYBACK_TICK_MS = 33 -def _bgr_to_qimage(frame: np.ndarray) -> QImage: - rgb = frame[:, :, ::-1].copy() - h, w, ch = rgb.shape - bytes_per_line = ch * w - return QImage(rgb.data, w, h, bytes_per_line, QImage.Format_RGB888).copy() +#: Kept under its historical name for ``tools/ui_snapshot.py``, which imports it from +#: here. There is one conversion in the application (``ui.arrow_panel.bgr_to_qimage``). +_bgr_to_qimage = bgr_to_qimage def _reveal_in_explorer(path: Path) -> None: @@ -1047,7 +1055,7 @@ class StudioMainWindow(QMainWindow): # the data, and the 2D panel shows what it produces. The pass starts # here so the switch is not a blank chart waiting for a second click. self.force_plots.clear_recording_source() - self.arrow_panel.set_source_kind("live") + self.arrow_panel.set_source_kind("recompute") self.toolbar.set_source_mode("video") self.toolbar.set_video_path(str(segment)) self._refresh_status_rail() @@ -1075,18 +1083,29 @@ class StudioMainWindow(QMainWindow): self._render_replay_frame() def _render_replay_frame(self) -> None: - """Draw what the 2D panel should show for the current replay cursor.""" + """Draw what the 2D panel should show for the current replay cursor. + + A recording stores the raw camera image and nothing else, so the frame this + builds has no flow and no depth — the plan then says so, in one line, and the + panel offers exactly the one view the frame can be. The processed picture of the + same footage is what 「从视频重算」 produces, through the *same* backend. + """ if self._view_preset != "replay" or not self.replay.has_video(): return if self._recompute_running: # The pipeline is drawing its own frames (see ``_replay_owns_image``). return - self.arrow_panel.set_depth_scale(None) - image = self.replay.video_image() + image = self.replay.video_frame_bgr() if image is None: self.arrow_panel.clear_preview(self.replay.video_message()) - else: - self.arrow_panel.show_frame(image) + return + self._show_image_frame( + ImageFrame( + image, + source=SOURCE_REPLAY, + t_rel_s=float(self.replay.position()), + ) + ) def _apply_replay_source(self, active: bool) -> None: """Point the charts at the recording (or back at the live session).""" @@ -2405,7 +2424,9 @@ class StudioMainWindow(QMainWindow): self._stop_preview(announce=True) self.statusBar().showMessage(t("status.preview_failed", error="no frame")) return - self.arrow_panel.show_frame(_bgr_to_qimage(frame)) + # A preview is the camera, unprocessed: it goes through the same sink as + # everything else, where the plan states that it carries no field. + self._show_image_frame(ImageFrame(frame, source=SOURCE_PREVIEW)) def _on_run_toggled(self, checked: bool) -> None: if self.toolbar.run_role() == "playback": @@ -2862,59 +2883,62 @@ class StudioMainWindow(QMainWindow): def _render_run_image(self, packet) -> None: if packet is None or self._replay_owns_image(): return - from orisys_app_shared.acquisition.arrows import ( - arrow_display_scale, - depth_map_to_bgr, - draw_arrow_display, - flow_colormap, - hex_to_bgr, + self._show_image_frame( + ImageFrame( + getattr(packet, "processed_frame", None), + flow=getattr(packet, "flow", None), + depth=getattr(packet, "depth", None), + valid_mask=getattr(packet, "valid_mask", None), + source=SOURCE_RECOMPUTE if self._recompute_running else SOURCE_LIVE, + frame_id=int(getattr(packet, "frame_id", 0) or 0), + t_rel_s=float(getattr(packet, "mono_t", 0.0) or 0.0), + ) ) - mode = self.arrow_panel.view_mode() + def _show_image_frame(self, frame: ImageFrame) -> None: + """The single 2D sink: plan the frame, render it, hand the panel one image. + + Live, recomputed, replayed and previewed frames all arrive here, so the mode + set and the annotations follow the frame's own fields rather than which view + happens to be on screen. + """ + from orisys_app_shared.acquisition.arrows import arrow_display_scale, hex_to_bgr + + plan = plan_image(frame) + self.arrow_panel.set_plan(plan) + mode = plan.resolve(self.arrow_panel.view_mode()) + self.arrow_panel.set_view_mode(mode) self._last_rendered_view_mode = mode draw_style = self._draw_style() # The legend is decided by the same inputs as the picture: the arrow # magnification of this sensor's draw style, and whether this frame # carries a valid-area mask. self.arrow_panel.set_arrow_scale(arrow_display_scale(draw_style)) - self.arrow_panel.set_valid_area(getattr(packet, "valid_mask", None) is not None) - colormap = flow_colormap(self._theme) - if mode == "depth": - self._last_depth_frame = packet.depth - depth_bgr = depth_map_to_bgr( - packet.depth, - value_range=self._depth_range(), - invalid_mask=packet.valid_mask, - ) - if depth_bgr is None: - # Never substitute the processed frame here: that made a missing - # depth map look like a depth reading. Say so instead. - self.arrow_panel.clear_preview(t("panels.depth_unavailable")) - self.arrow_panel.set_depth_scale(None) - return + self.arrow_panel.set_valid_area(frame.valid_mask is not None) + if mode == DEPTH: + self._last_depth_frame = frame.depth self.arrow_panel.set_depth_scale(self._depth_scale_text()) - frame = depth_bgr - elif mode == "arrows_stitched": - frame = draw_arrow_display( - packet.processed_frame, - packet.flow, - background="captured", - draw_style=draw_style, - valid_mask=packet.valid_mask, - colormap=colormap, - ) else: - canvas_bgr = hex_to_bgr(color(self._theme, "arrow_canvas", "#0B1830")) - frame = draw_arrow_display( - packet.processed_frame, - packet.flow, - background="pure_color", - canvas_bgr=canvas_bgr, - draw_style=draw_style, - valid_mask=packet.valid_mask, - colormap=colormap, + self.arrow_panel.set_depth_scale(None) + rendered = render_image( + frame, + plan, + mode=mode, + theme=self._theme, + draw_style=draw_style, + depth_range=self._depth_range(), + canvas_bgr=hex_to_bgr(color(self._theme, "arrow_canvas", "#0B1830")), + ) + if rendered is None: + # Never substitute another picture here: a missing depth map that showed + # the processed frame made "no depth" look like a depth reading. + self.arrow_panel.clear_preview( + t("panels.depth_unavailable") + if mode == DEPTH + else t("panels.arrow_empty") ) - self.arrow_panel.show_frame(_bgr_to_qimage(frame)) + return + self.arrow_panel.show_frame(bgr_to_qimage(rendered)) def _restore_geometry(self) -> None: geo = self.prefs.get("window_geometry") diff --git a/apps/studio/src/orisys_studio/replay.py b/apps/studio/src/orisys_studio/replay.py index ea4db37..8324c7a 100644 --- a/apps/studio/src/orisys_studio/replay.py +++ b/apps/studio/src/orisys_studio/replay.py @@ -15,10 +15,10 @@ from __future__ import annotations import logging from pathlib import Path +from typing import Optional import numpy as np from PyQt5.QtCore import QObject, pyqtSignal -from PyQt5.QtGui import QImage from orisys_app_shared.i18n import t from orisys_app_shared.record import RecordingReader, VideoOnlySource @@ -26,12 +26,6 @@ from orisys_app_shared.record import RecordingReader, VideoOnlySource logger = logging.getLogger(__name__) -def _bgr_to_qimage(frame: np.ndarray) -> QImage: - rgb = frame[:, :, ::-1].copy() - height, width, channels = rgb.shape - return QImage(rgb.data, width, height, channels * width, QImage.Format_RGB888).copy() - - def _sdk_version() -> str: """The SDK version in use, or "" when the SDK is not importable.""" try: @@ -64,7 +58,7 @@ class ReplayController(QObject): self._position = 0.0 #: The frame cache: one video image per cursor position. self._video_position: float | None = None - self._video_qimage: QImage | None = None + self._video_bgr: Optional[np.ndarray] = None self._video_reason = "" self._warnings = "" @@ -171,7 +165,7 @@ class ReplayController(QObject): def _reset_frames(self) -> None: self._video_position = None - self._video_qimage = None + self._video_bgr = None self._video_reason = "" # ---------------------------------------------------------------- cursor @@ -209,9 +203,21 @@ class ReplayController(QObject): return self._unit # ----------------------------------------------------------------- frames - def video_image(self) -> QImage | None: + def video_frame_bgr(self) -> Optional[np.ndarray]: + """The recorded frame under the cursor, as BGR. + + The engine hands over pixels, not a prepared widget image: the single 2D + backend (``acquisition.image_backend``) decides what the picture is. + """ self._refresh_video() - return self._video_qimage + return self._video_bgr + + def video_image(self): + """The same frame as a ``QImage`` (kept for callers that predate the plan).""" + from orisys_studio.ui.arrow_panel import bgr_to_qimage + + frame = self.video_frame_bgr() + return None if frame is None else bgr_to_qimage(frame) def video_message(self) -> str: self._refresh_video() @@ -220,7 +226,7 @@ class ReplayController(QObject): def _refresh_video(self) -> None: if self._reader is None and self._video is None: self._video_position = None - self._video_qimage = None + self._video_bgr = None self._video_reason = t("playback.none_found") return if self._video_position == self._position: @@ -229,15 +235,15 @@ class ReplayController(QObject): if self._video is not None: image, _frame = self._video.video_frame_at(self._position) if image is None: - self._video_qimage = None + self._video_bgr = None self._video_reason = t("playback.frame_unreadable") return self._video_reason = "" - self._video_qimage = _bgr_to_qimage(image) + self._video_bgr = image return image, frame = self._reader.video_frame_at(self._position) if image is None: - self._video_qimage = None + self._video_bgr = None self._video_reason = ( t("playback.frame_dropped") if frame is not None and not frame.in_video @@ -245,7 +251,7 @@ class ReplayController(QObject): ) return self._video_reason = "" - self._video_qimage = _bgr_to_qimage(image) + self._video_bgr = image # ---------------------------------------------------------------- notes def warnings_text(self) -> str: diff --git a/apps/studio/src/orisys_studio/ui/arrow_panel.py b/apps/studio/src/orisys_studio/ui/arrow_panel.py index a430bf2..fef006a 100644 --- a/apps/studio/src/orisys_studio/ui/arrow_panel.py +++ b/apps/studio/src/orisys_studio/ui/arrow_panel.py @@ -1,7 +1,17 @@ -"""2D image panel: arrows / depth view with mode selector.""" +"""2D image panel: arrows / depth view with mode selector. + +The modes offered are **not** decided here: they come from the frame itself, through +``orisys_app_shared.acquisition.image_backend.plan_image`` (see +``ArrowPanel.set_plan``). A recorded frame has no flow and no depth, so it has exactly +one thing it can be — a raw picture — and the panel says so instead of offering a view +that would have to invent the data. ``set_source_kind`` keeps its older role: naming +where the picture comes from (live / replay), which words the footer. +""" from __future__ import annotations +from typing import Optional + from PyQt5.QtCore import QSize, Qt, pyqtSignal from PyQt5.QtGui import QImage, QPainter, QPixmap, QTransform from PyQt5.QtWidgets import ( @@ -13,19 +23,38 @@ from PyQt5.QtWidgets import ( QVBoxLayout, ) +from orisys_app_shared.acquisition.image_backend import ( + ARROW_PURE, + ARROW_STITCHED, + DEPTH, + ImagePlan, +) from orisys_app_shared.i18n import t from orisys_app_shared.theme import color from orisys_studio.ui.fit_text import ShrinkableLabel -VIEW_MODES = ("arrows_pure", "arrows_stitched", "depth") +VIEW_MODES = (ARROW_PURE, ARROW_STITCHED, DEPTH) #: While replaying a recording there is no optical flow to draw and no depth is #: stored: the panel shows the recorded frames, and nothing else. REPLAY_VIEW_MODES = ("recorded",) + + +def bgr_to_qimage(frame) -> QImage: + """The one BGR→QImage conversion in the application. + + It used to exist twice (the window and the replay controller), which is how a + picture can look slightly different depending on which path produced it. + """ + rgb = frame[:, :, ::-1].copy() + height, width, channels = rgb.shape + return QImage(rgb.data, width, height, channels * width, QImage.Format_RGB888).copy() + + _VIEW_LABELS = { - "arrows_pure": "panels.view_arrows_pure", - "arrows_stitched": "panels.view_arrows_stitched", - "depth": "panels.view_depth", + ARROW_PURE: "panels.view_arrows_pure", + ARROW_STITCHED: "panels.view_arrows_stitched", + DEPTH: "panels.view_depth", "recorded": "panels.view_recorded", } _ROTATION_STEPS = (0, 90, 180, 270) @@ -141,6 +170,11 @@ class ArrowPanel(QFrame): self._arrow_scale = 2.0 self._valid_area = False self._source_kind = "live" + #: The modes the frame on screen can be shown as, when a plan has been given. + #: Empty means "no plan yet": the preset-based list is used, which is how the + #: panel behaved before the plans existed. + self._modes: tuple[str, ...] = () + self._plan_note = "" self.view_combo = QComboBox() self.view_combo.setMinimumWidth(140) self.view_combo.currentIndexChanged.connect(self._on_view_index_changed) @@ -180,6 +214,11 @@ class ArrowPanel(QFrame): self.scale_label = annotation() self.pipeline_label = annotation() self.area_label = annotation() + # One line that explains a *plan* the operator may not expect ("this recording + # stores only the raw camera image"), so the panel never looks like it forgot + # to process the picture. + self.note_label = annotation() + self.note_label.setVisible(False) # Shows the fixed depth colour-scale range, so two frames can be # compared instead of the scale silently re-fitting every frame. self.depth_scale_label = QLabel() @@ -190,6 +229,7 @@ class ArrowPanel(QFrame): footer.addWidget(self.scale_label) footer.addWidget(self.pipeline_label) footer.addWidget(self.area_label) + footer.addWidget(self.note_label) footer.addStretch(1) footer.addWidget(self.colorbar) footer.addWidget(self.depth_scale_label) @@ -200,9 +240,11 @@ class ArrowPanel(QFrame): return str(data or "arrows_pure") def set_view_mode(self, mode: str) -> None: - wanted = str(mode or "arrows_pure") - if wanted not in VIEW_MODES: - wanted = "arrows_pure" + wanted = str(mode or "") + if wanted not in self.view_modes(): + # A mode this frame cannot offer: fall back to the first one it can. + modes = self.view_modes() + wanted = modes[0] if modes else ARROW_PURE idx = self.view_combo.findData(wanted) if idx < 0: idx = 0 @@ -276,12 +318,17 @@ class ArrowPanel(QFrame): """Switch between live frames and a recording being replayed. A recording has no optical flow to draw, so the arrow entries disappear - rather than offering a view that would be a lie. + rather than offering a view that would be a lie. When a plan has been given + (``set_plan``) it is the plan that decides the modes; this only words the + footer: ``live`` (the acquisition pipeline), ``recompute`` (the same pipeline + being run over a recording) or ``replay`` (the recording's own frames). """ - wanted = "replay" if str(kind) == "replay" else "live" + wanted = str(kind) if str(kind) in ("live", "recompute", "replay") else "live" if wanted == self._source_kind: return self._source_kind = wanted + # No plan for this picture: fall back to the preset-based list. + self._modes = () self._populate_view_combo() self._refresh_pipeline_label() self._refresh_footer_visibility() @@ -289,10 +336,43 @@ class ArrowPanel(QFrame): self._placeholder = self._empty_message() self.image_label.setText(self._placeholder) + def set_plan(self, plan: Optional[ImagePlan]) -> None: + """Adopt what the frame on screen can be shown as. + + The modes come from the frame's own fields (see ``image_backend.plan_image``), + so the same video offers the same views whether it is being replayed or + recomputed — and a raw frame offers exactly one. The current selection is kept + when the new plan still allows it. + """ + modes = tuple(plan.available_modes) if plan is not None else () + note = "" + if plan is not None and plan.note_key: + if plan.note_fields: + names = "、".join( + t(f"panels.field_{name}") + for name in plan.note_fields + if name in ("flow", "depth") + ) + note = t(plan.note_key, fields=names) + else: + note = t(plan.note_key) + if modes != self._modes or note != self._plan_note: + self._modes = modes + self._plan_note = note + self.note_label.setText(note) + self._populate_view_combo() + self._refresh_footer_visibility() + + def plan_modes(self) -> tuple[str, ...]: + """The modes currently offered (empty means "no plan: preset-based").""" + return self._modes + def source_kind(self) -> str: return self._source_kind def view_modes(self) -> tuple[str, ...]: + if self._modes: + return self._modes return REPLAY_VIEW_MODES if self._source_kind == "replay" else VIEW_MODES def set_theme(self, theme: str) -> None: @@ -313,23 +393,28 @@ class ArrowPanel(QFrame): self.scale_label.setToolTip(t("panels.arrow_scale_tip")) def _refresh_footer_visibility(self) -> None: - """Arrow annotations and the depth legend are mutually exclusive.""" - depth = self.view_mode() == "depth" - replay = self._source_kind == "replay" - # The magnification and the valid area describe the live flow drawing; - # neither exists for a recorded frame. - self.scale_label.setVisible(not depth and not replay) + """Arrow annotations and the depth legend are mutually exclusive. + + What is shown follows the *modes this frame supports*: the magnification and + the valid area describe a flow drawing, so they are shown only when an arrow + mode is on offer. + """ + depth = self.view_mode() == DEPTH + modes = self.view_modes() + has_arrows = ARROW_PURE in modes or ARROW_STITCHED in modes + self.scale_label.setVisible(not depth and has_arrows) self.pipeline_label.setVisible(not depth) - self.area_label.setVisible(not depth and not replay) + self.area_label.setVisible(not depth and has_arrows) self.colorbar.setVisible(depth) self.depth_scale_label.setVisible(depth and bool(self._depth_scale_text)) + self.note_label.setVisible(bool(self._plan_note)) def _populate_view_combo(self) -> None: current = self.view_combo.currentData() self.view_combo.blockSignals(True) self.view_combo.clear() for mode in self.view_modes(): - self.view_combo.addItem(t(_VIEW_LABELS[mode]), mode) + self.view_combo.addItem(t(_VIEW_LABELS.get(mode, mode)), mode) if current: idx = self.view_combo.findData(current) if idx >= 0: @@ -344,11 +429,12 @@ class ArrowPanel(QFrame): self.title.setText(t("panels.arrow")) for i, mode in enumerate(self.view_modes()): if i < self.view_combo.count(): - self.view_combo.setItemText(i, t(_VIEW_LABELS[mode])) + self.view_combo.setItemText(i, t(_VIEW_LABELS.get(mode, mode))) self.view_combo.setToolTip(t("panels.view_mode_tip")) self.image_label.setToolTip(t("panels.arrow_empty_tip")) self._refresh_pipeline_label() self._refresh_scale_label() + self.note_label.setText(self._plan_note) self.set_valid_area(self._valid_area) if not self.image_label.pixmap() or self.image_label.pixmap().isNull(): self._placeholder = self._empty_message() @@ -359,6 +445,12 @@ class ArrowPanel(QFrame): self.pipeline_label.setText(t("panels.replay_pipeline")) self.pipeline_label.setToolTip(t("panels.replay_pipeline_tip")) return + if self._source_kind == "recompute": + # The pipeline is drawing again, over a recording: the same picture as a + # live run, about footage rather than about now. + self.pipeline_label.setText(t("panels.recompute_pipeline")) + self.pipeline_label.setToolTip(t("panels.recompute_pipeline_tip")) + return self.pipeline_label.setText(t("panels.arrow_pipeline")) self.pipeline_label.setToolTip(t("panels.arrow_pipeline_tip")) diff --git a/apps/studio/tests/test_ui_smoke.py b/apps/studio/tests/test_ui_smoke.py index b2ab149..df3524c 100644 --- a/apps/studio/tests/test_ui_smoke.py +++ b/apps/studio/tests/test_ui_smoke.py @@ -2151,6 +2151,102 @@ def test_a_bare_video_plays_even_though_it_is_not_a_recording(make_window, recor window.close() +def test_the_replay_and_the_live_paths_share_one_image_backend(make_window, record_root): + """Stage 7: one renderer for every 2D producer, and modes from the frame itself. + + A recording stores the raw camera image and nothing else, so its plan offers exactly + one view and says why; a recomputed (or live) frame offers the derived views and goes + through the *same* ``render_image``. Before this, the replay path handed a QImage + straight to the panel while live/recompute went through two different drawing calls. + """ + import cv2 + import numpy as np + from PyQt5.QtWidgets import QApplication + + import orisys_studio.app_window as app_module + from orisys_app_shared.acquisition.image_backend import ( + DERIVED_MODES, + ImageFrame, + SOURCE_LIVE, + SOURCE_RECOMPUTE, + ) + + bare = Path(record_root) / "backend_video.mp4" + writer = cv2.VideoWriter(str(bare), cv2.VideoWriter_fourcc(*"mp4v"), 30.0, (64, 48)) + for index in range(30): + writer.write(np.full((48, 64, 3), index * 8, dtype=np.uint8)) + writer.release() + assert bare.is_file() + + window = _show(make_window(), (1440, 880)) + original = app_module.render_image + seen: list = [] + + def spy(frame, plan=None, **kwargs): + seen.append((frame.source, kwargs.get("mode"))) + return original(frame, plan, **kwargs) + + app_module.render_image = spy + try: + window._apply_view_preset("replay") + assert window._load_replay_video(bare) is True + window._render_replay_frame() + QApplication.processEvents() + assert window.arrow_panel.plan_modes() == ("recorded",), ( + "a recording must not offer a derived view" + ) + assert seen and seen[-1][0] == "replay" + note = window.arrow_panel.note_label.text() + assert note and note != "panels.replay_raw_note", note + assert window.arrow_panel.view_combo.count() == 1 + + # The pipeline drawing over the same footage: derived modes, same backend. + window.arrow_panel.set_source_kind("recompute") + shape = (48, 64) + window._show_image_frame( + ImageFrame( + np.zeros((*shape, 3), dtype=np.uint8), + flow=np.ones((*shape, 2), dtype=np.float32), + depth=np.zeros(shape, dtype=np.float32), + source=SOURCE_RECOMPUTE, + frame_id=7, + ) + ) + assert window.arrow_panel.plan_modes() == DERIVED_MODES + assert seen[-1] == ("recompute", "arrows_pure") + assert window.arrow_panel.view_combo.count() == len(DERIVED_MODES) + + # And the live packet path is the same sink (through the same spy). + window._show_image_frame( + ImageFrame( + np.full((*shape, 3), 9, dtype=np.uint8), + flow=np.ones((*shape, 2), dtype=np.float32), + source=SOURCE_LIVE, + frame_id=8, + ) + ) + assert seen[-1][0] == "live" + finally: + app_module.render_image = original + window.close() + + +def test_the_application_converts_bgr_to_qimage_in_one_place(): + """Two conversions is how the same frame looks different on two paths.""" + import orisys_studio.app_window as app_module + import orisys_studio.ui.arrow_panel as arrow_panel + + assert app_module._bgr_to_qimage is arrow_panel.bgr_to_qimage + root = Path(app_module.__file__).resolve().parent + others = sorted( + path.name + for path in root.rglob("*.py") + if path.name != "arrow_panel.py" + and "def bgr_to_qimage" in path.read_text(encoding="utf-8") + ) + assert others == [], f"another BGR→QImage conversion exists: {others}" + + def test_a_file_that_is_not_a_video_at_all_is_refused_with_a_reason( make_window, record_root ): diff --git a/shared/src/orisys_app_shared/acquisition/image_backend.py b/shared/src/orisys_app_shared/acquisition/image_backend.py new file mode 100644 index 0000000..4cf7e9c --- /dev/null +++ b/shared/src/orisys_app_shared/acquisition/image_backend.py @@ -0,0 +1,220 @@ +"""One place that turns a frame into the picture the 2D panel shows. + +The application used to have four producers of that picture and two different imaging +stacks: live and recomputed frames went through ``draw_arrow_display``/ +``depth_map_to_bgr`` in the window, the preview and the replay path handed a raw BGR +frame straight to the panel, and the panel decided which view modes existed from the +*view preset* (``set_source_kind("replay")``) rather than from what the frame actually +carried. The same video therefore behaved differently in "replay" and "recompute", and +a mode that needed data nobody had could still be offered. + +This module is the single backend. Two rules make the difference: + +* **the frame says what it has** (:class:`ImageFrame`), and the plan is derived from + that — not from which view is on screen. A frame with no flow has no arrow modes; a + frame with no depth has no depth mode; a frame with neither is a raw picture; +* **rendering is one function** (:func:`render_image`), so live, recompute, replay and + preview cannot drift apart. A mode the frame cannot support falls back to the plan's + default instead of drawing something plausible. + +NumPy only: the Qt conversion belongs to the UI layer (``ui/arrow_panel.bgr_to_qimage``). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional, Sequence + +import numpy as np + +from orisys_app_shared.acquisition.arrows import ( + depth_map_to_bgr, + draw_arrow_display, + flow_colormap, +) +from orisys_app_shared.acquisition.models import ensure_bgr + +#: The view modes, in the order a panel should offer them. ``recorded`` is not a +#: choice: it is what is left when the frame carries no derived field at all. +ARROW_PURE = "arrows_pure" +ARROW_STITCHED = "arrows_stitched" +DEPTH = "depth" +RECORDED = "recorded" + +DERIVED_MODES = (ARROW_PURE, ARROW_STITCHED, DEPTH) +ALL_MODES = (*DERIVED_MODES, RECORDED) + +#: Where a frame came from. Only used for wording and for the recording; the modes +#: come from the frame's own fields. +SOURCE_LIVE = "live" +SOURCE_RECOMPUTE = "recompute" +SOURCE_REPLAY = "replay" +SOURCE_PREVIEW = "preview" + +#: Shown when the only thing a frame can be is a raw picture. +RAW_NOTE_KEY = "panels.replay_raw_note" +#: Shown when a frame of a *derived* source is missing a field (a dropped frame, a +#: failed tracker): the mode stays selectable — that is the operator's choice — and the +#: line says which field is absent instead of the view looking broken. +MISSING_NOTE_KEY = "panels.frame_missing_note" + +#: Sources whose frames carry derived fields. +DERIVED_SOURCES = (SOURCE_LIVE, SOURCE_RECOMPUTE) + + +@dataclass(frozen=True) +class ImageFrame: + """A picture to draw, plus whatever derived fields came with it. + + ``image_bgr`` is the *base* picture (the processed/unrolled frame for a live or + recomputed source, the recorded or camera frame otherwise). ``flow``/``depth``/ + ``valid_mask`` are the same arrays the rest of the pipeline uses; ``None`` means the + frame does not have them, and the plan will say so rather than offer a mode that + would have to invent them. + """ + + image_bgr: Optional[np.ndarray] = None + flow: Optional[np.ndarray] = None + depth: Optional[np.ndarray] = None + valid_mask: Optional[np.ndarray] = None + source: str = SOURCE_LIVE + frame_id: int = 0 + t_rel_s: float = 0.0 + #: True when the producer knows this frame's measurement is not current (the + #: coordinator's "stale" state); the panel can say so under the picture. + stale: bool = False + + def has_flow(self) -> bool: + return self.flow is not None + + def has_depth(self) -> bool: + return self.depth is not None + + +@dataclass(frozen=True) +class ImagePlan: + """What this frame can be shown as, and what is missing from it.""" + + available_modes: tuple[str, ...] + default_mode: str + missing: tuple[str, ...] + source: str + frame_id: int + t_rel_s: float + stale: bool + #: An i18n key for the one line that explains the plan, or ``""``. When + #: :attr:`note_fields` is non-empty the line takes it as ``fields``. + note_key: str = "" + note_fields: tuple[str, ...] = () + + def accepts(self, mode: str) -> bool: + return str(mode) in self.available_modes + + def resolve(self, mode: str) -> str: + """The mode to draw: the request, or the default when it cannot be drawn.""" + wanted = str(mode or "") + return wanted if self.accepts(wanted) else self.default_mode + + def needs(self, mode: str) -> Optional[str]: + """The field a mode needs that this frame does not have, if any.""" + wanted = str(mode or "") + if wanted in (ARROW_PURE, ARROW_STITCHED) and not self.has_flow(): + return "flow" + if wanted == DEPTH and not self.has_depth(): + return "depth" + return None + + def has_flow(self) -> bool: + return "flow" not in self.missing + + def has_depth(self) -> bool: + return "depth" not in self.missing + + +def plan_image(frame: ImageFrame) -> ImagePlan: + """Derive the modes from the frame's *source* and its own fields. + + Two different questions, deliberately answered in two different places: + + * **which views may the operator choose?** For a source that carries derived fields + (live, recompute) all of them — a single dropped frame must not silently switch + the operator's view. For a source that never does (a recording, a camera preview) + exactly one, because offering the others would be offering a lie; + * **what can this frame actually be drawn as?** ``missing`` says which field is + absent, and the panel states it in one line. + """ + derived = str(frame.source or SOURCE_LIVE) in DERIVED_SOURCES + missing: list[str] = [] + if not frame.has_flow(): + missing.append("flow") + if not frame.has_depth(): + missing.append("depth") + if not derived: + return ImagePlan( + available_modes=(RECORDED,), + default_mode=RECORDED, + missing=("flow", "depth"), + source=str(frame.source or SOURCE_LIVE), + frame_id=int(frame.frame_id), + t_rel_s=float(frame.t_rel_s), + stale=bool(frame.stale), + note_key=RAW_NOTE_KEY, + ) + default = ARROW_PURE if frame.has_flow() else (DEPTH if frame.has_depth() else ARROW_PURE) + return ImagePlan( + available_modes=DERIVED_MODES, + default_mode=default, + missing=tuple(missing), + source=str(frame.source or SOURCE_LIVE), + frame_id=int(frame.frame_id), + t_rel_s=float(frame.t_rel_s), + stale=bool(frame.stale), + note_key=MISSING_NOTE_KEY if missing else "", + note_fields=tuple(missing), + ) + + +def render_image( + frame: ImageFrame, + plan: Optional[ImagePlan] = None, + *, + mode: str = "", + theme: str = "dark", + draw_style: str = "box", + depth_range: Optional[tuple[float, float]] = None, + canvas_bgr: Optional[Sequence[int]] = None, +) -> Optional[np.ndarray]: + """Draw ``frame`` in ``mode``; ``None`` when there is nothing to draw. + + The mode is resolved through the plan first, so a caller that asks for a view the + frame cannot support gets the plan's default rather than a picture assembled from + an array that is not there. + """ + plan = plan or plan_image(frame) + wanted = plan.resolve(mode) + if wanted == DEPTH: + # Never substitute another picture for a missing depth map: that made "no + # depth" look like a depth reading. The panel says so instead. + return depth_map_to_bgr( + frame.depth, + flow_colormap(theme), + value_range=depth_range, + invalid_mask=frame.valid_mask, + ) + if frame.image_bgr is None: + return None + if wanted in (ARROW_PURE, ARROW_STITCHED): + if frame.flow is None: + # The base picture with no arrows, and the plan's ``missing`` says why. + return ensure_bgr(np.asarray(frame.image_bgr)) + return draw_arrow_display( + frame.image_bgr, + frame.flow, + background="captured" if wanted == ARROW_STITCHED else "pure_color", + canvas_bgr=canvas_bgr, + draw_style=draw_style, + valid_mask=frame.valid_mask, + colormap=flow_colormap(theme), + ) + # ``recorded``: the frame is the picture, with nothing drawn over it. + return ensure_bgr(np.asarray(frame.image_bgr)) diff --git a/shared/tests/test_image_backend.py b/shared/tests/test_image_backend.py new file mode 100644 index 0000000..186bb6b --- /dev/null +++ b/shared/tests/test_image_backend.py @@ -0,0 +1,192 @@ +"""Acceptance for the single 2D image backend. + +The point of this module is that live, recomputed, replayed and previewed frames go +through *one* renderer and that the view modes come from what the frame carries. These +tests pin both halves: a plan derived from fields (not from a view preset), and +pixel-for-pixel agreement with the primitives the application used before. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from orisys_app_shared.acquisition import image_backend as backend +from orisys_app_shared.acquisition.arrows import ( + depth_map_to_bgr, + draw_arrow_display, + flow_colormap, +) + + +def _image(h: int = 12, w: int = 16) -> np.ndarray: + return np.arange(h * w * 3, dtype=np.uint8).reshape(h, w, 3) + + +def _flow(h: int = 12, w: int = 16) -> np.ndarray: + flow = np.zeros((h, w, 2), dtype=np.float32) + flow[..., 0] = 3.0 + flow[..., 1] = -1.5 + return flow + + +# ------------------------------------------------------------------------- plans +def test_a_live_frame_offers_the_derived_modes_and_needs_no_note(): + plan = backend.plan_image( + backend.ImageFrame(_image(), flow=_flow(), depth=np.ones((12, 16))) + ) + assert plan.available_modes == ("arrows_pure", "arrows_stitched", "depth") + assert plan.default_mode == "arrows_pure" + assert plan.missing == () + assert plan.note_key == "" + + +def test_a_recorded_frame_is_raw_and_says_so(): + """No flow and no depth: the frame *is* the picture, and the note explains it.""" + plan = backend.plan_image(backend.ImageFrame(_image(), source=backend.SOURCE_REPLAY)) + assert plan.available_modes == ("recorded",) + assert plan.default_mode == "recorded" + assert plan.missing == ("flow", "depth") + assert plan.note_key == backend.RAW_NOTE_KEY + assert plan.source == "replay" + + +def test_a_derived_source_keeps_every_mode_and_reports_what_is_missing(): + """A dropped frame must not silently switch the operator's view. + + Live and recomputed frames may be missing a field for one frame; the mode stays + selectable and ``missing`` says which field is absent, so the panel can state it + instead of the view looking broken. + """ + flow_only = backend.plan_image(backend.ImageFrame(_image(), flow=_flow())) + assert flow_only.available_modes == backend.DERIVED_MODES + assert flow_only.missing == ("depth",) + assert flow_only.default_mode == "arrows_pure" + assert flow_only.note_key == backend.MISSING_NOTE_KEY + assert flow_only.note_fields == ("depth",) + assert flow_only.needs("depth") == "depth" + assert flow_only.needs("arrows_pure") is None + + depth_only = backend.plan_image(backend.ImageFrame(_image(), depth=np.ones((12, 16)))) + assert depth_only.available_modes == backend.DERIVED_MODES + assert depth_only.missing == ("flow",) + assert depth_only.default_mode == "depth" + assert depth_only.needs("arrows_pure") == "flow" + + complete = backend.plan_image( + backend.ImageFrame(_image(), flow=_flow(), depth=np.ones((12, 16))) + ) + assert complete.missing == () + assert complete.note_key == "" + + +def test_a_raw_source_cannot_offer_a_derived_view_at_all(): + """A recording and a camera preview have no derived field, ever.""" + for source in (backend.SOURCE_REPLAY, backend.SOURCE_PREVIEW): + plan = backend.plan_image(backend.ImageFrame(_image(), source=source)) + assert plan.available_modes == ("recorded",) + assert plan.note_key == backend.RAW_NOTE_KEY + assert plan.note_fields == () + + +def test_the_plan_carries_the_frame_identity_for_the_panel(): + plan = backend.plan_image( + backend.ImageFrame( + _image(), source=backend.SOURCE_RECOMPUTE, frame_id=42, t_rel_s=1.25, stale=True + ) + ) + assert (plan.source, plan.frame_id, plan.t_rel_s, plan.stale) == ("recompute", 42, 1.25, True) + + +def test_a_mode_the_plan_does_not_offer_falls_back_to_the_default(): + """Never draw a view the source cannot have at all.""" + recorded = backend.plan_image( + backend.ImageFrame(_image(), source=backend.SOURCE_REPLAY) + ) + assert recorded.resolve("arrows_pure") == "recorded" + assert recorded.resolve("nonsense") == "recorded" + live = backend.plan_image(backend.ImageFrame(_image(), flow=_flow())) + assert live.resolve("recorded") == "arrows_pure", "a derived source has no raw mode" + assert live.resolve("arrows_stitched") == "arrows_stitched" + + +# --------------------------------------------------------------- one renderer +def test_the_backend_renders_what_the_primitives_rendered_before(): + """Live went through these two calls; the backend must not change a pixel.""" + image = _image() + flow = _flow() + mask = np.ones((12, 16), dtype=bool) + mask[0, 0] = False + + frame = backend.ImageFrame(image, flow=flow, valid_mask=mask) + plan = backend.plan_image(frame) + for mode in ("arrows_pure", "arrows_stitched"): + expected = draw_arrow_display( + image, + flow, + background="captured" if mode == "arrows_stitched" else "pure_color", + draw_style="finger", + valid_mask=mask, + ) + got = backend.render_image(frame, plan, mode=mode, draw_style="finger") + assert np.array_equal(got, expected), mode + + depth = np.linspace(0.0, 1.0, 12 * 16, dtype=np.float32).reshape(12, 16) + depth_frame = backend.ImageFrame(image, depth=depth, valid_mask=mask) + # The depth picture uses the *theme's* ramp — the same table the panel's colour bar + # paints. The window used to leave this argument out, so the map was drawn with + # OpenCV's JET while the legend under it showed the brand ramp: two colour tables + # for one picture, and the legend claiming they were the same. + expected_depth = depth_map_to_bgr( + depth, flow_colormap("dark"), value_range=(0.0, 1.0), invalid_mask=mask + ) + got_depth = backend.render_image( + depth_frame, mode="depth", depth_range=(0.0, 1.0) + ) + assert np.array_equal(got_depth, expected_depth) + + +def test_a_recorded_frame_is_passed_through_and_never_resized_to_fit(): + image = _image(9, 7) + got = backend.render_image( + backend.ImageFrame(image, source=backend.SOURCE_REPLAY), mode="recorded" + ) + assert got is not None and got.shape == image.shape + assert np.array_equal(got, image) + + +def test_nothing_to_draw_is_none_not_a_blank_picture(): + assert backend.render_image(backend.ImageFrame(), mode="recorded") is None + assert backend.render_image( + backend.ImageFrame(None, flow=_flow()), mode="arrows_pure" + ) is None + assert backend.render_image(backend.ImageFrame(None, depth=None), mode="depth") is None + + +def test_a_grey_base_frame_is_widened_once_by_the_backend(): + """A 2-D frame must reach the shader as BGR, and only one place may do it.""" + grey = np.zeros((8, 6), dtype=np.uint8) + got = backend.render_image(backend.ImageFrame(grey), mode="recorded") + assert got is not None and got.shape == (8, 6, 3) + assert np.array_equal(got, np.stack([grey] * 3, axis=-1)) + + +@pytest.mark.parametrize("mode", list(backend.ALL_MODES)) +def test_every_offered_mode_renders_something_or_says_it_cannot(mode): + """A mode the plan offers must never come back empty.""" + frame = backend.ImageFrame(_image(), flow=_flow(), depth=np.ones((12, 16))) + plan = backend.plan_image(frame) + if not plan.accepts(mode): + assert plan.resolve(mode) in plan.available_modes + return + assert backend.render_image(frame, plan, mode=mode) is not None + + +def test_a_missing_field_is_reported_not_invented(): + """Depth with no depth says so; arrows with no flow show the base picture.""" + no_depth = backend.ImageFrame(_image(), flow=_flow()) + assert backend.render_image(no_depth, mode="depth") is None + no_flow = backend.ImageFrame(_image(), depth=np.ones((12, 16))) + base = backend.render_image(no_flow, mode="arrows_pure") + assert base is not None + assert np.array_equal(base, _image()), "the base picture must not be redrawn"