From 3433eb0fec97db75b14064bed187289e81e30246 Mon Sep 17 00:00:00 2001 From: ykcheungab Date: Mon, 5 Oct 2026 15:01:48 +0800 Subject: [PATCH] Record the live box arrow grid so replay matches live paint Box analysis used the finger sample grid, so replay packed arrows into the contact zone. The pass now stores draw_arrows cells; stale box caches miss and re-run. Verified with shared and Studio tests, including a live-vs-replay canvas check. --- .../src/orisys_studio/replay_analysis.py | 140 +++- .../src/orisys_studio/replay_controller.py | 39 +- apps/studio/tests/test_bare_video_analysis.py | 676 ++++++++++++++++++ apps/studio/tests/test_replay_analysis.py | 269 ++++++- .../orisys_app_shared/acquisition/arrows.py | 57 +- .../acquisition/image_backend.py | 34 +- .../src/orisys_app_shared/analysis/model.py | 211 +++++- .../src/orisys_app_shared/analysis/service.py | 122 +++- shared/tests/test_image_backend.py | 141 ++++ shared/tests/test_replay_analysis.py | 19 +- 10 files changed, 1614 insertions(+), 94 deletions(-) create mode 100644 apps/studio/tests/test_bare_video_analysis.py diff --git a/apps/studio/src/orisys_studio/replay_analysis.py b/apps/studio/src/orisys_studio/replay_analysis.py index d48e1ad..2016481 100644 --- a/apps/studio/src/orisys_studio/replay_analysis.py +++ b/apps/studio/src/orisys_studio/replay_analysis.py @@ -24,15 +24,12 @@ ever speaks the second one. and never a new pass. The colour range and the idle-dot switch are not even inputs here: they live in the draw call, so changing one redraws and writes nothing. -The picture is rebuilt from the stored samples: ``dx``/``dy`` are the canvas -displacements the pass selected (uv pixels, already divided by the policy's arrow -scale), so the flow field is recovered by dividing them back out by -:data:`~orisys_app_shared.scene3d.visual_policy.DISPLACEMENT_PX_PER_FLOW` and writing -those values at the recorded candidate locations. The 2D canvas then runs its ordinary -path — the shared selector, the same idle dots, the same colour ramp — over that field. -This is exact, not approximate, and ``apps/studio/tests/test_replay_analysis.py`` -asserts it against a pass over real footage: re-selecting the recovered field through -the display policy reproduces the stored sample set element for element. +The picture is rebuilt from the stored samples. For the shared painter, ``dx``/``dy`` +are the drawn displacements, and the field is recovered by dividing them back out so +re-selecting it reproduces those offsets. A box result on the live painter's grid is +recovered from ``magnitude`` instead, because that painter colours by the displacement +before the length cap. ``apps/studio/tests/test_replay_analysis.py`` asserts the shared +path against a pass over real footage. The 3D pane is handed the **stored** sample set directly (:attr:`AnalysedFrame.samples`) — the object Part A documents as "what the canvas and the composer already draw" — so @@ -49,6 +46,7 @@ from pathlib import Path from typing import Any import numpy as np +from orisys_app_shared.acquisition.arrows import uses_sdk_box_grid from orisys_app_shared.acquisition.models import SdkPacket from orisys_app_shared.analysis.reader import AnalysisReader from orisys_app_shared.scene3d.visual_policy import ( @@ -67,11 +65,9 @@ __all__ = [ "policy_divergence", ] -#: The UV domain to rebuild the flow field in when the result carries no validity mask -#: keyframe (the box chain has none). Only the recorded candidate locations are ever -#: read back out of it, so its size is a canvas for the samples rather than a claim -#: about the device — but it has to hold every candidate, which -#: :meth:`AnalysisSession._recover_flow` checks. +#: Last-resort canvas when a result has no mask, no depth plane and no grid. The box +#: chain's samples live on ``resolution_stitch``, not on this raw-camera frame; using it +#: whenever the mask is absent paints that grid into one corner. DEFAULT_DOMAIN_HW = (1080, 1920) #: The policy keys that decide *where* the candidate locations are. A display setting @@ -191,6 +187,7 @@ class AnalysisSession: self._times = np.asarray(reader.times, dtype=np.float64) self._mask: np.ndarray | None = None self._mask_loaded = False + self._domain_hw: tuple[int, int] | None = None self._cache_id: int | None = None self._cache: AnalysedFrame | None = None @@ -256,6 +253,42 @@ class AnalysisSession: def threshold(self) -> float: return float(self.display_policy.flow_on_px) + def paint_policy(self) -> VisualPolicy: + """The policy this result's **recovered** flow field can be drawn with. + + The two halves come from different places, on purpose. *Where* the candidates are is + the pass's own answer — the field is finite only at those cells, and + :meth:`serve_policy` refuses a display setting that moves the grid — so the grid + travels from the result's provenance. *Which* candidates are active is a display + question, and ``magnitude`` is stored precisely so it can be asked again later, so + the threshold travels from the display policy. + + Without this, a display setting that moves the grid would be refused by + ``serve_policy`` and then applied to the drawing anyway, re-selecting the recovered + field on a grid whose cells mostly hold ``NaN``: the canvas would go blank rather + than draw the result's own samples. + """ + stored = self.stored_policy() + display = self.display_policy + policy = VisualPolicy( + flow_on_px=display.flow_on_px, + flow_off_px=display.flow_off_px, + flow_grid_spacing=int( + stored.get("flow_grid_spacing", self.policy.flow_grid_spacing) + ), + flow_arrow_scale=float( + stored.get("flow_arrow_scale", self.policy.flow_arrow_scale) + ), + flow_edge_margin=int( + stored.get("flow_edge_margin", self.policy.flow_edge_margin) + ), + flow_length_margin=float( + stored.get("flow_length_margin", self.policy.flow_length_margin) + ), + max_arrows=int(stored.get("max_arrows", self.policy.max_arrows)), + ) + return policy.normalised() + # ------------------------------------------------------------------ looking def frame_id_at(self, t_rel_s: float) -> int | None: """The stored frame at or just before ``t_rel_s``, by *signal* time.""" @@ -351,10 +384,35 @@ class AnalysisSession: return self._mask def domain_hw(self) -> tuple[int, int]: + """The image the stored samples were measured on. + + A validity mask is that image. Without one (the box chain), the depth plane is + the stitch grid and the candidate coordinates live on it. The raw camera frame + is larger, and painting the stitch grid onto it leaves the arrows in a corner. + """ mask = self.mask() - if mask is None: - return DEFAULT_DOMAIN_HW - return (int(mask.shape[0]), int(mask.shape[1])) + if mask is not None: + return (int(mask.shape[0]), int(mask.shape[1])) + if self._domain_hw is None: + self._domain_hw = self._unmasked_domain_hw() + return self._domain_hw + + def _unmasked_domain_hw(self) -> tuple[int, int]: + height = width = 0 + if self._frame_ids.size: + depth = getattr(self.reader.frame(int(self._frame_ids[0])), "depth", None) + if depth is not None and int(getattr(depth, "ndim", 0)) == 2 and depth.size: + height, width = int(depth.shape[0]), int(depth.shape[1]) + for reference in self.reader.keyframes.values(): + if getattr(reference, "kind", "") != "grid": + continue + rows, cols = self.reader.grid(int(reference.id)) + if rows.size: + height = max(height, int(np.max(rows)) + 1) + width = max(width, int(np.max(cols)) + 1) + if height > 0 and width > 0: + return (height, width) + return DEFAULT_DOMAIN_HW def _compose(self, payload: Any) -> AnalysedFrame: mask = self.mask() @@ -387,15 +445,17 @@ class AnalysisSession: invalid, so no neighbouring cell can contribute a sample the pass did not record. - ``dx``/``dy`` are the **drawn** displacements, which the selector produced as - ``DISPLACEMENT_PX_PER_FLOW * flow / flow_arrow_scale`` and then scaled by - ``flow_length_margin`` when the arrow was longer than its cell. Recovering the - field is therefore a division by the arrow scale, the display constant *and* - that margin — measured, not assumed: dividing by the first two alone leaves every - arrow 20% short (the finger margin is 0.8), which changes 319 of 330 samples and - one frame's active set. With the margin included the selector's own - ``np.where(length > spacing * margin, ...)`` branch is idempotent, so - re-selecting the recovered field reproduces the stored samples exactly. + ``dx``/``dy`` are the **drawn** displacements: ``DISPLACEMENT_PX_PER_FLOW * flow + / flow_arrow_scale``, then multiplied by ``flow_length_margin`` (and capped). + The shared painter re-selects this field, so the inverse divides by that + product and the cap stays idempotent: the stored offsets come back exactly. + At ``flow_arrow_scale == 1`` that divisor is ``DISPLACEMENT_PX_PER_FLOW`` times + the margin. + + The box painter does not re-select. It colours by the displacement *before* + the cap, which ``magnitude`` still holds (``|DISPLACEMENT_PX_PER_FLOW * flow|``). + Direction comes from ``dx``/``dy``. Float32 magnitude is not exact enough to + reproduce a capped offset, which is why only the box grid uses it. """ rows = np.asarray(samples.rows, np.int64) cols = np.asarray(samples.cols, np.int64) @@ -405,18 +465,32 @@ class AnalysisSession: "the result's candidate locations do not fit the domain it declares " f"({height}x{width}); the mask keyframe and the grid keyframe disagree" ) - divisor = ( - DISPLACEMENT_PX_PER_FLOW - * float(self.policy.flow_arrow_scale) - * float(self.policy.flow_length_margin) - ) + dx = np.asarray(samples.dx, np.float64) + dy = np.asarray(samples.dy, np.float64) flow = np.full((height, width, 2), np.nan, dtype=np.float64) with np.errstate(invalid="ignore"): # ``NaN`` is the point of the mask, and the selector's tip arithmetic on a # discarded cell is defined to be discarded: say so here rather than let # numpy warn about an invalid cast on every frame. - flow[rows, cols, 0] = np.asarray(samples.dx, np.float64) / divisor - flow[rows, cols, 1] = np.asarray(samples.dy, np.float64) / divisor + if uses_sdk_box_grid(self.policy): + magnitude = np.asarray(samples.magnitude, np.float64) + length = np.sqrt(dx * dx + dy * dy) + flow_mag = magnitude / DISPLACEMENT_PX_PER_FLOW + safe = np.maximum(length, 1e-12) + moving = length > 0.0 + flow[rows, cols, 0] = np.where(moving, dx / safe * flow_mag, 0.0) + flow[rows, cols, 1] = np.where(moving, dy / safe * flow_mag, 0.0) + else: + scale = float(self.policy.flow_arrow_scale) + if scale <= 0.0: + scale = 1.0 + divisor = ( + DISPLACEMENT_PX_PER_FLOW + * float(self.policy.flow_length_margin) + / scale + ) + flow[rows, cols, 0] = dx / divisor + flow[rows, cols, 1] = dy / divisor return flow diff --git a/apps/studio/src/orisys_studio/replay_controller.py b/apps/studio/src/orisys_studio/replay_controller.py index c3f9c1d..36f8b24 100644 --- a/apps/studio/src/orisys_studio/replay_controller.py +++ b/apps/studio/src/orisys_studio/replay_controller.py @@ -698,8 +698,16 @@ class ReplayController(QObject): policy = policy_for_settings(settings) if not session.serve_policy(policy): - moved = policy_divergence(session.stored_policy(), policy) - self._host.notify(t("playback.analysis_grid_mismatch", keys="·".join(moved))) + # A box result's grid is the live painter's, not a display preference. + # The display policy is the shared selector's, so the two always differ, + # and saying so would scold the operator for a grid they cannot change. + from orisys_app_shared.acquisition.arrows import uses_sdk_box_grid + + if not uses_sdk_box_grid(session.stored_policy()): + moved = policy_divergence(session.stored_policy(), policy) + self._host.notify( + t("playback.analysis_grid_mismatch", keys="·".join(moved)) + ) self._engine.redraw() def on_analysis_requested(self) -> None: @@ -810,6 +818,19 @@ class ReplayController(QObject): return folder.parent return None + @staticmethod + def _host_chain_is_box(service) -> bool: + chain = getattr(service, "chain", None) + return str(getattr(chain, "sensor_type", "") or "") == "box" + + @staticmethod + def _stored_grid_is_sdk_box(reader) -> bool: + from orisys_app_shared.acquisition.arrows import uses_sdk_box_grid + + provenance = getattr(reader, "provenance", None) + stored = dict(getattr(provenance, "policy", {}) or {}) + return uses_sdk_box_grid(stored) + def _activate_stored_result( self, recording: Path, *, notify_reuse: bool = False, reanalyse: bool = False ) -> bool: @@ -844,6 +865,17 @@ class ReplayController(QObject): reader = None if reader is None: return False + # A box result recorded on the shared selector's grid cannot be drawn by the + # live painter. Treat it as a miss so the pass runs again and records the + # painter's own cells. The key does not include the grid, so the new pass + # replaces this directory. + if self._host_chain_is_box(service) and not self._stored_grid_is_sdk_box(reader): + logger.info("stored box result uses a different sample grid; re-running") + try: + reader.close() + except Exception: # noqa: BLE001 - releasing is best effort + logger.debug("could not close the stale box result", exc_info=True) + return False try: session = AnalysisSession( reader, @@ -1153,7 +1185,8 @@ class ReplayController(QObject): # drawable at all; the plan reads it to decide what to offer. result_available=True, # ...and the field it carries is *recovered*: finite only at the candidate - # cells the pass recorded, so it can only be re-selected on the recorded grid. + # cells the pass recorded. A finger result is re-selected on that grid; a + # box result recorded on the live painter's grid is filled and drawn by it. # The display's own policy is the caller's for a live field, and wrong here. arrow_policy=self._session_policy(), ) diff --git a/apps/studio/tests/test_bare_video_analysis.py b/apps/studio/tests/test_bare_video_analysis.py new file mode 100644 index 0000000..9e8c6f1 --- /dev/null +++ b/apps/studio/tests/test_bare_video_analysis.py @@ -0,0 +1,676 @@ +"""T1: a bare video, analysed end to end, through the window's own action. + +The workflow this defends is the minimum one an operator needs today: point Studio at a +video that has no recording beside it, answer one prompt, and get a **measurement** — arrows, +a depth map and a force trace that can be scrubbed in both directions, because they are read +from a stored result rather than from a re-run of the chain. + +What makes it a *bare* video is the input: the clip is copied into a folder of its own with +nothing else in it (no ``forces.csv``, no ``frames.csv``, no ``session.json``). What makes it +analysable is the workspace the app generates for it (``analysis.bare_video``), written under +the app's own derived-data root — which this module points at a temporary directory, so the +test can assert *where* it went instead of trusting it. + +Where a failure would be invisible: + +* the prompt's answers decide what is measured, so the profile, the calibration and the + reference frame are read back from the dialog's own widgets; +* a recomputed force trace and a recorded one are different claims, so the charts' source + kind and the provenance's ``sources``/``synthetic`` are asserted, not just the pixels; +* a box result is recorded on the live painter's grid, so the canvas is compared with + ``draw_arrows`` on the recovered field once its gaps are filled with zero. + +The fixture clip is the project's recorded box footage under ``dist/``, copied into the test's +own directory. If it is absent the test **fails** (``_require_clip``) rather than skipping: a +silently skipped end-to-end test is how a broken workflow ships. +""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import numpy as np +import pytest +from orisys_app_shared import paths as paths_module +from orisys_app_shared.acquisition.arrows import ( + depth_map_to_bgr, + draw_arrow_display, + flow_colormap, + hex_to_bgr, +) +from orisys_app_shared.acquisition.image_backend import ( + ARROW_PURE, + DEPTH, + RECORDED, + RESULT_FAILED, + SOURCE_REPLAY, + ImageFrame, + render_image, +) +from orisys_app_shared.analysis import bare_video +from orisys_app_shared.analysis.cache import scan_results +from orisys_app_shared.i18n import t +from orisys_app_shared.scene3d.contracts import DisplaySettings +from orisys_app_shared.sensors import ( + CatalogEntry, + SensorCatalog, + UserRootDecision, + scan_sensor_catalog, +) +from orisys_app_shared.theme import color +from PySide6.QtWidgets import QApplication, QDialog, QDialogButtonBox + +from orisys_studio.ui.video_analysis_dialog import VideoAnalysisDialog + +APP = Path(__file__).resolve().parents[1] +REPO = APP.parents[1] +EVIDENCE = ( + REPO + / "dist" + / "OrisysStudio" + / "outputs" + / "recordings" + / "20260930T015028Z_session0002_40cb8e7441434feab48bb646afdba011" +) +SOURCE_CLIP = EVIDENCE / "raw_0001.mp4" + + +def _require_clip() -> Path: + """The one real input: recorded box footage, copied rather than faked. + + A synthetic clip cannot stand in for it. The box chain's stitch probe finds its lenslet + grid in a real image; on generated pixels it does not converge — measured, a synthetic + 896² clip sent the chain into minutes of work where the real 33-frame clip takes two + seconds. So the input is the project's own recording, and its absence is a **failure**: + skipping would leave "a bare video can be analysed" untested, which is the state this + task exists to change. + """ + assert SOURCE_CLIP.is_file(), ( + "the bare-video test needs recorded box footage at " + f"{SOURCE_CLIP}; copy it there or run the build that produces dist/" + ) + pytest.importorskip("orisys", reason="the analysis pass drives the SDK chain") + return SOURCE_CLIP + + +# ------------------------------------------------------------------------ fixtures +@pytest.fixture(scope="module") +def cache_root(tmp_path_factory): + """The analysis cache for this module, in place of the user's own.""" + root = tmp_path_factory.mktemp("bare-video-cache") / "analysis-cache" + root.mkdir(parents=True) + patch = pytest.MonkeyPatch() + patch.setattr(paths_module, "analysis_cache_dir", lambda: root) + yield root + patch.undo() + + +@pytest.fixture +def own_cache(tmp_path, monkeypatch) -> Path: + """A cache root this test alone owns, so its first open really does run a pass.""" + root = tmp_path / "own-cache" + root.mkdir() + monkeypatch.setattr(paths_module, "analysis_cache_dir", lambda: root) + return root + + +@pytest.fixture(scope="module") +def workspace_root(tmp_path_factory): + """The app-owned root the generated workspace goes to, in place of the user's own.""" + root = tmp_path_factory.mktemp("bare-video-workspaces") / "video-workspaces" + patch = pytest.MonkeyPatch() + patch.setattr(paths_module, "video_workspace_dir", lambda: root) + yield root + patch.undo() + + +@pytest.fixture +def bare_clip(tmp_path) -> Path: + """One video, alone in a folder: a bare video and nothing else.""" + folder = tmp_path / "operator-footage" + folder.mkdir() + clip = folder / "session-clip.mp4" + shutil.copy2(_require_clip(), clip) + return clip + + +@pytest.fixture +def window(make_window, qt_app, cache_root, workspace_root): + """The application, in the replay view, as the operator meets it.""" + view = make_window(sensor_type="box", source_mode="camera") + view.resize(1440, 880) + view.show() + QApplication.processEvents() + view.brand_bar.btn_playback.click() + QApplication.processEvents() + assert view.layout_ctl.in_replay_view(), "the window did not switch to the replay view" + return view + + +# ------------------------------------------------------------------------- helpers +def answer_prompt( + monkeypatch, + *, + profile: str = "shipped/box", + calibration: str = "default", + reference: int | None = None, + accept: bool = True, + seen: list | None = None, +): + """Answer the setup prompt through the real dialog's own widgets. + + The dialog class is the real one and the window builds it; only ``exec`` is replaced, so + what a test answers is exactly what an operator could have answered, and the defaults can + be read back from the widgets the operator sees. + """ + + def _answer(dialog_self): + if seen is not None: + seen.append(dialog_self) + assert isinstance(dialog_self, VideoAnalysisDialog) + assert dialog_self.select_profile(profile), ( + f"the prompt does not offer {profile}: " + f"{[entry.ref for entry in dialog_self.entries]}" + ) + if reference is not None: + dialog_self.reference_spin.setValue(int(reference)) + if calibration == "none": + index = dialog_self.calibration_combo.findData("none") + assert index >= 0 + dialog_self.calibration_combo.setCurrentIndex(index) + elif calibration != "default": + dialog_self.select_calibration(Path(calibration)) + return QDialog.DialogCode.Accepted if accept else QDialog.DialogCode.Rejected + + monkeypatch.setattr(VideoAnalysisDialog, "exec", _answer) + + +def panel_bgr(window) -> np.ndarray: + """The 2D panel's picture, in BGR, from the end of the real display path.""" + image = window.arrow_panel._source_image + assert image is not None and not image.isNull(), "the 2D panel shows no picture" + width, height = image.width(), image.height() + rows = np.frombuffer(bytes(image.constBits()), dtype=np.uint8) + rows = rows.reshape(height, image.bytesPerLine()) + return rows[:, : width * 3].reshape(height, width, 3)[:, :, ::-1].copy() + + +def sink_frames(window, monkeypatch) -> list: + captured: list = [] + original = window._show_image_frame + + def spy(frame): + captured.append(frame) + return original(frame) + + monkeypatch.setattr(window, "_show_image_frame", spy) + return captured + + +def set_mode(window, mode: str) -> None: + index = window.arrow_panel.view_combo.findData(mode) + assert index >= 0, f"the mode {mode} is not offered: {window.arrow_panel.view_modes()}" + window.arrow_panel.view_combo.setCurrentIndex(index) + QApplication.processEvents() + assert window.arrow_panel.view_mode() == mode + + +def canvas_bgr(window) -> tuple[int, int, int]: + return hex_to_bgr(color(window._theme, "arrow_canvas", "#0B1830")) + + +def painted(image, canvas) -> int: + array = np.asarray(image) + return int(np.count_nonzero(np.any(array != np.asarray(canvas, np.uint8), axis=-1))) + + +def entry_set(directory: Path) -> set: + """Every entry under ``directory`` as ``(relative path, size)``.""" + found = set() + for path in Path(directory).rglob("*"): + relative = str(path.relative_to(directory)) + found.add((relative, -1 if path.is_dir() else path.stat().st_size)) + return found + + +def open_bare_video(window, clip: Path, monkeypatch, **answer) -> None: + """The operator's sequence up to "the pass is running": open, and answer the prompt.""" + answer_prompt(monkeypatch, **answer) + assert window.replay_ctl.load_video(clip) is True, "the video did not open" + QApplication.processEvents() + + +def settle(window, *, timeout_s: float = 300.0) -> None: + """Let a pass finish if one is running, and check that a result came out of it.""" + if window.replay_ctl.analysis_running(): + assert window.replay_ctl.wait_for_analysis(timeout_s), "the pass did not finish" + QApplication.processEvents() + assert window.replay_ctl.analysis_error() == "", ( + f"the pass left no usable result: {window.replay_ctl.analysis_error()}" + ) + assert window.replay_ctl.available_result() is not None, "no stored result was adopted" + + +# --------------------------------------------------------------------- the workflow +def test_a_bare_video_is_analysed_and_scrubbed_in_both_directions( + bare_clip, window, monkeypatch, own_cache, workspace_root +): + """Open a bare video → answer the prompt → the pass runs → scrub it backwards.""" + before = entry_set(bare_clip.parent) + captured = sink_frames(window, monkeypatch) + seen: list = [] + open_bare_video(window, bare_clip, monkeypatch, seen=seen) + + assert window.replay_ctl.analysis_running(), ( + "no pass was started for a bare video with no stored result" + ) + assert window.analysis_progress.state() == "running" + settle(window) + + # ---- the prompt's fields and their defaults ----------------------------- + dialog = seen[0] + assert [entry.ref for entry in dialog.entries] == [ + entry.ref for entry in bare_video.profiles_for_video(scan_sensor_catalog().entries) + ], "the prompt offered a different set of profiles than the catalog can analyse with" + assert "flow" in dialog.capability_label.text(), ( + "the row does not say what the profile can do" + ) + assert dialog.calibration_combo.currentData() == "default" + assert dialog.calibration_combo.currentText() == t( + "video_analysis.calibration_default", name="force_calibration.json" + ), "the calibration did not default to the profile's declared one" + assert dialog.reference_spin.value() == 1, ( + "the reference frame did not default to the first" + ) + chosen = dialog.choice() + assert chosen is not None and chosen.reference_frame is None, ( + "the first frame must stay the pass's own default, not a reset it cannot apply" + ) + assert chosen.calibration is not None and chosen.calibration.name == "force_calibration.json" + + # ---- the workspace, and where it went ----------------------------------- + workspace = Path(window.replay_ctl.recording_directory() or "") + assert workspace.is_dir(), "the pass was not given a generated workspace" + assert workspace.parent == workspace_root, f"the workspace went to {workspace.parent}" + assert workspace.parent != bare_clip.parent + assert sorted(path.name for path in workspace.iterdir()) == [ + "frames.csv", + "raw_0001.mp4", + "session.json", + ] + assert entry_set(bare_clip.parent) == before, ( + "opening and analysing a bare video wrote beside the operator's footage" + ) + + session = window.replay_ctl.available_result() + provenance = session.reader.provenance + assert provenance.synthetic is True, ( + "the result does not say its input was a generated workspace" + ) + assert provenance.synthetic_source == str(bare_clip), ( + "the provenance does not name the video the numbers were computed from" + ) + assert provenance.sources["flow"] == "recomputed" + assert provenance.sources["forces"] == "recomputed", ( + "a recomputed force trace was recorded as a recorded measurement" + ) + + # ---- the charts: the recomputed force trace ----------------------------- + assert window.force_plots.curve_kind() == "analysis", ( + "the charts are not reading the result the pass wrote" + ) + assert window.force_plots.is_recording_source() + assert window.force_plots.mode_label.text() == t("chart.mode_recomputed"), ( + "the charts do not say that this trace is a recomputation" + ) + table = session.reader.forces() + assert table is not None and len(table) > 1, "the result stores no force at all" + assert np.isfinite(table.newtons).any(), "the stored force trace holds no number" + unit = window.force_plots.unit() + xs, ys = window.force_plots.curves[0].getData() + assert xs is not None and len(xs) == len(table), "the FN curve does not hold the trace" + assert np.allclose( + np.asarray(ys, float), table.value_for_unit(unit)[:, 0], equal_nan=True + ) + + # ---- the 2D canvas: arrows and depth, from the cache -------------------- + canvas = canvas_bgr(window) + ids = [int(value) for value in session.frame_ids] + last_id, middle_id = ids[-1], ids[len(ids) // 2] + assert session.frame(last_id).active_count > 0 + + set_mode(window, ARROW_PURE) + window.replay.set_position(session.time_of(middle_id)) + QApplication.processEvents() + assert captured[-1].frame_id == middle_id + assert captured[-1].source == SOURCE_REPLAY + assert captured[-1].result_available is True + + # ...then scrub *backwards*, from the end of the timeline to that same frame: it must + # resolve to the stored frame for the position, drawn from the cache, with no pass. + window.replay.set_position(window.replay.duration_s() + 1.0) + QApplication.processEvents() + assert captured[-1].frame_id == last_id, ( + "the cursor at the end of the timeline did not resolve to the last stored frame" + ) + window.replay.set_position(session.time_of(middle_id)) + QApplication.processEvents() + assert captured[-1].frame_id == middle_id, ( + "scrubbing backwards did not resolve to the stored frame at that position" + ) + assert not window.replay_ctl.analysis_running(), "scrubbing ran an analysis pass" + middle_shot = panel_bgr(window) + + frame = session.frame(middle_id) + policy = session.paint_policy() + expected = render_image( + ImageFrame( + frame.base_bgr, + flow=frame.flow, + depth=frame.depth, + valid_mask=frame.valid_mask, + source=SOURCE_REPLAY, + frame_id=int(frame.frame_id), + flow_valid=True, + depth_valid=bool(frame.has_depth), + result_available=True, + arrow_policy=policy, + ), + mode=ARROW_PURE, + theme=window._theme, + draw_style=window._draw_style(), + canvas_bgr=canvas, + domain_mask=window._domain_mask.mask, + arrow_policy=policy, + ) + assert expected is not None + assert painted(expected, canvas) > 0, "the cached arrows painted nothing" + assert np.array_equal(middle_shot, np.asarray(expected)), ( + "the panel's arrow canvas is not the stored frame's own drawing" + ) + raw = window.replay.video_frame_bgr() + assert raw is not None and not np.array_equal(middle_shot, np.asarray(raw)), ( + "the arrow view is the video's raw picture" + ) + + # ---- the depth view, from the same stored frame ------------------------- + set_mode(window, DEPTH) + QApplication.processEvents() + expected_depth = depth_map_to_bgr( + frame.depth, + flow_colormap(window._theme), + value_range=window._depth_range(), + invalid_mask=window._domain_mask.mask, + ) + assert expected_depth is not None + assert np.array_equal(panel_bgr(window), expected_depth), ( + "the depth view is not depth_map_to_bgr of the stored frame" + ) + + # ---- the recording's own picture is still one step away ----------------- + set_mode(window, RECORDED) + assert np.array_equal(panel_bgr(window), np.asarray(raw)) + assert window.arrow_panel.note_label.text() == "", ( + "the panel says there is nothing to draw while it is drawing a result" + ) + + +def test_the_box_arrow_canvas_is_painted_by_the_live_painter( + bare_clip, window, monkeypatch +): + """A box pass stores the live painter's grid, so the window draws ``draw_arrows``. + + The recovered field is still NaN off the recorded cells. Filling those gaps with zero + is what the live painter reads as "no motion", and that picture is the panel. + """ + open_bare_video(window, bare_clip, monkeypatch) + settle(window) + session = window.replay_ctl.available_result() + canvas = canvas_bgr(window) + policy = session.paint_policy() + assert int(policy.flow_grid_spacing) == 20 + assert abs(float(policy.flow_arrow_scale) - 0.5) < 1e-9 + common = { + "background": "pure_color", + "canvas_bgr": canvas, + "draw_style": window._draw_style(), + "colormap": flow_colormap(window._theme), + "policy": policy, + "shared_painter": False, + } + + set_mode(window, ARROW_PURE) + first = next(frame for frame in session.frames() if frame.active_count > 0) + window.replay.set_position(session.time_of(first.frame_id)) + QApplication.processEvents() + shown = panel_bgr(window) + assert painted(shown, canvas) > 0, "the operator's arrow canvas is blank" + filled = np.nan_to_num(np.asarray(first.flow, dtype=np.float64), nan=0.0) + expected = draw_arrow_display( + first.base_bgr, + filled, + valid_mask=first.valid_mask, + **common, + ) + assert np.array_equal(shown, np.asarray(expected)), ( + "the panel's canvas is not the live painter's output for this frame" + ) + + +def test_declining_the_prompt_leaves_the_video_playing_with_nothing_promised( + bare_clip, window, monkeypatch +): + """Cancelling is a state, not a crash: the picture plays and the panel says no pass ran.""" + before = entry_set(bare_clip.parent) + captured = sink_frames(window, monkeypatch) + open_bare_video(window, bare_clip, monkeypatch, accept=False) + + assert window.replay.has_video(), "declining the prompt closed the video" + assert window.replay_ctl.available_result() is None + assert window.replay_ctl.recording_directory() is None, ( + "a dismissed prompt still claimed a workspace as this timeline's input" + ) + assert not window.replay_ctl.analysis_running(), "a pass ran after the prompt was dismissed" + state, reason = window.replay_ctl.result_state() + assert state == RESULT_FAILED + assert reason == t("playback.analysis_setup_declined") + assert captured and captured[-1].result_state == RESULT_FAILED + assert entry_set(bare_clip.parent) == before + + +def test_a_split_recording_is_refused_with_a_clear_message( + bare_clip, window, monkeypatch +): + """A split recording is a measurement error: refuse it rather than half-analyse it. + + The files are named the way a split recording is (``take_0001.mp4`` beside + ``take_0002.mp4``), and the chosen one is opened as a video first — the refusal comes + from the setup step, which is where "what exactly is this input" is asked. + """ + folder = bare_clip.parent + bare_clip.unlink() + chosen = folder / "take_0001.mp4" + shutil.copy2(_require_clip(), chosen) + shutil.copy2(_require_clip(), folder / "take_0002.mp4") + captured = sink_frames(window, monkeypatch) + + open_bare_video(window, chosen, monkeypatch) + + assert not window.replay_ctl.analysis_running(), "a split recording was analysed anyway" + _state, reason = window.replay_ctl.result_state() + assert "split recording" in reason, f"the refusal does not say why: {reason!r}" + assert "take_0001.mp4" in reason and "take_0002.mp4" in reason + assert window.replay.has_video(), "the refusal closed the video" + assert captured, "the refusal left the panel with nothing" + + +def test_reopening_the_video_reuses_the_result_and_runs_no_second_pass( + bare_clip, window, monkeypatch, cache_root +): + """The cache is the point: a second open is a file read, not a second pass. + + ``recording_identity`` is derived from the video and the chosen profile, so a workspace + generated again is the *same* input — the key does not move and no pass runs. + """ + open_bare_video(window, bare_clip, monkeypatch) + settle(window) + directories = {entry.directory for entry in scan_results(cache_root)} + assert directories, "the first open stored no result" + workspace = window.replay_ctl.recording_directory() + + window.replay.close() + QApplication.processEvents() + assert window.replay_ctl.recording_directory() is None + + started: list = [] + original_start = window.analysis_progress.start + + def spy_start(*args, **kwargs): + started.append(True) + return original_start(*args, **kwargs) + + monkeypatch.setattr(window.analysis_progress, "start", spy_start) + open_bare_video(window, bare_clip, monkeypatch) + settle(window) + + assert not started, "a second pass ran for a video that already had a result" + assert window.analysis_progress.state() == "reused" + assert {entry.directory for entry in scan_results(cache_root)} == directories + assert window.replay_ctl.recording_directory() == workspace, ( + "the second open generated a different workspace" + ) + assert window.force_plots.curve_kind() == "analysis" + + +def test_a_setting_that_moves_the_grid_redraws_and_writes_nothing( + bare_clip, window, monkeypatch, cache_root +): + """A display setting is a read; a grid setting cannot be served and must not blank it.""" + open_bare_video(window, bare_clip, monkeypatch) + settle(window) + session = window.replay_ctl.available_result() + set_mode(window, ARROW_PURE) + before = {entry.directory for entry in scan_results(cache_root)} + captured = sink_frames(window, monkeypatch) + + window.replay_ctl.on_display_settings(DisplaySettings(flow_on_px=0.9)) + QApplication.processEvents() + assert captured, "the threshold change drew nothing" + assert session.threshold() == pytest.approx(0.9) + assert {entry.directory for entry in scan_results(cache_root)} == before, ( + "a display change wrote an analysis result" + ) + + window.replay_ctl.on_display_settings(DisplaySettings(max_arrows=50)) + QApplication.processEvents() + policy = captured[-1].arrow_policy + assert policy is not None, ( + "the frame does not say which policy its recovered field can be sampled with" + ) + assert policy.max_arrows == session.paint_policy().max_arrows, ( + "the display's grid setting reached the recovered field's sampling policy" + ) + assert painted(panel_bgr(window), canvas_bgr(window)) > 0, ( + "a refused grid setting blanked the canvas" + ) + assert {entry.directory for entry in scan_results(cache_root)} == before + + +def test_the_workspace_carries_the_profile_the_operator_chose( + bare_clip, window, monkeypatch +): + """The prompt's answer is what the pass is driven with — read back from the workspace.""" + open_bare_video(window, bare_clip, monkeypatch, calibration="none") + settle(window) + workspace = Path(window.replay_ctl.recording_directory()) + document = json.loads((workspace / "session.json").read_text(encoding="utf-8")) + + assert document["device"]["sensor_type"] == "box" + config_ref = Path(document["device"]["config_ref"]) + assert config_ref.name == "sensor.json" and "sensors" in config_ref.parts + assert document["synthetic"]["profile_ref"] == "shipped/box" + assert document["synthetic"]["calibration"] == "" + assert document["calibration_id"] == "", ( + "choosing raw force must not leave a calibration in the result's key" + ) + assert document["stats"]["segments"] == 1 + assert (workspace / "frames.csv").is_file() + # The chain was told, too: no calibration means the raw triplet, not a substitute curve. + assert window.replay_ctl.available_result() is not None + + +# ------------------------------------------------------------------- the prompt itself +def _catalog(entry: CatalogEntry) -> SensorCatalog: + return SensorCatalog( + roots=(), + entries=(entry,), + user_root_decision=UserRootDecision( + path=Path("."), kind="explicit", writable=True, reason="test" + ), + ) + + +def _blocked_entry() -> CatalogEntry: + """The shipped box bundle as the catalog reports one the SDK's validation rejected. + + Built from the *real* entry so the profile behind it is real too: the verdict is what is + replaced, and the reason is the sentence the SDK would have produced. + """ + import dataclasses + + real = next( + entry for entry in scan_sensor_catalog().entries if entry.folder == "box" + ) + return dataclasses.replace( + real, + valid=False, + state="failed", + reason="assets.computational-missing: the required unroll map is missing", + ) + + +def test_a_profile_the_sdk_rejected_cannot_be_chosen_and_says_why( + bare_clip, window, qt_app, tmp_path +): + """The catalog's ``start_blocked_reason`` reaches the prompt instead of no caller.""" + facts = bare_video.probe(bare_video.resolve_video(bare_clip)) + dialog = VideoAnalysisDialog( + video=bare_clip, facts=facts, catalog=_catalog(_blocked_entry()), window=window + ) + try: + entry = dialog.entries[0] + blocked = bare_video.setup_blocked_reason(entry) + assert blocked is not None and "unroll map" in blocked + item = dialog.profile_combo.model().item(0) + assert not item.isEnabled(), "a profile the SDK rejected is offered as startable" + assert not dialog.buttons.button(QDialogButtonBox.StandardButton.Ok).isEnabled(), ( + "OK is offered for a profile that cannot be started" + ) + assert dialog.note_label.text() == t("video_analysis.blocked", reason=blocked), ( + "the prompt does not carry the SDK's own reason" + ) + assert dialog.choice() is None, "the refused profile produced a setup anyway" + finally: + dialog.deleteLater() + + +def test_the_first_frame_is_the_prompt_s_default_reference( + bare_clip, window, qt_app +): + """A reference frame that is not asked for must not become a reset the pass cannot make.""" + facts = bare_video.probe(bare_video.resolve_video(bare_clip)) + dialog = VideoAnalysisDialog( + video=bare_clip, facts=facts, catalog=scan_sensor_catalog(), window=window + ) + try: + assert dialog.reference_spin.minimum() == 1 + assert dialog.reference_spin.maximum() == facts.frame_count + assert dialog.reference_spin.value() == 1 + assert dialog.choice().reference_frame is None + dialog.reference_spin.setValue(4) + assert dialog.choice().reference_frame == 4 + assert dialog.reference_spin.value() == 4 + finally: + dialog.deleteLater() diff --git a/apps/studio/tests/test_replay_analysis.py b/apps/studio/tests/test_replay_analysis.py index 5be20d6..9336038 100644 --- a/apps/studio/tests/test_replay_analysis.py +++ b/apps/studio/tests/test_replay_analysis.py @@ -40,19 +40,26 @@ from orisys_app_shared.acquisition.image_backend import ( RESULT_NONE, RESULT_RUNNING, SOURCE_ANALYSIS, + SOURCE_LIVE, SOURCE_REPLAY, ImageFrame, plan_image, render_image, ) +from orisys_app_shared.analysis import service as analysis_service_mod from orisys_app_shared.analysis.cache import ResultCache, scan_results from orisys_app_shared.analysis.reader import ( AnalysisIncompleteError, AnalysisNotFoundError, AnalysisReader, ) -from orisys_app_shared.analysis.service import AnalysisService, ChainSpec +from orisys_app_shared.analysis.service import ( + AnalysisService, + ChainSpec, + chain_spec_for, +) from orisys_app_shared.i18n import t +from orisys_app_shared.record.playback import RecordingReader from orisys_app_shared.scene3d.contracts import DisplaySettings from orisys_app_shared.scene3d.visual_policy import ( policy_for_settings, @@ -141,6 +148,33 @@ def session(reader) -> AnalysisSession: ) +@pytest.fixture(scope="module") +def box_pass(tmp_path_factory): + """One box-chain pass over the evidence recording, for live/replay paint checks.""" + _require_inputs() + root = tmp_path_factory.mktemp("replay-analysis-box") + chain = chain_spec_for(RecordingReader(RECORDING).session()) + service = AnalysisService(RECORDING, chain=chain, cache=ResultCache(root)) + outcome = service.run() + assert outcome.complete, outcome.reason + return service, outcome + + +@pytest.fixture(scope="module") +def box_session(box_pass): + service, outcome = box_pass + reader = AnalysisReader(outcome.directory) + session = AnalysisSession( + reader, + policy=service.policy, + display_policy=service.policy, + session_id=2, + recording=RECORDING, + ) + yield session + reader.close() + + def _same_samples(left, right) -> dict[str, bool]: """Compare two ``FlowSamples`` (or two frames that carry one) array by array.""" left = getattr(left, "samples", left) @@ -341,13 +375,27 @@ class _Progress: self.states.append("idle") + +def _wire_replay(controller, host): + controller.view_requested.connect(lambda preset: ( + host.presets.append(preset), + setattr(host, "view", preset), + controller.set_active_view(preset), + )) + controller.acquire_requested.connect(host.start_acquire) + controller.acquire_stop_requested.connect( + lambda reason: host.stop_acquire(reason=reason) + ) + controller.set_active_view(host.view) + return controller + class _Host: def __init__(self, service_factory=None) -> None: self.prefs: dict = {} self.view = "replay" - self._running = False - self._preview_open = False - self._session_active = False + self._running_flag = False + self._preview_open_flag = False + self._session_active_flag = False self.notes: list[str] = [] self.frames: list = [] self.analysis_packets: list = [] @@ -356,52 +404,99 @@ class _Host: self.service_factory = service_factory self.services_built = 0 - def _in_replay_view(self) -> bool: - return self.view == "replay" + @property + def running(self) -> bool: + return self._running_flag - def _current_source_kind(self) -> str: + @running.setter + def running(self, value: bool) -> None: + self._running_flag = bool(value) + + @property + def preview_open(self) -> bool: + return self._preview_open_flag + + @preview_open.setter + def preview_open(self, value: bool) -> None: + self._preview_open_flag = bool(value) + + @property + def session_active(self) -> bool: + return self._session_active_flag + + @session_active.setter + def session_active(self, value: bool) -> None: + self._session_active_flag = bool(value) + + @property + def _running(self) -> bool: + return self._running_flag + + @_running.setter + def _running(self, value: bool) -> None: + self._running_flag = bool(value) + + @property + def _preview_open(self) -> bool: + return self._preview_open_flag + + @_preview_open.setter + def _preview_open(self, value: bool) -> None: + self._preview_open_flag = bool(value) + + @property + def _session_active(self) -> bool: + return self._session_active_flag + + @_session_active.setter + def _session_active(self, value: bool) -> None: + self._session_active_flag = bool(value) + + def current_source_kind(self) -> str: return "replay" - def _apply_view_preset(self, preset: str) -> None: - self.presets.append(preset) - self.view = preset - - def _set_display_unit(self, unit: str) -> None: + def set_display_unit(self, unit: str) -> None: self.units.append(unit) - def _set_source_mode(self, mode: str) -> None: + def set_source_mode(self, mode: str) -> None: pass - def _show_image_frame(self, frame) -> None: + def show_frame(self, frame) -> None: self.frames.append(frame) - def _show_analysis_frame(self, packet) -> None: + def show_analysis_packet(self, packet) -> None: self.analysis_packets.append(packet) - def _drop_render_marker(self) -> None: + def drop_render_marker(self) -> None: pass - def _notify(self, text: str, ms: int = 0) -> None: + def notify(self, text: str, ms: int = 0) -> None: self.notes.append(str(text)) - def _start_run(self) -> None: + def start_acquire(self) -> None: pass - def _stop_run(self, *, reason: str = "user_stop", wait: bool = False) -> None: + def stop_acquire(self, *, reason: str = "user_stop", wait: bool = False) -> None: pass - def _update_run_enabled(self) -> None: + def update_run_enabled(self) -> None: pass - def _refresh_status_rail(self) -> None: + def refresh_status_rail(self) -> None: pass - def _analysis_service(self, recording: Path, cancelled=None, *, on_progress=None): + def persist_prefs(self) -> None: + pass + + def analysis_service(self, recording: Path, cancelled=None, *, on_progress=None): self.services_built += 1 if self.service_factory is None: raise AssertionError("the test did not provide an analysis service") return self.service_factory(recording, cancelled, on_progress) + def bare_video_workspace(self, video): + return None + class _RecordingReader: """The recording reader as the engine and the controller use it.""" @@ -446,14 +541,17 @@ def _controller(engine=None, host=None, **kwargs): panel = kwargs.pop("image_panel", None) or _ImagePanel() mesh = kwargs.pop("mesh_panel", None) or _Mesh() progress = kwargs.pop("progress", None) or _Progress() - controller = ReplayController( - engine=engine, - host=host, - toolbar=toolbar, - charts=charts, - image_panel=panel, - mesh_panel=mesh, - analysis_progress=progress, + controller = _wire_replay( + ReplayController( + engine=engine, + host=host, + toolbar=toolbar, + charts=charts, + image_panel=panel, + mesh_panel=mesh, + analysis_progress=progress, + ), + host, ) actions = { "open_video": _Action(), @@ -610,6 +708,74 @@ def test_the_recovered_field_reselects_to_the_stored_samples(session): assert checked == 33 +def test_box_live_and_replay_arrow_canvases_do_not_diverge(box_pass, box_session, monkeypatch): + """Live ``draw_arrows`` on the dense field equals the replay canvas, frame for frame. + + This is the discrepancy the box arrow work exists to close: the operator's live view + paints the chain's dense flow, and the replay paints a field recovered from the + stored samples. Both go through :func:`render_image` with ``draw_style="box"``. + """ + service, outcome = box_pass + assert outcome.complete, outcome.reason + original = analysis_service_mod.select_flow_samples + dense_flows: list[np.ndarray] = [] + + def _capture(flow, mask, policy, **kwargs): + dense_flows.append(np.asarray(flow, dtype=np.float64).copy()) + return original(flow, mask, policy, **kwargs) + + monkeypatch.setattr(analysis_service_mod, "select_flow_samples", _capture) + lives = list(service.recompute()) + assert len(lives) == len(dense_flows) == outcome.frames_written + + canvas_bgr = (0x16, 0x13, 0x12) + policy = box_session.paint_policy() + assert int(policy.flow_grid_spacing) == 20 + painted_any = 0 + for live, dense in zip(lives, dense_flows, strict=True): + frame_id = int(live.row.frame_id) + analysed = box_session.frame(frame_id) + height, width = int(dense.shape[0]), int(dense.shape[1]) + base = np.empty((height, width, 3), dtype=np.uint8) + base[:] = canvas_bgr + live_canvas = render_image( + ImageFrame( + base, + flow=dense, + source=SOURCE_LIVE, + flow_valid=True, + valid_mask=live.mask, + ), + mode=ARROW_PURE, + draw_style="box", + arrow_policy=policy, + canvas_bgr=canvas_bgr, + ) + replay_canvas = render_image( + ImageFrame( + base, + flow=analysed.flow, + source=SOURCE_ANALYSIS, + flow_valid=True, + result_available=True, + valid_mask=analysed.valid_mask, + ), + mode=ARROW_PURE, + draw_style="box", + arrow_policy=policy, + canvas_bgr=canvas_bgr, + ) + assert live_canvas is not None and replay_canvas is not None + assert np.array_equal(np.asarray(live_canvas), np.asarray(replay_canvas)), ( + f"frame {frame_id}: live and replay arrow canvases diverge" + ) + painted_any = max( + painted_any, + int(np.count_nonzero(np.any(np.asarray(live_canvas) != canvas_bgr, axis=2))), + ) + assert painted_any > 0, "every frame painted nothing; the discrepancy was vacuous" + + def test_a_seek_equals_playing_forward_to_the_same_frame(session): """Seek to *N* and play to *N* must produce the same picture. @@ -1372,3 +1538,46 @@ def test_a_derived_mode_is_served_from_the_result_even_on_the_recording_timeline assert plan_image(host.frames[-1]).note_key == "" finally: reader.close() + + +def test_a_stitch_grid_is_not_painted_onto_the_camera_frame(): + """Box results have no mask. The canvas is the stitch image, not 1080p. + + Candidates and the depth plane live on ``resolution_stitch``. The raw camera + frame is the old stand-in, and it leaves the arrows in one corner. + """ + from types import SimpleNamespace + + class _Reader: + complete = True + key = "stitch" + directory = Path(".") + frame_ids = np.array([1], dtype=np.int64) + times = np.array([0.0]) + keyframes: dict = {} + + def mask(self): + return None + + def frame(self, frame_id): + return SimpleNamespace(depth=np.zeros((365, 365), dtype=np.float32)) + + def grid(self, grid_id): + raise AssertionError("a depth plane already names the canvas") + + session = AnalysisSession(_Reader()) + assert session.domain_hw() == (365, 365) + + rows = np.array([15, 355], dtype=np.int32) + cols = np.array([15, 300], dtype=np.int32) + + class _GridReader(_Reader): + keyframes = {0: SimpleNamespace(id=0, kind="grid")} + + def frame(self, frame_id): + return SimpleNamespace(depth=None) + + def grid(self, grid_id): + return rows, cols + + assert AnalysisSession(_GridReader()).domain_hw() == (356, 301) diff --git a/shared/src/orisys_app_shared/acquisition/arrows.py b/shared/src/orisys_app_shared/acquisition/arrows.py index 30994a4..7d19666 100644 --- a/shared/src/orisys_app_shared/acquisition/arrows.py +++ b/shared/src/orisys_app_shared/acquisition/arrows.py @@ -77,6 +77,52 @@ _ARROW_DRAW_KW = dict( arrow_scale=0.5, below_threshold_color=hex_to_bgr(FLOW_IDLE_HEX), ) + +#: ``draw_arrows`` reads every point of this grid, including those under its threshold +#: (they become the idle dots). The cap is only a guard: a stitch image does not have +#: this many points, and a lower cap would drop cells the live painter still draws. +SDK_BOX_MAX_ARROWS = 100_000 + + +def sdk_box_policy(): + """The sample grid :func:`orisys.util.draw_arrows` walks for a box frame. + + Live box arrows are not the shared selector's default (spacing 10, 400 strongest). + A stored result has to record *this* grid or the replay cannot hand the same cells + back to the same painter. + """ + from orisys_app_shared.scene3d.visual_policy import VisualPolicy + + return VisualPolicy( + flow_on_px=float(_ARROW_DRAW_KW["threshold"]), + flow_off_px=float(_ARROW_DRAW_KW["threshold"]) * 0.75, + flow_grid_spacing=int(_ARROW_DRAW_KW["grid_spacing"]), + flow_arrow_scale=float(_ARROW_DRAW_KW["arrow_scale"]), + # ``draw_arrows`` still folds the border cells into its colour range, then + # refuses to draw them. Recording them (edge 0) is what keeps that range. + flow_edge_margin=0, + flow_length_margin=0.8, + max_arrows=SDK_BOX_MAX_ARROWS, + ).normalised() + + +def uses_sdk_box_grid(policy) -> bool: + """True when ``policy`` is the box painter's grid, not the shared selector's.""" + if policy is None: + return False + spacing = getattr(policy, "flow_grid_spacing", None) + scale = getattr(policy, "flow_arrow_scale", None) + if spacing is None and isinstance(policy, dict): + spacing = policy.get("flow_grid_spacing") + scale = policy.get("flow_arrow_scale", _ARROW_DRAW_KW["arrow_scale"]) + try: + return int(spacing) == int(_ARROW_DRAW_KW["grid_spacing"]) and abs( + float(scale) - float(_ARROW_DRAW_KW["arrow_scale"]) + ) < 1e-9 + except (TypeError, ValueError): + return False + + _FINGER_ARROW_DRAW_KW = dict( threshold=0.4, grid_spacing=10, @@ -426,6 +472,7 @@ def draw_arrow_display( magnitude_range: tuple[float, float] | None = None, per_frame_colour: bool = False, idle_dots: bool | None = None, + shared_painter: bool = False, ) -> np.ndarray: """Draw flow arrows on a solid canvas or over the processed frame. @@ -464,11 +511,19 @@ def draw_arrow_display( that shows it without reading the attribute reports "nothing measured" for "nothing could be drawn". A ``flow`` of ``None`` is a legitimate empty frame and leaves the attribute ``None``. + + ``shared_painter`` forces this module's per-sample painter for a field that came from + somewhere other than a live chain — a *recovered* field, finite only at the cells a + stored analysis result recorded. The SDK painter walks the dense grid, which on such a + field is almost entirely ``NaN``; measured on one stored box frame it paints **0 px** + where the shared painter paints **1,486 px** (the frame's own candidate cells). It is a + statement about the input, not about the sensor: the box *style* still decides the + magnification and the idle-dot tint, and a live box frame keeps the SDK path. """ base = _ensure_bgr(processed_frame) if flow is None: return _flow_result(base) - finger = str(draw_style).lower().startswith("f") + finger = str(draw_style).lower().startswith("f") or bool(shared_painter) draw_kw = dict(_FINGER_ARROW_DRAW_KW if finger else _ARROW_DRAW_KW) if colormap is not None: draw_kw["colormap"] = colormap diff --git a/shared/src/orisys_app_shared/acquisition/image_backend.py b/shared/src/orisys_app_shared/acquisition/image_backend.py index 7125e59..cb5035b 100644 --- a/shared/src/orisys_app_shared/acquisition/image_backend.py +++ b/shared/src/orisys_app_shared/acquisition/image_backend.py @@ -24,6 +24,7 @@ from __future__ import annotations from collections.abc import Sequence from dataclasses import dataclass +from typing import Any import numpy as np @@ -31,6 +32,7 @@ from orisys_app_shared.acquisition.arrows import ( depth_map_to_bgr, draw_arrow_display, flow_colormap, + uses_sdk_box_grid, ) from orisys_app_shared.acquisition.display_orientation import ( ORIENTATION_IDENTITY, @@ -96,6 +98,14 @@ RESULT_NOTE_KEYS = { #: Sources whose frames carry derived fields. DERIVED_SOURCES = (SOURCE_LIVE, SOURCE_RECOMPUTE, SOURCE_ANALYSIS) +#: Sources whose flow field was **recovered from a stored analysis result**: it holds a +#: value at the recorded candidate cells and ``NaN`` everywhere else. ``SOURCE_ANALYSIS`` is +#: the result's own timeline, ``SOURCE_REPLAY`` a derived view drawn from the result while +#: the cursor is on the recording's. Both are read from a result, and the distinction that +#: matters to a painter is *where the field came from* — a dense live field and a recovered +#: one are not the same input, though they are the same array type. +RESULT_SOURCES = (SOURCE_ANALYSIS, SOURCE_REPLAY) + #: How much a frame's domain mask must overlap the latched one before the latched one #: is kept. Measured on the shipped finger island (32937 px, 39.2% of the 300x280 #: domain): a rebuilt mask that wobbles by one pixel of rim overlap the held one at @@ -225,6 +235,13 @@ class ImageFrame: #: the states were told apart. result_state: str = "" result_reason: str = "" + #: The sample policy this frame may be drawn with, when the producer knows better than + #: the caller. A field recovered from a stored result is finite only at the pass's + #: recorded candidate cells, so it can only be re-selected on the *recorded* grid: a + #: display setting that moves the grid would otherwise sample between those cells and + #: draw nothing. ``None`` means "the caller's own policy", which is right for a field + #: the chain computed just now — it is dense, so any grid finds it. + arrow_policy: Any | None = None def has_flow(self) -> bool: if self.flow is None: @@ -420,13 +437,28 @@ def render_image( if frame.flow is None or frame.flow_valid is False: # The base picture with no arrows, and the plan's ``missing`` says why. return apply_orientation(ensure_bgr(np.asarray(frame.image_bgr)), turn) + # A recovered field is finite only at the cells the pass stored. The shared + # painter draws those cells; the SDK painter walks its own grid and finds + # nothing wherever that grid and the stored cells disagree. A box result whose + # grid *is* the SDK grid is the exception: fill the gaps with zero (the cells + # the painter does not read) and call the same ``draw_arrows`` live uses. + box_style = not str(draw_style).lower().startswith("f") + sdk_result = ( + box_style + and str(frame.source or "") in RESULT_SOURCES + and uses_sdk_box_grid(arrow_policy) + ) + flow = frame.flow + if sdk_result: + flow = np.nan_to_num(np.asarray(flow, dtype=np.float64), nan=0.0) return apply_orientation( draw_arrow_display( frame.image_bgr, - frame.flow, + flow, background="captured" if wanted == ARROW_STITCHED else "pure_color", canvas_bgr=canvas_bgr, draw_style=draw_style, + shared_painter=str(frame.source or "") in RESULT_SOURCES and not sdk_result, valid_mask=frame.valid_mask, domain_mask=domain_mask, colormap=flow_colormap(theme), diff --git a/shared/src/orisys_app_shared/analysis/model.py b/shared/src/orisys_app_shared/analysis/model.py index 30a760d..9f9376e 100644 --- a/shared/src/orisys_app_shared/analysis/model.py +++ b/shared/src/orisys_app_shared/analysis/model.py @@ -17,10 +17,11 @@ than a guess: keyframe too — and the per-frame record still carries an 8-byte digest of the mask it was computed with, so a frame that *did* change is detectable instead of being silently reconstructed against the wrong mask; -* a location set that is *not* static (the box chain has no valid mask, so its - ``max_arrows`` cap orders by magnitude and moves frame to frame) gets its own - keyframe, deduplicated by digest. The format has one code path, and "stored once" - is true by construction wherever it can be. +* a location set that is *not* static gets its own keyframe, deduplicated by digest. + The box painter's grid is static (spacing 20, every cell kept). A grid that is + still cut by ``max_arrows`` moves with the strongest cells and is stored once per + distinct set. The format has one code path, and "stored once" is true wherever + the set does not change. Display settings are deliberately **not** part of the invalidation key. The record keeps ``magnitude`` as well as the packed ``active`` bits, so a threshold change is @@ -43,6 +44,19 @@ the largest error that can be introduced is half a quantisation step, below half of one step of the 256-entry colour ramp the depth view paints with, so a reconstructed frame cannot be coloured differently from the original. +**Force is stored too, and it is a recomputation, not a recording.** When the input is +not a recording — a bare video in a generated workspace — ``forces.csv`` does not exist, +so the only force there is the one the chain computes from the frames, read through the +same ``read_forces_from_sensor`` the live loop uses. It is stored *per frame*, behind +:data:`FRAME_FLAG_FORCE` and appended after the depth payload, so a build that does not +know the flag still reads every record correctly: a chunk's own offset table delimits a +record, and trailing bytes were always ignored. Six float32 values plus two flags cost +28 B/frame — 15 MB per ten minutes, against the 4,048 B the frame already costs — and +the format version is unchanged because nothing that could be read before reads +differently now. This is the one channel whose ``Provenance.sources`` entry is +``recomputed`` on the *synthetic* path and ``recorded`` beside a real ``forces.csv``: +the two must never be confused, which is what the field is for. + Sparse storage was measured and rejected: 4-byte indices per non-zero pixel cost 910 B for the worst non-empty frame against 486 B for the whole quantised map, because a mostly-zero byte plane is nearly free for zlib while pixel indices are not. Island @@ -78,8 +92,10 @@ __all__ = [ "CODEC_ZLIB", "DEPTH_BITS", "DEPTH_HEADER", + "FORCE_PAYLOAD", "FRAME_FLAG_DEPTH", "FRAME_FLAG_DROPPED", + "FRAME_FLAG_FORCE", "FRAME_FLAG_VALIDITY", "FRAME_HEADER", "KEYFRAME_GRID", @@ -89,6 +105,7 @@ __all__ = [ "MAX_DEPTH_PIXELS", "UNIT", "DecodedDepth", + "ForceSample", "FramePayload", "KeyframeRef", "Provenance", @@ -134,6 +151,18 @@ FRAME_FLAG_DROPPED = 0x02 #: header describe it and a depth payload follows. Absent flag means *no depth #: channel at all*, which is a different fact from "the channel was all zeros". FRAME_FLAG_DEPTH = 0x04 +#: The chain produced a force sample for this frame and a :data:`FORCE_PAYLOAD` +#: follows the depth payload. Absent means the frame carries no force — either the +#: pass was not asked to collect it (a recording, whose forces live in its own +#: ``forces.csv``) or the chain exposed none. +FRAME_FLAG_FORCE = 0x08 + +#: The force payload: FN, FX, FY in newtons, then the raw triplet, then one byte of +#: validity and one of "a calibration curve was applied". Appended **last** in the +#: record so a reader that does not know this flag still decodes every earlier field +#: at the same offsets: a chunk's offset table delimits a record and the decoder has +#: always ignored trailing bytes. +FORCE_PAYLOAD = struct.Struct(" np.ndarra return _dequantise(codes, float(low), float(high)) +# ------------------------------------------------------------------ force channel +_NAN = float("nan") + + +@dataclass(frozen=True) +class ForceSample: + """One frame's force, as the chain computed it for that frame. + + Six values: the calibrated triplet in newtons and the raw triplet in device counts. + A non-finite value is *absent* — ``read_forces_from_sensor`` reports a failed readout + as ``NaN`` rather than zero, because a zero is a measurement and "no measurement" is + not. ``calibrated`` and ``valid`` are the chain's own verdicts, so a chart can label a + trace instead of inferring one from the numbers it happens to hold. + """ + + normal: float = _NAN + shear_x: float = _NAN + shear_y: float = _NAN + normal_raw: float = _NAN + shear_x_raw: float = _NAN + shear_y_raw: float = _NAN + valid: bool = True + calibrated: bool = False + + @property + def any_finite(self) -> bool: + """True when at least one of the six values is a real number.""" + return any( + value == value + for value in ( + self.normal, + self.shear_x, + self.shear_y, + self.normal_raw, + self.shear_x_raw, + self.shear_y_raw, + ) + ) + + def newtons(self) -> tuple[float, float, float]: + return (self.normal, self.shear_x, self.shear_y) + + def raw(self) -> tuple[float, float, float]: + return (self.normal_raw, self.shear_x_raw, self.shear_y_raw) + + def values(self, unit: str = "raw") -> tuple[float, float, float]: + """The triplet in ``unit``: newtons when a calibration was applied, else raw. + + Asking for newtons from a sample the chain produced without a curve returns the + missing values rather than the raw counts in their place: a count is not a force. + """ + return self.newtons() if str(unit).upper().startswith("N") else self.raw() + + @classmethod + def from_reading(cls, reading: Mapping[str, Any]) -> ForceSample: + """One :func:`~orisys_app_shared.acquisition.sdk_worker.read_forces_from_sensor` + reading, as this record's own type.""" + return cls( + normal=float(reading.get("normal_force", _NAN)), + shear_x=float(reading.get("shear_x", _NAN)), + shear_y=float(reading.get("shear_y", _NAN)), + normal_raw=float(reading.get("normal_force_raw", _NAN)), + shear_x_raw=float(reading.get("shear_x_raw", _NAN)), + shear_y_raw=float(reading.get("shear_y_raw", _NAN)), + valid=bool(reading.get("valid", True)), + calibrated=bool(reading.get("force_calibrated", False)), + ) + + def encode(self) -> bytes: + return FORCE_PAYLOAD.pack( + float(self.normal), + float(self.shear_x), + float(self.shear_y), + float(self.normal_raw), + float(self.shear_x_raw), + float(self.shear_y_raw), + 1 if self.valid else 0, + 1 if self.calibrated else 0, + ) + + @classmethod + def decode(cls, buffer: bytes | bytearray | memoryview, offset: int = 0) -> ForceSample: + ( + normal, + shear_x, + shear_y, + normal_raw, + shear_x_raw, + shear_y_raw, + valid, + calibrated, + ) = FORCE_PAYLOAD.unpack_from(memoryview(buffer), offset) + return cls( + normal=float(normal), + shear_x=float(shear_x), + shear_y=float(shear_y), + normal_raw=float(normal_raw), + shear_x_raw=float(shear_x_raw), + shear_y_raw=float(shear_y_raw), + valid=bool(valid), + calibrated=bool(calibrated), + ) + + # ------------------------------------------------------------------- frame encoding def encode_frame( *, @@ -454,8 +603,9 @@ def encode_frame( depth: Any | None = None, depth_nonzero: int = 0, dropped: bool = False, + force: ForceSample | None = None, ) -> bytes: - """One frame record: a fixed header, then the value arrays and the depth payload. + """One frame record: a fixed header, then the value arrays, depth and force. Arrays are little-endian regardless of host order, so a result written on one machine reads back identically on another. ``dx``/``dy``/``magnitude`` are stored @@ -467,6 +617,10 @@ def encode_frame( first. ``depth_nonzero`` is only consulted when ``depth`` is ``None``, which is how a caller that can count but cannot afford to hand the array over still records the count exactly. + + ``force`` is the chain's own sample for this frame, or ``None`` when the pass did not + collect one (``FRAME_FLAG_FORCE``). It is written last, after every field an older + reader knows, so the record stays readable by a build without this flag. """ dx = np.ascontiguousarray(dx, dtype=" np.ndarray | None: """This frame's depth map, or ``None`` when it had no depth channel.""" @@ -580,6 +741,12 @@ def decode_frame( depth = decode_depth( view, cursor, low=float(depth_low), high=float(depth_high) ) + cursor += depth.payload_bytes + force = None + if flags & FRAME_FLAG_FORCE: + if len(view) < cursor + FORCE_PAYLOAD.size: + raise ValueError("a frame says it carries force but is shorter than one sample") + force = ForceSample.decode(view, cursor) return DecodedFrame( frame_id=int(frame_id), t_rel_s=float(t_rel), @@ -593,6 +760,7 @@ def decode_frame( active=active.astype(bool), validity_digest=validity, depth=depth, + force=force, ) @@ -602,16 +770,19 @@ def frame_payload_bytes( with_validity: bool = True, with_depth: bool = False, depth_stored: int = 0, + with_force: bool = False, ) -> int: """Stored bytes of one frame record at ``candidates`` sample locations. ``with_depth`` adds the 6-byte depth header every frame that carries a channel pays; ``depth_stored`` adds a payload of that many bytes on top, for a caller sizing the - memory bound rather than the common case. + memory bound rather than the common case. ``with_force`` adds the fixed force sample. """ total = FRAME_HEADER.size + int(candidates) * 12 + (int(candidates) + 7) // 8 if with_validity: total += 8 + if with_force: + total += FORCE_PAYLOAD.size if with_depth: total += DEPTH_HEADER.size + max(0, int(depth_stored)) return total @@ -672,6 +843,11 @@ class FramePayload: #: header plus plane otherwise, and 0 when the frame had no depth channel. It is #: what a caller adds to the fixed part of a record to say what depth cost. depth_stored_bytes: int = 0 + #: The chain's own force sample for this frame, or ``None`` when the pass stored none. + #: It is a **recomputation** whenever it is present (see :class:`Provenance.synthetic`), + #: which is why the provenance — not this field — is what a chart must label a trace + #: from. + force: ForceSample | None = None @property def has_depth(self) -> bool: @@ -727,6 +903,7 @@ class FramePayload: depth=self.depth, depth_nonzero=self.depth_nonzero, dropped=self.dropped, + force=self.force, ) @@ -814,6 +991,14 @@ class Provenance: #: Per channel: ``recomputed`` (re-run from the video) or ``recorded`` (read from a #: CSV the recorder wrote). No channel is both. sources: dict[str, str] = field(default_factory=dict) + #: True when the input was not a recording: a bare video whose ``frames.csv`` and + #: ``session.json`` were **manufactured** for it (see + #: :mod:`orisys_app_shared.analysis.bare_video`). Every number in such a result is a + #: recomputation, so this flag is what keeps one from ever being read as a recorded + #: measurement — and ``synthetic_source`` names the video it was computed from, which + #: the workspace's own path cannot say (that path is app-owned and disposable). + synthetic: bool = False + synthetic_source: str = "" versions: dict[str, str] = field(default_factory=dict) config: dict[str, Any] = field(default_factory=dict) open_working_directory: str = "" diff --git a/shared/src/orisys_app_shared/analysis/service.py b/shared/src/orisys_app_shared/analysis/service.py index 6211da1..e8d9876 100644 --- a/shared/src/orisys_app_shared/analysis/service.py +++ b/shared/src/orisys_app_shared/analysis/service.py @@ -62,6 +62,7 @@ from orisys_app_shared.analysis.cache import ( from orisys_app_shared.analysis.model import ( ALGORITHM_VERSION, KEYFRAME_MASK, + ForceSample, Provenance, ResultKey, digest8, @@ -166,6 +167,14 @@ class ChainSpec: export_dir: Path | None = None flip_source_vertical: bool = True mirror: bool = False + #: An explicit ``force_calibration.json`` for the box chain, or ``None`` for whatever + #: the configuration's own bundle resolves. It is *not* part of :meth:`identity`, + #: deliberately: a recording was taken under one calibration and the identity of *what + #: it is* must not move when a different curve is tried against it. The calibration + #: half of the invalidation key is the recording's own ``calibration_id`` (see + #: :meth:`AnalysisService.dependency_report`), which the generated workspace sets from + #: the chosen file's content. + calibration: Path | None = None def __post_init__(self) -> None: if self.sensor_type not in ("finger", "box"): @@ -200,12 +209,16 @@ def chain_spec_for( sensor_type: str = "", config_ref: str | Path | None = None, export_dir: str | Path | None = None, + calibration: str | Path | None = None, ) -> ChainSpec: """The chain a recording was made with, unless the caller names a different one. Defaulting from ``session.json`` is what makes a stored result reproducible: the recorded ``device.config_ref`` is the configuration the footage was taken under, - not whatever happens to be installed now. + not whatever happens to be installed now. ``calibration`` is the one fact a *recording* + cannot supply — it names the curve in force at record time, not the file it lived in — + so a caller that knows which file it means (the generated workspace for a bare video) + passes it, and everything else keeps the bundle's own resolution. """ device = dict(session.get("device") or {}) kind = str(sensor_type or device.get("sensor_type") or "") @@ -221,7 +234,12 @@ def chain_spec_for( bundle = Path(str(export_dir)) elif kind == "finger" and resolved is not None: bundle = resolve_finger_export_dir(resolved) - return ChainSpec(sensor_type=kind, config_ref=resolved, export_dir=bundle) + return ChainSpec( + sensor_type=kind, + config_ref=resolved, + export_dir=bundle, + calibration=None if calibration is None else Path(str(calibration)), + ) @contextlib.contextmanager @@ -278,6 +296,15 @@ class _OpenChain: if self.spec.sensor_type == "finger": from orisys.finger import FingerRuntime + if self.spec.calibration is not None: + # The finger runtime opens its ``Sensor`` without a calibration path, + # so a chosen curve cannot be applied to this chain. Refusing is the + # only honest answer: a pass that ran anyway would store numbers from a + # different calibration than the one the caller asked for. + raise RuntimeError( + "the finger chain cannot take a chosen force calibration; it uses " + "the one the bundle resolves" + ) kwargs: dict[str, Any] = { "video": str(self.video), "verbose": False, @@ -299,6 +326,14 @@ class _OpenChain: config_name=str(self.spec.config_ref) if self.spec.config_ref else "box", + # ``cal_path`` overrides the bundle's own ``force_calibration.json``. + # Without it a caller's chosen curve would be silently ignored, since + # the SDK resolves the bundle by the configuration's *name*. + cal_path=( + None + if self.spec.calibration is None + else str(self.spec.calibration) + ), verbose=False, backend="auto", ) @@ -374,6 +409,10 @@ class LiveFrame: #: The chain's depth map for this frame, or ``None`` when it exposed none. Held only #: until the writer has encoded it: one frame at a time, never accumulated. depth: np.ndarray | None = None + #: The chain's own force sample for this frame, or ``None`` when the pass was not asked + #: to collect one. It is a *recomputation* of what the chain says about this footage — + #: never the recorded ``forces.csv`` — which the result's provenance labels. + force: ForceSample | None = None def payload_matches(self, stored: Any) -> bool: """True when a stored payload equals this frame's selection, in stored precision. @@ -479,6 +518,7 @@ class AnalysisService: chunk_frames: int = DEFAULT_CHUNK_FRAMES, max_bytes: int = DEFAULT_MAX_BYTES, working_directory: str | Path | None = None, + store_force: bool = False, on_progress: Callable[[Progress], None] | None = None, cancelled: threading.Event | None = None, ) -> None: @@ -489,13 +529,24 @@ class AnalysisService: ) #: The policy the samples are selected with. Recorded, never keyed (see #: ``model.py``): the shipped defaults make a stored result independent of the - #: operator's display preferences. + #: operator's display preferences. A box chain records the live painter's own + #: grid; the shared defaults are the finger painter's and would drop every cell + #: ``draw_arrows`` still draws. + if policy is None and chain.sensor_type == "box": + from orisys_app_shared.acquisition.arrows import sdk_box_policy + + policy = sdk_box_policy() self.policy = policy or VisualPolicy() self.reference_frame = None if reference_frame is None else int(reference_frame) self.chunk_frames = int(chunk_frames) self.working_directory = ( None if working_directory is None else Path(working_directory) ) + #: Whether each frame also stores the chain's own force sample. Off for a recording, + #: whose force is a *measurement in its own file* (``forces.csv``) and must not be + #: overwritten in the reader's mind by a recomputation of it; on for a bare video, + #: where the chain's force is the only force there is (see ``bare_video``). + self.store_force = bool(store_force) self.on_progress = on_progress self.cancelled = cancelled self._reader: RecordingReader | None = None @@ -689,6 +740,17 @@ class AnalysisService: state.reasons.append(f"{type(exc).__name__}: {exc}") finally: self._discard_working_directory(working) + if ( + not state.reasons + and self.reference_frame is not None + and not provenance.counts.get("reset_count") + ): + # The frame was checked against ``frames.csv`` and the chain never reached it: + # a box link consumes the first decoded frames, so a reference that names one + # of them is a request the pass cannot honour. Storing the result anyway would + # claim ``reference_is_explicit`` while the tracker was never cleared, which is + # the one thing the reference frame is *for*. + state.reasons.append(f"reference_not_applied:{self.reference_frame}") counts = provenance.counts counts.update(state.plan) counts["frames_written"] = state.frames_written @@ -807,6 +869,7 @@ class AnalysisService: provenance.warnings.append(f"chain:{error}") state.reasons.append(f"chain_failed:{row.frame_id}") return + force = self._read_force(chain) flow = getattr(chain.sensor, "flow", None) if flow is None or not getattr(flow, "size", 0): state.reasons.append(f"no_flow:{row.frame_id}") @@ -840,9 +903,28 @@ class AnalysisService: reset_applied=reset_applied, segment=int(number), depth=depth_array, + force=force, ) # ------------------------------------------------------------------- internals + def _read_force(self, chain: _OpenChain) -> ForceSample | None: + """This frame's force sample, or ``None`` when the pass was not asked for one. + + Read after the chain has computed, through the same + :func:`~orisys_app_shared.acquisition.sdk_worker.read_forces_from_sensor` the live + loop uses, so a recomputed trace is the live measurement re-run rather than a second + opinion about it. A readout the sensor cannot give comes back as ``NaN`` with + ``valid=False`` — recorded as-is, because a zero in its place would be a measurement + nothing made. + """ + if not self.store_force: + return None + from orisys_app_shared.acquisition.sdk_worker import read_forces_from_sensor + + return ForceSample.from_reading( + read_forces_from_sensor(chain.sensor, config_ref=self.chain.config_ref) + ) + def _make_working_directory(self, key: ResultKey) -> Path: if self.working_directory is not None: self.working_directory.mkdir(parents=True, exist_ok=True) @@ -934,9 +1016,12 @@ class AnalysisService: depth=live.depth, depth_nonzero=live.depth_nonzero, dropped=row.dropped, + force=live.force, ) counts = provenance.counts counts["frame_record_bytes"] = counts.get("frame_record_bytes", 0) + len(record) + if live.force is not None: + self._accumulate_force(counts, live.force) if live.depth is not None: counts["depth_frames"] = counts.get("depth_frames", 0) + 1 counts["depth_pixels"] = int( @@ -989,9 +1074,29 @@ class AnalysisService: float(counts.get("depth_abs_sum", 0.0)) + float(depth_abs_sum), 6 ) + @staticmethod + def _accumulate_force(counts: dict[str, Any], force: ForceSample) -> None: + """Count what the stored force trace is, so a chart can promise it or say why not. + + ``force_finite_frames`` is the number a caller has to see before offering a trace: + a chain that answers every frame with ``NaN`` has stored no measurement, and an + empty chart with no explanation is what this counter exists to prevent. + """ + counts["force_frames"] = counts.get("force_frames", 0) + 1 + counts["force_valid_frames"] = counts.get("force_valid_frames", 0) + ( + 1 if force.valid else 0 + ) + counts["force_finite_frames"] = counts.get("force_finite_frames", 0) + ( + 1 if force.any_finite else 0 + ) + counts["force_calibrated_frames"] = counts.get( + "force_calibrated_frames", 0 + ) + (1 if force.calibrated else 0) + def _provenance(self, key: ResultKey, facts: dict[str, Any]) -> Provenance: session = facts["session"] policy = self.policy.normalised() + synthetic = dict(session.get("synthetic") or {}) return Provenance( format_version=model_module.ANALYSIS_FORMAT_VERSION, algorithm=ALGORITHM_VERSION, @@ -1019,6 +1124,7 @@ class AnalysisService: "flow_on_px": policy.flow_on_px, "flow_off_px": policy.flow_off_px, "flow_grid_spacing": policy.flow_grid_spacing, + "flow_arrow_scale": policy.flow_arrow_scale, "flow_edge_margin": policy.flow_edge_margin, "flow_length_margin": policy.flow_length_margin, "max_arrows": policy.max_arrows, @@ -1026,8 +1132,16 @@ class AnalysisService: sources={ "flow": "recomputed", "depth": "recomputed", - "forces": "recorded", + # The one channel whose answer depends on what the input *is*. Beside a + # recording, ``forces.csv`` is the measurement and the result stores none of + # it; over a bare video there is no CSV, so the chain's own recomputation is + # what a chart can draw — and it says so. + "forces": "recomputed" + if self.store_force + else "recorded", }, + synthetic=bool(synthetic), + synthetic_source=str(synthetic.get("source_video") or ""), versions={ "sdk": _version_of("orisys"), "shared": _version_of("orisys_app_shared"), diff --git a/shared/tests/test_image_backend.py b/shared/tests/test_image_backend.py index ffbfe56..5e8a007 100644 --- a/shared/tests/test_image_backend.py +++ b/shared/tests/test_image_backend.py @@ -919,3 +919,144 @@ def test_resetting_the_latch_clears_the_session_boundary(): assert np.array_equal(latch.update(other), other), ( "a new session kept the previous session's island" ) + + +# --------------------------------------------- a field recovered from a stored result +def _policy(): + from orisys_app_shared.scene3d.visual_policy import VisualPolicy + + return VisualPolicy(flow_grid_spacing=10, max_arrows=400).normalised() + + +def _recovered_flow(height: int = 60, width: int = 80): + """A flow field as an analysis result hands one over: finite only at its own cells. + + This is what the replay's reconstruction produces — the pass recorded candidate + *locations* and their displacements, and the field is rebuilt with those values at + those cells and ``NaN`` everywhere else. A dense field is the opposite case, and the + difference between them is what the painter has to be chosen by. + """ + from orisys_app_shared.scene3d.visual_policy import select_flow_samples + + dense = np.zeros((height, width, 2), dtype=np.float64) + dense[..., 0] = 1.5 + dense[..., 1] = -0.75 + samples = select_flow_samples(dense, None, _policy()) + sparse = np.full((height, width, 2), np.nan, dtype=np.float64) + sparse[samples.rows, samples.cols, 0] = samples.dx + sparse[samples.rows, samples.cols, 1] = samples.dy + return sparse, samples + + +def _painted(image, canvas=(0x16, 0x13, 0x12)) -> int: + """Pixels the painter changed from the canvas colour.""" + array = np.asarray(image) + return int( + np.count_nonzero(np.any(array != np.asarray(canvas, np.uint8), axis=-1)) + ) + + +def test_a_recovered_field_is_drawn_by_the_shared_painter_whatever_the_style_is(): + """Where a frame came from decides the painter, not which sensor's style is selected. + + The SDK painter walks the dense sampling grid, so on a field that is finite only at a + stored result's candidate cells it finds no sample at all and paints nothing: measured + on a real box recording, **0 px** against the shared painter's **1,486 px** on the same + frame. So the routing is by ``ImageFrame.source`` — the frame came from a result — and + the box *style* still owns the magnification and the idle-dot tint. + """ + from orisys_app_shared.scene3d.visual_policy import select_flow_samples + + sparse, samples = _recovered_flow() + base = np.zeros((60, 80, 3), dtype=np.uint8) + base[:, :] = (0x16, 0x13, 0x12) + + result_frame = backend.ImageFrame( + base, flow=sparse, source=backend.SOURCE_ANALYSIS, flow_valid=True + ) + live_frame = backend.ImageFrame( + base, flow=sparse, source=backend.SOURCE_LIVE, flow_valid=True + ) + drawn = backend.render_image( + result_frame, mode=backend.ARROW_PURE, draw_style="box", arrow_policy=_policy() + ) + expected = draw_arrow_display( + base, + sparse, + background="pure_color", + draw_style="box", + # The backend hands the theme's ramp to the painter, so the reference call has to + # as well; the default is ``cv2.COLORMAP_JET``, which is a different picture. + colormap=flow_colormap("dark"), + policy=_policy(), + shared_painter=True, + ) + assert np.array_equal(np.asarray(drawn), np.asarray(expected)), ( + "a result's canvas is not the shared painter's output" + ) + assert _painted(drawn) > 0, "the recovered field painted nothing" + + # The same array handed over as a *live* field keeps the SDK path — which is what the + # sensor family's draw style has always meant, and which paints nothing here. + sdk = backend.render_image( + live_frame, mode=backend.ARROW_PURE, draw_style="box", arrow_policy=_policy() + ) + assert _painted(sdk) == 0, "the SDK painter drew a recovered field" + assert not np.array_equal(np.asarray(drawn), np.asarray(sdk)) + + # One marker per recorded cell at most: the recovered field carries no other value, so + # nothing outside those cells can become a sample. + assert select_flow_samples(sparse, None, _policy()).rows.size == samples.rows.size + + +def test_a_box_result_on_the_sdk_grid_matches_the_live_painter(): + """A box pass stores the live painter's grid, so filling the gaps and calling it + back draws the same picture as the dense field.""" + from orisys_app_shared.acquisition.arrows import sdk_box_policy + from orisys_app_shared.scene3d.visual_policy import ( + DISPLACEMENT_PX_PER_FLOW, + select_flow_samples, + ) + + policy = sdk_box_policy() + dense = np.zeros((80, 100, 2), dtype=np.float64) + dense[20:60, 20:80, 0] = 1.5 + dense[30:40, 30:50, 0] = 8.0 + samples = select_flow_samples(dense, None, policy) + assert samples.rows.size > 0 + dx = np.asarray(samples.dx, np.float64) + dy = np.asarray(samples.dy, np.float64) + length = np.sqrt(dx * dx + dy * dy) + flow_mag = np.asarray(samples.magnitude, np.float64) / DISPLACEMENT_PX_PER_FLOW + safe = np.maximum(length, 1e-12) + moving = length > 0.0 + sparse = np.full(dense.shape, np.nan, dtype=np.float64) + sparse[samples.rows, samples.cols, 0] = np.where(moving, dx / safe * flow_mag, 0.0) + sparse[samples.rows, samples.cols, 1] = np.where(moving, dy / safe * flow_mag, 0.0) + base = np.zeros(dense.shape[:2] + (3,), np.uint8) + base[:] = (0x16, 0x13, 0x12) + cmap = flow_colormap("dark") + live = draw_arrow_display( + base, + dense, + draw_style="box", + background="pure_color", + canvas_bgr=(0x16, 0x13, 0x12), + colormap=cmap, + ) + frame = backend.ImageFrame( + base, + flow=sparse, + source=backend.SOURCE_ANALYSIS, + flow_valid=True, + result_available=True, + ) + drawn = backend.render_image( + frame, + mode=backend.ARROW_PURE, + draw_style="box", + arrow_policy=policy, + canvas_bgr=(0x16, 0x13, 0x12), + ) + assert np.array_equal(np.asarray(drawn), np.asarray(live)) + assert _painted(drawn) > 0 diff --git a/shared/tests/test_replay_analysis.py b/shared/tests/test_replay_analysis.py index c65f58c..4390c76 100644 --- a/shared/tests/test_replay_analysis.py +++ b/shared/tests/test_replay_analysis.py @@ -13,9 +13,8 @@ The inputs are real: the 33-frame box recording under ``dist/`` (its own ``frame ``session.json`` and ``raw_0001.mp4``) and the shipped finger configuration and asset bundle. The pass is driven with the **box** chain on it for the offset and round-trip checks, and with the **finger** chain on the same footage for the keyframe, reset and -size checks — the finger chain over box footage is exactly what the Step 0 measurement -did, and it is the only way to get a validity mask (and therefore a static candidate -grid) out of this recording. +size checks. The finger chain is what produces a validity mask; the box chain's +painter grid is static without one. """ from __future__ import annotations @@ -530,14 +529,16 @@ def test_the_stored_size_is_the_measured_encoding(box_pass, finger_pass): _, box = box_pass box_reader = AnalysisReader(box.directory) try: - # The box chain has no validity mask, so there is no digest and every frame - # carries its own grid keyframe: 4,882 B of record per frame at 400 candidates, - # plus whatever depth it carries. + # The box chain has no validity mask. Its sample grid is the live painter's + # (spacing 20 over the 365 stitch, every cell kept): 18 x 18 = 324 candidates, + # one keyframe, and the per-frame record is values plus whatever depth it carries. + frames = list(box_reader.frames()) + assert {item.candidates for item in frames} == {324} assert box_reader.frame_record_bytes == sum( - 32 + 400 * 12 + 50 + item.depth_stored_bytes - for item in box_reader.frames() + frame_payload_bytes(324, with_validity=False) + item.depth_stored_bytes + for item in frames ) - assert box_reader.keyframe_counts == {"grid": 26} + assert box_reader.keyframe_counts == {"grid": 1} assert box_reader.mask() is None finally: box_reader.close()