From 3daee6d40753319da921983d16c88396412d5a1f Mon Sep 17 00:00:00 2001 From: ykcheungab Date: Fri, 25 Sep 2026 17:53:07 +0800 Subject: [PATCH] fixed display problem --- .gitignore | 5 + apps/studio/src/orisys_studio/app_window.py | 29 ++- apps/studio/tests/test_ui_smoke.py | 53 ++++++ apps/studio/tools/ui_check.py | 4 + .../orisys_app_shared/acquisition/arrows.py | 12 +- .../acquisition/image_backend.py | 100 ++++++++++- shared/tests/test_image_backend.py | 169 ++++++++++++++++++ 7 files changed, 367 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 14cf82f..b25be0f 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,11 @@ apps/studio/preference.json shared/tests/_scratch/ shared/tests/_scratch_*/ +# Working artefacts: screenshots and probe output from 3D/UI diagnosis, and assets parked +# out of the shipping path. None of these are inputs to a build or a release. +.snap/ +_parked_stale_assets/ + # IDE / OS .idea/ .vscode/ diff --git a/apps/studio/src/orisys_studio/app_window.py b/apps/studio/src/orisys_studio/app_window.py index a95befe..0613042 100644 --- a/apps/studio/src/orisys_studio/app_window.py +++ b/apps/studio/src/orisys_studio/app_window.py @@ -30,6 +30,7 @@ from orisys_app_shared.acquisition.image_backend import ( SOURCE_LIVE, SOURCE_PREVIEW, SOURCE_RECOMPUTE, + DomainMaskLatch, ImageFrame, plan_image, render_image, @@ -222,6 +223,11 @@ class StudioMainWindow(QMainWindow): self._last_calibration_export = "" self._depth_range_value: tuple[float, float] | None = None self._last_depth_frame = None + #: The 2D domain boundary, latched for the session the way the depth range is: + #: the island is the device's own geometry and is rebuilt from the same remap + #: every frame, so re-deriving the edge per frame redrew a boundary that had not + #: moved (see ``image_backend.DomainMaskLatch``). + self._domain_mask = DomainMaskLatch() icon = QIcon(str(logo_path())) if not icon.isNull(): @@ -2028,6 +2034,10 @@ class StudioMainWindow(QMainWindow): self._last_display_available = None self._depth_range_value = None self._last_depth_frame = None + # The new session has not established a domain boundary yet; the previous + # session's island must not survive into it (a different sensor is a different + # island, and the latch compares against what it holds). + self._domain_mask.reset() # The session's own bookkeeping (its counter and telemetry) is the # controller's: telling it a run starts is what drops the previous run's # numbers, so none of them can be read as this run's. @@ -2461,6 +2471,19 @@ class StudioMainWindow(QMainWindow): return "" return t("panels.depth_scale", low=f"{span[0]:.3f}", high=f"{span[1]:.3f}") + def _latched_domain_mask(self, frame: "ImageFrame"): + """The 2D domain boundary to draw, latched for the session. + + Exactly the treatment ``_depth_range`` gives the colour scale, and for the same + reason: the boundary is not a per-frame measurement. It is the device's own UV + island — geometry, rebuilt from the same remap on every frame — and the depth + view paints it as its "no depth" grey, so rebuilding it per frame redrew an edge + that had not moved while any difference in ``frame.valid_mask`` repainted the + whole boundary, which reads as lag. The latch holds the boundary until a frame + no longer describes the same island (see ``DomainMaskLatch``). + """ + return self._domain_mask.update(frame.valid_mask) + def _replay_owns_image(self) -> bool: """True while the 2D panel shows the replay, not the live camera. @@ -2518,7 +2541,10 @@ class StudioMainWindow(QMainWindow): draw_style = self._draw_style() # The legend is decided by the same inputs as the picture: the arrow # magnification of this sensor's draw style, and whether this frame - # carries a valid-area mask. + # carries a valid-area mask. It stays the *frame's* statement rather than the + # latched boundary's, because that is what the chip claims: the arrows are + # gated by this frame's own mask on every path, latched boundary or not. + boundary = self._latched_domain_mask(frame) self.arrow_panel.set_arrow_scale(arrow_display_scale(draw_style)) self.arrow_panel.set_valid_area(frame.valid_mask is not None) if mode == DEPTH: @@ -2534,6 +2560,7 @@ class StudioMainWindow(QMainWindow): draw_style=draw_style, depth_range=self._depth_range(), canvas_bgr=hex_to_bgr(color(self._theme, "arrow_canvas", "#0B1830")), + domain_mask=boundary, ) if rendered is None: # Never substitute another picture here: a missing depth map that showed diff --git a/apps/studio/tests/test_ui_smoke.py b/apps/studio/tests/test_ui_smoke.py index 3f3e472..25467ed 100644 --- a/apps/studio/tests/test_ui_smoke.py +++ b/apps/studio/tests/test_ui_smoke.py @@ -925,6 +925,59 @@ def test_depth_view_reports_missing_data_and_fixes_its_scale(make_window): _finish(window, stuck) +def test_the_domain_boundary_is_latched_for_the_session(make_window): + """The 2D boundary is the device's island: rebuilt per frame, decided once. + + Same treatment as the depth colour scale above, and for the same reason — a mask + that is re-derived every frame redraws an edge that has not moved, and any pixel of + difference repaints the whole boundary (measured: a 0.73% area wobble repaints 69% + of the boundary band). + """ + import numpy as np + + window, stuck = _running_window(make_window) + window.arrow_panel.set_view_mode("arrows_pure") + island = np.zeros((8, 8), dtype=bool) + island[1:7, 1:7] = True + island_wobble = island.copy() + island_wobble[1, 1] = False # one pixel of rim, i.e. 1.6% of a 36-pixel island + + window._last_rendered_view_mode = None + stuck.emit(ui_check._fake_packet(0, valid_mask=island)) + window._on_run_tick() + assert np.array_equal(window._domain_mask.mask, island), ( + "the session never established the device's island" + ) + + window._last_rendered_view_mode = None + stuck.emit(ui_check._fake_packet(1, valid_mask=island_wobble)) + window._on_run_tick() + assert np.array_equal(window._domain_mask.mask, island), ( + "a rebuilt mask moved the session's boundary" + ) + assert (window._domain_mask.adoptions, window._domain_mask.held) == (1, 1) + + # An island that really changed is adopted rather than frozen out. + moved = np.zeros((8, 8), dtype=bool) + moved[2:8, 2:8] = True + window._last_rendered_view_mode = None + stuck.emit(ui_check._fake_packet(2, valid_mask=moved)) + window._on_run_tick() + assert np.array_equal(window._domain_mask.mask, moved), ( + "a moved island kept the old boundary on screen" + ) + assert window._domain_mask.adoptions == 2 + + # A new run is a new session: the previous island must not survive into it. + _finish(window, stuck) + window._start_run() + assert window._domain_mask.mask is None, ( + "the previous session's island survived into a new run" + ) + window._run_timer.stop() + window._stop_run(reason="user_stop") + + # ------------------------------------------------------------- experiment record def test_recording_writes_the_documented_experiment_layout(make_window, record_root): import csv diff --git a/apps/studio/tools/ui_check.py b/apps/studio/tools/ui_check.py index 062a1c8..c43d036 100644 --- a/apps/studio/tools/ui_check.py +++ b/apps/studio/tools/ui_check.py @@ -177,6 +177,7 @@ def _fake_packet( mono_t: float | None = None, shear_n: float | None = None, frame=None, + valid_mask=None, ): """A packet shaped like the real worker's, for the record-channel checks. @@ -213,6 +214,9 @@ def _fake_packet( invalid_reason=reason, raw_available=True, depth=depth, + # The device's own UV-island mask, which the real workers pass through from the + # runtime (``runtime.valid_mask``); the 2D boundary is drawn from it. + valid_mask=valid_mask, # Per-field provenance: a producer has to say whether the arrays it hands over # are this frame's own (the 3D sink refuses an unstated field rather than # colouring something it cannot claim). A failed frame is stale here, exactly diff --git a/shared/src/orisys_app_shared/acquisition/arrows.py b/shared/src/orisys_app_shared/acquisition/arrows.py index 14714e6..662e284 100644 --- a/shared/src/orisys_app_shared/acquisition/arrows.py +++ b/shared/src/orisys_app_shared/acquisition/arrows.py @@ -199,6 +199,7 @@ def draw_arrow_display( canvas_bgr: Sequence[int] | None = None, draw_style: str = "box", valid_mask: Optional[np.ndarray] = None, + domain_mask: Optional[np.ndarray] = None, colormap: Optional[np.ndarray] = None, exterior_bgr: Sequence[int] | None = None, island_bgr: Sequence[int] | None = None, @@ -214,6 +215,11 @@ def draw_arrow_display( inside, slightly darker outside). Without a mask the whole rectangle uses ``canvas_bgr`` / ``island_bgr`` for backward compatibility. + ``domain_mask`` is the boundary to paint that canvas from, when the caller holds a + latched one; ``None`` means this frame's own ``valid_mask``. It never reaches + :func:`select_flow_samples`: the arrows stay this frame's own measurement, so + latching the boundary cannot put an arrow on a sample the device does not vouch for. + Finger style uses the shared :mod:`visual_policy` sampler so 2D and 3D arrows share the same UV sample points and thresholds. Box style keeps the SDK path. """ @@ -233,14 +239,15 @@ def draw_arrow_display( if exterior_bgr is None else tuple(int(v) for v in exterior_bgr) ) - if valid_mask is not None: + boundary = valid_mask if domain_mask is None else domain_mask + if boundary is not None: from orisys_app_shared.scene3d.visual_policy import ( build_domain_mask, paint_domain_background, ) domain, _ = build_domain_mask( - valid_mask, shape_hw=base.shape[:2] + boundary, shape_hw=base.shape[:2] ) canvas = paint_domain_background( base.shape[:2], @@ -248,7 +255,6 @@ def draw_arrow_display( island_bgr=inside, # type: ignore[arg-type] exterior_bgr=outside, # type: ignore[arg-type] ) - valid_mask = domain else: canvas = np.empty_like(base) canvas[:, :] = inside diff --git a/shared/src/orisys_app_shared/acquisition/image_backend.py b/shared/src/orisys_app_shared/acquisition/image_backend.py index 67402a3..bbb906c 100644 --- a/shared/src/orisys_app_shared/acquisition/image_backend.py +++ b/shared/src/orisys_app_shared/acquisition/image_backend.py @@ -61,6 +61,94 @@ MISSING_NOTE_KEY = "panels.frame_missing_note" #: Sources whose frames carry derived fields. DERIVED_SOURCES = (SOURCE_LIVE, SOURCE_RECOMPUTE) +#: How much a frame's domain mask must overlap the latched one before the latched one +#: is kept. Measured on the shipped finger island (32937 px, 39.2% of the 300x280 +#: domain): a rebuilt mask that wobbles by one pixel of rim overlap the held one at +#: IoU >= 0.9633, a 200-pixel speckle at 0.9879, while an island that really moved +#: 6 px falls to 0.9102. 0.95 therefore absorbs the wobble -- which is what repaints +#: the edge, 69% of the boundary band for a 0.73% area flip -- and still adopts a real +#: move. A per-pixel frame-count hysteresis was rejected: the measured flips are spread +#: around the whole rim rather than persistent at any pixel, so it would need a long +#: count to suppress them and would then delay a genuine move by that many frames. +DOMAIN_MASK_MIN_IOU = 0.95 + + +def mask_overlap(left: np.ndarray, right: np.ndarray) -> float: + """Intersection over union of two equally shaped boolean masks. + + Both empty counts as total agreement: two frames that agree there is no island at + all are not a change. + """ + a = np.asarray(left, dtype=bool) + b = np.asarray(right, dtype=bool) + union = int(np.count_nonzero(a | b)) + if union == 0: + return 1.0 + return float(np.count_nonzero(a & b)) / float(union) + + +class DomainMaskLatch: + """The 2D domain boundary, kept until it stops describing the same island. + + ``build_domain_mask``/``paint_domain_background`` rebuild the island/exterior canvas + from ``ImageFrame.valid_mask`` on every frame, and the depth view paints that same + boundary as its "no depth" grey. The mask is the device's own UV island: geometry, + rebuilt from the same remap every frame, and the divergence map the depth view reads + is zero-filled everywhere outside it. So the edge being redrawn is the same edge, + and any difference in the mask repaints the whole boundary, which reads as the + picture lagging rather than as the boundary moving. + + Latching it is the same treatment the depth *colour scale* already gets + (``StudioMainWindow._depth_range``): one decision, kept for the session, so frames + stay comparable. The latch compares against the mask it is holding — not against the + previous frame — so a slow drift accumulates and is adopted once it is material, + while a per-frame flicker around the same island never is. + + ``adoptions`` and ``held`` make the behaviour observable: a session that keeps + re-latching a moving boundary and a session that correctly holds a static one look + the same on screen, and only these counters tell them apart. + + The array is held, not copied. A producer that hands a mask over must not write into + it again — the same contract ``scene3d.contracts.freeze`` enforces for every other + array a frame carries, which is what makes holding it safe. + """ + + __slots__ = ("_mask", "min_iou", "adoptions", "held") + + def __init__(self, *, min_iou: float = DOMAIN_MASK_MIN_IOU) -> None: + self._mask: Optional[np.ndarray] = None + self.min_iou = float(min_iou) + self.adoptions = 0 + self.held = 0 + + @property + def mask(self) -> Optional[np.ndarray]: + return self._mask + + def reset(self) -> None: + """Drop the latched boundary: a new session has not established one yet.""" + self._mask = None + self.adoptions = 0 + self.held = 0 + + def update(self, mask: Optional[np.ndarray]) -> Optional[np.ndarray]: + """The mask to draw the boundary with, given this frame's own mask. + + A frame that carries no mask cannot move the boundary, so it inherits the + session's: the island belongs to the sensor, not to the frame that happens not + to state it (a preview frame, a frame from a source that never had one). + """ + if mask is None: + return self._mask + current = np.asarray(mask, dtype=bool) + if self._mask is not None and current.shape == self._mask.shape: + if mask_overlap(self._mask, current) >= self.min_iou: + self.held += 1 + return self._mask + self._mask = current + self.adoptions += 1 + return self._mask + @dataclass(frozen=True) class ImageFrame: @@ -196,15 +284,24 @@ def render_image( draw_style: str = "box", depth_range: Optional[tuple[float, float]] = None, canvas_bgr: Optional[Sequence[int]] = None, + domain_mask: Optional[np.ndarray] = None, ) -> Optional[np.ndarray]: """Draw ``frame`` in ``mode``; ``None`` when there is nothing to draw. The mode is resolved through the plan first, so a caller that asks for a view the frame cannot support gets the plan's default rather than a picture assembled from an array that is not there. + + ``domain_mask`` is the *latched* boundary to paint the island/exterior canvas and + the depth view's "no depth" grey from (see :class:`DomainMaskLatch`); ``None`` means + "use this frame's own ``valid_mask``", which is what a caller with no session to + latch to should do. It never gates the arrows or the depth values: those stay this + frame's own data, so a sample the device does not vouch for is still not drawn just + because the boundary around it was latched. """ plan = plan or plan_image(frame) wanted = plan.resolve(mode) + boundary = frame.valid_mask if domain_mask is None else domain_mask if wanted == DEPTH: # Never substitute another picture for a missing depth map: that made "no # depth" look like a depth reading. The panel says so instead. @@ -214,7 +311,7 @@ def render_image( frame.depth, flow_colormap(theme), value_range=depth_range, - invalid_mask=frame.valid_mask, + invalid_mask=boundary, ) if frame.image_bgr is None: return None @@ -229,6 +326,7 @@ def render_image( canvas_bgr=canvas_bgr, draw_style=draw_style, valid_mask=frame.valid_mask, + domain_mask=domain_mask, colormap=flow_colormap(theme), ) # ``recorded``: the frame is the picture, with nothing drawn over it. diff --git a/shared/tests/test_image_backend.py b/shared/tests/test_image_backend.py index 3c02f7f..b409771 100644 --- a/shared/tests/test_image_backend.py +++ b/shared/tests/test_image_backend.py @@ -191,3 +191,172 @@ def test_a_missing_field_is_reported_not_invented(): base = backend.render_image(no_flow, mode="arrows_pure") assert base is not None assert np.array_equal(base, _image()), "the base picture must not be redrawn" + + +# ------------------------------------------------------- the latched boundary +#: A domain the same size and the same kind of shape as the shipped finger sensor's +#: 300x280 UV island: one large blob with a long smooth rim, so "a one-pixel wobble" +#: and "the island moved" have the overlap the real package has (measured on the +#: shipped island: a 1 px wobble scores IoU 0.9633, a 6 px move 0.9102). +DOMAIN_HW = (300, 280) +ISLAND_RADIUS_PX = 95 + + +def _island(h: int = DOMAIN_HW[0], w: int = DOMAIN_HW[1]) -> np.ndarray: + yy, xx = np.mgrid[0:h, 0:w] + return ((yy - h / 2.0) ** 2 + (xx - w / 2.0) ** 2) <= float(ISLAND_RADIUS_PX) ** 2 + + +def _blank_image(h: int = DOMAIN_HW[0], w: int = DOMAIN_HW[1]) -> np.ndarray: + """A base picture the two views do not read: only depth and the mask are drawn.""" + return np.zeros((h, w, 3), dtype=np.uint8) + + +def _shift(mask: np.ndarray, dy: int, dx: int) -> np.ndarray: + out = np.zeros_like(mask) + h, w = mask.shape + ys = slice(max(0, dy), h + min(0, dy)) + xs = slice(max(0, dx), w + min(0, dx)) + ys2 = slice(max(0, -dy), h + min(0, -dy)) + xs2 = slice(max(0, -dx), w + min(0, -dx)) + out[ys, xs] = mask[ys2, xs2] + return out + + +def test_overlap_of_two_identical_masks_is_one(): + mask = _island() + assert backend.mask_overlap(mask, mask) == 1.0 + assert backend.mask_overlap(mask, ~mask) == 0.0 + # Two frames that agree there is no island at all are not a change. + empty = np.zeros((4, 4), dtype=bool) + assert backend.mask_overlap(empty, empty) == 1.0 + + +def test_the_gate_separates_a_rebuilt_mask_from_a_moved_one(): + """The threshold itself, on the two cases it exists to tell apart.""" + island = _island() + for dy, dx in ((1, 0), (0, 1), (1, 1), (-1, 1), (2, 0)): + assert backend.mask_overlap(island, _shift(island, dy, dx)) >= ( + backend.DOMAIN_MASK_MIN_IOU + ), f"a {dy},{dx} rebuild would be taken for a moved boundary" + assert backend.mask_overlap(island, _shift(island, 6, 0)) < backend.DOMAIN_MASK_MIN_IOU + + +def test_a_stable_domain_mask_is_latched_once_for_the_session(): + """The device's island is geometry: rebuilt per frame, decided once.""" + latch = backend.DomainMaskLatch() + mask = _island() + seen = {latch.update(mask).tobytes() for _ in range(20)} + assert len(seen) == 1, "a stable mask was rebuilt into a different array" + assert np.array_equal(latch.mask, mask) + assert (latch.adoptions, latch.held) == (1, 19) + + +def test_a_wobbling_domain_mask_does_not_repaint_the_boundary(): + """A one-pixel rim wobble is the mask being rebuilt, not the island moving. + + The measured case: on the shipped finger island this is a 0.73% area flip that + repaints 69% of the boundary band and 35% of the rendered edge pixels. + """ + latch = backend.DomainMaskLatch() + image = _blank_image() + # A zero field, so nothing but the boundary can change the picture. + flow = np.zeros((*DOMAIN_HW, 2), dtype=np.float32) + depth = np.ones(DOMAIN_HW, dtype=np.float32) + wobbles = [(0, 0), (1, 0), (0, -1), (1, 1), (-1, 0), (0, 1), (-1, -1)] + arrows, maps = [], [] + for dy, dx in wobbles: + mask = _shift(_island(), dy, dx) + frame = backend.ImageFrame(image, flow=flow, depth=depth, valid_mask=mask) + used = latch.update(mask) + arrows.append( + backend.render_image( + frame, mode="arrows_pure", draw_style="finger", domain_mask=used + ) + ) + maps.append( + backend.render_image( + frame, mode="depth", depth_range=(0.0, 1.0), domain_mask=used + ) + ) + assert all(np.array_equal(arrows[0], other) for other in arrows), ( + "the arrow canvas repainted a boundary that had not moved" + ) + assert all(np.array_equal(maps[0], other) for other in maps), ( + "the depth view's no-data boundary repainted a boundary that had not moved" + ) + assert latch.adoptions == 1, "a rim wobble was taken for a moved boundary" + + +def test_a_real_move_of_the_domain_is_followed(): + """The latch must not freeze a boundary that actually moved.""" + latch = backend.DomainMaskLatch() + island = _island() + latch.update(island) + moved = _shift(island, 6, 0) + used = latch.update(moved) + assert np.array_equal(used, moved), "a moved island kept the old boundary" + assert latch.adoptions == 2 + + frame = backend.ImageFrame( + _blank_image(), flow=np.zeros((*DOMAIN_HW, 2), np.float32) + ) + before = backend.render_image( + frame, mode="arrows_pure", draw_style="finger", domain_mask=island + ) + after = backend.render_image( + frame, mode="arrows_pure", draw_style="finger", domain_mask=used + ) + assert not np.array_equal(before, after) + + +def test_a_frame_without_a_mask_cannot_move_the_boundary(): + latch = backend.DomainMaskLatch() + assert latch.update(None) is None, "a session with no mask invented one" + island = _island() + latch.update(island) + assert np.array_equal(latch.update(None), island), ( + "a frame that states no mask dropped the session's boundary" + ) + assert (latch.adoptions, latch.held) == (1, 0) + + +def test_the_latched_boundary_never_gates_the_arrows(): + """Latching the boundary must not put an arrow on an unvouched-for sample.""" + image = _blank_image() + flow = np.zeros((*DOMAIN_HW, 2), dtype=np.float32) + flow[::10, ::10, 0] = 3.0 + # The frame claims nothing; the session's boundary is the whole domain. + frame = backend.ImageFrame(image, flow=flow, valid_mask=np.zeros(DOMAIN_HW, bool)) + boundary = np.ones(DOMAIN_HW, dtype=bool) + got = backend.render_image( + frame, mode="arrows_pure", draw_style="finger", domain_mask=boundary + ) + expected = draw_arrow_display( + image, + flow, + background="pure_color", + draw_style="finger", + valid_mask=frame.valid_mask, + domain_mask=boundary, + colormap=flow_colormap("dark"), + ) + assert np.array_equal(got, expected) + # A whole-domain boundary paints one tone; an empty frame mask means no arrows, + # so the picture stays that one tone rather than gaining arrow pixels. + tones = {tuple(int(v) for v in tone) for tone in np.unique(got.reshape(-1, 3), axis=0)} + assert tones == {(0x16, 0x13, 0x12)}, ( + f"arrows were drawn from a mask the frame does not claim: {tones}" + ) + + +def test_resetting_the_latch_clears_the_session_boundary(): + latch = backend.DomainMaskLatch() + latch.update(_island()) + latch.reset() + assert latch.mask is None + assert (latch.adoptions, latch.held) == (0, 0) + other = _shift(_island(), 5, 5) + assert np.array_equal(latch.update(other), other), ( + "a new session kept the previous session's island" + )