orisys-tech/orisys-app
3
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fixed display problem

Browse Source
This commit is contained in:
ykcheungab
2026-09-25 17:53:07 +08:00
parent 12781854ca
commit 3daee6d407
7 changed files with 367 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
shared/src/orisys_app_shared/acquisition/arrows.py
+9 -3
View File
@@ -199,6 +199,7 @@ def draw_arrow_display(
canvas_bgr: Sequence[int] | None = None,
draw_style: str = "box",
valid_mask: Optional[np.ndarray] = None,
domain_mask: Optional[np.ndarray] = None,
colormap: Optional[np.ndarray] = None,
exterior_bgr: Sequence[int] | None = None,
island_bgr: Sequence[int] | None = None,
@@ -214,6 +215,11 @@ def draw_arrow_display(
inside, slightly darker outside). Without a mask the whole rectangle uses
``canvas_bgr`` / ``island_bgr`` for backward compatibility.
``domain_mask`` is the boundary to paint that canvas from, when the caller holds a
latched one; ``None`` means this frame's own ``valid_mask``. It never reaches
:func:`select_flow_samples`: the arrows stay this frame's own measurement, so
latching the boundary cannot put an arrow on a sample the device does not vouch for.
Finger style uses the shared :mod:`visual_policy` sampler so 2D and 3D arrows
share the same UV sample points and thresholds. Box style keeps the SDK path.
"""
@@ -233,14 +239,15 @@ def draw_arrow_display(
if exterior_bgr is None
else tuple(int(v) for v in exterior_bgr)
)
if valid_mask is not None:
boundary = valid_mask if domain_mask is None else domain_mask
if boundary is not None:
from orisys_app_shared.scene3d.visual_policy import (
build_domain_mask,
paint_domain_background,
)
domain, _ = build_domain_mask(
valid_mask, shape_hw=base.shape[:2]
boundary, shape_hw=base.shape[:2]
)
canvas = paint_domain_background(
base.shape[:2],
@@ -248,7 +255,6 @@ def draw_arrow_display(
island_bgr=inside, # type: ignore[arg-type]
exterior_bgr=outside, # type: ignore[arg-type]
)
valid_mask = domain
else:
canvas = np.empty_like(base)
canvas[:, :] = inside
shared/src/orisys_app_shared/acquisition/image_backend.py
+99 -1
View File
@@ -61,6 +61,94 @@ MISSING_NOTE_KEY = "panels.frame_missing_note"
#: Sources whose frames carry derived fields.
DERIVED_SOURCES = (SOURCE_LIVE, SOURCE_RECOMPUTE)
#: How much a frame's domain mask must overlap the latched one before the latched one
#: is kept. Measured on the shipped finger island (32937 px, 39.2% of the 300x280
#: domain): a rebuilt mask that wobbles by one pixel of rim overlap the held one at
#: IoU >= 0.9633, a 200-pixel speckle at 0.9879, while an island that really moved
#: 6 px falls to 0.9102. 0.95 therefore absorbs the wobble -- which is what repaints
#: the edge, 69% of the boundary band for a 0.73% area flip -- and still adopts a real
#: move. A per-pixel frame-count hysteresis was rejected: the measured flips are spread
#: around the whole rim rather than persistent at any pixel, so it would need a long
#: count to suppress them and would then delay a genuine move by that many frames.
DOMAIN_MASK_MIN_IOU = 0.95
def mask_overlap(left: np.ndarray, right: np.ndarray) -> float:
"""Intersection over union of two equally shaped boolean masks.
Both empty counts as total agreement: two frames that agree there is no island at
all are not a change.
"""
a = np.asarray(left, dtype=bool)
b = np.asarray(right, dtype=bool)
union = int(np.count_nonzero(a | b))
if union == 0:
return 1.0
return float(np.count_nonzero(a & b)) / float(union)
class DomainMaskLatch:
"""The 2D domain boundary, kept until it stops describing the same island.
``build_domain_mask``/``paint_domain_background`` rebuild the island/exterior canvas
from ``ImageFrame.valid_mask`` on every frame, and the depth view paints that same
boundary as its "no depth" grey. The mask is the device's own UV island: geometry,
rebuilt from the same remap every frame, and the divergence map the depth view reads
is zero-filled everywhere outside it. So the edge being redrawn is the same edge,
and any difference in the mask repaints the whole boundary, which reads as the
picture lagging rather than as the boundary moving.
Latching it is the same treatment the depth *colour scale* already gets
(``StudioMainWindow._depth_range``): one decision, kept for the session, so frames
stay comparable. The latch compares against the mask it is holding — not against the
previous frame — so a slow drift accumulates and is adopted once it is material,
while a per-frame flicker around the same island never is.
``adoptions`` and ``held`` make the behaviour observable: a session that keeps
re-latching a moving boundary and a session that correctly holds a static one look
the same on screen, and only these counters tell them apart.
The array is held, not copied. A producer that hands a mask over must not write into
it again — the same contract ``scene3d.contracts.freeze`` enforces for every other
array a frame carries, which is what makes holding it safe.
"""
__slots__ = ("_mask", "min_iou", "adoptions", "held")
def __init__(self, *, min_iou: float = DOMAIN_MASK_MIN_IOU) -> None:
self._mask: Optional[np.ndarray] = None
self.min_iou = float(min_iou)
self.adoptions = 0
self.held = 0
@property
def mask(self) -> Optional[np.ndarray]:
return self._mask
def reset(self) -> None:
"""Drop the latched boundary: a new session has not established one yet."""
self._mask = None
self.adoptions = 0
self.held = 0
def update(self, mask: Optional[np.ndarray]) -> Optional[np.ndarray]:
"""The mask to draw the boundary with, given this frame's own mask.
A frame that carries no mask cannot move the boundary, so it inherits the
session's: the island belongs to the sensor, not to the frame that happens not
to state it (a preview frame, a frame from a source that never had one).
"""
if mask is None:
return self._mask
current = np.asarray(mask, dtype=bool)
if self._mask is not None and current.shape == self._mask.shape:
if mask_overlap(self._mask, current) >= self.min_iou:
self.held += 1
return self._mask
self._mask = current
self.adoptions += 1
return self._mask
@dataclass(frozen=True)
class ImageFrame:
@@ -196,15 +284,24 @@ def render_image(
draw_style: str = "box",
depth_range: Optional[tuple[float, float]] = None,
canvas_bgr: Optional[Sequence[int]] = None,
domain_mask: Optional[np.ndarray] = None,
) -> Optional[np.ndarray]:
"""Draw ``frame`` in ``mode``; ``None`` when there is nothing to draw.
The mode is resolved through the plan first, so a caller that asks for a view the
frame cannot support gets the plan's default rather than a picture assembled from
an array that is not there.
``domain_mask`` is the *latched* boundary to paint the island/exterior canvas and
the depth view's "no depth" grey from (see :class:`DomainMaskLatch`); ``None`` means
"use this frame's own ``valid_mask``", which is what a caller with no session to
latch to should do. It never gates the arrows or the depth values: those stay this
frame's own data, so a sample the device does not vouch for is still not drawn just
because the boundary around it was latched.
"""
plan = plan or plan_image(frame)
wanted = plan.resolve(mode)
boundary = frame.valid_mask if domain_mask is None else domain_mask
if wanted == DEPTH:
# Never substitute another picture for a missing depth map: that made "no
# depth" look like a depth reading. The panel says so instead.
@@ -214,7 +311,7 @@ def render_image(
frame.depth,
flow_colormap(theme),
value_range=depth_range,
invalid_mask=frame.valid_mask,
invalid_mask=boundary,
)
if frame.image_bgr is None:
return None
@@ -229,6 +326,7 @@ def render_image(
canvas_bgr=canvas_bgr,
draw_style=draw_style,
valid_mask=frame.valid_mask,
domain_mask=domain_mask,
colormap=flow_colormap(theme),
)
# ``recorded``: the frame is the picture, with nothing drawn over it.
shared/tests/test_image_backend.py
+169
View File
@@ -191,3 +191,172 @@ def test_a_missing_field_is_reported_not_invented():
base = backend.render_image(no_flow, mode="arrows_pure")
assert base is not None
assert np.array_equal(base, _image()), "the base picture must not be redrawn"
# ------------------------------------------------------- the latched boundary
#: A domain the same size and the same kind of shape as the shipped finger sensor's
#: 300x280 UV island: one large blob with a long smooth rim, so "a one-pixel wobble"
#: and "the island moved" have the overlap the real package has (measured on the
#: shipped island: a 1 px wobble scores IoU 0.9633, a 6 px move 0.9102).
DOMAIN_HW = (300, 280)
ISLAND_RADIUS_PX = 95
def _island(h: int = DOMAIN_HW[0], w: int = DOMAIN_HW[1]) -> np.ndarray:
yy, xx = np.mgrid[0:h, 0:w]
return ((yy - h / 2.0) ** 2 + (xx - w / 2.0) ** 2) <= float(ISLAND_RADIUS_PX) ** 2
def _blank_image(h: int = DOMAIN_HW[0], w: int = DOMAIN_HW[1]) -> np.ndarray:
"""A base picture the two views do not read: only depth and the mask are drawn."""
return np.zeros((h, w, 3), dtype=np.uint8)
def _shift(mask: np.ndarray, dy: int, dx: int) -> np.ndarray:
out = np.zeros_like(mask)
h, w = mask.shape
ys = slice(max(0, dy), h + min(0, dy))
xs = slice(max(0, dx), w + min(0, dx))
ys2 = slice(max(0, -dy), h + min(0, -dy))
xs2 = slice(max(0, -dx), w + min(0, -dx))
out[ys, xs] = mask[ys2, xs2]
return out
def test_overlap_of_two_identical_masks_is_one():
mask = _island()
assert backend.mask_overlap(mask, mask) == 1.0
assert backend.mask_overlap(mask, ~mask) == 0.0
# Two frames that agree there is no island at all are not a change.
empty = np.zeros((4, 4), dtype=bool)
assert backend.mask_overlap(empty, empty) == 1.0
def test_the_gate_separates_a_rebuilt_mask_from_a_moved_one():
"""The threshold itself, on the two cases it exists to tell apart."""
island = _island()
for dy, dx in ((1, 0), (0, 1), (1, 1), (-1, 1), (2, 0)):
assert backend.mask_overlap(island, _shift(island, dy, dx)) >= (
backend.DOMAIN_MASK_MIN_IOU
), f"a {dy},{dx} rebuild would be taken for a moved boundary"
assert backend.mask_overlap(island, _shift(island, 6, 0)) < backend.DOMAIN_MASK_MIN_IOU
def test_a_stable_domain_mask_is_latched_once_for_the_session():
"""The device's island is geometry: rebuilt per frame, decided once."""
latch = backend.DomainMaskLatch()
mask = _island()
seen = {latch.update(mask).tobytes() for _ in range(20)}
assert len(seen) == 1, "a stable mask was rebuilt into a different array"
assert np.array_equal(latch.mask, mask)
assert (latch.adoptions, latch.held) == (1, 19)
def test_a_wobbling_domain_mask_does_not_repaint_the_boundary():
"""A one-pixel rim wobble is the mask being rebuilt, not the island moving.
The measured case: on the shipped finger island this is a 0.73% area flip that
repaints 69% of the boundary band and 35% of the rendered edge pixels.
"""
latch = backend.DomainMaskLatch()
image = _blank_image()
# A zero field, so nothing but the boundary can change the picture.
flow = np.zeros((*DOMAIN_HW, 2), dtype=np.float32)
depth = np.ones(DOMAIN_HW, dtype=np.float32)
wobbles = [(0, 0), (1, 0), (0, -1), (1, 1), (-1, 0), (0, 1), (-1, -1)]
arrows, maps = [], []
for dy, dx in wobbles:
mask = _shift(_island(), dy, dx)
frame = backend.ImageFrame(image, flow=flow, depth=depth, valid_mask=mask)
used = latch.update(mask)
arrows.append(
backend.render_image(
frame, mode="arrows_pure", draw_style="finger", domain_mask=used
)
)
maps.append(
backend.render_image(
frame, mode="depth", depth_range=(0.0, 1.0), domain_mask=used
)
)
assert all(np.array_equal(arrows[0], other) for other in arrows), (
"the arrow canvas repainted a boundary that had not moved"
)
assert all(np.array_equal(maps[0], other) for other in maps), (
"the depth view's no-data boundary repainted a boundary that had not moved"
)
assert latch.adoptions == 1, "a rim wobble was taken for a moved boundary"
def test_a_real_move_of_the_domain_is_followed():
"""The latch must not freeze a boundary that actually moved."""
latch = backend.DomainMaskLatch()
island = _island()
latch.update(island)
moved = _shift(island, 6, 0)
used = latch.update(moved)
assert np.array_equal(used, moved), "a moved island kept the old boundary"
assert latch.adoptions == 2
frame = backend.ImageFrame(
_blank_image(), flow=np.zeros((*DOMAIN_HW, 2), np.float32)
)
before = backend.render_image(
frame, mode="arrows_pure", draw_style="finger", domain_mask=island
)
after = backend.render_image(
frame, mode="arrows_pure", draw_style="finger", domain_mask=used
)
assert not np.array_equal(before, after)
def test_a_frame_without_a_mask_cannot_move_the_boundary():
latch = backend.DomainMaskLatch()
assert latch.update(None) is None, "a session with no mask invented one"
island = _island()
latch.update(island)
assert np.array_equal(latch.update(None), island), (
"a frame that states no mask dropped the session's boundary"
)
assert (latch.adoptions, latch.held) == (1, 0)
def test_the_latched_boundary_never_gates_the_arrows():
"""Latching the boundary must not put an arrow on an unvouched-for sample."""
image = _blank_image()
flow = np.zeros((*DOMAIN_HW, 2), dtype=np.float32)
flow[::10, ::10, 0] = 3.0
# The frame claims nothing; the session's boundary is the whole domain.
frame = backend.ImageFrame(image, flow=flow, valid_mask=np.zeros(DOMAIN_HW, bool))
boundary = np.ones(DOMAIN_HW, dtype=bool)
got = backend.render_image(
frame, mode="arrows_pure", draw_style="finger", domain_mask=boundary
)
expected = draw_arrow_display(
image,
flow,
background="pure_color",
draw_style="finger",
valid_mask=frame.valid_mask,
domain_mask=boundary,
colormap=flow_colormap("dark"),
)
assert np.array_equal(got, expected)
# A whole-domain boundary paints one tone; an empty frame mask means no arrows,
# so the picture stays that one tone rather than gaining arrow pixels.
tones = {tuple(int(v) for v in tone) for tone in np.unique(got.reshape(-1, 3), axis=0)}
assert tones == {(0x16, 0x13, 0x12)}, (
f"arrows were drawn from a mask the frame does not claim: {tones}"
)
def test_resetting_the_latch_clears_the_session_boundary():
latch = backend.DomainMaskLatch()
latch.update(_island())
latch.reset()
assert latch.mask is None
assert (latch.adoptions, latch.held) == (0, 0)
other = _shift(_island(), 5, 5)
assert np.array_equal(latch.update(other), other), (
"a new session kept the previous session's island"
)