@@ -5,7 +5,7 @@ only what a replay draws: the flow samples the shared selector returns, not the
field they came from. The fields a frame produces are far too large to cache — the
measured cost is 1,176,000 B per frame (``flow`` 672,000 + ``depth_map`` 336,000 +
``valid_mask`` 84,000 + the gray ``img`` 84,000), which is 21.2 GB for ten minutes at
30 fps. The samples themselves cost 4,03 4 B per frame at the shipped finger grid.
30 fps. The samples themselves cost 4,048 B per frame at the shipped finger grid.
Three structural facts decide the layout, and each one is a measured result rather
than a guess:
@@ -28,6 +28,28 @@ served by re-deriving activity from magnitude; only a change to the sample *grid
(``flow_grid_spacing``, ``flow_edge_margin``, ``max_arrows``) moves the candidate
locations, and that policy is recorded in the provenance for comparison instead of
being baked into the key.
**Depth is stored, and it is stored quantised.** A frame ' s own ``depth_map`` is
336,000 B (300 ×
obvious encoding is out — and on the two real recordings it is empty far more often
than not: 241 of 264 frames of the eight-times clip and 32 of 33 frames of the box
recording carry a map that is *exactly* all zeros, with the worst non-empty frame at
336 of 84,000 pixels. So the record states the depth channel ' s own statistics every
frame (present, non-zero count, minimum, maximum) and appends a payload only for a
frame that has one, as uint8 over that frame ' s own range, zlib ' d at level 6. The
measured cost is 14 B per empty frame and 486 B for the worst non-empty frame seen;
the largest error that can be introduced is half a quantisation step,
``(max - min) / 510``, measured at 6.6e-5 on the clip whose peak is 0.034. That is
below half of one step of the 256-entry colour ramp the depth view paints with, so a
reconstructed frame cannot be coloured differently from the original.
Sparse storage was measured and rejected: 4-byte indices per non-zero pixel cost
910 B for the worst non-empty frame against 486 B for the whole quantised map, because
a mostly-zero byte plane is nearly free for zlib while pixel indices are not. Island
restriction was measured and rejected too — 418 B against 486 B, a saving of 5.2 B per
frame of a 4,048 B record — because it would make the reconstruction depend on the
validity-mask keyframe (the box chain has none) and would silently drop a depth value
that fell outside the island. See ``tests/test_analysis_depth.py`` for the numbers.
"""
from __future__ import annotations
@@ -35,6 +57,7 @@ from __future__ import annotations
import hashlib
import json
import struct
import zlib
from collections . abc import Mapping , Sequence
from dataclasses import dataclass , field
from typing import Any
@@ -50,8 +73,12 @@ __all__ = [
" CHUNK_HEADER " ,
" CHUNK_MAGIC " ,
" CHUNK_TRAILER " ,
" CODEC_EMPTY " ,
" CODEC_RAW " ,
" CODEC_ZLIB " ,
" DEPTH_BITS " ,
" DEPTH_HEADER " ,
" FRAME_FLAG_DEPTH " ,
" FRAME_FLAG_DROPPED " ,
" FRAME_FLAG_VALIDITY " ,
" FRAME_HEADER " ,
@@ -59,12 +86,16 @@ __all__ = [
" KEYFRAME_HEADER " ,
" KEYFRAME_MAGIC " ,
" KEYFRAME_MASK " ,
" MAX_DEPTH_PIXELS " ,
" UNIT " ,
" DecodedDepth " ,
" FramePayload " ,
" KeyframeRef " ,
" Provenance " ,
" ResultKey " ,
" decode_depth " ,
" decode_frame " ,
" encode_depth " ,
" encode_frame " ,
" frame_payload_bytes " ,
" keyframe_payload " ,
@@ -78,8 +109,9 @@ RESULT_DIRNAME = "analysis"
#: On-disk result format. Bump only for an incompatible layout change; a change to
#: what the *chain computes* belongs in :data:`ALGORITHM_VERSION` instead.
#: Version 2 adds the depth channel to the frame record (see the module docstring).
ANALYSIS_FORMAT = " orisys-replay-analysis "
ANALYSIS_FORMAT_VERSION = 1
ANALYSIS_FORMAT_VERSION = 2
#: Identity of the analysis the pass runs: the SDK calls it makes, in order.
#: ``compute_deformation(decompose=True, check_motion=False)`` then
@@ -92,11 +124,16 @@ ALGORITHM_VERSION = "flow-hhd-contact-1"
#: unit :class:`~orisys_app_shared.scene3d.visual_policy.FlowSamples` documents.
UNIT = " uv_px "
#: ``frame_id``, ``t_rel_s``, grid keyframe id, flags, candidate count, depth statistic.
#: The four trailing pad bytes ke ep the value arrays 8-byte aligned inside a record.
FRAME_HEADER = struct . Struct ( " <idHBHI3x " )
#: ``frame_id``, ``t_rel_s``, grid keyframe id, flags, candidate count, depth
#: non-zero count, depth minimum, d epth maximum. The three trailing pad bytes keep
#: the value arrays 8-byte aligned inside a record.
FRAME_HEADER = struct . Struct ( " <idHBHIff3x " )
FRAME_FLAG_VALIDITY = 0x01
FRAME_FLAG_DROPPED = 0x02
#: The chain exposed a depth map for this frame: the three depth statistics in the
#: header describe it and a depth payload follows. Absent flag means *no depth
#: channel at all*, which is a different fact from "the channel was all zeros".
FRAME_FLAG_DEPTH = 0x04
#: magic, kind, codec, count, height, width, flags, reserved, raw bytes, digest
KEYFRAME_HEADER = struct . Struct ( " <4sBBIIIBBQ8s " )
@@ -104,8 +141,22 @@ KEYFRAME_MAGIC = b"ORAK"
KEYFRAME_GRID = 0
KEYFRAME_MASK = 1
#: codec, bits per stored value, height, width. A depth payload is this header
#: followed by its stored bytes — none at all for :data:`CODEC_EMPTY`.
DEPTH_HEADER = struct . Struct ( " <BBHH " )
DEPTH_BITS = 8
CODEC_RAW = 0
CODEC_ZLIB = 1
#: The map is exactly all zeros: ``height ×
#: the common case by a wide margin, and saying so explicitly is what lets a reader
#: explain an empty depth view instead of drawing a blank one.
CODEC_EMPTY = 2
#: Refuse a depth header that asks for an implausible array before allocating it. The
#: shipped grid is 300 ×
#: a corrupt header to become a 4 GB allocation.
MAX_DEPTH_PIXELS = 1 << 26
#: magic, version, reserved, chunk index, frame count, header size
CHUNK_HEADER = struct . Struct ( " <4sHHIII " )
@@ -207,6 +258,188 @@ def recording_identity(
return canonical_digest8 ( payload ) . hex ( )
# ------------------------------------------------------------------- depth channel
def _depth_plane ( depth : Any ) - > np . ndarray :
""" A depth map as a 2-D float32 array, whatever leading singletons it arrives with.
The finger chain hands ``depth_map`` over as ``(1, H, W)`` — a batch axis nobody
batches — and the validity mask it belongs beside is ``(H, W)``. Squeezing only
leading singletons keeps a genuinely 3-D field (a future multi-plane depth) an
error rather than something silently reinterpreted.
"""
array = np . asarray ( depth , dtype = np . float32 )
while array . ndim > 2 and array . shape [ 0 ] == 1 :
array = array [ 0 ]
if array . ndim != 2 :
raise ValueError ( f " a depth map is 2-D, got shape { np . shape ( depth ) } " )
return np . ascontiguousarray ( array )
@dataclass ( frozen = True )
class DecodedDepth :
""" One frame ' s depth channel: how it was stored, and the statistics that describe it. """
codec : int
height : int
width : int
nonzero : int
minimum : float
maximum : float
#: The stored bytes below the header — empty for :data:`CODEC_EMPTY`, and the
#: quantised plane decompressed for the other codecs.
stored : bytes = b " "
#: How many bytes those were on disk, before decompression. Kept separate from
#: ``len(stored)`` because the compressed and the decoded length are different
#: numbers and only one of them is what the record costs.
stored_bytes : int = 0
@property
def pixels ( self ) - > int :
return int ( self . height ) * int ( self . width )
@property
def payload_bytes ( self ) - > int :
""" Every byte this frame ' s depth channel occupies, header included. """
return DEPTH_HEADER . size + int ( self . stored_bytes )
@property
def is_empty ( self ) - > bool :
return self . codec == CODEC_EMPTY
def _quantise ( depth : np . ndarray ) - > tuple [ np . ndarray , float , float ] :
""" uint8 codes over the array ' s own ``[min, max]``, plus that range.
The range is the frame ' s own rather than a session-wide one on purpose: a depth
response that is large on one frame and small on the next keeps its full 256 levels
either way. The price is that the range has to travel with the frame, which is why
it lives in the frame header (8 B) instead of in each payload.
"""
low = float ( depth . min ( ) )
high = float ( depth . max ( ) )
span = high - low
if span < = 0.0 :
return np . zeros ( depth . shape , dtype = np . uint8 ) , low , high
scaled = ( depth . astype ( np . float64 ) - low ) / span
codes = np . clip ( np . rint ( scaled * 255.0 ) , 0.0 , 255.0 ) . astype ( np . uint8 )
return codes , low , high
def _dequantise ( codes : np . ndarray , low : float , high : float ) - > np . ndarray :
span = float ( high ) - float ( low )
if span < = 0.0 :
return np . full ( codes . shape , float ( low ) , dtype = np . float32 )
return ( float ( low ) + codes . astype ( np . float64 ) * ( span / 255.0 ) ) . astype ( np . float32 )
def encode_depth ( depth : Any ) - > tuple [ bytes , int , float , float ] :
""" One frame ' s depth channel as ``(payload, nonzero, min, max)``.
The payload is a :data:`DEPTH_HEADER` and, unless the map is exactly all zeros, the
zlib ' d uint8 plane. An empty map stores the header alone: 6 bytes that state the
shape, which is what a reader needs to rebuild the zeros rather than guess at them.
"""
array = _depth_plane ( depth )
height , width = int ( array . shape [ 0 ] ) , int ( array . shape [ 1 ] )
if height * width > MAX_DEPTH_PIXELS :
raise ValueError ( f " a { height } x { width } depth map is larger than this format holds " )
nonzero = int ( np . count_nonzero ( array ) )
low = float ( array . min ( ) )
high = float ( array . max ( ) )
if nonzero == 0 :
return (
DEPTH_HEADER . pack ( CODEC_EMPTY , DEPTH_BITS , height , width ) ,
nonzero ,
low ,
high ,
)
codes , low , high = _quantise ( array )
raw = codes . tobytes ( )
stored = zlib . compress ( raw , 6 )
codec = CODEC_ZLIB
if len ( stored ) > = len ( raw ) :
codec = CODEC_RAW
stored = raw
return (
DEPTH_HEADER . pack ( codec , DEPTH_BITS , height , width ) + stored ,
nonzero ,
low ,
high ,
)
def decode_depth (
buffer : bytes | bytearray | memoryview ,
offset : int = 0 ,
* ,
low : float = 0.0 ,
high : float = 0.0 ,
) - > DecodedDepth :
""" Read one depth payload. Raises :class:`ValueError` on anything inconsistent.
The stored plane is checked against the shape its own header declares, so a
truncated or mis-sized payload is refused here rather than surfacing later as an
array of the wrong shape.
"""
view = memoryview ( buffer )
if len ( view ) < offset + DEPTH_HEADER . size :
raise ValueError ( " a depth payload is shorter than its header " )
codec , bits , height , width = DEPTH_HEADER . unpack_from ( view , offset )
cursor = offset + DEPTH_HEADER . size
pixels = int ( height ) * int ( width )
if pixels < = 0 or pixels > MAX_DEPTH_PIXELS :
raise ValueError ( f " a depth payload declares an implausible shape { height } x { width } " )
if int ( bits ) != DEPTH_BITS :
raise ValueError ( f " a depth payload stores { bits } bits per value, not { DEPTH_BITS } " )
stored = bytes ( view [ cursor : ] )
if int ( codec ) == CODEC_EMPTY :
return DecodedDepth (
codec = int ( codec ) ,
height = int ( height ) ,
width = int ( width ) ,
nonzero = 0 ,
minimum = 0.0 ,
maximum = 0.0 ,
)
if int ( codec ) not in ( CODEC_RAW , CODEC_ZLIB ) :
raise ValueError ( f " a depth payload uses codec { codec } " )
if int ( codec ) == CODEC_ZLIB :
try :
raw = zlib . decompress ( stored )
except zlib . error as exc :
raise ValueError ( f " a depth payload does not decompress: { exc } " ) from exc
else :
raw = stored
if len ( raw ) != pixels :
raise ValueError (
f " a depth payload decoded to { len ( raw ) } bytes for a { height } x { width } map "
)
return DecodedDepth (
codec = int ( codec ) ,
height = int ( height ) ,
width = int ( width ) ,
nonzero = pixels ,
minimum = float ( low ) ,
maximum = float ( high ) ,
stored = raw ,
stored_bytes = len ( stored ) ,
)
def depth_values ( decoded : DecodedDepth , * , low : float , high : float ) - > np . ndarray :
""" The depth map a decoded payload describes, as float32.
A :data:`CODEC_EMPTY` payload becomes ``height ×
chain produced, not a placeholder — so the depth view and its statistics describe
the same array.
"""
shape = ( int ( decoded . height ) , int ( decoded . width ) )
if decoded . codec == CODEC_EMPTY :
return np . zeros ( shape , dtype = np . float32 )
codes = np . frombuffer ( decoded . stored , dtype = np . uint8 ) . reshape ( shape )
return _dequantise ( codes , float ( low ) , float ( high ) )
# ------------------------------------------------------------------- frame encoding
def encode_frame (
* ,
@@ -218,15 +451,22 @@ def encode_frame(
magnitude : np . ndarray ,
active : np . ndarray ,
validity_digest : bytes | None = None ,
depth : Any | None = None ,
depth_nonzero : int = 0 ,
dropped : bool = False ,
) - > bytes :
""" One frame record: a fixed header, then the value arrays.
""" One frame record: a fixed header, then the value arrays and the depth payload .
Arrays are little-endian regardless of host order, so a result written on one
machine reads back identically on another. ``dx``/``dy``/``magnitude`` are stored
as float32 — the precision the display needs and the reason a stored payload is
compared to a live selection *cast to float32* rather than in float64.
``depth`` is the frame ' s own map, or ``None`` when the chain exposed none; the two
are different facts (``FRAME_FLAG_DEPTH``) and the header ' s statistics belong to the
first. ``depth_nonzero`` is only consulted when ``depth`` is ``None``, which is how
a caller that can count but cannot afford to hand the array over still records the
count exactly.
"""
dx = np . ascontiguousarray ( dx , dtype = " <f4 " ) . reshape ( - 1 )
dy = np . ascontiguousarray ( dy , dtype = " <f4 " ) . reshape ( - 1 )
@@ -247,6 +487,11 @@ def encode_frame(
flags | = FRAME_FLAG_VALIDITY
if dropped :
flags | = FRAME_FLAG_DROPPED
depth_payload = b " "
depth_low = depth_high = 0.0
if depth is not None :
flags | = FRAME_FLAG_DEPTH
depth_payload , depth_nonzero , depth_low , depth_high = encode_depth ( depth )
parts = [
FRAME_HEADER . pack (
int ( frame_id ) ,
@@ -255,6 +500,8 @@ def encode_frame(
flags ,
count ,
max ( 0 , int ( depth_nonzero ) ) ,
float ( depth_low ) ,
float ( depth_high ) ,
) ,
dx . tobytes ( ) ,
dy . tobytes ( ) ,
@@ -266,6 +513,7 @@ def encode_frame(
if len ( digest ) != 8 :
raise ValueError ( f " validity digest must be 8 bytes, got { len ( digest ) } " )
parts . append ( digest )
parts . append ( depth_payload )
return b " " . join ( parts )
@@ -284,6 +532,17 @@ class DecodedFrame:
magnitude : np . ndarray
active : np . ndarray
validity_digest : bytes | None
#: ``None`` when the frame carried no depth channel; otherwise the payload, whose
#: ``stored`` bytes are still quantised and need :func:`depth_values`.
depth : DecodedDepth | None = None
def depth_map ( self ) - > np . ndarray | None :
""" This frame ' s depth map, or ``None`` when it had no depth channel. """
if self . depth is None :
return None
return depth_values (
self . depth , low = self . depth . minimum , high = self . depth . maximum
)
def decode_frame (
@@ -291,9 +550,16 @@ def decode_frame(
) - > DecodedFrame :
""" Decode one frame record starting at ``offset``. Ignores any trailing bytes. """
view = memoryview ( buffer )
frame_id , t_rel , grid_id , flags , count , depth_nonzero = FRAME_HEADER . unpack_from (
view , offset
)
(
frame_id ,
t_rel ,
grid_id ,
flags ,
count ,
depth_nonzero ,
depth_low ,
depth_high ,
) = FRAME_HEADER . unpack_from ( view , offset )
cursor = offset + FRAME_HEADER . size
arrays = [ ]
for _ in range ( 3 ) :
@@ -308,6 +574,12 @@ def decode_frame(
validity = None
if flags & FRAME_FLAG_VALIDITY :
validity = bytes ( view [ cursor : cursor + 8 ] )
cursor + = 8
depth = None
if flags & FRAME_FLAG_DEPTH :
depth = decode_depth (
view , cursor , low = float ( depth_low ) , high = float ( depth_high )
)
return DecodedFrame (
frame_id = int ( frame_id ) ,
t_rel_s = float ( t_rel ) ,
@@ -320,13 +592,29 @@ def decode_frame(
magnitude = arrays [ 2 ] ,
active = active . astype ( bool ) ,
validity_digest = validity ,
depth = depth ,
)
def frame_payload_bytes ( candidates : int , * , with_validity : bool = True ) - > int :
""" Stored bytes of one frame record at ``candidates`` sample locations. """
def frame_payload_bytes (
candidates : int ,
* ,
with_validity : bool = True ,
with_depth : bool = False ,
depth_stored : int = 0 ,
) - > int :
""" Stored bytes of one frame record at ``candidates`` sample locations.
``with_depth`` adds the 6-byte depth header every frame that carries a channel pays;
``depth_stored`` adds a payload of that many bytes on top, for a caller sizing the
memory bound rather than the common case.
"""
total = FRAME_HEADER . size + int ( candidates ) * 12 + ( int ( candidates ) + 7 ) / / 8
return total + 8 if with_validity else total
if with_validity :
total + = 8
if with_depth :
total + = DEPTH_HEADER . size + max ( 0 , int ( depth_stored ) )
return total
def keyframe_payload ( kind : int , array : np . ndarray ) - > bytes :
@@ -373,6 +661,30 @@ class FramePayload:
validity_digest : bytes | None = None
depth_nonzero : int = 0
dropped : bool = False
#: This frame's depth map at the chain's own resolution, or ``None`` when the frame
#: carried no depth channel. An all-zero map is *not* ``None``: "the response was
#: zero everywhere" and "there was no response" are different answers and a depth
#: view that shows the same blank picture for both is the one that misleads.
depth : np . ndarray | None = None
depth_min : float = 0.0
depth_max : float = 0.0
#: Stored bytes of this frame's depth payload — the 6-byte header for an empty map,
#: header plus plane otherwise, and 0 when the frame had no depth channel. It is
#: what a caller adds to the fixed part of a record to say what depth cost.
depth_stored_bytes : int = 0
@property
def has_depth ( self ) - > bool :
return self . depth is not None
@property
def depth_is_empty ( self ) - > bool :
""" True when a depth channel exists and is exactly zero everywhere. """
return self . depth is not None and self . depth_nonzero == 0
@property
def depth_pixels ( self ) - > int :
return 0 if self . depth is None else int ( self . depth . size )
@property
def candidates ( self ) - > int :
@@ -412,6 +724,7 @@ class FramePayload:
magnitude = self . magnitude ,
active = self . active ,
validity_digest = self . validity_digest ,
depth = self . depth ,
depth_nonzero = self . depth_nonzero ,
dropped = self . dropped ,
)
ykcheungab