ykcheungab
720 lines
28 KiB
Python
720 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Real-time UV-based perspective correction for the exported finger mesh.
|
|
|
|
This version replaces the previous cylindrical approximation with a remap built
|
|
from the mesh UV parameterization. The remap is precomputed once and then
|
|
applied to each live frame with cv2.remap().
|
|
"""
|
|
|
|
import argparse
|
|
import sys
|
|
import time
|
|
|
|
import cv2
|
|
import numpy as np
|
|
import orisys
|
|
from orisys.configs import load_config
|
|
from orisys.frame_processors import UVUnrollFrameProcessor
|
|
from orisys.unroll import (
|
|
barycentric_grid,
|
|
build_or_load_domain,
|
|
compute_uv_distortion_metrics,
|
|
face_uvs_from_loop_data,
|
|
load_calib_json,
|
|
load_exported_data,
|
|
recommend_edge_expand,
|
|
resolve_uv_runtime_sizes,
|
|
scale_camera_geometry,
|
|
scaled_map_cache_path,
|
|
uv_triangle_to_image,
|
|
)
|
|
|
|
|
|
class StepTimer:
|
|
"""Accumulate per-step wall times for rolling runtime averages."""
|
|
|
|
def __init__(self):
|
|
self._starts = {}
|
|
self.totals = {}
|
|
self.counts = {}
|
|
|
|
def start(self, name):
|
|
self._starts[name] = time.perf_counter()
|
|
|
|
def stop(self, name):
|
|
dt = time.perf_counter() - self._starts.pop(name)
|
|
self.totals[name] = self.totals.get(name, 0.0) + dt
|
|
self.counts[name] = self.counts.get(name, 0) + 1
|
|
return dt
|
|
|
|
def avg_ms(self, name):
|
|
if self.counts.get(name, 0) == 0:
|
|
return 0.0
|
|
return 1000.0 * self.totals[name] / self.counts[name]
|
|
|
|
|
|
def benchmark_step(label, func, *args, **kwargs):
|
|
"""Run func once and print its wall time."""
|
|
t0 = time.perf_counter()
|
|
result = func(*args, **kwargs)
|
|
ms = (time.perf_counter() - t0) * 1000.0
|
|
print(f"[benchmark] {label}: {ms:.1f} ms")
|
|
return result
|
|
|
|
|
|
def print_runtime_benchmark(timer):
|
|
"""Format rolling average timings for the live loop."""
|
|
parts = [
|
|
f"capture={timer.avg_ms('capture'):.1f}ms",
|
|
f"uv_prep={timer.avg_ms('uv_prepare'):.1f}ms",
|
|
f"uv_remap={timer.avg_ms('uv_remap'):.1f}ms",
|
|
f"get_img={timer.avg_ms('get_img'):.1f}ms",
|
|
f"flow={timer.avg_ms('flow'):.1f}ms",
|
|
f"hhd={timer.avg_ms('hhd'):.1f}ms",
|
|
f"contact={timer.avg_ms('contact'):.1f}ms",
|
|
f"viz={timer.avg_ms('viz'):.1f}ms",
|
|
]
|
|
return "bench[" + ", ".join(parts) + "]"
|
|
|
|
|
|
def resize_to_height(image, target_height):
|
|
"""Resize an image to a target height while preserving aspect ratio."""
|
|
if image.shape[0] == target_height:
|
|
return image
|
|
scale = target_height / float(image.shape[0])
|
|
target_width = max(1, int(round(image.shape[1] * scale)))
|
|
return cv2.resize(image, (target_width, target_height), interpolation=cv2.INTER_AREA)
|
|
|
|
|
|
def resize_to_fit(image, target_width, target_height):
|
|
"""Resize with aspect ratio preserved and pad to exact tile size."""
|
|
h, w = image.shape[:2]
|
|
if h <= 0 or w <= 0:
|
|
return np.zeros((target_height, target_width, 3), dtype=np.uint8)
|
|
|
|
scale = min(target_width / float(w), target_height / float(h))
|
|
new_w = max(1, int(round(w * scale)))
|
|
new_h = max(1, int(round(h * scale)))
|
|
resized = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_AREA)
|
|
|
|
if resized.ndim == 2:
|
|
resized = cv2.cvtColor(resized, cv2.COLOR_GRAY2BGR)
|
|
elif resized.ndim == 3 and resized.shape[2] == 1:
|
|
resized = cv2.cvtColor(resized[:, :, 0], cv2.COLOR_GRAY2BGR)
|
|
|
|
canvas = np.zeros((target_height, target_width, 3), dtype=np.uint8)
|
|
y0 = (target_height - new_h) // 2
|
|
x0 = (target_width - new_w) // 2
|
|
canvas[y0:y0 + new_h, x0:x0 + new_w] = resized
|
|
return canvas
|
|
|
|
|
|
def add_panel_title(image, title):
|
|
"""Overlay a simple title onto a display tile."""
|
|
out = image.copy()
|
|
cv2.rectangle(out, (0, 0), (out.shape[1], 36), (0, 0, 0), thickness=-1)
|
|
cv2.putText(out, title, (12, 24), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2, cv2.LINE_AA)
|
|
return out
|
|
|
|
|
|
def apply_display_mask(image, valid_mask, border_value=0):
|
|
"""Zero invalid UV pixels in visualization panels."""
|
|
if image is None or valid_mask is None:
|
|
return image
|
|
out = image.copy()
|
|
mask = valid_mask.astype(np.uint8)
|
|
if mask.shape != out.shape[:2]:
|
|
mask = cv2.resize(mask, (out.shape[1], out.shape[0]), interpolation=cv2.INTER_NEAREST)
|
|
invalid = ~mask.astype(bool)
|
|
out[invalid] = border_value
|
|
return out
|
|
|
|
|
|
def contact_inside_mask(centroid, valid_mask):
|
|
if centroid is None or valid_mask is None:
|
|
return None
|
|
if len(centroid) < 2:
|
|
return None
|
|
cx, cy = int(centroid[0]), int(centroid[1])
|
|
if cy < 0 or cy >= valid_mask.shape[0] or cx < 0 or cx >= valid_mask.shape[1]:
|
|
return False
|
|
return bool(valid_mask[cy, cx])
|
|
|
|
|
|
def overlay_runtime_status(image, sensor, centroid=None):
|
|
"""Annotate a panel with mask-aware runtime diagnostics."""
|
|
out = image.copy()
|
|
stats = getattr(sensor, "mask_stats", {}) or {}
|
|
coverage = 100.0 * float(stats.get("coverage", 0.0))
|
|
flow_valid = float(stats.get("flow_mean_valid", 0.0))
|
|
flow_invalid = float(stats.get("flow_mean_invalid", 0.0))
|
|
centroid_ok = contact_inside_mask(centroid, getattr(sensor, "valid_mask", None))
|
|
centroid_label = "n/a" if centroid_ok is None else ("inside" if centroid_ok else "outside")
|
|
lines = [
|
|
f"mask {coverage:.1f}%",
|
|
f"flow valid {flow_valid:.3f}",
|
|
f"flow invalid {flow_invalid:.3f}",
|
|
f"contact {centroid_label}",
|
|
]
|
|
y = 54
|
|
for line in lines:
|
|
cv2.putText(out, line, (12, y), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (255, 255, 255), 2, cv2.LINE_AA)
|
|
y += 24
|
|
return out
|
|
|
|
|
|
def make_tiled_dashboard(panels, tile_width=640, tile_height=360):
|
|
"""Arrange four panels into a 2x2 dashboard."""
|
|
tiles = [
|
|
add_panel_title(resize_to_fit(image, tile_width, tile_height), title)
|
|
for title, image in panels
|
|
]
|
|
top_row = np.hstack(tiles[:2])
|
|
bottom_row = np.hstack(tiles[2:4])
|
|
return np.vstack([top_row, bottom_row])
|
|
|
|
|
|
def make_uv_debug_image(data, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0):
|
|
"""Create a UV-domain reference image to verify orientation of the remap."""
|
|
face_uvs = face_uvs_from_loop_data(data["faces"], data["uv_data"], data["loop_to_vertex"])
|
|
uv_img = np.zeros((out_height, out_width, 3), dtype=np.uint8)
|
|
label_color = (255, 255, 255)
|
|
|
|
for uv_tri in face_uvs:
|
|
tri = uv_triangle_to_image(
|
|
uv_tri,
|
|
out_width,
|
|
out_height,
|
|
flip_v=flip_v,
|
|
edge_band=edge_band,
|
|
edge_expand=edge_expand,
|
|
)
|
|
pts = np.round(tri).astype(np.int32)
|
|
|
|
mask = np.zeros((out_height, out_width), dtype=np.uint8)
|
|
cv2.fillConvexPoly(mask, pts, 255)
|
|
|
|
x0 = max(0, int(np.floor(np.min(tri[:, 0]))))
|
|
x1 = min(out_width - 1, int(np.ceil(np.max(tri[:, 0]))))
|
|
y0 = max(0, int(np.floor(np.min(tri[:, 1]))))
|
|
y1 = min(out_height - 1, int(np.ceil(np.max(tri[:, 1]))))
|
|
if x1 < x0 or y1 < y0:
|
|
continue
|
|
|
|
yy, xx = np.mgrid[y0:y1 + 1, x0:x1 + 1]
|
|
inside = mask[y0:y1 + 1, x0:x1 + 1] > 0
|
|
if not np.any(inside):
|
|
continue
|
|
|
|
bary = barycentric_grid(xx + 0.5, yy + 0.5, tri)
|
|
if bary is None:
|
|
continue
|
|
w0, w1, w2 = bary
|
|
valid = inside & (w0 >= -1e-6) & (w1 >= -1e-6) & (w2 >= -1e-6)
|
|
if not np.any(valid):
|
|
continue
|
|
|
|
uv = (
|
|
w0[valid, None] * uv_tri[0]
|
|
+ w1[valid, None] * uv_tri[1]
|
|
+ w2[valid, None] * uv_tri[2]
|
|
)
|
|
colors = np.zeros((uv.shape[0], 3), dtype=np.uint8)
|
|
colors[:, 0] = np.clip(uv[:, 0] * 255.0, 0, 255).astype(np.uint8)
|
|
colors[:, 1] = np.clip(uv[:, 1] * 255.0, 0, 255).astype(np.uint8)
|
|
colors[:, 2] = np.clip((1.0 - 0.5 * (uv[:, 0] + uv[:, 1])) * 255.0, 0, 255).astype(np.uint8)
|
|
uv_img[yy[valid], xx[valid]] = colors
|
|
|
|
cv2.polylines(uv_img, [pts.reshape(-1, 1, 2)], isClosed=True, color=(255, 255, 255), thickness=1)
|
|
|
|
cv2.arrowedLine(uv_img, (20, out_height - 20), (min(out_width - 20, 140), out_height - 20), (0, 0, 255), 2, tipLength=0.15)
|
|
cv2.arrowedLine(uv_img, (20, out_height - 20), (20, max(20, out_height - 140)), (0, 255, 0), 2, tipLength=0.15)
|
|
cv2.putText(uv_img, "U", (min(out_width - 30, 145), out_height - 28), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2, cv2.LINE_AA)
|
|
cv2.putText(uv_img, "V", (28, max(30, out_height - 145)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2, cv2.LINE_AA)
|
|
cv2.putText(uv_img, "UV Map", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.9, label_color, 2, cv2.LINE_AA)
|
|
if edge_expand > 1.0:
|
|
cv2.putText(uv_img, f"Edge expand {edge_expand:.2f}x", (20, 64), cv2.FONT_HERSHEY_SIMPLEX, 0.55, label_color, 2, cv2.LINE_AA)
|
|
return uv_img
|
|
|
|
|
|
def make_uv_distortion_image(data, distortion, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0):
|
|
"""Render UV distortion heatmap; red means more 3D surface per UV area."""
|
|
face_uvs = face_uvs_from_loop_data(data["faces"], data["uv_data"], data["loop_to_vertex"])
|
|
uv_img = np.zeros((out_height, out_width, 3), dtype=np.uint8)
|
|
rel = distortion["relative_scale"]
|
|
edge_mask = distortion["edge_mask"]
|
|
valid_rel = np.isfinite(rel) & (rel > 0)
|
|
if not np.any(valid_rel):
|
|
return uv_img
|
|
|
|
log_rel = np.log2(rel[valid_rel])
|
|
clip = max(0.25, float(np.percentile(np.abs(log_rel), 95)))
|
|
|
|
for idx, uv_tri in enumerate(face_uvs):
|
|
tri = uv_triangle_to_image(
|
|
uv_tri,
|
|
out_width,
|
|
out_height,
|
|
flip_v=flip_v,
|
|
edge_band=edge_band,
|
|
edge_expand=edge_expand,
|
|
)
|
|
pts = np.round(tri).astype(np.int32)
|
|
|
|
if not np.isfinite(rel[idx]) or rel[idx] <= 0:
|
|
color = (64, 64, 64)
|
|
else:
|
|
value = np.clip((np.log2(rel[idx]) + clip) / (2.0 * clip), 0.0, 1.0)
|
|
color_idx = np.uint8(round(value * 255.0))
|
|
color = tuple(int(c) for c in cv2.applyColorMap(np.array([[color_idx]], dtype=np.uint8), cv2.COLORMAP_JET)[0, 0])
|
|
|
|
cv2.fillConvexPoly(uv_img, pts, color)
|
|
edge_color = (255, 255, 255) if edge_mask[idx] else (180, 180, 180)
|
|
edge_thickness = 2 if edge_mask[idx] else 1
|
|
cv2.polylines(uv_img, [pts.reshape(-1, 1, 2)], isClosed=True, color=edge_color, thickness=edge_thickness)
|
|
|
|
cv2.putText(uv_img, "UV Distortion", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 255), 2, cv2.LINE_AA)
|
|
cv2.putText(uv_img, "Blue: denser UV Red: sparser UV", (20, 64), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (220, 220, 220), 2, cv2.LINE_AA)
|
|
cv2.putText(uv_img, "White outlines: edge-band faces", (20, 92), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (220, 220, 220), 2, cv2.LINE_AA)
|
|
middle_stats = distortion["middle_stats"]
|
|
edge_stats = distortion["edge_stats"]
|
|
cv2.putText(
|
|
uv_img,
|
|
f"middle p05/p95: {middle_stats['relative_scale_p05']:.2f}x / {middle_stats['relative_scale_p95']:.2f}x",
|
|
(20, 120),
|
|
cv2.FONT_HERSHEY_SIMPLEX,
|
|
0.55,
|
|
(220, 220, 220),
|
|
2,
|
|
cv2.LINE_AA,
|
|
)
|
|
cv2.putText(
|
|
uv_img,
|
|
f"edge p05/p95: {edge_stats['relative_scale_p05']:.2f}x / {edge_stats['relative_scale_p95']:.2f}x",
|
|
(20, 148),
|
|
cv2.FONT_HERSHEY_SIMPLEX,
|
|
0.55,
|
|
(220, 220, 220),
|
|
2,
|
|
cv2.LINE_AA,
|
|
)
|
|
return uv_img
|
|
|
|
|
|
def open_source(video_path, camera_id):
|
|
return cv2.VideoCapture(video_path) if video_path else cv2.VideoCapture(camera_id)
|
|
|
|
|
|
PERF_TUNING_EPILOG = """
|
|
Performance tuning (config JSON + CLI):
|
|
camera.resolution_stitch [280, 300] UV output + SDK flow/HHD grid
|
|
camera.source_scale 0.4 shrink source before uv_prep/remap
|
|
tracker.dis_preset medium -> fast faster DIS optical flow
|
|
tracker.hhd_downsample 5 -> 4 smaller HHD Poisson grid
|
|
contact.denoise_sigma 12 -> 6 or 0 faster FoE contact step
|
|
--source_scale 0.4 override config source scale
|
|
--masked_poisson_solver sparse vs capfft benchmark HHD solve time
|
|
--no-viz skip dashboard ~20 ms/frame savings
|
|
|
|
Recommended dd02-ov fast profile:
|
|
resolution_stitch [280, 300], source_scale 0.4, hhd_downsample 5
|
|
Delete or rename stale unroll_map_uv.npz after changing output scale.
|
|
"""
|
|
|
|
|
|
def run_deformation_timed(sensor, timer, check_motion=True, threshold=0, tail_frames=10):
|
|
"""Run flow + gated HHD with separate benchmark buckets."""
|
|
timer.start("flow")
|
|
sensor.compute_flow()
|
|
timer.stop("flow")
|
|
|
|
timer.start("hhd")
|
|
run_hhd = True
|
|
if check_motion:
|
|
is_motion = sensor.detect_motion(threshold=threshold)
|
|
try:
|
|
tail_frames = max(0, int(tail_frames))
|
|
except (TypeError, ValueError):
|
|
tail_frames = 0
|
|
if is_motion:
|
|
sensor.tail_frames_count = 0
|
|
elif sensor.tail_frames_count < tail_frames:
|
|
sensor.tail_frames_count += 1
|
|
else:
|
|
run_hhd = False
|
|
sensor.tail_frames_count = tail_frames
|
|
if run_hhd:
|
|
sensor.compute_hhd()
|
|
timer.stop("hhd")
|
|
|
|
|
|
def configure_masked_poisson(sensor, args):
|
|
"""Apply masked-HHD solver settings to the externally fed SDK sensor."""
|
|
sensor.masked_poisson_solver = args.masked_poisson_solver
|
|
sensor.masked_poisson_max_boundary = args.masked_poisson_max_boundary
|
|
sensor.masked_poisson_boundary_stride = args.masked_poisson_boundary_stride
|
|
sensor.masked_poisson_max_iters = args.masked_poisson_max_iters
|
|
sensor.masked_poisson_tol = args.masked_poisson_tol
|
|
sensor.masked_poisson_reg = args.masked_poisson_reg
|
|
|
|
|
|
def resolve_runtime_geometry(args):
|
|
"""Apply config defaults and optional source-scale to UV/camera geometry."""
|
|
sdk_config = load_config(args.sdk_config)
|
|
camera_config = sdk_config.get("camera", {})
|
|
out_width, out_height, source_scale = resolve_uv_runtime_sizes(
|
|
camera_config,
|
|
out_width=args.out_width,
|
|
out_height=args.out_height,
|
|
source_scale=args.source_scale,
|
|
)
|
|
stitch = camera_config.get("resolution_stitch")
|
|
if stitch and len(stitch) >= 2 and (int(stitch[0]) != out_width or int(stitch[1]) != out_height):
|
|
print(
|
|
"Warning: UV output "
|
|
f"{out_width}x{out_height} differs from resolution_stitch "
|
|
f"{int(stitch[0])}x{int(stitch[1])} in {args.sdk_config}; "
|
|
"Sensor.get_img will resize again."
|
|
)
|
|
|
|
args.out_width = out_width
|
|
args.out_height = out_height
|
|
args.source_scale = source_scale
|
|
args.map_cache = scaled_map_cache_path(args.map_cache, source_scale)
|
|
return sdk_config, camera_config
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Real-time UV-based finger unrolling",
|
|
epilog=PERF_TUNING_EPILOG,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
)
|
|
parser.add_argument("--mesh", required=True, help="Exported mesh.npz path")
|
|
parser.add_argument("--K", dest="intrinsics_path", required=True, help="Exported camera_intrinsics.npz path")
|
|
parser.add_argument("--pose", required=True, help="Exported world_to_cam.npy path")
|
|
parser.add_argument("--calib", default=None, help="OpenCV calibration JSON for the live camera")
|
|
# With a 12% UV edge band on each side, only about 76% of each axis is the
|
|
# lower-distortion middle region. Use a slightly larger full remap so the
|
|
# useful center keeps roughly the same sampling density as the prior 506x550.
|
|
parser.add_argument("--out_width", type=int, default=None, help="UV flatten width (default: camera.resolution_stitch in sdk config)")
|
|
parser.add_argument("--out_height", type=int, default=None, help="UV flatten height (default: camera.resolution_stitch in sdk config)")
|
|
parser.add_argument(
|
|
"--source_scale",
|
|
type=float,
|
|
default=None,
|
|
help="Resize live source to this fraction before uv_prep/uv_remap; scales K accordingly (default: camera.source_scale or work4[3] in sdk config, else 1.0)",
|
|
)
|
|
parser.add_argument("--camera_id", type=int, default=0)
|
|
parser.add_argument("--video", help="Video file path (overrides camera)")
|
|
parser.add_argument("--map_cache", default="unroll_map_uv.npz")
|
|
parser.add_argument("--no_flip_v", action="store_true", help="Keep UV V increasing downward")
|
|
parser.add_argument("--no_flip_source_vertical", action="store_true", help="Do not vertically flip the live source before UV remap")
|
|
parser.add_argument("--edge_band", type=float, default=0.12, help="Normalized UV band near the boundary treated as high-distortion edge")
|
|
parser.add_argument("--edge_expand", type=float, default=0.0, help="Extra UV allocation for edge band; 0 chooses automatically from distortion stats")
|
|
parser.add_argument("--sdk_config", default="./config/dd02-ov.json", help="Orisys SDK config path/name")
|
|
parser.add_argument("--sdk_source", default=None, help="Dummy source used to initialize Sensor; defaults to --video or --camera_id")
|
|
parser.add_argument("--masked_poisson_solver", choices=["sparse", "capfft"], default="sparse", help="Masked HHD Poisson solver used inside the SDK")
|
|
parser.add_argument("--masked_poisson_max_boundary", type=int, default=96, help="Maximum number of UV-boundary samples used by the capfft solver")
|
|
parser.add_argument("--masked_poisson_boundary_stride", type=int, default=1, help="Subsample stride applied before capfft boundary sampling")
|
|
parser.add_argument("--masked_poisson_max_iters", type=int, default=3, help="Maximum capfft correction iterations per Poisson solve")
|
|
parser.add_argument("--masked_poisson_tol", type=float, default=1e-4, help="Residual tolerance used by the capfft correction loop")
|
|
parser.add_argument("--masked_poisson_reg", type=float, default=1e-8, help="Diagonal regularization applied to the capacitance matrix")
|
|
parser.add_argument("--verbose", action="store_true", help="Enable verbose SDK logs")
|
|
parser.add_argument(
|
|
"--no-viz",
|
|
action="store_true",
|
|
help="Skip OpenCV dashboard; print forces and benchmark timings only",
|
|
)
|
|
args = parser.parse_args()
|
|
_, camera_config = resolve_runtime_geometry(args)
|
|
|
|
print("[benchmark] Startup timing:")
|
|
data = benchmark_step("load_exported_data", load_exported_data, args.mesh, args.intrinsics_path, args.pose)
|
|
flip_v = not args.no_flip_v
|
|
flip_source_vertical = not args.no_flip_source_vertical
|
|
distortion = benchmark_step(
|
|
"compute_uv_distortion_metrics",
|
|
compute_uv_distortion_metrics,
|
|
data,
|
|
edge_band=args.edge_band,
|
|
)
|
|
distortion_stats = distortion["stats"]
|
|
middle_stats = distortion["middle_stats"]
|
|
edge_stats = distortion["edge_stats"]
|
|
edge_expand = args.edge_expand if args.edge_expand > 1.0 else recommend_edge_expand(distortion)
|
|
print(
|
|
"UV distortion stats: "
|
|
f"median mm/uv={distortion_stats['median_mm_per_uv']:.3f}, "
|
|
f"relative scale p05/p95={distortion_stats['relative_scale_p05']:.2f}x/"
|
|
f"{distortion_stats['relative_scale_p95']:.2f}x, "
|
|
f"anisotropy p50/p95={distortion_stats['anisotropy_p50']:.2f}x/"
|
|
f"{distortion_stats['anisotropy_p95']:.2f}x"
|
|
)
|
|
print(f"Using UV edge warp: band={args.edge_band:.2f}, expand={edge_expand:.2f}x")
|
|
print(
|
|
"UV middle-region stats: "
|
|
f"faces={middle_stats['face_count']}, "
|
|
f"relative scale p05/p95={middle_stats['relative_scale_p05']:.2f}x/"
|
|
f"{middle_stats['relative_scale_p95']:.2f}x, "
|
|
f"anisotropy p50/p95={middle_stats['anisotropy_p50']:.2f}x/"
|
|
f"{middle_stats['anisotropy_p95']:.2f}x"
|
|
)
|
|
print(
|
|
"UV edge-band stats: "
|
|
f"faces={edge_stats['face_count']}, "
|
|
f"relative scale p05/p95={edge_stats['relative_scale_p05']:.2f}x/"
|
|
f"{edge_stats['relative_scale_p95']:.2f}x, "
|
|
f"anisotropy p50/p95={edge_stats['anisotropy_p50']:.2f}x/"
|
|
f"{edge_stats['anisotropy_p95']:.2f}x"
|
|
)
|
|
|
|
if args.calib:
|
|
source_K, dist_coeffs, img_size = benchmark_step("load_calib_json", load_calib_json, args.calib)
|
|
native_width, native_height = img_size
|
|
print(f"Calibration loaded. Live image size: {(native_width, native_height)}")
|
|
else:
|
|
source_K = data["K"]
|
|
dist_coeffs = None
|
|
native_width = data["width"]
|
|
native_height = data["height"]
|
|
print("No calibration file provided; using exported intrinsics for projection.")
|
|
|
|
source_K, source_width, source_height = scale_camera_geometry(
|
|
source_K,
|
|
native_width,
|
|
native_height,
|
|
args.source_scale,
|
|
)
|
|
print(
|
|
"UV runtime geometry: "
|
|
f"native={native_width}x{native_height}, "
|
|
f"source={source_width}x{source_height} (scale={args.source_scale:g}), "
|
|
f"output={args.out_width}x{args.out_height}, "
|
|
f"map_cache={args.map_cache}"
|
|
)
|
|
|
|
domain = benchmark_step(
|
|
"build_or_load_domain",
|
|
build_or_load_domain,
|
|
args.mesh,
|
|
args.intrinsics_path,
|
|
args.pose,
|
|
args.map_cache,
|
|
source_K,
|
|
source_width,
|
|
source_height,
|
|
args.out_width,
|
|
args.out_height,
|
|
flip_v=flip_v,
|
|
edge_band=args.edge_band,
|
|
edge_expand=edge_expand,
|
|
include_surface=True,
|
|
source_scale=args.source_scale,
|
|
)
|
|
remap = domain.remap
|
|
remap.ensure_fast_maps()
|
|
edge_expand = float(remap.metadata.get("edge_expand", edge_expand))
|
|
|
|
uv_debug = benchmark_step(
|
|
"make_uv_debug_image",
|
|
make_uv_debug_image,
|
|
data,
|
|
out_width=args.out_width,
|
|
out_height=args.out_height,
|
|
flip_v=flip_v,
|
|
edge_band=args.edge_band,
|
|
edge_expand=edge_expand,
|
|
)
|
|
uv_distortion = benchmark_step(
|
|
"make_uv_distortion_image",
|
|
make_uv_distortion_image,
|
|
data,
|
|
distortion,
|
|
out_width=args.out_width,
|
|
out_height=args.out_height,
|
|
flip_v=flip_v,
|
|
edge_band=args.edge_band,
|
|
edge_expand=edge_expand,
|
|
)
|
|
uv_reference = benchmark_step(
|
|
"uv_reference_compose",
|
|
lambda: np.hstack(
|
|
[
|
|
resize_to_height(uv_debug, args.out_height),
|
|
resize_to_height(uv_distortion, args.out_height),
|
|
]
|
|
),
|
|
)
|
|
|
|
cap = open_source(args.video, args.camera_id)
|
|
if not cap.isOpened():
|
|
print("Cannot open camera/video source")
|
|
sys.exit(1)
|
|
|
|
cap.set(cv2.CAP_PROP_FRAME_WIDTH, native_width)
|
|
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, native_height)
|
|
|
|
sensor = benchmark_step(
|
|
"sensor_init",
|
|
lambda: orisys.Sensor(
|
|
"external",
|
|
config_name=args.sdk_config,
|
|
verbose=args.verbose,
|
|
mirror=False,
|
|
),
|
|
)
|
|
configure_masked_poisson(sensor, args)
|
|
print(
|
|
"Masked HHD solver: "
|
|
f"{sensor.masked_poisson_solver} "
|
|
f"(max_boundary={sensor.masked_poisson_max_boundary}, "
|
|
f"stride={sensor.masked_poisson_boundary_stride}, "
|
|
f"iters={sensor.masked_poisson_max_iters}, "
|
|
f"tol={sensor.masked_poisson_tol:.1e})"
|
|
)
|
|
|
|
frame_processor = UVUnrollFrameProcessor(
|
|
domain.remap,
|
|
camera_matrix=source_K if dist_coeffs is not None else None,
|
|
dist_coeffs=dist_coeffs,
|
|
source_width=source_width,
|
|
source_height=source_height,
|
|
flip_source_vertical=flip_source_vertical,
|
|
border_value=(0, 0, 0),
|
|
)
|
|
|
|
ok, warmup_frame = cap.read()
|
|
if ok:
|
|
warmup_prepared = frame_processor.prepare_source(warmup_frame)
|
|
warmup_unrolled = frame_processor.flatten_prepared(warmup_prepared)
|
|
sensor.get_img(
|
|
frame=warmup_unrolled,
|
|
already_processed=True,
|
|
valid_mask=frame_processor.valid_mask,
|
|
)
|
|
benchmark_step("hhd_warmup", sensor.warmup_deformation)
|
|
else:
|
|
print("[benchmark] hhd_warmup: skipped (no frame from source)")
|
|
|
|
if args.no_viz:
|
|
print("Running with --no-viz (dashboard disabled).")
|
|
else:
|
|
print("Press 'q' to quit.")
|
|
runtime_timer = StepTimer()
|
|
try:
|
|
while True:
|
|
runtime_timer.start("capture")
|
|
ok, frame = cap.read()
|
|
runtime_timer.stop("capture")
|
|
if not ok:
|
|
break
|
|
|
|
runtime_timer.start("uv_prepare")
|
|
frame_source = frame_processor.prepare_source(frame)
|
|
runtime_timer.stop("uv_prepare")
|
|
|
|
runtime_timer.start("uv_remap")
|
|
unrolled = frame_processor.flatten_prepared(frame_source)
|
|
runtime_timer.stop("uv_remap")
|
|
|
|
runtime_timer.start("get_img")
|
|
sensor.get_img(
|
|
frame=unrolled,
|
|
already_processed=True,
|
|
valid_mask=frame_processor.valid_mask,
|
|
)
|
|
runtime_timer.stop("get_img")
|
|
|
|
run_deformation_timed(sensor, runtime_timer, check_motion=True, threshold=0)
|
|
|
|
runtime_timer.start("contact")
|
|
sensor.compute_contact()
|
|
runtime_timer.stop("contact")
|
|
|
|
fps = sensor._fps
|
|
fnormal = sensor.fnormal
|
|
fshearx = sensor.fshearx
|
|
fsheary = sensor.fsheary
|
|
contact_ok = contact_inside_mask(sensor.centroid, sensor.valid_mask)
|
|
|
|
if not args.no_viz:
|
|
runtime_timer.start("viz")
|
|
fps, flow, vnormal, vshear, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = sensor.read_info(
|
|
sensor.info.FPS,
|
|
sensor.info.VRAW,
|
|
sensor.info.VNORMAL,
|
|
sensor.info.VSHEAR,
|
|
sensor.info.IMG,
|
|
sensor.info.CONTOUR,
|
|
sensor.info.CENTROID,
|
|
sensor.info.DEPTH,
|
|
sensor.info.FNORMAL,
|
|
sensor.info.FSHEARX,
|
|
sensor.info.FSHEARY,
|
|
)
|
|
|
|
arrows = orisys.util.draw_arrows(
|
|
img,
|
|
flow,
|
|
threshold=0.4,
|
|
grid_spacing=10,
|
|
arrow_scale=1.0,
|
|
)
|
|
contact_img = orisys.util.draw_contact(img, contour, centroid)
|
|
|
|
depth_norm = depth_map
|
|
if depth_norm.size > 0 and depth_norm.max() > depth_norm.min():
|
|
depth_u8 = ((depth_norm - depth_norm.min()) / (depth_norm.max() - depth_norm.min()) * 255).astype(np.uint8)
|
|
else:
|
|
depth_u8 = np.zeros_like(depth_norm, dtype=np.uint8)
|
|
depth_colored = cv2.applyColorMap(depth_u8, cv2.COLORMAP_JET)
|
|
|
|
display_mask = sensor.valid_mask
|
|
arrows = apply_display_mask(arrows, display_mask, border_value=(0, 0, 0))
|
|
contact_img = apply_display_mask(contact_img, display_mask, border_value=(0, 0, 0))
|
|
depth_colored = apply_display_mask(depth_colored, display_mask, border_value=(0, 0, 0))
|
|
arrows = overlay_runtime_status(arrows, sensor, centroid=centroid)
|
|
contact_ok = contact_inside_mask(centroid, display_mask)
|
|
|
|
dashboard = make_tiled_dashboard(
|
|
[
|
|
("Original", frame_source),
|
|
("UV Map | Distortion", uv_reference),
|
|
("UV Unrolled", unrolled),
|
|
("Flow", arrows),
|
|
],
|
|
tile_width=640,
|
|
tile_height=360,
|
|
)
|
|
cv2.imshow("UV Unroll Dashboard", dashboard)
|
|
runtime_timer.stop("viz")
|
|
|
|
print(
|
|
f"FPS={fps:.2f}, 法向力={fnormal:.4f}, 切向力X={fshearx:.4f}, "
|
|
f"切向力Y={fsheary:.4f}, 展开图尺寸={unrolled.shape[:2]}, "
|
|
f"solver={sensor.masked_poisson_solver}, "
|
|
f"mask={sensor.mask_stats['coverage']:.3f}, "
|
|
f"flow(valid/invalid)=({sensor.mask_stats['flow_mean_valid']:.3f}/"
|
|
f"{sensor.mask_stats['flow_mean_invalid']:.3f}), "
|
|
f"contact_in_mask={contact_ok}, "
|
|
f"{print_runtime_benchmark(runtime_timer)}",
|
|
end="\r",
|
|
)
|
|
|
|
key = cv2.waitKey(1) & 0xFF
|
|
if key == ord("q"):
|
|
print("\n正在退出程序...")
|
|
break
|
|
if key == ord("r"):
|
|
sensor.reset()
|
|
print("\n追踪器已重置。")
|
|
finally:
|
|
cap.release()
|
|
sensor.disconnect()
|
|
cv2.destroyAllWindows()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |