#!/usr/bin/env python3 """ Real-time UV-based perspective correction for the exported finger mesh. This version replaces the previous cylindrical approximation with a remap built from the mesh UV parameterization. The remap is precomputed once and then applied to each live frame with cv2.remap(). """ import argparse import json import os import sys import time import cv2 import numpy as np import orisys def resize_to_height(image, target_height): """Resize an image to a target height while preserving aspect ratio.""" if image.shape[0] == target_height: return image scale = target_height / float(image.shape[0]) target_width = max(1, int(round(image.shape[1] * scale))) return cv2.resize(image, (target_width, target_height), interpolation=cv2.INTER_AREA) def resize_to_fit(image, target_width, target_height): """Resize with aspect ratio preserved and pad to exact tile size.""" h, w = image.shape[:2] if h <= 0 or w <= 0: return np.zeros((target_height, target_width, 3), dtype=np.uint8) scale = min(target_width / float(w), target_height / float(h)) new_w = max(1, int(round(w * scale))) new_h = max(1, int(round(h * scale))) resized = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_AREA) if resized.ndim == 2: resized = cv2.cvtColor(resized, cv2.COLOR_GRAY2BGR) elif resized.ndim == 3 and resized.shape[2] == 1: resized = cv2.cvtColor(resized[:, :, 0], cv2.COLOR_GRAY2BGR) canvas = np.zeros((target_height, target_width, 3), dtype=np.uint8) y0 = (target_height - new_h) // 2 x0 = (target_width - new_w) // 2 canvas[y0:y0 + new_h, x0:x0 + new_w] = resized return canvas def add_panel_title(image, title): """Overlay a simple title onto a display tile.""" out = image.copy() cv2.rectangle(out, (0, 0), (out.shape[1], 36), (0, 0, 0), thickness=-1) cv2.putText(out, title, (12, 24), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2, cv2.LINE_AA) return out def make_tiled_dashboard(panels, tile_width=640, tile_height=360): """Arrange four panels into a 2x2 dashboard.""" tiles = [ add_panel_title(resize_to_fit(image, tile_width, tile_height), title) for title, image in panels ] top_row = np.hstack(tiles[:2]) bottom_row = np.hstack(tiles[2:4]) return np.vstack([top_row, bottom_row]) def load_calib_json(json_path): with open(json_path, "r", encoding="utf-8") as f: calib = json.load(f) K = np.array(calib["camera_matrix"], dtype=np.float64) dist = np.array(calib["distortion"], dtype=np.float64).flatten() img_size = tuple(int(v) for v in calib["image_size"]) return K, dist, img_size def load_exported_data(mesh_path, intrinsics_path, pose_path): mesh = np.load(mesh_path) cam = np.load(intrinsics_path) return { "vertices": mesh["vertices"].astype(np.float64), "faces": mesh["faces"].astype(np.int32), "uv_data": mesh["uv_data"].astype(np.float64), "loop_to_vertex": mesh["loop_to_vertex"].astype(np.int32), "K": cam["K"].astype(np.float64), "width": int(cam["width"]), "height": int(cam["height"]), "world_to_cam": np.load(pose_path).astype(np.float64), } def face_uvs_from_loop_data(faces, uv_data, loop_to_vertex): if len(uv_data) == 0: raise ValueError("mesh.npz does not contain UV data; UV flattening is unavailable.") if len(uv_data) != len(faces) * 3: raise ValueError("Expected a triangulated mesh with exactly 3 UV loops per face.") face_loop_vertices = loop_to_vertex.reshape(-1, 3) if face_loop_vertices.shape != faces.shape or not np.array_equal(face_loop_vertices, faces): raise ValueError("UV loop order does not match exported face vertex order.") return uv_data.reshape(-1, 3, 2) def project_world_to_image(points_world, K, world_to_cam): """Project world points into the image using Blender/OpenCV-compatible intrinsics.""" points_h = np.column_stack((points_world, np.ones(len(points_world), dtype=np.float64))) points_cam = (world_to_cam @ points_h.T).T[:, :3] z = -points_cam[:, 2] x = K[0, 0] * (points_cam[:, 0] / z) + K[0, 2] y = K[1, 1] * (points_cam[:, 1] / z) + K[1, 2] return np.column_stack((x, y)), z def barycentric_grid(px, py, tri): a, b, c = tri denom = (b[1] - c[1]) * (a[0] - c[0]) + (c[0] - b[0]) * (a[1] - c[1]) if abs(denom) < 1e-12: return None w0 = ((b[1] - c[1]) * (px - c[0]) + (c[0] - b[0]) * (py - c[1])) / denom w1 = ((c[1] - a[1]) * (px - c[0]) + (a[0] - c[0]) * (py - c[1])) / denom w2 = 1.0 - w0 - w1 return w0, w1, w2 def triangle_edge_lengths_3d(tri): return np.array( [ np.linalg.norm(tri[1] - tri[0]), np.linalg.norm(tri[2] - tri[1]), np.linalg.norm(tri[0] - tri[2]), ], dtype=np.float64, ) def triangle_edge_lengths_2d(tri): return np.array( [ np.linalg.norm(tri[1] - tri[0]), np.linalg.norm(tri[2] - tri[1]), np.linalg.norm(tri[0] - tri[2]), ], dtype=np.float64, ) def triangle_area_3d(tri): return 0.5 * np.linalg.norm(np.cross(tri[1] - tri[0], tri[2] - tri[0])) def triangle_area_2d(tri): return 0.5 * abs( (tri[1, 0] - tri[0, 0]) * (tri[2, 1] - tri[0, 1]) - (tri[1, 1] - tri[0, 1]) * (tri[2, 0] - tri[0, 0]) ) def summarize_uv_region(relative_scale, anisotropy, mask): mask_scale = mask & np.isfinite(relative_scale) & (relative_scale > 0) mask_aniso = mask & np.isfinite(anisotropy) & (anisotropy > 0) if not np.any(mask_scale): return { "face_count": 0, "relative_scale_p05": float("nan"), "relative_scale_p50": float("nan"), "relative_scale_p95": float("nan"), "relative_scale_min": float("nan"), "relative_scale_max": float("nan"), "anisotropy_p50": float("nan"), "anisotropy_p95": float("nan"), "anisotropy_max": float("nan"), } return { "face_count": int(np.count_nonzero(mask_scale)), "relative_scale_p05": float(np.percentile(relative_scale[mask_scale], 5)), "relative_scale_p50": float(np.percentile(relative_scale[mask_scale], 50)), "relative_scale_p95": float(np.percentile(relative_scale[mask_scale], 95)), "relative_scale_min": float(np.min(relative_scale[mask_scale])), "relative_scale_max": float(np.max(relative_scale[mask_scale])), "anisotropy_p50": float(np.percentile(anisotropy[mask_aniso], 50)) if np.any(mask_aniso) else float("nan"), "anisotropy_p95": float(np.percentile(anisotropy[mask_aniso], 95)) if np.any(mask_aniso) else float("nan"), "anisotropy_max": float(np.max(anisotropy[mask_aniso])) if np.any(mask_aniso) else float("nan"), } def compute_uv_distortion_metrics(data, edge_band=0.12): """Estimate local UV stretch/compression and anisotropy per face.""" faces = data["faces"] vertices = data["vertices"] face_uvs = face_uvs_from_loop_data(faces, data["uv_data"], data["loop_to_vertex"]) local_scale = np.full(len(faces), np.nan, dtype=np.float64) relative_scale = np.full(len(faces), np.nan, dtype=np.float64) anisotropy = np.full(len(faces), np.nan, dtype=np.float64) uv_centers = np.full((len(faces), 2), np.nan, dtype=np.float64) edge_mask = np.zeros(len(faces), dtype=bool) for idx, (face, uv_tri) in enumerate(zip(faces, face_uvs)): tri_3d = vertices[face] tri_uv = uv_tri uv_centers[idx] = np.mean(tri_uv, axis=0) edges_3d = triangle_edge_lengths_3d(tri_3d) edges_uv = triangle_edge_lengths_2d(tri_uv) valid_edges = edges_uv > 1e-9 if np.count_nonzero(valid_edges) < 2: continue edge_scales = edges_3d[valid_edges] / edges_uv[valid_edges] local_scale[idx] = float(np.exp(np.mean(np.log(np.clip(edge_scales, 1e-12, None))))) anisotropy[idx] = float(np.max(edge_scales) / max(np.min(edge_scales), 1e-12)) area_uv = triangle_area_2d(tri_uv) area_3d = triangle_area_3d(tri_3d) if area_uv > 1e-12 and area_3d > 1e-12: local_scale[idx] = float(np.sqrt(area_3d / area_uv)) valid_scale = np.isfinite(local_scale) & (local_scale > 0) if not np.any(valid_scale): raise RuntimeError("Could not compute UV distortion metrics; UV triangles may be degenerate.") median_scale = float(np.median(local_scale[valid_scale])) relative_scale[valid_scale] = local_scale[valid_scale] / median_scale min_edge_distance = np.min( np.column_stack( [ uv_centers[:, 0], 1.0 - uv_centers[:, 0], uv_centers[:, 1], 1.0 - uv_centers[:, 1], ] ), axis=1, ) edge_mask = min_edge_distance < edge_band middle_mask = ~edge_mask valid_aniso = np.isfinite(anisotropy) & (anisotropy > 0) stats = { "median_mm_per_uv": median_scale, "relative_scale_p05": float(np.percentile(relative_scale[valid_scale], 5)), "relative_scale_p50": float(np.percentile(relative_scale[valid_scale], 50)), "relative_scale_p95": float(np.percentile(relative_scale[valid_scale], 95)), "relative_scale_min": float(np.min(relative_scale[valid_scale])), "relative_scale_max": float(np.max(relative_scale[valid_scale])), "anisotropy_p50": float(np.percentile(anisotropy[valid_aniso], 50)) if np.any(valid_aniso) else float("nan"), "anisotropy_p95": float(np.percentile(anisotropy[valid_aniso], 95)) if np.any(valid_aniso) else float("nan"), "anisotropy_max": float(np.max(anisotropy[valid_aniso])) if np.any(valid_aniso) else float("nan"), "edge_band": float(edge_band), } return { "relative_scale": relative_scale, "anisotropy": anisotropy, "uv_centers": uv_centers, "edge_mask": edge_mask, "middle_mask": middle_mask, "stats": stats, "middle_stats": summarize_uv_region(relative_scale, anisotropy, middle_mask), "edge_stats": summarize_uv_region(relative_scale, anisotropy, edge_mask), } def recommend_edge_expand(distortion, max_expand=3.0): """Suggest how much more UV space to allocate to the edge band.""" middle_p95 = distortion["middle_stats"]["relative_scale_p95"] edge_p95 = distortion["edge_stats"]["relative_scale_p95"] if not np.isfinite(middle_p95) or not np.isfinite(edge_p95) or middle_p95 <= 0: return 1.0 # Use a square-root response so large edge distortion does not over-expand. return float(np.clip(np.sqrt(edge_p95 / middle_p95), 1.0, max_expand)) def warp_uv_axis(values, edge_band, edge_expand): """Allocate more normalized output span to UV edge regions.""" u = np.asarray(values, dtype=np.float64) e = float(np.clip(edge_band, 0.0, 0.49)) s = float(max(edge_expand, 1.0)) if e <= 0.0 or s <= 1.0: return u.copy() middle = 1.0 - 2.0 * e total = middle + 2.0 * e * s out = np.empty_like(u) left = u < e right = u > (1.0 - e) center = ~(left | right) out[left] = (u[left] * s) / total out[center] = (e * s + (u[center] - e)) / total out[right] = (e * s + middle + (u[right] - (1.0 - e)) * s) / total return out def uv_triangle_to_image(uv_tri, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0): """Map UV triangle into output image coordinates with optional edge expansion.""" tri = np.empty_like(uv_tri) u_warp = warp_uv_axis(uv_tri[:, 0], edge_band=edge_band, edge_expand=edge_expand) v_warp = warp_uv_axis(uv_tri[:, 1], edge_band=edge_band, edge_expand=edge_expand) tri[:, 0] = u_warp * (out_width - 1) tri[:, 1] = ((1.0 - v_warp) if flip_v else v_warp) * (out_height - 1) return tri def build_uv_remap( data, source_K, source_width, source_height, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0, ): faces = data["faces"] vertices = data["vertices"] face_uvs = face_uvs_from_loop_data(faces, data["uv_data"], data["loop_to_vertex"]) map_x = np.full((out_height, out_width), -1.0, dtype=np.float32) map_y = np.full((out_height, out_width), -1.0, dtype=np.float32) valid_mask = np.zeros((out_height, out_width), dtype=bool) depth_buffer = np.full((out_height, out_width), np.inf, dtype=np.float64) print("Building UV remap ...") t_start = time.time() for face_idx, (face, uv_tri) in enumerate(zip(faces, face_uvs)): tri = uv_triangle_to_image( uv_tri, out_width, out_height, flip_v=flip_v, edge_band=edge_band, edge_expand=edge_expand, ) x0 = max(0, int(np.floor(np.min(tri[:, 0])))) x1 = min(out_width - 1, int(np.ceil(np.max(tri[:, 0])))) y0 = max(0, int(np.floor(np.min(tri[:, 1])))) y1 = min(out_height - 1, int(np.ceil(np.max(tri[:, 1])))) if x1 < x0 or y1 < y0: continue yy, xx = np.mgrid[y0:y1 + 1, x0:x1 + 1] bary = barycentric_grid(xx + 0.5, yy + 0.5, tri) if bary is None: continue w0, w1, w2 = bary inside = (w0 >= -1e-6) & (w1 >= -1e-6) & (w2 >= -1e-6) if not np.any(inside): continue weights = np.stack((w0[inside], w1[inside], w2[inside]), axis=1) points_world = weights @ vertices[face] pixels, depth = project_world_to_image(points_world, source_K, data["world_to_cam"]) src_x = pixels[:, 0] src_y = pixels[:, 1] target_y = yy[inside] target_x = xx[inside] valid = ( (depth > 0) & (src_x >= 0) & (src_x <= source_width - 1) & (src_y >= 0) & (src_y <= source_height - 1) & (depth < depth_buffer[target_y, target_x]) ) if not np.any(valid): continue tx = target_x[valid] ty = target_y[valid] map_x[ty, tx] = src_x[valid].astype(np.float32) map_y[ty, tx] = src_y[valid].astype(np.float32) valid_mask[ty, tx] = True depth_buffer[ty, tx] = depth[valid] if (face_idx % 200) == 0: print(f" face {face_idx}/{len(faces)}") print(f"Remap build took {time.time() - t_start:.1f} s") return { "map_x": map_x, "map_y": map_y, "valid_mask": valid_mask, "source_width": np.array(source_width, dtype=np.int32), "source_height": np.array(source_height, dtype=np.int32), "out_width": np.array(out_width, dtype=np.int32), "out_height": np.array(out_height, dtype=np.int32), "flip_v": np.array(int(flip_v), dtype=np.int32), "edge_band": np.array(edge_band, dtype=np.float64), "edge_expand": np.array(edge_expand, dtype=np.float64), "projection_K": source_K.astype(np.float64), "method": np.array("uv"), } def remap_valid_ratio(remap): return float(np.mean(remap["valid_mask"])) def load_cached_remap(path): data = np.load(path) return {key: data[key] for key in data.files} def cache_matches(remap, source_K, source_width, source_height, out_width, out_height, flip_v, edge_band, edge_expand): required = {"map_x", "map_y", "valid_mask", "source_width", "source_height", "out_width", "out_height"} if not required.issubset(remap.keys()): return False if remap["map_x"].shape != (out_height, out_width): return False if int(remap["source_width"]) != source_width or int(remap["source_height"]) != source_height: return False if int(remap["out_width"]) != out_width or int(remap["out_height"]) != out_height: return False if "flip_v" in remap and int(remap["flip_v"]) != int(flip_v): return False if "edge_band" in remap and not np.isclose(float(remap["edge_band"]), float(edge_band), rtol=1e-6, atol=1e-6): return False if "edge_expand" in remap and not np.isclose(float(remap["edge_expand"]), float(edge_expand), rtol=1e-6, atol=1e-6): return False if "projection_K" in remap and not np.allclose(remap["projection_K"], source_K, rtol=1e-6, atol=1e-6): return False return True def flatten_frame(frame, remap, border_value=0): flat = cv2.remap( frame, remap["map_x"].astype(np.float32), remap["map_y"].astype(np.float32), interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=border_value, ) flat = flat.copy() flat[~remap["valid_mask"].astype(bool)] = border_value return flat def make_uv_debug_image(data, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0): """Create a UV-domain reference image to verify orientation of the remap.""" face_uvs = face_uvs_from_loop_data(data["faces"], data["uv_data"], data["loop_to_vertex"]) uv_img = np.zeros((out_height, out_width, 3), dtype=np.uint8) label_color = (255, 255, 255) for uv_tri in face_uvs: tri = uv_triangle_to_image( uv_tri, out_width, out_height, flip_v=flip_v, edge_band=edge_band, edge_expand=edge_expand, ) pts = np.round(tri).astype(np.int32) mask = np.zeros((out_height, out_width), dtype=np.uint8) cv2.fillConvexPoly(mask, pts, 255) x0 = max(0, int(np.floor(np.min(tri[:, 0])))) x1 = min(out_width - 1, int(np.ceil(np.max(tri[:, 0])))) y0 = max(0, int(np.floor(np.min(tri[:, 1])))) y1 = min(out_height - 1, int(np.ceil(np.max(tri[:, 1])))) if x1 < x0 or y1 < y0: continue yy, xx = np.mgrid[y0:y1 + 1, x0:x1 + 1] inside = mask[y0:y1 + 1, x0:x1 + 1] > 0 if not np.any(inside): continue bary = barycentric_grid(xx + 0.5, yy + 0.5, tri) if bary is None: continue w0, w1, w2 = bary valid = inside & (w0 >= -1e-6) & (w1 >= -1e-6) & (w2 >= -1e-6) if not np.any(valid): continue uv = ( w0[valid, None] * uv_tri[0] + w1[valid, None] * uv_tri[1] + w2[valid, None] * uv_tri[2] ) colors = np.zeros((uv.shape[0], 3), dtype=np.uint8) colors[:, 0] = np.clip(uv[:, 0] * 255.0, 0, 255).astype(np.uint8) colors[:, 1] = np.clip(uv[:, 1] * 255.0, 0, 255).astype(np.uint8) colors[:, 2] = np.clip((1.0 - 0.5 * (uv[:, 0] + uv[:, 1])) * 255.0, 0, 255).astype(np.uint8) uv_img[yy[valid], xx[valid]] = colors cv2.polylines(uv_img, [pts.reshape(-1, 1, 2)], isClosed=True, color=(255, 255, 255), thickness=1) cv2.arrowedLine(uv_img, (20, out_height - 20), (min(out_width - 20, 140), out_height - 20), (0, 0, 255), 2, tipLength=0.15) cv2.arrowedLine(uv_img, (20, out_height - 20), (20, max(20, out_height - 140)), (0, 255, 0), 2, tipLength=0.15) cv2.putText(uv_img, "U", (min(out_width - 30, 145), out_height - 28), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2, cv2.LINE_AA) cv2.putText(uv_img, "V", (28, max(30, out_height - 145)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2, cv2.LINE_AA) cv2.putText(uv_img, "UV Map", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.9, label_color, 2, cv2.LINE_AA) if edge_expand > 1.0: cv2.putText(uv_img, f"Edge expand {edge_expand:.2f}x", (20, 64), cv2.FONT_HERSHEY_SIMPLEX, 0.55, label_color, 2, cv2.LINE_AA) return uv_img def make_uv_distortion_image(data, distortion, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0): """Render UV distortion heatmap; red means more 3D surface per UV area.""" face_uvs = face_uvs_from_loop_data(data["faces"], data["uv_data"], data["loop_to_vertex"]) uv_img = np.zeros((out_height, out_width, 3), dtype=np.uint8) rel = distortion["relative_scale"] edge_mask = distortion["edge_mask"] valid_rel = np.isfinite(rel) & (rel > 0) if not np.any(valid_rel): return uv_img log_rel = np.log2(rel[valid_rel]) clip = max(0.25, float(np.percentile(np.abs(log_rel), 95))) for idx, uv_tri in enumerate(face_uvs): tri = uv_triangle_to_image( uv_tri, out_width, out_height, flip_v=flip_v, edge_band=edge_band, edge_expand=edge_expand, ) pts = np.round(tri).astype(np.int32) if not np.isfinite(rel[idx]) or rel[idx] <= 0: color = (64, 64, 64) else: value = np.clip((np.log2(rel[idx]) + clip) / (2.0 * clip), 0.0, 1.0) color_idx = np.uint8(round(value * 255.0)) color = tuple(int(c) for c in cv2.applyColorMap(np.array([[color_idx]], dtype=np.uint8), cv2.COLORMAP_JET)[0, 0]) cv2.fillConvexPoly(uv_img, pts, color) edge_color = (255, 255, 255) if edge_mask[idx] else (180, 180, 180) edge_thickness = 2 if edge_mask[idx] else 1 cv2.polylines(uv_img, [pts.reshape(-1, 1, 2)], isClosed=True, color=edge_color, thickness=edge_thickness) cv2.putText(uv_img, "UV Distortion", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 255), 2, cv2.LINE_AA) cv2.putText(uv_img, "Blue: denser UV Red: sparser UV", (20, 64), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (220, 220, 220), 2, cv2.LINE_AA) cv2.putText(uv_img, "White outlines: edge-band faces", (20, 92), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (220, 220, 220), 2, cv2.LINE_AA) middle_stats = distortion["middle_stats"] edge_stats = distortion["edge_stats"] cv2.putText( uv_img, f"middle p05/p95: {middle_stats['relative_scale_p05']:.2f}x / {middle_stats['relative_scale_p95']:.2f}x", (20, 120), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (220, 220, 220), 2, cv2.LINE_AA, ) cv2.putText( uv_img, f"edge p05/p95: {edge_stats['relative_scale_p05']:.2f}x / {edge_stats['relative_scale_p95']:.2f}x", (20, 148), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (220, 220, 220), 2, cv2.LINE_AA, ) return uv_img def open_source(video_path, camera_id): return cv2.VideoCapture(video_path) if video_path else cv2.VideoCapture(camera_id) def parse_video_source(value): try: return int(value) except (TypeError, ValueError): return value def align_frame_to_uv_map(frame_bgr, flip_vertical=True): """Match live-frame orientation to the Blender-exported UV remap convention.""" if not flip_vertical: return frame_bgr return np.flipud(frame_bgr).copy() def main(): parser = argparse.ArgumentParser(description="Real-time UV-based finger unrolling") parser.add_argument("--mesh", required=True, help="Exported mesh.npz path") parser.add_argument("--K", dest="intrinsics_path", required=True, help="Exported camera_intrinsics.npz path") parser.add_argument("--pose", required=True, help="Exported world_to_cam.npy path") parser.add_argument("--calib", default=None, help="OpenCV calibration JSON for the live camera") # With a 12% UV edge band on each side, only about 76% of each axis is the # lower-distortion middle region. Use a slightly larger full remap so the # useful center keeps roughly the same sampling density as the prior 506x550. parser.add_argument("--out_width", type=int, default=672) parser.add_argument("--out_height", type=int, default=728) parser.add_argument("--camera_id", type=int, default=0) parser.add_argument("--video", help="Video file path (overrides camera)") parser.add_argument("--map_cache", default="unroll_map_uv.npz") parser.add_argument("--no_flip_v", action="store_true", help="Keep UV V increasing downward") parser.add_argument("--no_flip_source_vertical", action="store_true", help="Do not vertically flip the live source before UV remap") parser.add_argument("--edge_band", type=float, default=0.12, help="Normalized UV band near the boundary treated as high-distortion edge") parser.add_argument("--edge_expand", type=float, default=0.0, help="Extra UV allocation for edge band; 0 chooses automatically from distortion stats") parser.add_argument("--sdk_config", default="./config/dd02-ov.json", help="Orisys SDK config path/name") parser.add_argument("--sdk_source", default=None, help="Dummy source used to initialize Sensor; defaults to --video or --camera_id") parser.add_argument("--verbose", action="store_true", help="Enable verbose SDK logs") args = parser.parse_args() data = load_exported_data(args.mesh, args.intrinsics_path, args.pose) flip_v = not args.no_flip_v flip_source_vertical = not args.no_flip_source_vertical distortion = compute_uv_distortion_metrics(data, edge_band=args.edge_band) distortion_stats = distortion["stats"] middle_stats = distortion["middle_stats"] edge_stats = distortion["edge_stats"] edge_expand = args.edge_expand if args.edge_expand > 1.0 else recommend_edge_expand(distortion) print( "UV distortion stats: " f"median mm/uv={distortion_stats['median_mm_per_uv']:.3f}, " f"relative scale p05/p95={distortion_stats['relative_scale_p05']:.2f}x/" f"{distortion_stats['relative_scale_p95']:.2f}x, " f"anisotropy p50/p95={distortion_stats['anisotropy_p50']:.2f}x/" f"{distortion_stats['anisotropy_p95']:.2f}x" ) print(f"Using UV edge warp: band={args.edge_band:.2f}, expand={edge_expand:.2f}x") print( "UV middle-region stats: " f"faces={middle_stats['face_count']}, " f"relative scale p05/p95={middle_stats['relative_scale_p05']:.2f}x/" f"{middle_stats['relative_scale_p95']:.2f}x, " f"anisotropy p50/p95={middle_stats['anisotropy_p50']:.2f}x/" f"{middle_stats['anisotropy_p95']:.2f}x" ) print( "UV edge-band stats: " f"faces={edge_stats['face_count']}, " f"relative scale p05/p95={edge_stats['relative_scale_p05']:.2f}x/" f"{edge_stats['relative_scale_p95']:.2f}x, " f"anisotropy p50/p95={edge_stats['anisotropy_p50']:.2f}x/" f"{edge_stats['anisotropy_p95']:.2f}x" ) if args.calib: source_K, dist_coeffs, img_size = load_calib_json(args.calib) source_width, source_height = img_size print(f"Calibration loaded. Live image size: {(source_width, source_height)}") else: source_K = data["K"] dist_coeffs = None source_width = data["width"] source_height = data["height"] print("No calibration file provided; using exported intrinsics for projection.") remap = None if os.path.exists(args.map_cache): cached = load_cached_remap(args.map_cache) if cache_matches( cached, source_K=source_K, source_width=source_width, source_height=source_height, out_width=args.out_width, out_height=args.out_height, flip_v=flip_v, edge_band=args.edge_band, edge_expand=edge_expand, ): remap = cached print(f"Loaded cached UV remap. Valid ratio: {remap_valid_ratio(remap):.3f}") else: print("Cached UV remap is stale. Regenerating.") if remap is None: remap = build_uv_remap( data, source_K=source_K, source_width=source_width, source_height=source_height, out_width=args.out_width, out_height=args.out_height, flip_v=flip_v, edge_band=args.edge_band, edge_expand=edge_expand, ) valid_ratio = remap_valid_ratio(remap) print(f"Generated UV remap valid ratio: {valid_ratio:.3f}") if valid_ratio == 0.0: raise RuntimeError("Generated UV remap contains no valid samples.") np.savez_compressed(args.map_cache, **remap) uv_debug = make_uv_debug_image( data, out_width=args.out_width, out_height=args.out_height, flip_v=flip_v, edge_band=args.edge_band, edge_expand=edge_expand, ) uv_distortion = make_uv_distortion_image( data, distortion, out_width=args.out_width, out_height=args.out_height, flip_v=flip_v, edge_band=args.edge_band, edge_expand=edge_expand, ) uv_reference = np.hstack( [ resize_to_height(uv_debug, args.out_height), resize_to_height(uv_distortion, args.out_height), ] ) cap = open_source(args.video, args.camera_id) if not cap.isOpened(): print("Cannot open camera/video source") sys.exit(1) cap.set(cv2.CAP_PROP_FRAME_WIDTH, source_width) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, source_height) sensor = orisys.Sensor( "external", config_name=args.sdk_config, verbose=args.verbose, mirror=False, ) print("Press 'q' to quit.") try: while True: ok, frame = cap.read() if not ok: break if frame.shape[1] != source_width or frame.shape[0] != source_height: frame = cv2.resize(frame, (source_width, source_height), interpolation=cv2.INTER_LINEAR) if dist_coeffs is not None: frame_source = cv2.undistort(frame, source_K, dist_coeffs) else: frame_source = frame frame_source = align_frame_to_uv_map(frame_source, flip_vertical=flip_source_vertical) unrolled = flatten_frame(frame_source, remap, border_value=(0, 0, 0)) sensor.get_img(frame=unrolled, already_processed=True) sensor.compute_deformation(check_motion=True, threshold=0) sensor.compute_contact() fps, flow, vnormal, vshear, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = sensor.read_info( sensor.info.FPS, sensor.info.VRAW, sensor.info.VNORMAL, sensor.info.VSHEAR, sensor.info.IMG, sensor.info.CONTOUR, sensor.info.CENTROID, sensor.info.DEPTH, sensor.info.FNORMAL, sensor.info.FSHEARX, sensor.info.FSHEARY, ) arrows = orisys.util.draw_arrows( img, flow, threshold=2, grid_spacing=20, arrow_scale=1.0, ) contact_img = orisys.util.draw_contact(img, contour, centroid) depth_norm = depth_map if depth_norm.size > 0 and depth_norm.max() > depth_norm.min(): depth_u8 = ((depth_norm - depth_norm.min()) / (depth_norm.max() - depth_norm.min()) * 255).astype(np.uint8) else: depth_u8 = np.zeros_like(depth_norm, dtype=np.uint8) depth_colored = cv2.applyColorMap(depth_u8, cv2.COLORMAP_JET) dashboard = make_tiled_dashboard( [ ("Original", frame_source), ("UV Map | Distortion", uv_reference), ("UV Unrolled", unrolled), ("Flow", arrows), ], tile_width=640, tile_height=360, ) cv2.imshow("UV Unroll Dashboard", dashboard) print( f"FPS={fps:.2f}, 法向力={fnormal:.4f}, 切向力X={fshearx:.4f}, " f"切向力Y={fsheary:.4f}, 展开图尺寸={unrolled.shape[:2]}", end="\r", ) key = cv2.waitKey(1) & 0xFF if key == ord("q"): print("\n正在退出程序...") break if key == ord("r"): sensor.reset() print("\n追踪器已重置。") finally: cap.release() sensor.disconnect() cv2.destroyAllWindows() if __name__ == "__main__": main()