diff --git a/config/dd02-ov.json b/config/dd02-ov.json index 654b67c..0252e68 100644 --- a/config/dd02-ov.json +++ b/config/dd02-ov.json @@ -5,7 +5,7 @@ "camera": { "resolution_raw": [1280, 720], "fps": 30, - "resolution_stitch": [1280, 720] + "resolution_stitch": [672, 728] }, "tracker": { "adaptive": true, diff --git a/src/orisys/sensor.py b/src/orisys/sensor.py index 83eede0..9020696 100644 --- a/src/orisys/sensor.py +++ b/src/orisys/sensor.py @@ -85,6 +85,8 @@ class Sensor: if vid_src is None: raise Exception("User must specify a video source.") + self._external_only = isinstance(vid_src, str) and vid_src.lower() == "external" + # Store settings self.vid_src = vid_src self.isstitch = isstitch @@ -171,7 +173,7 @@ class Sensor: self._is_live_camera = ( isinstance(self.vid_src, int) or (isinstance(self.vid_src, str) and self.vid_src.startswith("/dev/video")) - ) + ) and not self._external_only self._capture_thread = None self._capture_running = False self._latest_frame = None @@ -179,8 +181,13 @@ class Sensor: self._last_consumed_seq = -1 self._latest_frame_lock = threading.Lock() - # Setup video capture - self._open_video_source(override, box) + # Setup video capture unless this sensor is fed externally. + if self._external_only: + self.stitch_valid = False + if self.verbose: + print("Sensor initialized in external-frame mode.") + else: + self._open_video_source(override, box) if self.verbose: print(f" Video source: {vid_src}") @@ -197,9 +204,19 @@ class Sensor: # Public API - Core Methods (Most Frequently Used) # ========================================================================= - def get_img(self): + def get_img(self, frame=None, already_processed=False): """ Capture and preprocess a frame. + + Parameters + ---------- + frame : np.ndarray | None + Optional externally supplied frame. When provided, the sensor skips + reading from its own video source and uses this frame instead. + already_processed : bool + If True, treat `frame` as the final BGR tactile image and skip the + configured frame processor. This is useful for pipelines that + generate a derived image, such as an unrolled tactile view. Returns: np.ndarray | None: Processed image, or None on capture failure. @@ -207,7 +224,10 @@ class Sensor: # Simple direct read - buffer size of 1 should minimize frame accumulation # Timing measurement helps identify if capture or processing is the bottleneck t_capture_start = time.perf_counter() - ret, frame = self._read_frame() + if frame is None: + ret, frame = self._read_frame() + else: + ret = frame is not None t_capture_end = time.perf_counter() self._last_capture_time = t_capture_end - t_capture_start @@ -218,8 +238,15 @@ class Sensor: # Start processing timing t_processing_start = time.perf_counter() - self.frame = frame - img_bgr = self.frame_processor.process(frame) + self.frame = frame.copy() if isinstance(frame, np.ndarray) else frame + if already_processed: + img_bgr = frame + if img_bgr.ndim == 2: + img_bgr = cv2.cvtColor(img_bgr, cv2.COLOR_GRAY2BGR) + elif img_bgr.ndim == 3 and img_bgr.shape[2] == 1: + img_bgr = cv2.cvtColor(img_bgr[:, :, 0], cv2.COLOR_GRAY2BGR) + else: + img_bgr = self.frame_processor.process(frame) t_after_extract = time.perf_counter() t_after_stitch = time.perf_counter() diff --git a/tools/3d_patch_generation_blender/dd02c_0607.blend b/tools/3d_patch_generation_blender/dd02c_0607.blend new file mode 100644 index 0000000..5e97939 Binary files /dev/null and b/tools/3d_patch_generation_blender/dd02c_0607.blend differ diff --git a/tools/3d_patch_generation_blender/dd02c_0607.blend1 b/tools/3d_patch_generation_blender/dd02c_0607.blend1 new file mode 100644 index 0000000..e8e70e4 Binary files /dev/null and b/tools/3d_patch_generation_blender/dd02c_0607.blend1 differ diff --git a/tools/3d_patch_generation_blender/dd02c_v3.blend b/tools/3d_patch_generation_blender/dd02c_v3.blend deleted file mode 100644 index e7c4e00..0000000 Binary files a/tools/3d_patch_generation_blender/dd02c_v3.blend and /dev/null differ diff --git a/tools/3d_patch_generation_blender/dd02c_v3.blend1 b/tools/3d_patch_generation_blender/dd02c_v3.blend1 deleted file mode 100644 index 725a69b..0000000 Binary files a/tools/3d_patch_generation_blender/dd02c_v3.blend1 and /dev/null differ diff --git a/tools/3d_patch_generation_blender/patch_texture.png b/tools/3d_patch_generation_blender/patch_texture.png new file mode 100644 index 0000000..7d3d9fa Binary files /dev/null and b/tools/3d_patch_generation_blender/patch_texture.png differ diff --git a/tools/3d_patch_generation_blender/patch_texture_0607_X80Y45.png b/tools/3d_patch_generation_blender/patch_texture_0607_X80Y45.png new file mode 100644 index 0000000..830ec01 Binary files /dev/null and b/tools/3d_patch_generation_blender/patch_texture_0607_X80Y45.png differ diff --git a/tools/3d_patch_generation_blender/unroll_finger.py b/tools/3d_patch_generation_blender/unroll_finger.py deleted file mode 100644 index a08488f..0000000 --- a/tools/3d_patch_generation_blender/unroll_finger.py +++ /dev/null @@ -1,217 +0,0 @@ -"""Flatten captured finger images into the mesh UV surface domain. - -The generated remap is expensive to build but cheap to use: live code only -needs cv2.remap(frame, map_x, map_y, ...). -""" - -import argparse -import os - -import cv2 -import numpy as np - - -def load_exported_data(data_dir): - mesh = np.load(os.path.join(data_dir, "mesh.npz")) - cam = np.load(os.path.join(data_dir, "camera_intrinsics.npz")) - return { - "vertices": mesh["vertices"].astype(np.float64), - "faces": mesh["faces"].astype(np.int32), - "uv_data": mesh["uv_data"].astype(np.float64), - "loop_to_vertex": mesh["loop_to_vertex"].astype(np.int32), - "K": cam["K"].astype(np.float64), - "width": int(cam["width"]), - "height": int(cam["height"]), - "world_to_cam": np.load(os.path.join(data_dir, "world_to_cam.npy")).astype(np.float64), - } - - -def face_uvs_from_loop_data(faces, uv_data, loop_to_vertex): - if len(uv_data) == 0: - raise ValueError("mesh.npz does not contain UV data; UV flattening is unavailable.") - if len(uv_data) != len(faces) * 3: - raise ValueError("Expected a triangulated mesh with exactly 3 UV loops per face.") - - face_loop_vertices = loop_to_vertex.reshape(-1, 3) - if face_loop_vertices.shape != faces.shape or not np.array_equal(face_loop_vertices, faces): - raise ValueError("UV loop order does not match exported face vertex order.") - return uv_data.reshape(-1, 3, 2) - - -def project_world_to_image(points_world, K, world_to_cam): - points_h = np.column_stack((points_world, np.ones(len(points_world)))) - points_cam = (world_to_cam @ points_h.T).T[:, :3] - z = -points_cam[:, 2] - x = K[0, 0] * (points_cam[:, 0] / z) + K[0, 2] - y = K[1, 1] * (points_cam[:, 1] / z) + K[1, 2] - return np.column_stack((x, y)), z - - -def barycentric_grid(px, py, tri): - a, b, c = tri - denom = (b[1] - c[1]) * (a[0] - c[0]) + (c[0] - b[0]) * (a[1] - c[1]) - if abs(denom) < 1e-12: - return None - w0 = ((b[1] - c[1]) * (px - c[0]) + (c[0] - b[0]) * (py - c[1])) / denom - w1 = ((c[1] - a[1]) * (px - c[0]) + (a[0] - c[0]) * (py - c[1])) / denom - w2 = 1.0 - w0 - w1 - return w0, w1, w2 - - -def build_uv_remap(data, out_width=1024, out_height=1024, flip_v=True): - faces = data["faces"] - vertices = data["vertices"] - face_uvs = face_uvs_from_loop_data(faces, data["uv_data"], data["loop_to_vertex"]) - - map_x = np.full((out_height, out_width), -1.0, dtype=np.float32) - map_y = np.full((out_height, out_width), -1.0, dtype=np.float32) - flat_xyz = np.full((out_height, out_width, 3), np.nan, dtype=np.float32) - depth_buffer = np.full((out_height, out_width), np.inf, dtype=np.float64) - - for face, uv_tri in zip(faces, face_uvs): - tri = np.empty_like(uv_tri) - tri[:, 0] = uv_tri[:, 0] * (out_width - 1) - tri[:, 1] = (1.0 - uv_tri[:, 1] if flip_v else uv_tri[:, 1]) * (out_height - 1) - - x0 = max(0, int(np.floor(np.min(tri[:, 0])))) - x1 = min(out_width - 1, int(np.ceil(np.max(tri[:, 0])))) - y0 = max(0, int(np.floor(np.min(tri[:, 1])))) - y1 = min(out_height - 1, int(np.ceil(np.max(tri[:, 1])))) - if x1 < x0 or y1 < y0: - continue - - yy, xx = np.mgrid[y0:y1 + 1, x0:x1 + 1] - bary = barycentric_grid(xx + 0.5, yy + 0.5, tri) - if bary is None: - continue - - w0, w1, w2 = bary - inside = (w0 >= -1e-6) & (w1 >= -1e-6) & (w2 >= -1e-6) - if not np.any(inside): - continue - - weights = np.stack((w0[inside], w1[inside], w2[inside]), axis=1) - points_world = weights @ vertices[face] - pixels, depth = project_world_to_image(points_world, data["K"], data["world_to_cam"]) - - src_x = pixels[:, 0] - src_y = pixels[:, 1] - target_y = yy[inside] - target_x = xx[inside] - valid = ( - (depth > 0) - & (src_x >= 0) - & (src_x <= data["width"] - 1) - & (src_y >= 0) - & (src_y <= data["height"] - 1) - & (depth < depth_buffer[target_y, target_x]) - ) - if not np.any(valid): - continue - - tx = target_x[valid] - ty = target_y[valid] - map_x[ty, tx] = src_x[valid].astype(np.float32) - map_y[ty, tx] = src_y[valid].astype(np.float32) - flat_xyz[ty, tx] = points_world[valid].astype(np.float32) - depth_buffer[ty, tx] = depth[valid] - - valid_mask = map_x >= 0 - return { - "map_x": map_x, - "map_y": map_y, - "valid_mask": valid_mask, - "flat_xyz": flat_xyz, - "source_width": np.array(data["width"], dtype=np.int32), - "source_height": np.array(data["height"], dtype=np.int32), - "out_width": np.array(out_width, dtype=np.int32), - "out_height": np.array(out_height, dtype=np.int32), - "method": np.array("uv"), - } - - -def save_remap(path, remap): - os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) - np.savez_compressed(path, **remap) - - -def load_remap(path): - data = np.load(path) - return {key: data[key] for key in data.files} - - -def flatten_frame(frame, remap, border_value=0): - flat = cv2.remap( - frame, - remap["map_x"].astype(np.float32), - remap["map_y"].astype(np.float32), - interpolation=cv2.INTER_LINEAR, - borderMode=cv2.BORDER_CONSTANT, - borderValue=border_value, - ) - valid_mask = remap.get("valid_mask") - if valid_mask is not None: - flat = flat.copy() - flat[~valid_mask.astype(bool)] = border_value - return flat - - -def load_capture_frame(capture_path, frame_index=0, flip_vertical=True): - image = cv2.imread(capture_path, cv2.IMREAD_COLOR) - if image is not None: - if flip_vertical: - image = np.flipud(image).copy() - return image - - cap = cv2.VideoCapture(capture_path, flip_vertical=True) - if not cap.isOpened(): - raise FileNotFoundError(f"Could not read image or video: {capture_path}") - try: - if frame_index > 0: - cap.set(cv2.CAP_PROP_POS_FRAMES, frame_index) - ok, frame = cap.read() - - - if not ok: - raise RuntimeError(f"Could not read frame {frame_index} from {capture_path}") - return frame - finally: - cap.release() - - -def main(): - parser = argparse.ArgumentParser(description="Build and apply a UV flattening remap for finger images.") - parser.add_argument("--data", required=True, help="Path to Blender exported_data directory.") - parser.add_argument("--remap", default=None, help="Output/input flatten_remap.npz path.") - parser.add_argument("--capture", default=None, help="Image or video file to flatten for verification.") - parser.add_argument("--output", default="unrolled.png", help="Flattened verification image output.") - parser.add_argument("--out-width", type=int, default=1024) - parser.add_argument("--out-height", type=int, default=1024) - parser.add_argument("--frame-index", type=int, default=0) - parser.add_argument("--no-flip-v", action="store_true", help="Keep UV V increasing downward.") - parser.add_argument("--reuse-remap", action="store_true", help="Load an existing remap instead of rebuilding it.") - args = parser.parse_args() - - remap_path = args.remap or os.path.join(args.data, "flatten_remap.npz") - if args.reuse_remap and os.path.exists(remap_path): - remap = load_remap(remap_path) - else: - remap = build_uv_remap( - load_exported_data(args.data), - out_width=args.out_width, - out_height=args.out_height, - flip_v=not args.no_flip_v, - ) - save_remap(remap_path, remap) - valid_ratio = float(np.mean(remap["valid_mask"])) - print(f"Saved remap: {remap_path} valid={valid_ratio:.1%}") - - if args.capture: - frame = load_capture_frame(args.capture, args.frame_index) - flat = flatten_frame(frame, remap) - cv2.imwrite(args.output, flat) - print(f"Saved flattened image: {args.output}") - - -if __name__ == "__main__": - main() diff --git a/tools/3d_patch_generation_blender/unroll_realtime.py b/tools/3d_patch_generation_blender/unroll_realtime.py new file mode 100644 index 0000000..b580c25 --- /dev/null +++ b/tools/3d_patch_generation_blender/unroll_realtime.py @@ -0,0 +1,821 @@ +#!/usr/bin/env python3 +""" +Real-time UV-based perspective correction for the exported finger mesh. + +This version replaces the previous cylindrical approximation with a remap built +from the mesh UV parameterization. The remap is precomputed once and then +applied to each live frame with cv2.remap(). +""" + +import argparse +import json +import os +import sys +import time + +import cv2 +import numpy as np +import orisys + + +def resize_to_height(image, target_height): + """Resize an image to a target height while preserving aspect ratio.""" + if image.shape[0] == target_height: + return image + scale = target_height / float(image.shape[0]) + target_width = max(1, int(round(image.shape[1] * scale))) + return cv2.resize(image, (target_width, target_height), interpolation=cv2.INTER_AREA) + + +def resize_to_fit(image, target_width, target_height): + """Resize with aspect ratio preserved and pad to exact tile size.""" + h, w = image.shape[:2] + if h <= 0 or w <= 0: + return np.zeros((target_height, target_width, 3), dtype=np.uint8) + + scale = min(target_width / float(w), target_height / float(h)) + new_w = max(1, int(round(w * scale))) + new_h = max(1, int(round(h * scale))) + resized = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_AREA) + + if resized.ndim == 2: + resized = cv2.cvtColor(resized, cv2.COLOR_GRAY2BGR) + elif resized.ndim == 3 and resized.shape[2] == 1: + resized = cv2.cvtColor(resized[:, :, 0], cv2.COLOR_GRAY2BGR) + + canvas = np.zeros((target_height, target_width, 3), dtype=np.uint8) + y0 = (target_height - new_h) // 2 + x0 = (target_width - new_w) // 2 + canvas[y0:y0 + new_h, x0:x0 + new_w] = resized + return canvas + + +def add_panel_title(image, title): + """Overlay a simple title onto a display tile.""" + out = image.copy() + cv2.rectangle(out, (0, 0), (out.shape[1], 36), (0, 0, 0), thickness=-1) + cv2.putText(out, title, (12, 24), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2, cv2.LINE_AA) + return out + + +def make_tiled_dashboard(panels, tile_width=640, tile_height=360): + """Arrange four panels into a 2x2 dashboard.""" + tiles = [ + add_panel_title(resize_to_fit(image, tile_width, tile_height), title) + for title, image in panels + ] + top_row = np.hstack(tiles[:2]) + bottom_row = np.hstack(tiles[2:4]) + return np.vstack([top_row, bottom_row]) + + +def load_calib_json(json_path): + with open(json_path, "r", encoding="utf-8") as f: + calib = json.load(f) + K = np.array(calib["camera_matrix"], dtype=np.float64) + dist = np.array(calib["distortion"], dtype=np.float64).flatten() + img_size = tuple(int(v) for v in calib["image_size"]) + return K, dist, img_size + + +def load_exported_data(mesh_path, intrinsics_path, pose_path): + mesh = np.load(mesh_path) + cam = np.load(intrinsics_path) + return { + "vertices": mesh["vertices"].astype(np.float64), + "faces": mesh["faces"].astype(np.int32), + "uv_data": mesh["uv_data"].astype(np.float64), + "loop_to_vertex": mesh["loop_to_vertex"].astype(np.int32), + "K": cam["K"].astype(np.float64), + "width": int(cam["width"]), + "height": int(cam["height"]), + "world_to_cam": np.load(pose_path).astype(np.float64), + } + + +def face_uvs_from_loop_data(faces, uv_data, loop_to_vertex): + if len(uv_data) == 0: + raise ValueError("mesh.npz does not contain UV data; UV flattening is unavailable.") + if len(uv_data) != len(faces) * 3: + raise ValueError("Expected a triangulated mesh with exactly 3 UV loops per face.") + + face_loop_vertices = loop_to_vertex.reshape(-1, 3) + if face_loop_vertices.shape != faces.shape or not np.array_equal(face_loop_vertices, faces): + raise ValueError("UV loop order does not match exported face vertex order.") + return uv_data.reshape(-1, 3, 2) + + +def project_world_to_image(points_world, K, world_to_cam): + """Project world points into the image using Blender/OpenCV-compatible intrinsics.""" + points_h = np.column_stack((points_world, np.ones(len(points_world), dtype=np.float64))) + points_cam = (world_to_cam @ points_h.T).T[:, :3] + z = -points_cam[:, 2] + x = K[0, 0] * (points_cam[:, 0] / z) + K[0, 2] + y = K[1, 1] * (points_cam[:, 1] / z) + K[1, 2] + return np.column_stack((x, y)), z + + +def barycentric_grid(px, py, tri): + a, b, c = tri + denom = (b[1] - c[1]) * (a[0] - c[0]) + (c[0] - b[0]) * (a[1] - c[1]) + if abs(denom) < 1e-12: + return None + w0 = ((b[1] - c[1]) * (px - c[0]) + (c[0] - b[0]) * (py - c[1])) / denom + w1 = ((c[1] - a[1]) * (px - c[0]) + (a[0] - c[0]) * (py - c[1])) / denom + w2 = 1.0 - w0 - w1 + return w0, w1, w2 + + +def triangle_edge_lengths_3d(tri): + return np.array( + [ + np.linalg.norm(tri[1] - tri[0]), + np.linalg.norm(tri[2] - tri[1]), + np.linalg.norm(tri[0] - tri[2]), + ], + dtype=np.float64, + ) + + +def triangle_edge_lengths_2d(tri): + return np.array( + [ + np.linalg.norm(tri[1] - tri[0]), + np.linalg.norm(tri[2] - tri[1]), + np.linalg.norm(tri[0] - tri[2]), + ], + dtype=np.float64, + ) + + +def triangle_area_3d(tri): + return 0.5 * np.linalg.norm(np.cross(tri[1] - tri[0], tri[2] - tri[0])) + + +def triangle_area_2d(tri): + return 0.5 * abs( + (tri[1, 0] - tri[0, 0]) * (tri[2, 1] - tri[0, 1]) + - (tri[1, 1] - tri[0, 1]) * (tri[2, 0] - tri[0, 0]) + ) + + +def summarize_uv_region(relative_scale, anisotropy, mask): + mask_scale = mask & np.isfinite(relative_scale) & (relative_scale > 0) + mask_aniso = mask & np.isfinite(anisotropy) & (anisotropy > 0) + if not np.any(mask_scale): + return { + "face_count": 0, + "relative_scale_p05": float("nan"), + "relative_scale_p50": float("nan"), + "relative_scale_p95": float("nan"), + "relative_scale_min": float("nan"), + "relative_scale_max": float("nan"), + "anisotropy_p50": float("nan"), + "anisotropy_p95": float("nan"), + "anisotropy_max": float("nan"), + } + return { + "face_count": int(np.count_nonzero(mask_scale)), + "relative_scale_p05": float(np.percentile(relative_scale[mask_scale], 5)), + "relative_scale_p50": float(np.percentile(relative_scale[mask_scale], 50)), + "relative_scale_p95": float(np.percentile(relative_scale[mask_scale], 95)), + "relative_scale_min": float(np.min(relative_scale[mask_scale])), + "relative_scale_max": float(np.max(relative_scale[mask_scale])), + "anisotropy_p50": float(np.percentile(anisotropy[mask_aniso], 50)) if np.any(mask_aniso) else float("nan"), + "anisotropy_p95": float(np.percentile(anisotropy[mask_aniso], 95)) if np.any(mask_aniso) else float("nan"), + "anisotropy_max": float(np.max(anisotropy[mask_aniso])) if np.any(mask_aniso) else float("nan"), + } + + +def compute_uv_distortion_metrics(data, edge_band=0.12): + """Estimate local UV stretch/compression and anisotropy per face.""" + faces = data["faces"] + vertices = data["vertices"] + face_uvs = face_uvs_from_loop_data(faces, data["uv_data"], data["loop_to_vertex"]) + + local_scale = np.full(len(faces), np.nan, dtype=np.float64) + relative_scale = np.full(len(faces), np.nan, dtype=np.float64) + anisotropy = np.full(len(faces), np.nan, dtype=np.float64) + uv_centers = np.full((len(faces), 2), np.nan, dtype=np.float64) + edge_mask = np.zeros(len(faces), dtype=bool) + + for idx, (face, uv_tri) in enumerate(zip(faces, face_uvs)): + tri_3d = vertices[face] + tri_uv = uv_tri + uv_centers[idx] = np.mean(tri_uv, axis=0) + + edges_3d = triangle_edge_lengths_3d(tri_3d) + edges_uv = triangle_edge_lengths_2d(tri_uv) + valid_edges = edges_uv > 1e-9 + if np.count_nonzero(valid_edges) < 2: + continue + + edge_scales = edges_3d[valid_edges] / edges_uv[valid_edges] + local_scale[idx] = float(np.exp(np.mean(np.log(np.clip(edge_scales, 1e-12, None))))) + anisotropy[idx] = float(np.max(edge_scales) / max(np.min(edge_scales), 1e-12)) + + area_uv = triangle_area_2d(tri_uv) + area_3d = triangle_area_3d(tri_3d) + if area_uv > 1e-12 and area_3d > 1e-12: + local_scale[idx] = float(np.sqrt(area_3d / area_uv)) + + valid_scale = np.isfinite(local_scale) & (local_scale > 0) + if not np.any(valid_scale): + raise RuntimeError("Could not compute UV distortion metrics; UV triangles may be degenerate.") + + median_scale = float(np.median(local_scale[valid_scale])) + relative_scale[valid_scale] = local_scale[valid_scale] / median_scale + min_edge_distance = np.min( + np.column_stack( + [ + uv_centers[:, 0], + 1.0 - uv_centers[:, 0], + uv_centers[:, 1], + 1.0 - uv_centers[:, 1], + ] + ), + axis=1, + ) + edge_mask = min_edge_distance < edge_band + middle_mask = ~edge_mask + + valid_aniso = np.isfinite(anisotropy) & (anisotropy > 0) + stats = { + "median_mm_per_uv": median_scale, + "relative_scale_p05": float(np.percentile(relative_scale[valid_scale], 5)), + "relative_scale_p50": float(np.percentile(relative_scale[valid_scale], 50)), + "relative_scale_p95": float(np.percentile(relative_scale[valid_scale], 95)), + "relative_scale_min": float(np.min(relative_scale[valid_scale])), + "relative_scale_max": float(np.max(relative_scale[valid_scale])), + "anisotropy_p50": float(np.percentile(anisotropy[valid_aniso], 50)) if np.any(valid_aniso) else float("nan"), + "anisotropy_p95": float(np.percentile(anisotropy[valid_aniso], 95)) if np.any(valid_aniso) else float("nan"), + "anisotropy_max": float(np.max(anisotropy[valid_aniso])) if np.any(valid_aniso) else float("nan"), + "edge_band": float(edge_band), + } + return { + "relative_scale": relative_scale, + "anisotropy": anisotropy, + "uv_centers": uv_centers, + "edge_mask": edge_mask, + "middle_mask": middle_mask, + "stats": stats, + "middle_stats": summarize_uv_region(relative_scale, anisotropy, middle_mask), + "edge_stats": summarize_uv_region(relative_scale, anisotropy, edge_mask), + } + + +def recommend_edge_expand(distortion, max_expand=3.0): + """Suggest how much more UV space to allocate to the edge band.""" + middle_p95 = distortion["middle_stats"]["relative_scale_p95"] + edge_p95 = distortion["edge_stats"]["relative_scale_p95"] + if not np.isfinite(middle_p95) or not np.isfinite(edge_p95) or middle_p95 <= 0: + return 1.0 + # Use a square-root response so large edge distortion does not over-expand. + return float(np.clip(np.sqrt(edge_p95 / middle_p95), 1.0, max_expand)) + + +def warp_uv_axis(values, edge_band, edge_expand): + """Allocate more normalized output span to UV edge regions.""" + u = np.asarray(values, dtype=np.float64) + e = float(np.clip(edge_band, 0.0, 0.49)) + s = float(max(edge_expand, 1.0)) + if e <= 0.0 or s <= 1.0: + return u.copy() + + middle = 1.0 - 2.0 * e + total = middle + 2.0 * e * s + out = np.empty_like(u) + + left = u < e + right = u > (1.0 - e) + center = ~(left | right) + + out[left] = (u[left] * s) / total + out[center] = (e * s + (u[center] - e)) / total + out[right] = (e * s + middle + (u[right] - (1.0 - e)) * s) / total + return out + + +def uv_triangle_to_image(uv_tri, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0): + """Map UV triangle into output image coordinates with optional edge expansion.""" + tri = np.empty_like(uv_tri) + u_warp = warp_uv_axis(uv_tri[:, 0], edge_band=edge_band, edge_expand=edge_expand) + v_warp = warp_uv_axis(uv_tri[:, 1], edge_band=edge_band, edge_expand=edge_expand) + tri[:, 0] = u_warp * (out_width - 1) + tri[:, 1] = ((1.0 - v_warp) if flip_v else v_warp) * (out_height - 1) + return tri + + +def build_uv_remap( + data, + source_K, + source_width, + source_height, + out_width, + out_height, + flip_v=True, + edge_band=0.12, + edge_expand=1.0, +): + faces = data["faces"] + vertices = data["vertices"] + face_uvs = face_uvs_from_loop_data(faces, data["uv_data"], data["loop_to_vertex"]) + + map_x = np.full((out_height, out_width), -1.0, dtype=np.float32) + map_y = np.full((out_height, out_width), -1.0, dtype=np.float32) + valid_mask = np.zeros((out_height, out_width), dtype=bool) + depth_buffer = np.full((out_height, out_width), np.inf, dtype=np.float64) + + print("Building UV remap ...") + t_start = time.time() + for face_idx, (face, uv_tri) in enumerate(zip(faces, face_uvs)): + tri = uv_triangle_to_image( + uv_tri, + out_width, + out_height, + flip_v=flip_v, + edge_band=edge_band, + edge_expand=edge_expand, + ) + + x0 = max(0, int(np.floor(np.min(tri[:, 0])))) + x1 = min(out_width - 1, int(np.ceil(np.max(tri[:, 0])))) + y0 = max(0, int(np.floor(np.min(tri[:, 1])))) + y1 = min(out_height - 1, int(np.ceil(np.max(tri[:, 1])))) + if x1 < x0 or y1 < y0: + continue + + yy, xx = np.mgrid[y0:y1 + 1, x0:x1 + 1] + bary = barycentric_grid(xx + 0.5, yy + 0.5, tri) + if bary is None: + continue + + w0, w1, w2 = bary + inside = (w0 >= -1e-6) & (w1 >= -1e-6) & (w2 >= -1e-6) + if not np.any(inside): + continue + + weights = np.stack((w0[inside], w1[inside], w2[inside]), axis=1) + points_world = weights @ vertices[face] + pixels, depth = project_world_to_image(points_world, source_K, data["world_to_cam"]) + + src_x = pixels[:, 0] + src_y = pixels[:, 1] + target_y = yy[inside] + target_x = xx[inside] + valid = ( + (depth > 0) + & (src_x >= 0) + & (src_x <= source_width - 1) + & (src_y >= 0) + & (src_y <= source_height - 1) + & (depth < depth_buffer[target_y, target_x]) + ) + if not np.any(valid): + continue + + tx = target_x[valid] + ty = target_y[valid] + map_x[ty, tx] = src_x[valid].astype(np.float32) + map_y[ty, tx] = src_y[valid].astype(np.float32) + valid_mask[ty, tx] = True + depth_buffer[ty, tx] = depth[valid] + + if (face_idx % 200) == 0: + print(f" face {face_idx}/{len(faces)}") + + print(f"Remap build took {time.time() - t_start:.1f} s") + return { + "map_x": map_x, + "map_y": map_y, + "valid_mask": valid_mask, + "source_width": np.array(source_width, dtype=np.int32), + "source_height": np.array(source_height, dtype=np.int32), + "out_width": np.array(out_width, dtype=np.int32), + "out_height": np.array(out_height, dtype=np.int32), + "flip_v": np.array(int(flip_v), dtype=np.int32), + "edge_band": np.array(edge_band, dtype=np.float64), + "edge_expand": np.array(edge_expand, dtype=np.float64), + "projection_K": source_K.astype(np.float64), + "method": np.array("uv"), + } + + +def remap_valid_ratio(remap): + return float(np.mean(remap["valid_mask"])) + + +def load_cached_remap(path): + data = np.load(path) + return {key: data[key] for key in data.files} + + +def cache_matches(remap, source_K, source_width, source_height, out_width, out_height, flip_v, edge_band, edge_expand): + required = {"map_x", "map_y", "valid_mask", "source_width", "source_height", "out_width", "out_height"} + if not required.issubset(remap.keys()): + return False + if remap["map_x"].shape != (out_height, out_width): + return False + if int(remap["source_width"]) != source_width or int(remap["source_height"]) != source_height: + return False + if int(remap["out_width"]) != out_width or int(remap["out_height"]) != out_height: + return False + if "flip_v" in remap and int(remap["flip_v"]) != int(flip_v): + return False + if "edge_band" in remap and not np.isclose(float(remap["edge_band"]), float(edge_band), rtol=1e-6, atol=1e-6): + return False + if "edge_expand" in remap and not np.isclose(float(remap["edge_expand"]), float(edge_expand), rtol=1e-6, atol=1e-6): + return False + if "projection_K" in remap and not np.allclose(remap["projection_K"], source_K, rtol=1e-6, atol=1e-6): + return False + return True + + +def flatten_frame(frame, remap, border_value=0): + flat = cv2.remap( + frame, + remap["map_x"].astype(np.float32), + remap["map_y"].astype(np.float32), + interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=border_value, + ) + flat = flat.copy() + flat[~remap["valid_mask"].astype(bool)] = border_value + return flat + + +def make_uv_debug_image(data, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0): + """Create a UV-domain reference image to verify orientation of the remap.""" + face_uvs = face_uvs_from_loop_data(data["faces"], data["uv_data"], data["loop_to_vertex"]) + uv_img = np.zeros((out_height, out_width, 3), dtype=np.uint8) + label_color = (255, 255, 255) + + for uv_tri in face_uvs: + tri = uv_triangle_to_image( + uv_tri, + out_width, + out_height, + flip_v=flip_v, + edge_band=edge_band, + edge_expand=edge_expand, + ) + pts = np.round(tri).astype(np.int32) + + mask = np.zeros((out_height, out_width), dtype=np.uint8) + cv2.fillConvexPoly(mask, pts, 255) + + x0 = max(0, int(np.floor(np.min(tri[:, 0])))) + x1 = min(out_width - 1, int(np.ceil(np.max(tri[:, 0])))) + y0 = max(0, int(np.floor(np.min(tri[:, 1])))) + y1 = min(out_height - 1, int(np.ceil(np.max(tri[:, 1])))) + if x1 < x0 or y1 < y0: + continue + + yy, xx = np.mgrid[y0:y1 + 1, x0:x1 + 1] + inside = mask[y0:y1 + 1, x0:x1 + 1] > 0 + if not np.any(inside): + continue + + bary = barycentric_grid(xx + 0.5, yy + 0.5, tri) + if bary is None: + continue + w0, w1, w2 = bary + valid = inside & (w0 >= -1e-6) & (w1 >= -1e-6) & (w2 >= -1e-6) + if not np.any(valid): + continue + + uv = ( + w0[valid, None] * uv_tri[0] + + w1[valid, None] * uv_tri[1] + + w2[valid, None] * uv_tri[2] + ) + colors = np.zeros((uv.shape[0], 3), dtype=np.uint8) + colors[:, 0] = np.clip(uv[:, 0] * 255.0, 0, 255).astype(np.uint8) + colors[:, 1] = np.clip(uv[:, 1] * 255.0, 0, 255).astype(np.uint8) + colors[:, 2] = np.clip((1.0 - 0.5 * (uv[:, 0] + uv[:, 1])) * 255.0, 0, 255).astype(np.uint8) + uv_img[yy[valid], xx[valid]] = colors + + cv2.polylines(uv_img, [pts.reshape(-1, 1, 2)], isClosed=True, color=(255, 255, 255), thickness=1) + + cv2.arrowedLine(uv_img, (20, out_height - 20), (min(out_width - 20, 140), out_height - 20), (0, 0, 255), 2, tipLength=0.15) + cv2.arrowedLine(uv_img, (20, out_height - 20), (20, max(20, out_height - 140)), (0, 255, 0), 2, tipLength=0.15) + cv2.putText(uv_img, "U", (min(out_width - 30, 145), out_height - 28), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2, cv2.LINE_AA) + cv2.putText(uv_img, "V", (28, max(30, out_height - 145)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2, cv2.LINE_AA) + cv2.putText(uv_img, "UV Map", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.9, label_color, 2, cv2.LINE_AA) + if edge_expand > 1.0: + cv2.putText(uv_img, f"Edge expand {edge_expand:.2f}x", (20, 64), cv2.FONT_HERSHEY_SIMPLEX, 0.55, label_color, 2, cv2.LINE_AA) + return uv_img + + +def make_uv_distortion_image(data, distortion, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0): + """Render UV distortion heatmap; red means more 3D surface per UV area.""" + face_uvs = face_uvs_from_loop_data(data["faces"], data["uv_data"], data["loop_to_vertex"]) + uv_img = np.zeros((out_height, out_width, 3), dtype=np.uint8) + rel = distortion["relative_scale"] + edge_mask = distortion["edge_mask"] + valid_rel = np.isfinite(rel) & (rel > 0) + if not np.any(valid_rel): + return uv_img + + log_rel = np.log2(rel[valid_rel]) + clip = max(0.25, float(np.percentile(np.abs(log_rel), 95))) + + for idx, uv_tri in enumerate(face_uvs): + tri = uv_triangle_to_image( + uv_tri, + out_width, + out_height, + flip_v=flip_v, + edge_band=edge_band, + edge_expand=edge_expand, + ) + pts = np.round(tri).astype(np.int32) + + if not np.isfinite(rel[idx]) or rel[idx] <= 0: + color = (64, 64, 64) + else: + value = np.clip((np.log2(rel[idx]) + clip) / (2.0 * clip), 0.0, 1.0) + color_idx = np.uint8(round(value * 255.0)) + color = tuple(int(c) for c in cv2.applyColorMap(np.array([[color_idx]], dtype=np.uint8), cv2.COLORMAP_JET)[0, 0]) + + cv2.fillConvexPoly(uv_img, pts, color) + edge_color = (255, 255, 255) if edge_mask[idx] else (180, 180, 180) + edge_thickness = 2 if edge_mask[idx] else 1 + cv2.polylines(uv_img, [pts.reshape(-1, 1, 2)], isClosed=True, color=edge_color, thickness=edge_thickness) + + cv2.putText(uv_img, "UV Distortion", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 255), 2, cv2.LINE_AA) + cv2.putText(uv_img, "Blue: denser UV Red: sparser UV", (20, 64), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (220, 220, 220), 2, cv2.LINE_AA) + cv2.putText(uv_img, "White outlines: edge-band faces", (20, 92), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (220, 220, 220), 2, cv2.LINE_AA) + middle_stats = distortion["middle_stats"] + edge_stats = distortion["edge_stats"] + cv2.putText( + uv_img, + f"middle p05/p95: {middle_stats['relative_scale_p05']:.2f}x / {middle_stats['relative_scale_p95']:.2f}x", + (20, 120), + cv2.FONT_HERSHEY_SIMPLEX, + 0.55, + (220, 220, 220), + 2, + cv2.LINE_AA, + ) + cv2.putText( + uv_img, + f"edge p05/p95: {edge_stats['relative_scale_p05']:.2f}x / {edge_stats['relative_scale_p95']:.2f}x", + (20, 148), + cv2.FONT_HERSHEY_SIMPLEX, + 0.55, + (220, 220, 220), + 2, + cv2.LINE_AA, + ) + return uv_img + + +def open_source(video_path, camera_id): + return cv2.VideoCapture(video_path) if video_path else cv2.VideoCapture(camera_id) + + +def parse_video_source(value): + try: + return int(value) + except (TypeError, ValueError): + return value + + +def align_frame_to_uv_map(frame_bgr, flip_vertical=True): + """Match live-frame orientation to the Blender-exported UV remap convention.""" + if not flip_vertical: + return frame_bgr + return np.flipud(frame_bgr).copy() + + +def main(): + parser = argparse.ArgumentParser(description="Real-time UV-based finger unrolling") + parser.add_argument("--mesh", required=True, help="Exported mesh.npz path") + parser.add_argument("--K", dest="intrinsics_path", required=True, help="Exported camera_intrinsics.npz path") + parser.add_argument("--pose", required=True, help="Exported world_to_cam.npy path") + parser.add_argument("--calib", default=None, help="OpenCV calibration JSON for the live camera") + # With a 12% UV edge band on each side, only about 76% of each axis is the + # lower-distortion middle region. Use a slightly larger full remap so the + # useful center keeps roughly the same sampling density as the prior 506x550. + parser.add_argument("--out_width", type=int, default=672) + parser.add_argument("--out_height", type=int, default=728) + parser.add_argument("--camera_id", type=int, default=0) + parser.add_argument("--video", help="Video file path (overrides camera)") + parser.add_argument("--map_cache", default="unroll_map_uv.npz") + parser.add_argument("--no_flip_v", action="store_true", help="Keep UV V increasing downward") + parser.add_argument("--no_flip_source_vertical", action="store_true", help="Do not vertically flip the live source before UV remap") + parser.add_argument("--edge_band", type=float, default=0.12, help="Normalized UV band near the boundary treated as high-distortion edge") + parser.add_argument("--edge_expand", type=float, default=0.0, help="Extra UV allocation for edge band; 0 chooses automatically from distortion stats") + parser.add_argument("--sdk_config", default="./config/dd02-ov.json", help="Orisys SDK config path/name") + parser.add_argument("--sdk_source", default=None, help="Dummy source used to initialize Sensor; defaults to --video or --camera_id") + parser.add_argument("--verbose", action="store_true", help="Enable verbose SDK logs") + args = parser.parse_args() + + data = load_exported_data(args.mesh, args.intrinsics_path, args.pose) + flip_v = not args.no_flip_v + flip_source_vertical = not args.no_flip_source_vertical + distortion = compute_uv_distortion_metrics(data, edge_band=args.edge_band) + distortion_stats = distortion["stats"] + middle_stats = distortion["middle_stats"] + edge_stats = distortion["edge_stats"] + edge_expand = args.edge_expand if args.edge_expand > 1.0 else recommend_edge_expand(distortion) + print( + "UV distortion stats: " + f"median mm/uv={distortion_stats['median_mm_per_uv']:.3f}, " + f"relative scale p05/p95={distortion_stats['relative_scale_p05']:.2f}x/" + f"{distortion_stats['relative_scale_p95']:.2f}x, " + f"anisotropy p50/p95={distortion_stats['anisotropy_p50']:.2f}x/" + f"{distortion_stats['anisotropy_p95']:.2f}x" + ) + print(f"Using UV edge warp: band={args.edge_band:.2f}, expand={edge_expand:.2f}x") + print( + "UV middle-region stats: " + f"faces={middle_stats['face_count']}, " + f"relative scale p05/p95={middle_stats['relative_scale_p05']:.2f}x/" + f"{middle_stats['relative_scale_p95']:.2f}x, " + f"anisotropy p50/p95={middle_stats['anisotropy_p50']:.2f}x/" + f"{middle_stats['anisotropy_p95']:.2f}x" + ) + print( + "UV edge-band stats: " + f"faces={edge_stats['face_count']}, " + f"relative scale p05/p95={edge_stats['relative_scale_p05']:.2f}x/" + f"{edge_stats['relative_scale_p95']:.2f}x, " + f"anisotropy p50/p95={edge_stats['anisotropy_p50']:.2f}x/" + f"{edge_stats['anisotropy_p95']:.2f}x" + ) + + if args.calib: + source_K, dist_coeffs, img_size = load_calib_json(args.calib) + source_width, source_height = img_size + print(f"Calibration loaded. Live image size: {(source_width, source_height)}") + else: + source_K = data["K"] + dist_coeffs = None + source_width = data["width"] + source_height = data["height"] + print("No calibration file provided; using exported intrinsics for projection.") + + remap = None + if os.path.exists(args.map_cache): + cached = load_cached_remap(args.map_cache) + if cache_matches( + cached, + source_K=source_K, + source_width=source_width, + source_height=source_height, + out_width=args.out_width, + out_height=args.out_height, + flip_v=flip_v, + edge_band=args.edge_band, + edge_expand=edge_expand, + ): + remap = cached + print(f"Loaded cached UV remap. Valid ratio: {remap_valid_ratio(remap):.3f}") + else: + print("Cached UV remap is stale. Regenerating.") + + if remap is None: + remap = build_uv_remap( + data, + source_K=source_K, + source_width=source_width, + source_height=source_height, + out_width=args.out_width, + out_height=args.out_height, + flip_v=flip_v, + edge_band=args.edge_band, + edge_expand=edge_expand, + ) + valid_ratio = remap_valid_ratio(remap) + print(f"Generated UV remap valid ratio: {valid_ratio:.3f}") + if valid_ratio == 0.0: + raise RuntimeError("Generated UV remap contains no valid samples.") + np.savez_compressed(args.map_cache, **remap) + + uv_debug = make_uv_debug_image( + data, + out_width=args.out_width, + out_height=args.out_height, + flip_v=flip_v, + edge_band=args.edge_band, + edge_expand=edge_expand, + ) + uv_distortion = make_uv_distortion_image( + data, + distortion, + out_width=args.out_width, + out_height=args.out_height, + flip_v=flip_v, + edge_band=args.edge_band, + edge_expand=edge_expand, + ) + uv_reference = np.hstack( + [ + resize_to_height(uv_debug, args.out_height), + resize_to_height(uv_distortion, args.out_height), + ] + ) + + cap = open_source(args.video, args.camera_id) + if not cap.isOpened(): + print("Cannot open camera/video source") + sys.exit(1) + + cap.set(cv2.CAP_PROP_FRAME_WIDTH, source_width) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, source_height) + + sensor = orisys.Sensor( + "external", + config_name=args.sdk_config, + verbose=args.verbose, + mirror=False, + ) + + print("Press 'q' to quit.") + try: + while True: + ok, frame = cap.read() + if not ok: + break + + if frame.shape[1] != source_width or frame.shape[0] != source_height: + frame = cv2.resize(frame, (source_width, source_height), interpolation=cv2.INTER_LINEAR) + + if dist_coeffs is not None: + frame_source = cv2.undistort(frame, source_K, dist_coeffs) + else: + frame_source = frame + frame_source = align_frame_to_uv_map(frame_source, flip_vertical=flip_source_vertical) + + unrolled = flatten_frame(frame_source, remap, border_value=(0, 0, 0)) + + sensor.get_img(frame=unrolled, already_processed=True) + sensor.compute_deformation(check_motion=True, threshold=0) + sensor.compute_contact() + + fps, flow, vnormal, vshear, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = sensor.read_info( + sensor.info.FPS, + sensor.info.VRAW, + sensor.info.VNORMAL, + sensor.info.VSHEAR, + sensor.info.IMG, + sensor.info.CONTOUR, + sensor.info.CENTROID, + sensor.info.DEPTH, + sensor.info.FNORMAL, + sensor.info.FSHEARX, + sensor.info.FSHEARY, + ) + + arrows = orisys.util.draw_arrows( + img, + flow, + threshold=2, + grid_spacing=20, + arrow_scale=1.0, + ) + contact_img = orisys.util.draw_contact(img, contour, centroid) + + depth_norm = depth_map + if depth_norm.size > 0 and depth_norm.max() > depth_norm.min(): + depth_u8 = ((depth_norm - depth_norm.min()) / (depth_norm.max() - depth_norm.min()) * 255).astype(np.uint8) + else: + depth_u8 = np.zeros_like(depth_norm, dtype=np.uint8) + depth_colored = cv2.applyColorMap(depth_u8, cv2.COLORMAP_JET) + + dashboard = make_tiled_dashboard( + [ + ("Original", frame_source), + ("UV Map | Distortion", uv_reference), + ("UV Unrolled", unrolled), + ("Flow", arrows), + ], + tile_width=640, + tile_height=360, + ) + cv2.imshow("UV Unroll Dashboard", dashboard) + + print( + f"FPS={fps:.2f}, 法向力={fnormal:.4f}, 切向力X={fshearx:.4f}, " + f"切向力Y={fsheary:.4f}, 展开图尺寸={unrolled.shape[:2]}", + end="\r", + ) + + key = cv2.waitKey(1) & 0xFF + if key == ord("q"): + print("\n正在退出程序...") + break + if key == ord("r"): + sensor.reset() + print("\n追踪器已重置。") + finally: + cap.release() + sensor.disconnect() + cv2.destroyAllWindows() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/unroll_map_uv.npz b/unroll_map_uv.npz new file mode 100644 index 0000000..c16da64 Binary files /dev/null and b/unroll_map_uv.npz differ