orisys-tech/orisys-sdk
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update using angle-based unwarp for display but projection for printing

Browse Source
This commit is contained in:
ykcheungab
2026-06-11 17:07:39 +08:00
parent 4cff6a1625
commit 667bdf1961
11 changed files with 856 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/dd02-ov.json
+1 -1
View File
@@ -5,7 +5,7 @@
"camera": {
"resolution_raw": [1280, 720],
"fps": 30,
"resolution_stitch": [1280, 720]
"resolution_stitch": [672, 728]
},
"tracker": {
"adaptive": true,
src/orisys/sensor.py
+34 -7
View File
@@ -85,6 +85,8 @@ class Sensor:
if vid_src is None:
raise Exception("User must specify a video source.")
self._external_only = isinstance(vid_src, str) and vid_src.lower() == "external"
# Store settings
self.vid_src = vid_src
self.isstitch = isstitch
@@ -171,7 +173,7 @@ class Sensor:
self._is_live_camera = (
isinstance(self.vid_src, int)
or (isinstance(self.vid_src, str) and self.vid_src.startswith("/dev/video"))
)
) and not self._external_only
self._capture_thread = None
self._capture_running = False
self._latest_frame = None
@@ -179,8 +181,13 @@ class Sensor:
self._last_consumed_seq = -1
self._latest_frame_lock = threading.Lock()
# Setup video capture
self._open_video_source(override, box)
# Setup video capture unless this sensor is fed externally.
if self._external_only:
self.stitch_valid = False
if self.verbose:
print("Sensor initialized in external-frame mode.")
else:
self._open_video_source(override, box)
if self.verbose:
print(f" Video source: {vid_src}")
@@ -197,9 +204,19 @@ class Sensor:
# Public API - Core Methods (Most Frequently Used)
# =========================================================================
def get_img(self):
def get_img(self, frame=None, already_processed=False):
"""
Capture and preprocess a frame.
Parameters
----------
frame : np.ndarray | None
Optional externally supplied frame. When provided, the sensor skips
reading from its own video source and uses this frame instead.
already_processed : bool
If True, treat `frame` as the final BGR tactile image and skip the
configured frame processor. This is useful for pipelines that
generate a derived image, such as an unrolled tactile view.
Returns:
np.ndarray | None: Processed image, or None on capture failure.
@@ -207,7 +224,10 @@ class Sensor:
# Simple direct read - buffer size of 1 should minimize frame accumulation
# Timing measurement helps identify if capture or processing is the bottleneck
t_capture_start = time.perf_counter()
ret, frame = self._read_frame()
if frame is None:
ret, frame = self._read_frame()
else:
ret = frame is not None
t_capture_end = time.perf_counter()
self._last_capture_time = t_capture_end - t_capture_start
@@ -218,8 +238,15 @@ class Sensor:
# Start processing timing
t_processing_start = time.perf_counter()
self.frame = frame
img_bgr = self.frame_processor.process(frame)
self.frame = frame.copy() if isinstance(frame, np.ndarray) else frame
if already_processed:
img_bgr = frame
if img_bgr.ndim == 2:
img_bgr = cv2.cvtColor(img_bgr, cv2.COLOR_GRAY2BGR)
elif img_bgr.ndim == 3 and img_bgr.shape[2] == 1:
img_bgr = cv2.cvtColor(img_bgr[:, :, 0], cv2.COLOR_GRAY2BGR)
else:
img_bgr = self.frame_processor.process(frame)
t_after_extract = time.perf_counter()
t_after_stitch = time.perf_counter()
tools/3d_patch_generation_blender/dd02c_0607.blend
BIN
View File
Binary file not shown.
tools/3d_patch_generation_blender/dd02c_0607.blend1
BIN
View File
Binary file not shown.
tools/3d_patch_generation_blender/dd02c_v3.blend
BIN
View File
Binary file not shown.
tools/3d_patch_generation_blender/dd02c_v3.blend1
BIN
View File
Binary file not shown.
tools/3d_patch_generation_blender/patch_texture.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 631 KiB

tools/3d_patch_generation_blender/patch_texture_0607_X80Y45.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 272 KiB

tools/3d_patch_generation_blender/unroll_finger.py
-217
View File
@@ -1,217 +0,0 @@
"""Flatten captured finger images into the mesh UV surface domain.
The generated remap is expensive to build but cheap to use: live code only
needs cv2.remap(frame, map_x, map_y, ...).
"""
import argparse
import os
import cv2
import numpy as np
def load_exported_data(data_dir):
mesh = np.load(os.path.join(data_dir, "mesh.npz"))
cam = np.load(os.path.join(data_dir, "camera_intrinsics.npz"))
return {
"vertices": mesh["vertices"].astype(np.float64),
"faces": mesh["faces"].astype(np.int32),
"uv_data": mesh["uv_data"].astype(np.float64),
"loop_to_vertex": mesh["loop_to_vertex"].astype(np.int32),
"K": cam["K"].astype(np.float64),
"width": int(cam["width"]),
"height": int(cam["height"]),
"world_to_cam": np.load(os.path.join(data_dir, "world_to_cam.npy")).astype(np.float64),
}
def face_uvs_from_loop_data(faces, uv_data, loop_to_vertex):
if len(uv_data) == 0:
raise ValueError("mesh.npz does not contain UV data; UV flattening is unavailable.")
if len(uv_data) != len(faces) * 3:
raise ValueError("Expected a triangulated mesh with exactly 3 UV loops per face.")
face_loop_vertices = loop_to_vertex.reshape(-1, 3)
if face_loop_vertices.shape != faces.shape or not np.array_equal(face_loop_vertices, faces):
raise ValueError("UV loop order does not match exported face vertex order.")
return uv_data.reshape(-1, 3, 2)
def project_world_to_image(points_world, K, world_to_cam):
points_h = np.column_stack((points_world, np.ones(len(points_world))))
points_cam = (world_to_cam @ points_h.T).T[:, :3]
z = -points_cam[:, 2]
x = K[0, 0] * (points_cam[:, 0] / z) + K[0, 2]
y = K[1, 1] * (points_cam[:, 1] / z) + K[1, 2]
return np.column_stack((x, y)), z
def barycentric_grid(px, py, tri):
a, b, c = tri
denom = (b[1] - c[1]) * (a[0] - c[0]) + (c[0] - b[0]) * (a[1] - c[1])
if abs(denom) < 1e-12:
return None
w0 = ((b[1] - c[1]) * (px - c[0]) + (c[0] - b[0]) * (py - c[1])) / denom
w1 = ((c[1] - a[1]) * (px - c[0]) + (a[0] - c[0]) * (py - c[1])) / denom
w2 = 1.0 - w0 - w1
return w0, w1, w2
def build_uv_remap(data, out_width=1024, out_height=1024, flip_v=True):
faces = data["faces"]
vertices = data["vertices"]
face_uvs = face_uvs_from_loop_data(faces, data["uv_data"], data["loop_to_vertex"])
map_x = np.full((out_height, out_width), -1.0, dtype=np.float32)
map_y = np.full((out_height, out_width), -1.0, dtype=np.float32)
flat_xyz = np.full((out_height, out_width, 3), np.nan, dtype=np.float32)
depth_buffer = np.full((out_height, out_width), np.inf, dtype=np.float64)
for face, uv_tri in zip(faces, face_uvs):
tri = np.empty_like(uv_tri)
tri[:, 0] = uv_tri[:, 0] * (out_width - 1)
tri[:, 1] = (1.0 - uv_tri[:, 1] if flip_v else uv_tri[:, 1]) * (out_height - 1)
x0 = max(0, int(np.floor(np.min(tri[:, 0]))))
x1 = min(out_width - 1, int(np.ceil(np.max(tri[:, 0]))))
y0 = max(0, int(np.floor(np.min(tri[:, 1]))))
y1 = min(out_height - 1, int(np.ceil(np.max(tri[:, 1]))))
if x1 < x0 or y1 < y0:
continue
yy, xx = np.mgrid[y0:y1 + 1, x0:x1 + 1]
bary = barycentric_grid(xx + 0.5, yy + 0.5, tri)
if bary is None:
continue
w0, w1, w2 = bary
inside = (w0 >= -1e-6) & (w1 >= -1e-6) & (w2 >= -1e-6)
if not np.any(inside):
continue
weights = np.stack((w0[inside], w1[inside], w2[inside]), axis=1)
points_world = weights @ vertices[face]
pixels, depth = project_world_to_image(points_world, data["K"], data["world_to_cam"])
src_x = pixels[:, 0]
src_y = pixels[:, 1]
target_y = yy[inside]
target_x = xx[inside]
valid = (
(depth > 0)
& (src_x >= 0)
& (src_x <= data["width"] - 1)
& (src_y >= 0)
& (src_y <= data["height"] - 1)
& (depth < depth_buffer[target_y, target_x])
)
if not np.any(valid):
continue
tx = target_x[valid]
ty = target_y[valid]
map_x[ty, tx] = src_x[valid].astype(np.float32)
map_y[ty, tx] = src_y[valid].astype(np.float32)
flat_xyz[ty, tx] = points_world[valid].astype(np.float32)
depth_buffer[ty, tx] = depth[valid]
valid_mask = map_x >= 0
return {
"map_x": map_x,
"map_y": map_y,
"valid_mask": valid_mask,
"flat_xyz": flat_xyz,
"source_width": np.array(data["width"], dtype=np.int32),
"source_height": np.array(data["height"], dtype=np.int32),
"out_width": np.array(out_width, dtype=np.int32),
"out_height": np.array(out_height, dtype=np.int32),
"method": np.array("uv"),
}
def save_remap(path, remap):
os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
np.savez_compressed(path, **remap)
def load_remap(path):
data = np.load(path)
return {key: data[key] for key in data.files}
def flatten_frame(frame, remap, border_value=0):
flat = cv2.remap(
frame,
remap["map_x"].astype(np.float32),
remap["map_y"].astype(np.float32),
interpolation=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_CONSTANT,
borderValue=border_value,
)
valid_mask = remap.get("valid_mask")
if valid_mask is not None:
flat = flat.copy()
flat[~valid_mask.astype(bool)] = border_value
return flat
def load_capture_frame(capture_path, frame_index=0, flip_vertical=True):
image = cv2.imread(capture_path, cv2.IMREAD_COLOR)
if image is not None:
if flip_vertical:
image = np.flipud(image).copy()
return image
cap = cv2.VideoCapture(capture_path, flip_vertical=True)
if not cap.isOpened():
raise FileNotFoundError(f"Could not read image or video: {capture_path}")
try:
if frame_index > 0:
cap.set(cv2.CAP_PROP_POS_FRAMES, frame_index)
ok, frame = cap.read()
if not ok:
raise RuntimeError(f"Could not read frame {frame_index} from {capture_path}")
return frame
finally:
cap.release()
def main():
parser = argparse.ArgumentParser(description="Build and apply a UV flattening remap for finger images.")
parser.add_argument("--data", required=True, help="Path to Blender exported_data directory.")
parser.add_argument("--remap", default=None, help="Output/input flatten_remap.npz path.")
parser.add_argument("--capture", default=None, help="Image or video file to flatten for verification.")
parser.add_argument("--output", default="unrolled.png", help="Flattened verification image output.")
parser.add_argument("--out-width", type=int, default=1024)
parser.add_argument("--out-height", type=int, default=1024)
parser.add_argument("--frame-index", type=int, default=0)
parser.add_argument("--no-flip-v", action="store_true", help="Keep UV V increasing downward.")
parser.add_argument("--reuse-remap", action="store_true", help="Load an existing remap instead of rebuilding it.")
args = parser.parse_args()
remap_path = args.remap or os.path.join(args.data, "flatten_remap.npz")
if args.reuse_remap and os.path.exists(remap_path):
remap = load_remap(remap_path)
else:
remap = build_uv_remap(
load_exported_data(args.data),
out_width=args.out_width,
out_height=args.out_height,
flip_v=not args.no_flip_v,
)
save_remap(remap_path, remap)
valid_ratio = float(np.mean(remap["valid_mask"]))
print(f"Saved remap: {remap_path} valid={valid_ratio:.1%}")
if args.capture:
frame = load_capture_frame(args.capture, args.frame_index)
flat = flatten_frame(frame, remap)
cv2.imwrite(args.output, flat)
print(f"Saved flattened image: {args.output}")
if __name__ == "__main__":
main()
tools/3d_patch_generation_blender/unroll_realtime.py
+821
View File
@@ -0,0 +1,821 @@
#!/usr/bin/env python3
"""
Real-time UV-based perspective correction for the exported finger mesh.
This version replaces the previous cylindrical approximation with a remap built
from the mesh UV parameterization. The remap is precomputed once and then
applied to each live frame with cv2.remap().
"""
import argparse
import json
import os
import sys
import time
import cv2
import numpy as np
import orisys
def resize_to_height(image, target_height):
"""Resize an image to a target height while preserving aspect ratio."""
if image.shape[0] == target_height:
return image
scale = target_height / float(image.shape[0])
target_width = max(1, int(round(image.shape[1] * scale)))
return cv2.resize(image, (target_width, target_height), interpolation=cv2.INTER_AREA)
def resize_to_fit(image, target_width, target_height):
"""Resize with aspect ratio preserved and pad to exact tile size."""
h, w = image.shape[:2]
if h <= 0 or w <= 0:
return np.zeros((target_height, target_width, 3), dtype=np.uint8)
scale = min(target_width / float(w), target_height / float(h))
new_w = max(1, int(round(w * scale)))
new_h = max(1, int(round(h * scale)))
resized = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_AREA)
if resized.ndim == 2:
resized = cv2.cvtColor(resized, cv2.COLOR_GRAY2BGR)
elif resized.ndim == 3 and resized.shape[2] == 1:
resized = cv2.cvtColor(resized[:, :, 0], cv2.COLOR_GRAY2BGR)
canvas = np.zeros((target_height, target_width, 3), dtype=np.uint8)
y0 = (target_height - new_h) // 2
x0 = (target_width - new_w) // 2
canvas[y0:y0 + new_h, x0:x0 + new_w] = resized
return canvas
def add_panel_title(image, title):
"""Overlay a simple title onto a display tile."""
out = image.copy()
cv2.rectangle(out, (0, 0), (out.shape[1], 36), (0, 0, 0), thickness=-1)
cv2.putText(out, title, (12, 24), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2, cv2.LINE_AA)
return out
def make_tiled_dashboard(panels, tile_width=640, tile_height=360):
"""Arrange four panels into a 2x2 dashboard."""
tiles = [
add_panel_title(resize_to_fit(image, tile_width, tile_height), title)
for title, image in panels
]
top_row = np.hstack(tiles[:2])
bottom_row = np.hstack(tiles[2:4])
return np.vstack([top_row, bottom_row])
def load_calib_json(json_path):
with open(json_path, "r", encoding="utf-8") as f:
calib = json.load(f)
K = np.array(calib["camera_matrix"], dtype=np.float64)
dist = np.array(calib["distortion"], dtype=np.float64).flatten()
img_size = tuple(int(v) for v in calib["image_size"])
return K, dist, img_size
def load_exported_data(mesh_path, intrinsics_path, pose_path):
mesh = np.load(mesh_path)
cam = np.load(intrinsics_path)
return {
"vertices": mesh["vertices"].astype(np.float64),
"faces": mesh["faces"].astype(np.int32),
"uv_data": mesh["uv_data"].astype(np.float64),
"loop_to_vertex": mesh["loop_to_vertex"].astype(np.int32),
"K": cam["K"].astype(np.float64),
"width": int(cam["width"]),
"height": int(cam["height"]),
"world_to_cam": np.load(pose_path).astype(np.float64),
}
def face_uvs_from_loop_data(faces, uv_data, loop_to_vertex):
if len(uv_data) == 0:
raise ValueError("mesh.npz does not contain UV data; UV flattening is unavailable.")
if len(uv_data) != len(faces) * 3:
raise ValueError("Expected a triangulated mesh with exactly 3 UV loops per face.")
face_loop_vertices = loop_to_vertex.reshape(-1, 3)
if face_loop_vertices.shape != faces.shape or not np.array_equal(face_loop_vertices, faces):
raise ValueError("UV loop order does not match exported face vertex order.")
return uv_data.reshape(-1, 3, 2)
def project_world_to_image(points_world, K, world_to_cam):
"""Project world points into the image using Blender/OpenCV-compatible intrinsics."""
points_h = np.column_stack((points_world, np.ones(len(points_world), dtype=np.float64)))
points_cam = (world_to_cam @ points_h.T).T[:, :3]
z = -points_cam[:, 2]
x = K[0, 0] * (points_cam[:, 0] / z) + K[0, 2]
y = K[1, 1] * (points_cam[:, 1] / z) + K[1, 2]
return np.column_stack((x, y)), z
def barycentric_grid(px, py, tri):
a, b, c = tri
denom = (b[1] - c[1]) * (a[0] - c[0]) + (c[0] - b[0]) * (a[1] - c[1])
if abs(denom) < 1e-12:
return None
w0 = ((b[1] - c[1]) * (px - c[0]) + (c[0] - b[0]) * (py - c[1])) / denom
w1 = ((c[1] - a[1]) * (px - c[0]) + (a[0] - c[0]) * (py - c[1])) / denom
w2 = 1.0 - w0 - w1
return w0, w1, w2
def triangle_edge_lengths_3d(tri):
return np.array(
[
np.linalg.norm(tri[1] - tri[0]),
np.linalg.norm(tri[2] - tri[1]),
np.linalg.norm(tri[0] - tri[2]),
],
dtype=np.float64,
)
def triangle_edge_lengths_2d(tri):
return np.array(
[
np.linalg.norm(tri[1] - tri[0]),
np.linalg.norm(tri[2] - tri[1]),
np.linalg.norm(tri[0] - tri[2]),
],
dtype=np.float64,
)
def triangle_area_3d(tri):
return 0.5 * np.linalg.norm(np.cross(tri[1] - tri[0], tri[2] - tri[0]))
def triangle_area_2d(tri):
return 0.5 * abs(
(tri[1, 0] - tri[0, 0]) * (tri[2, 1] - tri[0, 1])
- (tri[1, 1] - tri[0, 1]) * (tri[2, 0] - tri[0, 0])
)
def summarize_uv_region(relative_scale, anisotropy, mask):
mask_scale = mask & np.isfinite(relative_scale) & (relative_scale > 0)
mask_aniso = mask & np.isfinite(anisotropy) & (anisotropy > 0)
if not np.any(mask_scale):
return {
"face_count": 0,
"relative_scale_p05": float("nan"),
"relative_scale_p50": float("nan"),
"relative_scale_p95": float("nan"),
"relative_scale_min": float("nan"),
"relative_scale_max": float("nan"),
"anisotropy_p50": float("nan"),
"anisotropy_p95": float("nan"),
"anisotropy_max": float("nan"),
}
return {
"face_count": int(np.count_nonzero(mask_scale)),
"relative_scale_p05": float(np.percentile(relative_scale[mask_scale], 5)),
"relative_scale_p50": float(np.percentile(relative_scale[mask_scale], 50)),
"relative_scale_p95": float(np.percentile(relative_scale[mask_scale], 95)),
"relative_scale_min": float(np.min(relative_scale[mask_scale])),
"relative_scale_max": float(np.max(relative_scale[mask_scale])),
"anisotropy_p50": float(np.percentile(anisotropy[mask_aniso], 50)) if np.any(mask_aniso) else float("nan"),
"anisotropy_p95": float(np.percentile(anisotropy[mask_aniso], 95)) if np.any(mask_aniso) else float("nan"),
"anisotropy_max": float(np.max(anisotropy[mask_aniso])) if np.any(mask_aniso) else float("nan"),
}
def compute_uv_distortion_metrics(data, edge_band=0.12):
"""Estimate local UV stretch/compression and anisotropy per face."""
faces = data["faces"]
vertices = data["vertices"]
face_uvs = face_uvs_from_loop_data(faces, data["uv_data"], data["loop_to_vertex"])
local_scale = np.full(len(faces), np.nan, dtype=np.float64)
relative_scale = np.full(len(faces), np.nan, dtype=np.float64)
anisotropy = np.full(len(faces), np.nan, dtype=np.float64)
uv_centers = np.full((len(faces), 2), np.nan, dtype=np.float64)
edge_mask = np.zeros(len(faces), dtype=bool)
for idx, (face, uv_tri) in enumerate(zip(faces, face_uvs)):
tri_3d = vertices[face]
tri_uv = uv_tri
uv_centers[idx] = np.mean(tri_uv, axis=0)
edges_3d = triangle_edge_lengths_3d(tri_3d)
edges_uv = triangle_edge_lengths_2d(tri_uv)
valid_edges = edges_uv > 1e-9
if np.count_nonzero(valid_edges) < 2:
continue
edge_scales = edges_3d[valid_edges] / edges_uv[valid_edges]
local_scale[idx] = float(np.exp(np.mean(np.log(np.clip(edge_scales, 1e-12, None)))))
anisotropy[idx] = float(np.max(edge_scales) / max(np.min(edge_scales), 1e-12))
area_uv = triangle_area_2d(tri_uv)
area_3d = triangle_area_3d(tri_3d)
if area_uv > 1e-12 and area_3d > 1e-12:
local_scale[idx] = float(np.sqrt(area_3d / area_uv))
valid_scale = np.isfinite(local_scale) & (local_scale > 0)
if not np.any(valid_scale):
raise RuntimeError("Could not compute UV distortion metrics; UV triangles may be degenerate.")
median_scale = float(np.median(local_scale[valid_scale]))
relative_scale[valid_scale] = local_scale[valid_scale] / median_scale
min_edge_distance = np.min(
np.column_stack(
[
uv_centers[:, 0],
1.0 - uv_centers[:, 0],
uv_centers[:, 1],
1.0 - uv_centers[:, 1],
]
),
axis=1,
)
edge_mask = min_edge_distance < edge_band
middle_mask = ~edge_mask
valid_aniso = np.isfinite(anisotropy) & (anisotropy > 0)
stats = {
"median_mm_per_uv": median_scale,
"relative_scale_p05": float(np.percentile(relative_scale[valid_scale], 5)),
"relative_scale_p50": float(np.percentile(relative_scale[valid_scale], 50)),
"relative_scale_p95": float(np.percentile(relative_scale[valid_scale], 95)),
"relative_scale_min": float(np.min(relative_scale[valid_scale])),
"relative_scale_max": float(np.max(relative_scale[valid_scale])),
"anisotropy_p50": float(np.percentile(anisotropy[valid_aniso], 50)) if np.any(valid_aniso) else float("nan"),
"anisotropy_p95": float(np.percentile(anisotropy[valid_aniso], 95)) if np.any(valid_aniso) else float("nan"),
"anisotropy_max": float(np.max(anisotropy[valid_aniso])) if np.any(valid_aniso) else float("nan"),
"edge_band": float(edge_band),
}
return {
"relative_scale": relative_scale,
"anisotropy": anisotropy,
"uv_centers": uv_centers,
"edge_mask": edge_mask,
"middle_mask": middle_mask,
"stats": stats,
"middle_stats": summarize_uv_region(relative_scale, anisotropy, middle_mask),
"edge_stats": summarize_uv_region(relative_scale, anisotropy, edge_mask),
}
def recommend_edge_expand(distortion, max_expand=3.0):
"""Suggest how much more UV space to allocate to the edge band."""
middle_p95 = distortion["middle_stats"]["relative_scale_p95"]
edge_p95 = distortion["edge_stats"]["relative_scale_p95"]
if not np.isfinite(middle_p95) or not np.isfinite(edge_p95) or middle_p95 <= 0:
return 1.0
# Use a square-root response so large edge distortion does not over-expand.
return float(np.clip(np.sqrt(edge_p95 / middle_p95), 1.0, max_expand))
def warp_uv_axis(values, edge_band, edge_expand):
"""Allocate more normalized output span to UV edge regions."""
u = np.asarray(values, dtype=np.float64)
e = float(np.clip(edge_band, 0.0, 0.49))
s = float(max(edge_expand, 1.0))
if e <= 0.0 or s <= 1.0:
return u.copy()
middle = 1.0 - 2.0 * e
total = middle + 2.0 * e * s
out = np.empty_like(u)
left = u < e
right = u > (1.0 - e)
center = ~(left | right)
out[left] = (u[left] * s) / total
out[center] = (e * s + (u[center] - e)) / total
out[right] = (e * s + middle + (u[right] - (1.0 - e)) * s) / total
return out
def uv_triangle_to_image(uv_tri, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0):
"""Map UV triangle into output image coordinates with optional edge expansion."""
tri = np.empty_like(uv_tri)
u_warp = warp_uv_axis(uv_tri[:, 0], edge_band=edge_band, edge_expand=edge_expand)
v_warp = warp_uv_axis(uv_tri[:, 1], edge_band=edge_band, edge_expand=edge_expand)
tri[:, 0] = u_warp * (out_width - 1)
tri[:, 1] = ((1.0 - v_warp) if flip_v else v_warp) * (out_height - 1)
return tri
def build_uv_remap(
data,
source_K,
source_width,
source_height,
out_width,
out_height,
flip_v=True,
edge_band=0.12,
edge_expand=1.0,
):
faces = data["faces"]
vertices = data["vertices"]
face_uvs = face_uvs_from_loop_data(faces, data["uv_data"], data["loop_to_vertex"])
map_x = np.full((out_height, out_width), -1.0, dtype=np.float32)
map_y = np.full((out_height, out_width), -1.0, dtype=np.float32)
valid_mask = np.zeros((out_height, out_width), dtype=bool)
depth_buffer = np.full((out_height, out_width), np.inf, dtype=np.float64)
print("Building UV remap ...")
t_start = time.time()
for face_idx, (face, uv_tri) in enumerate(zip(faces, face_uvs)):
tri = uv_triangle_to_image(
uv_tri,
out_width,
out_height,
flip_v=flip_v,
edge_band=edge_band,
edge_expand=edge_expand,
)
x0 = max(0, int(np.floor(np.min(tri[:, 0]))))
x1 = min(out_width - 1, int(np.ceil(np.max(tri[:, 0]))))
y0 = max(0, int(np.floor(np.min(tri[:, 1]))))
y1 = min(out_height - 1, int(np.ceil(np.max(tri[:, 1]))))
if x1 < x0 or y1 < y0:
continue
yy, xx = np.mgrid[y0:y1 + 1, x0:x1 + 1]
bary = barycentric_grid(xx + 0.5, yy + 0.5, tri)
if bary is None:
continue
w0, w1, w2 = bary
inside = (w0 >= -1e-6) & (w1 >= -1e-6) & (w2 >= -1e-6)
if not np.any(inside):
continue
weights = np.stack((w0[inside], w1[inside], w2[inside]), axis=1)
points_world = weights @ vertices[face]
pixels, depth = project_world_to_image(points_world, source_K, data["world_to_cam"])
src_x = pixels[:, 0]
src_y = pixels[:, 1]
target_y = yy[inside]
target_x = xx[inside]
valid = (
(depth > 0)
& (src_x >= 0)
& (src_x <= source_width - 1)
& (src_y >= 0)
& (src_y <= source_height - 1)
& (depth < depth_buffer[target_y, target_x])
)
if not np.any(valid):
continue
tx = target_x[valid]
ty = target_y[valid]
map_x[ty, tx] = src_x[valid].astype(np.float32)
map_y[ty, tx] = src_y[valid].astype(np.float32)
valid_mask[ty, tx] = True
depth_buffer[ty, tx] = depth[valid]
if (face_idx % 200) == 0:
print(f" face {face_idx}/{len(faces)}")
print(f"Remap build took {time.time() - t_start:.1f} s")
return {
"map_x": map_x,
"map_y": map_y,
"valid_mask": valid_mask,
"source_width": np.array(source_width, dtype=np.int32),
"source_height": np.array(source_height, dtype=np.int32),
"out_width": np.array(out_width, dtype=np.int32),
"out_height": np.array(out_height, dtype=np.int32),
"flip_v": np.array(int(flip_v), dtype=np.int32),
"edge_band": np.array(edge_band, dtype=np.float64),
"edge_expand": np.array(edge_expand, dtype=np.float64),
"projection_K": source_K.astype(np.float64),
"method": np.array("uv"),
}
def remap_valid_ratio(remap):
return float(np.mean(remap["valid_mask"]))
def load_cached_remap(path):
data = np.load(path)
return {key: data[key] for key in data.files}
def cache_matches(remap, source_K, source_width, source_height, out_width, out_height, flip_v, edge_band, edge_expand):
required = {"map_x", "map_y", "valid_mask", "source_width", "source_height", "out_width", "out_height"}
if not required.issubset(remap.keys()):
return False
if remap["map_x"].shape != (out_height, out_width):
return False
if int(remap["source_width"]) != source_width or int(remap["source_height"]) != source_height:
return False
if int(remap["out_width"]) != out_width or int(remap["out_height"]) != out_height:
return False
if "flip_v" in remap and int(remap["flip_v"]) != int(flip_v):
return False
if "edge_band" in remap and not np.isclose(float(remap["edge_band"]), float(edge_band), rtol=1e-6, atol=1e-6):
return False
if "edge_expand" in remap and not np.isclose(float(remap["edge_expand"]), float(edge_expand), rtol=1e-6, atol=1e-6):
return False
if "projection_K" in remap and not np.allclose(remap["projection_K"], source_K, rtol=1e-6, atol=1e-6):
return False
return True
def flatten_frame(frame, remap, border_value=0):
flat = cv2.remap(
frame,
remap["map_x"].astype(np.float32),
remap["map_y"].astype(np.float32),
interpolation=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_CONSTANT,
borderValue=border_value,
)
flat = flat.copy()
flat[~remap["valid_mask"].astype(bool)] = border_value
return flat
def make_uv_debug_image(data, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0):
"""Create a UV-domain reference image to verify orientation of the remap."""
face_uvs = face_uvs_from_loop_data(data["faces"], data["uv_data"], data["loop_to_vertex"])
uv_img = np.zeros((out_height, out_width, 3), dtype=np.uint8)
label_color = (255, 255, 255)
for uv_tri in face_uvs:
tri = uv_triangle_to_image(
uv_tri,
out_width,
out_height,
flip_v=flip_v,
edge_band=edge_band,
edge_expand=edge_expand,
)
pts = np.round(tri).astype(np.int32)
mask = np.zeros((out_height, out_width), dtype=np.uint8)
cv2.fillConvexPoly(mask, pts, 255)
x0 = max(0, int(np.floor(np.min(tri[:, 0]))))
x1 = min(out_width - 1, int(np.ceil(np.max(tri[:, 0]))))
y0 = max(0, int(np.floor(np.min(tri[:, 1]))))
y1 = min(out_height - 1, int(np.ceil(np.max(tri[:, 1]))))
if x1 < x0 or y1 < y0:
continue
yy, xx = np.mgrid[y0:y1 + 1, x0:x1 + 1]
inside = mask[y0:y1 + 1, x0:x1 + 1] > 0
if not np.any(inside):
continue
bary = barycentric_grid(xx + 0.5, yy + 0.5, tri)
if bary is None:
continue
w0, w1, w2 = bary
valid = inside & (w0 >= -1e-6) & (w1 >= -1e-6) & (w2 >= -1e-6)
if not np.any(valid):
continue
uv = (
w0[valid, None] * uv_tri[0]
+ w1[valid, None] * uv_tri[1]
+ w2[valid, None] * uv_tri[2]
)
colors = np.zeros((uv.shape[0], 3), dtype=np.uint8)
colors[:, 0] = np.clip(uv[:, 0] * 255.0, 0, 255).astype(np.uint8)
colors[:, 1] = np.clip(uv[:, 1] * 255.0, 0, 255).astype(np.uint8)
colors[:, 2] = np.clip((1.0 - 0.5 * (uv[:, 0] + uv[:, 1])) * 255.0, 0, 255).astype(np.uint8)
uv_img[yy[valid], xx[valid]] = colors
cv2.polylines(uv_img, [pts.reshape(-1, 1, 2)], isClosed=True, color=(255, 255, 255), thickness=1)
cv2.arrowedLine(uv_img, (20, out_height - 20), (min(out_width - 20, 140), out_height - 20), (0, 0, 255), 2, tipLength=0.15)
cv2.arrowedLine(uv_img, (20, out_height - 20), (20, max(20, out_height - 140)), (0, 255, 0), 2, tipLength=0.15)
cv2.putText(uv_img, "U", (min(out_width - 30, 145), out_height - 28), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2, cv2.LINE_AA)
cv2.putText(uv_img, "V", (28, max(30, out_height - 145)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2, cv2.LINE_AA)
cv2.putText(uv_img, "UV Map", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.9, label_color, 2, cv2.LINE_AA)
if edge_expand > 1.0:
cv2.putText(uv_img, f"Edge expand {edge_expand:.2f}x", (20, 64), cv2.FONT_HERSHEY_SIMPLEX, 0.55, label_color, 2, cv2.LINE_AA)
return uv_img
def make_uv_distortion_image(data, distortion, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0):
"""Render UV distortion heatmap; red means more 3D surface per UV area."""
face_uvs = face_uvs_from_loop_data(data["faces"], data["uv_data"], data["loop_to_vertex"])
uv_img = np.zeros((out_height, out_width, 3), dtype=np.uint8)
rel = distortion["relative_scale"]
edge_mask = distortion["edge_mask"]
valid_rel = np.isfinite(rel) & (rel > 0)
if not np.any(valid_rel):
return uv_img
log_rel = np.log2(rel[valid_rel])
clip = max(0.25, float(np.percentile(np.abs(log_rel), 95)))
for idx, uv_tri in enumerate(face_uvs):
tri = uv_triangle_to_image(
uv_tri,
out_width,
out_height,
flip_v=flip_v,
edge_band=edge_band,
edge_expand=edge_expand,
)
pts = np.round(tri).astype(np.int32)
if not np.isfinite(rel[idx]) or rel[idx] <= 0:
color = (64, 64, 64)
else:
value = np.clip((np.log2(rel[idx]) + clip) / (2.0 * clip), 0.0, 1.0)
color_idx = np.uint8(round(value * 255.0))
color = tuple(int(c) for c in cv2.applyColorMap(np.array([[color_idx]], dtype=np.uint8), cv2.COLORMAP_JET)[0, 0])
cv2.fillConvexPoly(uv_img, pts, color)
edge_color = (255, 255, 255) if edge_mask[idx] else (180, 180, 180)
edge_thickness = 2 if edge_mask[idx] else 1
cv2.polylines(uv_img, [pts.reshape(-1, 1, 2)], isClosed=True, color=edge_color, thickness=edge_thickness)
cv2.putText(uv_img, "UV Distortion", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 255), 2, cv2.LINE_AA)
cv2.putText(uv_img, "Blue: denser UV Red: sparser UV", (20, 64), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (220, 220, 220), 2, cv2.LINE_AA)
cv2.putText(uv_img, "White outlines: edge-band faces", (20, 92), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (220, 220, 220), 2, cv2.LINE_AA)
middle_stats = distortion["middle_stats"]
edge_stats = distortion["edge_stats"]
cv2.putText(
uv_img,
f"middle p05/p95: {middle_stats['relative_scale_p05']:.2f}x / {middle_stats['relative_scale_p95']:.2f}x",
(20, 120),
cv2.FONT_HERSHEY_SIMPLEX,
0.55,
(220, 220, 220),
2,
cv2.LINE_AA,
)
cv2.putText(
uv_img,
f"edge p05/p95: {edge_stats['relative_scale_p05']:.2f}x / {edge_stats['relative_scale_p95']:.2f}x",
(20, 148),
cv2.FONT_HERSHEY_SIMPLEX,
0.55,
(220, 220, 220),
2,
cv2.LINE_AA,
)
return uv_img
def open_source(video_path, camera_id):
return cv2.VideoCapture(video_path) if video_path else cv2.VideoCapture(camera_id)
def parse_video_source(value):
try:
return int(value)
except (TypeError, ValueError):
return value
def align_frame_to_uv_map(frame_bgr, flip_vertical=True):
"""Match live-frame orientation to the Blender-exported UV remap convention."""
if not flip_vertical:
return frame_bgr
return np.flipud(frame_bgr).copy()
def main():
parser = argparse.ArgumentParser(description="Real-time UV-based finger unrolling")
parser.add_argument("--mesh", required=True, help="Exported mesh.npz path")
parser.add_argument("--K", dest="intrinsics_path", required=True, help="Exported camera_intrinsics.npz path")
parser.add_argument("--pose", required=True, help="Exported world_to_cam.npy path")
parser.add_argument("--calib", default=None, help="OpenCV calibration JSON for the live camera")
# With a 12% UV edge band on each side, only about 76% of each axis is the
# lower-distortion middle region. Use a slightly larger full remap so the
# useful center keeps roughly the same sampling density as the prior 506x550.
parser.add_argument("--out_width", type=int, default=672)
parser.add_argument("--out_height", type=int, default=728)
parser.add_argument("--camera_id", type=int, default=0)
parser.add_argument("--video", help="Video file path (overrides camera)")
parser.add_argument("--map_cache", default="unroll_map_uv.npz")
parser.add_argument("--no_flip_v", action="store_true", help="Keep UV V increasing downward")
parser.add_argument("--no_flip_source_vertical", action="store_true", help="Do not vertically flip the live source before UV remap")
parser.add_argument("--edge_band", type=float, default=0.12, help="Normalized UV band near the boundary treated as high-distortion edge")
parser.add_argument("--edge_expand", type=float, default=0.0, help="Extra UV allocation for edge band; 0 chooses automatically from distortion stats")
parser.add_argument("--sdk_config", default="./config/dd02-ov.json", help="Orisys SDK config path/name")
parser.add_argument("--sdk_source", default=None, help="Dummy source used to initialize Sensor; defaults to --video or --camera_id")
parser.add_argument("--verbose", action="store_true", help="Enable verbose SDK logs")
args = parser.parse_args()
data = load_exported_data(args.mesh, args.intrinsics_path, args.pose)
flip_v = not args.no_flip_v
flip_source_vertical = not args.no_flip_source_vertical
distortion = compute_uv_distortion_metrics(data, edge_band=args.edge_band)
distortion_stats = distortion["stats"]
middle_stats = distortion["middle_stats"]
edge_stats = distortion["edge_stats"]
edge_expand = args.edge_expand if args.edge_expand > 1.0 else recommend_edge_expand(distortion)
print(
"UV distortion stats: "
f"median mm/uv={distortion_stats['median_mm_per_uv']:.3f}, "
f"relative scale p05/p95={distortion_stats['relative_scale_p05']:.2f}x/"
f"{distortion_stats['relative_scale_p95']:.2f}x, "
f"anisotropy p50/p95={distortion_stats['anisotropy_p50']:.2f}x/"
f"{distortion_stats['anisotropy_p95']:.2f}x"
)
print(f"Using UV edge warp: band={args.edge_band:.2f}, expand={edge_expand:.2f}x")
print(
"UV middle-region stats: "
f"faces={middle_stats['face_count']}, "
f"relative scale p05/p95={middle_stats['relative_scale_p05']:.2f}x/"
f"{middle_stats['relative_scale_p95']:.2f}x, "
f"anisotropy p50/p95={middle_stats['anisotropy_p50']:.2f}x/"
f"{middle_stats['anisotropy_p95']:.2f}x"
)
print(
"UV edge-band stats: "
f"faces={edge_stats['face_count']}, "
f"relative scale p05/p95={edge_stats['relative_scale_p05']:.2f}x/"
f"{edge_stats['relative_scale_p95']:.2f}x, "
f"anisotropy p50/p95={edge_stats['anisotropy_p50']:.2f}x/"
f"{edge_stats['anisotropy_p95']:.2f}x"
)
if args.calib:
source_K, dist_coeffs, img_size = load_calib_json(args.calib)
source_width, source_height = img_size
print(f"Calibration loaded. Live image size: {(source_width, source_height)}")
else:
source_K = data["K"]
dist_coeffs = None
source_width = data["width"]
source_height = data["height"]
print("No calibration file provided; using exported intrinsics for projection.")
remap = None
if os.path.exists(args.map_cache):
cached = load_cached_remap(args.map_cache)
if cache_matches(
cached,
source_K=source_K,
source_width=source_width,
source_height=source_height,
out_width=args.out_width,
out_height=args.out_height,
flip_v=flip_v,
edge_band=args.edge_band,
edge_expand=edge_expand,
):
remap = cached
print(f"Loaded cached UV remap. Valid ratio: {remap_valid_ratio(remap):.3f}")
else:
print("Cached UV remap is stale. Regenerating.")
if remap is None:
remap = build_uv_remap(
data,
source_K=source_K,
source_width=source_width,
source_height=source_height,
out_width=args.out_width,
out_height=args.out_height,
flip_v=flip_v,
edge_band=args.edge_band,
edge_expand=edge_expand,
)
valid_ratio = remap_valid_ratio(remap)
print(f"Generated UV remap valid ratio: {valid_ratio:.3f}")
if valid_ratio == 0.0:
raise RuntimeError("Generated UV remap contains no valid samples.")
np.savez_compressed(args.map_cache, **remap)
uv_debug = make_uv_debug_image(
data,
out_width=args.out_width,
out_height=args.out_height,
flip_v=flip_v,
edge_band=args.edge_band,
edge_expand=edge_expand,
)
uv_distortion = make_uv_distortion_image(
data,
distortion,
out_width=args.out_width,
out_height=args.out_height,
flip_v=flip_v,
edge_band=args.edge_band,
edge_expand=edge_expand,
)
uv_reference = np.hstack(
[
resize_to_height(uv_debug, args.out_height),
resize_to_height(uv_distortion, args.out_height),
]
)
cap = open_source(args.video, args.camera_id)
if not cap.isOpened():
print("Cannot open camera/video source")
sys.exit(1)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, source_width)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, source_height)
sensor = orisys.Sensor(
"external",
config_name=args.sdk_config,
verbose=args.verbose,
mirror=False,
)
print("Press 'q' to quit.")
try:
while True:
ok, frame = cap.read()
if not ok:
break
if frame.shape[1] != source_width or frame.shape[0] != source_height:
frame = cv2.resize(frame, (source_width, source_height), interpolation=cv2.INTER_LINEAR)
if dist_coeffs is not None:
frame_source = cv2.undistort(frame, source_K, dist_coeffs)
else:
frame_source = frame
frame_source = align_frame_to_uv_map(frame_source, flip_vertical=flip_source_vertical)
unrolled = flatten_frame(frame_source, remap, border_value=(0, 0, 0))
sensor.get_img(frame=unrolled, already_processed=True)
sensor.compute_deformation(check_motion=True, threshold=0)
sensor.compute_contact()
fps, flow, vnormal, vshear, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = sensor.read_info(
sensor.info.FPS,
sensor.info.VRAW,
sensor.info.VNORMAL,
sensor.info.VSHEAR,
sensor.info.IMG,
sensor.info.CONTOUR,
sensor.info.CENTROID,
sensor.info.DEPTH,
sensor.info.FNORMAL,
sensor.info.FSHEARX,
sensor.info.FSHEARY,
)
arrows = orisys.util.draw_arrows(
img,
flow,
threshold=2,
grid_spacing=20,
arrow_scale=1.0,
)
contact_img = orisys.util.draw_contact(img, contour, centroid)
depth_norm = depth_map
if depth_norm.size > 0 and depth_norm.max() > depth_norm.min():
depth_u8 = ((depth_norm - depth_norm.min()) / (depth_norm.max() - depth_norm.min()) * 255).astype(np.uint8)
else:
depth_u8 = np.zeros_like(depth_norm, dtype=np.uint8)
depth_colored = cv2.applyColorMap(depth_u8, cv2.COLORMAP_JET)
dashboard = make_tiled_dashboard(
[
("Original", frame_source),
("UV Map | Distortion", uv_reference),
("UV Unrolled", unrolled),
("Flow", arrows),
],
tile_width=640,
tile_height=360,
)
cv2.imshow("UV Unroll Dashboard", dashboard)
print(
f"FPS={fps:.2f}, 法向力={fnormal:.4f}, 切向力X={fshearx:.4f}, "
f"切向力Y={fsheary:.4f}, 展开图尺寸={unrolled.shape[:2]}",
end="\r",
)
key = cv2.waitKey(1) & 0xFF
if key == ord("q"):
print("\n正在退出程序...")
break
if key == ord("r"):
sensor.reset()
print("\n追踪器已重置。")
finally:
cap.release()
sensor.disconnect()
cv2.destroyAllWindows()
if __name__ == "__main__":
main()
unroll_map_uv.npz
BIN
View File
Binary file not shown.