diff --git a/config/dd02-ov.json b/config/dd02-ov.json index 2a0c695..654b67c 100644 --- a/config/dd02-ov.json +++ b/config/dd02-ov.json @@ -16,11 +16,11 @@ "dis_preset": "medium", "dis_mode": 3, "algorithm_list": ["raft", "dis"], - "hhd_downsample": 3 + "hhd_downsample": 1 }, "contact": { "min_contour_area": 3000, - "flow_threshold": 5, + "flow_threshold": 1, "div_threshold": 0.05, "expansion_threshold": 0.01, "kernel_size": [5, 5], diff --git a/tools/3d_patch_generation_blender/captured_image.png b/tools/3d_patch_generation_blender/captured_image.png new file mode 100644 index 0000000..cb97cc9 Binary files /dev/null and b/tools/3d_patch_generation_blender/captured_image.png differ diff --git a/tools/3d_patch_generation_blender/captured_image_1.jpg b/tools/3d_patch_generation_blender/captured_image_1.jpg new file mode 100644 index 0000000..6a738a1 Binary files /dev/null and b/tools/3d_patch_generation_blender/captured_image_1.jpg differ diff --git a/tools/3d_patch_generation_blender/dd02c_v2.blend b/tools/3d_patch_generation_blender/dd02c_v2.blend new file mode 100644 index 0000000..692b1aa Binary files /dev/null and b/tools/3d_patch_generation_blender/dd02c_v2.blend differ diff --git a/tools/3d_patch_generation_blender/dd02c_v2.blend1 b/tools/3d_patch_generation_blender/dd02c_v2.blend1 new file mode 100644 index 0000000..fbac531 Binary files /dev/null and b/tools/3d_patch_generation_blender/dd02c_v2.blend1 differ diff --git a/tools/3d_patch_generation_blender/texture/patch_texture_0525_N120.png b/tools/3d_patch_generation_blender/texture/patch_texture_0525_N120.png new file mode 100644 index 0000000..769f728 Binary files /dev/null and b/tools/3d_patch_generation_blender/texture/patch_texture_0525_N120.png differ diff --git a/tools/3d_patch_generation_blender/texture/patch_texture_0525_N60.png b/tools/3d_patch_generation_blender/texture/patch_texture_0525_N60.png new file mode 100644 index 0000000..bf4adc6 Binary files /dev/null and b/tools/3d_patch_generation_blender/texture/patch_texture_0525_N60.png differ diff --git a/tools/3d_patch_generation_blender/texture/patch_texture_0525_N75.png b/tools/3d_patch_generation_blender/texture/patch_texture_0525_N75.png new file mode 100644 index 0000000..5356d64 Binary files /dev/null and b/tools/3d_patch_generation_blender/texture/patch_texture_0525_N75.png differ diff --git a/tools/3d_patch_generation_blender/texture/patch_texture_0525_N90.png b/tools/3d_patch_generation_blender/texture/patch_texture_0525_N90.png new file mode 100644 index 0000000..5775e41 Binary files /dev/null and b/tools/3d_patch_generation_blender/texture/patch_texture_0525_N90.png differ diff --git a/tools/3d_patch_generation_blender/unroll_finger.py b/tools/3d_patch_generation_blender/unroll_finger.py new file mode 100644 index 0000000..a08488f --- /dev/null +++ b/tools/3d_patch_generation_blender/unroll_finger.py @@ -0,0 +1,217 @@ +"""Flatten captured finger images into the mesh UV surface domain. + +The generated remap is expensive to build but cheap to use: live code only +needs cv2.remap(frame, map_x, map_y, ...). +""" + +import argparse +import os + +import cv2 +import numpy as np + + +def load_exported_data(data_dir): + mesh = np.load(os.path.join(data_dir, "mesh.npz")) + cam = np.load(os.path.join(data_dir, "camera_intrinsics.npz")) + return { + "vertices": mesh["vertices"].astype(np.float64), + "faces": mesh["faces"].astype(np.int32), + "uv_data": mesh["uv_data"].astype(np.float64), + "loop_to_vertex": mesh["loop_to_vertex"].astype(np.int32), + "K": cam["K"].astype(np.float64), + "width": int(cam["width"]), + "height": int(cam["height"]), + "world_to_cam": np.load(os.path.join(data_dir, "world_to_cam.npy")).astype(np.float64), + } + + +def face_uvs_from_loop_data(faces, uv_data, loop_to_vertex): + if len(uv_data) == 0: + raise ValueError("mesh.npz does not contain UV data; UV flattening is unavailable.") + if len(uv_data) != len(faces) * 3: + raise ValueError("Expected a triangulated mesh with exactly 3 UV loops per face.") + + face_loop_vertices = loop_to_vertex.reshape(-1, 3) + if face_loop_vertices.shape != faces.shape or not np.array_equal(face_loop_vertices, faces): + raise ValueError("UV loop order does not match exported face vertex order.") + return uv_data.reshape(-1, 3, 2) + + +def project_world_to_image(points_world, K, world_to_cam): + points_h = np.column_stack((points_world, np.ones(len(points_world)))) + points_cam = (world_to_cam @ points_h.T).T[:, :3] + z = -points_cam[:, 2] + x = K[0, 0] * (points_cam[:, 0] / z) + K[0, 2] + y = K[1, 1] * (points_cam[:, 1] / z) + K[1, 2] + return np.column_stack((x, y)), z + + +def barycentric_grid(px, py, tri): + a, b, c = tri + denom = (b[1] - c[1]) * (a[0] - c[0]) + (c[0] - b[0]) * (a[1] - c[1]) + if abs(denom) < 1e-12: + return None + w0 = ((b[1] - c[1]) * (px - c[0]) + (c[0] - b[0]) * (py - c[1])) / denom + w1 = ((c[1] - a[1]) * (px - c[0]) + (a[0] - c[0]) * (py - c[1])) / denom + w2 = 1.0 - w0 - w1 + return w0, w1, w2 + + +def build_uv_remap(data, out_width=1024, out_height=1024, flip_v=True): + faces = data["faces"] + vertices = data["vertices"] + face_uvs = face_uvs_from_loop_data(faces, data["uv_data"], data["loop_to_vertex"]) + + map_x = np.full((out_height, out_width), -1.0, dtype=np.float32) + map_y = np.full((out_height, out_width), -1.0, dtype=np.float32) + flat_xyz = np.full((out_height, out_width, 3), np.nan, dtype=np.float32) + depth_buffer = np.full((out_height, out_width), np.inf, dtype=np.float64) + + for face, uv_tri in zip(faces, face_uvs): + tri = np.empty_like(uv_tri) + tri[:, 0] = uv_tri[:, 0] * (out_width - 1) + tri[:, 1] = (1.0 - uv_tri[:, 1] if flip_v else uv_tri[:, 1]) * (out_height - 1) + + x0 = max(0, int(np.floor(np.min(tri[:, 0])))) + x1 = min(out_width - 1, int(np.ceil(np.max(tri[:, 0])))) + y0 = max(0, int(np.floor(np.min(tri[:, 1])))) + y1 = min(out_height - 1, int(np.ceil(np.max(tri[:, 1])))) + if x1 < x0 or y1 < y0: + continue + + yy, xx = np.mgrid[y0:y1 + 1, x0:x1 + 1] + bary = barycentric_grid(xx + 0.5, yy + 0.5, tri) + if bary is None: + continue + + w0, w1, w2 = bary + inside = (w0 >= -1e-6) & (w1 >= -1e-6) & (w2 >= -1e-6) + if not np.any(inside): + continue + + weights = np.stack((w0[inside], w1[inside], w2[inside]), axis=1) + points_world = weights @ vertices[face] + pixels, depth = project_world_to_image(points_world, data["K"], data["world_to_cam"]) + + src_x = pixels[:, 0] + src_y = pixels[:, 1] + target_y = yy[inside] + target_x = xx[inside] + valid = ( + (depth > 0) + & (src_x >= 0) + & (src_x <= data["width"] - 1) + & (src_y >= 0) + & (src_y <= data["height"] - 1) + & (depth < depth_buffer[target_y, target_x]) + ) + if not np.any(valid): + continue + + tx = target_x[valid] + ty = target_y[valid] + map_x[ty, tx] = src_x[valid].astype(np.float32) + map_y[ty, tx] = src_y[valid].astype(np.float32) + flat_xyz[ty, tx] = points_world[valid].astype(np.float32) + depth_buffer[ty, tx] = depth[valid] + + valid_mask = map_x >= 0 + return { + "map_x": map_x, + "map_y": map_y, + "valid_mask": valid_mask, + "flat_xyz": flat_xyz, + "source_width": np.array(data["width"], dtype=np.int32), + "source_height": np.array(data["height"], dtype=np.int32), + "out_width": np.array(out_width, dtype=np.int32), + "out_height": np.array(out_height, dtype=np.int32), + "method": np.array("uv"), + } + + +def save_remap(path, remap): + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + np.savez_compressed(path, **remap) + + +def load_remap(path): + data = np.load(path) + return {key: data[key] for key in data.files} + + +def flatten_frame(frame, remap, border_value=0): + flat = cv2.remap( + frame, + remap["map_x"].astype(np.float32), + remap["map_y"].astype(np.float32), + interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=border_value, + ) + valid_mask = remap.get("valid_mask") + if valid_mask is not None: + flat = flat.copy() + flat[~valid_mask.astype(bool)] = border_value + return flat + + +def load_capture_frame(capture_path, frame_index=0, flip_vertical=True): + image = cv2.imread(capture_path, cv2.IMREAD_COLOR) + if image is not None: + if flip_vertical: + image = np.flipud(image).copy() + return image + + cap = cv2.VideoCapture(capture_path, flip_vertical=True) + if not cap.isOpened(): + raise FileNotFoundError(f"Could not read image or video: {capture_path}") + try: + if frame_index > 0: + cap.set(cv2.CAP_PROP_POS_FRAMES, frame_index) + ok, frame = cap.read() + + + if not ok: + raise RuntimeError(f"Could not read frame {frame_index} from {capture_path}") + return frame + finally: + cap.release() + + +def main(): + parser = argparse.ArgumentParser(description="Build and apply a UV flattening remap for finger images.") + parser.add_argument("--data", required=True, help="Path to Blender exported_data directory.") + parser.add_argument("--remap", default=None, help="Output/input flatten_remap.npz path.") + parser.add_argument("--capture", default=None, help="Image or video file to flatten for verification.") + parser.add_argument("--output", default="unrolled.png", help="Flattened verification image output.") + parser.add_argument("--out-width", type=int, default=1024) + parser.add_argument("--out-height", type=int, default=1024) + parser.add_argument("--frame-index", type=int, default=0) + parser.add_argument("--no-flip-v", action="store_true", help="Keep UV V increasing downward.") + parser.add_argument("--reuse-remap", action="store_true", help="Load an existing remap instead of rebuilding it.") + args = parser.parse_args() + + remap_path = args.remap or os.path.join(args.data, "flatten_remap.npz") + if args.reuse_remap and os.path.exists(remap_path): + remap = load_remap(remap_path) + else: + remap = build_uv_remap( + load_exported_data(args.data), + out_width=args.out_width, + out_height=args.out_height, + flip_v=not args.no_flip_v, + ) + save_remap(remap_path, remap) + valid_ratio = float(np.mean(remap["valid_mask"])) + print(f"Saved remap: {remap_path} valid={valid_ratio:.1%}") + + if args.capture: + frame = load_capture_frame(args.capture, args.frame_index) + flat = flatten_frame(frame, remap) + cv2.imwrite(args.output, flat) + print(f"Saved flattened image: {args.output}") + + +if __name__ == "__main__": + main() diff --git a/tools/3d_patch_generation_blender/unrolled.png b/tools/3d_patch_generation_blender/unrolled.png new file mode 100644 index 0000000..1de9fcf Binary files /dev/null and b/tools/3d_patch_generation_blender/unrolled.png differ diff --git a/tools/dd02_3dviewer/exported_data/flatten_remap.npz b/tools/dd02_3dviewer/exported_data/flatten_remap.npz new file mode 100644 index 0000000..02ca287 Binary files /dev/null and b/tools/dd02_3dviewer/exported_data/flatten_remap.npz differ diff --git a/tools/dd02_3dviewer/sdk_mapto3d.py b/tools/dd02_3dviewer/sdk_mapto3d.py index 2c79caa..068a6c8 100644 --- a/tools/dd02_3dviewer/sdk_mapto3d.py +++ b/tools/dd02_3dviewer/sdk_mapto3d.py @@ -45,6 +45,12 @@ def load_exported_data(data_dir): def parse_source(value): + value = str(value).strip() + for prefix in ("camera:", "cam:", "webcam:"): + if value.lower().startswith(prefix): + value = value[len(prefix):].strip() + break + try: return int(value) except ValueError: @@ -53,13 +59,54 @@ def parse_source(value): def prepare_static_points(data, display_stride): xyz_map = data["xyz_map"] + height, width = xyz_map.shape[:2] valid_mask = ~np.isnan(xyz_map[:, :, 0]) valid_points = xyz_map[valid_mask].reshape(-1, 3) if len(valid_points) == 0: raise ValueError("pixel_to_xyz.npz contains no valid mesh hit points.") display_indices = np.arange(len(valid_points))[::max(1, int(display_stride))] - return valid_mask, display_indices, valid_points[display_indices] + flat_indices = np.flatnonzero(valid_mask.ravel())[display_indices] + display_ys = flat_indices // width + display_xs = flat_indices % width + return ( + valid_mask, + display_indices, + valid_points[display_indices], + display_ys, + display_xs, + ) + + +def mesh_pixel_to_world_scale(xyz_map, valid_mask): + valid_points = xyz_map[valid_mask] + if len(valid_points) == 0: + return 1e-3 + extent = np.nanmax(valid_points, axis=0) - np.nanmin(valid_points, axis=0) + mesh_span = float(np.max(extent)) + height, width = xyz_map.shape[:2] + return mesh_span / max(height, width, 1) + + +def sample_field_at_display_points(field, display_ys, display_xs): + if field.ndim == 2: + return field[display_ys, display_xs] + return field[display_ys, display_xs, :] + + +def deform_point_cloud(rest_points, displacement, pixel_to_world, deform_scale): + return rest_points + displacement * pixel_to_world * float(deform_scale) + + +def lift_flow_to_3d(flow, tangent_x, tangent_y, normals): + """Lift raw 2D optical flow to tangent-plane 3D displacement.""" + tx_norm = np.linalg.norm(tangent_x, axis=2, keepdims=True) + ty_norm = np.linalg.norm(tangent_y, axis=2, keepdims=True) + t1 = np.divide(tangent_x, tx_norm, out=np.zeros_like(tangent_x), where=tx_norm > 1e-8) + t2 = np.divide(tangent_y, ty_norm, out=np.zeros_like(tangent_y), where=ty_norm > 1e-8) + flow_3d = flow[:, :, 0:1] * t1 + flow[:, :, 1:2] * t2 + normal_component = np.sum(flow_3d * normals, axis=2, keepdims=True) + return flow_3d - normal_component * normals def align_image_to_map(img_bgr, data, valid_mask, display_indices, flip_vertical=True): @@ -98,6 +145,47 @@ def magnitude_map_colors(flow, valid_mask, display_indices, max_magnitude=None): ).astype(np.float64) +def align_scalar_field_to_map(field, data, flip_vertical=True): + if field.ndim == 3: + field = np.linalg.norm(field, axis=2) + + target_w = int(data["width"]) + target_h = int(data["height"]) + h, w = field.shape[:2] + if (w, h) != (target_w, target_h): + field = cv2.resize(field, (target_w, target_h), interpolation=cv2.INTER_LINEAR) + else: + field = field.copy() + + if flip_vertical: + field = np.flipud(field).copy() + return field + + +def scalar_map_colors(field, valid_mask, display_indices, value_min=None, value_max=None): + valid_values = field[valid_mask].reshape(-1) + display_values = valid_values[display_indices] + finite_values = display_values[np.isfinite(display_values)] + if len(finite_values) == 0: + return arrow_map_reference_colors(len(display_indices)) + + if value_min is None: + value_min = float(np.percentile(finite_values, 5)) + if value_max is None: + value_max = float(np.percentile(finite_values, 95)) + if value_max <= value_min: + value_max = value_min + 1e-6 + + value = np.clip((display_values - value_min) / (value_max - value_min), 0.0, 1.0) + return np.column_stack( + ( + value, + np.clip(1.0 - np.abs(value - 0.5) * 2.0, 0.0, 1.0), + 1.0 - value, + ) + ).astype(np.float64) + + def align_vector_field_to_map(field, data, flip_vertical=True, scale_as_flow=True): target_w = int(data["width"]) target_h = int(data["height"]) @@ -131,6 +219,37 @@ def estimate_surface_normals(xyz_map): return normals +def compute_surface_frames(xyz_map): + valid = ~np.isnan(xyz_map[:, :, 0]) + filled = xyz_map.copy() + filled[~valid] = 0 + d_y = np.gradient(filled, axis=0) + d_x = np.gradient(filled, axis=1) + normals = np.cross(d_x, d_y) + norm = np.linalg.norm(normals, axis=2, keepdims=True) + normals = np.divide(normals, norm, out=np.zeros_like(normals), where=norm > 1e-8) + d_x[~valid] = 0 + d_y[~valid] = 0 + normals[~valid] = 0 + return d_x, d_y, normals, valid + + +def lift_deformation_to_3d(vshear, vnormal, tangent_x, tangent_y, normals, normal_gain=1.0): + # Unit tangents: map 2D pixel shear to 3D direction without world-scale Jacobian blow-up. + tx_norm = np.linalg.norm(tangent_x, axis=2, keepdims=True) + ty_norm = np.linalg.norm(tangent_y, axis=2, keepdims=True) + t1 = np.divide(tangent_x, tx_norm, out=np.zeros_like(tangent_x), where=tx_norm > 1e-8) + t2 = np.divide(tangent_y, ty_norm, out=np.zeros_like(tangent_y), where=ty_norm > 1e-8) + + shear_3d = vshear[:, :, 0:1] * t1 + vshear[:, :, 1:2] * t2 + normal_mag = np.linalg.norm(vnormal, axis=2, keepdims=True) * float(normal_gain) + normal_3d = normal_mag * normals + + normal_component = np.sum(shear_3d * normals, axis=2, keepdims=True) + shear_tangent = shear_3d - normal_component * normals + return shear_tangent + normal_3d + + def line_colors(mag, arrow_map_style): if len(mag) == 0: return np.empty((0, 3), dtype=np.float64) @@ -141,13 +260,17 @@ def line_colors(mag, arrow_map_style): return np.column_stack((norm_mag, 1.0 - norm_mag, np.zeros_like(norm_mag))).astype(np.float64) -def lines_from_vectors(starts, vectors, mag, flow_scale, arrow_map_style): +def lines_from_vectors(starts, vectors, mag, flow_scale, arrow_map_style, unit_direction=False): vec_norm = np.linalg.norm(vectors, axis=1) good = np.isfinite(vec_norm) & (vec_norm > 1e-8) starts, vectors, mag = starts[good], vectors[good], mag[good] if len(starts) == 0: return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) + if unit_direction: + vec_norm = vec_norm[:, None] + vectors = np.divide(vectors, vec_norm, out=np.zeros_like(vectors), where=vec_norm > 1e-8) + points = np.empty((len(starts) * 2, 3), dtype=np.float64) points[0::2] = starts points[1::2] = starts + vectors * flow_scale @@ -189,81 +312,46 @@ def build_flow_lines( return lines_from_vectors(starts, vectors, mag, flow_scale, arrow_map_style) -def build_shear_normal_lines( +def build_resultant_force_lines( xyz_map, valid_mask, - vshear, - vnormal, - normals, + resultant_force, arrow_stride=32, flow_scale=1.0, min_magnitude=0.01, - normal_gain=1.0, arrow_map_style=False, ): - h, w = vshear.shape[:2] + h, w = resultant_force.shape[:2] ys, xs = np.mgrid[0:h:arrow_stride, 0:w:arrow_stride] ys = ys.reshape(-1) xs = xs.reshape(-1) - shear = vshear[ys, xs] - normal_field = vnormal[ys, xs] - shear_mag = np.linalg.norm(shear, axis=1) - normal_mag = np.linalg.norm(normal_field, axis=1) * normal_gain - combined_mag = shear_mag + normal_mag - keep = valid_mask[ys, xs] & (combined_mag >= min_magnitude) - xs, ys, shear, shear_mag, normal_mag, combined_mag = ( + vectors = resultant_force[ys, xs] + mag = np.linalg.norm(vectors, axis=1) + keep = valid_mask[ys, xs] & (mag >= min_magnitude) + xs, ys, vectors, mag = ( xs[keep], ys[keep], - shear[keep], - shear_mag[keep], - normal_mag[keep], - combined_mag[keep], + vectors[keep], + mag[keep], ) if len(xs) == 0: return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) - - tx = np.rint(xs + shear[:, 0]).astype(np.int32) - ty = np.rint(ys + shear[:, 1]).astype(np.int32) - inside = (0 <= tx) & (tx < w) & (0 <= ty) & (ty < h) - xs, ys, tx, ty, shear_mag, normal_mag, combined_mag = ( - xs[inside], - ys[inside], - tx[inside], - ty[inside], - shear_mag[inside], - normal_mag[inside], - combined_mag[inside], - ) - if len(xs) == 0: - return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) - - valid_target = valid_mask[ty, tx] - xs, ys, tx, ty, shear_mag, normal_mag, combined_mag = ( - xs[valid_target], - ys[valid_target], - tx[valid_target], - ty[valid_target], - shear_mag[valid_target], - normal_mag[valid_target], - combined_mag[valid_target], - ) - starts = xyz_map[ys, xs] - tangent = xyz_map[ty, tx] - starts - tangent_norm = np.linalg.norm(tangent, axis=1, keepdims=True) - tangent_unit = np.divide(tangent, tangent_norm, out=np.zeros_like(tangent), where=tangent_norm > 1e-8) - normal_unit = normals[ys, xs] - - shear_ratio = shear_mag / np.maximum(combined_mag, 1e-8) - normal_ratio = normal_mag / np.maximum(combined_mag, 1e-8) - vectors = tangent_unit * shear_ratio[:, None] + normal_unit * normal_ratio[:, None] - vector_norm = np.linalg.norm(vectors, axis=1, keepdims=True) - vectors = np.divide(vectors, vector_norm, out=np.zeros_like(vectors), where=vector_norm > 1e-8) - return lines_from_vectors(starts, vectors, combined_mag, flow_scale, arrow_map_style) + return lines_from_vectors( + starts, vectors, mag, flow_scale, arrow_map_style, unit_direction=True + ) -def init_viewer(cloud_points, cloud_colors, flow_points, flow_lines, flow_colors, arrow_map_style=False): +def init_viewer( + cloud_points, + cloud_colors, + flow_points, + flow_lines, + flow_colors, + arrow_map_style=False, + show_arrows=True, +): import open3d as o3d vis = o3d.visualization.Visualizer() @@ -274,11 +362,13 @@ def init_viewer(cloud_points, cloud_colors, flow_points, flow_lines, flow_colors pcd.colors = o3d.utility.Vector3dVector(cloud_colors.astype(np.float64)) vis.add_geometry(pcd) - line_set = o3d.geometry.LineSet() - line_set.points = o3d.utility.Vector3dVector(flow_points.astype(np.float64)) - line_set.lines = o3d.utility.Vector2iVector(flow_lines) - line_set.colors = o3d.utility.Vector3dVector(flow_colors.astype(np.float64)) - vis.add_geometry(line_set) + line_set = None + if show_arrows: + line_set = o3d.geometry.LineSet() + line_set.points = o3d.utility.Vector3dVector(flow_points.astype(np.float64)) + line_set.lines = o3d.utility.Vector2iVector(flow_lines) + line_set.colors = o3d.utility.Vector3dVector(flow_colors.astype(np.float64)) + vis.add_geometry(line_set) render_option = vis.get_render_option() render_option.point_size = 1.0 if arrow_map_style else 2.0 @@ -291,8 +381,13 @@ def init_viewer(cloud_points, cloud_colors, flow_points, flow_lines, flow_colors def run_sdk_3d_flow(args): data = load_exported_data(args.data_dir) - valid_mask, display_indices, cloud_points = prepare_static_points(data, args.display_stride) - surface_normals = estimate_surface_normals(data["xyz_map"]) + valid_mask, display_indices, rest_cloud_points, display_ys, display_xs = prepare_static_points( + data, args.display_stride + ) + tangent_x, tangent_y, surface_normals, valid_mask = compute_surface_frames(data["xyz_map"]) + pixel_to_world = mesh_pixel_to_world_scale(data["xyz_map"], valid_mask) + show_arrows = args.display_mode == "arrows" + deform_cloud = args.display_mode == "deform" sensor = orisys.Sensor( parse_source(args.source), @@ -314,19 +409,32 @@ def run_sdk_3d_flow(args): if sensor.get_img() is None: break - sensor.compute_deformation(decompose=(args.vector_mode == "shear-normal"), check_motion=False) - flow, img_bgr, vshear, vnormal = sensor.read_info( + needs_decomposition = args.vector_mode in ("shear-normal", "depth-map") + sensor.compute_deformation(decompose=needs_decomposition, check_motion=False) + if args.vector_mode == "depth-map": + sensor.compute_contact() + flow, img_bgr, vshear, vnormal, depth_map = sensor.read_info( sensor.info.VRAW, sensor.info.IMG, sensor.info.VSHEAR, sensor.info.VNORMAL, + sensor.info.DEPTH, ) flow_map = align_vector_field_to_map(flow, data, flip_vertical=not args.no_flip_vertical) if args.vector_mode == "vraw-magnitude": cloud_colors = magnitude_map_colors(flow_map, valid_mask, display_indices, args.magnitude_max) + elif args.vector_mode == "depth-map": + depth_map = align_scalar_field_to_map(depth_map, data, flip_vertical=not args.no_flip_vertical) + cloud_colors = scalar_map_colors( + depth_map, + valid_mask, + display_indices, + value_min=args.depth_min, + value_max=args.depth_max, + ) elif args.arrow_map_style: - cloud_colors = arrow_map_reference_colors(len(cloud_points)) + cloud_colors = arrow_map_reference_colors(len(rest_cloud_points)) else: cloud_colors = align_image_to_map( img_bgr, @@ -336,35 +444,81 @@ def run_sdk_3d_flow(args): flip_vertical=not args.no_flip_vertical, ) + flip_vertical = not args.no_flip_vertical + displacement_map = None if args.vector_mode == "shear-normal": - vshear_map = align_vector_field_to_map(vshear, data, flip_vertical=not args.no_flip_vertical) - vnormal_map = align_vector_field_to_map(vnormal, data, flip_vertical=not args.no_flip_vertical) - flow_points, flow_lines, flow_colors = build_shear_normal_lines( - data["xyz_map"], - valid_mask, + vshear_map = align_vector_field_to_map(vshear, data, flip_vertical=flip_vertical) + vnormal_map = align_vector_field_to_map(vnormal, data, flip_vertical=flip_vertical) + displacement_map = lift_deformation_to_3d( vshear_map, vnormal_map, + tangent_x, + tangent_y, surface_normals, - arrow_stride=args.arrow_stride, - flow_scale=args.flow_scale, - min_magnitude=args.min_flow, normal_gain=args.normal_gain, - arrow_map_style=args.arrow_map_style, ) - elif args.vector_mode == "vraw-magnitude": + elif args.vector_mode == "depth-map" and deform_cloud: + vnormal_map = align_vector_field_to_map(vnormal, data, flip_vertical=flip_vertical) + zero_shear = np.zeros_like(vnormal_map) + displacement_map = lift_deformation_to_3d( + zero_shear, + vnormal_map, + tangent_x, + tangent_y, + surface_normals, + normal_gain=args.normal_gain, + ) + elif args.vector_mode in ("vraw", "vraw-magnitude"): + displacement_map = lift_flow_to_3d(flow_map, tangent_x, tangent_y, surface_normals) + + if deform_cloud and args.deform_color == "magnitude" and displacement_map is not None: + cloud_colors = magnitude_map_colors( + displacement_map, valid_mask, display_indices, args.magnitude_max + ) + + if deform_cloud: + if displacement_map is None: + raise ValueError( + f"display-mode deform is not supported with vector-mode {args.vector_mode!r}" + ) + point_displacement = sample_field_at_display_points( + displacement_map, display_ys, display_xs + ) + cloud_points = deform_point_cloud( + rest_cloud_points, + point_displacement, + pixel_to_world, + args.deform_scale, + ) flow_points = np.empty((0, 3), dtype=np.float64) flow_lines = np.empty((0, 2), dtype=np.int32) flow_colors = np.empty((0, 3), dtype=np.float64) else: - flow_points, flow_lines, flow_colors = build_flow_lines( - data["xyz_map"], - valid_mask, - flow_map, - arrow_stride=args.arrow_stride, - flow_scale=args.flow_scale, - min_magnitude=args.min_flow, - arrow_map_style=args.arrow_map_style, - ) + cloud_points = rest_cloud_points + if args.vector_mode == "shear-normal": + flow_points, flow_lines, flow_colors = build_resultant_force_lines( + data["xyz_map"], + valid_mask, + displacement_map, + arrow_stride=args.arrow_stride, + flow_scale=args.flow_scale, + min_magnitude=args.min_flow, + arrow_map_style=args.arrow_map_style, + ) + elif args.vector_mode in ("vraw-magnitude", "depth-map"): + flow_points = np.empty((0, 3), dtype=np.float64) + flow_lines = np.empty((0, 2), dtype=np.int32) + flow_colors = np.empty((0, 3), dtype=np.float64) + else: + flow_points, flow_lines, flow_colors = build_flow_lines( + data["xyz_map"], + valid_mask, + flow_map, + arrow_stride=args.arrow_stride, + flow_scale=args.flow_scale, + min_magnitude=args.min_flow, + arrow_map_style=args.arrow_map_style, + ) if vis is None: vis, pcd, line_set, o3d = init_viewer( @@ -374,21 +528,26 @@ def run_sdk_3d_flow(args): flow_lines, flow_colors, arrow_map_style=args.arrow_map_style, + show_arrows=show_arrows, ) - print("SDK 3D flow viewer started. Close the Open3D window to stop.") + mode_label = "deformed cloud" if deform_cloud else "arrows" + print(f"SDK 3D flow viewer started ({mode_label}). Close the Open3D window to stop.") else: + pcd.points = o3d.utility.Vector3dVector(cloud_points.astype(np.float64)) pcd.colors = o3d.utility.Vector3dVector(cloud_colors) - line_set.points = o3d.utility.Vector3dVector(flow_points.astype(np.float64)) - line_set.lines = o3d.utility.Vector2iVector(flow_lines) - line_set.colors = o3d.utility.Vector3dVector(flow_colors.astype(np.float64)) + if line_set is not None: + line_set.points = o3d.utility.Vector3dVector(flow_points.astype(np.float64)) + line_set.lines = o3d.utility.Vector2iVector(flow_lines) + line_set.colors = o3d.utility.Vector3dVector(flow_colors.astype(np.float64)) + vis.update_geometry(line_set) vis.update_geometry(pcd) - vis.update_geometry(line_set) vis.update_renderer() frame_count += 1 if frame_count % 30 == 0: elapsed = max(time.perf_counter() - t0, 1e-6) - print(f"frames={frame_count}, viewer FPS={frame_count / elapsed:.1f}, arrows={len(flow_lines)}") + arrow_count = len(flow_lines) if show_arrows else 0 + print(f"frames={frame_count}, viewer FPS={frame_count / elapsed:.1f}, arrows={arrow_count}") finally: sensor.disconnect() if vis is not None: @@ -398,16 +557,57 @@ def run_sdk_3d_flow(args): def parse_args(): parser = argparse.ArgumentParser(description="Map SDK flow fields to a 3D point cloud.") parser.add_argument("--data-dir", default=os.path.join(SCRIPT_DIR, "exported_data")) - parser.add_argument("--source", default=os.path.join(SCRIPT_DIR, "20260602-094916.mp4")) + parser.add_argument( + "--source", + default=os.path.join(SCRIPT_DIR, "20260602-094916.mp4"), + help="Video path or live camera ID, for example 0, 1, camera:0, or cam:1.", + ) parser.add_argument("--config", default=os.path.join(REPO_ROOT, "config", "dd02-ov.json")) parser.add_argument("--display-stride", type=int, default=4) parser.add_argument("--arrow-stride", type=int, default=8) - parser.add_argument("--flow-scale", type=float, default=3.0) - parser.add_argument("--min-flow", type=float, default=0.5) + parser.add_argument( + "--flow-scale", + type=float, + default=3.0, + help="Arrow length in 3D viewer units (shear-normal uses unit direction; magnitude shown by color).", + ) + parser.add_argument( + "--min-flow", + type=float, + default=0.5, + help="Minimum 2D-equivalent resultant magnitude before drawing an arrow.", + ) parser.add_argument("--normal-gain", type=float, default=1.0) + parser.add_argument( + "--display-mode", + choices=["arrows", "deform"], + default="arrows", + help="arrows: line vectors; deform: displace the point cloud by the 3D deformation field.", + ) + parser.add_argument( + "--deform-scale", + type=float, + default=1.0, + help="Gain on cloud displacement (multiplied by mesh pixel-to-world scale).", + ) + parser.add_argument( + "--deform-color", + choices=["texture", "magnitude"], + default="texture", + help="Point colors in deform mode: camera texture or deformation magnitude heat-map.", + ) parser.add_argument("--magnitude-max", type=float, default=None, help="Upper VRAW magnitude for heat-map color scaling; default uses frame percentile.") - parser.add_argument("--vector-mode", choices=["vraw", "vraw-magnitude", "shear-normal"], default="vraw") + parser.add_argument("--depth-min", type=float, default=None, + help="Minimum depth value for heat-map color scaling; default uses frame percentile.") + parser.add_argument("--depth-max", type=float, default=None, + help="Maximum depth value for heat-map color scaling; default uses frame percentile.") + parser.add_argument( + "--vector-mode", + choices=["vraw", "vraw-magnitude", "shear-normal", "depth-map"], + default="vraw", + help="Vector display mode. shear-normal shows 3D resultant vectors (surface-tangent shear + surface-normal component).", + ) parser.add_argument( "--arrow-map-style", action="store_true", diff --git a/tools/dd02_3dviewer/sdk_mapto3d_flatten.py b/tools/dd02_3dviewer/sdk_mapto3d_flatten.py new file mode 100644 index 0000000..822b739 --- /dev/null +++ b/tools/dd02_3dviewer/sdk_mapto3d_flatten.py @@ -0,0 +1,562 @@ +import argparse +import os +import sys +import time + +import cv2 +import numpy as np + + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..")) +SRC_DIR = os.path.join(REPO_ROOT, "src") +if SRC_DIR not in sys.path: + sys.path.insert(0, SRC_DIR) + +import orisys + + +def load_flatten_remap(path): + remap = np.load(path) + result = { + "map_x": remap["map_x"].astype(np.float32), + "map_y": remap["map_y"].astype(np.float32), + "valid_mask": remap["valid_mask"].astype(bool), + } + if "flat_xyz" in remap: + result["flat_xyz"] = remap["flat_xyz"].astype(np.float32) + return result + + +def flip_sensor_frames_vertical(sensor): + """Flip captured BGR/gray frames to match Blender-exported map orientation.""" + if sensor._img_bgr_raw is not None: + sensor._img_bgr_raw = cv2.flip(sensor._img_bgr_raw, 0) + if sensor.img is not None: + cv2.flip(sensor.img, 0, dst=sensor.img) + + +def flatten_frame(frame, remap, border_value=0): + flat = cv2.remap( + frame, + remap["map_x"], + remap["map_y"], + interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=border_value, + ) + flat = flat.copy() + flat[~remap["valid_mask"]] = border_value + return flat + + +def load_exported_data(data_dir): + mesh = np.load(os.path.join(data_dir, "mesh.npz")) + cam_data = np.load(os.path.join(data_dir, "camera_intrinsics.npz")) + xyz_map_path = os.path.join(data_dir, "pixel_to_xyz.npz") + if not os.path.exists(xyz_map_path): + raise FileNotFoundError(f"Missing precomputed 3D map: {xyz_map_path}") + + xyz_map = np.load(xyz_map_path)["xyz_map"] + width = int(cam_data["width"]) + height = int(cam_data["height"]) + if xyz_map.shape[:2] != (height, width): + raise ValueError( + f"pixel_to_xyz shape {xyz_map.shape[:2]} does not match camera size {(height, width)}" + ) + + return { + "vertices": mesh["vertices"], + "faces": mesh["faces"], + "uv_data": mesh["uv_data"], + "loop_to_vertex": mesh["loop_to_vertex"], + "K": cam_data["K"], + "width": width, + "height": height, + "world_to_cam": np.load(os.path.join(data_dir, "world_to_cam.npy")), + "xyz_map": xyz_map, + } + + +def parse_source(value): + value = str(value).strip() + for prefix in ("camera:", "cam:", "webcam:"): + if value.lower().startswith(prefix): + value = value[len(prefix):].strip() + break + + try: + return int(value) + except ValueError: + return value + + +def prepare_static_points(data, display_stride): + xyz_map = data["xyz_map"] + valid_mask = ~np.isnan(xyz_map[:, :, 0]) + valid_points = xyz_map[valid_mask].reshape(-1, 3) + if len(valid_points) == 0: + raise ValueError("pixel_to_xyz.npz contains no valid mesh hit points.") + + display_indices = np.arange(len(valid_points))[::max(1, int(display_stride))] + return valid_mask, display_indices, valid_points[display_indices] + + +def align_image_to_map(img_bgr, data, valid_mask, display_indices, flip_vertical=True, flip_horizontal=True): + target_w = int(data["width"]) + target_h = int(data["height"]) + h, w = img_bgr.shape[:2] + if (w, h) != (target_w, target_h): + img_bgr = cv2.resize(img_bgr, (target_w, target_h), interpolation=cv2.INTER_LINEAR) + + img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB) + if flip_vertical: + img_rgb = np.flipud(img_rgb).copy() + if flip_horizontal: + img_rgb = np.fliplr(img_rgb).copy() + + colors = img_rgb.astype(np.float64) / 255.0 + return colors[valid_mask].reshape(-1, 3)[display_indices] + + +def arrow_map_reference_colors(count, color=(0.04, 0.04, 0.04)): + return np.tile(np.asarray(color, dtype=np.float64), (count, 1)) + + +def magnitude_map_colors(flow, valid_mask, display_indices, max_magnitude=None): + mag = np.linalg.norm(flow, axis=2) + valid_mag = mag[valid_mask].reshape(-1) + display_mag = valid_mag[display_indices] + if max_magnitude is None or max_magnitude <= 0: + max_magnitude = max(float(np.percentile(display_mag, 95)), 1e-6) + + value = np.clip(display_mag / max_magnitude, 0.0, 1.0) + return np.column_stack( + ( + value, + np.clip(1.0 - np.abs(value - 0.5) * 2.0, 0.0, 1.0), + 1.0 - value, + ) + ).astype(np.float64) + + +def align_scalar_field_to_map(field, data, flip_vertical=True): + if field.ndim == 3: + field = np.linalg.norm(field, axis=2) + + target_w = int(data["width"]) + target_h = int(data["height"]) + h, w = field.shape[:2] + if (w, h) != (target_w, target_h): + field = cv2.resize(field, (target_w, target_h), interpolation=cv2.INTER_LINEAR) + else: + field = field.copy() + + if flip_vertical: + field = np.flipud(field).copy() + return field + + +def scalar_map_colors(field, valid_mask, display_indices, value_min=None, value_max=None): + valid_values = field[valid_mask].reshape(-1) + display_values = valid_values[display_indices] + finite_values = display_values[np.isfinite(display_values)] + if len(finite_values) == 0: + return arrow_map_reference_colors(len(display_indices)) + + if value_min is None: + value_min = float(np.percentile(finite_values, 5)) + if value_max is None: + value_max = float(np.percentile(finite_values, 95)) + if value_max <= value_min: + value_max = value_min + 1e-6 + + value = np.clip((display_values - value_min) / (value_max - value_min), 0.0, 1.0) + return np.column_stack( + ( + value, + np.clip(1.0 - np.abs(value - 0.5) * 2.0, 0.0, 1.0), + 1.0 - value, + ) + ).astype(np.float64) + + +def align_vector_field_to_map(field, data, flip_vertical=True, scale_as_flow=True): + target_w = int(data["width"]) + target_h = int(data["height"]) + h, w = field.shape[:2] + if (w, h) != (target_w, target_h): + sx = target_w / max(w, 1) + sy = target_h / max(h, 1) + field = cv2.resize(field, (target_w, target_h), interpolation=cv2.INTER_LINEAR) + if scale_as_flow: + field[:, :, 0] *= sx + field[:, :, 1] *= sy + else: + field = field.copy() + + if flip_vertical: + field = np.flipud(field).copy() + field[:, :, 1] *= -1.0 + return field + + +def estimate_surface_normals(xyz_map): + valid = ~np.isnan(xyz_map[:, :, 0]) + filled = xyz_map.copy() + filled[~valid] = 0 + d_y = np.gradient(filled, axis=0) + d_x = np.gradient(filled, axis=1) + normals = np.cross(d_x, d_y) + norm = np.linalg.norm(normals, axis=2, keepdims=True) + normals = np.divide(normals, norm, out=np.zeros_like(normals), where=norm > 1e-8) + normals[~valid] = 0 + return normals + + +def line_colors(mag, arrow_map_style): + if len(mag) == 0: + return np.empty((0, 3), dtype=np.float64) + if arrow_map_style: + return np.tile(np.array([0.0, 0.9, 1.0], dtype=np.float64), (len(mag), 1)) + norm_mag = mag / max(float(np.percentile(mag, 95)), 1e-6) + norm_mag = np.clip(norm_mag, 0.0, 1.0) + return np.column_stack((norm_mag, 1.0 - norm_mag, np.zeros_like(norm_mag))).astype(np.float64) + + +def lines_from_vectors(starts, vectors, mag, flow_scale, arrow_map_style): + vec_norm = np.linalg.norm(vectors, axis=1) + good = np.isfinite(vec_norm) & (vec_norm > 1e-8) + starts, vectors, mag = starts[good], vectors[good], mag[good] + if len(starts) == 0: + return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) + + points = np.empty((len(starts) * 2, 3), dtype=np.float64) + points[0::2] = starts + points[1::2] = starts + vectors * flow_scale + lines = np.column_stack((np.arange(0, len(points), 2), np.arange(1, len(points), 2))).astype(np.int32) + return points, lines, line_colors(mag, arrow_map_style) + + +def build_flow_lines( + xyz_map, + valid_mask, + flow, + arrow_stride=32, + flow_scale=1.0, + min_magnitude=0.5, + arrow_map_style=False, +): + h, w = flow.shape[:2] + ys, xs = np.mgrid[0:h:arrow_stride, 0:w:arrow_stride] + ys = ys.reshape(-1) + xs = xs.reshape(-1) + sample_flow = flow[ys, xs] + mag = np.linalg.norm(sample_flow, axis=1) + keep = valid_mask[ys, xs] & (mag >= min_magnitude) + xs, ys, sample_flow, mag = xs[keep], ys[keep], sample_flow[keep], mag[keep] + if len(xs) == 0: + return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) + + tx = np.rint(xs + sample_flow[:, 0]).astype(np.int32) + ty = np.rint(ys + sample_flow[:, 1]).astype(np.int32) + inside = (0 <= tx) & (tx < w) & (0 <= ty) & (ty < h) + xs, ys, tx, ty, mag = xs[inside], ys[inside], tx[inside], ty[inside], mag[inside] + if len(xs) == 0: + return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) + + valid_target = valid_mask[ty, tx] + xs, ys, tx, ty, mag = xs[valid_target], ys[valid_target], tx[valid_target], ty[valid_target], mag[valid_target] + starts = xyz_map[ys, xs] + vectors = xyz_map[ty, tx] - starts + return lines_from_vectors(starts, vectors, mag, flow_scale, arrow_map_style) + + +def build_shear_normal_lines( + xyz_map, + valid_mask, + vshear, + vnormal, + normals, + arrow_stride=32, + flow_scale=1.0, + min_magnitude=0.01, + normal_gain=1.0, + arrow_map_style=False, +): + h, w = vshear.shape[:2] + ys, xs = np.mgrid[0:h:arrow_stride, 0:w:arrow_stride] + ys = ys.reshape(-1) + xs = xs.reshape(-1) + + shear = vshear[ys, xs] + normal_field = vnormal[ys, xs] + shear_mag = np.linalg.norm(shear, axis=1) + normal_mag = np.linalg.norm(normal_field, axis=1) * normal_gain + combined_mag = shear_mag + normal_mag + keep = valid_mask[ys, xs] & (combined_mag >= min_magnitude) + xs, ys, shear, shear_mag, normal_mag, combined_mag = ( + xs[keep], + ys[keep], + shear[keep], + shear_mag[keep], + normal_mag[keep], + combined_mag[keep], + ) + if len(xs) == 0: + return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) + + tx = np.rint(xs + shear[:, 0]).astype(np.int32) + ty = np.rint(ys + shear[:, 1]).astype(np.int32) + inside = (0 <= tx) & (tx < w) & (0 <= ty) & (ty < h) + xs, ys, tx, ty, shear_mag, normal_mag, combined_mag = ( + xs[inside], + ys[inside], + tx[inside], + ty[inside], + shear_mag[inside], + normal_mag[inside], + combined_mag[inside], + ) + if len(xs) == 0: + return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) + + valid_target = valid_mask[ty, tx] + xs, ys, tx, ty, shear_mag, normal_mag, combined_mag = ( + xs[valid_target], + ys[valid_target], + tx[valid_target], + ty[valid_target], + shear_mag[valid_target], + normal_mag[valid_target], + combined_mag[valid_target], + ) + + starts = xyz_map[ys, xs] + tangent = xyz_map[ty, tx] - starts + tangent_norm = np.linalg.norm(tangent, axis=1, keepdims=True) + tangent_unit = np.divide(tangent, tangent_norm, out=np.zeros_like(tangent), where=tangent_norm > 1e-8) + normal_unit = normals[ys, xs] + + shear_ratio = shear_mag / np.maximum(combined_mag, 1e-8) + normal_ratio = normal_mag / np.maximum(combined_mag, 1e-8) + vectors = tangent_unit * shear_ratio[:, None] + normal_unit * normal_ratio[:, None] + vector_norm = np.linalg.norm(vectors, axis=1, keepdims=True) + vectors = np.divide(vectors, vector_norm, out=np.zeros_like(vectors), where=vector_norm > 1e-8) + return lines_from_vectors(starts, vectors, combined_mag, flow_scale, arrow_map_style) + + +def init_viewer(cloud_points, cloud_colors, flow_points, flow_lines, flow_colors, arrow_map_style=False): + import open3d as o3d + + vis = o3d.visualization.Visualizer() + vis.create_window(window_name="SDK 3D Flow", width=1280, height=720) + + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(cloud_points.astype(np.float64)) + pcd.colors = o3d.utility.Vector3dVector(cloud_colors.astype(np.float64)) + vis.add_geometry(pcd) + + line_set = o3d.geometry.LineSet() + line_set.points = o3d.utility.Vector3dVector(flow_points.astype(np.float64)) + line_set.lines = o3d.utility.Vector2iVector(flow_lines) + line_set.colors = o3d.utility.Vector3dVector(flow_colors.astype(np.float64)) + vis.add_geometry(line_set) + + render_option = vis.get_render_option() + render_option.point_size = 1.0 if arrow_map_style else 2.0 + render_option.line_width = 2.0 + render_option.background_color = np.array([0.0, 0.0, 0.0]) + vis.poll_events() + vis.update_renderer() + return vis, pcd, line_set, o3d + + +def run_sdk_3d_flow(args): + data = load_exported_data(args.data_dir) + flatten_remap = load_flatten_remap(args.flatten_remap) if args.flatten_remap else None + view_data = data + if flatten_remap is not None: + if "flat_xyz" not in flatten_remap: + raise ValueError("Flatten remap must contain flat_xyz for 3D visualization.") + view_data = dict(data) + view_data["xyz_map"] = flatten_remap["flat_xyz"] + view_data["height"], view_data["width"] = view_data["xyz_map"].shape[:2] + + valid_mask, display_indices, cloud_points = prepare_static_points(view_data, args.display_stride) + surface_normals = estimate_surface_normals(view_data["xyz_map"]) + flip_vertical = not args.no_flip_vertical + + sensor = orisys.Sensor( + parse_source(args.source), + config_name=args.config, + isstitch=False, + cuda=args.cuda, + verbose=args.verbose, + mirror=not args.no_mirror, + ) + + vis = pcd = line_set = o3d = None + frame_count = 0 + t0 = time.perf_counter() + + try: + while True: + if vis is not None and not vis.poll_events(): + break + + if sensor.get_img() is None: + break + if flip_vertical: + flip_sensor_frames_vertical(sensor) + flat_bgr = None + if flatten_remap is not None: + flat_bgr = flatten_frame(sensor.read_info(sensor.info.IMG)[0], flatten_remap) + flat_gray = cv2.cvtColor(flat_bgr, cv2.COLOR_BGR2GRAY) + sensor.img = flat_gray + sensor._img_bgr_raw = flat_bgr + sensor.img_size_raw = flat_bgr.shape + needs_decomposition = args.vector_mode in ("shear-normal", "depth-map") + sensor.compute_deformation(decompose=needs_decomposition, check_motion=False) + if args.vector_mode == "depth-map": + sensor.compute_contact() + flow, img_bgr, vshear, vnormal, depth_map = sensor.read_info( + sensor.info.VRAW, + sensor.info.IMG, + sensor.info.VSHEAR, + sensor.info.VNORMAL, + sensor.info.DEPTH, + ) + if flat_bgr is not None: + img_bgr = flat_bgr + + active_data = view_data if flatten_remap is not None else data + # Input is already flipped above; flattened domain uses flat_xyz directly. + flip_for_map = False if (flatten_remap is not None or flip_vertical) else True + flow_map = align_vector_field_to_map(flow, active_data, flip_vertical=flip_for_map) + if args.vector_mode in ("vraw-magnitude", "vraw-magnitude-arrows"): + cloud_colors = magnitude_map_colors(flow_map, valid_mask, display_indices, args.magnitude_max) + elif args.vector_mode == "depth-map": + depth_map = align_scalar_field_to_map(depth_map, active_data, flip_vertical=flip_for_map) + cloud_colors = scalar_map_colors( + depth_map, + valid_mask, + display_indices, + value_min=args.depth_min, + value_max=args.depth_max, + ) + elif args.arrow_map_style: + cloud_colors = arrow_map_reference_colors(len(cloud_points)) + else: + cloud_colors = align_image_to_map( + img_bgr, + active_data, + valid_mask, + display_indices, + flip_vertical=flip_for_map, + ) + + if args.vector_mode == "shear-normal": + vshear_map = align_vector_field_to_map(vshear, active_data, flip_vertical=flip_for_map) + vnormal_map = align_vector_field_to_map(vnormal, active_data, flip_vertical=flip_for_map) + flow_points, flow_lines, flow_colors = build_shear_normal_lines( + active_data["xyz_map"], + valid_mask, + vshear_map, + vnormal_map, + surface_normals, + arrow_stride=args.arrow_stride, + flow_scale=args.flow_scale, + min_magnitude=args.min_flow, + normal_gain=args.normal_gain, + arrow_map_style=args.arrow_map_style, + ) + elif args.vector_mode in ("vraw-magnitude", "depth-map"): + flow_points = np.empty((0, 3), dtype=np.float64) + flow_lines = np.empty((0, 2), dtype=np.int32) + flow_colors = np.empty((0, 3), dtype=np.float64) + else: + flow_points, flow_lines, flow_colors = build_flow_lines( + active_data["xyz_map"], + valid_mask, + flow_map, + arrow_stride=args.arrow_stride, + flow_scale=args.flow_scale, + min_magnitude=args.min_flow, + arrow_map_style=args.arrow_map_style, + ) + + if vis is None: + vis, pcd, line_set, o3d = init_viewer( + cloud_points, + cloud_colors, + flow_points, + flow_lines, + flow_colors, + arrow_map_style=args.arrow_map_style, + ) + print("SDK 3D flow viewer started. Close the Open3D window to stop.") + else: + pcd.colors = o3d.utility.Vector3dVector(cloud_colors) + line_set.points = o3d.utility.Vector3dVector(flow_points.astype(np.float64)) + line_set.lines = o3d.utility.Vector2iVector(flow_lines) + line_set.colors = o3d.utility.Vector3dVector(flow_colors.astype(np.float64)) + vis.update_geometry(pcd) + vis.update_geometry(line_set) + vis.update_renderer() + + frame_count += 1 + if frame_count % 30 == 0: + elapsed = max(time.perf_counter() - t0, 1e-6) + print(f"frames={frame_count}, viewer FPS={frame_count / elapsed:.1f}, arrows={len(flow_lines)}") + finally: + sensor.disconnect() + if vis is not None: + vis.destroy_window() + + +def parse_args(): + parser = argparse.ArgumentParser(description="Map SDK flow fields to a 3D point cloud.") + parser.add_argument("--data-dir", default=os.path.join(SCRIPT_DIR, "exported_data")) + parser.add_argument( + "--source", + default=os.path.join(SCRIPT_DIR, "20260602-094916.mp4"), + help="Video path or live camera ID, for example 0, 1, camera:0, or cam:1.", + ) + parser.add_argument("--config", default=os.path.join(REPO_ROOT, "config", "dd02-ov.json")) + parser.add_argument("--display-stride", type=int, default=4) + parser.add_argument("--arrow-stride", type=int, default=32) + parser.add_argument("--flow-scale", type=float, default=3.0) + parser.add_argument("--min-flow", type=float, default=0.5) + parser.add_argument("--normal-gain", type=float, default=1.0) + parser.add_argument("--magnitude-max", type=float, default=None, + help="Upper VRAW magnitude for heat-map color scaling; default uses frame percentile.") + parser.add_argument("--depth-min", type=float, default=None, + help="Minimum depth value for heat-map color scaling; default uses frame percentile.") + parser.add_argument("--depth-max", type=float, default=None, + help="Maximum depth value for heat-map color scaling; default uses frame percentile.") + parser.add_argument("--flatten-remap", default=None, + help="Optional flatten_remap.npz to run optical flow in UV-flattened image space.") + parser.add_argument( + "--vector-mode", + choices=["vraw", "vraw-magnitude", "vraw-magnitude-arrows", "shear-normal", "depth-map"], + default="vraw", + ) + parser.add_argument( + "--arrow-map-style", + action="store_true", + help="Hide real texture and use a dark Qt-viewer-like arrow map style.", + ) + parser.add_argument( + "--no-flip-vertical", + action="store_true", + help="Do not flip captured video vertically before processing.", + ) + parser.add_argument("--no-mirror", action="store_true") + parser.add_argument("--cuda", action="store_true") + parser.add_argument("--verbose", action="store_true") + return parser.parse_args() + + +if __name__ == "__main__": + run_sdk_3d_flow(parse_args()) diff --git a/tools/dd02_3dviewer/texture/patch_texture_0525_N120.png b/tools/dd02_3dviewer/texture/patch_texture_0525_N120.png new file mode 100644 index 0000000..769f728 Binary files /dev/null and b/tools/dd02_3dviewer/texture/patch_texture_0525_N120.png differ diff --git a/tools/dd02_3dviewer/texture/patch_texture_0525_N60.png b/tools/dd02_3dviewer/texture/patch_texture_0525_N60.png new file mode 100644 index 0000000..bf4adc6 Binary files /dev/null and b/tools/dd02_3dviewer/texture/patch_texture_0525_N60.png differ diff --git a/tools/dd02_3dviewer/texture/patch_texture_0525_N75.png b/tools/dd02_3dviewer/texture/patch_texture_0525_N75.png new file mode 100644 index 0000000..5356d64 Binary files /dev/null and b/tools/dd02_3dviewer/texture/patch_texture_0525_N75.png differ diff --git a/tools/dd02_3dviewer/texture/patch_texture_0525_N90.png b/tools/dd02_3dviewer/texture/patch_texture_0525_N90.png new file mode 100644 index 0000000..5775e41 Binary files /dev/null and b/tools/dd02_3dviewer/texture/patch_texture_0525_N90.png differ diff --git a/tools/patch_generation/checker_pattern.png b/tools/patch_generation/checker_pattern.png new file mode 100644 index 0000000..8051c8b Binary files /dev/null and b/tools/patch_generation/checker_pattern.png differ diff --git a/tools/patch_generation/color_pattern.png b/tools/patch_generation/color_pattern.png new file mode 100644 index 0000000..9733268 Binary files /dev/null and b/tools/patch_generation/color_pattern.png differ diff --git a/tools/patch_generation/dd01_checker.svg b/tools/patch_generation/dd01_checker.svg new file mode 100644 index 0000000..5367551 --- /dev/null +++ b/tools/patch_generation/dd01_checker.svg @@ -0,0 +1,49 @@ + + + + diff --git a/tools/patch_generation/patchgen_DD01 checker.py b/tools/patch_generation/patchgen_DD01 checker.py new file mode 100644 index 0000000..3105639 --- /dev/null +++ b/tools/patch_generation/patchgen_DD01 checker.py @@ -0,0 +1,46 @@ +#%% +import numpy as np +import cv2 + + +def generate_checker_pattern(height, width, patch_size): + """ + Generate a black-and-white checkerboard pattern. + + Each `patch_size x patch_size` block alternates between black and white. + Partial blocks at the image borders are also filled correctly. + """ + pattern = np.zeros((height, width, 3), dtype=np.uint8) + + num_patches_x = (height + patch_size - 1) // patch_size + num_patches_y = (width + patch_size - 1) // patch_size + + for i in range(num_patches_x): + for j in range(num_patches_y): + color = 255 if (i + j) % 2 == 0 else 0 + y0 = i * patch_size + y1 = min((i + 1) * patch_size, height) + x0 = j * patch_size + x1 = min((j + 1) * patch_size, width) + pattern[y0:y1, x0:x1] = color + print("Patch ({}, {}) -> {}".format(i, j, "white" if color == 255 else "black")) + + return pattern + + +def main(): + height = 1200 + width = 1000 + patch_size = 12 + + pattern = generate_checker_pattern(height, width, patch_size) + + cv2.imshow("Checker Pattern", pattern) + cv2.waitKey(0) + cv2.destroyAllWindows() + + cv2.imwrite("checker_pattern.png", pattern) + + +if __name__ == '__main__': + main() diff --git a/tools/patch_generation/patchgen_DD03.py b/tools/patch_generation/patchgen_DD03.py new file mode 100644 index 0000000..27341d9 --- /dev/null +++ b/tools/patch_generation/patchgen_DD03.py @@ -0,0 +1,141 @@ +#%% +import numpy as np +import cv2 + +def merge_intervals(intervals): + # Merge overlapping intervals. + if not intervals: + return [] + intervals.sort(key=lambda x: x[0]) + merged = [intervals[0]] + for current in intervals[1:]: + last = merged[-1] + if current[0] <= last[1]: + merged[-1] = (last[0], max(last[1], current[1])) + else: + merged.append(current) + return merged + +def get_allowed_intervals(neighbor_values, delta): + # Compute forbidden intervals for this channel from neighbor values. + forbidden = [] + for c in neighbor_values: + low = max(0, c - delta) + high = min(1, c + delta) + forbidden.append((low, high)) + # Merge overlapping forbidden intervals. + merged = merge_intervals(forbidden) + allowed = [] + # Allowed region before the first forbidden interval. + if merged: + if merged[0][0] > 0: + allowed.append((0, merged[0][0])) + # Allowed regions between forbidden intervals. + for i in range(1, len(merged)): + allowed.append((merged[i-1][1], merged[i][0])) + # Allowed region after the last forbidden interval. + if merged[-1][1] < 1: + allowed.append((merged[-1][1], 1)) + else: + allowed.append((0, 1)) + return allowed + +def sample_from_allowed(allowed): + # Choose a random value from the allowed intervals with probability proportional to interval length. + lengths = [end - start for start, end in allowed] + total_length = sum(lengths) + if total_length <= 0: + return None + point = np.random.rand() * total_length + cumulative = 0 + for (start, end), length in zip(allowed, lengths): + cumulative += length + if point <= cumulative: + return np.random.uniform(start, end) + return allowed[-1][0] + +def sample_channel(neighbor_values, delta): + # Compute allowed intervals and sample a value. + allowed = get_allowed_intervals(neighbor_values, delta) + if not allowed: + return None + return sample_from_allowed(allowed) + +def generate_valid_color(neighbors, r): + """ + Generate a random color that satisfies, for each neighbor, + |R - R_neighbor| >= sqrt(r), and similarly for G and B. + neighbors: list of neighbor colors (each is an array of shape (3,)) + r: threshold for squared difference. + """ + delta = np.sqrt(r) + color = np.zeros(3) + for channel in range(3): + neighbor_channel = [n[channel] for n in neighbors] + sampled = sample_channel(neighbor_channel, delta) + # If the allowed intervals are empty, use a fallback. + if sampled is None: + sampled = np.random.rand() + color[channel] = sampled + return color + +def generate_color_pattern(h, w, d, r): + """ + Generate a dense color pattern. + h, w: height and width of the output image in pixels. + d: patch size in pixels (each patch is d x d). + r: randomness regulation factor. For each neighbor the condition is: + (|channel_new - channel_neighbor|)^2 >= r. + """ + pattern = np.zeros((h, w, 3), dtype=np.float32) + num_patches_x = h // d + num_patches_y = w // d + + for i in range(num_patches_x): + for j in range(num_patches_y): + # Collect neighbor patch colors (using mean because patches have uniform color). + neighbors = [] + if i > 0: + patch_above = pattern[(i-1)*d:i*d, j*d:(j+1)*d] + neighbors.append(np.mean(patch_above, axis=(0, 1))) + if j > 0: + patch_left = pattern[i*d:(i+1)*d, (j-1)*d:j*d] + neighbors.append(np.mean(patch_left, axis=(0, 1))) + + if len(neighbors) == 0: + # For the first patch use a random color. + new_color = np.random.rand(3) + else: + new_color = generate_valid_color(neighbors, r) + + pattern[i*d:(i+1)*d, j*d:(j+1)*d] = new_color + print('Patch ({}, {})'.format(i, j)) + return pattern + +def main(): + # Parameters for the pattern. + # For example, using 100x100 pixels image with patches of 3x3 pixels. + # height = 2750 + # width = 1950 + height = 1340 + width = 1360 + patch_size = 12 + # Set r to a value that gives sufficient difference. + # Note: r is the threshold for squared difference per channel, and the condition is + # |c - c_neighbor| >= sqrt(r). Very high r may make it impossible to satisfy. + r = 1000 + + pattern = generate_color_pattern(height, width, patch_size, r) + # Scale to 0-255 for display and saving. + pattern_display = (pattern * 255).astype(np.uint8) + + # Display the pattern with OpenCV. + cv2.imshow('Pattern', pattern_display) + cv2.waitKey(0) + cv2.destroyAllWindows() + + # Save the pattern as an image file. + cv2.imwrite('color_pattern.png', pattern_display) + +if __name__ == '__main__': + main() diff --git a/tools/patch_generation/w19.5_l27.5_p12.pdf b/tools/patch_generation/w19.5_l27.5_p12.pdf new file mode 100644 index 0000000..8a1bf4d Binary files /dev/null and b/tools/patch_generation/w19.5_l27.5_p12.pdf differ diff --git a/tools/patch_generation/w19.5_l27.5_p12_checker.pdf b/tools/patch_generation/w19.5_l27.5_p12_checker.pdf new file mode 100644 index 0000000..bb22801 Binary files /dev/null and b/tools/patch_generation/w19.5_l27.5_p12_checker.pdf differ diff --git a/tools/patch_generation/彩色+黑白-svg-pdf.zip b/tools/patch_generation/彩色+黑白-svg-pdf.zip new file mode 100644 index 0000000..8d68d23 Binary files /dev/null and b/tools/patch_generation/彩色+黑白-svg-pdf.zip differ