orisys-tech/orisys-sdk
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update dd02 3d viewer v1

Browse Source
This commit is contained in:
ykcheungab
2026-06-02 12:15:36 +08:00
parent 7224fea411
commit 8a50492465
16 changed files with 2269 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/3d_patch_generation_blender/Readme.md
+121
View File
@@ -0,0 +1,121 @@
# 3D Patch Generation Blender 工具
本文件夹包含 Blender 场景文件以及辅助脚本，用于导出网格体/相机数据，并将一张采集到的 2D 图像重新映射回 3D 表面。
## 主要文件
- `dd02c_v2.blend`
DD02C 曲面与相机配置所使用的主要 Blender 工程文件。
- `blender_export.py`
在 Blender 中运行，用于导出以下数据：
- 网格几何信息
- 与 UV 相关的 loop 数据
- 相机内参
- 相机位姿
- 预计算的像素到 3D 查找表
- `mapto3d.py`
读取导出的数据和一张采集图像，并生成带颜色的 PLY 格式 3D 点云。
## 导出 / 生成的数据
- `exported_data/`
由 `blender_export.py` 生成。通常应包含以下文件：
- `mesh.npz`
- `camera_intrinsics.npz`
- `world_to_cam.npy`
- `pixel_to_xyz.npz`
- `captured_image.png`
要映射回导出 3D 表面的输入图像。当前该图像由 Blender 渲染得到。
- `mapped_points.ply`
由 `mapto3d.py` 生成的输出点云文件。
- `texture/`
保存实验过程中生成的纹理图片。
## 工作流程
### 1. 准备 Blender 场景
在 Blender 中打开 `dd02c_v2.blend`。
请确认：
- 已选中目标网格
- 相机存在且位置正确
- 渲染分辨率与采集图像分辨率一致
### 2. 导出几何和相机数据
在 Blender 的里面上面转到“scripts” 模式，然后点击运行 `blender_export.py`。
该脚本会将所需数据导出到：
```text
exported_data/
```
脚本会写出以下内容：
- 网格顶点和面信息
- UV loop 数据
- OpenCV 风格的相机内参
- world-to-camera 变换矩阵
- 可选的逐像素 3D 查找表
### 3. 准备输入图像
将需要投影到 3D 表面的图像放到当前文件夹中，可以在Blender 渲染生成，例如：
```text
captured_image.png
```
`mapto3d.py` 目前在其 `__main__` 代码块中直接指定输入图像路径。
### 4. 将图像像素映射到 3D
运行：
```bash
python mapto3d.py
```
该脚本会：
- 加载 Blender 导出的数据
- 加载输入图像
- 在需要时对图像做上下翻转，以匹配导出的像素映射顺序
- 优先使用预计算的 `pixel_to_xyz.npz`
- 将结果保存为 `mapped_points.ply`
如果缺少预计算查找表，脚本会退回到逐像素射线求交，运行速度会慢很多。
### 5. 查看输出结果
生成的点云可以使用支持 PLY 的工具打开，例如：
- Open3D
- MeshLab
- CloudCompare
`mapto3d.py` 中也包含可选的 Open3D 可视化步骤。
## 说明
- `mapto3d.py` 目前假设导出的相机模型与输入图像分辨率一致。
- 脚本现在默认会对加载的图像做上下翻转，以保证映射颜色与 Blender 导出的像素顺序一致。
- 为了获得更好的性能，建议在 `exported_data/` 中保留 `pixel_to_xyz.npz`。
## 依赖
### Blender 侧
- Blender Python API：`bpy`
- `numpy`
### Python 侧
- `numpy`
- `opencv-python`
- `open3d`（可选，用于可视化）
tools/3d_patch_generation_blender/dd02c.blend
BIN
View File
Binary file not shown.
tools/3d_patch_generation_blender/mapto3d_video.py
+411
View File
@@ -0,0 +1,411 @@
import argparse
import os
import time
import cv2
import numpy as np
def load_exported_data(data_dir):
"""Load Blender-exported mesh/camera files."""
mesh = np.load(os.path.join(data_dir, "mesh.npz"))
cam_data = np.load(os.path.join(data_dir, "camera_intrinsics.npz"))
xyz_map_path = os.path.join(data_dir, "pixel_to_xyz.npz")
if not os.path.exists(xyz_map_path):
raise FileNotFoundError(
f"Real-time video viewing requires precomputed pixel_to_xyz.npz: {xyz_map_path}"
)
xyz_map = np.load(xyz_map_path)["xyz_map"]
width = int(cam_data["width"])
height = int(cam_data["height"])
if xyz_map.shape[:2] != (height, width):
raise ValueError(
f"pixel_to_xyz shape {xyz_map.shape[:2]} does not match camera size {(height, width)}"
)
return {
"vertices": mesh["vertices"],
"faces": mesh["faces"],
"uv_data": mesh["uv_data"],
"loop_to_vertex": mesh["loop_to_vertex"],
"K": cam_data["K"],
"width": width,
"height": height,
"world_to_cam": np.load(os.path.join(data_dir, "world_to_cam.npy")),
"xyz_map": xyz_map,
}
def prepare_static_points(data, display_stride=4):
"""
Prepare fixed 3D points from the precomputed pixel-to-3D map.
The geometry does not change between video frames, so real-time viewing only
needs to update point colors.
"""
xyz_map = data["xyz_map"]
valid_mask = ~np.isnan(xyz_map[:, :, 0])
valid_points = xyz_map[valid_mask].reshape(-1, 3)
if len(valid_points) == 0:
raise ValueError("pixel_to_xyz.npz contains no valid mesh hit points.")
display_stride = max(1, int(display_stride))
display_indices = np.arange(len(valid_points))[::display_stride]
return valid_mask, display_indices, valid_points[display_indices]
def frame_to_rgb_colors(frame_bgr, data, valid_mask, display_indices, flip_vertical=True):
"""Convert a video frame to RGB point colors aligned with the exported map."""
target_w = int(data["width"])
target_h = int(data["height"])
h, w = frame_bgr.shape[:2]
if (w, h) != (target_w, target_h):
frame_bgr = cv2.resize(frame_bgr, (target_w, target_h), interpolation=cv2.INTER_LINEAR)
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
if flip_vertical:
frame_rgb = np.flipud(frame_rgb).copy()
colors = frame_rgb.astype(np.float64) / 255.0
valid_colors = colors[valid_mask].reshape(-1, 3)
return valid_colors[display_indices]
def init_open3d_viewer(points, colors):
import open3d as o3d
vis = o3d.visualization.Visualizer()
vis.create_window(window_name="Real-time 3D Point Cloud", width=1280, height=720)
pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(points.astype(np.float64))
pcd.colors = o3d.utility.Vector3dVector(colors.astype(np.float64))
vis.add_geometry(pcd)
render_option = vis.get_render_option()
render_option.point_size = 2.0
render_option.background_color = np.array([0.0, 0.0, 0.0])
vis.poll_events()
vis.update_renderer()
return vis, pcd, o3d
def save_colored_pointcloud(points, colors, output_path):
"""Save displayed points as a PLY file."""
valid_colors = (colors * 255).clip(0, 255).astype(np.uint8)
with open(output_path, "w", encoding="utf-8") as f:
f.write("ply\n")
f.write("format ascii 1.0\n")
f.write(f"element vertex {len(points)}\n")
f.write("property float x\n")
f.write("property float y\n")
f.write("property float z\n")
f.write("property uchar red\n")
f.write("property uchar green\n")
f.write("property uchar blue\n")
f.write("end_header\n")
for pt, col in zip(points, valid_colors):
f.write(f"{pt[0]} {pt[1]} {pt[2]} {col[0]} {col[1]} {col[2]}\n")
print(f"Saved point cloud to {output_path}")
def view_video_pointcloud(
video_path,
data,
display_stride=4,
frame_step=1,
flip_vertical=True,
loop=False,
save_snapshot_path=None,
):
"""Open a video and show it as a real-time colored 3D point cloud."""
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
raise FileNotFoundError(f"Could not open video: {video_path}")
valid_mask, display_indices, display_points = prepare_static_points(
data,
display_stride=display_stride,
)
ok, frame = cap.read()
if not ok:
cap.release()
raise RuntimeError(f"Could not read first frame from video: {video_path}")
display_colors = frame_to_rgb_colors(
frame,
data,
valid_mask,
display_indices,
flip_vertical=flip_vertical,
)
vis, pcd, o3d = init_open3d_viewer(display_points, display_colors)
print("Real-time point cloud viewer started.")
print("Close the Open3D window or press Ctrl+C in the terminal to stop.")
print(f"Displayed points: {len(display_points)} (stride={display_stride})")
frame_idx = 0
shown_frames = 0
t0 = time.perf_counter()
try:
while True:
alive = vis.poll_events()
if not alive:
break
ok, frame = cap.read()
if not ok:
if loop:
cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
continue
break
frame_idx += 1
if frame_idx % max(1, frame_step) != 0:
continue
display_colors = frame_to_rgb_colors(
frame,
data,
valid_mask,
display_indices,
flip_vertical=flip_vertical,
)
pcd.colors = o3d.utility.Vector3dVector(display_colors)
vis.update_geometry(pcd)
vis.update_renderer()
shown_frames += 1
if shown_frames % 30 == 0:
elapsed = max(time.perf_counter() - t0, 1e-6)
print(f"Displayed {shown_frames} frames, viewer FPS={shown_frames / elapsed:.1f}")
finally:
if save_snapshot_path:
save_colored_pointcloud(display_points, display_colors, save_snapshot_path)
cap.release()
vis.destroy_window()
def parse_args():
parser = argparse.ArgumentParser(description="View a video as a real-time 3D point cloud.")
parser.add_argument("--data-dir", default="./exported_data", help="Blender exported_data folder")
parser.add_argument("--video", default="20260602-094916.mp4", help="Input video path")
parser.add_argument("--display-stride", type=int, default=4,
help="Show every Nth valid point. Increase for faster viewing.")
parser.add_argument("--frame-step", type=int, default=1,
help="Process every Nth video frame.")
parser.add_argument("--no-flip-vertical", action="store_true",
help="Disable vertical flip before color mapping.")
parser.add_argument("--loop", action="store_true", help="Loop video playback.")
parser.add_argument("--save-snapshot", default="",
help="Optional PLY path for saving the last displayed frame.")
return parser.parse_args()
if __name__ == "__main__":
args = parse_args()
exported = load_exported_data(args.data_dir)
view_video_pointcloud(
args.video,
exported,
display_stride=args.display_stride,
frame_step=args.frame_step,
flip_vertical=not args.no_flip_vertical,
loop=args.loop,
save_snapshot_path=args.save_snapshot or None,
)
import numpy as np
import cv2
import os
def load_exported_data(data_dir):
"""Load all exported files."""
# Mesh
mesh = np.load(os.path.join(data_dir, "mesh.npz"))
vertices = mesh['vertices']
faces = mesh['faces']
uv_data = mesh['uv_data']
loop_to_vertex = mesh['loop_to_vertex']
# Camera intrinsics
cam_data = np.load(os.path.join(data_dir, "camera_intrinsics.npz"))
K = cam_data['K']
width = int(cam_data['width'])
height = int(cam_data['height'])
# Camera pose (world-to-camera)
world_to_cam = np.load(os.path.join(data_dir, "world_to_cam.npy"))
# Precomputed pixel-to-3D map (optional)
xyz_map_path = os.path.join(data_dir, "pixel_to_xyz.npz")
if os.path.exists(xyz_map_path):
xyz_map = np.load(xyz_map_path)['xyz_map']
else:
xyz_map = None
return {
'vertices': vertices,
'faces': faces,
'uv_data': uv_data,
'loop_to_vertex': loop_to_vertex,
'K': K,
'width': width,
'height': height,
'world_to_cam': world_to_cam,
'xyz_map': xyz_map
}
def ray_intersect_triangle(ray_origin, ray_dir, v0, v1, v2):
"""Möller–Trumbore intersection algorithm. Returns t (distance) or None."""
epsilon = 1e-8
edge1 = v1 - v0
edge2 = v2 - v0
h = np.cross(ray_dir, edge2)
a = np.dot(edge1, h)
if abs(a) < epsilon:
return None
f = 1.0 / a
s = ray_origin - v0
u = f * np.dot(s, h)
if u < 0.0 or u > 1.0:
return None
q = np.cross(s, edge1)
v = f * np.dot(ray_dir, q)
if v < 0.0 or u + v > 1.0:
return None
t = f * np.dot(edge2, q)
if t > epsilon:
return t
return None
def pixel_to_3d_ray(u, v, K, world_to_cam):
"""
Given pixel coordinates (u,v) and camera intrinsics K,
return a ray in world coordinates: origin and direction.
"""
# Invert K to get normalized image coordinates
fx = K[0,0]; fy = K[1,1]; cx = K[0,2]; cy = K[1,2]
x_norm = (u - cx) / fx
y_norm = (v - cy) / fy
# Ray in camera space: (x_norm, y_norm, 1.0)
ray_cam = np.array([x_norm, y_norm, 1.0], dtype=np.float64)
# Transform to world
cam_to_world = np.linalg.inv(world_to_cam)
R = cam_to_world[:3, :3]
t = cam_to_world[:3, 3]
ray_world_dir = R @ ray_cam
ray_world_dir /= np.linalg.norm(ray_world_dir)
ray_world_origin = t
return ray_world_origin, ray_world_dir
def map_image_to_3d(image_path, data, use_precomputed=True, flip_vertical=True):
"""
Map each pixel of the image to a 3D world point.
If use_precomputed and xyz_map exists, return that directly.
If `flip_vertical` is True, flip the loaded image top-to-bottom so its
row order matches the exported Blender camera/pixel mapping.
Otherwise, perform raycasting on the mesh.
Returns:
points: (H, W, 3) array of 3D points (NaN for misses).
colors: (H, W, 3) array of RGB colors from image.
"""
img = cv2.imread(image_path)
if img is None:
raise FileNotFoundError(f"Could not read image: {image_path}")
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # to RGB
if flip_vertical:
img = np.flipud(img).copy()
h, w = img.shape[:2]
if use_precomputed and data['xyz_map'] is not None:
# Use precomputed map (must match resolution)
if h != data['height'] or w != data['width']:
raise ValueError("Image resolution does not match precomputed map.")
points = data['xyz_map'].copy()
colors = img.astype(np.float32) / 255.0
return points, colors
# Otherwise, perform raycasting on the mesh
vertices = data['vertices']
faces = data['faces']
K = data['K']
world_to_cam = data['world_to_cam']
points = np.full((h, w, 3), np.nan, dtype=np.float32)
colors = img.astype(np.float32) / 255.0
# For speed, we can use a bounding volume hierarchy (BVH).
# Here we implement a simple brute-force over all faces (slow for large meshes).
# In practice, precomputing the xyz_map is strongly recommended.
for y in range(h):
for x in range(w):
origin, direction = pixel_to_3d_ray(x, y, K, world_to_cam)
best_t = np.inf
best_hit = None
for face in faces:
v0 = vertices[face[0]]
v1 = vertices[face[1]]
v2 = vertices[face[2]]
t = ray_intersect_triangle(origin, direction, v0, v1, v2)
if t is not None and t < best_t:
best_t = t
best_hit = origin + direction * t
if best_hit is not None:
points[y, x] = best_hit
return points, colors
def save_colored_pointcloud(points, colors, output_path):
"""Save as PLY file (colored point cloud)."""
# Flatten valid points
mask = ~np.isnan(points[:,:,0])
valid_points = points[mask].reshape(-1, 3)
valid_colors = (colors[mask].reshape(-1, 3) * 255).astype(np.uint8)
with open(output_path, 'w') as f:
f.write("ply\n")
f.write("format ascii 1.0\n")
f.write(f"element vertex {len(valid_points)}\n")
f.write("property float x\n")
f.write("property float y\n")
f.write("property float z\n")
f.write("property uchar red\n")
f.write("property uchar green\n")
f.write("property uchar blue\n")
f.write("end_header\n")
for pt, col in zip(valid_points, valid_colors):
f.write(f"{pt[0]} {pt[1]} {pt[2]} {col[0]} {col[1]} {col[2]}\n")
print(f"Saved point cloud to {output_path}")
# if __name__ == "__main__":
# data_dir = "./exported_data" # Path to exported files
# image_path = "captured_image_1.jpg" # Input camera image
# output_ply = "mapped_points.ply"
# data = load_exported_data(data_dir)
# points, colors = map_image_to_3d(
# image_path,
# data,
# use_precomputed=True,
# flip_vertical=True,
# )
# save_colored_pointcloud(points, colors, output_ply)
# # Optional: Visualize using Open3D
# import open3d as o3d
# pcd = o3d.geometry.PointCloud()
# mask = ~np.isnan(points[:,:,0])
# pcd.points = o3d.utility.Vector3dVector(points[mask].reshape(-1,3))
# pcd.colors = o3d.utility.Vector3dVector(colors[mask].reshape(-1,3))
# o3d.visualization.draw_geometries([pcd])
tools/3d_patch_generation_blender/random_patch_dd02c.py
+362 -35
View File
@@ -1,6 +1,7 @@
"""
Texture Generator for Arbitrary 3D Surfaces (Blender Script)
- Mode: "patches" (colored patches) or "dots" (circular markers)
- Mode: "patches" (colored patches), "checker" (black/white patches),
or "dots" (circular markers)
- Mesh UVs are filled white so that non‑visible parts become white.
- Texture background (no‑mesh areas) stays black.
"""
@@ -91,6 +92,36 @@ def fill_polygon(image, pixels, width, height, polygon, color):
idx = (y * width + x) * 4
pixels[idx:idx+4] = color
def fill_polygon_value(grid, width, height, polygon, value):
"""Fill a polygon into a 2D scalar grid."""
poly = [(int(round(x)), int(round(y))) for x, y in polygon]
if len(poly) < 3:
return
min_x = max(0, min(p[0] for p in poly))
max_x = min(width - 1, max(p[0] for p in poly))
min_y = max(0, min(p[1] for p in poly))
max_y = min(height - 1, max(p[1] for p in poly))
for y in range(min_y, max_y + 1):
intersections = []
for i in range(len(poly)):
x1, y1 = poly[i]
x2, y2 = poly[(i + 1) % len(poly)]
if y1 == y2:
continue
if (y1 <= y < y2) or (y2 <= y < y1):
t = (y - y1) / (y2 - y1)
x = x1 + t * (x2 - x1)
intersections.append(x)
intersections.sort()
for k in range(0, len(intersections), 2):
if k + 1 >= len(intersections):
break
x_start = int(round(intersections[k]))
x_end = int(round(intersections[k + 1]))
x_start = max(0, min(width - 1, x_start))
x_end = max(0, min(width - 1, x_end))
grid[y, x_start:x_end + 1] = value
def draw_ellipse(pixels, width, height, cx, cy, a, b, angle, color):
"""Filled ellipse with semi‑axes a,b (pixels), rotation angle (rad)."""
cos_a = np.cos(angle)
@@ -152,6 +183,85 @@ def raycast_from_camera(scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, f
return location, face_index
return None, None
def raycast_uv_from_camera(scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height):
"""Camera raycast through normalized sensor coordinates and return UV."""
loc, face_idx = raycast_from_camera(
scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height
)
if loc is None:
return None
return get_uv_from_face(obj, face_idx, loc)
def convex_hull_uv(points):
"""Return UV points ordered around their 2D convex hull."""
unique = sorted({(float(p.x), float(p.y)) for p in points})
if len(unique) < 3:
return None
def cross(o, a, b):
return (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0])
lower = []
for p in unique:
while len(lower) >= 2 and cross(lower[-2], lower[-1], p) <= 0:
lower.pop()
lower.append(p)
upper = []
for p in reversed(unique):
while len(upper) >= 2 and cross(upper[-2], upper[-1], p) <= 0:
upper.pop()
upper.append(p)
hull = lower[:-1] + upper[:-1]
if len(hull) < 3:
return None
return [Vector((u, v)) for u, v in hull]
def sample_patch_uv_polygon(
scene, depsgraph, cam, obj,
u_start, u_end, v_start, v_end,
frame_width, frame_height,
edge_sample_grid=5
):
"""
Build a UV polygon for one camera-space patch.
Full interior cells use the four corners. Edge cells that only partially hit
the mesh are sampled more densely and converted to a convex hull so they are
not dropped just because some corners miss the silhouette.
"""
corners_ndc = [
(u_start, v_start),
(u_end, v_start),
(u_end, v_end),
(u_start, v_end),
]
corner_uvs = []
for u_ndc, v_ndc in corners_ndc:
uv = raycast_uv_from_camera(
scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height
)
if uv is not None:
corner_uvs.append(uv)
if len(corner_uvs) >= 3:
return corner_uvs
sampled_uvs = list(corner_uvs)
for sy in range(edge_sample_grid):
v_ndc = v_start + (v_end - v_start) * sy / (edge_sample_grid - 1)
for sx in range(edge_sample_grid):
u_ndc = u_start + (u_end - u_start) * sx / (edge_sample_grid - 1)
uv = raycast_uv_from_camera(
scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height
)
if uv is not None:
sampled_uvs.append(uv)
return convex_hull_uv(sampled_uvs)
def rasterize_mesh_uvs(obj, pixels, tex_width, tex_height, color=(1,1,1,1)):
"""
Fill all triangles of the active UV map with 'color' (default white).
@@ -175,6 +285,74 @@ def rasterize_mesh_uvs(obj, pixels, tex_width, tex_height, color=(1,1,1,1)):
pixel_poly = [(u * (tex_width - 1), v * (tex_height - 1)) for u, v in uv_poly]
fill_polygon(None, pixels, tex_width, tex_height, pixel_poly, color)
def rasterize_mesh_mask(obj, tex_width, tex_height):
"""Rasterize the active UV map to a boolean mesh mask."""
mask = np.zeros((tex_height, tex_width), dtype=bool)
me = obj.data
uv_layer = me.uv_layers.active
if not uv_layer:
return mask
uv_data = uv_layer.data
for poly in me.polygons:
uv_poly = []
for loop_idx in poly.loop_indices:
uv = uv_data[loop_idx].uv
uv_poly.append((uv.x, uv.y))
pixel_poly = [(u * (tex_width - 1), v * (tex_height - 1)) for u, v in uv_poly]
fill_polygon_value(mask, tex_width, tex_height, pixel_poly, True)
return mask
def paint_mask_color(pixels, width, height, mask, color):
"""Paint all True pixels of a boolean mask with an RGBA color."""
r, g, b, a = color
ys, xs = np.nonzero(mask)
for y, x in zip(ys, xs):
idx = (y * width + x) * 4
pixels[idx:idx+4] = (r, g, b, a)
def complete_checker_gaps(pixels, width, height, mesh_mask, patch_state, max_gap_px=6):
"""
Fill small white slivers inside the mesh by expanding nearby checker colors.
`patch_state`: -1 = unassigned, 0 = black patch, 1 = white patch.
"""
for _ in range(max_gap_px):
candidate = mesh_mask & (patch_state < 0)
if not np.any(candidate):
break
black_counts = np.zeros_like(patch_state, dtype=np.uint8)
white_counts = np.zeros_like(patch_state, dtype=np.uint8)
for dy in (-1, 0, 1):
for dx in (-1, 0, 1):
if dx == 0 and dy == 0:
continue
src_y0 = max(0, -dy)
src_y1 = height - max(0, dy)
src_x0 = max(0, -dx)
src_x1 = width - max(0, dx)
dst_y0 = max(0, dy)
dst_y1 = height - max(0, -dy)
dst_x0 = max(0, dx)
dst_x1 = width - max(0, -dx)
neighbor_vals = patch_state[src_y0:src_y1, src_x0:src_x1]
black_counts[dst_y0:dst_y1, dst_x0:dst_x1] += (neighbor_vals == 0)
white_counts[dst_y0:dst_y1, dst_x0:dst_x1] += (neighbor_vals == 1)
fillable = candidate & ((black_counts + white_counts) > 0)
if not np.any(fillable):
break
patch_state[fillable] = np.where(
black_counts[fillable] > white_counts[fillable], 0, 1
).astype(np.int8)
paint_mask_color(pixels, width, height, patch_state == 0, (0.0, 0.0, 0.0, 1.0))
paint_mask_color(pixels, width, height, patch_state == 1, (1.0, 1.0, 1.0, 1.0))
# ----------------------------------------------------------------------
# Color generation (for patch mode)
# ----------------------------------------------------------------------
@@ -237,6 +415,12 @@ def generate_valid_color(neighbors, r):
color[channel] = sampled
return color
def checker_patch_color(i, j):
"""Return a black/white checker color for patch index (i, j)."""
if (i + j) % 2 == 0:
return np.array([1.0, 1.0, 1.0], dtype=np.float32)
return np.array([0.0, 0.0, 0.0], dtype=np.float32)
# ----------------------------------------------------------------------
# Mode 1: Colored Patches
# ----------------------------------------------------------------------
@@ -244,7 +428,8 @@ def generate_patch_texture(
obj, cam, scene, depsgraph,
nx=40, ny=30, r=0.05,
texture_width=2048, texture_height=2048,
output_path="//patch_texture.png",uv_centre=None, cross_length=40, cross_thickness=3, cross_color=(1,0,0,1)
output_path="//patch_texture.png", uv_centre=None, cross_length=40,
cross_thickness=3, cross_color=(1,0,0,1), edge_sample_grid=5
):
cam_data = cam.data
render = scene.render
@@ -260,8 +445,10 @@ def generate_patch_texture(
frame_width = sensor_width / focal
frame_height = sensor_height / focal
edge_sample_grid = max(2, int(edge_sample_grid))
patch_uv_polygons = []
patch_colors = np.zeros((ny, nx, 3), dtype=np.float32)
skipped_patches = 0
print(f"Raycasting {nx*ny} patches...")
for j in range(ny):
@@ -270,18 +457,15 @@ def generate_patch_texture(
for i in range(nx):
u_start = i / nx
u_end = (i+1) / nx
corners_ndc = [(u_start, v_start), (u_end, v_start),
(u_end, v_end), (u_start, v_end)]
uv_poly = []
for (u_ndc, v_ndc) in corners_ndc:
loc, face_idx = raycast_from_camera(scene, depsgraph, cam, obj,
u_ndc, v_ndc, frame_width, frame_height)
if loc is not None:
uv = get_uv_from_face(obj, face_idx, loc)
if uv is not None:
uv_poly.append(uv)
if len(uv_poly) < 3:
uv_poly = sample_patch_uv_polygon(
scene, depsgraph, cam, obj,
u_start, u_end, v_start, v_end,
frame_width, frame_height,
edge_sample_grid=edge_sample_grid
)
if uv_poly is None:
patch_uv_polygons.append(None)
skipped_patches += 1
continue
patch_uv_polygons.append(uv_poly)
@@ -290,6 +474,7 @@ def generate_patch_texture(
if i > 0: neighbors.append(patch_colors[j, i-1])
color = np.random.rand(3) if len(neighbors)==0 else generate_valid_color(neighbors, r)
patch_colors[j, i] = color
print(f"Skipped {skipped_patches} patches with fewer than 3 mesh samples.")
# Create image – start with black background
img_name = "PatchTexture"
@@ -331,7 +516,105 @@ def generate_patch_texture(
return image
# ----------------------------------------------------------------------
# Mode 2: Circular Dot Markers
# Mode 2: Black/White Checker Patches
# ----------------------------------------------------------------------
def generate_checker_texture(
obj, cam, scene, depsgraph,
nx=40, ny=40,
texture_width=2048, texture_height=2048,
output_path="//checker_texture.png", uv_centre=None, cross_length=40,
cross_thickness=3, cross_color=(1,0,0,1), edge_sample_grid=5,
gap_fill_px=6
):
cam_data = cam.data
render = scene.render
res_x = render.resolution_x
res_y = render.resolution_y
if cam_data.sensor_fit == 'HORIZONTAL':
sensor_width = cam_data.sensor_width
sensor_height = sensor_width / (res_x / res_y)
else:
sensor_height = cam_data.sensor_height
sensor_width = sensor_height * (res_x / res_y)
focal = cam_data.lens
frame_width = sensor_width / focal
frame_height = sensor_height / focal
edge_sample_grid = max(2, int(edge_sample_grid))
patch_uv_polygons = []
patch_colors = np.zeros((ny, nx, 3), dtype=np.float32)
skipped_patches = 0
print(f"Raycasting {nx*ny} checker patches...")
for j in range(ny):
v_start = j / ny
v_end = (j+1) / ny
for i in range(nx):
u_start = i / nx
u_end = (i+1) / nx
uv_poly = sample_patch_uv_polygon(
scene, depsgraph, cam, obj,
u_start, u_end, v_start, v_end,
frame_width, frame_height,
edge_sample_grid=edge_sample_grid
)
if uv_poly is None:
patch_uv_polygons.append(None)
skipped_patches += 1
continue
patch_uv_polygons.append(uv_poly)
patch_colors[j, i] = checker_patch_color(i, j)
print(f"Skipped {skipped_patches} checker patches with fewer than 3 mesh samples.")
img_name = "CheckerTexture"
if img_name in bpy.data.images:
bpy.data.images.remove(bpy.data.images[img_name])
image = bpy.data.images.new(img_name, width=texture_width, height=texture_height, alpha=True)
pixels = list(image.pixels)
mesh_mask = rasterize_mesh_mask(obj, texture_width, texture_height)
patch_state = np.full((texture_height, texture_width), -1, dtype=np.int8)
fill_background(pixels, texture_width, texture_height, (0,0,0,1))
paint_mask_color(pixels, texture_width, texture_height, mesh_mask, (1,1,1,1))
idx = 0
for j in range(ny):
for i in range(nx):
uv_poly = patch_uv_polygons[idx]; idx += 1
if uv_poly is None:
continue
pixel_poly = [(u*(texture_width-1), v*(texture_height-1)) for u,v in uv_poly]
color = patch_colors[j,i]
rgba = (color[0], color[1], color[2], 1.0)
fill_polygon(image, pixels, texture_width, texture_height, pixel_poly, rgba)
fill_polygon_value(
patch_state,
texture_width,
texture_height,
pixel_poly,
1 if np.mean(color) >= 0.5 else 0,
)
complete_checker_gaps(
pixels, texture_width, texture_height, mesh_mask, patch_state, max_gap_px=gap_fill_px
)
if uv_centre is not None:
draw_cross(pixels, texture_width, texture_height,
uv_centre[0], uv_centre[1],
cross_length, cross_thickness, cross_color)
image.pixels[:] = pixels
image.update()
abs_path = bpy.path.abspath(output_path)
image.filepath_raw = abs_path
image.file_format = 'PNG'
image.save()
print(f"Checker texture saved to {abs_path}")
return image
# ----------------------------------------------------------------------
# Mode 3: Circular Dot Markers
# ----------------------------------------------------------------------
def generate_dot_texture(
obj, cam, scene, depsgraph,
@@ -381,34 +664,49 @@ def generate_dot_texture(
( u_ndc, v_ndc - dv_ndc)
]
uv_offsets = []
valid = True
for (u_off, v_off) in offsets:
loc_off, f_idx_off = raycast_from_camera(scene, depsgraph, cam, obj,
u_off, v_off, frame_width, frame_height)
if loc_off is None:
valid = False; break
uv_off = get_uv_from_face(obj, f_idx_off, loc_off)
if uv_off is None:
valid = False; break
uv_off = raycast_uv_from_camera(
scene, depsgraph, cam, obj,
u_off, v_off, frame_width, frame_height
)
uv_offsets.append(uv_off)
if not valid: continue
scale_u = 1.0 / res_x
scale_v = 1.0 / res_y
du_dx = (uv_offsets[0] - uv_offsets[1]) / (2 * offset_pix * scale_u)
dv_dx = (uv_offsets[2] - uv_offsets[3]) / (2 * offset_pix * scale_v)
r_pix = dot_diameter_pixels / 2.0
a_uv = r_pix * du_dx.length * scale_u
b_uv = r_pix * dv_dx.length * scale_v
right_uv, left_uv, up_uv, down_uv = uv_offsets
if right_uv is not None and left_uv is not None:
du_dx = (right_uv - left_uv) / (2 * offset_pix * scale_u)
a_uv = r_pix * du_dx.length * scale_u
elif right_uv is not None:
du_dx = (right_uv - center_uv) / (offset_pix * scale_u)
a_uv = r_pix * du_dx.length * scale_u
elif left_uv is not None:
du_dx = (center_uv - left_uv) / (offset_pix * scale_u)
a_uv = r_pix * du_dx.length * scale_u
else:
du_dx = Vector((1.0, 0.0))
a_uv = r_pix / texture_width
if up_uv is not None and down_uv is not None:
dv_dx = (up_uv - down_uv) / (2 * offset_pix * scale_v)
b_uv = r_pix * dv_dx.length * scale_v
elif up_uv is not None:
dv_dx = (up_uv - center_uv) / (offset_pix * scale_v)
b_uv = r_pix * dv_dx.length * scale_v
elif down_uv is not None:
dv_dx = (center_uv - down_uv) / (offset_pix * scale_v)
b_uv = r_pix * dv_dx.length * scale_v
else:
b_uv = r_pix / texture_height
a_uv = max(a_uv, 1.0 / texture_width)
b_uv = max(b_uv, 1.0 / texture_height)
angle = np.arctan2(du_dx.y, du_dx.x) if du_dx.length > 0 else 0.0
dots.append((center_uv.x, center_uv.y, a_uv, b_uv, angle))
print(f"Found {len(dots)} valid dots.")
if uv_centre is not None:
draw_cross(pixels, texture_width, texture_height,
uv_centre[0], uv_centre[1],
cross_length, cross_thickness, cross_color)
# Create image
img_name = "DotTexture"
@@ -430,6 +728,11 @@ def generate_dot_texture(
draw_ellipse(pixels, texture_width, texture_height,
cx_px, cy_px, a_px, b_px, angle_uv, dot_color)
if uv_centre is not None:
draw_cross(pixels, texture_width, texture_height,
uv_centre[0], uv_centre[1],
cross_length, cross_thickness, cross_color)
image.pixels[:] = pixels
image.update()
abs_path = bpy.path.abspath(output_path)
@@ -444,15 +747,23 @@ def generate_dot_texture(
# ----------------------------------------------------------------------
def main():
# ==================== CONFIGURATION ====================
MODE = "patches"
MODE = "patches" # "patches", "checker", or "dots"
OBJ_NAME = "FingerMesh"
CAM_NAME = "Camera"
TEXTURE_WIDTH = 4096
TEXTURE_HEIGHT = 4096
OUTPUT_PATH = "//" + ("patch_texture.png" if MODE=="patches" else "dot_texture.png")
OUTPUT_NAMES = {
"patches": "patch_texture.png",
"checker": "checker_texture.png",
"dots": "dot_texture.png",
}
OUTPUT_PATH = "//" + OUTPUT_NAMES.get(MODE, "texture.png")
# Patch mode
PATCH_NX = 60; PATCH_NY = 50; COLOR_R = 0.05
EDGE_SAMPLE_GRID = 5
# Checker mode
CHECKER_NX = 40; CHECKER_NY = 40; CHECKER_GAP_FILL_PX = 6
# Dot mode
DOT_DIAMETER_PIX = 12; DOT_SPACING_PIX = 40; DOT_COLOR = (0,0,0,1)
# Cross
@@ -506,7 +817,21 @@ def main():
uv_centre=uv_centre,
cross_length=CROSS_LENGTH_PX,
cross_thickness=CROSS_THICKNESS_PX,
cross_color=CROSS_COLOR
cross_color=CROSS_COLOR,
edge_sample_grid=EDGE_SAMPLE_GRID
)
elif MODE == "checker":
generate_checker_texture(
obj, cam, scene, depsgraph,
nx=CHECKER_NX, ny=CHECKER_NY,
texture_width=TEXTURE_WIDTH, texture_height=TEXTURE_HEIGHT,
output_path=OUTPUT_PATH,
uv_centre=uv_centre,
cross_length=CROSS_LENGTH_PX,
cross_thickness=CROSS_THICKNESS_PX,
cross_color=CROSS_COLOR,
edge_sample_grid=EDGE_SAMPLE_GRID,
gap_fill_px=CHECKER_GAP_FILL_PX
)
elif MODE == "dots":
generate_dot_texture(
@@ -521,6 +846,8 @@ def main():
cross_thickness=CROSS_THICKNESS_PX,
cross_color=CROSS_COLOR
)
else:
raise ValueError(f"Unknown MODE: {MODE!r}. Use 'patches', 'checker', or 'dots'.")
print(f"Completed in {time.time()-start_time:.2f} seconds.")
tools/3d_patch_generation_blender/uv_to_printer.py
-180
View File
@@ -1,180 +0,0 @@
"""
Convert a UV texture map to a physically scaled image for UV printing.
Usage: python uv_to_printer.py --model model.obj --texture texture.png --resolution 100 --output print_ready.png
"""
import argparse
import numpy as np
import cv2
import trimesh
import os
def compute_uv_physical_scale(mesh, uv_layer_name=None):
"""
Compute the physical dimensions (mm) of the UV space.
Assumes the UV map is a reasonably isometric unwrapping (e.g., cylindrical).
Returns: (width_mm, height_mm, scale_factor_u, scale_factor_v)
where scale_factor = physical length per UV unit.
"""
# Get UV coordinates and corresponding 3D positions
if not hasattr(mesh.visual, 'uv') or mesh.visual.uv is None:
raise ValueError("Mesh has no UV coordinates.")
uvs = mesh.visual.uv
faces = mesh.faces
vertices = mesh.vertices
# For each face, we have 3 UVs and 3 vertex positions
# We'll compute the physical edge lengths for all edges in UV space
# and compare to 3D edge lengths to estimate scaling.
total_uv_length = 0.0
total_3d_length = 0.0
edge_count = 0
# Also track UV bounding box for overall dimensions
uv_min = uvs.min(axis=0)
uv_max = uvs.max(axis=0)
uv_range = uv_max - uv_min
# Estimate scale factor by sampling edges
for face in faces:
for i in range(3):
j = (i + 1) % 3
# UV edge
uv0 = uvs[face[i]]
uv1 = uvs[face[j]]
uv_len = np.linalg.norm(uv1 - uv0)
# 3D edge
v0 = vertices[face[i]]
v1 = vertices[face[j]]
len3d = np.linalg.norm(v1 - v0)
if uv_len > 1e-6:
total_uv_length += uv_len
total_3d_length += len3d
edge_count += 1
if edge_count == 0:
raise ValueError("No valid edges found for scale estimation.")
# Average scale factor (physical mm per UV unit)
scale_factor = total_3d_length / total_uv_length
# Physical dimensions of the UV layout
width_mm = uv_range[0] * scale_factor
height_mm = uv_range[1] * scale_factor
print(f"UV bounding box: {uv_range[0]:.3f} x {uv_range[1]:.3f} UV units")
print(f"Scale factor: {scale_factor:.3f} mm per UV unit")
print(f"Physical size: {width_mm:.2f} mm x {height_mm:.2f} mm")
return width_mm, height_mm, scale_factor
def generate_print_image(texture_path, width_mm, height_mm, pixels_per_mm, output_path):
"""
Load the texture image, scale it to match the physical dimensions at the
specified resolution, and save the result.
"""
# Read texture (with alpha if present)
texture = cv2.imread(texture_path, cv2.IMREAD_UNCHANGED)
if texture is None:
raise FileNotFoundError(f"Cannot load texture: {texture_path}")
# Compute target pixel dimensions
target_width_px = int(round(width_mm * pixels_per_mm))
target_height_px = int(round(height_mm * pixels_per_mm))
print(f"Target image size: {target_width_px} x {target_height_px} pixels")
# Resize texture (using INTER_LINEAR for up/down scaling)
resized = cv2.resize(texture, (target_width_px, target_height_px),
interpolation=cv2.INTER_LINEAR)
# Save output (preserve alpha if present)
cv2.imwrite(output_path, resized)
print(f"Print-ready image saved to: {output_path}")
def process_exported_npz(npz_path, texture_path, pixels_per_mm, output_path):
"""
Alternative: load data exported from Blender script (vertices, faces, UVs).
"""
data = np.load(npz_path)
vertices = data['vertices']
faces = data['faces']
uv_data = data['uv_data']
loop_to_vertex = data['loop_to_vertex']
# Reconstruct per-vertex UVs (average)
# For simplicity, we'll compute physical scale using face edges as in trimesh method.
# We need to build a temporary mesh-like structure.
# Count number of faces
num_faces = len(faces)
# Build face UVs array (3 per face)
# The uv_data is per-loop, we need to group by face.
# Assuming faces are triangles.
face_uvs = []
face_vertices = []
uv_idx = 0
for face in faces:
f_uvs = []
f_verts = []
for _ in range(len(face)):
f_uvs.append(uv_data[uv_idx])
v_idx = loop_to_vertex[uv_idx]
f_verts.append(vertices[v_idx])
uv_idx += 1
face_uvs.append(np.array(f_uvs))
face_vertices.append(np.array(f_verts))
total_uv_len = 0.0
total_3d_len = 0.0
count = 0
uv_min = np.min(uv_data, axis=0)
uv_max = np.max(uv_data, axis=0)
uv_range = uv_max - uv_min
for f_uv, f_v in zip(face_uvs, face_vertices):
for i in range(3):
j = (i+1)%3
uv_len = np.linalg.norm(f_uv[i] - f_uv[j])
v_len = np.linalg.norm(f_v[i] - f_v[j])
if uv_len > 1e-6:
total_uv_len += uv_len
total_3d_len += v_len
count += 1
scale = total_3d_len / total_uv_len if count>0 else 1.0
width_mm = uv_range[0] * scale
height_mm = uv_range[1] * scale
print(f"Physical size from .npz: {width_mm:.2f} x {height_mm:.2f} mm")
generate_print_image(texture_path, width_mm, height_mm, pixels_per_mm, output_path)
def main():
parser = argparse.ArgumentParser(description="Convert UV texture to physically scaled print image.")
parser.add_argument("--model", help="3D model file (OBJ, PLY, STL) with UVs")
parser.add_argument("--npz", help="Alternatively, use exported .npz from Blender script")
parser.add_argument("--texture", required=True, help="Input texture image (PNG, JPG)")
parser.add_argument("--resolution", type=float, default=100.0,
help="Pixels per millimeter (default: 100)")
parser.add_argument("--output", default="print_ready.png", help="Output image path")
args = parser.parse_args()
if not args.model and not args.npz:
parser.error("Either --model or --npz must be provided.")
if args.npz:
process_exported_npz(args.npz, args.texture, args.resolution, args.output)
else:
# Load mesh with trimesh
mesh = trimesh.load(args.model, force='mesh')
if not isinstance(mesh, trimesh.Trimesh):
raise ValueError("Loaded object is not a single mesh.")
width_mm, height_mm, scale = compute_uv_physical_scale(mesh)
generate_print_image(args.texture, width_mm, height_mm, args.resolution, args.output)
if __name__ == "__main__":
main()
tools/dd02_3dviewer/exported_data/camera_intrinsics.npz
BIN
View File
Binary file not shown.
tools/dd02_3dviewer/exported_data/mesh.npz
BIN
View File
Binary file not shown.
tools/dd02_3dviewer/exported_data/pixel_to_xyz.npz
BIN
View File
Binary file not shown.
tools/dd02_3dviewer/exported_data/world_to_cam.npy
BIN
View File
Binary file not shown.
tools/3d_patch_generation_blender/mapto3d.py → tools/dd02_3dviewer/mapto3d.py
+14 -3
View File
@@ -85,17 +85,23 @@ def pixel_to_3d_ray(u, v, K, world_to_cam):
return ray_world_origin, ray_world_dir
def map_image_to_3d(image_path, data, use_precomputed=True):
def map_image_to_3d(image_path, data, use_precomputed=True, flip_vertical=True):
"""
Map each pixel of the image to a 3D world point.
If use_precomputed and xyz_map exists, return that directly.
If `flip_vertical` is True, flip the loaded image top-to-bottom so its
row order matches the exported Blender camera/pixel mapping.
Otherwise, perform raycasting on the mesh.
Returns:
points: (H, W, 3) array of 3D points (NaN for misses).
colors: (H, W, 3) array of RGB colors from image.
"""
img = cv2.imread(image_path)
if img is None:
raise FileNotFoundError(f"Could not read image: {image_path}")
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # to RGB
if flip_vertical:
img = np.flipud(img).copy()
h, w = img.shape[:2]
if use_precomputed and data['xyz_map'] is not None:
@@ -161,11 +167,16 @@ def save_colored_pointcloud(points, colors, output_path):
if __name__ == "__main__":
data_dir = "./exported_data" # Path to exported files
image_path = "captured_image.png" # Input camera image
image_path = "captured_image_1.jpg" # Input camera image
output_ply = "mapped_points.ply"
data = load_exported_data(data_dir)
points, colors = map_image_to_3d(image_path, data, use_precomputed=True)
points, colors = map_image_to_3d(
image_path,
data,
use_precomputed=True,
flip_vertical=True,
)
save_colored_pointcloud(points, colors, output_ply)
# Optional: Visualize using Open3D
tools/dd02_3dviewer/sdk_mapto3d.py
+424
View File
@@ -0,0 +1,424 @@
import argparse
import os
import sys
import time
import cv2
import numpy as np
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
REPO_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
SRC_DIR = os.path.join(REPO_ROOT, "src")
if SRC_DIR not in sys.path:
sys.path.insert(0, SRC_DIR)
import orisys
def load_exported_data(data_dir):
mesh = np.load(os.path.join(data_dir, "mesh.npz"))
cam_data = np.load(os.path.join(data_dir, "camera_intrinsics.npz"))
xyz_map_path = os.path.join(data_dir, "pixel_to_xyz.npz")
if not os.path.exists(xyz_map_path):
raise FileNotFoundError(f"Missing precomputed 3D map: {xyz_map_path}")
xyz_map = np.load(xyz_map_path)["xyz_map"]
width = int(cam_data["width"])
height = int(cam_data["height"])
if xyz_map.shape[:2] != (height, width):
raise ValueError(
f"pixel_to_xyz shape {xyz_map.shape[:2]} does not match camera size {(height, width)}"
)
return {
"vertices": mesh["vertices"],
"faces": mesh["faces"],
"uv_data": mesh["uv_data"],
"loop_to_vertex": mesh["loop_to_vertex"],
"K": cam_data["K"],
"width": width,
"height": height,
"world_to_cam": np.load(os.path.join(data_dir, "world_to_cam.npy")),
"xyz_map": xyz_map,
}
def parse_source(value):
try:
return int(value)
except ValueError:
return value
def prepare_static_points(data, display_stride):
xyz_map = data["xyz_map"]
valid_mask = ~np.isnan(xyz_map[:, :, 0])
valid_points = xyz_map[valid_mask].reshape(-1, 3)
if len(valid_points) == 0:
raise ValueError("pixel_to_xyz.npz contains no valid mesh hit points.")
display_indices = np.arange(len(valid_points))[::max(1, int(display_stride))]
return valid_mask, display_indices, valid_points[display_indices]
def align_image_to_map(img_bgr, data, valid_mask, display_indices, flip_vertical=True):
target_w = int(data["width"])
target_h = int(data["height"])
h, w = img_bgr.shape[:2]
if (w, h) != (target_w, target_h):
img_bgr = cv2.resize(img_bgr, (target_w, target_h), interpolation=cv2.INTER_LINEAR)
img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
if flip_vertical:
img_rgb = np.flipud(img_rgb).copy()
colors = img_rgb.astype(np.float64) / 255.0
return colors[valid_mask].reshape(-1, 3)[display_indices]
def arrow_map_reference_colors(count, color=(0.04, 0.04, 0.04)):
return np.tile(np.asarray(color, dtype=np.float64), (count, 1))
def magnitude_map_colors(flow, valid_mask, display_indices, max_magnitude=None):
mag = np.linalg.norm(flow, axis=2)
valid_mag = mag[valid_mask].reshape(-1)
display_mag = valid_mag[display_indices]
if max_magnitude is None or max_magnitude <= 0:
max_magnitude = max(float(np.percentile(display_mag, 95)), 1e-6)
value = np.clip(display_mag / max_magnitude, 0.0, 1.0)
return np.column_stack(
(
value,
np.clip(1.0 - np.abs(value - 0.5) * 2.0, 0.0, 1.0),
1.0 - value,
)
).astype(np.float64)
def align_vector_field_to_map(field, data, flip_vertical=True, scale_as_flow=True):
target_w = int(data["width"])
target_h = int(data["height"])
h, w = field.shape[:2]
if (w, h) != (target_w, target_h):
sx = target_w / max(w, 1)
sy = target_h / max(h, 1)
field = cv2.resize(field, (target_w, target_h), interpolation=cv2.INTER_LINEAR)
if scale_as_flow:
field[:, :, 0] *= sx
field[:, :, 1] *= sy
else:
field = field.copy()
if flip_vertical:
field = np.flipud(field).copy()
field[:, :, 1] *= -1.0
return field
def estimate_surface_normals(xyz_map):
valid = ~np.isnan(xyz_map[:, :, 0])
filled = xyz_map.copy()
filled[~valid] = 0
d_y = np.gradient(filled, axis=0)
d_x = np.gradient(filled, axis=1)
normals = np.cross(d_x, d_y)
norm = np.linalg.norm(normals, axis=2, keepdims=True)
normals = np.divide(normals, norm, out=np.zeros_like(normals), where=norm > 1e-8)
normals[~valid] = 0
return normals
def line_colors(mag, arrow_map_style):
if len(mag) == 0:
return np.empty((0, 3), dtype=np.float64)
if arrow_map_style:
return np.tile(np.array([0.0, 0.9, 1.0], dtype=np.float64), (len(mag), 1))
norm_mag = mag / max(float(np.percentile(mag, 95)), 1e-6)
norm_mag = np.clip(norm_mag, 0.0, 1.0)
return np.column_stack((norm_mag, 1.0 - norm_mag, np.zeros_like(norm_mag))).astype(np.float64)
def lines_from_vectors(starts, vectors, mag, flow_scale, arrow_map_style):
vec_norm = np.linalg.norm(vectors, axis=1)
good = np.isfinite(vec_norm) & (vec_norm > 1e-8)
starts, vectors, mag = starts[good], vectors[good], mag[good]
if len(starts) == 0:
return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3))
points = np.empty((len(starts) * 2, 3), dtype=np.float64)
points[0::2] = starts
points[1::2] = starts + vectors * flow_scale
lines = np.column_stack((np.arange(0, len(points), 2), np.arange(1, len(points), 2))).astype(np.int32)
return points, lines, line_colors(mag, arrow_map_style)
def build_flow_lines(
xyz_map,
valid_mask,
flow,
arrow_stride=32,
flow_scale=1.0,
min_magnitude=0.5,
arrow_map_style=False,
):
h, w = flow.shape[:2]
ys, xs = np.mgrid[0:h:arrow_stride, 0:w:arrow_stride]
ys = ys.reshape(-1)
xs = xs.reshape(-1)
sample_flow = flow[ys, xs]
mag = np.linalg.norm(sample_flow, axis=1)
keep = valid_mask[ys, xs] & (mag >= min_magnitude)
xs, ys, sample_flow, mag = xs[keep], ys[keep], sample_flow[keep], mag[keep]
if len(xs) == 0:
return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3))
tx = np.rint(xs + sample_flow[:, 0]).astype(np.int32)
ty = np.rint(ys + sample_flow[:, 1]).astype(np.int32)
inside = (0 <= tx) & (tx < w) & (0 <= ty) & (ty < h)
xs, ys, tx, ty, mag = xs[inside], ys[inside], tx[inside], ty[inside], mag[inside]
if len(xs) == 0:
return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3))
valid_target = valid_mask[ty, tx]
xs, ys, tx, ty, mag = xs[valid_target], ys[valid_target], tx[valid_target], ty[valid_target], mag[valid_target]
starts = xyz_map[ys, xs]
vectors = xyz_map[ty, tx] - starts
return lines_from_vectors(starts, vectors, mag, flow_scale, arrow_map_style)
def build_shear_normal_lines(
xyz_map,
valid_mask,
vshear,
vnormal,
normals,
arrow_stride=32,
flow_scale=1.0,
min_magnitude=0.01,
normal_gain=1.0,
arrow_map_style=False,
):
h, w = vshear.shape[:2]
ys, xs = np.mgrid[0:h:arrow_stride, 0:w:arrow_stride]
ys = ys.reshape(-1)
xs = xs.reshape(-1)
shear = vshear[ys, xs]
normal_field = vnormal[ys, xs]
shear_mag = np.linalg.norm(shear, axis=1)
normal_mag = np.linalg.norm(normal_field, axis=1) * normal_gain
combined_mag = shear_mag + normal_mag
keep = valid_mask[ys, xs] & (combined_mag >= min_magnitude)
xs, ys, shear, shear_mag, normal_mag, combined_mag = (
xs[keep],
ys[keep],
shear[keep],
shear_mag[keep],
normal_mag[keep],
combined_mag[keep],
)
if len(xs) == 0:
return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3))
tx = np.rint(xs + shear[:, 0]).astype(np.int32)
ty = np.rint(ys + shear[:, 1]).astype(np.int32)
inside = (0 <= tx) & (tx < w) & (0 <= ty) & (ty < h)
xs, ys, tx, ty, shear_mag, normal_mag, combined_mag = (
xs[inside],
ys[inside],
tx[inside],
ty[inside],
shear_mag[inside],
normal_mag[inside],
combined_mag[inside],
)
if len(xs) == 0:
return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3))
valid_target = valid_mask[ty, tx]
xs, ys, tx, ty, shear_mag, normal_mag, combined_mag = (
xs[valid_target],
ys[valid_target],
tx[valid_target],
ty[valid_target],
shear_mag[valid_target],
normal_mag[valid_target],
combined_mag[valid_target],
)
starts = xyz_map[ys, xs]
tangent = xyz_map[ty, tx] - starts
tangent_norm = np.linalg.norm(tangent, axis=1, keepdims=True)
tangent_unit = np.divide(tangent, tangent_norm, out=np.zeros_like(tangent), where=tangent_norm > 1e-8)
normal_unit = normals[ys, xs]
shear_ratio = shear_mag / np.maximum(combined_mag, 1e-8)
normal_ratio = normal_mag / np.maximum(combined_mag, 1e-8)
vectors = tangent_unit * shear_ratio[:, None] + normal_unit * normal_ratio[:, None]
vector_norm = np.linalg.norm(vectors, axis=1, keepdims=True)
vectors = np.divide(vectors, vector_norm, out=np.zeros_like(vectors), where=vector_norm > 1e-8)
return lines_from_vectors(starts, vectors, combined_mag, flow_scale, arrow_map_style)
def init_viewer(cloud_points, cloud_colors, flow_points, flow_lines, flow_colors, arrow_map_style=False):
import open3d as o3d
vis = o3d.visualization.Visualizer()
vis.create_window(window_name="SDK 3D Flow", width=1280, height=720)
pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(cloud_points.astype(np.float64))
pcd.colors = o3d.utility.Vector3dVector(cloud_colors.astype(np.float64))
vis.add_geometry(pcd)
line_set = o3d.geometry.LineSet()
line_set.points = o3d.utility.Vector3dVector(flow_points.astype(np.float64))
line_set.lines = o3d.utility.Vector2iVector(flow_lines)
line_set.colors = o3d.utility.Vector3dVector(flow_colors.astype(np.float64))
vis.add_geometry(line_set)
render_option = vis.get_render_option()
render_option.point_size = 1.0 if arrow_map_style else 2.0
render_option.line_width = 2.0
render_option.background_color = np.array([0.0, 0.0, 0.0])
vis.poll_events()
vis.update_renderer()
return vis, pcd, line_set, o3d
def run_sdk_3d_flow(args):
data = load_exported_data(args.data_dir)
valid_mask, display_indices, cloud_points = prepare_static_points(data, args.display_stride)
surface_normals = estimate_surface_normals(data["xyz_map"])
sensor = orisys.Sensor(
parse_source(args.source),
config_name=args.config,
isstitch=False,
cuda=args.cuda,
verbose=args.verbose,
mirror=not args.no_mirror,
)
vis = pcd = line_set = o3d = None
frame_count = 0
t0 = time.perf_counter()
try:
while True:
if vis is not None and not vis.poll_events():
break
if sensor.get_img() is None:
break
sensor.compute_deformation(decompose=(args.vector_mode == "shear-normal"), check_motion=False)
flow, img_bgr, vshear, vnormal = sensor.read_info(
sensor.info.VRAW,
sensor.info.IMG,
sensor.info.VSHEAR,
sensor.info.VNORMAL,
)
flow_map = align_vector_field_to_map(flow, data, flip_vertical=not args.no_flip_vertical)
if args.vector_mode == "vraw-magnitude":
cloud_colors = magnitude_map_colors(flow_map, valid_mask, display_indices, args.magnitude_max)
elif args.arrow_map_style:
cloud_colors = arrow_map_reference_colors(len(cloud_points))
else:
cloud_colors = align_image_to_map(
img_bgr,
data,
valid_mask,
display_indices,
flip_vertical=not args.no_flip_vertical,
)
if args.vector_mode == "shear-normal":
vshear_map = align_vector_field_to_map(vshear, data, flip_vertical=not args.no_flip_vertical)
vnormal_map = align_vector_field_to_map(vnormal, data, flip_vertical=not args.no_flip_vertical)
flow_points, flow_lines, flow_colors = build_shear_normal_lines(
data["xyz_map"],
valid_mask,
vshear_map,
vnormal_map,
surface_normals,
arrow_stride=args.arrow_stride,
flow_scale=args.flow_scale,
min_magnitude=args.min_flow,
normal_gain=args.normal_gain,
arrow_map_style=args.arrow_map_style,
)
elif args.vector_mode == "vraw-magnitude":
flow_points = np.empty((0, 3), dtype=np.float64)
flow_lines = np.empty((0, 2), dtype=np.int32)
flow_colors = np.empty((0, 3), dtype=np.float64)
else:
flow_points, flow_lines, flow_colors = build_flow_lines(
data["xyz_map"],
valid_mask,
flow_map,
arrow_stride=args.arrow_stride,
flow_scale=args.flow_scale,
min_magnitude=args.min_flow,
arrow_map_style=args.arrow_map_style,
)
if vis is None:
vis, pcd, line_set, o3d = init_viewer(
cloud_points,
cloud_colors,
flow_points,
flow_lines,
flow_colors,
arrow_map_style=args.arrow_map_style,
)
print("SDK 3D flow viewer started. Close the Open3D window to stop.")
else:
pcd.colors = o3d.utility.Vector3dVector(cloud_colors)
line_set.points = o3d.utility.Vector3dVector(flow_points.astype(np.float64))
line_set.lines = o3d.utility.Vector2iVector(flow_lines)
line_set.colors = o3d.utility.Vector3dVector(flow_colors.astype(np.float64))
vis.update_geometry(pcd)
vis.update_geometry(line_set)
vis.update_renderer()
frame_count += 1
if frame_count % 30 == 0:
elapsed = max(time.perf_counter() - t0, 1e-6)
print(f"frames={frame_count}, viewer FPS={frame_count / elapsed:.1f}, arrows={len(flow_lines)}")
finally:
sensor.disconnect()
if vis is not None:
vis.destroy_window()
def parse_args():
parser = argparse.ArgumentParser(description="Map SDK flow fields to a 3D point cloud.")
parser.add_argument("--data-dir", default=os.path.join(SCRIPT_DIR, "exported_data"))
parser.add_argument("--source", default=os.path.join(SCRIPT_DIR, "20260602-094916.mp4"))
parser.add_argument("--config", default=os.path.join(REPO_ROOT, "config", "dd02-ov.json"))
parser.add_argument("--display-stride", type=int, default=4)
parser.add_argument("--arrow-stride", type=int, default=8)
parser.add_argument("--flow-scale", type=float, default=3.0)
parser.add_argument("--min-flow", type=float, default=0.5)
parser.add_argument("--normal-gain", type=float, default=1.0)
parser.add_argument("--magnitude-max", type=float, default=None,
help="Upper VRAW magnitude for heat-map color scaling; default uses frame percentile.")
parser.add_argument("--vector-mode", choices=["vraw", "vraw-magnitude", "shear-normal"], default="vraw")
parser.add_argument(
"--arrow-map-style",
action="store_true",
help="Hide real texture and use a dark Qt-viewer-like arrow map style.",
)
parser.add_argument("--no-flip-vertical", action="store_true")
parser.add_argument("--no-mirror", action="store_true")
parser.add_argument("--cuda", action="store_true")
parser.add_argument("--verbose", action="store_true")
return parser.parse_args()
if __name__ == "__main__":
run_sdk_3d_flow(parse_args())