split example visualizaiton pipeline
This commit is contained in:
ykcheungab
+41
-162
@@ -3,7 +3,7 @@
|
||||
Minimum UV-unroll finger tactile pipeline.
|
||||
|
||||
Offline: load Blender export -> build_or_load_domain -> unroll_map_uv.npz
|
||||
Runtime: camera frame -> UVUnrollFrameProcessor -> Sensor -> forces/contact
|
||||
Runtime: camera frame -> UVUnrollFrameProcessor -> TactileProcessor -> forces/contact
|
||||
|
||||
For the full demo (UV debug panels, dashboard, benchmarks) see:
|
||||
tools/3d_patch_generation_blender/unroll_realtime.py
|
||||
@@ -17,190 +17,67 @@ os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0")
|
||||
|
||||
import cv2
|
||||
import orisys
|
||||
from orisys.configs import load_config
|
||||
from orisys.frame_processors import UVUnrollFrameProcessor
|
||||
from orisys.unroll import (
|
||||
build_or_load_domain,
|
||||
compute_uv_distortion_metrics,
|
||||
load_calib_json,
|
||||
load_exported_data,
|
||||
recommend_edge_expand,
|
||||
resolve_uv_runtime_sizes,
|
||||
scale_camera_geometry,
|
||||
scaled_map_cache_path,
|
||||
)
|
||||
|
||||
|
||||
def open_source(video_path, camera_id):
|
||||
return cv2.VideoCapture(video_path) if video_path else cv2.VideoCapture(camera_id)
|
||||
|
||||
|
||||
def resolve_export_paths(export_dir):
|
||||
"""Return Blender export paths expected inside one folder."""
|
||||
export_dir = os.path.abspath(export_dir)
|
||||
paths = {
|
||||
"mesh": os.path.join(export_dir, "mesh.npz"),
|
||||
"intrinsics": os.path.join(export_dir, "camera_intrinsics.npz"),
|
||||
"pose": os.path.join(export_dir, "world_to_cam.npy"),
|
||||
}
|
||||
missing = [name for name, path in paths.items() if not os.path.isfile(path)]
|
||||
if missing:
|
||||
raise FileNotFoundError(
|
||||
f"Missing export file(s) in {export_dir}: {', '.join(missing)}"
|
||||
)
|
||||
return paths["mesh"], paths["intrinsics"], paths["pose"]
|
||||
from orisys.finger import FingerRuntime
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Minimum UV-unroll finger tactile pipeline")
|
||||
parser.add_argument(
|
||||
"--export-dir",
|
||||
required=True,
|
||||
help="Blender export folder containing mesh.npz, camera_intrinsics.npz, world_to_cam.npy",
|
||||
)
|
||||
parser.add_argument("--calib", default=None, help="OpenCV calibration JSON for the live camera")
|
||||
parser.add_argument("--map_cache", default="unroll_map_uv.npz", help="Cached UV domain .npz")
|
||||
parser.add_argument("--camera_id", type=int, default=0)
|
||||
parser.add_argument("--video", help="Video file path (overrides camera)")
|
||||
parser.add_argument("--config", "-c", default="./config/dd02-ov.json", help="Orisys SDK config")
|
||||
parser.add_argument(
|
||||
"--source_scale",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Resize live source before UV remap (default: camera.source_scale in config)",
|
||||
)
|
||||
parser.add_argument("--no_flip_source_vertical", action="store_true", help="Skip vertical flip before UV remap")
|
||||
parser.add_argument("--show", action="store_true", help="Show unrolled image window")
|
||||
parser.add_argument("--show", action="store_true", default=True, help="Show unrolled image window")
|
||||
parser.add_argument("--verbose", action="store_true", help="SDK verbose logs")
|
||||
args = parser.parse_args()
|
||||
|
||||
sdk_config = load_config(args.config)
|
||||
camera_config = sdk_config.get("camera", {})
|
||||
out_width, out_height, source_scale = resolve_uv_runtime_sizes(
|
||||
camera_config,
|
||||
source_scale=args.source_scale,
|
||||
)
|
||||
map_cache = scaled_map_cache_path(args.map_cache, source_scale)
|
||||
try:
|
||||
video = args.camera_id
|
||||
if args.video:
|
||||
video = args.video
|
||||
runtime = FingerRuntime.open(
|
||||
config=args.config,
|
||||
video=video,
|
||||
verbose=args.verbose,
|
||||
)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
mesh_path, intrinsics_path, pose_path = resolve_export_paths(args.export_dir)
|
||||
|
||||
data = load_exported_data(mesh_path, intrinsics_path, pose_path)
|
||||
distortion = compute_uv_distortion_metrics(data)
|
||||
edge_expand = recommend_edge_expand(distortion)
|
||||
|
||||
if args.calib:
|
||||
source_K, dist_coeffs, img_size = load_calib_json(args.calib)
|
||||
native_width, native_height = img_size
|
||||
else:
|
||||
source_K = data["K"]
|
||||
dist_coeffs = None
|
||||
native_width = data["width"]
|
||||
native_height = data["height"]
|
||||
|
||||
source_K, source_width, source_height = scale_camera_geometry(
|
||||
source_K,
|
||||
native_width,
|
||||
native_height,
|
||||
source_scale,
|
||||
)
|
||||
if args.verbose:
|
||||
geometry = runtime.geometry
|
||||
print(
|
||||
f"UV geometry: native={native_width}x{native_height}, "
|
||||
f"source={source_width}x{source_height} (scale={source_scale:g}), "
|
||||
f"output={out_width}x{out_height}, map_cache={map_cache}"
|
||||
f"UV geometry: native={geometry.native_width}x{geometry.native_height}, "
|
||||
f"source={geometry.source_width}x{geometry.source_height} "
|
||||
f"output={geometry.out_width}x{geometry.out_height} "
|
||||
)
|
||||
|
||||
domain = build_or_load_domain(
|
||||
mesh_path,
|
||||
intrinsics_path,
|
||||
pose_path,
|
||||
map_cache,
|
||||
source_K,
|
||||
source_width,
|
||||
source_height,
|
||||
out_width,
|
||||
out_height,
|
||||
edge_expand=edge_expand,
|
||||
include_surface=True,
|
||||
verbose=args.verbose,
|
||||
source_scale=source_scale,
|
||||
)
|
||||
domain.remap.ensure_fast_maps()
|
||||
|
||||
cap = open_source(args.video, args.camera_id)
|
||||
if not cap.isOpened():
|
||||
print("Cannot open camera/video source", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
cap.set(cv2.CAP_PROP_FRAME_WIDTH, native_width)
|
||||
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, native_height)
|
||||
|
||||
sensor = orisys.Sensor("external", config_name=args.config, verbose=args.verbose, mirror=False)
|
||||
processor = UVUnrollFrameProcessor(
|
||||
domain.remap,
|
||||
camera_matrix=source_K if dist_coeffs is not None else None,
|
||||
dist_coeffs=dist_coeffs,
|
||||
source_width=source_width,
|
||||
source_height=source_height,
|
||||
flip_source_vertical=not args.no_flip_source_vertical,
|
||||
)
|
||||
|
||||
ok, frame = cap.read()
|
||||
if ok:
|
||||
if args.show:
|
||||
prepared = processor.prepare_source(frame)
|
||||
sensor.get_img(
|
||||
frame=processor.flatten_prepared(prepared),
|
||||
already_processed=True,
|
||||
valid_mask=processor.valid_mask,
|
||||
)
|
||||
else:
|
||||
prepared_gray = processor.prepare_source_gray(frame)
|
||||
sensor.get_img(
|
||||
frame=processor.flatten_prepared_gray(prepared_gray),
|
||||
already_processed=True,
|
||||
already_gray=True,
|
||||
valid_mask=processor.valid_mask,
|
||||
)
|
||||
sensor.warmup_deformation()
|
||||
runtime.warmup(color=False)
|
||||
tactile = runtime.tactile
|
||||
|
||||
print("Press 'q' to quit, 'r' to reset tracker.")
|
||||
try:
|
||||
while True:
|
||||
ok, frame = cap.read()
|
||||
ok, unrolled = runtime.grab_and_process(color=False)
|
||||
if not ok:
|
||||
break
|
||||
tactile.compute_deformation(check_motion=True, threshold=0)
|
||||
tactile.compute_contact()
|
||||
|
||||
if args.show:
|
||||
prepared = processor.prepare_source(frame)
|
||||
unrolled = processor.flatten_prepared(prepared)
|
||||
sensor.get_img(
|
||||
frame=unrolled,
|
||||
already_processed=True,
|
||||
valid_mask=processor.valid_mask,
|
||||
)
|
||||
else:
|
||||
prepared_gray = processor.prepare_source_gray(frame)
|
||||
unrolled = processor.flatten_prepared_gray(prepared_gray)
|
||||
sensor.get_img(
|
||||
frame=unrolled,
|
||||
already_processed=True,
|
||||
already_gray=True,
|
||||
valid_mask=processor.valid_mask,
|
||||
)
|
||||
sensor.compute_deformation(check_motion=True, threshold=0)
|
||||
sensor.compute_contact()
|
||||
|
||||
fps, flow,fnormal, fshearx, fsheary = sensor.read_info(
|
||||
sensor.info.FPS,
|
||||
sensor.info.VRAW,
|
||||
sensor.info.FNORMAL,
|
||||
sensor.info.FSHEARX,
|
||||
sensor.info.FSHEARY,
|
||||
fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = tactile.read_info(
|
||||
tactile.info.FPS,
|
||||
tactile.info.VRAW,
|
||||
tactile.info.VNORMAL,
|
||||
tactile.info.IMG,
|
||||
tactile.info.CONTOUR,
|
||||
tactile.info.CENTROID,
|
||||
tactile.info.DEPTH,
|
||||
tactile.info.FNORMAL,
|
||||
tactile.info.FSHEARX,
|
||||
tactile.info.FSHEARY
|
||||
)
|
||||
|
||||
print(
|
||||
f"FPS={fps:.2f} normal={fnormal:.2f} shear=({fshearx:.2f}, {fsheary:.2f}) "
|
||||
f"mask={sensor.mask_stats['coverage']:.2f}",
|
||||
f"mask={tactile.mask_stats['coverage']:.2f}",
|
||||
end="\r",
|
||||
)
|
||||
|
||||
@@ -214,6 +91,9 @@ def main():
|
||||
grid_spacing=10,
|
||||
arrow_scale=1.0,
|
||||
)
|
||||
# 步骤 3.2：显示接触区域和质心
|
||||
image_with_foe = orisys.util.draw_contact(img, contour, centroid)
|
||||
cv2.imshow("接触区域与质心", image_with_foe)
|
||||
cv2.imshow("UV Unrolled", arrows)
|
||||
|
||||
key = cv2.waitKey(1) & 0xFF
|
||||
@@ -221,11 +101,10 @@ def main():
|
||||
print("\nExiting.")
|
||||
break
|
||||
if key == ord("r"):
|
||||
sensor.reset()
|
||||
tactile.reset()
|
||||
print("\nTracker reset.")
|
||||
finally:
|
||||
cap.release()
|
||||
sensor.disconnect()
|
||||
runtime.close()
|
||||
cv2.destroyAllWindows()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user