orisys-sdk/examples/finger_pipeline_3dview.py

#!/usr/bin/env python3
"""
UV-unroll finger tactile pipeline with 3D mesh visualization.

Offline: load Blender export -> build_or_load_domain -> unroll_map_uv.npz
Runtime:  camera frame -> UVUnrollFrameProcessor -> TactileProcessor -> forces/contact
          UV-space flow/HHD -> grey-white 3D finger mesh (Open3D)

For the full 2D dashboard demo see:
  tools/3d_patch_generation_blender/unroll_realtime.py
"""

import argparse
import os
import sys
from pathlib import Path

os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0")

import cv2
from orisys.finger import DEFAULT_FINGER_EXPORT_DIR, DEFAULT_FINGER_MAP_CACHE, FingerRuntime

_EXAMPLES_DIR = Path(__file__).resolve().parent
if str(_EXAMPLES_DIR) not in sys.path:
    sys.path.insert(0, str(_EXAMPLES_DIR))

from viz import UVMeshViewer, build_uv_surface_mesh, show_finger_uv_panels


def open_mesh_viewer(runtime, args):
    domain = runtime.domain
    if domain.surface is None:
        raise RuntimeError(
            "UV domain has no world_xyz surface map. Rebuild cache with include_surface=True."
        )

    uv_mesh = build_uv_surface_mesh(domain.surface.world_xyz, domain.valid_mask)
    if args.no_3d:
        return None, uv_mesh

    viewer = UVMeshViewer(
        uv_mesh,
        max_magnitude=args.magnitude_max,
        threshold=args.depth_threshold if args.mesh_map == "depth" else args.flow_threshold,
        color_mode=args.mesh_map,
    )
    print(
        f"3D mesh viewer: {len(uv_mesh.rest_vertices)} vertices, "
        f"{len(uv_mesh.triangles)} triangles, {args.mesh_map} surface mode"
    )
    return viewer, uv_mesh


def main():
    parser = argparse.ArgumentParser(
        description="UV-unroll finger tactile pipeline with 3D mesh viewer"
    )
    parser.add_argument(
        "--export-dir",
        default=DEFAULT_FINGER_EXPORT_DIR,
        help="Blender export folder containing mesh.npz, camera_intrinsics.npz, world_to_cam.npy",
    )
    parser.add_argument("--map_cache", default=DEFAULT_FINGER_MAP_CACHE, help="Cached UV domain .npz")
    parser.add_argument("--camera_id", type=int, default=0)
    parser.add_argument("--video", help="Video file path (overrides camera)")
    parser.add_argument("--config", "-c", default="./config/dd02-ov.json", help="Orisys SDK config")
    parser.add_argument(
        "--source_scale",
        type=float,
        default=None,
        help="Resize live source before UV remap (default: camera.source_scale in config)",
    )
    parser.add_argument("--no_flip_source_vertical", action="store_true", help="Skip vertical flip before UV remap")
    parser.add_argument("--show", action="store_true", help="Show 2D unrolled image window with flow arrows")
    parser.add_argument("--verbose", action="store_true", help="SDK verbose logs")
    parser.add_argument(
        "--magnitude-max",
        type=float,
        default=None,
        help="Upper optical-flow magnitude for 3D surface heat-map scaling; default uses frame percentile.",
    )
    parser.add_argument(
        "--mesh-map",
        choices=["flow", "depth"],
        default="depth",
        help="Color the 3D mesh from optical-flow magnitude or the contact depth/divergence map.",
    )
    parser.add_argument(
        "--flow-threshold",
        type=float,
        default=0.4,
        help="Minimum optical-flow magnitude to color 3D surface / draw 2D arrows (matches draw_arrows).",
    )
    parser.add_argument(
        "--depth-threshold",
        type=float,
        default=0.0,
        help="Minimum depth/divergence value to color on the 3D mesh.",
    )
    parser.add_argument("--no-3d", action="store_true", help="Disable Open3D mesh viewer")
    args = parser.parse_args()

    video = args.camera_id
    if args.video:
        video = args.video

    try:
        runtime = FingerRuntime.open(
            export_dir=args.export_dir,
            config=args.config,
            video=video,
            verbose=args.verbose,
        )
    except RuntimeError as exc:
        print(str(exc), file=sys.stderr)
        sys.exit(1)

    if args.verbose:
        geometry = runtime.geometry
        print(
            f"UV geometry: native={geometry.native_width}x{geometry.native_height}, "
            f"source={geometry.source_width}x{geometry.source_height} "
            f"(scale={geometry.source_scale:g}), output={geometry.out_width}x{geometry.out_height}, "
            f"map_cache={geometry.map_cache}"
        )

    mesh_viewer, _uv_mesh = open_mesh_viewer(runtime, args)
    runtime.warmup(color=False)
    tactile = runtime.tactile

    print("Press 'q' to quit, 'r' to reset tracker. Close the 3D window to stop.")
    try:
        while True:
            ok, unrolled = runtime.grab_and_process(color=False)
            if not ok:
                break

            tactile.compute_deformation(decompose=True, check_motion=True, threshold=0)
            tactile.compute_contact()

            fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = tactile.read_info(
                tactile.info.FPS,
                tactile.info.VRAW,
                tactile.info.VNORMAL,
                tactile.info.IMG,
                tactile.info.CONTOUR,
                tactile.info.CENTROID,
                tactile.info.DEPTH,
                tactile.info.FNORMAL,
                tactile.info.FSHEARX,
                tactile.info.FSHEARY,
            )
            flow_mag = float((flow[:, :, 0] ** 2 + flow[:, :, 1] ** 2).mean() ** 0.5) if flow.size else 0.0

            print(
                f"FPS={fps:.2f} flow_rms={flow_mag:.2f} "
                f"mask={tactile.mask_stats['coverage']:.2f} normal={fnormal:.2f} shear=({fshearx:.2f}, {fsheary:.2f})",
                end="\r",
            )

            if mesh_viewer is not None:
                if not mesh_viewer.update(flow=flow, scalar_map=depth_map):
                    print("\n3D viewer closed.")
                    break

            if args.show:
                show_finger_uv_panels(
                    unrolled,
                    vnormal,
                    depth_map,
                    flow_threshold=args.flow_threshold,
                )

            key = cv2.waitKey(1) & 0xFF
            if key == ord("q"):
                print("\nExiting.")
                break
            if key == ord("r"):
                tactile.reset()
                print("\nTracker reset.")
    finally:
        runtime.close()
        if mesh_viewer is not None:
            mesh_viewer.close()
        cv2.destroyAllWindows()


if __name__ == "__main__":
    main()