From fbee1c37ed1a5780df403b2f561a9ef469fd22e6 Mon Sep 17 00:00:00 2001 From: ykcheungab Date: Sun, 14 Jun 2026 20:58:46 +0800 Subject: [PATCH] tuned pipeline to 60 fps --- config/dd02-ov.json | 10 ++- examples/finger_pipeline.py | 56 ++++++++++-- src/orisys/hhd/api.py | 7 ++ src/orisys/hhd/domain.py | 62 +++++++++++-- src/orisys/hhd/poisson.py | 16 ++++ src/orisys/hhd/solvers/capacitance.py | 26 +++++- src/orisys/hhd/solvers/masked_sparse.py | 15 ++++ src/orisys/unroll/__init__.py | 4 + src/orisys/unroll/geometry.py | 80 +++++++++++++++++ src/orisys/unroll/precompute.py | 5 ++ src/orisys/unroll/remap.py | 5 ++ .../unroll_realtime.py | 90 +++++++++++++++---- 12 files changed, 339 insertions(+), 37 deletions(-) create mode 100644 src/orisys/unroll/geometry.py diff --git a/config/dd02-ov.json b/config/dd02-ov.json index 23ea55e..35e39c4 100644 --- a/config/dd02-ov.json +++ b/config/dd02-ov.json @@ -5,9 +5,13 @@ "camera": { "resolution_raw": [1280, 720], "fps": 30, - "resolution_stitch": [504, 546], + "resolution_stitch": [280, 300], + "source_scale": 0.4, "old":[672, 728], - "work": [504, 546] + "work": [504, 546,1], + "work2": [420, 450,0.8], + "work3": [350, 375,4,0.7], + "work4": [280, 300,4,0.4] }, "tracker": { "adaptive": true, @@ -18,7 +22,7 @@ "dis_preset": "medium", "dis_mode": 3, "algorithm_list": ["raft", "dis"], - "hhd_downsample": 3 + "hhd_downsample": 5 }, "contact": { "min_contour_area": 3000, diff --git a/examples/finger_pipeline.py b/examples/finger_pipeline.py index 00f0cc4..c7ec68f 100644 --- a/examples/finger_pipeline.py +++ b/examples/finger_pipeline.py @@ -25,6 +25,9 @@ from orisys.unroll import ( load_calib_json, load_exported_data, recommend_edge_expand, + resolve_uv_runtime_sizes, + scale_camera_geometry, + scaled_map_cache_path, ) @@ -60,13 +63,24 @@ def main(): parser.add_argument("--camera_id", type=int, default=0) parser.add_argument("--video", help="Video file path (overrides camera)") parser.add_argument("--config", "-c", default="./config/dd02-ov.json", help="Orisys SDK config") + parser.add_argument( + "--source_scale", + type=float, + default=None, + help="Resize live source before UV remap (default: camera.source_scale in config)", + ) parser.add_argument("--no_flip_source_vertical", action="store_true", help="Skip vertical flip before UV remap") parser.add_argument("--show", action="store_true", help="Show unrolled image window") parser.add_argument("--verbose", action="store_true", help="SDK verbose logs") args = parser.parse_args() sdk_config = load_config(args.config) - out_width, out_height = sdk_config["camera"]["resolution_stitch"] + camera_config = sdk_config.get("camera", {}) + out_width, out_height, source_scale = resolve_uv_runtime_sizes( + camera_config, + source_scale=args.source_scale, + ) + map_cache = scaled_map_cache_path(args.map_cache, source_scale) mesh_path, intrinsics_path, pose_path = resolve_export_paths(args.export_dir) @@ -76,18 +90,31 @@ def main(): if args.calib: source_K, dist_coeffs, img_size = load_calib_json(args.calib) - source_width, source_height = img_size + native_width, native_height = img_size else: source_K = data["K"] dist_coeffs = None - source_width = data["width"] - source_height = data["height"] + native_width = data["width"] + native_height = data["height"] + + source_K, source_width, source_height = scale_camera_geometry( + source_K, + native_width, + native_height, + source_scale, + ) + if args.verbose: + print( + f"UV geometry: native={native_width}x{native_height}, " + f"source={source_width}x{source_height} (scale={source_scale:g}), " + f"output={out_width}x{out_height}, map_cache={map_cache}" + ) domain = build_or_load_domain( mesh_path, intrinsics_path, pose_path, - args.map_cache, + map_cache, source_K, source_width, source_height, @@ -96,6 +123,7 @@ def main(): edge_expand=edge_expand, include_surface=True, verbose=args.verbose, + source_scale=source_scale, ) domain.remap.ensure_fast_maps() @@ -103,8 +131,8 @@ def main(): if not cap.isOpened(): print("Cannot open camera/video source", file=sys.stderr) sys.exit(1) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, source_width) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, source_height) + cap.set(cv2.CAP_PROP_FRAME_WIDTH, native_width) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, native_height) sensor = orisys.Sensor("external", config_name=args.config, verbose=args.verbose, mirror=False) processor = UVUnrollFrameProcessor( @@ -162,8 +190,9 @@ def main(): sensor.compute_deformation(check_motion=True, threshold=0) sensor.compute_contact() - fps, fnormal, fshearx, fsheary = sensor.read_info( + fps, flow,fnormal, fshearx, fsheary = sensor.read_info( sensor.info.FPS, + sensor.info.VRAW, sensor.info.FNORMAL, sensor.info.FSHEARX, sensor.info.FSHEARY, @@ -175,8 +204,17 @@ def main(): end="\r", ) + if args.show: - cv2.imshow("UV Unrolled", unrolled) + + arrows = orisys.util.draw_arrows( + unrolled, + flow, + threshold=0.4, + grid_spacing=10, + arrow_scale=1.0, + ) + cv2.imshow("UV Unrolled", arrows) key = cv2.waitKey(1) & 0xFF if key == ord("q"): diff --git a/src/orisys/hhd/api.py b/src/orisys/hhd/api.py index 42d1a0d..4d89ccc 100644 --- a/src/orisys/hhd/api.py +++ b/src/orisys/hhd/api.py @@ -166,6 +166,13 @@ class nHHD(object): # compute potentials if mask is None and self.poisson_backend == 'opencl' and ('OPENCL_AVAILABLE' in globals() and OPENCL_AVAILABLE): self.nD, self.nRu = self.psolver.solve_opencl_batch2(self.div, self.curlw, verbose > 0, profile=self.debug) + elif mask is not None and self.masked_poisson_solver == 'sparse': + self.nD, self.nRu = self.psolver.solve_masked_pair( + self.div, + self.curlw, + mask, + verbose > 0, + ) else: self.nD = self._solve_poisson_backend(self.div, verbose > 0, valid_mask=mask) self.nRu = self._solve_poisson_backend(self.curlw, verbose > 0, valid_mask=mask) diff --git a/src/orisys/hhd/domain.py b/src/orisys/hhd/domain.py index ade666e..dbaf415 100644 --- a/src/orisys/hhd/domain.py +++ b/src/orisys/hhd/domain.py @@ -57,6 +57,11 @@ class MaskedDomain: "num_active", "pinned", "active_coords", + "active_linear_indices", + "_rhs_buffer", + "_rhs_pair_buffer", + "_scatter_buffer", + "_scatter_pair_buffer", ) def __init__( @@ -69,6 +74,7 @@ class MaskedDomain: num_active, pinned, active_coords, + active_linear_indices, ): self.shape = shape self.mask = mask @@ -78,6 +84,11 @@ class MaskedDomain: self.num_active = num_active self.pinned = pinned self.active_coords = active_coords + self.active_linear_indices = active_linear_indices + self._rhs_buffer = None + self._rhs_pair_buffer = None + self._scatter_buffer = None + self._scatter_pair_buffer = None @classmethod def build(cls, mask, spacings): @@ -101,10 +112,12 @@ class MaskedDomain: num_active=0, pinned=0, active_coords=active_coords, + active_linear_indices=numpy.flatnonzero(active), ) index_map = numpy.full(active.shape, -1, dtype=numpy.int32) index_map[active] = numpy.arange(num_active, dtype=numpy.int32) + active_linear_indices = numpy.flatnonzero(active) dy, dx = float(spacings[0]), float(spacings[1]) coeff_y = 1.0 / (dy * dy) @@ -153,6 +166,7 @@ class MaskedDomain: num_active=num_active, pinned=pinned, active_coords=active_coords, + active_linear_indices=active_linear_indices, ) def matches(self, mask): @@ -166,13 +180,34 @@ class MaskedDomain: f: (H, W) scalar source field returns: (N,) RHS on active pixels; empty if no active pixels """ - rhs = numpy.asarray(f, dtype=numpy.float64)[self.mask] - if rhs.size == 0: - return rhs + if self.num_active == 0: + return numpy.zeros(0, dtype=numpy.float64) + if self._rhs_buffer is None or self._rhs_buffer.shape != (self.num_active,): + self._rhs_buffer = numpy.empty(self.num_active, dtype=numpy.float64) + numpy.take( + numpy.asarray(f, dtype=numpy.float64).ravel(), + self.active_linear_indices, + out=self._rhs_buffer, + ) + rhs = self._rhs_buffer rhs -= rhs.mean() rhs[self.pinned] = 0.0 return rhs + def prepare_rhs_pair(self, f0, f1): + """Extract two gauge-fixed RHS vectors into one reusable (N, 2) buffer.""" + if self.num_active == 0: + return numpy.zeros((0, 2), dtype=numpy.float64) + if self._rhs_pair_buffer is None or self._rhs_pair_buffer.shape != (self.num_active, 2): + self._rhs_pair_buffer = numpy.empty((self.num_active, 2), dtype=numpy.float64) + flat0 = numpy.asarray(f0, dtype=numpy.float64).ravel() + flat1 = numpy.asarray(f1, dtype=numpy.float64).ravel() + numpy.take(flat0, self.active_linear_indices, out=self._rhs_pair_buffer[:, 0]) + numpy.take(flat1, self.active_linear_indices, out=self._rhs_pair_buffer[:, 1]) + self._rhs_pair_buffer -= self._rhs_pair_buffer.mean(axis=0, keepdims=True) + self._rhs_pair_buffer[self.pinned, :] = 0.0 + return self._rhs_pair_buffer + def scatter(self, sol_active, template): """Map reduced solution back to full grid. @@ -180,10 +215,25 @@ class MaskedDomain: template: (H, W) array used for output shape/dtype returns: (H, W) potential; zero outside mask """ - out = numpy.zeros_like(template, dtype=numpy.float32) + if self._scatter_buffer is None or self._scatter_buffer.shape != template.shape: + self._scatter_buffer = numpy.zeros(template.shape, dtype=numpy.float32) + else: + self._scatter_buffer.fill(0) if sol_active.size: - out[self.mask] = numpy.asarray(sol_active, dtype=numpy.float32) - return out + self._scatter_buffer.ravel()[self.active_linear_indices] = numpy.asarray(sol_active, dtype=numpy.float32) + return self._scatter_buffer.copy() + + def scatter_pair(self, sol_active_pair, template): + """Map two reduced solutions back to a pair of full-grid outputs.""" + if self._scatter_pair_buffer is None or self._scatter_pair_buffer.shape != (2,) + template.shape: + self._scatter_pair_buffer = numpy.zeros((2,) + template.shape, dtype=numpy.float32) + else: + self._scatter_pair_buffer.fill(0) + if sol_active_pair.size: + values = numpy.asarray(sol_active_pair, dtype=numpy.float32) + self._scatter_pair_buffer[0].ravel()[self.active_linear_indices] = values[:, 0] + self._scatter_pair_buffer[1].ravel()[self.active_linear_indices] = values[:, 1] + return self._scatter_pair_buffer[0].copy(), self._scatter_pair_buffer[1].copy() def residual(self, field, rhs_active): """Compute masked Laplacian residual L u - rhs on active pixels. diff --git a/src/orisys/hhd/poisson.py b/src/orisys/hhd/poisson.py index 984b91c..793475e 100644 --- a/src/orisys/hhd/poisson.py +++ b/src/orisys/hhd/poisson.py @@ -237,6 +237,22 @@ class PoissonSolver(object): return self._cap_solver.solve(f, domain) return self._sparse_solver.solve(f, domain) + def solve_masked_pair(self, f0, f1, mask, verbose=False): + """Solve two masked RHS fields, using a batched sparse path when available.""" + mask_arr = self._normalize_mask(mask, f0.shape) + if mask_arr is None: + return None, None + if f1.shape != f0.shape: + raise ValueError("Both masked RHS fields must have the same shape") + if not numpy.any(mask_arr): + empty = numpy.zeros_like(f0, dtype=numpy.float32) + return empty, empty.copy() + + domain = self._get_or_build_domain(mask_arr) + if self.masked_config.method == "capfft": + return self._cap_solver.solve(f0, domain), self._cap_solver.solve(f1, domain) + return self._sparse_solver.solve_pair(f0, f1, domain) + def solve(self, f, verbose=False, mask=None): """Solve Poisson equation on the prepared grid. diff --git a/src/orisys/hhd/solvers/capacitance.py b/src/orisys/hhd/solvers/capacitance.py index 7452ba5..6dec36a 100644 --- a/src/orisys/hhd/solvers/capacitance.py +++ b/src/orisys/hhd/solvers/capacitance.py @@ -1,6 +1,7 @@ import numpy from scipy import linalg as dense_linalg +from ...backend_availability import CUPY_AVAILABLE, cp from ..config import MaskedPoissonConfig from ..domain import MaskedDomain, compute_boundary_mask, sample_boundary_coords @@ -14,6 +15,25 @@ class CapacitanceFFTSolver: self._capfft_cache = None self._capfft_domain = None + def _use_cupy_backend(self): + return ( + getattr(self.bulk_solver, "poisson_backend", "cpu") == "cupy" + and CUPY_AVAILABLE + ) + + def _bulk_solve(self, rhs): + """Solve one full-grid Poisson problem, using CuPy when configured.""" + if not self._use_cupy_backend(): + return self.bulk_solver.solve(rhs) + + if not getattr(self.bulk_solver, "ready", False): + self.bulk_solver.prepare(False) + + rhs_gpu = cp.asarray(rhs, dtype=cp.float32) + field_gpu = self.bulk_solver._solve_gpu_internal(rhs_gpu) + cp.cuda.get_current_stream().synchronize() + return cp.asnumpy(field_gpu) + def prepare(self, domain: MaskedDomain): """Precompute capacitance matrix for boundary correction. @@ -40,7 +60,7 @@ class CapacitanceFFTSolver: for col, (row, col_idx) in enumerate(boundary_coords): charge = numpy.zeros(active.shape, dtype=numpy.float32) charge[row, col_idx] = 1.0 - response = numpy.asarray(self.bulk_solver.solve(charge), dtype=numpy.float64) + response = numpy.asarray(self._bulk_solve(charge), dtype=numpy.float64) residual = domain.residual(response, zero_rhs) cap_matrix[:, col] = residual[boundary_active_indices] @@ -82,7 +102,7 @@ class CapacitanceFFTSolver: rhs_full = numpy.zeros_like(f, dtype=numpy.float32) rhs_full[domain.mask] = rhs_active.astype(numpy.float32) - field = numpy.asarray(self.bulk_solver.solve(rhs_full), dtype=numpy.float64) + field = numpy.asarray(self._bulk_solve(rhs_full), dtype=numpy.float64) cap_cache = self.get_cache(domain) if not cap_cache or cap_cache["boundary_count"] == 0: field_active = field[domain.mask] @@ -103,7 +123,7 @@ class CapacitanceFFTSolver: charges = numpy.zeros_like(f, dtype=numpy.float32) coords = cap_cache["boundary_coords"] charges[coords[:, 0], coords[:, 1]] = alpha.astype(numpy.float32) - correction = numpy.asarray(self.bulk_solver.solve(charges), dtype=numpy.float64) + correction = numpy.asarray(self._bulk_solve(charges), dtype=numpy.float64) field -= correction field_active = field[domain.mask] diff --git a/src/orisys/hhd/solvers/masked_sparse.py b/src/orisys/hhd/solvers/masked_sparse.py index 634691a..58c98b2 100644 --- a/src/orisys/hhd/solvers/masked_sparse.py +++ b/src/orisys/hhd/solvers/masked_sparse.py @@ -19,3 +19,18 @@ class MaskedSparseSolver: sol_active = domain.factor(rhs) # (N,) reduced-system potential return domain.scatter(sol_active, f) + + def solve_pair(self, f0, f1, domain: MaskedDomain): + """Solve two masked Poisson RHS with one reused sparse factorization call.""" + rhs_pair = domain.prepare_rhs_pair(f0, f1) + if rhs_pair.shape[0] == 0: + empty = numpy.zeros_like(f0, dtype=numpy.float32) + return empty, empty.copy() + + try: + sol_pair = domain.factor(rhs_pair) # (N, 2) reduced-system potentials + except Exception: + sol_pair = numpy.column_stack( + [domain.factor(rhs_pair[:, 0]), domain.factor(rhs_pair[:, 1])] + ) + return domain.scatter_pair(sol_pair, f0) diff --git a/src/orisys/unroll/__init__.py b/src/orisys/unroll/__init__.py index dba757e..22e82f5 100644 --- a/src/orisys/unroll/__init__.py +++ b/src/orisys/unroll/__init__.py @@ -15,6 +15,7 @@ from .precompute import ( load_exported_data, recommend_edge_expand, ) +from .geometry import resolve_uv_runtime_sizes, scale_camera_geometry, scaled_map_cache_path from .remap import UVRemap, flatten_frame, remap_valid_ratio from .surface import UVSurfaceMap @@ -33,5 +34,8 @@ __all__ = [ "load_exported_data", "recommend_edge_expand", "remap_valid_ratio", + "resolve_uv_runtime_sizes", + "scale_camera_geometry", + "scaled_map_cache_path", "uv_triangle_to_image", ] diff --git a/src/orisys/unroll/geometry.py b/src/orisys/unroll/geometry.py new file mode 100644 index 0000000..f1d1b4d --- /dev/null +++ b/src/orisys/unroll/geometry.py @@ -0,0 +1,80 @@ +"""Camera geometry scaling for faster UV remap at reduced source resolution.""" + +import numpy as np + + +def scale_camera_geometry(source_K, source_width, source_height, scale): + """Scale pinhole intrinsics and image size for a downsampled source frame. + + When the live frame is resized by ``scale`` before undistort/remap, ``fx``, + ``fy``, ``cx``, and ``cy`` must be scaled by the same factor. OpenCV + distortion coefficients are unchanged. + + source_K: (3, 3) camera matrix + source_width, source_height: native calibration / capture size + scale: (0, 1] resize factor applied to width and height + returns: (K_scaled, width_scaled, height_scaled) + """ + scale = float(scale) + if scale <= 0.0 or scale > 1.0: + raise ValueError("source scale must be in (0, 1]") + if np.isclose(scale, 1.0): + return np.asarray(source_K, dtype=np.float64), int(source_width), int(source_height) + + k = np.asarray(source_K, dtype=np.float64).copy() + k[0, 0] *= scale + k[1, 1] *= scale + k[0, 2] *= scale + k[1, 2] *= scale + + width_scaled = max(1, int(round(int(source_width) * scale))) + height_scaled = max(1, int(round(int(source_height) * scale))) + return k, width_scaled, height_scaled + + +def resolve_uv_runtime_sizes(camera_config, out_width=None, out_height=None, source_scale=None): + """Resolve UV output size and source scale from config and CLI overrides. + + Prefers explicit CLI values, then ``camera.uv_output`` or + ``camera.resolution_stitch``, then legacy ``camera.work4``. + Source scale prefers CLI, then ``camera.source_scale``, then ``work4[3]``. + """ + camera_config = camera_config or {} + + stitch = camera_config.get("resolution_stitch") + uv_output = camera_config.get("uv_output") + if out_width is None or out_height is None: + if uv_output and len(uv_output) >= 2: + out_width = out_width if out_width is not None else int(uv_output[0]) + out_height = out_height if out_height is not None else int(uv_output[1]) + elif stitch and len(stitch) >= 2: + out_width = out_width if out_width is not None else int(stitch[0]) + out_height = out_height if out_height is not None else int(stitch[1]) + + if source_scale is None: + if "source_scale" in camera_config: + source_scale = float(camera_config["source_scale"]) + else: + work4 = camera_config.get("work4") + if work4 and len(work4) >= 4: + source_scale = float(work4[3]) + + if out_width is None or out_height is None: + raise ValueError("UV output width/height must be set via CLI or config resolution_stitch") + if source_scale is None: + source_scale = 1.0 + + source_scale = float(source_scale) + if source_scale <= 0.0 or source_scale > 1.0: + raise ValueError("source_scale must be in (0, 1]") + + return int(out_width), int(out_height), source_scale + + +def scaled_map_cache_path(path, source_scale): + """Append a scale suffix so caches for different source scales do not collide.""" + if path is None or np.isclose(source_scale, 1.0): + return path + scale_tag = f"{source_scale:.3f}".rstrip("0").rstrip(".") + base, ext = path.rsplit(".", 1) if "." in path else (path, "npz") + return f"{base}_s{scale_tag}.{ext}" diff --git a/src/orisys/unroll/precompute.py b/src/orisys/unroll/precompute.py index b0403d6..2bd7059 100644 --- a/src/orisys/unroll/precompute.py +++ b/src/orisys/unroll/precompute.py @@ -181,6 +181,7 @@ def build_uv_domain( edge_expand=1.0, include_surface=True, verbose=True, + source_scale=1.0, ): """Build image-to-UV remap and optional UV-to-world surface map.""" faces = data["faces"] @@ -270,6 +271,7 @@ def build_uv_domain( "edge_band": np.array(edge_band, dtype=np.float64), "edge_expand": np.array(edge_expand, dtype=np.float64), "projection_K": source_K.astype(np.float64), + "source_scale": np.array(float(source_scale), dtype=np.float64), "method": np.array("uv"), }, ) @@ -292,6 +294,7 @@ def build_or_load_domain( edge_expand=0.0, include_surface=True, verbose=True, + source_scale=1.0, ): """Build or load full UV domain (2D remap + optional 3D surface). @@ -317,6 +320,7 @@ def build_or_load_domain( flip_v=flip_v, edge_band=edge_band, edge_expand=resolved_edge_expand, + source_scale=source_scale, ) surface_ok = (not include_surface) or (cached.surface is not None) if cache_ok and surface_ok: @@ -338,6 +342,7 @@ def build_or_load_domain( edge_expand=resolved_edge_expand, include_surface=include_surface, verbose=verbose, + source_scale=source_scale, ) valid_ratio = remap_valid_ratio(domain.remap) if verbose: diff --git a/src/orisys/unroll/remap.py b/src/orisys/unroll/remap.py index 27aad68..01767d1 100644 --- a/src/orisys/unroll/remap.py +++ b/src/orisys/unroll/remap.py @@ -37,6 +37,7 @@ class UVRemap: meta_keys = { "source_width", "source_height", + "source_scale", "out_width", "out_height", "flip_v", @@ -121,6 +122,7 @@ def _cache_matches( flip_v, edge_band, edge_expand, + source_scale=1.0, ) -> bool: """True if cached remap matches the requested build parameters.""" if isinstance(remap, UVRemap): @@ -145,6 +147,9 @@ def _cache_matches( return False if "projection_K" in rd and not np.allclose(rd["projection_K"], source_K, rtol=1e-6, atol=1e-6): return False + cached_scale = float(rd["source_scale"]) if "source_scale" in rd else 1.0 + if not np.isclose(cached_scale, float(source_scale), rtol=1e-6, atol=1e-6): + return False return True diff --git a/tools/3d_patch_generation_blender/unroll_realtime.py b/tools/3d_patch_generation_blender/unroll_realtime.py index 5193a9f..57bec0f 100644 --- a/tools/3d_patch_generation_blender/unroll_realtime.py +++ b/tools/3d_patch_generation_blender/unroll_realtime.py @@ -14,6 +14,7 @@ import time import cv2 import numpy as np import orisys +from orisys.configs import load_config from orisys.frame_processors import UVUnrollFrameProcessor from orisys.unroll import ( barycentric_grid, @@ -23,6 +24,9 @@ from orisys.unroll import ( load_calib_json, load_exported_data, recommend_edge_expand, + resolve_uv_runtime_sizes, + scale_camera_geometry, + scaled_map_cache_path, uv_triangle_to_image, ) @@ -304,12 +308,18 @@ def open_source(video_path, camera_id): PERF_TUNING_EPILOG = """ Performance tuning (config JSON + CLI): - tracker.dis_preset medium -> fast/ultrafast faster DIS optical flow - tracker.loop_closure_interval 1 -> 3-5 fewer loop-closure remaps - tracker.hhd_downsample 3 -> 4 smaller HHD Poisson grid - contact.denoise_sigma 12 -> 6 or 0 faster FoE contact step - --masked_poisson_solver sparse vs capfft benchmark HHD solve time - --no-viz skip dashboard/imshow ~20 ms/frame savings + camera.resolution_stitch [280, 300] UV output + SDK flow/HHD grid + camera.source_scale 0.4 shrink source before uv_prep/remap + tracker.dis_preset medium -> fast faster DIS optical flow + tracker.hhd_downsample 5 -> 4 smaller HHD Poisson grid + contact.denoise_sigma 12 -> 6 or 0 faster FoE contact step + --source_scale 0.4 override config source scale + --masked_poisson_solver sparse vs capfft benchmark HHD solve time + --no-viz skip dashboard ~20 ms/frame savings + +Recommended dd02-ov fast profile: + resolution_stitch [280, 300], source_scale 0.4, hhd_downsample 5 + Delete or rename stale unroll_map_uv.npz after changing output scale. """ @@ -349,6 +359,32 @@ def configure_masked_poisson(sensor, args): sensor.masked_poisson_reg = args.masked_poisson_reg +def resolve_runtime_geometry(args): + """Apply config defaults and optional source-scale to UV/camera geometry.""" + sdk_config = load_config(args.sdk_config) + camera_config = sdk_config.get("camera", {}) + out_width, out_height, source_scale = resolve_uv_runtime_sizes( + camera_config, + out_width=args.out_width, + out_height=args.out_height, + source_scale=args.source_scale, + ) + stitch = camera_config.get("resolution_stitch") + if stitch and len(stitch) >= 2 and (int(stitch[0]) != out_width or int(stitch[1]) != out_height): + print( + "Warning: UV output " + f"{out_width}x{out_height} differs from resolution_stitch " + f"{int(stitch[0])}x{int(stitch[1])} in {args.sdk_config}; " + "Sensor.get_img will resize again." + ) + + args.out_width = out_width + args.out_height = out_height + args.source_scale = source_scale + args.map_cache = scaled_map_cache_path(args.map_cache, source_scale) + return sdk_config, camera_config + + def main(): parser = argparse.ArgumentParser( description="Real-time UV-based finger unrolling", @@ -362,8 +398,14 @@ def main(): # With a 12% UV edge band on each side, only about 76% of each axis is the # lower-distortion middle region. Use a slightly larger full remap so the # useful center keeps roughly the same sampling density as the prior 506x550. - parser.add_argument("--out_width", type=int, default=504) - parser.add_argument("--out_height", type=int, default=506) + parser.add_argument("--out_width", type=int, default=None, help="UV flatten width (default: camera.resolution_stitch in sdk config)") + parser.add_argument("--out_height", type=int, default=None, help="UV flatten height (default: camera.resolution_stitch in sdk config)") + parser.add_argument( + "--source_scale", + type=float, + default=None, + help="Resize live source to this fraction before uv_prep/uv_remap; scales K accordingly (default: camera.source_scale or work4[3] in sdk config, else 1.0)", + ) parser.add_argument("--camera_id", type=int, default=0) parser.add_argument("--video", help="Video file path (overrides camera)") parser.add_argument("--map_cache", default="unroll_map_uv.npz") @@ -386,6 +428,7 @@ def main(): help="Skip OpenCV dashboard; print forces and benchmark timings only", ) args = parser.parse_args() + _, camera_config = resolve_runtime_geometry(args) print("[benchmark] Startup timing:") data = benchmark_step("load_exported_data", load_exported_data, args.mesh, args.intrinsics_path, args.pose) @@ -429,15 +472,29 @@ def main(): if args.calib: source_K, dist_coeffs, img_size = benchmark_step("load_calib_json", load_calib_json, args.calib) - source_width, source_height = img_size - print(f"Calibration loaded. Live image size: {(source_width, source_height)}") + native_width, native_height = img_size + print(f"Calibration loaded. Live image size: {(native_width, native_height)}") else: source_K = data["K"] dist_coeffs = None - source_width = data["width"] - source_height = data["height"] + native_width = data["width"] + native_height = data["height"] print("No calibration file provided; using exported intrinsics for projection.") + source_K, source_width, source_height = scale_camera_geometry( + source_K, + native_width, + native_height, + args.source_scale, + ) + print( + "UV runtime geometry: " + f"native={native_width}x{native_height}, " + f"source={source_width}x{source_height} (scale={args.source_scale:g}), " + f"output={args.out_width}x{args.out_height}, " + f"map_cache={args.map_cache}" + ) + domain = benchmark_step( "build_or_load_domain", build_or_load_domain, @@ -454,6 +511,7 @@ def main(): edge_band=args.edge_band, edge_expand=edge_expand, include_surface=True, + source_scale=args.source_scale, ) remap = domain.remap remap.ensure_fast_maps() @@ -495,8 +553,8 @@ def main(): print("Cannot open camera/video source") sys.exit(1) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, source_width) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, source_height) + cap.set(cv2.CAP_PROP_FRAME_WIDTH, native_width) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, native_height) sensor = benchmark_step( "sensor_init", @@ -600,8 +658,8 @@ def main(): arrows = orisys.util.draw_arrows( img, flow, - threshold=1, - grid_spacing=15, + threshold=0.4, + grid_spacing=10, arrow_scale=1.0, ) contact_img = orisys.util.draw_contact(img, contour, centroid)