diff --git a/build/build_nuitka.py b/build/build_nuitka.py index 23249bf..6b20ed2 100644 --- a/build/build_nuitka.py +++ b/build/build_nuitka.py @@ -20,7 +20,7 @@ from paths import NUITKA_COMPILE_DIR, NUITKA_WHEEL_DIR, REPO_ROOT, repo_path from pyproject_utils import bundle_items, extras_require, get_version, install_requires from verify import verify_built_wheel from wheel_build import publish_wheel_to_sdk, run_bdist_wheel, write_wheel_setup -from wheel_layout import copy_bundle_items, package_data_from_bundle +from wheel_layout import copy_bundle_items, copy_orisys_blend, package_data_from_bundle def _reset_dir(path) -> None: @@ -51,6 +51,10 @@ def build() -> None: ext_suffix = copy_compiled_orisys(pkg_dir, ext_file) + wheel_root = NUITKA_WHEEL_DIR / "wheel_root" + blend_count = copy_orisys_blend(REPO_ROOT, wheel_root) + print(f"Staged orisys_blend Python modules: {blend_count}") + # Bundle rules live in pyproject.toml → [[tool.orisys.bundle.items]] bundle = bundle_items(REPO_ROOT) copied_counts = copy_bundle_items(REPO_ROOT, pkg_dir, bundle) diff --git a/build/nuitka_compile.py b/build/nuitka_compile.py index cd87e0d..0217987 100644 --- a/build/nuitka_compile.py +++ b/build/nuitka_compile.py @@ -19,6 +19,7 @@ NUITKA_INCLUDE_PACKAGES = ( ) NUITKA_NOFOLLOW_IMPORTS = ( + "orisys_blend", "cupy", "cupyx", "pyopencl", diff --git a/build/wheel_build.py b/build/wheel_build.py index e23edaa..0fd1f33 100644 --- a/build/wheel_build.py +++ b/build/wheel_build.py @@ -31,7 +31,7 @@ setup( version="{version}", description="Orisys - Optical Tactile Sensor SDK (Nuitka Compiled)", author="Orisys", - packages=['orisys', 'orisys.examples'], + packages=['orisys', 'orisys.examples', 'orisys_blend'], package_dir={{'': 'wheel_root'}}, package_data={package_data}, include_package_data=True, diff --git a/build/wheel_layout.py b/build/wheel_layout.py index 8409458..66549e6 100644 --- a/build/wheel_layout.py +++ b/build/wheel_layout.py @@ -83,9 +83,22 @@ def package_data_from_bundle(ext_suffix: str, items: list[dict]) -> dict[str, li package_data: dict[str, list[str]] = { "orisys": [f"*{ext_suffix}"], "orisys.examples": ["*.py", "README.md", "viz/**/*.py", "pipeline/**/*.py", "gui/**/*.py"], + "orisys_blend": ["*.py"], } for item in items: if item["name"] == "examples": continue package_data["orisys"].extend(package_data_paths_for_item(item)) return package_data + + +def copy_orisys_blend(repo_root: Path, wheel_root: Path) -> int: + """Copy pure-Python orisys_blend (Numba kernels) next to the Nuitka orisys .pyd.""" + src = repo_root / "src" / "orisys_blend" + dst = wheel_root / "orisys_blend" + if dst.exists(): + shutil.rmtree(dst) + shutil.copytree(src, dst) + count = sum(1 for p in dst.rglob("*.py") if p.is_file()) + print(f"Copied orisys_blend ({count} .py files) to {dst}") + return count diff --git a/config/ddjx01.json b/config/ddjx01.json index 21b5e96..af88413 100644 --- a/config/ddjx01.json +++ b/config/ddjx01.json @@ -18,7 +18,9 @@ "nominal_ov_pos": 100, "n": [3,3], "row_tol": 0.6, - "box_size": [250, 250] + "box_size": [250, 250], + "blend":"numba", + "blend_list": ["numpy", "numba","none"] }, "tracker": { "adaptive": true, diff --git a/config/ddjx01_480p.json b/config/ddjx01_480p.json index 6083c90..e58f998 100644 --- a/config/ddjx01_480p.json +++ b/config/ddjx01_480p.json @@ -18,7 +18,9 @@ "nominal_ov_pos": 44, "n": [3,3], "row_tol": 0.6, - "box_size": [98, 98] + "box_size": [98, 98], + "blend":"numpy", + "blend_list": ["numpy", "numba","none"] }, "tracker": { "adaptive": true, diff --git a/config/ddjx01_q6a.json b/config/ddjx01_q6a.json index f267045..d6a1335 100644 --- a/config/ddjx01_q6a.json +++ b/config/ddjx01_q6a.json @@ -18,7 +18,9 @@ "nominal_ov_pos": 67, "n": [3,3], "row_tol": 0.6, - "box_size": [180, 180] + "box_size": [180, 180], + "blend":"numpy", + "blend_list": ["numpy", "numba","none"] }, "tracker": { "adaptive": true, diff --git a/config/ddjx01_wuxi.json b/config/ddjx01_wuxi.json index 0f0a874..cbbe434 100644 --- a/config/ddjx01_wuxi.json +++ b/config/ddjx01_wuxi.json @@ -18,7 +18,9 @@ "nominal_ov_pos": 67, "n": [3,3], "row_tol": 0.6, - "box_size": [180, 180] + "box_size": [180, 180], + "blend":"numpy", + "blend_list": ["numpy", "numba","none"] }, "tracker": { "adaptive": true, diff --git a/config/prototype_of3.json b/config/prototype_of3.json deleted file mode 100644 index 8d94471..0000000 --- a/config/prototype_of3.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "name": "prototype", - "version": "1.0", - "camera": { - "resolution_raw": [1920, 1080], - "fps": 30, - "units": 15, - "units_used": 9, - "resolution_stitch": [365, 365], - "stitch_res_list": [365,263,123,63] - }, - "stitcher": { - "min_area_frac": 0.01, - "max_area_frac": 0.04, - "max_aspect_ratio": 100, - "index": [1,2,3,6,7,8,11,12,13], - "margin": 10, - "nominal_ov_pos": 100, - "n": [3,3], - "row_tol": 0.6, - "box_size": [220, 220] - }, - "tracker": { - "adaptive": true, - "lower_threshold": 12.0, - "upper_threshold": 18.0, - "loop_closure_interval": 1, - "algorithm": "dis", - "dis_preset": "medium", - "dis_mode": 3, - "algorithm_list": ["raft", "dis"], - "hhd_downsample": 3 - }, - "contact": { - "min_contour_area": 3000, - "flow_threshold":1, - "div_threshold":0.02, - "expansion_threshold": 0.01, - "kernel_size": [5,5], - "denoise_sigma": 15 - } -} - - diff --git a/config/prototype_of3_480p.json b/config/prototype_of3_480p.json deleted file mode 100644 index 5c321a3..0000000 --- a/config/prototype_of3_480p.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "name": "prototype", - "version": "1.0", - "camera": { - "resolution_raw": [854, 480], - "fps": 30, - "units": 15, - "units_used": 9, - "resolution_stitch": [123, 123], - "stitch_res_list": [365,263,123,63] - }, - "stitcher": { - "min_area_frac": 0.01, - "max_area_frac": 0.06, - "max_aspect_ratio": 100, - "index": [1,2,3,6,7,8,11,12,13], - "margin": 4, - "nominal_ov_pos": 44, - "n": [3,3], - "row_tol": 0.6, - "box_size": [98, 98] - }, - "tracker": { - "adaptive": true, - "lower_threshold": 12.0, - "upper_threshold": 18.0, - "loop_closure_interval": 1, - "algorithm": "dis", - "dis_preset": "medium", - "dis_mode": 2, - "algorithm_list": ["raft", "dis"], - "hhd_downsample": 3 - }, - "contact": { - "min_contour_area": 3000, - "flow_threshold":1, - "div_threshold":0.02, - "expansion_threshold": 0.01, - "kernel_size": [5,5], - "denoise_sigma": 6 - } -} - - diff --git a/config/prototype_of3_720p.json b/config/prototype_of3_720p.json deleted file mode 100644 index 185ccbf..0000000 --- a/config/prototype_of3_720p.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "name": "prototype", - "version": "1.0", - "camera": { - "resolution_raw": [1280, 720], - "fps": 30, - "units": 15, - "units_used": 9, - "resolution_stitch": [263, 263], - "stitch_res_list": [365,263,123,63] - }, - "stitcher": { - "min_area_frac": 0.01, - "max_area_frac": 0.04, - "max_aspect_ratio": 25, - "index": [1,2,3,6,7,8,11,12,13], - "margin": 10, - "nominal_ov_pos": 67, - "n": [3,3], - "row_tol": 0.6, - "box_size": [180, 180] - }, - "tracker": { - "adaptive": true, - "lower_threshold": 12.0, - "upper_threshold": 18.0, - "loop_closure_interval": 1, - "algorithm": "dis", - "dis_preset": "medium", - "dis_mode": 2, - "algorithm_list": ["raft", "dis"], - "hhd_downsample": 3 - }, - "contact": { - "min_contour_area": 3000, - "flow_threshold":1, - "div_threshold":0.02, - "expansion_threshold": 0.01, - "kernel_size": [5,5], - "denoise_sigma": 10 - } -} - - diff --git a/config/prototype_raft.json b/config/prototype_raft.json deleted file mode 100644 index 480995b..0000000 --- a/config/prototype_raft.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "name": "prototype", - "version": "1.0", - "camera": { - "resolution_raw": [1920, 1080], - "fps": 30, - "units": 15, - "units_used": 9, - "resolution_stitch": [384, 384] - }, - "stitcher": { - "min_area_frac": 0.01, - "max_area_frac": 0.04, - "max_aspect_ratio": 100, - "index": [1,2,3,6,7,8,11,12,13], - "margin": 10, - "nominal_ov_pos": 100, - "n": [3,3], - "row_tol": 0.6, - "box_size": [220, 220] - }, - "tracker": { - "adaptive": true, - "lower_threshold": 12.0, - "upper_threshold": 18.0, - "loop_closure_interval": 1, - "algorithm": "raft", - "dis_preset": "medium", - "dis_mode": 3, - "algorithm_list": ["raft", "dis"], - "hhd_downsample": 3 - }, - "contact": { - "min_contour_area": 3000, - "flow_threshold":1.5, - "div_threshold":0.08, - "expansion_threshold": 0.01, - "kernel_size": [5,5], - "denoise_sigma": 15 - } -} - - diff --git a/pyproject.toml b/pyproject.toml index c627639..226ad5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ cuda13 = ["cupy-cuda13x>=13.0.0"] # Backward compatibility: 'cuda' defaults to CUDA 12.x cuda = ["cupy-cuda12x>=12.3.0"] gui = ["pyqt5>=5.15", "pyqtgraph>=0.13"] +numba = ["numba>=0.59.0"] [tool.setuptools] package-dir = { "" = "src" } diff --git a/src/orisys/Stitcher.py b/src/orisys/Stitcher.py index b4d1068..02a2719 100644 --- a/src/orisys/Stitcher.py +++ b/src/orisys/Stitcher.py @@ -5,45 +5,92 @@ from scipy import ndimage import cv2 try: - from numba import njit + from orisys_blend.kernel import _NUMBA_AVAILABLE, blend_kernel as _blend_kernel_numba +except Exception: + _NUMBA_AVAILABLE = False + _blend_kernel_numba = None - _NUMBA_AVAILABLE = True -except ImportError: +# Re-export availability from the pure-Python kernel package when import succeeded +# but numba itself was missing (blend_kernel is None). +if _blend_kernel_numba is None: _NUMBA_AVAILABLE = False - def njit(*args, **kwargs): - def decorator(fn): - return fn - return decorator +def _blend_kernel_numpy(blend_patch, wsum_patch, cov_patch, src_patch, w_patch): + """Incremental weighted-average composite (pure NumPy, no JIT). + + Uses contiguous arithmetic instead of boolean fancy indexing. Callers must + zero ``blend`` / ``wsum`` on uncovered pixels (as the stitcher reset does); + then the covered lerp also paints first-touch pixels correctly when ``w > 0``. + """ + w = w_patch if w_patch.dtype == np.float32 else np.asarray(w_patch, dtype=np.float32) + src = src_patch if src_patch.dtype == np.float32 else np.asarray(src_patch, dtype=np.float32) + + uncovered = cov_patch == 0 + if uncovered.all(): + # First paint over a cleared patch: direct copy. + np.copyto(blend_patch, src) + np.copyto(wsum_patch, w) + cov_patch.fill(1) + return + + # Contiguous lerp. With zeroed uncovered pixels this also equals a first paint + # when w > 0 (a ≈ 1 → blend becomes src). + ws = wsum_patch + w + a = w / np.maximum(ws, np.float32(1e-6)) + # Channel loop keeps temps HxW instead of one HxWx3 gather/scatter. + for c in range(src.shape[2]): + bc = blend_patch[..., c] + bc += (src[..., c] - bc) * a + np.copyto(wsum_patch, ws) + + # Match numba on rare first-touch pixels with ~0 weight (lerp would leave 0). + if uncovered.any(): + need_fix = uncovered & (w < np.float32(1e-6)) + if need_fix.any(): + blend_patch[need_fix] = src[need_fix] + wsum_patch[need_fix] = w[need_fix] + + cov_patch.fill(1) -@njit(cache=False) -def _blend_kernel(blend_patch, wsum_patch, cov_patch, src_patch, w_patch): - """Fused incremental weighted-average composite (numba-accelerated).""" - sh = src_patch.shape[0] - sw = src_patch.shape[1] - for y in range(sh): - for x in range(sw): - w = w_patch[y, x] - s0 = np.float32(src_patch[y, x, 0]) - s1 = np.float32(src_patch[y, x, 1]) - s2 = np.float32(src_patch[y, x, 2]) - if cov_patch[y, x] == 0: - blend_patch[y, x, 0] = s0 - blend_patch[y, x, 1] = s1 - blend_patch[y, x, 2] = s2 - wsum_patch[y, x] = w - else: - ws = wsum_patch[y, x] + w - if ws < 1e-6: - ws = 1e-6 - a = w / ws - blend_patch[y, x, 0] += (s0 - blend_patch[y, x, 0]) * a - blend_patch[y, x, 1] += (s1 - blend_patch[y, x, 1]) * a - blend_patch[y, x, 2] += (s2 - blend_patch[y, x, 2]) * a - wsum_patch[y, x] = ws - cov_patch[y, x] = 1 +# Backward-compatible alias used by older call sites / tests. +_blend_kernel = _blend_kernel_numpy + +# JSON ``stitcher.blend`` / ``blend_list``: none = fast paste; numpy/numba = feathered. +_BLEND_MODES = frozenset({"none", "numpy", "numba"}) +_DEFAULT_BLEND_MODE = "none" + + +def _resolve_blend_mode(config_stitcher=None): + """Resolve stitcher blend mode from env, then JSON ``blend``. + + Returns ``(mode, fell_back_from_numba)`` where mode is one of + ``none`` / ``numpy`` / ``numba``. + + Priority: ``ORISYS_STITCH_BLEND``, ``ORISYS_STITCH_BLEND_BACKEND``, + then ``config_stitcher["blend"]`` / legacy ``blend_backend``, then ``none``. + """ + requested = None + for key in ("ORISYS_STITCH_BLEND", "ORISYS_STITCH_BLEND_BACKEND"): + env = os.environ.get(key, "").strip().lower() + if env in _BLEND_MODES: + requested = env + break + + if requested is None and isinstance(config_stitcher, dict): + for key in ("blend", "blend_backend"): + cfg = str(config_stitcher.get(key, "")).strip().lower() + if cfg in _BLEND_MODES: + requested = cfg + break + + if requested is None: + requested = _DEFAULT_BLEND_MODE + + if requested == "numba" and not _NUMBA_AVAILABLE: + return "numpy", True + return requested, False class Stitcher: @@ -85,25 +132,100 @@ class Stitcher: self._feature_stitch_print_every = 30 self._feature_stitch_pad = 5 self._feature_stitch_match_brightness = False + self._blend_mode, fell_back = _resolve_blend_mode(config_stitcher) + # Feather path uses numpy/numba; ``none`` keeps a numpy fallback if forced on. + self._blend_backend = "numpy" if self._blend_mode == "none" else self._blend_mode + self._numba_failed = False - if self.isstitch and not _NUMBA_AVAILABLE: + if fell_back and self.isstitch: print( - "Warning: numba is not installed; feature stitching will run a slow " - "Python blend path (~1 FPS). Install numba for real-time performance." + "Warning: stitcher blend=numba requested but numba is " + "unavailable; using numpy blend path." ) + elif self.isstitch and self.verbose: + print(f"Stitch blend: {self._blend_mode}") if self.verbose: print(f"Stitcher initialized with config: {self.config_stitcher}") - def configure_feature_stitch(self, *, match_brightness=None, pad=None, print_every=None): - """Update runtime options for feature-based stitching.""" + @property + def blend_mode(self): + """Configured blend mode: ``none``, ``numpy``, or ``numba``.""" + return self._blend_mode + + def configure_feature_stitch( + self, + *, + match_brightness=None, + pad=None, + print_every=None, + blend=None, + blend_backend=None, + ): + """Update runtime options for feature-based stitching. + + ``blend`` accepts ``none`` / ``numpy`` / ``numba`` (same as JSON ``stitcher.blend``). + ``blend_backend`` is a legacy alias for the same values. + """ if match_brightness is not None: self._feature_stitch_match_brightness = bool(match_brightness) if pad is not None: self._feature_stitch_pad = max(0, int(pad)) if print_every is not None: self._feature_stitch_print_every = max(1, int(print_every)) - + mode = blend if blend is not None else blend_backend + if mode is not None: + mode = str(mode).strip().lower() + if mode not in _BLEND_MODES: + raise ValueError( + f"Unknown blend={mode!r}; expected one of {sorted(_BLEND_MODES)}" + ) + if mode == "numba" and (not _NUMBA_AVAILABLE or self._numba_failed): + print("Warning: numba unavailable; using numpy blend.") + mode = "numpy" + self._blend_mode = mode + self._blend_backend = "numpy" if mode == "none" else mode + if mode == "numba": + self._numba_failed = False + + def _resolve_call_blend(self, blend): + """Map a ``stitch_rt_optimized(blend=...)`` arg to ``(use_feather, backend)``.""" + if blend is None: + mode = self._blend_mode + elif isinstance(blend, bool): + if blend: + mode = self._blend_mode if self._blend_mode != "none" else "numpy" + else: + return False, self._blend_backend + else: + mode = str(blend).strip().lower() + if mode not in _BLEND_MODES: + raise ValueError( + f"Unknown blend={mode!r}; expected one of {sorted(_BLEND_MODES)}" + ) + if mode == "numba" and not _NUMBA_AVAILABLE: + mode = "numpy" + + if mode == "numba" and self._numba_failed: + mode = "numpy" + + if mode == "none": + return False, self._blend_backend + return True, mode + + def _mark_numba_failed(self, exc): + """Stick to numpy after a Numba JIT/runtime failure (log once).""" + if self._numba_failed: + return + self._numba_failed = True + self._blend_backend = "numpy" + if self._blend_mode == "numba": + self._blend_mode = "numpy" + print( + f"Warning: numba blend failed ({type(exc).__name__}: {exc}); " + f"falling back to numpy." + ) + def _get_blend_coef(self, ov_pos, dim, ndim): key = (ov_pos, dim, ndim) coef = self._blend_coef_cache.get(key) @@ -117,13 +239,30 @@ class Stitcher: self._blend_coef_cache[key] = blend_coef return blend_coef - def stitch_rt_optimized(self, change_sv=False, s_mean=28., v_mean=180., blend=True): - feature_stitched = self.stitch_feature_optimized( - blend=blend, - match_brightness=self._feature_stitch_match_brightness, - pad=self._feature_stitch_pad, - print_every=self._feature_stitch_print_every, - ) + def stitch_rt_optimized(self, change_sv=False, s_mean=28., v_mean=180., blend=None): + """Stitch the current tile list. + + ``blend`` overrides JSON/env for this call: + - ``None`` (default): use configured ``stitcher.blend`` + - ``True`` / ``False``: force feathered blend on/off + - ``\"none\"`` / ``\"numpy\"`` / ``\"numba\"``: force that mode for this call + """ + use_feather, blend_backend = self._resolve_call_blend(blend) + prev_backend = self._blend_backend + self._blend_backend = blend_backend + try: + feature_stitched = self.stitch_feature_optimized( + blend=use_feather, + match_brightness=self._feature_stitch_match_brightness, + pad=self._feature_stitch_pad, + print_every=self._feature_stitch_print_every, + ) + finally: + # Keep sticky numpy fallback after a Numba failure; do not restore numba. + if self._numba_failed: + self._blend_backend = "numpy" + else: + self._blend_backend = prev_backend if feature_stitched is not None: return feature_stitched @@ -455,7 +594,7 @@ class Stitcher: return wy[:, None] * wx[None, :] def _blend_tile_inplace(self, blend, wsum, cov, src, weight, y0, x0, scratch=None): - """Composite a tile onto the canvas using numba-fused weighted average.""" + """Composite a tile onto the canvas using weighted average blending.""" h, w = src.shape[:2] H, W = blend.shape[:2] y0d, x0d = max(0, y0), max(0, x0) @@ -467,13 +606,20 @@ class Stitcher: sh, sw = y1d - y0d, x1d - x0d src_f = src[sy0:sy0+sh, sx0:sx0+sw].astype(np.float32, copy=False) - _blend_kernel( + args = ( blend[y0d:y1d, x0d:x1d], wsum[y0d:y1d, x0d:x1d], cov[y0d:y1d, x0d:x1d], src_f, weight[sy0:sy0+sh, sx0:sx0+sw], ) + if self._blend_backend == "numba" and _blend_kernel_numba is not None: + try: + _blend_kernel_numba(*args) + return + except Exception as exc: + self._mark_numba_failed(exc) + _blend_kernel_numpy(*args) def _mean_gray_bgr(self, bgr): ch = cv2.mean(bgr) @@ -535,6 +681,214 @@ class Stitcher: shapes = tuple(tb.shape[:2] for tb in tiles_bgr_c) return ref_h, ref_w, shapes, pad + def _ensure_feature_geom_cache( + self, + canvas_cache, + t_ref_bgr, + tiles_bgr_c, + H_use_list, + pad, + *, + allocate_blend_bufs=True, + ): + """Populate/reuse fixed-window geometry in canvas_cache. Returns geom_cache_hit.""" + h0c, w0c = t_ref_bgr.shape[:2] + geom_key = self._geom_cache_key(t_ref_bgr, tiles_bgr_c, pad) + geom_cache_hit = canvas_cache.get("geom_key") == geom_key + if geom_cache_hit: + if allocate_blend_bufs and ( + "blend_buf" not in canvas_cache + or "wsum_buf" not in canvas_cache + or "cov_buf" not in canvas_cache + ): + out_h = canvas_cache["out_h"] + out_w = canvas_cache["out_w"] + canvas_cache["blend_buf"] = np.zeros((out_h, out_w, 3), dtype=np.float32) + canvas_cache["wsum_buf"] = np.zeros((out_h, out_w), dtype=np.float32) + canvas_cache["cov_buf"] = np.zeros((out_h, out_w), dtype=np.uint8) + return True + + ref_corners = np.array([[0, 0], [w0c, 0], [w0c, h0c], [0, h0c]], dtype=np.float64) + pts = [ref_corners] + for tile_bgr, H_use in zip(tiles_bgr_c, H_use_list): + if H_use is None: + continue + hi, wi = tile_bgr.shape[:2] + tx, ty = self._translation_from_H_to_ref(H_use) + pts.append( + np.array( + [[tx, ty], [tx + wi, ty], [tx + wi, ty + hi], [tx, ty + hi]], + dtype=np.float64, + ) + ) + + all_xy = np.concatenate(pts, axis=0) + x_min_f = float(all_xy[:, 0].min()) - pad + y_min_f = float(all_xy[:, 1].min()) - pad + x_max_f = float(all_xy[:, 0].max()) + pad + y_max_f = float(all_xy[:, 1].max()) + pad + out_w = max(1, int(np.ceil(x_max_f - x_min_f))) + out_h = max(1, int(np.ceil(y_max_f - y_min_f))) + ref_y0 = int(np.floor(-y_min_f)) + ref_x0 = int(np.floor(-x_min_f)) + + pastes = [] + for H_use in H_use_list: + if H_use is None: + pastes.append(None) + continue + tx, ty = self._translation_from_H_to_ref(H_use) + pastes.append((int(np.floor(ty - y_min_f)), int(np.floor(tx - x_min_f)))) + + _dr_y0 = max(0, ref_y0) + _dr_x0 = max(0, ref_x0) + _dr_y1 = min(out_h, ref_y0 + h0c) + _dr_x1 = min(out_w, ref_x0 + w0c) + for _tb, _pa in zip(tiles_bgr_c, pastes): + if _pa is None: + continue + _py, _px = _pa + _th, _tw = _tb.shape[:2] + _dr_y0 = min(_dr_y0, max(0, _py)) + _dr_x0 = min(_dr_x0, max(0, _px)) + _dr_y1 = max(_dr_y1, min(out_h, _py + _th)) + _dr_x1 = max(_dr_x1, min(out_w, _px + _tw)) + + canvas_cache["geom_key"] = geom_key + canvas_cache["out_w"] = out_w + canvas_cache["out_h"] = out_h + canvas_cache["ref_y0"] = ref_y0 + canvas_cache["ref_x0"] = ref_x0 + canvas_cache["pastes"] = pastes + canvas_cache["x_min_f"] = x_min_f + canvas_cache["y_min_f"] = y_min_f + canvas_cache["crop_rect"] = None + canvas_cache["dirty_rect"] = (_dr_y0, _dr_y1, _dr_x0, _dr_x1) + canvas_cache["weight_maps"] = {} + canvas_cache["uint8_buf"] = np.zeros((out_h, out_w, 3), dtype=np.uint8) + if allocate_blend_bufs: + canvas_cache["blend_buf"] = np.zeros((out_h, out_w, 3), dtype=np.float32) + canvas_cache["wsum_buf"] = np.zeros((out_h, out_w), dtype=np.float32) + canvas_cache["cov_buf"] = np.zeros((out_h, out_w), dtype=np.uint8) + else: + # Coverage buffer only needed once to compute crop_rect for paste path. + canvas_cache["cov_buf"] = np.zeros((out_h, out_w), dtype=np.uint8) + canvas_cache.pop("blend_buf", None) + canvas_cache.pop("wsum_buf", None) + return False + + @staticmethod + def _paste_tile_clipped(canvas, tile, y0, x0, cov=None): + """Paste tile into canvas with clipping; optionally mark coverage.""" + th, tw = tile.shape[:2] + H, W = canvas.shape[:2] + y0d, x0d = max(0, y0), max(0, x0) + y1d, x1d = min(H, y0 + th), min(W, x0 + tw) + if y0d >= y1d or x0d >= x1d: + return + sy0, sx0 = y0d - y0, x0d - x0 + sh, sw = y1d - y0d, x1d - x0d + np.copyto(canvas[y0d:y1d, x0d:x1d], tile[sy0:sy0 + sh, sx0:sx0 + sw]) + if cov is not None: + cov[y0d:y1d, x0d:x1d] = 1 + + def _paste_feature_tiles_fast( + self, + t_ref_bgr, + tiles_bgr_c, + H_use_list, + canvas_cache, + *, + geom_cache_hit, + frame_idx=0, + print_every=30, + verbose=False, + t_profile0=0.0, + t_after_brightness=0.0, + t_after_geom=0.0, + ): + """Fixed-geometry uint8 paste path used when blend=False.""" + do_profile = verbose and frame_idx % print_every == 0 + out_w = canvas_cache["out_w"] + out_h = canvas_cache["out_h"] + ref_y0 = canvas_cache["ref_y0"] + ref_x0 = canvas_cache["ref_x0"] + pastes = canvas_cache["pastes"] + + canvas = canvas_cache.get("uint8_buf") + if canvas is None or canvas.shape[0] != out_h or canvas.shape[1] != out_w: + canvas = np.zeros((out_h, out_w, 3), dtype=np.uint8) + canvas_cache["uint8_buf"] = canvas + + dirty = canvas_cache.get("dirty_rect") + if geom_cache_hit and dirty is not None: + dy0, dy1, dx0, dx1 = dirty + canvas[dy0:dy1, dx0:dx1] = 0 + elif not geom_cache_hit: + canvas.fill(0) + else: + canvas.fill(0) + t_after_reset = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 + + need_crop = canvas_cache.get("crop_rect") is None + cov = canvas_cache.get("cov_buf") if need_crop else None + if need_crop: + if cov is None or cov.shape != (out_h, out_w): + cov = np.zeros((out_h, out_w), dtype=np.uint8) + canvas_cache["cov_buf"] = cov + else: + if dirty is not None: + dy0, dy1, dx0, dx1 = dirty + cov[dy0:dy1, dx0:dx1] = 0 + else: + cov.fill(0) + + pasted_tiles = 0 + for tile_bgr, H_use, paste in zip(tiles_bgr_c, H_use_list, pastes): + if H_use is None or paste is None: + continue + y0, x0 = paste + self._paste_tile_clipped(canvas, tile_bgr, y0, x0, cov=cov) + pasted_tiles += 1 + self._paste_tile_clipped(canvas, t_ref_bgr, ref_y0, ref_x0, cov=cov) + t_after_paste = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 + + crop_rect = canvas_cache.get("crop_rect") + if crop_rect is None and cov is not None: + crop_rect = self._compute_final_crop_rect(cov) + canvas_cache["crop_rect"] = crop_rect + if crop_rect is not None: + y0, y1, x0, x1 = crop_rect + roi = canvas[y0:y1, x0:x1] + else: + roi = canvas + + rh, rw = roi.shape[:2] + stitch_buf = canvas_cache.get("stitch_buf") + if stitch_buf is None or stitch_buf.shape[0] != rh or stitch_buf.shape[1] != rw: + stitch_buf = np.empty((rh, rw, 3), dtype=np.uint8) + canvas_cache["stitch_buf"] = stitch_buf + np.copyto(stitch_buf, roi) + t_after_finalize = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 + + if do_profile: + xmf = canvas_cache.get("x_min_f", 0.0) + ymf = canvas_cache.get("y_min_f", 0.0) + print( + f"stitch_feature_optimized: canvas out_w={out_w}, out_h={out_h}, " + f"x_min={xmf:.2f}, y_min={ymf:.2f} (fast_paste)" + ) + print( + "stitch_feature_optimized steps: " + f"brightness={t_after_brightness - t_profile0:.6f}s, " + f"geom={t_after_geom - t_after_brightness:.6f}s ({'hit' if geom_cache_hit else 'miss'}), " + f"reset={t_after_reset - t_after_geom:.6f}s, " + f"paste={t_after_paste - t_after_reset:.6f}s ({pasted_tiles + 1} tiles incl ref), " + f"finalize={t_after_finalize - t_after_paste:.6f}s, " + f"total={t_after_finalize - t_profile0:.6f}s" + ) + return stitch_buf + def _render_feature_tiles( self, t_ref_bgr, @@ -579,77 +933,37 @@ class Stitcher: tiles_bgr_c = tiles_bgr t_after_brightness = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 - geom_key = self._geom_cache_key(t_ref_bgr, tiles_bgr_c, pad) - geom_cache_hit = canvas_cache.get("geom_key") == geom_key - if not geom_cache_hit: - ref_corners = np.array([[0, 0], [w0c, 0], [w0c, h0c], [0, h0c]], dtype=np.float64) - pts = [ref_corners] - for tile_bgr, H_use in zip(tiles_bgr_c, H_use_list): - if H_use is None: - continue - hi, wi = tile_bgr.shape[:2] - tx, ty = self._translation_from_H_to_ref(H_use) - pts.append( - np.array( - [[tx, ty], [tx + wi, ty], [tx + wi, ty + hi], [tx, ty + hi]], - dtype=np.float64, - ) - ) - - all_xy = np.concatenate(pts, axis=0) - x_min_f = float(all_xy[:, 0].min()) - pad - y_min_f = float(all_xy[:, 1].min()) - pad - x_max_f = float(all_xy[:, 0].max()) + pad - y_max_f = float(all_xy[:, 1].max()) + pad - out_w = max(1, int(np.ceil(x_max_f - x_min_f))) - out_h = max(1, int(np.ceil(y_max_f - y_min_f))) - ref_y0 = int(np.floor(-y_min_f)) - ref_x0 = int(np.floor(-x_min_f)) - - pastes = [] - for H_use in H_use_list: - if H_use is None: - pastes.append(None) - continue - tx, ty = self._translation_from_H_to_ref(H_use) - pastes.append((int(np.floor(ty - y_min_f)), int(np.floor(tx - x_min_f)))) - - _dr_y0 = max(0, ref_y0) - _dr_x0 = max(0, ref_x0) - _dr_y1 = min(out_h, ref_y0 + h0c) - _dr_x1 = min(out_w, ref_x0 + w0c) - for _tb, _pa in zip(tiles_bgr_c, pastes): - if _pa is None: - continue - _py, _px = _pa - _th, _tw = _tb.shape[:2] - _dr_y0 = min(_dr_y0, max(0, _py)) - _dr_x0 = min(_dr_x0, max(0, _px)) - _dr_y1 = max(_dr_y1, min(out_h, _py + _th)) - _dr_x1 = max(_dr_x1, min(out_w, _px + _tw)) - - canvas_cache["geom_key"] = geom_key - canvas_cache["out_w"] = out_w - canvas_cache["out_h"] = out_h - canvas_cache["ref_y0"] = ref_y0 - canvas_cache["ref_x0"] = ref_x0 - canvas_cache["pastes"] = pastes - canvas_cache["x_min_f"] = x_min_f - canvas_cache["y_min_f"] = y_min_f - canvas_cache["crop_rect"] = None - canvas_cache["dirty_rect"] = (_dr_y0, _dr_y1, _dr_x0, _dr_x1) - canvas_cache["blend_buf"] = np.zeros((out_h, out_w, 3), dtype=np.float32) - canvas_cache["wsum_buf"] = np.zeros((out_h, out_w), dtype=np.float32) - canvas_cache["cov_buf"] = np.zeros((out_h, out_w), dtype=np.uint8) - canvas_cache["weight_maps"] = {} - else: - out_w = canvas_cache["out_w"] - out_h = canvas_cache["out_h"] - ref_y0 = canvas_cache["ref_y0"] - ref_x0 = canvas_cache["ref_x0"] - pastes = canvas_cache["pastes"] + geom_cache_hit = self._ensure_feature_geom_cache( + canvas_cache, + t_ref_bgr, + tiles_bgr_c, + H_use_list, + pad, + allocate_blend_bufs=bool(blend), + ) t_after_geom = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 + if not blend: + return self._paste_feature_tiles_fast( + t_ref_bgr, + tiles_bgr_c, + H_use_list, + canvas_cache, + geom_cache_hit=geom_cache_hit, + frame_idx=frame_idx, + print_every=print_every, + verbose=verbose, + t_profile0=t_profile0, + t_after_brightness=t_after_brightness, + t_after_geom=t_after_geom, + ) + + out_w = canvas_cache["out_w"] + out_h = canvas_cache["out_h"] + ref_y0 = canvas_cache["ref_y0"] + ref_x0 = canvas_cache["ref_x0"] + pastes = canvas_cache["pastes"] + blend_buf = canvas_cache["blend_buf"] wsum = canvas_cache["wsum_buf"] cov = canvas_cache["cov_buf"] @@ -667,7 +981,7 @@ class Stitcher: cov.fill(0) t_after_reset = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 - blend_margin = 16 if blend else 0 + blend_margin = 16 blended_tiles = 0 for tile_bgr, H_use, paste in zip(tiles_bgr_c, H_use_list, pastes): diff --git a/src/orisys/frame_processors.py b/src/orisys/frame_processors.py index 331ce1e..3b7e946 100644 --- a/src/orisys/frame_processors.py +++ b/src/orisys/frame_processors.py @@ -16,7 +16,7 @@ class CompoundEyeFrameProcessor: def process(self, frame): self.stitcher.get_stitch_unit(frame) self.stitcher.extract_tiles() - return self.stitcher.stitch_rt_optimized(blend=False) + return self.stitcher.stitch_rt_optimized() class SingleCameraFrameProcessor: diff --git a/src/orisys_blend/__init__.py b/src/orisys_blend/__init__.py new file mode 100644 index 0000000..3b05a79 --- /dev/null +++ b/src/orisys_blend/__init__.py @@ -0,0 +1,10 @@ +"""Pure-Python Numba stitch kernels kept outside the Nuitka-compiled orisys .pyd. + +Nuitka cannot reliably JIT ``@njit`` functions baked into a compiled extension. +This package is shipped as source next to the Nuitka wheel so Numba can see +real CPython bytecode at runtime. +""" + +from .kernel import blend_kernel + +__all__ = ["blend_kernel"] diff --git a/src/orisys_blend/kernel.py b/src/orisys_blend/kernel.py new file mode 100644 index 0000000..8c2ef0d --- /dev/null +++ b/src/orisys_blend/kernel.py @@ -0,0 +1,51 @@ +"""Fused stitch blend kernel (Numba JIT when available).""" + +from __future__ import annotations + +import numpy as np + +try: + from numba import njit + + _NUMBA_AVAILABLE = True +except ImportError: + _NUMBA_AVAILABLE = False + + def njit(*args, **kwargs): + def decorator(fn): + return fn + + return decorator + + +if _NUMBA_AVAILABLE: + + @njit(cache=True) + def blend_kernel(blend_patch, wsum_patch, cov_patch, src_patch, w_patch): + """Fused incremental weighted-average composite (numba-accelerated).""" + sh = src_patch.shape[0] + sw = src_patch.shape[1] + for y in range(sh): + for x in range(sw): + w = w_patch[y, x] + s0 = np.float32(src_patch[y, x, 0]) + s1 = np.float32(src_patch[y, x, 1]) + s2 = np.float32(src_patch[y, x, 2]) + if cov_patch[y, x] == 0: + blend_patch[y, x, 0] = s0 + blend_patch[y, x, 1] = s1 + blend_patch[y, x, 2] = s2 + wsum_patch[y, x] = w + else: + ws = wsum_patch[y, x] + w + if ws < 1e-6: + ws = 1e-6 + a = w / ws + blend_patch[y, x, 0] += (s0 - blend_patch[y, x, 0]) * a + blend_patch[y, x, 1] += (s1 - blend_patch[y, x, 1]) * a + blend_patch[y, x, 2] += (s2 - blend_patch[y, x, 2]) * a + wsum_patch[y, x] = ws + cov_patch[y, x] = 1 + +else: + blend_kernel = None