From 966fc7431339785b22840a2381a1623b8ba5505a Mon Sep 17 00:00:00 2001 From: ykcheungab Date: Sat, 21 Mar 2026 19:27:07 +0800 Subject: [PATCH] add blend control on stitch --- config/cal.npy | Bin 1640 -> 0 bytes config/dd01a_0.npy | Bin 1640 -> 0 bytes config/dd01a_1.npy | Bin 1640 -> 0 bytes config/dd01a_2.npy | Bin 1640 -> 0 bytes config/ddjx01_480p.json | 43 +++++++ config/prototype_of3_720p.json | 4 +- example2_two_sensor_pipeline.py | 12 +- src/orisys/Stitcher.py | 214 ++++++++++++++++++-------------- src/orisys/sensor.py | 19 ++- 9 files changed, 189 insertions(+), 103 deletions(-) delete mode 100644 config/cal.npy delete mode 100644 config/dd01a_0.npy delete mode 100644 config/dd01a_1.npy delete mode 100644 config/dd01a_2.npy create mode 100644 config/ddjx01_480p.json diff --git a/config/cal.npy b/config/cal.npy deleted file mode 100644 index e99c32de719815f37c572a48dd40554e2411097b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1640 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Z6KZ6ruA`uymS0p-l$aNvUzCyxl5k7RDNY57 z7iT0EqyqUGmO2VXnmP)#3giN=3}3YVg(>J0AfmkA{Ft?z)6J!Ua z0Az+K5L-}c5R(2fAUhR^sTrQs3`dw#(ZiD(b_}I>2gNxx(hw{>(c_&OX44})sTJ>L iz<@A;1qKyp^z=!s@C4;|P}xK+fFAEgR4|wp@eTm2>@Mv9 diff --git a/config/dd01a_0.npy b/config/dd01a_0.npy deleted file mode 100644 index 36d734419d1a92680424eb90de41ba826c13e15d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1640 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Z6KZ6ruA`uymS0p-l$aNvUzCyxl5k7RDNY57 z7iT0EqyqUGmO2VXnmP)#3giN=3}3YVg(>J0AfmkA{Ft?z)6J!Ua zfEMm713D@dh^ZN#)C@;hn4yO!HS7R|rv(s$;+|Rn-JR5mcaS^HspUwZIq30DO?R47 h(VghwNv(JvD)}80?`BjCPxN>Lk2>{UCF8}}l diff --git a/config/dd01a_1.npy b/config/dd01a_1.npy deleted file mode 100644 index 36d734419d1a92680424eb90de41ba826c13e15d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1640 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Z6KZ6ruA`uymS0p-l$aNvUzCyxl5k7RDNY57 z7iT0EqyqUGmO2VXnmP)#3giN=3}3YVg(>J0AfmkA{Ft?z)6J!Ua zfEMm713D@dh^ZN#)C@;hn4yO!HS7R|rv(s$;+|Rn-JR5mcaS^HspUwZIq30DO?R47 h(VghwNv(JvD)}80?`BjCPxN>Lk2>{UCF8}}l diff --git a/config/dd01a_2.npy b/config/dd01a_2.npy deleted file mode 100644 index 720658e4d7811103496e1eea0c794962c78a8adb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1640 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Z6KZ6ruA`uymS0p-l$aNvUzCyxl5k7RDNY57 z7iT0EqyqUGmO2VXnmP)#3giN=3}3YVg(>J0AfmkA{Ft?z)6J!Ua zfEMm713D@dh^ZN#)C@;hn4yO!HSC~8c%r*=sKonVbtigwj 0: - # Pad right side of the narrower image - padding = ((0, 0), (0, wdiff), (0, 0)) if len(img_stitched_width.shape) == 3 else ((0, 0), (0, wdiff)) + padding = ( + ((0, 0), (0, wdiff), (0, 0)) + if len(img_stitched_width.shape) == 3 + else ((0, 0), (0, wdiff)) + ) img_stitched_width = np.pad(img_stitched_width, padding, mode='edge') else: - # Crop the wider image img_stitched_width = img_stitched_width[:, :width_h] ov_pos_h = int(stitch_calibration[(i + 1) * nrow - 1][1]) - ov_pos_h, ov_h1, ov_h2 = self.__find_ov(img_stitched_height, img_stitched_width, ov_pos_h, dim=0) + ov_pos_h, ov_h1, ov_h2 = self.__find_ov( + img_stitched_height, img_stitched_width, ov_pos_h, dim=0 + ) img_stitched_height = self.__stitch_optimized( - img_stitched_height, img_stitched_width, ov_pos_h, ov_h1, ov_h2, dim=0, blend=True + img_stitched_height, img_stitched_width, ov_pos_h, ov_h1, ov_h2, dim=0, blend=blend ) return img_stitched_height @@ -118,6 +134,16 @@ class Stitcher: # return self.boxes + def _copy_into_tile_buffer(self, src_view, dst, flip_x=True, flip_y=True): + if flip_x and flip_y: + cv2.flip(src_view, -1, dst) + elif flip_x: + cv2.flip(src_view, 1, dst) + elif flip_y: + cv2.flip(src_view, 0, dst) + else: + np.copyto(dst, src_view) + def get_stitch_unit(self, img, flip_x=True, flip_y=True,index = np.arange(15)): """ Extract each tile given by boxes (x0,y0,x1,y1), default flip both X and Y due to inverting lens. @@ -129,43 +155,35 @@ class Stitcher: Returns: canvas image and list of flipped tiles as (x0,y0,flipped_tile) """ h, w = img.shape[:2] - if self.units is None: - flipped_units = [] - for i, (x0, y0, x1, y1) in enumerate(self.boxes): - - # clamp coordinates - x0 = max(0, int(x0)); y0 = max(0, int(y0)) - x1 = min(w, int(x1)); y1 = min(h, int(y1)) - if x1 <= x0 or y1 <= y0: - continue - tile = img[y0:y1, x0:x1].copy() - # flip X then Y in default - if flip_x: - tile = cv2.flip(tile, 1) # horizontal - if flip_y: - tile = cv2.flip(tile, 0) # vertical - flipped_units.append((x0, y0, tile)) - self.units = flipped_units + active_index = index if self.units is None else self.index + if isinstance(active_index, int): + selected_index = {active_index} else: - flipped_units = self.units - index = self.index - for i, (x0, y0, x1, y1) in enumerate(self.boxes): - if i not in index: - continue - # clamp coordinates - x0 = max(0, int(x0)); y0 = max(0, int(y0)) - x1 = min(w, int(x1)); y1 = min(h, int(y1)) - if x1 <= x0 or y1 <= y0: - continue - tile = img[y0:y1, x0:x1].copy() - # flip X then Y in default - if flip_x: - tile = cv2.flip(tile, 1) # horizontal - if flip_y: - tile = cv2.flip(tile, 0) # vertical - # Assign instead of append + selected_index = set(active_index) + flipped_units = [] if self.units is None else self.units + + for i, (x0, y0, x1, y1) in enumerate(self.boxes): + if i not in selected_index: + continue + + x0 = max(0, int(x0)); y0 = max(0, int(y0)) + x1 = min(w, int(x1)); y1 = min(h, int(y1)) + if x1 <= x0 or y1 <= y0: + continue + + src_view = img[y0:y1, x0:x1] + if self.units is None: + tile = np.empty_like(src_view) + self._copy_into_tile_buffer(src_view, tile, flip_x=flip_x, flip_y=flip_y) + flipped_units.append((x0, y0, tile)) + else: + _, _, tile = flipped_units[i] + if tile.shape != src_view.shape or tile.dtype != src_view.dtype: + tile = np.empty_like(src_view) + self._copy_into_tile_buffer(src_view, tile, flip_x=flip_x, flip_y=flip_y) flipped_units[i] = (x0, y0, tile) - self.units = flipped_units + + self.units = flipped_units return True @@ -238,56 +256,64 @@ class Stitcher: def __stitch_optimized(self, img1, img2, ov_pos, ov1, ov2, blend=True, dim=1): _dim = len(img1.shape) - - # Early return if no blending needed - if not blend: - ov = ov1 - elif ov_pos <= 1: - ov = ov1 - else: - # Vectorized blending - blend_coef = np.linspace(1.0, 0.0, ov_pos, dtype=np.float32) - - # Prepare blend_coef for broadcasting - if dim == 1: # Horizontal blending - if _dim == 3: - # Shape: (1, ov_pos, 1) - blend_coef = blend_coef.reshape(1, ov_pos, 1) - else: - # Shape: (1, ov_pos) - blend_coef = blend_coef.reshape(1, ov_pos) - else: # Vertical blending - if _dim == 3: - # Shape: (ov_pos, 1, 1) - blend_coef = blend_coef.reshape(ov_pos, 1, 1) - else: - # Shape: (ov_pos, 1) - blend_coef = blend_coef.reshape(ov_pos, 1) - - # Compute blended overlap in one operation - ov = (ov1.astype(np.float32) * blend_coef + - ov2.astype(np.float32) * (1 - blend_coef)).astype(np.uint8) - - # Avoid unnecessary .copy() calls when possible + + # Clamp overlap for safety if dim == 1: - if _dim == 3: - # Use np.take or slicing without copy when safe - left = img1[:, :img1.shape[1] - ov_pos, :] if img1.shape[1] > ov_pos else img1[:, :0, :] - right = img2[:, ov_pos:, :] - stitched_img = np.concatenate((left, ov, right), axis=1) - else: - left = img1[:, :img1.shape[1] - ov_pos] if img1.shape[1] > ov_pos else img1[:, :0] - right = img2[:, ov_pos:] - stitched_img = np.concatenate((left, ov, right), axis=1) + ov_pos = int(max(0, min(ov_pos, img1.shape[1], img2.shape[1]))) else: + ov_pos = int(max(0, min(ov_pos, img1.shape[0], img2.shape[0]))) + + # Fast blend path with cached coefficient tensors + if not blend or ov_pos <= 1: + ov = ov1 + else: + blend_coef = self._get_blend_coef(ov_pos, dim, _dim) + ov = ( + ov1.astype(np.float32, copy=False) * blend_coef + + ov2.astype(np.float32, copy=False) * (1.0 - blend_coef) + ).astype(np.uint8) + + # Build stitched output with one target allocation and slice writes + if dim == 1: + left_w = img1.shape[1] - ov_pos + right_w = img2.shape[1] - ov_pos + out_w = left_w + ov_pos + right_w if _dim == 3: - top = img1[:img1.shape[0] - ov_pos, :, :] if img1.shape[0] > ov_pos else img1[:0, :, :] - bottom = img2[ov_pos:, :, :] - stitched_img = np.concatenate((top, ov, bottom), axis=0) + stitched_img = np.empty((img1.shape[0], out_w, img1.shape[2]), dtype=img1.dtype) + if left_w > 0: + stitched_img[:, :left_w, :] = img1[:, :left_w, :] + if ov_pos > 0: + stitched_img[:, left_w:left_w + ov_pos, :] = ov + if right_w > 0: + stitched_img[:, left_w + ov_pos:, :] = img2[:, ov_pos:, :] else: - top = img1[:img1.shape[0] - ov_pos, :] if img1.shape[0] > ov_pos else img1[:0, :] - bottom = img2[ov_pos:, :] - stitched_img = np.concatenate((top, ov, bottom), axis=0) + stitched_img = np.empty((img1.shape[0], out_w), dtype=img1.dtype) + if left_w > 0: + stitched_img[:, :left_w] = img1[:, :left_w] + if ov_pos > 0: + stitched_img[:, left_w:left_w + ov_pos] = ov + if right_w > 0: + stitched_img[:, left_w + ov_pos:] = img2[:, ov_pos:] + else: + top_h = img1.shape[0] - ov_pos + bottom_h = img2.shape[0] - ov_pos + out_h = top_h + ov_pos + bottom_h + if _dim == 3: + stitched_img = np.empty((out_h, img1.shape[1], img1.shape[2]), dtype=img1.dtype) + if top_h > 0: + stitched_img[:top_h, :, :] = img1[:top_h, :, :] + if ov_pos > 0: + stitched_img[top_h:top_h + ov_pos, :, :] = ov + if bottom_h > 0: + stitched_img[top_h + ov_pos:, :, :] = img2[ov_pos:, :, :] + else: + stitched_img = np.empty((out_h, img1.shape[1]), dtype=img1.dtype) + if top_h > 0: + stitched_img[:top_h, :] = img1[:top_h, :] + if ov_pos > 0: + stitched_img[top_h:top_h + ov_pos, :] = ov + if bottom_h > 0: + stitched_img[top_h + ov_pos:, :] = img2[ov_pos:, :] return stitched_img diff --git a/src/orisys/sensor.py b/src/orisys/sensor.py index 610e8d4..4453098 100644 --- a/src/orisys/sensor.py +++ b/src/orisys/sensor.py @@ -134,6 +134,13 @@ class Sensor: self._fps = 0 self._last_capture_time = 0.0 self._last_processing_time = 0.0 + self._last_stage_times = { + "extract_tiles": 0.0, + "stitch": 0.0, + "resize": 0.0, + "mirror": 0.0, + "astype": 0.0, + } # Initialize components self.stitcher = Stitcher(isstitch=isstitch, config_stitcher=self.__config_stitcher, verbose=self.verbose) @@ -202,20 +209,30 @@ class Sensor: # Extract and stitch tiles # self.stitcher.get_stitch_unit(frame) self._extract_tiles(frame) + t_after_extract = time.perf_counter() - img = np.ascontiguousarray(self.stitcher.stitch_rt_optimized()) + img = np.ascontiguousarray(self.stitcher.stitch_rt_optimized(blend=True)) + t_after_stitch = time.perf_counter() self.img_size_raw = img.shape # Resize img = cv2.resize(img, tuple(self._config_resolution_stitch), interpolation=cv2.INTER_LINEAR) + t_after_resize = time.perf_counter() if self.mirror: img = cv2.flip(img, 1) + t_after_mirror = time.perf_counter() self.img = img.astype(np.uint8) + t_after_astype = time.perf_counter() t_processing_end = time.perf_counter() self._last_processing_time = t_processing_end - t_processing_start + self._last_stage_times["extract_tiles"] = t_after_extract - t_processing_start + self._last_stage_times["stitch"] = t_after_stitch - t_after_extract + self._last_stage_times["resize"] = t_after_resize - t_after_stitch + self._last_stage_times["mirror"] = t_after_mirror - t_after_resize + self._last_stage_times["astype"] = t_after_astype - t_after_mirror # FPS calculation self._fps = 1.0 / (time.perf_counter() - self.last_log_time)