diff --git a/Readme.md b/Readme.md index e51976c..74812e6 100644 --- a/Readme.md +++ b/Readme.md @@ -1,7 +1,8 @@ # OrisysSDK API 文档 本文件提供了 OrisysSDK 的各类方法 - +## 使用方法 +pip install . --- ## `Sensor` 类别 ### 描述 @@ -18,7 +19,7 @@ ```python # Example 1： 用相机编号创建 -from xensesdk import Sensor +from orisys import Sensor from src.sensor import Sensor sensor = Sensor(0) diff --git a/config/compound_eye.png b/config/compound_eye.png index ef3b17b..bac700b 100644 Binary files a/config/compound_eye.png and b/config/compound_eye.png differ diff --git a/example_contact.py b/example_contact.py index 163a5c4..0f7a8b2 100644 --- a/example_contact.py +++ b/example_contact.py @@ -7,10 +7,10 @@ import time def main(): # 方式1: 从视频文件读取 - sensor = orisys.Sensor("./s1.mp4", isstitch=True, cuda=True, verbose=True) + # sensor = orisys.Sensor("./s1.mp4", isstitch=True, cuda=True, verbose=True) # 方式2: 从摄像头读取（取消注释使用） - # sensor = orisys.Sensor(0, isstitch=True, cuda=True, verbose=True) + sensor = orisys.Sensor(0, isstitch=True, cuda=True, verbose=True) print("\n按 'q' 退出程序，按 'r' 重置光流追踪器") @@ -36,34 +36,25 @@ def main(): ) cv2.imshow("形变场 Deformation", arrows) - mag_img = orisys.util.draw_magnitude_map( - sensor.img.copy(), - vnormal, - threshold=5, - colormap=cv2.COLORMAP_HOT, - downsample_factor = 1 - ) - cv2.imshow("magnitude map", mag_img) - # 计算触碰点 - contact_result = sensor.get_contact(min_magnitude=1, expansion_threshold=0.01,denoise_sigma=15) + sensor.compute_contact(min_magnitude=0.1,expansion_threshold=0.01,denoise_sigma=15) # 在图像上触碰点和区域 - image_with_foe = orisys.util.draw_foe(sensor.img.copy(), contact_result) + image_with_foe = orisys.util.draw_contact(sensor.img.copy(), sensor.contour, sensor.centroid) cv2.imshow("Contact Centroid", image_with_foe) # 计算触碰区域 - divergence = contact_result['divergence_map'] + depth_map = sensor.depth_map # 归一化散度值到0-255范围（使用绝对值，然后归一化） - div_abs = divergence + div_abs = depth_map if div_abs.max() > div_abs.min(): div_normalized = ((div_abs - div_abs.min()) / (div_abs.max() - div_abs.min()) * 255).astype(np.uint8) else: div_normalized = np.zeros_like(div_abs, dtype=np.uint8) # 应用颜色映射（使用JET：蓝色=低散度，红色=高散度） - cv2.imshow("Divergence Map", cv2.applyColorMap(div_normalized, cv2.COLORMAP_JET)) + cv2.imshow("Depth Map", cv2.applyColorMap(div_normalized, cv2.COLORMAP_JET)) print(f"fps: {fps}") diff --git a/requirements.txt b/requirements.txt index 1de2d31..ef33b23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,4 @@ opencv_python>=4.10.0.84 opencv-contrib-python scipy>=1.10.1 setuptools>=45.2.0 -PyQt5>=5.15.0 -pyqtgraph>=0.12.0 -PyInstaller>=5.0 + diff --git a/src/orisys/Stitcher.py b/src/orisys/Stitcher.py index 66dac21..817ebbc 100644 --- a/src/orisys/Stitcher.py +++ b/src/orisys/Stitcher.py @@ -8,9 +8,14 @@ class Stitcher: def __init__(self,isstitch=False, n=[3,3], index=np.array([1,2,3,6,7,8,11,12,13])): - # Settings + # Settings self.isstitch = isstitch + # Find Boxes Related + self.min_area_frac = 0.01 + self.max_area_frac = 0.05 + self.max_aspect_ratio = 100 + # stitch related self.stitch_valid = False self.boxes = self.units = None @@ -21,6 +26,7 @@ class Stitcher: self.nominal_ov_pos=100 self.margin=10 self.bound=self.nominal_ov_pos - self.margin + # for sensor 1 self.pos_default = [(260, 56), (577, 62), (897, 69), (1214, 75), (1530, 81), (256, 391), (570, 397), (889, 403), (1208, 409), (1523, 416), (248, 724), (563, 730), (882, 738), (1201, 743), (1516, 751)] @@ -542,7 +548,7 @@ class Stitcher: - def __filter_boxes_by_size(self, boxes, img_shape, min_area_frac=0.018, max_area_frac=0.035): + def __filter_boxes_by_size(self, boxes, img_shape): """ Remove boxes that are too small or too large relative to the image. - boxes: list of (x,y,w,h) @@ -552,12 +558,12 @@ class Stitcher: """ h, w = img_shape img_area = h * w - min_area = img_area * float(min_area_frac) - max_area = img_area * float(max_area_frac) + min_area = img_area * float(self.min_area_frac) + max_area = img_area * float(self.max_area_frac) filtered = [] for (x,y,ww,hh) in boxes: area = ww * hh - if area < min_area or area > max_area or (abs(ww - hh) > 20): + if area < min_area or area > max_area or (abs(ww - hh) > self.max_aspect_ratio): continue filtered.append((x,y,ww,hh)) return filtered \ No newline at end of file diff --git a/src/orisys/configs/prototype.json b/src/orisys/configs/prototype.json index 1d6c253..f2f81c8 100644 --- a/src/orisys/configs/prototype.json +++ b/src/orisys/configs/prototype.json @@ -9,6 +9,19 @@ "resolution_stitch": [400, 400], "index": [1,2,3,6,7,8,11,12,13], "pos_default": [[260, 56], [577, 62], [897, 69], [1214, 75], [1530, 81], [256, 391], [570, 397], [889, 403], [1208, 409], [1523, 416], [248, 724], [563, 730], [882, 738], [1201, 743], [1516, 751]] + }, + "stitcher": { + "min_area_frac": 0.018, + "max_area_frac": 0.035, + "max_aspect_ratio": 20 + }, + "contact": { + "min_contour_area": 3000, + "min_magnitude": 0.02, + "expansion_threshold": 0.01, + "downsample_factor": 1, + "kernel_size": [5,5], + "denoise_sigma": 15 } } diff --git a/src/orisys/configs/sample_1.json b/src/orisys/configs/sample_1.json new file mode 100644 index 0000000..7dc1e1a --- /dev/null +++ b/src/orisys/configs/sample_1.json @@ -0,0 +1,23 @@ +{ + "name": "prototype", + "version": "1.0", + "camera": { + "resolution_raw": [1920, 1080], + "fps": 60, + "units": 15, + "units_used": 9, + "resolution_stitch": [400, 400], + "index": [1,2,3,6,7,8,11,12,13], + "pos_default": [[260, 56], [577, 62], [897, 69], [1214, 75], [1530, 81], [256, 391], [570, 397], [889, 403], [1208, 409], [1523, 416], [248, 724], [563, 730], [882, 738], [1201, 743], [1516, 751]] + }, + "contact": { + "min_contour_area": 3000, + "min_magnitude": 1.0, + "expansion_threshold": 0.01, + "downsample_factor": 1, + "kernel_size": [5,5], + "denoise_sigma": 15 + } +} + + diff --git a/src/orisys/get_depth.py b/src/orisys/get_depth.py index 568d195..02d9503 100644 --- a/src/orisys/get_depth.py +++ b/src/orisys/get_depth.py @@ -12,6 +12,41 @@ except ImportError: # 缓存形态学操作的核，避免重复创建 _morphology_kernels = {} +def _create_foe_result(expansion_mask, contraction_mask, divergence_map, + centroid=None, centroid_found=False, contour=None, + all_contours=None): + """ + 创建标准化的FOE结果字典，统一所有函数的返回格式。 + + Args: + expansion_mask: 扩张区域掩码 + contraction_mask: 收缩区域掩码 + divergence_map: 散度图 + centroid: 质心坐标，可以是单个元组 (x, y) 或列表 [(x1, y1), (x2, y2), ...] + centroid_found: 是否找到质心 + contour: 轮廓，可以是单个轮廓或轮廓列表 + all_contours: 所有轮廓（仅在return_all_contours=True时使用） + + Returns: + dict: 标准化的FOE结果字典 + """ + result = { + 'expansion_mask': expansion_mask, + 'contraction_mask': contraction_mask, + 'divergence_map': divergence_map, + 'centroid': centroid if centroid is not None else [0,0], + 'centroid_found': centroid_found, + } + + # 可选字段 + if contour is not None: + result['contour'] = contour + + if all_contours is not None: + result['_all_contours'] = all_contours + + return result + def compute_foe(flow_field, min_magnitude=1.0, expansion_threshold=1, downsample_factor=1, return_all_contours=False, kernel_size=(5,5), denoise_sigma=10): """ 计算光流场的扩张焦点(Focus of Expansion)。 @@ -73,20 +108,15 @@ def compute_foe(flow_field, min_magnitude=1.0, expansion_threshold=1, downsample divergence = cv2.GaussianBlur(divergence, (0, 0), denoise_sigma) # 优化：使用原地操作 - np.where(divergence > 0.02, divergence, 0, out=divergence) + np.where(divergence > min_magnitude, divergence, 0, out=divergence) # 创建掩码（使用原地操作优化） valid_mask = magnitude > min_magnitude expansion_mask = (valid_mask & (divergence > expansion_threshold)) contraction_mask = (valid_mask & (divergence < -expansion_threshold)) - result = { - 'expansion_mask': expansion_mask, - 'contraction_mask': contraction_mask, - 'divergence_map': divergence, - 'foe_points': [0, 0], - 'is_foe_found': False - } + # 初始化结果字典 + result = _create_foe_result(expansion_mask, contraction_mask, divergence) # 寻找扩张区域轮廓 # 优化：缓存形态学操作的核 @@ -100,16 +130,16 @@ def compute_foe(flow_field, min_magnitude=1.0, expansion_threshold=1, downsample contours, _ = cv2.findContours(expansion_binary_closed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if contours: - result['is_foe_found'] = True + result['centroid_found'] = True if return_all_contours: result['_all_contours'] = contours largest_contour = max(contours, key=cv2.contourArea) - result['expansion_region'] = largest_contour + result['contour'] = largest_contour M = cv2.moments(largest_contour) if M["m00"] > 0: - result['foe_point'] = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"])) + result['centroid'] = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"])) return result @@ -206,7 +236,7 @@ def compute_foe_gpu(flow_field, min_magnitude=1.0, expansion_threshold=1, downsa # 优化：在GPU上完成掩码操作，减少数据传输 # CuPy的where不支持out参数，使用赋值代替 - divergence_gpu = cp.where(divergence_gpu > 0.02, divergence_gpu, 0) + divergence_gpu = cp.where(divergence_gpu > min_magnitude, divergence_gpu, 0) # 在GPU上创建掩码 valid_mask_gpu = magnitude_gpu > min_magnitude @@ -219,13 +249,8 @@ def compute_foe_gpu(flow_field, min_magnitude=1.0, expansion_threshold=1, downsa expansion_mask = cp.asnumpy(expansion_mask_gpu) contraction_mask = cp.asnumpy(contraction_mask_gpu) - result = { - 'expansion_mask': expansion_mask, - 'contraction_mask': contraction_mask, - 'divergence_map': divergence, - 'foe_points': [0, 0], - 'is_foe_found': False - } + # 初始化结果字典 + result = _create_foe_result(expansion_mask, contraction_mask, divergence) # 寻找扩张区域轮廓（在CPU上进行，因为OpenCV的轮廓检测需要CPU） # 优化：缓存形态学操作的核 @@ -239,16 +264,16 @@ def compute_foe_gpu(flow_field, min_magnitude=1.0, expansion_threshold=1, downsa contours, _ = cv2.findContours(expansion_binary_closed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if contours: - result['is_foe_found'] = True + result['centroid_found'] = True if return_all_contours: result['_all_contours'] = contours largest_contour = max(contours, key=cv2.contourArea) - result['expansion_region'] = largest_contour + result['contour'] = largest_contour M = cv2.moments(largest_contour) if M["m00"] > 0: - result['foe_point'] = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"])) + result['centroid'] = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"])) return result @@ -275,7 +300,7 @@ def compute_multiple_foe(flow_field, min_magnitude=1.0, expansion_threshold=0.00 - 'divergence_map': 散度图 """ # 调用 compute_foe 获取基础数据和所有轮廓 - foe_result = compute_foe( + foe_result = compute_foe_gpu( flow_field, min_magnitude=min_magnitude, expansion_threshold=expansion_threshold, @@ -286,13 +311,12 @@ def compute_multiple_foe(flow_field, min_magnitude=1.0, expansion_threshold=0.00 ) # 构建多焦点结果，复用基础数据 - result = { - 'expansion_mask': foe_result['expansion_mask'], - 'contraction_mask': foe_result['contraction_mask'], - 'divergence_map': foe_result['divergence_map'], - 'foe_points': [], - 'expansion_regions': [] - } + result = _create_foe_result( + expansion_mask=foe_result['expansion_mask'], + contraction_mask=foe_result['contraction_mask'], + divergence_map=foe_result['divergence_map'], + centroid_found=foe_result['centroid_found'] + ) # 处理所有轮廓，过滤并计算焦点 if '_all_contours' in foe_result: @@ -304,7 +328,9 @@ def compute_multiple_foe(flow_field, min_magnitude=1.0, expansion_threshold=0.00 if M["m00"] > 0: cx = int(M["m10"] / M["m00"]) cy = int(M["m01"] / M["m00"]) - result['foe_points'].append((cx, cy)) - result['expansion_regions'].append(contour) + result['centroid'].append((cx, cy)) + if 'contour' not in result: + result['contour'] = [] + result['contour'].append(contour) return result diff --git a/src/orisys/sensor.py b/src/orisys/sensor.py index b5e5ebe..5281c64 100644 --- a/src/orisys/sensor.py +++ b/src/orisys/sensor.py @@ -45,6 +45,8 @@ class Sensor: FNORMAL = auto() FSHEARX = auto() FSHEARY = auto() + CENTROID = auto() + CONTOUR = auto() DEPTH = auto() # ========================================================================= @@ -91,15 +93,22 @@ class Sensor: self.vupdate = False self.stitch_calibration = [] - # Data variables + # Data Variables self.img = None + self.img_size_raw = None self.img_filtered = None self.flow = None self.vnormal = None - self.shear = None - self.depth = None + self.vshear = None + # self.depth = None self.n_boxes = None self.fnormal = self.fshearx = self.fsheary = self.ftor = 0.0 + + # Contact Data Variables + self.centroid = None + self.centroid_found = False + self.contour = None + self.depth_map = None # Timing self.last_log_time = time.perf_counter() @@ -116,7 +125,11 @@ class Sensor: self._stop_event = None # Load configuration + self.config_cam = None + self.config_stitcher = None + self.config_contact = None self._load_config() + # Initialize components self._stitcher = Stitcher(isstitch=isstitch, index=self.config_index) @@ -167,7 +180,8 @@ class Sensor: if log_timing: t0 = time.perf_counter() img = np.ascontiguousarray(self._stitcher.stitch_rt_optimized()) if log_timing: timings['stitch'] = (time.perf_counter() - t0) * 1000 - + + self.img_size_raw = img.shape # Resize img = cv2.resize(img, tuple(self.config_resolution_stitch), interpolation=cv2.INTER_CUBIC) @@ -222,7 +236,7 @@ class Sensor: # Get components self.vnormal = self.hhd.get_normal() - self.shear = self.hhd.get_shear() + self.vshear = self.hhd.get_shear() # # Depth computation # if log_timing: t0 = time.perf_counter() @@ -244,21 +258,31 @@ class Sensor: return self.flow - def get_contact(self, multiple_foe=False, min_contour_area=3000, min_magnitude=1.0, expansion_threshold=0.01, downsample_factor=1, kernel_size=(5,5), denoise_sigma=15): + def compute_contact(self, multiple_foe=False, min_contour_area=3000, min_magnitude=1.0, expansion_threshold=0.01, downsample_factor=1, kernel_size=(5,5), denoise_sigma=15): if multiple_foe: - return compute_multiple_foe(self.vnormal, min_contour_area=min_contour_area, min_magnitude=min_magnitude, expansion_threshold=expansion_threshold, downsample_factor=downsample_factor, kernel_size=kernel_size, denoise_sigma=denoise_sigma) - else: - if self.cuda: - return compute_foe_gpu(self.vnormal, min_magnitude=min_magnitude, expansion_threshold=expansion_threshold, downsample_factor=downsample_factor, kernel_size=kernel_size, denoise_sigma=denoise_sigma) - - else: - return compute_foe(self.vnormal, min_magnitude=min_magnitude, expansion_threshold=expansion_threshold, downsample_factor=downsample_factor, kernel_size=kernel_size, denoise_sigma=denoise_sigma) - + result = compute_multiple_foe(self.vnormal, min_contour_area=min_contour_area, min_magnitude=min_magnitude, expansion_threshold=expansion_threshold, downsample_factor=downsample_factor, kernel_size=kernel_size, denoise_sigma=denoise_sigma) + + else: + if self.cuda: + result = compute_foe_gpu(self.vnormal, min_magnitude=min_magnitude, expansion_threshold=expansion_threshold, downsample_factor=downsample_factor, kernel_size=kernel_size, denoise_sigma=denoise_sigma) + + else: + result = compute_foe(self.vnormal, min_magnitude=min_magnitude, expansion_threshold=expansion_threshold, downsample_factor=downsample_factor, kernel_size=kernel_size, denoise_sigma=denoise_sigma) + + self.centroid = result['centroid'] + self.centroid_found = result['centroid_found'] + if 'contour' in result: + self.contour = result['contour'] + self.depth_map = result['divergence_map'] + + return result + + + # def get_depth(self, denoise_sigma=15): + # return getFilteredDensity(self.vnormal, use_cuda=self.cuda) + # self.depth = getFilteredDensity(self.vnormal, use_cuda=self.cuda) - def get_depth(self, denoise_sigma=15): - return getFilteredDensity(self.vnormal, use_cuda=self.cuda) - self.depth = getFilteredDensity(self.vnormal, use_cuda=self.cuda) def update(self): """ Convenience method: get_img + compute_deformation in one call. @@ -270,7 +294,7 @@ class Sensor: self.compute_deformation() return { 'img': self.img, 'flow': self.flow, - 'normal': self.vnormal, 'shear': self.shear, 'depth': self.depth, + 'normal': self.vnormal, 'shear': self.vshear, 'depth': self.depth, 'fps': self.fps, 'fnormal': self.fnormal, 'fshearx': self.fshearx, 'fsheary': self.fsheary, 'ftor': self.ftor, } @@ -291,13 +315,15 @@ class Sensor: mapping = { self.info.VRAW: lambda: self.flow if self.flow is not None else np.array([]), self.info.VNORMAL: lambda: self.vnormal if self.vnormal is not None else np.array([]), - self.info.VSHEAR: lambda: self.shear if self.shear is not None else np.array([]), + self.info.VSHEAR: lambda: self.vshear if self.vshear is not None else np.array([]), self.info.FNORMAL: lambda: self.fnormal, self.info.FSHEARX: lambda: self.fshearx, self.info.FSHEARY: lambda: self.fsheary, self.info.FPS: lambda: self.fps, self.info.TIMESTAMP: lambda: time.strftime('%H:%M:%S'), - self.info.DEPTH: lambda: self.depth if self.depth is not None else np.array([]), + self.info.DEPTH: lambda: self.depth_map if self.depth_map is not None else np.array([]), + self.info.CENTROID: lambda: self.centroid if self.centroid is not None else np.array([]), + self.info.CONTOUR: lambda: self.contour if self.contour is not None else np.array([]), } return tuple(mapping.get(arg, lambda: None)() for arg in args if isinstance(arg, self.info)) @@ -431,6 +457,12 @@ class Sensor: self.config_index = cam["index"] self.config_pos_default = cam["pos_default"] + config_stitcher = self.config["stitcher"] + + config_contact = self.config["contact"] + + + if self.verbose: print(f"Config: {self.config_name}") print(f" Resolution: {self.config_resolution_raw} → {self.config_resolution_stitch}") @@ -579,7 +611,7 @@ class Sensor: while not self._stop_event.is_set(): try: img, flow = self._flow_queue.get(timeout=0.1) - contact_result = self.get_contact(flow) + contact_result = self.compute_contact(flow) self._queue_put(self._result_queue, contact_result) except Empty: continue diff --git a/src/orisys/util.py b/src/orisys/util.py index 8356736..7be998c 100644 --- a/src/orisys/util.py +++ b/src/orisys/util.py @@ -132,37 +132,35 @@ def draw_fps(image, fps): return image -def draw_foe(image, foe_result, point_color=(0, 255, 0), region_color=(0, 255, 255)): +def draw_contact(image, contour, centroid, point_color=(0, 255, 0), region_color=(0, 255, 255)): """ 在图像上绘制扩张焦点和扩张区域。 Args: image: 背景图像 - foe_result: compute_foe 函数的返回结果 - point_color: 焦点点的颜色 (BGR) - region_color: 扩张区域轮廓的颜色 (BGR) + contour: 轮廓 + centroid: 触碰点 + point_color: 触碰点的颜色 (BGR) + region_color: 轮廓的颜色 (BGR) Returns: - np.ndarray: 绘制了焦点和区域的图像 + np.ndarray: 绘制了触碰点和轮廓的图像 """ image = image.copy() # 绘制扩张区域轮廓 - if 'expansion_region' in foe_result: - cv2.drawContours(image, [foe_result['expansion_region']], -1, region_color, 2) + if contour is not None: + cv2.drawContours(image, [contour], -1, region_color, 2) - # 绘制焦点点 - if 'foe_point' in foe_result: - cx, cy = foe_result['foe_point'] + # 绘制触碰点 + if centroid is not None: + cx, cy = centroid cv2.circle(image, (cx, cy), 8, point_color, -1) cv2.circle(image, (cx, cy), 10, point_color, 2) - cv2.putText(image, "FOE", (cx + 15, cy), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, point_color, 2) return image - def draw_multiple_foe(image, foe_result, point_color=(0, 255, 0), region_color=(0, 255, 255), label_colors=None): """