diff --git a/examples/README.md b/examples/README.md index d4d974a..96a159f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,75 +1,87 @@ # Examples -Runnable demos live at the `examples/` root. Shared helpers stay in -`pipeline/`, `viz/`, and `gui/`. Run commands from the repository root. +Runnable demos live at the `examples/` root. Shared helpers stay in `pipeline/`, `viz/`, and +`gui/`. Run commands from the repository root. -## Product Entry Points +## Start here + +Both recommended examples create one object, then read a frame at a time. Everything else - +camera reading, stitching, preprocessing, the optical-flow and force stages - happens inside +`StreamPipeline`, and each frame arrives as one `FrameSnapshot`. There is no stage sequence to +remember and no tuple to unpack, and no backend setting to tune. | Product | Entry | Default config | |---------|-------|----------------| | ORY-BOX | `basic_pipeline.py` | `box` | -| ORY-MINI | `mini_pipeline.py` | `mini` | -| ORY-FINGER (2D) | `finger_pipeline.py` | `finger` | -| ORY-FINGER (3D Qt) | `finger_pipeline_3dview.py` | `finger` | +| ORY-MINI | `basic_pipeline.py -c mini` | `mini` | +| ORY-FINGER | `finger_pipeline.py` | `finger` | -`finger` is the public ORY-FINGER config stem. `dd02` remains a compatibility alias for `dd02-ov`. +```python +# ORY-BOX / ORY-MINI +sensor = orisys.Sensor(0, config_name="box") +with orisys.StreamPipeline(sensor, sensor.get_img, contact=True) as pipe: + while (snap := pipe.wait()) is not None: + print(snap.normal_force, snap.shear_x, snap.shear_y) -## Typical Usage - -ORY-BOX: - -```bash -python examples/basic_pipeline.py -v 0 -c box +# ORY-FINGER - the runtime owns camera, Blender export, unroll map and valid mask +runtime = FingerRuntime.open(config="finger", video=0) +with runtime.pipeline(contact=True) as pipe: + while (snap := pipe.wait()) is not None: + print(snap.normal_force, snap.shear_x, snap.shear_y) ``` -ORY-MINI: - -```bash -python examples/mini_pipeline.py -v 0 -c mini -``` - -ORY-FINGER 2D: +`contact=True` runs contact analysis; `include=(...)` chooses which arrays each snapshot carries +(`image`, `flow`, `vnormal`, `vshear`, `depth`, `centroid`, `contour`). Arrays you do not include +are `None`, so you pay only for what you use. `pipe.reset()` resets the tracker safely from your +loop; `snap.image_size_raw` reports the capture resolution before UV unroll or resize. + +### Typical usage ```bash +python examples/basic_pipeline.py -v 0 -c box # first camera, ORY-BOX +python examples/basic_pipeline.py -v 0 -c mini # ORY-MINI +python examples/basic_pipeline.py -v recording.mp4 -c box +python examples/basic_pipeline.py --no-show # headless: print only python examples/finger_pipeline.py --camera_id 0 --config finger +python examples/finger_pipeline.py --video data/dd02-0703/1.mp4 ``` -ORY-FINGER video file: +Keys in both: `q` quits, `r` resets the tracker. -```bash -python examples/finger_pipeline.py --video ./data/sample.mp4 --config finger +## The manual form, for when you need the stages + +`examples/archive/` holds the previous versions of these two examples. They call the stages by +hand: + +```python +sensor.get_img() +sensor.compute_deformation() +sensor.compute_contact() +fps, flow, vnormal, img, ... = sensor.read_info( ... ) # positional, 13 elements ``` -ORY-FINGER 3D Qt dashboard: +That path is still supported and still correct - it is the right choice if you are implementing +your own scheduling. It is no longer the recommended starting point, and the archived files carry +a header explaining the difference. They are kept so that code copied from them keeps working. -```bash -pip install -e ".[gui]" -python examples/finger_pipeline_3dview.py --video 0 --config finger -``` +## Other root examples -## Other Root Examples - -- `plot_data.py` — single-sensor OpenCV views plus PyQtGraph force charts -- `two_sensor_pipeline.py` — two-sensor multiprocessing demo +- `multi_stream.py` - several sensors at once, using `StreamPipeline` with `StreamRecorder` +- `two_sensor_pipeline.py` - two-sensor multiprocessing demo, manual staging +- `plot_data.py` - single-sensor OpenCV views plus PyQtGraph force charts +- `finger_pipeline_3dview.py` - the Qt/open3d dashboard; see the note below ## Requirements -- SDK installed from the repository root, for example `pip install -e .` -- camera index or test video path -- matching config names under `./config/` (or packaged `orisys/configs/`) +- SDK installed from the repository root: `pip install -e .` +- a camera index or a test video path +- matching config names under `./config/` or the packaged `orisys/configs/` -Qt / PyQtGraph examples additionally require: +Qt / PyQtGraph examples additionally need `pip install -e ".[gui]"`. -```bash -pip install -e ".[gui]" -``` +## The GUI is moving out -## Notes - -- `finger_pipeline_3dview.py` is the main interactive finger demo. - It uses `gui/app_paths.py` so the same entry works in editable and packaged runs. -- Dashboard controls: click **启动** to start acquisition; use **Record CSV / Stop CSV** - and the CSV save-directory browser to log contact/force data; `Q` quits, `R` resets, - `S` starts video recording, `E` stops video recording, and `Esc` toggles fullscreen. -- `finger_pipeline.py` is the leanest finger example if you want to - validate UV unroll + force/contact processing without the full dashboard. +`examples/gui/` - the 2D viewer, the 3D dashboard and their widgets - is a full Qt + open3d +application, not an example. It is being moved to the **orisys-studio** application, which is +where the GUI belongs. Until that lands, `finger_pipeline_3dview.py` and `examples/gui/` still +work as before; new work should not build on them. diff --git a/examples/_display.py b/examples/_display.py new file mode 100644 index 0000000..9f136a7 --- /dev/null +++ b/examples/_display.py @@ -0,0 +1,52 @@ +"""Display helpers shared by the pipeline examples. + +These exist so an example can show a result without hand-rolling OpenCV arithmetic. Everything +here is presentation only: the SDK never draws, and nothing in ``orisys`` imports this module. + +The examples import it through a sibling import, which works when the examples are run from the +examples directory or with ``examples/`` on ``sys.path``. +""" + +from __future__ import annotations + +import cv2 +import numpy as np + + +def draw_flow_arrows(image, flow, *, threshold=0.4, grid_spacing=10, arrow_scale=1.0, + valid_mask=None): + """Return the deformation field drawn over ``image``.""" + import orisys + + return orisys.util.draw_arrows( + image, + flow, + threshold=threshold, + grid_spacing=grid_spacing, + arrow_scale=arrow_scale, + valid_mask=valid_mask, + ) + + +def depth_to_color(depth_map): + """Normalise a depth/divergence map to 8-bit and apply a colormap for the ``depth`` window. + + Returns ``None`` for an empty map, which is what ``read_info`` gives when the field was never + generated, so a caller can skip the window instead of showing saturated noise. + """ + if depth_map is None or not isinstance(depth_map, np.ndarray) or depth_map.size == 0: + return None + lo = float(depth_map.min()) + hi = float(depth_map.max()) + if hi <= lo: + scaled = np.zeros_like(depth_map, dtype=np.uint8) + else: + scaled = ((depth_map - lo) / (hi - lo) * 255.0).astype(np.uint8) + return cv2.applyColorMap(scaled, cv2.COLORMAP_JET) + + +def draw_contact(image, contour, centroid): + """Return ``image`` with the contact region and centroid drawn on it.""" + import orisys + + return orisys.util.draw_contact(image, contour, centroid) diff --git a/examples/archive/basic_pipeline_manual.py b/examples/archive/basic_pipeline_manual.py new file mode 100644 index 0000000..3710e72 --- /dev/null +++ b/examples/archive/basic_pipeline_manual.py @@ -0,0 +1,159 @@ +""" +ORY-BOX:单传感器基础流程(**已归档的手写版本**) + +本示例演示 Orisys SDK 的标准处理流程: +1. 初始化传感器(摄像头或视频文件) +2. 获取图像并计算形变 +3. 按需计算接触区域 +4. 读取并可视化结果 + +--- + +**已归档。请改用 `examples/basic_pipeline.py`。** + +保留本文件是为了兼容已经复制过这段代码的用户。它逐级调用 +`get_img()` → `compute_deformation()` → `compute_contact()` → `read_info()`, +需要自己记住调用顺序,并按下标解包一个 13 元组。 + +推荐写法是交给 `orisys.StreamPipeline`:读取、预处理、力/接触解算在内部并发完成,每一帧以 +单个 `FrameSnapshot` 交回: + + with orisys.StreamPipeline(sensor, sensor.get_img, contact=True) as pipe: + snap = pipe.wait() # 一个对象,一次调用 + print(snap.normal_force, snap.shear_x, snap.shear_y) + +本文件里的 `sensor.reset()` 在 pipeline 中对应 `pipe.reset()`:它会排队到流程线程执行, +不会与光流阶段争用同一帧缓冲。 + +**注意**:本文件用中文 `print`,在 cp1252 控制台(未设 UTF-8 的 Windows 终端)会抛 +`UnicodeEncodeError`。这是归档前就存在的行为,没有在这里修正——新示例的输出去掉了非 ASCII +字符,因此在任何终端都能跑。 + +""" +import os + +os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" +import argparse + +import cv2 +import numpy as np + +import orisys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--video","-v", type=str, default="0", help="视频源:摄像头编号或视频文件路径") + parser.add_argument("--config","-c", type=str, default="box", help="配置文件名称或路径") + parser.add_argument("--verbose","-verbose", type=bool, default=True, help="是否输出详细日志") + parser.add_argument("--cal_path","-cal", type=str, default=None, help="标定文件路径") + args = parser.parse_args() + + # 步骤 1:创建传感器对象(必需) + # config_name 可以是内置配置名称,也可以是自定义配置文件路径 + # cal_path 为标定文件路径;每台设备建议使用独立的标定文件 + + # 输入既可以是摄像头编号,也可以是视频文件路径 + if args.video.isdigit(): + sensor = orisys.Sensor(int(args.video), config_name=args.config, verbose=args.verbose, cal_path=args.cal_path) + else: + sensor = orisys.Sensor(args.video, config_name=args.config, verbose=args.verbose, cal_path=args.cal_path) + + print("\n按键说明:'q' 退出程序,'r' 重置追踪器。") + + while True: + # 步骤 2:数据处理 + # 步骤 2.1:获取并拼接图像(必需) + sensor.get_img() + # 步骤 2.2:计算形变场(必需);返回值表示当前帧是否检测到接触/运动 + is_contact_now = sensor.compute_deformation() + # 步骤 2.3:仅在当前帧检测到接触/运动时计算接触区域 + # if is_contact_now: + sensor.compute_contact() + + + # 步骤 2.4:读取结果(FNORMAL/FSHEAR* 在有 sensors/box/force_calibration.json 时为标定后牛顿力; + # *_RAW 始终为原始积分量) + ( + fps, + flow, + vnormal, + img, + contour, + centroid, + depth_map, + fnormal, + fshearx, + fsheary, + fnormal_raw, + fshearx_raw, + fsheary_raw, + ) = sensor.read_info( + sensor.info.FPS, + sensor.info.VRAW, + sensor.info.VNORMAL, + sensor.info.IMG, + sensor.info.CONTOUR, + sensor.info.CENTROID, + sensor.info.DEPTH, + sensor.info.FNORMAL, + sensor.info.FSHEARX, + sensor.info.FSHEARY, + sensor.info.FNORMAL_RAW, + sensor.info.FSHEARX_RAW, + sensor.info.FSHEARY_RAW, + ) + + # 步骤 3:可视化显示 + # 步骤 3.1:显示形变矢量场 + arrows = orisys.util.draw_arrows( + img, + flow, + threshold=2, + grid_spacing=20, + arrow_scale=1.0 + ) + cv2.imshow("arrow", arrows) + + # 步骤 3.2:显示接触区域和质心 + image_with_foe = orisys.util.draw_contact(img, contour, centroid) + cv2.imshow("contact", image_with_foe) + + # 步骤 3.3:显示深度图 + # 将深度值归一化到 0-255 范围 + div_abs = depth_map + if div_abs.max() > div_abs.min(): + div_normalized = ((div_abs - div_abs.min()) / (div_abs.max() - div_abs.min()) * 255).astype(np.uint8) + else: + div_normalized = np.zeros_like(div_abs, dtype=np.uint8) + + # 应用伪彩色映射 + cv2.imshow("depth", cv2.applyColorMap(div_normalized, cv2.COLORMAP_JET)) + + # 步骤 3.4:打印关键结果(标定力 + 原始力) + print( + f"FPS={fps:.2f}, " + f"法向力={fnormal:.4f} N (raw={fnormal_raw:.4f}), " + f"切向力X={fshearx:.4f} N (raw={fshearx_raw:.4f}), " + f"切向力Y={fsheary:.4f} N (raw={fsheary_raw:.4f}), " + f"深度图尺寸={depth_map.shape}, 原始分辨率={sensor.img_size_raw}" + ) + + # 步骤 3.5:键盘控制 + key = cv2.waitKey(1) & 0xFF + if key == ord("q"): + print("\n正在退出示例程序...") + break + if key == ord("r"): + # 当光流追踪出现漂移时,可手动重置追踪器 + sensor.reset() + print("追踪器已重置。") + + # ========================================================================= + # 4. 释放资源 + # ========================================================================= + sensor.disconnect() # 释放视频源及相关资源 + + +if __name__ == '__main__': + main() diff --git a/examples/archive/finger_pipeline_manual.py b/examples/archive/finger_pipeline_manual.py new file mode 100644 index 0000000..87dcfaf --- /dev/null +++ b/examples/archive/finger_pipeline_manual.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +""" +ORY-FINGER: Minimum UV-unroll finger tactile pipeline. **ARCHIVED — manual version.** + +Offline: load Blender export -> build_or_load_domain -> unroll_map_uv.npz +Runtime: camera frame -> FrameUnwarper -> TactileProcessor -> forces/contact + +Recommended entry: + python examples/finger_pipeline.py --camera_id 0 --config finger + +Current product config: ``finger`` (``dd02`` still resolves to ``dd02-ov``). + +--- + +**ARCHIVED. Use ``examples/finger_pipeline.py`` instead.** + +Kept so code copied from this file keeps working. It hand-sequences ``grab_and_process()`` → +``compute_deformation()`` → ``compute_contact()`` → ``read_info()`` and unpacks a ten-element +tuple by position. + +The recommended form lets :class:`orisys.StreamPipeline` run those stages concurrently and hand +back one immutable snapshot per frame: + + with runtime.pipeline(contact=True, include=("image", "depth")) as pipe: + snap = pipe.wait() + print(snap.normal_force, snap.shear_x, snap.shear_y) + +``tactile.reset()`` becomes ``pipe.reset()``, which is queued onto the flow-stage thread instead of +racing it. + +This archived version also has a defect: ``--show`` is declared ``store_true`` with +``default=True``, so it can never be turned off. The replacement uses ``--no-show``. +""" + +import argparse +import os +import sys + +os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0") + +import cv2 + +import orisys +from orisys.finger import FingerRuntime + + +def main(): + parser = argparse.ArgumentParser(description="Minimum UV-unroll finger tactile pipeline") + parser.add_argument("--camera_id", type=int, default=0) + parser.add_argument("--video","-v", help="Video file path (overrides camera)") + parser.add_argument("--config", "-c", default="finger", help="Orisys SDK config name or path") + parser.add_argument("--show", action="store_true", default=True, help="Show unrolled image window") + parser.add_argument("--verbose", action="store_true", help="SDK verbose logs") + args = parser.parse_args() + + try: + video = args.camera_id + if args.video: + video = args.video + runtime = FingerRuntime.open( + config=args.config, + video=video, + verbose=args.verbose, + ) + except RuntimeError as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) + + if args.verbose: + geometry = runtime.geometry + print( + f"UV geometry: native={geometry.native_width}x{geometry.native_height}, " + f"source={geometry.source_width}x{geometry.source_height} " + f"output={geometry.out_width}x{geometry.out_height} " + ) + + runtime.warmup(color=False) + tactile = runtime.tactile + + print("Press 'q' to quit, 'r' to reset tracker.") + try: + while True: + ok, unrolled = runtime.grab_and_process(color=False) + if not ok: + break + + tactile.compute_deformation() + tactile.compute_contact() + + fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = tactile.read_info( + tactile.info.FPS, + tactile.info.VRAW, + tactile.info.VNORMAL, + tactile.info.IMG, + tactile.info.CONTOUR, + tactile.info.CENTROID, + tactile.info.DEPTH, + tactile.info.FNORMAL, + tactile.info.FSHEARX, + tactile.info.FSHEARY, + ) + + status = ( + f"FPS={fps:.2f} normal={fnormal:.2f} shear=({fshearx:.2f}, {fsheary:.2f}) " + f"mask={tactile.mask_stats['coverage']:.2f}" + ) + + if args.show: + arrows = orisys.util.draw_arrows( + unrolled, + flow, + threshold=0.4, + grid_spacing=10, + arrow_scale=1.0, + valid_mask=tactile.valid_mask, + ) + cv2.imshow("UV Unrolled", arrows) + + print(status, end="\r") + + key = cv2.waitKey(1) & 0xFF + if key == ord("q"): + print("\nExiting.") + break + if key == ord("r"): + tactile.reset() + print("\nTracker reset.") + finally: + runtime.close() + cv2.destroyAllWindows() + + +if __name__ == "__main__": + main() diff --git a/examples/basic_pipeline.py b/examples/basic_pipeline.py index 7f32846..9ee99e1 100644 --- a/examples/basic_pipeline.py +++ b/examples/basic_pipeline.py @@ -1,137 +1,120 @@ +"""ORY-BOX / ORY-MINI: read a sensor and get tactile information, in one loop. + +The whole recommended flow: + + sensor -> StreamPipeline -> FrameSnapshot per frame + +One object reads the camera, pre-processes (stitch, grey, resize), computes the deformation field +and the forces, and hands back one immutable snapshot per frame. There is no stage sequence to +remember and no tuple to unpack by position. + +What you do NOT need to touch: the compute backend, the capture queue, the stitch mode, the +low-pass filter, or the order in which the stages run. Those are the SDK's concern and +``StreamPipeline`` sequences them on three concurrent threads. If you later need that control, the +manual form is preserved in ``examples/archive/basic_pipeline_manual.py``. + +Run: + + python examples/basic_pipeline.py # first camera, box config + python examples/basic_pipeline.py -v 0 -c mini + python examples/basic_pipeline.py -v recording.mp4 -c box + python examples/basic_pipeline.py --no-show # headless: print only + +Keys: 'q' quit, 'r' reset the tracker. """ -ORY-BOX:单传感器基础流程 -本示例演示 Orisys SDK 的标准处理流程: -1. 初始化传感器(摄像头或视频文件) -2. 获取图像并计算形变 -3. 按需计算接触区域 -4. 读取并可视化结果 - -""" -import os - -os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" import argparse +import os +import sys + +# Set before OpenCV is imported: stops the capture backend applying its own transform on top of +# the SDK's preprocessing. This is a capture detail, not a tuning knob you need to own - it is +# here so the file can be copied as a whole. +os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0") import cv2 -import numpy as np import orisys +from _display import depth_to_color, draw_contact, draw_flow_arrows -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--video","-v", type=str, default="0", help="视频源:摄像头编号或视频文件路径") - parser.add_argument("--config","-c", type=str, default="box", help="配置文件名称或路径") - parser.add_argument("--verbose","-verbose", type=bool, default=True, help="是否输出详细日志") - parser.add_argument("--cal_path","-cal", type=str, default=None, help="标定文件路径") +def main() -> int: + parser = argparse.ArgumentParser(description="Read a sensor and print its tactile information.") + parser.add_argument("-v", "--video", default="0", + help="camera index or video file path (default: 0)") + parser.add_argument("-c", "--config", default="box", + help="product config: box, mini, finger, or a config path") + parser.add_argument("--cal", default=None, + help="force calibration file; omit to use the sensor bundle's own") + parser.add_argument("--no-show", dest="show", action="store_false", + help="do not open the display windows") + parser.add_argument("-verbose", action="store_true", help="SDK verbose logs") args = parser.parse_args() - # 步骤 1:创建传感器对象(必需) - # config_name 可以是内置配置名称,也可以是自定义配置文件路径 - # cal_path 为标定文件路径;每台设备建议使用独立的标定文件 + # ------------------------------------------------------------------ create the object + # That is the whole setup. The source may be a camera index or a video path. + source = int(args.video) if args.video.isdigit() else args.video + sensor = orisys.Sensor(source, config_name=args.config, cal_path=args.cal, + verbose=args.verbose) - # 输入既可以是摄像头编号,也可以是视频文件路径 - if args.video.isdigit(): - sensor = orisys.Sensor(int(args.video), config_name=args.config, verbose=args.verbose, cal_path=args.cal_path) - else: - sensor = orisys.Sensor(args.video, config_name=args.config, verbose=args.verbose, cal_path=args.cal_path) + # contact=True runs contact analysis. include=... selects which arrays each snapshot carries; + # arrays that are not included are None, so you pay only for what you use. + include = ("image", "flow", "depth", "centroid", "contour") if args.show else ("image",) - print("\n按键说明:'q' 退出程序,'r' 重置追踪器。") + if args.show: + print("Keys: 'q' quit, 'r' reset tracker.") - while True: - # 步骤 2:数据处理 - # 步骤 2.1:获取并拼接图像(必需) - sensor.get_img() - # 步骤 2.2:计算形变场(必需);返回值表示当前帧是否检测到接触/运动 - is_contact_now = sensor.compute_deformation() - # 步骤 2.3:仅在当前帧检测到接触/运动时计算接触区域 - # if is_contact_now: - sensor.compute_contact() + try: + with orisys.StreamPipeline(sensor, sensor.get_img, contact=True, + include=include) as pipe: + while True: + # ------------------------------------------------------ read one frame + # Blocks until a frame newer than the last one is ready. Everything in the + # snapshot belongs to the same frame. + snap = pipe.wait() + if snap is None: + print("\nInput ended.") + break - - # 步骤 2.4:读取结果(FNORMAL/FSHEAR* 在有 sensors/box/force_calibration.json 时为标定后牛顿力; - # *_RAW 始终为原始积分量) - ( - fps, - flow, - vnormal, - img, - contour, - centroid, - depth_map, - fnormal, - fshearx, - fsheary, - fnormal_raw, - fshearx_raw, - fsheary_raw, - ) = sensor.read_info( - sensor.info.FPS, - sensor.info.VRAW, - sensor.info.VNORMAL, - sensor.info.IMG, - sensor.info.CONTOUR, - sensor.info.CENTROID, - sensor.info.DEPTH, - sensor.info.FNORMAL, - sensor.info.FSHEARX, - sensor.info.FSHEARY, - sensor.info.FNORMAL_RAW, - sensor.info.FSHEARX_RAW, - sensor.info.FSHEARY_RAW, - ) - - # 步骤 3:可视化显示 - # 步骤 3.1:显示形变矢量场 - arrows = orisys.util.draw_arrows( - img, - flow, - threshold=2, - grid_spacing=20, - arrow_scale=1.0 - ) - cv2.imshow("arrow", arrows) - - # 步骤 3.2:显示接触区域和质心 - image_with_foe = orisys.util.draw_contact(img, contour, centroid) - cv2.imshow("contact", image_with_foe) + # ------------------------------------------------------ use the results + # Scalars are calibrated Newtons when the bundle ships a calibration; the *_raw + # pair is always the unfiltered integrator output. + touching = "yes" if snap.centroid is not None else "no" + print( + f"FPS={snap.fps:5.1f} " + f"normal={snap.normal_force:8.4f} N (raw {snap.normal_force_raw:10.2f}) " + f"shear=({snap.shear_x:7.4f}, {snap.shear_y:7.4f}) " + f"contact={touching} age={snap.age_ms:4.1f} ms", + end="\r", + ) - # 步骤 3.3:显示深度图 - # 将深度值归一化到 0-255 范围 - div_abs = depth_map - if div_abs.max() > div_abs.min(): - div_normalized = ((div_abs - div_abs.min()) / (div_abs.max() - div_abs.min()) * 255).astype(np.uint8) - else: - div_normalized = np.zeros_like(div_abs, dtype=np.uint8) - - # 应用伪彩色映射 - cv2.imshow("depth", cv2.applyColorMap(div_normalized, cv2.COLORMAP_JET)) + if args.show and snap.image is not None: + if snap.flow is not None: + cv2.imshow("deformation", draw_flow_arrows( + snap.image, snap.flow, threshold=2, grid_spacing=20)) + if snap.contour is not None and snap.centroid is not None: + cv2.imshow("contact", draw_contact( + snap.image, snap.contour, snap.centroid)) + colour = depth_to_color(snap.depth) + if colour is not None: + cv2.imshow("depth", colour) - # 步骤 3.4:打印关键结果(标定力 + 原始力) - print( - f"FPS={fps:.2f}, " - f"法向力={fnormal:.4f} N (raw={fnormal_raw:.4f}), " - f"切向力X={fshearx:.4f} N (raw={fshearx_raw:.4f}), " - f"切向力Y={fsheary:.4f} N (raw={fsheary_raw:.4f}), " - f"深度图尺寸={depth_map.shape}, 原始分辨率={sensor.img_size_raw}" - ) - - # 步骤 3.5:键盘控制 - key = cv2.waitKey(1) & 0xFF - if key == ord("q"): - print("\n正在退出示例程序...") - break - if key == ord("r"): - # 当光流追踪出现漂移时,可手动重置追踪器 - sensor.reset() - print("追踪器已重置。") - - # ========================================================================= - # 4. 释放资源 - # ========================================================================= - sensor.disconnect() # 释放视频源及相关资源 + key = cv2.waitKey(1) & 0xFF + if key == ord("q"): + print("\nExiting.") + break + if key == ord("r"): + # Queued onto the flow stage, so this is safe from the consumer loop. + pipe.reset() + print("\nTracker reset.") + except KeyboardInterrupt: + print("\nInterrupted.") + finally: + sensor.disconnect() + if args.show: + cv2.destroyAllWindows() + return 0 -if __name__ == '__main__': - main() +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/examples/finger_pipeline.py b/examples/finger_pipeline.py index 63af80b..466c096 100644 --- a/examples/finger_pipeline.py +++ b/examples/finger_pipeline.py @@ -1,14 +1,21 @@ -#!/usr/bin/env python3 -""" -ORY-FINGER:Minimum UV-unroll finger tactile pipeline. +"""ORY-FINGER: UV-unrolled finger tactile pipeline. -Offline: load Blender export -> build_or_load_domain -> unroll_map_uv.npz -Runtime: camera frame -> FrameUnwarper -> TactileProcessor -> forces/contact +Same shape as ``basic_pipeline.py`` - one object, one call per frame - with the finger-specific +capture path underneath: -Recommended entry: - python examples/finger_pipeline.py --camera_id 0 --config finger + FingerRuntime -> runtime.pipeline() -> FrameSnapshot per frame -Current product config: ``finger`` (``dd02`` still resolves to ``dd02-ov``). +``FingerRuntime`` owns the camera, the Blender export (mesh, intrinsics, world-to-camera), the UV +unroll map and the valid-region mask. ``runtime.pipeline()`` runs those stages concurrently and +returns one immutable snapshot per frame, so there is nothing to sequence and no tuple to unpack. + +Run: + + python examples/finger_pipeline.py --camera_id 0 --config finger + python examples/finger_pipeline.py --video path/to/clip.mp4 + python examples/finger_pipeline.py --no-show # headless: print only + +Keys: 'q' quit, 'r' reset the tracker. """ import argparse @@ -19,95 +26,83 @@ os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0") import cv2 -import orisys +from _display import draw_flow_arrows from orisys.finger import FingerRuntime -def main(): - parser = argparse.ArgumentParser(description="Minimum UV-unroll finger tactile pipeline") - parser.add_argument("--camera_id", type=int, default=0) - parser.add_argument("--video","-v", help="Video file path (overrides camera)") - parser.add_argument("--config", "-c", default="finger", help="Orisys SDK config name or path") - parser.add_argument("--show", action="store_true", default=True, help="Show unrolled image window") - parser.add_argument("--verbose", action="store_true", help="SDK verbose logs") +def main() -> int: + parser = argparse.ArgumentParser(description="UV-unrolled finger tactile pipeline") + parser.add_argument("--camera_id", type=int, default=0, help="camera index (default 0)") + parser.add_argument("-v", "--video", help="video file path; overrides --camera_id") + parser.add_argument("-c", "--config", default="finger", + help="product config name or path (default: finger)") + parser.add_argument("--no-show", dest="show", action="store_false", + help="do not open the display window") + parser.add_argument("-verbose", action="store_true", help="SDK verbose logs") args = parser.parse_args() + source = args.video if args.video else args.camera_id + + # ------------------------------------------------------------------ create the object try: - video = args.camera_id - if args.video: - video = args.video - runtime = FingerRuntime.open( - config=args.config, - video=video, - verbose=args.verbose, - ) + runtime = FingerRuntime.open(config=args.config, video=source, verbose=args.verbose) except RuntimeError as exc: print(str(exc), file=sys.stderr) - sys.exit(1) + return 1 if args.verbose: - geometry = runtime.geometry - print( - f"UV geometry: native={geometry.native_width}x{geometry.native_height}, " - f"source={geometry.source_width}x{geometry.source_height} " - f"output={geometry.out_width}x{geometry.out_height} " - ) + g = runtime.geometry + print(f"UV geometry: native={g.native_width}x{g.native_height} " + f"source={g.source_width}x{g.source_height} " + f"output={g.out_width}x{g.out_height}") + # Fill the flow/HHD caches once so the first published frame is not a warm-up outlier. runtime.warmup(color=False) - tactile = runtime.tactile - print("Press 'q' to quit, 'r' to reset tracker.") + if args.show: + print("Keys: 'q' quit, 'r' reset tracker.") + + include = ("image", "flow") if args.show else ("image",) try: - while True: - ok, unrolled = runtime.grab_and_process(color=False) - if not ok: - break + with runtime.pipeline(contact=True, include=include) as pipe: + while True: + # ------------------------------------------------------ read one frame + snap = pipe.wait() + if snap is None: + print("\nInput ended.") + break - tactile.compute_deformation() - tactile.compute_contact() - - fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = tactile.read_info( - tactile.info.FPS, - tactile.info.VRAW, - tactile.info.VNORMAL, - tactile.info.IMG, - tactile.info.CONTOUR, - tactile.info.CENTROID, - tactile.info.DEPTH, - tactile.info.FNORMAL, - tactile.info.FSHEARX, - tactile.info.FSHEARY, - ) - - status = ( - f"FPS={fps:.2f} normal={fnormal:.2f} shear=({fshearx:.2f}, {fsheary:.2f}) " - f"mask={tactile.mask_stats['coverage']:.2f}" - ) - - if args.show: - arrows = orisys.util.draw_arrows( - unrolled, - flow, - threshold=0.4, - grid_spacing=10, - arrow_scale=1.0, - valid_mask=tactile.valid_mask, + # ------------------------------------------------------ use the results + coverage = runtime.tactile.mask_stats.get("coverage") + coverage_text = f"{coverage:.2f}" if isinstance(coverage, float) else "n/a" + print( + f"FPS={snap.fps:5.1f} " + f"normal={snap.normal_force:8.4f} N " + f"shear=({snap.shear_x:7.4f}, {snap.shear_y:7.4f}) " + f"mask={coverage_text} age={snap.age_ms:4.1f} ms", + end="\r", ) - cv2.imshow("UV Unrolled", arrows) - print(status, end="\r") + if args.show and snap.image is not None and snap.flow is not None: + cv2.imshow("tactile (UV unrolled)", draw_flow_arrows( + snap.image, snap.flow, threshold=0.4, grid_spacing=10, + valid_mask=runtime.tactile.valid_mask)) - key = cv2.waitKey(1) & 0xFF - if key == ord("q"): - print("\nExiting.") - break - if key == ord("r"): - tactile.reset() - print("\nTracker reset.") + key = cv2.waitKey(1) & 0xFF + if key == ord("q"): + print("\nExiting.") + break + if key == ord("r"): + pipe.reset() + print("\nTracker reset.") + except KeyboardInterrupt: + print("\nInterrupted.") finally: runtime.close() - cv2.destroyAllWindows() + if args.show: + cv2.destroyAllWindows() + return 0 if __name__ == "__main__": - main() + sys.exit(main()) \ No newline at end of file diff --git a/src/orisys/finger/runtime.py b/src/orisys/finger/runtime.py index acae2f1..eaf4478 100644 --- a/src/orisys/finger/runtime.py +++ b/src/orisys/finger/runtime.py @@ -114,6 +114,21 @@ class TactileProcessor: def warmup(self): return self._sensor.warmup_deformation() + def pipeline(self, produce, **kwargs): + """Start a :class:`~orisys.pipeline.StreamPipeline` over this processor's own Sensor. + + ``produce`` is the frame source: a zero-argument callable returning the next unrolled + frame, or ``None`` at end of input. :class:`FingerRuntime` supplies it through + :meth:`FingerRuntime.pipeline`, so call this directly only when you are producing frames + yourself. + + Any keyword argument is forwarded to ``StreamPipeline``, so ``contact=True`` and + ``include=(...)`` work exactly as they do there. + """ + from orisys.pipeline import StreamPipeline + + return StreamPipeline(self._sensor, produce, **kwargs) + def compute_deformation(self, *args, **kwargs): return self._sensor.compute_deformation(*args, **kwargs) @@ -361,6 +376,27 @@ class FingerRuntime: self.tactile.warmup() return ok, unrolled + def pipeline(self, **kwargs): + """Return a :class:`~orisys.pipeline.StreamPipeline` for this runtime. + + The recommended way to run a finger sensor. Each finished frame arrives as a single + :class:`~orisys.pipeline.FrameSnapshot`, so there are no stage calls to sequence and no + tuple to unpack:: + + with runtime.pipeline(contact=True, include=("image", "depth")) as pipe: + while True: + snap = pipe.wait() + if snap is None: + break + print(snap.normal_force, snap.shear_x, snap.shear_y) + + Any keyword argument is forwarded to ``StreamPipeline``. Use ``contact=True`` to run + contact analysis, and ``include=(...)`` to select which arrays each snapshot carries. + """ + from orisys.pipeline import StreamPipeline + + return StreamPipeline.for_finger_runtime(self, **kwargs) + def close(self): if self.capture_source is not None: self.capture_source.close() diff --git a/src/orisys/pipeline.py b/src/orisys/pipeline.py index 236d0ee..c3bcde4 100644 --- a/src/orisys/pipeline.py +++ b/src/orisys/pipeline.py @@ -110,6 +110,10 @@ class FrameSnapshot: produce_ms: float = 0.0 flow_ms: float = 0.0 hhd_ms: float = 0.0 + #: Capture resolution before any stitch/unroll/resize, as ``(width, height)``, or ``None`` + #: when the producer did not report one. Read from ``Sensor.img_size_raw`` because callers + #: compare it with ``image.shape`` to see how much the frame was reduced. + image_size_raw: tuple[int, int] | None = None @dataclass @@ -180,6 +184,9 @@ class StreamPipeline: self._threads: list[threading.Thread] = [] self._error: BaseException | None = None self._running = False + #: Set by :meth:`reset` and consumed by the flow stage, so that resetting sensor state + #: never races the stage that owns that state. + self._reset_pending = threading.Event() self._lock = threading.Lock() self._latest: FrameSnapshot | None = None @@ -258,6 +265,19 @@ class StreamPipeline: with self._lock: return self._latest + def reset(self) -> None: + """Reset the sensor's tracker and force-filter state, safely. + + Call this from the consumer thread (after a key press, on a drift alarm). The reset is + queued and applied by the flow stage before its next frame, never inline: ``Sensor.reset`` + rebases ``initial_img`` on the current ``Sensor.img``, and the flow stage is reading that + buffer concurrently, so resetting from the caller's thread would capture a half-written + reference frame. + + Idempotent: a second call before the next frame does not queue twice. + """ + self._reset_pending.set() + def wait(self, timeout: float | None = None) -> FrameSnapshot | None: """Block until a snapshot newer than the last returned one exists. @@ -351,6 +371,11 @@ class StreamPipeline: item = self._get(self._q1) if item is None: break + # Applied here, on the thread that owns the sensor's flow state, before this frame is + # computed: see reset() for why this must not run on the caller's thread. + if self._reset_pending.is_set(): + self._reset_pending.clear() + self.sensor.reset() frame, t_cap, produce_s = item t0 = time.perf_counter() # The frame is passed explicitly: the SDK's self.img slot is shared @@ -406,6 +431,12 @@ class StreamPipeline: def maybe(name: str, selector: Callable[[], Any]) -> np.ndarray | None: return self._copy_array(selector) if name in include else None + raw_size = getattr(sensor, "img_size_raw", None) + if isinstance(raw_size, (tuple, list)) and len(raw_size) == 2: + image_size_raw: tuple[int, int] | None = (int(raw_size[0]), int(raw_size[1])) + else: + image_size_raw = None + return FrameSnapshot( seq=self.stats.published, timestamp=time.time(), @@ -431,6 +462,7 @@ class StreamPipeline: produce_ms=produce_s * 1000.0, flow_ms=flow_s * 1000.0, hhd_ms=hhd_s * 1000.0, + image_size_raw=image_size_raw, ) def _fps(self) -> float: diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index ddcf4ec..5c97497 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -2,6 +2,7 @@ from __future__ import annotations +import threading import time import numpy as np @@ -47,6 +48,12 @@ class FakeSensor: self._seen_images: list[np.ndarray] = [] self._seen_flow_values: list[float] = [] self.contact_calls = 0 + #: Mirrors ``Sensor.reset()``, which rebases ``initial_img`` on the current frame. Records + #: the frame sequence at each call so a test can prove the reset did not race the flow + #: stage's read of that buffer. + self.img_size_raw = None + self.reset_calls = 0 + self.reset_at_frame: list[int] = [] def make_frame(self, seq: int) -> np.ndarray: # writes INTO the shared buffer, exactly like cv2.resize(dst=...) @@ -88,6 +95,10 @@ class FakeSensor: self.contour = np.zeros((1, 1, 2), dtype=np.int32) self.depth_map = np.zeros((8, 8), dtype=np.float32) + def reset(self) -> None: + self.reset_calls += 1 + self.reset_at_frame.append(self._flow_calls) + def read_info(self, *args): source = { self.info.VRAW: lambda: self.flow.copy(), @@ -327,3 +338,70 @@ def test_for_finger_runtime_maps_eof_to_none(): pipe.stop() assert len(frames) == 3 assert pipe.stats.produced == 3 + + +def test_reset_is_applied_on_the_flow_stage_not_the_caller(): + """``reset()`` must not run on the caller's thread. + + ``Sensor.reset`` rebases ``initial_img`` on the current ``Sensor.img``, and the flow stage is + reading that buffer concurrently, so a caller-thread reset would capture a half-written + reference frame. The pipeline queues it and applies it before the next frame instead. + """ + sensor = FakeSensor() + ready = threading.Event() + release = threading.Event() + + def gated_produce(): + # Hold the producer until the caller has asked for a reset, so the reset is provably + # queued before any frame reaches the flow stage. + ready.set() + release.wait(timeout=5.0) + return sensor.make_frame(0) + + frames: list = [] + pipe = StreamPipeline(sensor, gated_produce, on_frame=frames.append) + pipe.start() + assert ready.wait(timeout=5.0) + + pipe.reset() + pipe.reset() # idempotent: a second request before the next frame is not doubled + assert sensor.reset_calls == 0, "reset must not run on the caller's thread" + + release.set() + deadline = time.perf_counter() + 5.0 + while not frames and time.perf_counter() < deadline: + time.sleep(0.001) + pipe.stop() + + assert sensor.reset_calls == 1, f"expected one queued reset, got {sensor.reset_calls}" + assert sensor.reset_at_frame == [0], ( + "the reset must land before the first compute_flow, not after it" + ) + + +def test_snapshot_carries_the_raw_capture_size(): + """``image_size_raw`` lets a caller compare the processed frame with the capture resolution.""" + sensor = FakeSensor() + sensor.img_size_raw = (1280, 720) + frames: list = [] + pipe = StreamPipeline(sensor, make_producer(sensor, 1), on_frame=frames.append) + pipe.start() + deadline = time.perf_counter() + 5.0 + while not frames and time.perf_counter() < deadline: + time.sleep(0.001) + pipe.stop() + assert frames and frames[0].image_size_raw == (1280, 720) + + +def test_snapshot_raw_capture_size_is_none_when_unreported(): + """A producer that never sets the size must not fabricate one.""" + sensor = FakeSensor() + sensor.img_size_raw = None + frames: list = [] + pipe = StreamPipeline(sensor, make_producer(sensor, 1), on_frame=frames.append) + pipe.start() + deadline = time.perf_counter() + 5.0 + while not frames and time.perf_counter() < deadline: + time.sleep(0.001) + pipe.stop() + assert frames and frames[0].image_size_raw is None