orisys-tech/orisys-sdk
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update dd02 3d viewer v1

Browse Source
This commit is contained in:
ykcheungab
2026-06-02 12:15:36 +08:00
parent 7224fea411
commit 8a50492465
16 changed files with 2269 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/orisys/__init__.py
+5
View File
@@ -12,6 +12,9 @@ from .sensor import Sensor
from . import util
from . import configs
CompoundEyeSensor = Sensor
CameraSensor = Sensor
# Re-export list_configs as list_config for convenience
list_config = configs.list_config
export_config = configs.export_config
@@ -19,6 +22,8 @@ export_config = configs.export_config
__all__ = [
# class
'Sensor',
'CompoundEyeSensor',
'CameraSensor',
# util functions
'util',
src/orisys/frame_processors.py
+36
View File
@@ -0,0 +1,36 @@
import cv2
import numpy as np
class CompoundEyeFrameProcessor:
"""Convert a raw compound-eye frame into one stitched BGR tactile image."""
def __init__(self, stitcher):
self.stitcher = stitcher
def process(self, frame):
self.stitcher.get_stitch_unit(frame)
self.stitcher.extract_tiles()
return self.stitcher.stitch_rt_optimized(blend=False)
class SingleCameraFrameProcessor:
"""Convert a normal camera frame into one BGR tactile image."""
def process(self, frame):
if frame.ndim == 2:
return cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR)
if frame.shape[2] == 1:
return cv2.cvtColor(frame[:, :, 0], cv2.COLOR_GRAY2BGR)
return frame
def create_frame_processor(sensor_type, stitcher=None):
sensor_type = (sensor_type or "compound_eye").lower()
if sensor_type in ("compound_eye", "compound-eye", "compound"):
if stitcher is None:
raise ValueError("Compound-eye sensor requires a Stitcher instance.")
return CompoundEyeFrameProcessor(stitcher)
if sensor_type in ("single_camera", "camera", "mono"):
return SingleCameraFrameProcessor()
raise ValueError(f"Unsupported sensor type: {sensor_type}")
src/orisys/sensor.py
+37 -34
View File
@@ -7,7 +7,6 @@ and Helmholtz-Hodge decomposition.
"""
import os
import platform
import numpy as np
import cv2
import time
@@ -23,10 +22,7 @@ from .hhd import nHHD
from . import configs
from .get_depth import Contact3D
from .backend_availability import (check_cupy_available, check_opencl_available)
try:
import av # Optional: used for Linux camera capture path.
except ImportError:
av = None
from .frame_processors import create_frame_processor
class Sensor:
"""
@@ -149,7 +145,12 @@ class Sensor:
}
# Initialize components
self.stitcher = Stitcher(isstitch=isstitch, config_stitcher=self.__config_stitcher, verbose=self.verbose)
self.stitcher = (
Stitcher(isstitch=isstitch, config_stitcher=self.__config_stitcher, verbose=self.verbose)
if self._is_compound_eye()
else None
)
self.frame_processor = create_frame_processor(self.sensor_type, stitcher=self.stitcher)
self.__tracker = Tracker(adaptive=True, cuda=cuda, verbose=self.verbose, config_tracker=self.__config_tracker)
self.__contact3d = Contact3D(config_contact=self.__config_contact, verbose=self.verbose)
self.__hhd = None
@@ -167,7 +168,6 @@ class Sensor:
self._av_stream = None
self._av_decoder = None
self.cap = None
# Treat both camera index (0) and V4L2 path ("/dev/video0") as live camera for threading/low-latency path.
self._is_live_camera = (
isinstance(self.vid_src, int)
or (isinstance(self.vid_src, str) and self.vid_src.startswith("/dev/video"))
@@ -219,10 +219,9 @@ class Sensor:
t_processing_start = time.perf_counter()
self.frame = frame
self._extract_tiles(frame)
img_bgr = self.frame_processor.process(frame)
t_after_extract = time.perf_counter()
img_bgr = self.stitcher.stitch_rt_optimized(blend=False)
t_after_stitch = time.perf_counter()
self.img_size_raw = img_bgr.shape
@@ -538,22 +537,29 @@ class Sensor:
def _load_config(self):
"""Load sensor configuration from JSON file."""
self._config = configs.load_config(self.config_name)
self.sensor_type = self._config.get(
"type",
"compound_eye" if "stitcher" in self._config else "single_camera",
)
self.__config_cam = self._config["camera"]
self._config_resolution_raw = self.__config_cam["resolution_raw"]
self._config_fps = self.__config_cam["fps"]
self._config_units = self.__config_cam["units"]
self._config_units = self.__config_cam.get("units", 1)
self._config_resolution_stitch = self.__config_cam["resolution_stitch"]
self._config_dis_preset = self._config["tracker"]["dis_preset"]
self.__config_tracker = self._config["tracker"]
self.__config_stitcher = self._config["stitcher"]
self.__config_stitcher = self._config.get("stitcher")
self.__config_contact = self._config["contact"]
if self.verbose:
print(f"Config: {self.config_name}")
return True
def _is_compound_eye(self):
return str(self.sensor_type).lower() in ("compound_eye", "compound-eye", "compound")
def _open_video_source(self, override, box):
"""Initialize video capture and calibration."""
# _is_camera_index = isinstance(self.vid_src, int)
@@ -589,16 +595,11 @@ class Sensor:
self.use_pyav = False
if not self._use_pyav:
self.cap = cv2.VideoCapture(self.vid_src)
# Set buffer size to 1 to reduce latency and avoid frame accumulation
# Note: Not all backends support this, but it helps when available
# try:
# self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
# except Exception:
# pass # Some backends don't support this property
self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M', 'J', 'P', 'G'))
self.cap.set(cv2.CAP_PROP_FPS, self._config_fps)
self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self._config_resolution_raw[0])
self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self._config_resolution_raw[1])
if self._is_live_camera:
self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M', 'J', 'P', 'G'))
self.cap.set(cv2.CAP_PROP_FPS, self._config_fps)
self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self._config_resolution_raw[0])
self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self._config_resolution_raw[1])
if self.verbose:
@@ -633,7 +634,10 @@ class Sensor:
if not ret:
time.sleep(0.1)
if override:
if not self._is_compound_eye():
self.stitch_valid = False
self.initial_img = self._make_initial_gray(frame)
elif override:
if box is not None:
self.stitcher.isstitch = self.isstitch = False
self.stitcher.boxes = box
@@ -641,24 +645,23 @@ class Sensor:
print("Override=True but no box provided.")
else:
self._find_stitch_boxes(frame)
self._extract_tiles(frame)
# Feature-based stitching no longer needs eager overlap calibration here.
# Keep stitch_cal() as a fallback so older overlap-based stitching still works.
img = self.stitcher.stitch_rt_optimized()
# if img is None:
# img = self.stitch_cal()
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
self.resolution_stitch = img.shape
self.initial_img = cv2.resize(img, tuple(self._config_resolution_stitch), interpolation=cv2.INTER_CUBIC)
if self.mirror:
self.initial_img = cv2.flip(self.initial_img, 1)
self.initial_img = self.initial_img.astype(np.uint8)
img = self.frame_processor.process(frame)
self.initial_img = self._make_initial_gray(img)
# For live cameras, start a latest-frame capture thread to avoid latency buildup.
if self._is_live_camera:
self._start_capture_thread()
def _make_initial_gray(self, img_bgr):
gray = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY)
self.resolution_stitch = gray.shape
initial = cv2.resize(gray, tuple(self._config_resolution_stitch), interpolation=cv2.INTER_CUBIC)
if self.mirror:
initial = cv2.flip(initial, 1)
return initial.astype(np.uint8)
def _find_stitch_boxes(self, img_src, show=False):
"""Detect vision tile boxes with camera stabilization."""
n_boxes, v_mean0, timeout = 0, 0, 5