orisys-tech/orisys-sdk
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GUI for finger v1

Browse Source
This commit is contained in:
ykcheungab
2026-06-18 22:18:19 +08:00
parent f19f092b64
commit 3fd9b839a9
42 changed files with 2326 additions and 1186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/README.md
+41 -41
View File
@@ -1,41 +1,41 @@
# Examples
User-facing demo scripts for the Orisys SDK. Run from the repository root.
## Layout
- `basic_pipeline.py` — compound-eye sensor, OpenCV visualization
- `plot_data.py` — compound-eye sensor, force charts (PyQtGraph)
- `two_sensor_pipeline.py` — dual-sensor multiprocessing demo
- `finger_pipeline.py` — finger UV minimal pipeline
- `finger_pipeline_3dview.py` — finger UV + Open3D mesh viewer
- `finger_plot_data.py` — finger UV + force charts
- `viz/` — shared visualization helpers used by plot and finger examples
- `mesh_viewer.py` — re-exports `UVMeshViewer` / `build_uv_surface_mesh` from `orisys.finger`
- `finger_panels.py` — 2D OpenCV panels for finger demos
## Typical usage
```bash
python examples/basic_pipeline.py -v 0 -c ./config/ddjx01.json
python examples/plot_data.py -v 0 -c ./config/ddjx01.json
python examples/finger_plot_data.py -v 0 -c ./config/dd02-ov.json
```
## Requirements
- SDK installed (`pip install -e .`)
- camera or test video
- JSON config under `./config/`
Plot examples additionally require:
```bash
pip install pyqt5 pyqtgraph
```
## Notes
- Press `q` to quit OpenCV windows where applicable.
- Plot examples: `s` starts recording, `e` stops recording.
# Examples
User-facing demo scripts for the Orisys SDK. Run from the repository root.
## Layout
- `basic_pipeline.py` — compound-eye sensor, OpenCV visualization
- `plot_data.py` — compound-eye sensor, force charts (PyQtGraph)
- `two_sensor_pipeline.py` — dual-sensor multiprocessing demo
- `finger_pipeline.py` — finger UV minimal pipeline
- `finger_pipeline_3dview.py` — finger UV unified Qt dashboard
- `pipeline/` — runtime acquisition (`FingerPipeline`, `FingerFrameView`)
- `gui/` — Qt dashboard composition (3D + 2D + charts in one window)
- `viz/` — rendering helpers (offscreen mesh, 2D panels, charts)
## Typical usage
```bash
pip install -e ".[gui]"
python examples/finger_pipeline_3dview.py -v 0 -c ./config/dd02-ov.json
# Sales-demo shim
python scripts/qt_viewer_finger.py -v 0 -c ./config/dd02-ov.json
```
## Requirements
- SDK installed (`pip install -e .`)
- camera or test video
- JSON config under `./config/`
GUI examples additionally require:
```bash
pip install -e ".[gui]"
```
## Notes
- Dashboard: click **启动** to start acquisition; `Q` quit, `R` reset, `S`/`E` record, `Esc` fullscreen.
examples/finger_pipeline_3dview.py
+29 -153
View File
@@ -1,13 +1,12 @@
#!/usr/bin/env python3
"""
UV-unroll finger tactile pipeline with 3D mesh visualization.
UV-unroll finger tactile pipeline with unified Qt dashboard.
Offline: load Blender export -> build_or_load_domain -> unroll_map_uv.npz
Runtime: camera frame -> UVUnrollFrameProcessor -> TactileProcessor -> forces/contact
UV-space flow/HHD -> grey-white 3D finger mesh (Open3D)
For the full 2D dashboard demo see:
tools/3d_patch_generation_blender/unroll_realtime.py
Where to customize the GUI:
Layout (sizes, proportions) -> examples/gui/layout.py <-- edit here
Colors / QSS -> examples/gui/styles.py
Widget rendering logic -> examples/gui/widgets/
Window behavior -> examples/gui/dashboard.py
"""
import argparse
@@ -17,172 +16,49 @@ from pathlib import Path
os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0")
import cv2
from orisys.finger import DEFAULT_FINGER_EXPORT_DIR, DEFAULT_FINGER_MAP_CACHE, FingerRuntime
_EXAMPLES_DIR = Path(__file__).resolve().parent
if str(_EXAMPLES_DIR) not in sys.path:
sys.path.insert(0, str(_EXAMPLES_DIR))
from viz import UVMeshViewer, build_uv_surface_mesh, show_finger_uv_panels
from gui import run_dashboard
from pipeline import PipelineConfig
from viz import DEFAULT_FINGER_VIZ_POSE
def open_mesh_viewer(runtime, args):
domain = runtime.domain
if domain.surface is None:
raise RuntimeError(
"UV domain has no world_xyz surface map. Rebuild cache with include_surface=True."
)
uv_mesh = build_uv_surface_mesh(domain.surface.world_xyz, domain.valid_mask)
if args.no_3d:
return None, uv_mesh
viewer = UVMeshViewer(
uv_mesh,
max_magnitude=args.magnitude_max,
threshold=args.depth_threshold if args.mesh_map == "depth" else args.flow_threshold,
color_mode=args.mesh_map,
)
print(
f"3D mesh viewer: {len(uv_mesh.rest_vertices)} vertices, "
f"{len(uv_mesh.triangles)} triangles, {args.mesh_map} surface mode"
)
return viewer, uv_mesh
def main():
def _parse_args():
parser = argparse.ArgumentParser(
description="UV-unroll finger tactile pipeline with 3D mesh viewer"
description="UV-unroll finger tactile pipeline with unified Qt dashboard"
)
parser.add_argument(
"--export-dir",
default=DEFAULT_FINGER_EXPORT_DIR,
help="Blender export folder containing mesh.npz, camera_intrinsics.npz, world_to_cam.npy",
)
parser.add_argument("--map_cache", default=DEFAULT_FINGER_MAP_CACHE, help="Cached UV domain .npz")
parser.add_argument("--camera_id", type=int, default=0)
parser.add_argument("--video", help="Video file path (overrides camera)")
parser.add_argument("--video", "-v", default="0", help="Camera index or video file path")
parser.add_argument("--config", "-c", default="./config/dd02-ov.json", help="Orisys SDK config")
parser.add_argument(
"--source_scale",
type=float,
default=None,
help="Resize live source before UV remap (default: camera.source_scale in config)",
)
parser.add_argument("--no_flip_source_vertical", action="store_true", help="Skip vertical flip before UV remap")
parser.add_argument("--show", action="store_true", help="Show 2D unrolled image window with flow arrows")
parser.add_argument("--cal", default="", help="Optional calibration file path")
parser.add_argument("--verbose", action="store_true", help="SDK verbose logs")
parser.add_argument(
"--magnitude-max",
type=float,
default=None,
help="Upper optical-flow magnitude for 3D surface heat-map scaling; default uses frame percentile.",
)
parser.add_argument(
"--mesh-map",
choices=["flow", "depth"],
default="depth",
help="Color the 3D mesh from optical-flow magnitude or the contact depth/divergence map.",
help="Color the 3D mesh from flow magnitude or contact depth.",
)
parser.add_argument("--threshold", type=float, default=None, help="Mesh/arrow threshold")
parser.add_argument("--no-3d", action="store_true", help="Disable 3D mesh view")
parser.add_argument(
"--viz-pose",
default=DEFAULT_FINGER_VIZ_POSE,
help="world_to_cam.npy for the default view; pass 'none' to skip",
)
parser.add_argument(
"--flow-threshold",
"--cutoff",
type=float,
default=0.4,
help="Minimum optical-flow magnitude to color 3D surface / draw 2D arrows (matches draw_arrows).",
default=5.0,
help="Low-pass cutoff for force curves (Hz)",
)
parser.add_argument(
"--depth-threshold",
type=float,
default=0.0,
help="Minimum depth/divergence value to color on the 3D mesh.",
)
parser.add_argument("--no-3d", action="store_true", help="Disable Open3D mesh viewer")
args = parser.parse_args()
return parser.parse_args()
video = args.camera_id
if args.video:
video = args.video
try:
runtime = FingerRuntime.open(
export_dir=args.export_dir,
config=args.config,
video=video,
verbose=args.verbose,
)
except RuntimeError as exc:
print(str(exc), file=sys.stderr)
sys.exit(1)
if args.verbose:
geometry = runtime.geometry
print(
f"UV geometry: native={geometry.native_width}x{geometry.native_height}, "
f"source={geometry.source_width}x{geometry.source_height} "
f"(scale={geometry.source_scale:g}), output={geometry.out_width}x{geometry.out_height}, "
f"map_cache={geometry.map_cache}"
)
mesh_viewer, _uv_mesh = open_mesh_viewer(runtime, args)
runtime.warmup(color=False)
tactile = runtime.tactile
print("Press 'q' to quit, 'r' to reset tracker. Close the 3D window to stop.")
try:
while True:
ok, unrolled = runtime.grab_and_process(color=False)
if not ok:
break
tactile.compute_deformation(decompose=True, check_motion=True, threshold=0)
tactile.compute_contact()
fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = tactile.read_info(
tactile.info.FPS,
tactile.info.VRAW,
tactile.info.VNORMAL,
tactile.info.IMG,
tactile.info.CONTOUR,
tactile.info.CENTROID,
tactile.info.DEPTH,
tactile.info.FNORMAL,
tactile.info.FSHEARX,
tactile.info.FSHEARY,
)
flow_mag = float((flow[:, :, 0] ** 2 + flow[:, :, 1] ** 2).mean() ** 0.5) if flow.size else 0.0
print(
f"FPS={fps:.2f} flow_rms={flow_mag:.2f} "
f"mask={tactile.mask_stats['coverage']:.2f} normal={fnormal:.2f} shear=({fshearx:.2f}, {fsheary:.2f})",
end="\r",
)
if mesh_viewer is not None:
if not mesh_viewer.update(flow=flow, scalar_map=depth_map):
print("\n3D viewer closed.")
break
if args.show:
show_finger_uv_panels(
unrolled,
vnormal,
depth_map,
flow_threshold=args.flow_threshold,
)
key = cv2.waitKey(1) & 0xFF
if key == ord("q"):
print("\nExiting.")
break
if key == ord("r"):
tactile.reset()
print("\nTracker reset.")
finally:
runtime.close()
if mesh_viewer is not None:
mesh_viewer.close()
cv2.destroyAllWindows()
def main():
args = _parse_args()
config = PipelineConfig.from_args(args)
sys.exit(run_dashboard(config))
if __name__ == "__main__":
examples/finger_plot_data.py
-101
View File
@@ -1,101 +0,0 @@
"""
Finger UV pipeline with real-time force charts and OpenCV panels.
Demonstrates:
1. UV-unrolled deformation arrows
2. Contact region overlay
3. PyQtGraph normal/shear force plots
4. Optional raw-frame recording (s/e keys)
"""
import argparse
import os
import sys
import time
from pathlib import Path
os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0"
import cv2
import orisys
from orisys.finger import FingerRuntime
_EXAMPLES_DIR = Path(__file__).resolve().parent
if str(_EXAMPLES_DIR) not in sys.path:
sys.path.insert(0, str(_EXAMPLES_DIR))
from viz import ForceChartWindow, FrameRecorder, lowpass_filter
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--video", "-v", type=str, default="0", help="视频源：摄像头编号或视频文件路径")
parser.add_argument("--config", "-c", type=str, default="./config/dd02-ov.json", help="配置文件名称或路径")
parser.add_argument("--verbose", "-verbose", type=bool, default=True, help="是否输出详细日志")
args = parser.parse_args()
charts = ForceChartWindow()
recorder = FrameRecorder()
runtime = FingerRuntime.open(
config=args.config,
video=args.video,
verbose=args.verbose,
)
runtime.warmup(color=True)
tactile = runtime.tactile
start_time = time.time()
print("\n按键说明：'q' 退出程序，'s' 开始录像，'e' 结束录像。")
try:
while True:
ok, unrolled = runtime.grab_and_process(color=True)
if not ok:
break
recorder.write_if_recording(tactile.frame)
tactile.compute_deformation(check_motion=True, threshold=0)
tactile.compute_contact()
fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = tactile.read_info(
tactile.info.FPS,
tactile.info.VRAW,
tactile.info.VNORMAL,
tactile.info.IMG,
tactile.info.CONTOUR,
tactile.info.CENTROID,
tactile.info.DEPTH,
tactile.info.FNORMAL,
tactile.info.FSHEARX,
tactile.info.FSHEARY,
)
print(f"FPS={fps:.2f}, 法向力={fnormal:.4f}, 切向力X={fshearx:.4f}, 切向力Y={fsheary:.4f}")
arrows = orisys.util.draw_arrows(unrolled, flow, threshold=3, grid_spacing=10, arrow_scale=1.0)
cv2.imshow("arrows", arrows)
cv2.imshow("contact", orisys.util.draw_contact(img, contour, centroid))
charts.push(time.time() - start_time, fnormal, fshearx, fsheary)
display = charts.data.copy()
for channel in range(3):
display[:, 1, channel] = lowpass_filter(
charts.data[:, 1, channel],
cutoff_hz=5,
sample_rate_hz=30,
)
charts.refresh(display)
key = cv2.waitKey(1)
if key & 0xFF == ord("q"):
print("\n正在退出示例程序...")
break
if key == ord("s"):
recorder.start(tactile.frame)
if key == ord("e"):
recorder.stop()
finally:
runtime.close()
if __name__ == "__main__":
main()
examples/gui/__init__.py
+30
View File
@@ -0,0 +1,30 @@
"""Public GUI entry points for finger examples."""
from .dashboard import FingerDashboardWindow, run_dashboard
from .layout import (
DashboardLayout,
LAYOUT,
LOG_PANEL,
PANEL_SETUP,
TOP_ROW,
PanelRow,
PanelSetup,
PanelSlot,
top_row_splitter_sizes,
top_row_slot_index,
)
__all__ = [
"DashboardLayout",
"FingerDashboardWindow",
"LAYOUT",
"LOG_PANEL",
"PANEL_SETUP",
"TOP_ROW",
"PanelRow",
"PanelSetup",
"PanelSlot",
"run_dashboard",
"top_row_splitter_sizes",
"top_row_slot_index",
]
examples/gui/app.py
+34
View File
@@ -0,0 +1,34 @@
"""Qt application bootstrap for Orisys example dashboards."""
from __future__ import annotations
import os
import sys
def fix_qt_plugin_path() -> None:
"""OpenCV redirects Qt to cv2/qt/plugins; use PyQt5's plugins instead."""
try:
from PyQt5.QtCore import QLibraryInfo
os.environ["QT_QPA_PLATFORM_PLUGIN_PATH"] = QLibraryInfo.location(
QLibraryInfo.PluginsPath
)
except Exception:
path = os.environ.get("QT_QPA_PLATFORM_PLUGIN_PATH", "")
if "cv2" in path.replace("\\", "/"):
os.environ.pop("QT_QPA_PLATFORM_PLUGIN_PATH", None)
def get_or_create_qapp(app_name: str = "Orisys Finger Dashboard"):
fix_qt_plugin_path()
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QApplication
QApplication.setAttribute(Qt.AA_EnableHighDpiScaling, True)
QApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True)
app = QApplication.instance()
if app is None:
app = QApplication(sys.argv)
app.setApplicationName(app_name)
return app
examples/gui/dashboard.py
+333
View File
@@ -0,0 +1,333 @@
"""Unified Qt dashboard for the finger tactile example."""
from __future__ import annotations
import sys
from pathlib import Path
from PyQt5.QtCore import Qt, QTimer
from PyQt5.QtGui import QIcon
from PyQt5.QtWidgets import (
QApplication,
QFileDialog,
QHBoxLayout,
QLineEdit,
QMainWindow,
QPushButton,
QSplitter,
QVBoxLayout,
QWidget,
)
from pipeline.config import PipelineConfig
from pipeline.service import FingerPipeline
from .layout import (
DashboardLayout,
LAYOUT,
PANEL_SETUP,
PanelSetup,
top_row_splitter_sizes,
)
from .styles import STYLE
from .widgets import (
ArrowPanelWidget,
ForceChartWidget,
MeshViewportWidget,
TeeStream,
WidgetLogHandler,
make_log_widget,
)
class FingerDashboardWindow(QMainWindow):
"""Single-window finger demo: 3D mesh | UV arrows | force charts."""
def __init__(
self,
config: PipelineConfig,
*,
layout: DashboardLayout = LAYOUT,
log_file=None,
):
super().__init__()
self.config = config
self.layout = layout
self.pipeline = FingerPipeline(config)
self._running = False
self.setWindowTitle(layout.window_title)
self.setStyleSheet(STYLE)
logo_path = Path(__file__).resolve().parents[2] / "logo.png"
if logo_path.is_file():
self.setWindowIcon(QIcon(str(logo_path)))
self._setup_ui()
self._wire_logging(log_file)
self.timer = QTimer()
self.timer.timeout.connect(self._on_timer)
self.camera_edit.setText(str(config.video))
self.config_edit.setText(config.config)
self.cal_edit.setText(config.calib or "")
def _setup_ui(self) -> None:
layout = self.layout
central = QWidget()
self.setCentralWidget(central)
root_layout = QVBoxLayout(central)
root_layout.setContentsMargins(layout.root_margin_px, layout.root_margin_px, layout.root_margin_px, layout.root_margin_px)
root_layout.setSpacing(layout.root_spacing_px)
bar = QHBoxLayout()
bar.setSpacing(layout.toolbar_spacing_px)
self.btn_camera = QPushButton("摄像机")
self.btn_camera.clicked.connect(self._select_camera)
bar.addWidget(self.btn_camera)
self.camera_edit = QLineEdit("0")
self.camera_edit.setPlaceholderText("摄像头编号 或 视频路径")
bar.addWidget(self.camera_edit, 1)
self.btn_config = QPushButton("配置")
self.btn_config.clicked.connect(self._select_config)
bar.addWidget(self.btn_config)
self.config_edit = QLineEdit("./config/dd02-ov.json")
bar.addWidget(self.config_edit, 1)
self.cal_edit = QLineEdit("")
bar.addWidget(self.cal_edit, 1)
self.btn_reset = QPushButton("重置")
self.btn_reset.setEnabled(False)
self.btn_reset.clicked.connect(self._on_reset)
bar.addWidget(self.btn_reset)
self.btn_start = QPushButton("启动")
self.btn_start.setObjectName("btnStart")
self.btn_start.clicked.connect(self._on_start)
bar.addWidget(self.btn_start)
self.btn_stop = QPushButton("暂停")
self.btn_stop.setObjectName("btnStop")
self.btn_stop.setEnabled(False)
self.btn_stop.clicked.connect(self._on_stop)
bar.addWidget(self.btn_stop)
root_layout.addLayout(bar)
setup = layout.panel_setup
self._panel_widgets: dict[str, QWidget] = {}
self.mesh_viewport = MeshViewportWidget(layout=layout)
self.arrow_panel = ArrowPanelWidget(flow_threshold=layout.arrow_flow_threshold, layout=layout)
self.force_charts = ForceChartWidget(layout=layout, lowpass_cutoff_hz=self.config.lowpass_cutoff_hz)
self.log_widget = make_log_widget()
self._panel_widgets["mesh"] = self.mesh_viewport
self._panel_widgets["arrows"] = self.arrow_panel
self._panel_widgets["charts"] = self.force_charts.widget
self._panel_widgets["log"] = self.log_widget
# Top row: mesh | arrows | charts (order from layout.PANEL_SETUP.top_row)
h_splitter = QSplitter(Qt.Horizontal)
for slot in setup.top_row.slots:
if not slot.enabled:
continue
h_splitter.addWidget(self._panel_widgets[slot.id])
h_splitter.setStretchFactor(h_splitter.count() - 1, slot.stretch)
h_splitter.setChildrenCollapsible(setup.top_row.collapsible)
self.h_splitter = h_splitter
# Vertical: top panel | log panel
v_splitter = QSplitter(Qt.Vertical)
v_splitter.addWidget(h_splitter)
if setup.log_panel.enabled:
v_splitter.addWidget(self.log_widget)
v_splitter.setStretchFactor(0, setup.top_stretch)
v_splitter.setStretchFactor(1, setup.log_stretch)
else:
v_splitter.setStretchFactor(0, 1)
self.v_splitter = v_splitter
root_layout.addWidget(v_splitter)
self.resize(layout.window_width, layout.window_height)
QTimer.singleShot(0, self._apply_panel_layout)
def _apply_panel_layout(self) -> None:
setup = self.layout.panel_setup
for slot in setup.top_row.slots:
if not slot.enabled:
continue
if slot.id in ("mesh", "arrows"):
self._panel_widgets[slot.id].set_fixed_size(slot.width_px, slot.height_px)
elif slot.id == "charts":
self.force_charts.set_panel_size(slot.width_px, slot.height_px)
if setup.log_panel.enabled and setup.log_panel.height_px > 0:
self.log_widget.setFixedHeight(setup.log_panel.height_px)
self.h_splitter.setSizes(list(top_row_splitter_sizes(self.layout)))
def resizeEvent(self, event):
super().resizeEvent(event)
QTimer.singleShot(0, self._apply_panel_layout)
def showEvent(self, event):
super().showEvent(event)
QTimer.singleShot(0, self._apply_panel_layout)
def _wire_logging(self, log_file) -> None:
streams = []
if log_file is not None:
streams.append(log_file)
self.log_handler = WidgetLogHandler(self.log_widget)
streams.append(self.log_handler)
sys.stdout = TeeStream(*streams)
sys.stderr = sys.stdout
def _on_start(self) -> None:
if self._running:
return
self.btn_start.setText("启动中...")
self.btn_start.setEnabled(False)
QApplication.processEvents()
self.config.video = self.camera_edit.text().strip()
self.config.config = self.config_edit.text().strip()
cal = self.cal_edit.text().strip()
self.config.calib = cal or None
try:
self.pipeline = FingerPipeline(self.config)
self.pipeline.open()
self.mesh_viewport.configure(self.pipeline)
self.force_charts.reset()
QTimer.singleShot(0, self._apply_panel_layout)
except Exception as exc:
self.mesh_viewport.close_renderer()
self.btn_start.setText("启动")
self.btn_start.setEnabled(True)
print(f"启动失败: {exc}")
return
self._running = True
self.timer.start(self.layout.timer_interval_ms)
self.btn_start.setText("启动")
self.btn_stop.setEnabled(True)
self.btn_reset.setEnabled(True)
print(
f"▶ 启动 — 摄像机={self.config.video}, 配置={self.config.config}, "
f"标定={self.config.calib or '(none)'}"
)
def _on_stop(self) -> None:
if not self._running:
return
self._running = False
self.timer.stop()
self.btn_stop.setText("停止中...")
self.btn_stop.setEnabled(False)
QApplication.processEvents()
self.mesh_viewport.close_renderer()
self.pipeline.close()
self.btn_start.setEnabled(True)
self.btn_stop.setText("暂停")
self.btn_reset.setEnabled(False)
print("⏸ 已暂停")
def _on_reset(self) -> None:
if not self._running:
print("请先启动传感器后再重置追踪器。")
return
self.pipeline.reset()
print("追踪器已重置。")
def _on_timer(self) -> None:
if not self._running:
return
frame = self.pipeline.tick()
if frame is None:
print("采集结束或读取失败。")
self._on_stop()
return
flow_mag = 0.0
if frame.flow.size:
flow_mag = float((frame.flow[:, :, 0] ** 2 + frame.flow[:, :, 1] ** 2).mean() ** 0.5)
print(
f"FPS={frame.fps:.2f} flow_rms={flow_mag:.2f} "
f"mask={frame.mask_coverage:.2f} normal={frame.fnormal:.2f} "
f"shear=({frame.fshearx:.2f}, {frame.fsheary:.2f})",
end="\r",
)
self.mesh_viewport.update_frame(frame)
self.arrow_panel.update_frame(frame)
self.force_charts.update_frame(frame)
def _select_camera(self) -> None:
path, _ = QFileDialog.getOpenFileName(
self,
"选择视频文件",
"data",
"视频 (*.mp4 *.avi *.mov);;所有文件 (*)",
)
if path:
self.camera_edit.setText(path)
def _select_config(self) -> None:
path, _ = QFileDialog.getOpenFileName(
self,
"选择配置文件",
"config",
"JSON (*.json);;所有文件 (*)",
)
if path:
self.config_edit.setText(path)
def keyPressEvent(self, event):
key = event.key()
if key == Qt.Key_Escape:
if self.isFullScreen():
self.showNormal()
else:
self.showFullScreen()
elif key == Qt.Key_Q:
print("\n正在退出...")
self.close()
elif key == Qt.Key_S:
if self.pipeline.start_recording():
print("开始录像。")
elif key == Qt.Key_E:
self.pipeline.stop_recording()
print("录像结束。")
elif key == Qt.Key_R:
self._on_reset()
else:
super().keyPressEvent(event)
def closeEvent(self, event):
self.timer.stop()
self.mesh_viewport.close_renderer()
self.pipeline.close()
event.accept()
def run_dashboard(config: PipelineConfig, *, layout: DashboardLayout = LAYOUT) -> int:
"""Launch the unified finger dashboard and block until the window closes."""
from .app import get_or_create_qapp
print("\n按键说明：Q 退出 | R 重置追踪器 | S 开始录像 | E 结束录像 | ESC 全屏")
app = get_or_create_qapp()
window = FingerDashboardWindow(config, layout=layout)
window.show()
QTimer.singleShot(0, window._apply_panel_layout)
return app.exec_()
examples/gui/layout.py
+141
View File
@@ -0,0 +1,141 @@
"""
Finger dashboard layout — edit this file to define the panel hierarchy and sizes.
Structure:
┌─ toolbar ─────────────────────────────────────────────┐
├─ TOP (stretch: top_stretch) ──────────────────────────┤
│ ┌ mesh ─┬─ arrows ─┬─ charts ─┐ (left → right) │
│ └───────┴───────────┴─────────┘ │
├─ LOG (stretch: log_stretch) ──────────────────────────┤
└───────────────────────────────────────────────────────┘
Each panel uses explicit width_px and height_px.
top_stretch / log_stretch control how extra vertical space is shared on resize.
Edit TOP_ROW.slots to reorder or resize panels.
Edit PANEL_SETUP to change top vs log proportions.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Literal
PanelId = Literal["mesh", "arrows", "charts", "log"]
@dataclass(frozen=True)
class PanelSlot:
"""One panel in the dashboard."""
id: PanelId
title: str
width_px: int
height_px: int
stretch: int = 0
enabled: bool = True
@dataclass(frozen=True)
class PanelRow:
"""Horizontal row of panels (the top visual strip)."""
slots: tuple[PanelSlot, ...]
collapsible: bool = False
@dataclass(frozen=True)
class PanelSetup:
"""Full panel hierarchy: top row + bottom log."""
top_row: PanelRow
log_panel: PanelSlot
top_stretch: int = 5
log_stretch: int = 1
# --- Top strip: mesh | arrows | charts (edit order and sizes here) ---
TOP_ROW = PanelRow(
slots=(
PanelSlot("mesh", "3D mesh", width_px=500, height_px=500),
PanelSlot("arrows", "UV Unrolled", width_px=500, height_px=500),
PanelSlot("charts", "Force charts", width_px=500, height_px=500),
),
)
LOG_PANEL = PanelSlot("log", "Log", width_px=0, height_px=100)
PANEL_SETUP = PanelSetup(
top_row=TOP_ROW,
log_panel=LOG_PANEL,
top_stretch=5,
log_stretch=1,
)
@dataclass(frozen=True)
class DashboardLayout:
"""Window chrome, chart styling, and panel hierarchy."""
panel_setup: PanelSetup = PANEL_SETUP
window_title: str = "Orisys Finger Dashboard"
window_width: int = 1500
window_height: int = 600
root_margin_px: int = 6
root_spacing_px: int = 6
toolbar_spacing_px: int = 8
plot_buffer_points: int = 120
plot_sample_rate_hz: float = 30.0
chart_title_font_pt: int = 11
chart_axis_label_font_pt: int = 9
chart_tick_font_pt: int = 9
y_axis_min_span: float = 20000.0
y_range_smooth_alpha: float = 0.5
curve_color: str = "#00E5FF"
# UV arrow panel: min flow magnitude to draw an arrow (display only; not dd02-ov.json contact.flow_threshold)
arrow_flow_threshold: float = 1.0
timer_interval_ms: int = 10
def slot(self, panel_id: PanelId) -> PanelSlot | None:
if self.panel_setup.log_panel.id == panel_id:
return self.panel_setup.log_panel
for slot in self.panel_setup.top_row.slots:
if slot.id == panel_id:
return slot
return None
def top_slot(self, panel_id: PanelId) -> PanelSlot:
for slot in self.panel_setup.top_row.slots:
if slot.id == panel_id:
return slot
raise KeyError(f"panel {panel_id!r} not in top row")
@property
def mesh_panel_title(self) -> str:
return self.top_slot("mesh").title
@property
def arrows_panel_title(self) -> str:
return self.top_slot("arrows").title
LAYOUT = DashboardLayout()
def top_row_splitter_sizes(layout: DashboardLayout = LAYOUT) -> tuple[int, ...]:
"""Horizontal splitter widths from panel setup."""
return tuple(slot.width_px for slot in layout.panel_setup.top_row.slots if slot.enabled)
def top_row_slot_index(panel_id: PanelId, layout: DashboardLayout = LAYOUT) -> int:
"""Index of a panel in the horizontal top splitter."""
for i, slot in enumerate(layout.panel_setup.top_row.slots):
if slot.id == panel_id:
return i
raise KeyError(f"panel {panel_id!r} not in top row")
examples/gui/styles.py
+114
View File
@@ -0,0 +1,114 @@
"""Qt stylesheet for the finger dashboard."""
STYLE = """
QMainWindow {
background-color: #000000;
}
QLabel#panelLabel, QLabel#meshViewport {
border: 1px solid #262626;
border-radius: 4px;
background-color: #0b0b0b;
}
QPushButton {
background-color: #1a1a1a;
color: #999999;
border: 1px solid #333333;
border-radius: 4px;
padding: 6px 16px;
font-family: "SimHei";
font-size: 13px;
}
QPushButton:hover {
background-color: #262626;
color: #cccccc;
border: 1px solid #555555;
}
QPushButton#btnStart {
background-color: #0d2818;
color: #4caf84;
border: 1px solid #1a4d30;
}
QPushButton#btnStart:hover {
background-color: #143d24;
color: #6fd4a0;
}
QPushButton#btnStart:disabled {
background-color: #0f0f0f;
color: #333333;
border: 1px solid #222222;
}
QPushButton#btnStop {
background-color: #2b1010;
color: #e05555;
border: 1px solid #4d1a1a;
}
QPushButton#btnStop:hover {
background-color: #3d1818;
color: #f07070;
}
QPushButton#btnStop:disabled {
background-color: #0f0f0f;
color: #333333;
border: 1px solid #222222;
}
QLineEdit {
background-color: #0d0d0d;
color: #999999;
border: 1px solid #333333;
border-radius: 4px;
padding: 5px 10px;
font-size: 13px;
}
GraphicsLayoutWidget {
border: 1px solid #262626;
border-radius: 4px;
}
QPlainTextEdit {
background-color: #0b0b0b;
color: #777777;
border: 1px solid #262626;
border-radius: 4px;
font-family: "Cascadia Code", "Consolas", "Courier New", monospace;
font-size: 13px;
padding: 8px;
selection-background-color: #1a3a4a;
selection-color: #cccccc;
}
QPlainTextEdit QScrollBar:vertical {
background-color: #0b0b0b;
width: 6px;
margin: 1px;
border-radius: 3px;
}
QPlainTextEdit QScrollBar::handle:vertical {
background-color: #262626;
border-radius: 3px;
min-height: 30px;
}
QPlainTextEdit QScrollBar::handle:vertical:hover {
background-color: #3a3a3a;
}
QPlainTextEdit QScrollBar::add-line:vertical,
QPlainTextEdit QScrollBar::sub-line:vertical {
height: 0px;
}
QPlainTextEdit QScrollBar::add-page:vertical,
QPlainTextEdit QScrollBar::sub-page:vertical {
background: none;
}
QSplitter::handle {
background-color: #1a1a1a;
}
QSplitter::handle:vertical {
height: 1px;
}
QSplitter::handle:horizontal {
width: 1px;
}
"""
examples/gui/widgets/__init__.py
+20
View File
@@ -0,0 +1,20 @@
"""Qt widgets for the finger dashboard."""
from .force_charts import ForceChartWidget
from .log_panel import TeeStream, WidgetLogHandler, make_log_widget
from .mesh_viewport import MeshViewportWidget
from .arrow_image import ArrowPanelWidget
from .qt_pixmap import ndarray_to_pixmap, pixmap_keep_aspect
from .image_panel import ImagePanel
__all__ = [
"ArrowPanelWidget",
"ForceChartWidget",
"MeshViewportWidget",
"ImagePanel",
"TeeStream",
"WidgetLogHandler",
"make_log_widget",
"ndarray_to_pixmap",
"pixmap_keep_aspect",
]
examples/gui/widgets/arrow_image.py
+44
View File
@@ -0,0 +1,44 @@
"""UV flow-arrow panel for the Qt dashboard."""
from __future__ import annotations
import cv2
from pipeline.frame import FingerFrameView
from viz.render_2d import render_uv_arrows
from ..layout import DashboardLayout, LAYOUT
from .image_panel import ImagePanel
from .qt_pixmap import ndarray_to_pixmap
class ArrowPanelWidget(ImagePanel):
"""UV unrolled flow-arrow panel."""
def __init__(
self,
*,
flow_threshold: float = 0.5,
layout: DashboardLayout = LAYOUT,
parent=None,
):
slot = layout.top_slot("arrows")
super().__init__(
title=layout.arrows_panel_title,
object_name="panelLabel",
default_side=min(slot.width_px, slot.height_px),
parent=parent,
)
self.flow_threshold = flow_threshold
def set_flow_threshold(self, threshold: float) -> None:
self.flow_threshold = threshold
def update_frame(self, frame: FingerFrameView) -> None:
arrows = render_uv_arrows(
frame.unrolled,
frame.vnormal,
flow_threshold=self.flow_threshold,
)
arrows = cv2.rotate(arrows, cv2.ROTATE_180)
self.set_source_pixmap(ndarray_to_pixmap(arrows, bgr=True))
examples/gui/widgets/force_charts.py
+202
View File
@@ -0,0 +1,202 @@
"""Embedded PyQtGraph force charts for the Qt dashboard."""
from __future__ import annotations
import numpy as np
from PyQt5.QtGui import QFont
from PyQt5.QtWidgets import QSizePolicy
from gui.layout import DashboardLayout, LAYOUT
from pipeline.frame import FingerFrameView
from viz.signal_filters import lowpass_filter
def _require_pyqtgraph():
try:
import pyqtgraph as pg
except ImportError as exc:
raise ImportError(
"PyQtGraph is not installed. Install with: pip install pyqt5 pyqtgraph"
) from exc
return pg
class ForceChartWidget:
"""Rolling normal/shear force charts as an embeddable QWidget."""
def __init__(
self,
*,
layout: DashboardLayout = LAYOUT,
lowpass_cutoff_hz: float = 5.0,
parent=None,
):
pg = _require_pyqtgraph()
pg.setConfigOption("background", "#000000")
pg.setConfigOption("foreground", "#777777")
pg.setConfigOptions(antialias=True)
self._layout = layout
self._pg = pg
self.npoints = layout.plot_buffer_points
self.sample_rate_hz = layout.plot_sample_rate_hz
self.lowpass_cutoff_hz = lowpass_cutoff_hz
self._plot_window_sec = self.npoints / self.sample_rate_hz
self._plot_sample_count = 0
self._y_range_alpha = layout.y_range_smooth_alpha
self._y_range_smooth: list[tuple[float, float] | None] = [None, None, None]
self.widget = pg.GraphicsLayoutWidget(parent=parent)
charts = layout.top_slot("charts")
self.set_panel_size(charts.width_px, charts.height_px)
self._init_plot_buffer()
self._create_plots()
def set_panel_size(self, width: int, height: int) -> None:
"""Apply layout width/height; height 0 means fill top-row height."""
width = max(int(width), 64)
self.widget.setFixedWidth(width)
if height > 0:
self.widget.setFixedHeight(height)
self.widget.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
else:
self.widget.setMaximumHeight(16777215)
self.widget.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Expanding)
def _init_plot_buffer(self) -> None:
self.data = np.full((self.npoints, 2, 3), np.nan, dtype=np.float64)
self._plot_sample_count = 0
self._y_range_smooth = [None, None, None]
def _create_plots(self) -> None:
pg = self._pg
layout = self._layout
titles = ["法向力", "切向力 X", "切向力 Y"]
self.plots = []
self.curves = []
self.dot_glows = []
self.dot_cores = []
axis_pen = pg.mkPen(color=(38, 38, 38), width=1)
label_font = QFont("SimHei", layout.chart_axis_label_font_pt)
title_pt = layout.chart_title_font_pt
for i, title in enumerate(titles):
plot = self.widget.addPlot(title=None)
plot.setTitle(title, color="#999999", size=f"{title_pt}pt")
plot.titleLabel.setFont(QFont("SimHei", title_pt))
plot.setLabel("bottom", "时间", units="s")
plot.getAxis("bottom").label.setFont(label_font)
plot.showAxis("top", False)
plot.showAxis("right", False)
plot.getAxis("left").setPen(axis_pen)
plot.getAxis("bottom").setPen(axis_pen)
plot.showGrid(x=False, y=True, alpha=0.12)
tick_font = QFont("Consolas", layout.chart_tick_font_pt)
tick_font.setStyleHint(QFont.Monospace)
plot.getAxis("left").setTickFont(tick_font)
plot.getAxis("bottom").setTickFont(tick_font)
line_pen = pg.mkPen(color=layout.curve_color, width=2)
curve = plot.plot(pen=line_pen, fillLevel=0)
dot_glow = plot.plot(
[],
[],
pen=None,
symbol="o",
symbolSize=8,
symbolPen=None,
symbolBrush=(0, 229, 255, 76),
)
dot_core = plot.plot(
[],
[],
pen=None,
symbol="o",
symbolSize=4,
symbolPen=None,
symbolBrush=(255, 255, 255, 255),
)
plot.setXRange(0, self._plot_window_sec, padding=0)
plot.enableAutoRange(x=False, y=False)
self.plots.append(plot)
self.curves.append(curve)
self.dot_glows.append(dot_glow)
self.dot_cores.append(dot_core)
if i < 2:
self.widget.nextRow()
def reset(self) -> None:
self._init_plot_buffer()
for plot in self.plots:
plot.setXRange(0, self._plot_window_sec, padding=0)
def update_frame(self, frame: FingerFrameView) -> None:
if self._plot_sample_count < self.npoints:
idx = self._plot_sample_count
self.data[idx, 0, :] = frame.timestamp
self.data[idx, 1, 0] = frame.fnormal
self.data[idx, 1, 1] = frame.fshearx
self.data[idx, 1, 2] = frame.fsheary
self._plot_sample_count += 1
else:
for channel in range(3):
self.data[:-1, 0, channel] = self.data[1:, 0, channel]
self.data[:-1, 1, channel] = self.data[1:, 1, channel]
self.data[-1, 0, :] = frame.timestamp
self.data[-1, 1, 0] = frame.fnormal
self.data[-1, 1, 1] = frame.fshearx
self.data[-1, 1, 2] = frame.fsheary
n = self._plot_sample_count
if n == 0:
return
layout = self._layout
data_plot = self.data[:n].copy()
times = data_plot[:, 0, 0] - float(data_plot[0, 0, 0])
for channel in range(3):
data_plot[:, 1, channel] = lowpass_filter(
data_plot[:, 1, channel],
self.lowpass_cutoff_hz,
sample_rate_hz=self.sample_rate_hz,
)
x_end = float(times[-1])
x_start = max(0.0, x_end - self._plot_window_sec)
if x_end - x_start < self._plot_window_sec:
x_start = 0.0
x_end = max(self._plot_window_sec, x_end)
for i, curve in enumerate(self.curves):
y = data_plot[:, 1, i]
curve.setData(times, y)
finite = np.isfinite(y)
if not np.any(finite):
continue
ymin, ymax = float(np.min(y[finite])), float(np.max(y[finite]))
if self._y_range_smooth[i] is None:
self._y_range_smooth[i] = (ymin, ymax)
else:
prev_min, prev_max = self._y_range_smooth[i]
alpha = self._y_range_alpha
ymin = alpha * ymin + (1 - alpha) * prev_min
ymax = alpha * ymax + (1 - alpha) * prev_max
span = ymax - ymin
if span < layout.y_axis_min_span:
mid = (ymin + ymax) / 2
half = layout.y_axis_min_span / 2
ymin = mid - half
ymax = mid + half
self._y_range_smooth[i] = (ymin, ymax)
pad = max((ymax - ymin) * 0.12, 0.01)
self.plots[i].setYRange(ymin - pad, ymax + pad)
last_x = times[-1:]
last_y = y[-1:]
self.dot_glows[i].setData(last_x, last_y)
self.dot_cores[i].setData(last_x, last_y)
for plot in self.plots:
plot.setXRange(x_start, x_end, padding=0)
examples/gui/widgets/image_panel.py
+71
View File
@@ -0,0 +1,71 @@
"""Square 1:1 image panel host for dashboard viewports."""
from __future__ import annotations
from PyQt5.QtCore import Qt
from PyQt5.QtGui import QPixmap
from PyQt5.QtWidgets import QLabel, QSizePolicy, QVBoxLayout, QWidget
from .qt_pixmap import pixmap_keep_aspect
class ImagePanel(QWidget):
"""Fixed square widget that displays an image at 1:1 aspect ratio."""
def __init__(
self,
*,
title: str = "",
object_name: str = "panelLabel",
default_side: int = 256,
min_side: int = 64,
parent=None,
):
super().__init__(parent)
self._side = default_side
self._min_side = min_side
self._source_pixmap: QPixmap | None = None
self.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
layout = QVBoxLayout(self)
layout.setContentsMargins(0, 0, 0, 0)
self.label = QLabel(title)
self.label.setObjectName(object_name)
self.label.setAlignment(Qt.AlignCenter)
layout.addWidget(self.label)
self.set_side_length(self._side)
def set_side_length(self, side: int) -> None:
self.set_fixed_size(side, side)
def set_fixed_size(self, width: int, height: int) -> None:
width = max(int(width), self._min_side)
height = max(int(height), self._min_side)
if width == self.width() and height == self.height() and width == self._side:
return
self._side = width
self.setFixedSize(width, height)
self.label.setFixedSize(width, height)
self.on_side_length_changed(min(width, height))
self._refresh_pixmap()
@property
def side_length(self) -> int:
return self._side
def on_side_length_changed(self, side: int) -> None:
"""Hook for subclasses (e.g. resize an offscreen renderer)."""
def set_source_pixmap(self, pixmap: QPixmap | None) -> None:
self._source_pixmap = pixmap
self._refresh_pixmap()
def _refresh_pixmap(self) -> None:
if self._source_pixmap is None or self._source_pixmap.isNull():
return
self.label.setPixmap(pixmap_keep_aspect(self._source_pixmap, self.label.size()))
def resizeEvent(self, event):
super().resizeEvent(event)
self._refresh_pixmap()
examples/gui/widgets/log_panel.py
+51
View File
@@ -0,0 +1,51 @@
"""Logging helpers for the finger dashboard."""
from __future__ import annotations
from PyQt5.QtGui import QFont
from PyQt5.QtWidgets import QPlainTextEdit
class TeeStream:
"""Write to multiple text streams."""
def __init__(self, *streams):
self.streams = streams
def write(self, text):
for stream in self.streams:
stream.write(text)
def flush(self):
for stream in self.streams:
stream.flush()
class WidgetLogHandler:
"""Mirror print output into a QPlainTextEdit."""
def __init__(self, widget: QPlainTextEdit):
self.widget = widget
self.buf = ""
def write(self, text: str):
self.buf += text
while "\n" in self.buf:
line, self.buf = self.buf.split("\n", 1)
if line:
self.widget.appendPlainText(line)
def flush(self):
if self.buf:
self.widget.appendPlainText(self.buf)
self.buf = ""
def make_log_widget() -> QPlainTextEdit:
widget = QPlainTextEdit()
widget.setReadOnly(True)
widget.setMaximumBlockCount(2000)
font = QFont("Consolas", 12)
font.setStyleHint(QFont.Monospace)
widget.setFont(font)
return widget
examples/gui/widgets/mesh_viewport.py
+79
View File
@@ -0,0 +1,79 @@
"""Embedded offscreen 3D mesh viewport for the Qt dashboard."""
from __future__ import annotations
from pipeline.frame import FingerFrameView
from viz.offscreen_mesh import OffscreenFingerMeshRenderer
from ..layout import DashboardLayout, LAYOUT
from .qt_pixmap import ndarray_to_pixmap
from .image_panel import ImagePanel
class MeshViewportWidget(ImagePanel):
"""Render the finger mesh offscreen in a square 1:1 viewport."""
def __init__(self, parent=None, *, layout: DashboardLayout = LAYOUT):
self._layout = layout
self._renderer: OffscreenFingerMeshRenderer | None = None
self._mesh_map = "depth"
self._threshold = 0.0
self._enable_3d = True
slot = layout.top_slot("mesh")
super().__init__(
title=layout.mesh_panel_title,
object_name="meshViewport",
default_side=min(slot.width_px, slot.height_px),
parent=parent,
)
def configure(self, pipeline) -> None:
self._mesh_map = pipeline.config.mesh_map
self._threshold = pipeline.config.flow_threshold
self._enable_3d = pipeline.config.enable_3d and pipeline.uv_mesh is not None
if self._renderer is not None:
self._renderer.close()
self._renderer = None
if not self._enable_3d:
self.label.setText(
"3D mesh disabled" if not pipeline.config.enable_3d else "3D mesh unavailable"
)
self.set_source_pixmap(None)
return
self.label.setText("")
self._init_renderer(pipeline)
def _init_renderer(self, pipeline) -> None:
if not self._enable_3d or pipeline.uv_mesh is None:
return
side = self.side_length
self._renderer = OffscreenFingerMeshRenderer(
pipeline.uv_mesh,
width=side,
height=side,
threshold=self._threshold,
color_mode=self._mesh_map,
world_to_cam=pipeline.world_to_cam,
lookat=pipeline.mesh_lookat,
)
def on_side_length_changed(self, side: int) -> None:
if self._renderer is not None:
self._renderer.resize(side, side)
def update_frame(self, frame: FingerFrameView) -> None:
if not self._enable_3d or self._renderer is None:
return
rgb = self._renderer.render(
flow=frame.flow,
scalar_map=frame.depth_map,
centroid=frame.centroid,
centroid_found=frame.centroid_found,
fnormal=frame.fnormal,
)
self.set_source_pixmap(ndarray_to_pixmap(rgb, bgr=False))
def close_renderer(self) -> None:
if self._renderer is not None:
self._renderer.close()
self._renderer = None
examples/gui/widgets/qt_pixmap.py
+32
View File
@@ -0,0 +1,32 @@
"""Qt pixmap conversion helpers (no widget dependencies)."""
from __future__ import annotations
import cv2
import numpy as np
from PyQt5.QtCore import Qt
from PyQt5.QtGui import QImage, QPixmap
def ndarray_to_pixmap(image: np.ndarray, *, bgr: bool = True) -> QPixmap:
"""Convert a numpy image to QPixmap for QLabel display."""
arr = np.ascontiguousarray(image)
if arr.ndim == 2:
arr = cv2.cvtColor(arr, cv2.COLOR_GRAY2BGR)
bgr = True
if arr.ndim == 3 and arr.shape[2] == 4:
arr = arr[:, :, :3]
if bgr:
rgb = cv2.cvtColor(arr, cv2.COLOR_BGR2RGB)
else:
rgb = arr
h, w, ch = rgb.shape
qimg = QImage(rgb.data, w, h, ch * w, QImage.Format_RGB888)
return QPixmap.fromImage(qimg.copy())
def pixmap_keep_aspect(pixmap: QPixmap, size) -> QPixmap:
"""Scale a pixmap to fit ``size`` without changing aspect ratio."""
if size.width() <= 0 or size.height() <= 0:
return pixmap
return pixmap.scaled(size, Qt.KeepAspectRatio, Qt.SmoothTransformation)
examples/pipeline/__init__.py
+12
View File
@@ -0,0 +1,12 @@
"""Finger example pipeline: runtime acquisition without GUI dependencies."""
from .config import PipelineConfig, mesh_threshold
from .frame import FingerFrameView
from .service import FingerPipeline
__all__ = [
"FingerFrameView",
"FingerPipeline",
"PipelineConfig",
"mesh_threshold",
]
examples/pipeline/config.py
+46
View File
@@ -0,0 +1,46 @@
"""Configuration for the finger tactile example pipeline."""
from __future__ import annotations
from dataclasses import dataclass
from viz.mesh_camera import DEFAULT_FINGER_VIZ_POSE
def mesh_threshold(mesh_map: str, threshold: float | None) -> float:
if threshold is not None:
return threshold
return 0.0 if mesh_map == "depth" else 0.4
@dataclass
class PipelineConfig:
video: str = "0"
config: str = "./config/dd02-ov.json"
calib: str | None = None
verbose: bool = False
mesh_map: str = "depth"
threshold: float | None = None
viz_pose: str = DEFAULT_FINGER_VIZ_POSE
enable_3d: bool = True
use_color: bool = True
lowpass_cutoff_hz: float = 5.0
force_sample_rate_hz: float = 30.0
@classmethod
def from_args(cls, args) -> "PipelineConfig":
return cls(
video=args.video,
config=args.config,
calib=getattr(args, "cal", None) or None,
verbose=args.verbose,
mesh_map=args.mesh_map,
threshold=args.threshold,
viz_pose=args.viz_pose,
enable_3d=not args.no_3d,
lowpass_cutoff_hz=getattr(args, "cutoff", 5.0),
)
@property
def flow_threshold(self) -> float:
return mesh_threshold(self.mesh_map, self.threshold)
examples/pipeline/frame.py
+26
View File
@@ -0,0 +1,26 @@
"""Display snapshot produced by one finger pipeline tick."""
from __future__ import annotations
from dataclasses import dataclass
import numpy as np
@dataclass
class FingerFrameView:
timestamp: float
fps: float
unrolled: np.ndarray
flow: np.ndarray
vnormal: np.ndarray
depth_map: np.ndarray
img: np.ndarray
contour: np.ndarray
centroid: np.ndarray
fnormal: float
fshearx: float
fsheary: float
centroid_found: bool
mask_coverage: float
raw_frame: np.ndarray | None = None
examples/pipeline/service.py
+123
View File
@@ -0,0 +1,123 @@
"""Finger tactile acquisition loop for example dashboards."""
from __future__ import annotations
import time
import numpy as np
from orisys.finger import FingerRuntime
from viz import FrameRecorder, build_uv_surface_mesh, resolve_viz_pose_path
from .config import PipelineConfig
from .frame import FingerFrameView
class FingerPipeline:
"""Owns FingerRuntime and produces FingerFrameView snapshots per tick."""
def __init__(self, config: PipelineConfig):
self.config = config
self.runtime: FingerRuntime | None = None
self.recorder = FrameRecorder()
self._start_time: float | None = None
self.uv_mesh = None
self.world_to_cam: np.ndarray | None = None
self.mesh_lookat: np.ndarray | None = None
@property
def is_open(self) -> bool:
return self.runtime is not None
@property
def tactile(self):
if self.runtime is None:
raise RuntimeError("FingerPipeline is not open")
return self.runtime.tactile
def open(self) -> None:
if self.runtime is not None:
return
self.runtime = FingerRuntime.open(
config=self.config.config,
calib=self.config.calib,
video=self.config.video,
verbose=self.config.verbose,
)
self._prepare_mesh_assets()
self.runtime.warmup(color=self.config.use_color)
self._start_time = time.time()
def _prepare_mesh_assets(self) -> None:
if self.runtime is None:
return
domain = self.runtime.domain
if domain.surface is None or getattr(domain.surface, "world_xyz", None) is None:
return
self.uv_mesh = build_uv_surface_mesh(domain.surface.world_xyz, domain.valid_mask)
if self.config.viz_pose.lower() != "none":
self.world_to_cam = np.load(resolve_viz_pose_path(self.config.viz_pose))
self.mesh_lookat = self.uv_mesh.rest_vertices.mean(axis=0)
def close(self) -> None:
if self.runtime is not None:
self.runtime.close()
self.runtime = None
self._start_time = None
def reset(self) -> None:
if self.runtime is not None:
self.runtime.tactile.reset()
def tick(self) -> FingerFrameView | None:
if self.runtime is None or self._start_time is None:
return None
ok, unrolled = self.runtime.grab_and_process(color=self.config.use_color)
if not ok:
return None
tactile = self.runtime.tactile
self.recorder.write_if_recording(tactile.frame)
tactile.compute_deformation(decompose=True, check_motion=True, threshold=0)
tactile.compute_contact()
fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = tactile.read_info(
tactile.info.FPS,
tactile.info.VRAW,
tactile.info.VNORMAL,
tactile.info.IMG,
tactile.info.CONTOUR,
tactile.info.CENTROID,
tactile.info.DEPTH,
tactile.info.FNORMAL,
tactile.info.FSHEARX,
tactile.info.FSHEARY,
)
return FingerFrameView(
timestamp=time.time() - self._start_time,
fps=float(fps),
unrolled=unrolled,
flow=flow,
vnormal=vnormal,
depth_map=depth_map,
img=img,
contour=contour,
centroid=centroid,
fnormal=float(fnormal),
fshearx=float(fshearx),
fsheary=float(fsheary),
centroid_found=bool(tactile.raw_sensor.centroid_found),
mask_coverage=float(tactile.mask_stats["coverage"]),
raw_frame=tactile.frame,
)
def start_recording(self) -> bool:
if self.runtime is None:
return False
return self.recorder.start(self.runtime.tactile.frame)
def stop_recording(self) -> None:
self.recorder.stop()
examples/viz/__init__.py
+20 -2
View File
@@ -1,20 +1,38 @@
"""Shared visualization helpers for Orisys example scripts."""
from .finger_panels import show_finger_uv_panels
from .force_charts import ForceChartWindow
from .mesh_camera import (
DEFAULT_FINGER_VIZ_POSE,
apply_mesh_lighting,
apply_view_from_world_to_cam,
camera_look_at_from_world_to_cam,
resolve_viz_pose_path,
)
from .offscreen_mesh import OffscreenFingerMeshRenderer
from .render_2d import render_contact_panel, render_depth_panel, render_uv_arrows
from .finger_mesh_viewer import FingerMeshViewer
from .mesh_viewer import UVMeshViewer, build_uv_surface_mesh
from .opencv_maps import depth_colormap, draw_magnitude_map, frame_to_bgr
from .recording import FrameRecorder
from .signal_filters import lowpass_filter
__all__ = [
"DEFAULT_FINGER_VIZ_POSE",
"FingerMeshViewer",
"ForceChartWindow",
"FrameRecorder",
"UVMeshViewer",
"apply_mesh_lighting",
"apply_view_from_world_to_cam",
"build_uv_surface_mesh",
"camera_look_at_from_world_to_cam",
"depth_colormap",
"draw_magnitude_map",
"frame_to_bgr",
"lowpass_filter",
"show_finger_uv_panels",
"OffscreenFingerMeshRenderer",
"render_contact_panel",
"render_depth_panel",
"render_uv_arrows",
"resolve_viz_pose_path",
]
examples/viz/contact_arrow.py
+223
View File
@@ -0,0 +1,223 @@
"""Contact normal-force arrow overlay for finger 3D mesh examples."""
from __future__ import annotations
import numpy as np
from orisys.finger.mesh import UVSurfaceMesh, compute_uv_surface_frames, mesh_uv_to_world_scale
DEFAULT_NORMAL_FORCE_SCALE = 1.0
MIN_ARROW_LENGTH_FRACTION = 0.02
MAX_ARROW_LENGTH_FRACTION = 0.30
REFERENCE_NORMAL_FORCE = 10000.0
MIN_ARROW_THICKNESS_SCALE = 0.20
MAX_ARROW_THICKNESS_SCALE = 1.0
CONTACT_ARROW_COLOR = (1.0, 0.92, 0.10)
def _mesh_span(vertices: np.ndarray) -> float:
extent = np.ptp(vertices, axis=0)
return float(max(np.max(extent), 1e-6))
def _rotation_matrix_z_to(direction: np.ndarray) -> np.ndarray:
"""Rotation matrix mapping unit +Z to ``direction``."""
direction = np.asarray(direction, dtype=np.float64).reshape(3)
norm = np.linalg.norm(direction)
if norm < 1e-8:
return np.eye(3, dtype=np.float64)
direction = direction / norm
z_axis = np.array([0.0, 0.0, 1.0], dtype=np.float64)
cross = np.cross(z_axis, direction)
dot = float(np.dot(z_axis, direction))
if dot > 0.9999:
return np.eye(3, dtype=np.float64)
if dot < -0.9999:
return np.diag([1.0, -1.0, -1.0]).astype(np.float64)
skew = np.array(
[
[0.0, -cross[2], cross[1]],
[cross[2], 0.0, -cross[0]],
[-cross[1], cross[0], 0.0],
],
dtype=np.float64,
)
return np.eye(3, dtype=np.float64) + skew + skew @ skew * (1.0 / (1.0 + dot))
def thickness_scale_from_force(force: float) -> float:
"""Map normal force to a radial scale factor (thin at low force, thicker at high force)."""
if force <= 0.0:
return 0.0
t = force / REFERENCE_NORMAL_FORCE
return float(
np.clip(
MIN_ARROW_THICKNESS_SCALE + (MAX_ARROW_THICKNESS_SCALE - MIN_ARROW_THICKNESS_SCALE) * t,
MIN_ARROW_THICKNESS_SCALE,
MAX_ARROW_THICKNESS_SCALE,
)
)
def _scale_arrow_template(
template_vertices: np.ndarray,
*,
length_scale: float,
thickness_scale: float,
) -> np.ndarray:
"""Scale a +Z arrow template axially and radially."""
scaled = np.asarray(template_vertices, dtype=np.float64).copy()
scaled[:, 0] *= thickness_scale
scaled[:, 1] *= thickness_scale
scaled[:, 2] *= length_scale
return scaled
def create_arrow_template(o3d):
"""Unit-height arrow mesh along +Z; radial size is scaled per-frame from force."""
arrow = o3d.geometry.TriangleMesh.create_arrow(
cylinder_radius=0.03,
cone_radius=0.06,
cylinder_height=0.72,
cone_height=0.28,
)
arrow.compute_vertex_normals()
return (
np.asarray(arrow.vertices, dtype=np.float64),
np.asarray(arrow.triangles, dtype=np.int32),
)
def build_contact_normal_arrow_mesh(
origin: np.ndarray,
normal: np.ndarray,
length: float,
force: float,
*,
mesh_span: float,
template_vertices: np.ndarray,
template_triangles: np.ndarray,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Build a yellow 3D arrow mesh pointing opposite the surface normal."""
origin = np.asarray(origin, dtype=np.float64).reshape(3)
normal = np.asarray(normal, dtype=np.float64).reshape(3)
norm = np.linalg.norm(normal)
color = np.tile(np.asarray(CONTACT_ARROW_COLOR, dtype=np.float64), (len(template_vertices), 1))
radial_scale = thickness_scale_from_force(force)
if norm < 1e-8 or length <= 0.0 or radial_scale <= 0.0 or not np.all(np.isfinite(origin)):
return np.zeros_like(template_vertices), template_triangles, color
direction = -normal / norm
length = float(np.clip(length, mesh_span * MIN_ARROW_LENGTH_FRACTION, mesh_span * MAX_ARROW_LENGTH_FRACTION))
if length <= 0.0:
return np.zeros_like(template_vertices), template_triangles, color
rotation = _rotation_matrix_z_to(direction)
scaled_template = _scale_arrow_template(
template_vertices,
length_scale=length,
thickness_scale=radial_scale,
)
vertices = scaled_template @ rotation.T + origin
return vertices, template_triangles, color
def create_arrow_mesh_geometry(o3d, template_vertices: np.ndarray, template_triangles: np.ndarray):
"""Create a persistent Open3D mesh for the contact arrow overlay."""
arrow_mesh = o3d.geometry.TriangleMesh()
arrow_mesh.vertices = o3d.utility.Vector3dVector(np.zeros_like(template_vertices))
arrow_mesh.triangles = o3d.utility.Vector3iVector(template_triangles)
arrow_mesh.vertex_colors = o3d.utility.Vector3dVector(
np.tile(np.asarray(CONTACT_ARROW_COLOR, dtype=np.float64), (len(template_vertices), 1))
)
arrow_mesh.compute_vertex_normals()
return arrow_mesh
class ContactArrowOverlay:
"""Example-layer contact arrow driven by centroid and normal force."""
def __init__(
self,
mesh: UVSurfaceMesh,
*,
normal_force_scale: float = DEFAULT_NORMAL_FORCE_SCALE,
enabled: bool = True,
):
import open3d as o3d
self.mesh = mesh
self.normal_force_scale = float(normal_force_scale)
self.enabled = enabled
self._mesh_span = _mesh_span(mesh.rest_vertices)
self._pixel_to_world = mesh_uv_to_world_scale(mesh.world_xyz, mesh.valid_mask)
_, _, self._surface_normals = compute_uv_surface_frames(mesh.world_xyz, mesh.valid_mask)
self._template_vertices, self._template_triangles = (
create_arrow_template(o3d) if enabled else (None, None)
)
self.arrow_mesh = (
create_arrow_mesh_geometry(o3d, self._template_vertices, self._template_triangles)
if enabled
else None
)
def compute(
self,
centroid,
centroid_found: bool,
fnormal: float,
) -> tuple[np.ndarray, np.ndarray, np.ndarray] | None:
if not self.enabled or self.arrow_mesh is None:
return None
template_vertices = self._template_vertices
template_triangles = self._template_triangles
color = np.tile(
np.asarray(CONTACT_ARROW_COLOR, dtype=np.float64),
(len(template_vertices), 1),
)
hidden = (np.zeros_like(template_vertices), template_triangles, color)
if not centroid_found or centroid is None:
return hidden
centroid = np.asarray(centroid, dtype=np.float64).reshape(-1)
if centroid.size < 2:
return hidden
cx, cy = int(round(centroid[0])), int(round(centroid[1]))
h, w = self.mesh.shape
if not (0 <= cy < h and 0 <= cx < w):
return hidden
if self.mesh.index_map[cy, cx] < 0:
return hidden
origin = self.mesh.world_xyz[cy, cx]
normal = self._surface_normals[cy, cx]
if not np.all(np.isfinite(origin)) or not np.all(np.isfinite(normal)):
return hidden
force = max(float(fnormal), 0.0)
if force <= 0.0:
return hidden
length = force * self.normal_force_scale * self._pixel_to_world
return build_contact_normal_arrow_mesh(
origin,
normal,
length,
force,
mesh_span=self._mesh_span,
template_vertices=template_vertices,
template_triangles=template_triangles,
)
def apply(self, arrow_vertices: np.ndarray, arrow_colors: np.ndarray) -> None:
import open3d as o3d
self.arrow_mesh.vertices = o3d.utility.Vector3dVector(arrow_vertices.astype(np.float64))
self.arrow_mesh.vertex_colors = o3d.utility.Vector3dVector(arrow_colors.astype(np.float64))
self.arrow_mesh.compute_vertex_normals()
examples/viz/finger_mesh_viewer.py
+66
View File
@@ -0,0 +1,66 @@
"""Finger 3D mesh viewer with example-only contact arrow overlay."""
from __future__ import annotations
from orisys.finger.viewer import UVMeshViewer
from .contact_arrow import ContactArrowOverlay, DEFAULT_NORMAL_FORCE_SCALE
class FingerMeshViewer(UVMeshViewer):
"""UV mesh viewer with a force-scaled contact normal arrow."""
def __init__(
self,
mesh,
*,
max_magnitude=None,
threshold: float = 0.0,
color_mode: str = "flow",
value_min=None,
value_max=None,
window_width: int = 1024,
window_height: int = 1024,
normal_force_scale: float = DEFAULT_NORMAL_FORCE_SCALE,
show_contact_arrow: bool = True,
):
super().__init__(
mesh,
max_magnitude=max_magnitude,
threshold=threshold,
color_mode=color_mode,
value_min=value_min,
value_max=value_max,
window_width=window_width,
window_height=window_height,
)
self._contact_arrow = ContactArrowOverlay(
mesh,
normal_force_scale=normal_force_scale,
enabled=show_contact_arrow,
)
def _extra_geometries_for_init(self):
if self._contact_arrow.arrow_mesh is None:
return []
return [self._contact_arrow.arrow_mesh]
def _extra_geometries_for_update(self):
if self._contact_arrow.arrow_mesh is None:
return []
return [self._contact_arrow.arrow_mesh]
def update(
self,
flow=None,
scalar_map=None,
*,
centroid=None,
centroid_found: bool = False,
fnormal: float = 0.0,
):
arrow = self._contact_arrow.compute(centroid, centroid_found, fnormal)
if arrow is not None and self._contact_arrow.arrow_mesh is not None:
arrow_vertices, _, arrow_colors = arrow
self._contact_arrow.apply(arrow_vertices, arrow_colors)
return super().update(flow=flow, scalar_map=scalar_map)
examples/viz/finger_panels.py
-29
View File
@@ -1,29 +0,0 @@
"""Finger-specific OpenCV panels for example scripts."""
from __future__ import annotations
import cv2
import orisys
from .opencv_maps import depth_colormap
def show_finger_uv_panels(
unrolled,
vnormal,
depth_map,
*,
flow_threshold: float = 0.4,
grid_spacing: int = 20,
arrow_scale: float = 1.0,
) -> None:
"""Show depth colormap and UV unrolled flow arrows."""
cv2.imshow("Depth", depth_colormap(depth_map))
arrows = orisys.util.draw_arrows(
unrolled,
vnormal,
threshold=flow_threshold,
grid_spacing=grid_spacing,
arrow_scale=arrow_scale,
)
cv2.imshow("UV Unrolled", arrows)
examples/viz/force_charts.py
+4 -3
View File
@@ -27,8 +27,8 @@ class ForceChartWindow:
app_name: str = "Orisys Plot Example",
) -> None:
pg = _require_pyqtgraph()
pg.setConfigOption("background", "w")
pg.setConfigOption("foreground", "k")
pg.setConfigOption("background", "k")
pg.setConfigOption("foreground", "w")
self._pg = pg
self.npoints = npoints
@@ -36,6 +36,7 @@ class ForceChartWindow:
self.app = pg.mkQApp(app_name)
self.win = pg.GraphicsLayoutWidget(show=True, title=title)
self.win.setWindowTitle(title)
self.pen = pg.mkPen(color="w")
self.data = np.zeros((npoints, 2, 3), dtype=np.float64)
for channel in range(3):
@@ -51,7 +52,7 @@ class ForceChartWindow:
p3.setLabel("bottom", "时间", units="s")
self._plots = (p1, p2, p3)
self._curves = (p1.plot(pen="k"), p2.plot(pen="k"), p3.plot(pen="k"))
self._curves = (p1.plot(pen=self.pen ), p2.plot(pen=self.pen ), p3.plot(pen=self.pen ))
def push(self, timestamp: float, fn: float, fx: float, fy: float) -> None:
for channel in range(3):
examples/viz/mesh_camera.py
+110
View File
@@ -0,0 +1,110 @@
"""Open3D camera helpers for finger mesh examples."""
from __future__ import annotations
from pathlib import Path
import numpy as np
DEFAULT_FINGER_VIZ_POSE = "assets/finger/visualization/world_to_cam.npy"
_CAMERA_X_ROT_90 = np.array(
[
[1.0, 0.0, 0.0],
[0.0, 0.0, -1.0],
[0.0, -1.0, 0.0],
],
dtype=np.float64,
)
_CAMERA_Z_ROT_180 = np.array(
[
[1.0, 0.0, 0.0],
[0.0, 1.0, -1],
[0.0, 0.0, -1],
],
dtype=np.float64,
)
def resolve_viz_pose_path(path: str | Path | None = None) -> Path:
"""Resolve world_to_cam.npy from cwd or packaged asset layout."""
raw = Path(path or DEFAULT_FINGER_VIZ_POSE).expanduser()
if raw.is_file():
return raw.resolve()
here = Path(__file__).resolve()
candidates = [
Path.cwd() / raw,
here.parents[2] / raw,
]
for candidate in candidates:
if candidate.is_file():
return candidate.resolve()
checked = ", ".join(str(p) for p in candidates)
raise FileNotFoundError(f"Visualization pose file not found: {raw}. Checked: {checked}")
def camera_look_at_from_world_to_cam(
world_to_cam: np.ndarray,
*,
lookat: np.ndarray,
distance_scale: float = 2.5,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Derive Open3D look_at(center, eye, up) from a Blender world_to_cam matrix."""
world_to_cam = np.asarray(world_to_cam, dtype=np.float64)
cam_to_world = np.linalg.inv(world_to_cam)
rotation = cam_to_world[:3, :3] @ _CAMERA_X_ROT_90 @ _CAMERA_Z_ROT_180
front = rotation @ np.array([0.0, 0.0, -1.0])
up = rotation @ np.array([0.0, 1.0, 0.0])
front_norm = np.linalg.norm(front)
up_norm = np.linalg.norm(up)
if front_norm > 0:
front = front / front_norm
if up_norm > 0:
up = up / up_norm
lookat = np.asarray(lookat, dtype=np.float64).reshape(3)
extent = float(np.linalg.norm(lookat - cam_to_world[:3, 3]))
if extent < 1e-6:
extent = 1.0
eye = lookat - front * (extent * distance_scale)
return lookat, eye, up
def apply_view_from_world_to_cam(vis, world_to_cam: np.ndarray, *, lookat: np.ndarray) -> None:
"""Apply Blender-exported world_to_cam as the Open3D default view."""
world_to_cam = np.asarray(world_to_cam, dtype=np.float64)
cam_to_world = np.linalg.inv(world_to_cam)
rotation = cam_to_world[:3, :3] @ _CAMERA_X_ROT_90 @ _CAMERA_Z_ROT_180
front = rotation @ np.array([0.0, 0.0, -1.0])
up = rotation @ np.array([0.0, 1.0, 0.0])
front_norm = np.linalg.norm(front)
up_norm = np.linalg.norm(up)
if front_norm > 0:
front = front / front_norm
if up_norm > 0:
up = up / up_norm
view_control = vis.get_view_control()
view_control.set_lookat(lookat.astype(np.float64))
view_control.set_front(front.astype(np.float64))
view_control.set_up(up.astype(np.float64))
vis.update_renderer()
def apply_mesh_lighting(vis) -> None:
"""Enable shaded lighting so the tactile mesh reads as a 3D surface."""
import open3d as o3d
render_option = vis.get_render_option()
if hasattr(render_option, "light_on"):
render_option.light_on = True
if hasattr(render_option, "mesh_color_option"):
render_option.mesh_color_option = o3d.visualization.MeshColorOption.Color
if hasattr(render_option, "mesh_shade_option"):
render_option.mesh_shade_option = o3d.visualization.MeshShadeOption.Color
vis.update_renderer()
examples/viz/mesh_viewer.py
+2 -2
View File
@@ -1,4 +1,4 @@
"""Re-exports for finger 3D mesh examples; implementation lives in orisys.finger."""
"""Re-exports for finger 3D mesh examples; core viewer lives in orisys.finger."""
from orisys.finger.mesh import * # noqa: F403
from orisys.finger.viewer import * # noqa: F403
from orisys.finger.viewer import UVMeshViewer, init_uv_mesh_viewer, update_uv_mesh_viewer # noqa: F401
examples/viz/offscreen_mesh.py
+357
View File
@@ -0,0 +1,357 @@
"""Embedded Open3D mesh renderer for Qt dashboards (offscreen or hidden-window capture)."""
from __future__ import annotations
import sys
import numpy as np
from orisys.finger.mesh import UVSurfaceMesh
from orisys.finger.viewer import flow_magnitude_vertex_colors, scalar_map_vertex_colors
from .contact_arrow import ContactArrowOverlay
from .mesh_camera import apply_view_from_world_to_cam, camera_look_at_from_world_to_cam
def _numpy_o3d_image_to_rgb(image) -> np.ndarray:
arr = np.asarray(image)
if arr.ndim == 3 and arr.shape[2] >= 3:
if arr.dtype == np.float32 or arr.dtype == np.float64:
return (np.clip(arr[:, :, :3], 0.0, 1.0) * 255.0).astype(np.uint8)
return arr[:, :, :3].astype(np.uint8)
return arr.astype(np.uint8)
def _make_unlit_material(o3d):
material = o3d.visualization.rendering.MaterialRecord()
material.shader = "defaultUnlit"
material.base_color = (1.0, 1.0, 1.0, 1.0)
return material
def _mesh_to_o3d(o3d, vertices: np.ndarray, triangles: np.ndarray, colors: np.ndarray):
mesh = o3d.geometry.TriangleMesh()
mesh.vertices = o3d.utility.Vector3dVector(vertices.astype(np.float64))
mesh.triangles = o3d.utility.Vector3iVector(triangles.astype(np.int32))
mesh.vertex_colors = o3d.utility.Vector3dVector(colors.astype(np.float64))
mesh.compute_vertex_normals()
return mesh
def _vertex_colors(mesh, *, color_mode, threshold, value_min, value_max, flow=None, scalar_map=None):
if color_mode == "depth":
return scalar_map_vertex_colors(
scalar_map,
mesh,
value_min=value_min,
value_max=value_max,
threshold=threshold,
)
return flow_magnitude_vertex_colors(
flow,
mesh,
threshold=threshold,
)
class _FilamentOffscreenBackend:
"""Filament OffscreenRenderer (Linux headless / platforms with EGL)."""
def __init__(
self,
mesh: UVSurfaceMesh,
*,
width: int,
height: int,
threshold: float,
color_mode: str,
value_min,
value_max,
world_to_cam: np.ndarray | None,
lookat: np.ndarray,
show_contact_arrow: bool,
):
import open3d as o3d
self._o3d = o3d
self.mesh = mesh
self.width = width
self.height = height
self.threshold = threshold
self.color_mode = color_mode
self.value_min = value_min
self.value_max = value_max
self.world_to_cam = world_to_cam
self.lookat = lookat
self._contact_arrow = ContactArrowOverlay(mesh, enabled=show_contact_arrow)
self._renderer = o3d.visualization.rendering.OffscreenRenderer(self.width, self.height)
self._renderer.scene.set_background([0.05, 0.05, 0.08, 1.0])
self._material = _make_unlit_material(o3d)
self._mesh_added = False
self._arrow_added = False
self._apply_camera()
def _apply_camera(self) -> None:
if self.world_to_cam is not None:
center, eye, up = camera_look_at_from_world_to_cam(self.world_to_cam, lookat=self.lookat)
else:
center = self.lookat.astype(np.float64)
extent = np.ptp(self.mesh.rest_vertices, axis=0)
span = float(np.max(extent))
eye = center + np.array([0.0, 0.0, span * 2.0], dtype=np.float64)
up = np.array([0.0, 1.0, 0.0], dtype=np.float64)
self._renderer.scene.camera.look_at(center, eye, up)
def resize(self, width: int, height: int) -> None:
width = max(int(width), 64)
height = max(int(height), 64)
if width == self.width and height == self.height:
return
self.width = width
self.height = height
self._renderer = self._o3d.visualization.rendering.OffscreenRenderer(self.width, self.height)
self._renderer.scene.set_background([0.05, 0.05, 0.08, 1.0])
self._mesh_added = False
self._arrow_added = False
self._apply_camera()
def render(self, *, flow=None, scalar_map=None, centroid=None, centroid_found=False, fnormal=0.0):
o3d = self._o3d
colors = _vertex_colors(
self.mesh,
color_mode=self.color_mode,
threshold=self.threshold,
value_min=self.value_min,
value_max=self.value_max,
flow=flow,
scalar_map=scalar_map,
)
body_mesh = _mesh_to_o3d(o3d, self.mesh.rest_vertices, self.mesh.triangles, colors)
if not self._mesh_added:
self._renderer.scene.add_geometry("finger_mesh", body_mesh, self._material)
self._mesh_added = True
else:
self._renderer.scene.remove_geometry("finger_mesh")
self._renderer.scene.add_geometry("finger_mesh", body_mesh, self._material)
arrow_result = self._contact_arrow.compute(centroid, centroid_found, fnormal)
if arrow_result is not None and self._contact_arrow.arrow_mesh is not None:
arrow_vertices, arrow_triangles, arrow_colors = arrow_result
if np.any(arrow_vertices):
arrow_mesh = _mesh_to_o3d(o3d, arrow_vertices, arrow_triangles, arrow_colors)
if not self._arrow_added:
self._renderer.scene.add_geometry("contact_arrow", arrow_mesh, self._material)
self._arrow_added = True
else:
self._renderer.scene.remove_geometry("contact_arrow")
self._renderer.scene.add_geometry("contact_arrow", arrow_mesh, self._material)
elif self._arrow_added:
self._renderer.scene.remove_geometry("contact_arrow")
self._arrow_added = False
return _numpy_o3d_image_to_rgb(self._renderer.render_to_image())
def close(self) -> None:
self._renderer = None
class _HiddenVisualizerBackend:
"""Invisible Open3D Visualizer + framebuffer capture (Windows-friendly)."""
def __init__(
self,
mesh: UVSurfaceMesh,
*,
width: int,
height: int,
threshold: float,
color_mode: str,
value_min,
value_max,
world_to_cam: np.ndarray | None,
lookat: np.ndarray,
show_contact_arrow: bool,
):
import open3d as o3d
self._o3d = o3d
self.mesh = mesh
self.width = width
self.height = height
self.threshold = threshold
self.color_mode = color_mode
self.value_min = value_min
self.value_max = value_max
self.world_to_cam = world_to_cam
self.lookat = lookat
self._contact_arrow = ContactArrowOverlay(mesh, enabled=show_contact_arrow)
self.vis = None
self.tri_mesh = None
self._create_window()
def _create_window(self) -> None:
o3d = self._o3d
if self.vis is not None:
self.vis.destroy_window()
colors = _vertex_colors(
self.mesh,
color_mode=self.color_mode,
threshold=self.threshold,
value_min=self.value_min,
value_max=self.value_max,
)
self.vis = o3d.visualization.Visualizer()
self.vis.create_window(
window_name="Orisys Finger Mesh (embedded)",
width=self.width,
height=self.height,
visible=False,
)
self.tri_mesh = o3d.geometry.TriangleMesh()
self.tri_mesh.vertices = o3d.utility.Vector3dVector(self.mesh.rest_vertices.astype(np.float64))
self.tri_mesh.triangles = o3d.utility.Vector3iVector(self.mesh.triangles.astype(np.int32))
self.tri_mesh.vertex_colors = o3d.utility.Vector3dVector(colors.astype(np.float64))
self.tri_mesh.compute_vertex_normals()
self.vis.add_geometry(self.tri_mesh)
if self._contact_arrow.arrow_mesh is not None:
self.vis.add_geometry(self._contact_arrow.arrow_mesh)
render_option = self.vis.get_render_option()
render_option.mesh_show_back_face = True
if hasattr(render_option, "light_on"):
render_option.light_on = False
render_option.background_color = np.array([0.05, 0.05, 0.08])
if self.world_to_cam is not None:
apply_view_from_world_to_cam(self.vis, self.world_to_cam, lookat=self.lookat)
else:
view_control = self.vis.get_view_control()
center = self.lookat.astype(np.float64)
extent = np.ptp(self.mesh.rest_vertices, axis=0)
span = float(np.max(extent))
view_control.set_lookat(center)
view_control.set_front(np.array([0.0, 0.0, -1.0]))
view_control.set_up(np.array([0.0, 1.0, 0.0]))
view_control.set_zoom(0.5 if span > 0 else 0.7)
self.vis.poll_events()
self.vis.update_renderer()
def resize(self, width: int, height: int) -> None:
width = max(int(width), 64)
height = max(int(height), 64)
if width == self.width and height == self.height:
return
self.width = width
self.height = height
self._create_window()
def render(self, *, flow=None, scalar_map=None, centroid=None, centroid_found=False, fnormal=0.0):
o3d = self._o3d
colors = _vertex_colors(
self.mesh,
color_mode=self.color_mode,
threshold=self.threshold,
value_min=self.value_min,
value_max=self.value_max,
flow=flow,
scalar_map=scalar_map,
)
vertices = self.mesh.rest_vertices
self.tri_mesh.vertices = o3d.utility.Vector3dVector(vertices.astype(np.float64))
self.tri_mesh.vertex_colors = o3d.utility.Vector3dVector(colors.astype(np.float64))
self.tri_mesh.compute_vertex_normals()
self.vis.update_geometry(self.tri_mesh)
arrow_result = self._contact_arrow.compute(centroid, centroid_found, fnormal)
if arrow_result is not None and self._contact_arrow.arrow_mesh is not None:
arrow_vertices, _, arrow_colors = arrow_result
self._contact_arrow.apply(arrow_vertices, arrow_colors)
self.vis.update_geometry(self._contact_arrow.arrow_mesh)
self.vis.poll_events()
self.vis.update_renderer()
return _numpy_o3d_image_to_rgb(self.vis.capture_screen_float_buffer(do_render=False))
def close(self) -> None:
if self.vis is not None:
self.vis.destroy_window()
self.vis = None
def _create_backend(mesh: UVSurfaceMesh, **kwargs):
# Filament OffscreenRenderer needs EGL headless (not available on Windows).
if sys.platform == "win32":
return _HiddenVisualizerBackend(mesh, **kwargs)
try:
return _FilamentOffscreenBackend(mesh, **kwargs)
except RuntimeError:
return _HiddenVisualizerBackend(mesh, **kwargs)
class OffscreenFingerMeshRenderer:
"""Render finger UV mesh + contact arrow to an RGB image for Qt embedding."""
def __init__(
self,
mesh: UVSurfaceMesh,
*,
width: int = 512,
height: int = 512,
threshold: float = 0.0,
color_mode: str = "depth",
value_min=None,
value_max=None,
world_to_cam: np.ndarray | None = None,
lookat: np.ndarray | None = None,
show_contact_arrow: bool = True,
):
lookat = lookat if lookat is not None else mesh.rest_vertices.mean(axis=0)
self._backend = _create_backend(
mesh,
width=int(width),
height=int(height),
threshold=threshold,
color_mode=color_mode,
value_min=value_min,
value_max=value_max,
world_to_cam=world_to_cam,
lookat=lookat,
show_contact_arrow=show_contact_arrow,
)
@property
def width(self) -> int:
return self._backend.width
@property
def height(self) -> int:
return self._backend.height
def resize(self, width: int, height: int) -> None:
self._backend.resize(width, height)
def render(
self,
*,
flow=None,
scalar_map=None,
centroid=None,
centroid_found: bool = False,
fnormal: float = 0.0,
) -> np.ndarray:
return self._backend.render(
flow=flow,
scalar_map=scalar_map,
centroid=centroid,
centroid_found=centroid_found,
fnormal=fnormal,
)
def close(self) -> None:
self._backend.close()
examples/viz/render_2d.py
+35
View File
@@ -0,0 +1,35 @@
"""Pure numpy→BGR render helpers for finger 2D panels (no window I/O)."""
from __future__ import annotations
import orisys
from .opencv_maps import depth_colormap
def render_depth_panel(depth_map):
"""Return a BGR depth colormap image."""
return depth_colormap(depth_map)
def render_uv_arrows(
unrolled,
vnormal,
*,
flow_threshold: float = 0.4,
grid_spacing: int = 20,
arrow_scale: float = 1.0,
):
"""Return a BGR image with UV unrolled flow arrows."""
return orisys.util.draw_arrows(
unrolled,
vnormal,
threshold=flow_threshold,
grid_spacing=grid_spacing,
arrow_scale=arrow_scale,
)
def render_contact_panel(img, contour, centroid):
"""Return a BGR contact overlay on the raw sensor image."""
return orisys.util.draw_contact(img, contour, centroid)
examples/viz/signal_filters.py
+7 -2
View File
@@ -20,8 +20,13 @@ def lowpass_filter(data, cutoff_hz, sample_rate_hz=30, order=2) -> np.ndarray:
)
b, a = butter(order, cutoff_hz / nyquist, btype="low")
padlen = 3 * (max(len(b), len(a)) - 1)
taplen = max(len(a), len(b))
# scipy.signal.filtfilt requires len(x) > padlen (default 3 * taplen for recent scipy).
padlen = 3 * taplen
if y.size <= padlen:
return y.copy()
return filtfilt(b, a, y)
try:
return filtfilt(b, a, y)
except ValueError:
return y.copy()