GUI for finger v1

This commit is contained in:
ykcheungab
2026-06-18 22:18:19 +08:00
parent f19f092b64
commit 3fd9b839a9
42 changed files with 2326 additions and 1186 deletions
+1
View File
@@ -29,6 +29,7 @@ __pycache__/
/docs/.quarto/ /docs/.quarto/
/docs/exported_data/ /docs/exported_data/
/tools/3d_patch_generation_blender/exported_data/ /tools/3d_patch_generation_blender/exported_data/
/scripts/logs/
# Local or generated assets kept out of git # Local or generated assets kept out of git
/config/compound_eye.png /config/compound_eye.png
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+1 -1
View File
@@ -26,7 +26,7 @@
}, },
"contact": { "contact": {
"min_contour_area": 3000, "min_contour_area": 3000,
"flow_threshold": 1, "flow_threshold": 0.5,
"div_threshold": 0.03, "div_threshold": 0.03,
"expansion_threshold": 0.01, "expansion_threshold": 0.01,
"kernel_size": [5, 5], "kernel_size": [5, 5],
+13 -13
View File
@@ -8,18 +8,20 @@ User-facing demo scripts for the Orisys SDK. Run from the repository root.
- `plot_data.py` — compound-eye sensor, force charts (PyQtGraph) - `plot_data.py` — compound-eye sensor, force charts (PyQtGraph)
- `two_sensor_pipeline.py` — dual-sensor multiprocessing demo - `two_sensor_pipeline.py` — dual-sensor multiprocessing demo
- `finger_pipeline.py` — finger UV minimal pipeline - `finger_pipeline.py` — finger UV minimal pipeline
- `finger_pipeline_3dview.py` — finger UV + Open3D mesh viewer - `finger_pipeline_3dview.py` — finger UV unified Qt dashboard
- `finger_plot_data.py` — finger UV + force charts - `pipeline/` — runtime acquisition (`FingerPipeline`, `FingerFrameView`)
- `viz/` — shared visualization helpers used by plot and finger examples - `gui/` — Qt dashboard composition (3D + 2D + charts in one window)
- `mesh_viewer.py` — re-exports `UVMeshViewer` / `build_uv_surface_mesh` from `orisys.finger` - `viz/` — rendering helpers (offscreen mesh, 2D panels, charts)
- `finger_panels.py` — 2D OpenCV panels for finger demos
## Typical usage ## Typical usage
```bash ```bash
python examples/basic_pipeline.py -v 0 -c ./config/ddjx01.json pip install -e ".[gui]"
python examples/plot_data.py -v 0 -c ./config/ddjx01.json
python examples/finger_plot_data.py -v 0 -c ./config/dd02-ov.json python examples/finger_pipeline_3dview.py -v 0 -c ./config/dd02-ov.json
# Sales-demo shim
python scripts/qt_viewer_finger.py -v 0 -c ./config/dd02-ov.json
``` ```
## Requirements ## Requirements
@@ -28,14 +30,12 @@ python examples/finger_plot_data.py -v 0 -c ./config/dd02-ov.json
- camera or test video - camera or test video
- JSON config under `./config/` - JSON config under `./config/`
Plot examples additionally require: GUI examples additionally require:
```bash ```bash
pip install pyqt5 pyqtgraph pip install -e ".[gui]"
``` ```
## Notes ## Notes
- Press `q` to quit OpenCV windows where applicable. - Dashboard: click **启动** to start acquisition; `Q` quit, `R` reset, `S`/`E` record, `Esc` fullscreen.
- Plot examples: `s` starts recording, `e` stops recording.
+29 -153
View File
@@ -1,13 +1,12 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
UV-unroll finger tactile pipeline with 3D mesh visualization. UV-unroll finger tactile pipeline with unified Qt dashboard.
Offline: load Blender export -> build_or_load_domain -> unroll_map_uv.npz Where to customize the GUI:
Runtime: camera frame -> UVUnrollFrameProcessor -> TactileProcessor -> forces/contact Layout (sizes, proportions) -> examples/gui/layout.py <-- edit here
UV-space flow/HHD -> grey-white 3D finger mesh (Open3D) Colors / QSS -> examples/gui/styles.py
Widget rendering logic -> examples/gui/widgets/
For the full 2D dashboard demo see: Window behavior -> examples/gui/dashboard.py
tools/3d_patch_generation_blender/unroll_realtime.py
""" """
import argparse import argparse
@@ -17,172 +16,49 @@ from pathlib import Path
os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0") os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0")
import cv2
from orisys.finger import DEFAULT_FINGER_EXPORT_DIR, DEFAULT_FINGER_MAP_CACHE, FingerRuntime
_EXAMPLES_DIR = Path(__file__).resolve().parent _EXAMPLES_DIR = Path(__file__).resolve().parent
if str(_EXAMPLES_DIR) not in sys.path: if str(_EXAMPLES_DIR) not in sys.path:
sys.path.insert(0, str(_EXAMPLES_DIR)) sys.path.insert(0, str(_EXAMPLES_DIR))
from viz import UVMeshViewer, build_uv_surface_mesh, show_finger_uv_panels from gui import run_dashboard
from pipeline import PipelineConfig
from viz import DEFAULT_FINGER_VIZ_POSE
def open_mesh_viewer(runtime, args): def _parse_args():
domain = runtime.domain
if domain.surface is None:
raise RuntimeError(
"UV domain has no world_xyz surface map. Rebuild cache with include_surface=True."
)
uv_mesh = build_uv_surface_mesh(domain.surface.world_xyz, domain.valid_mask)
if args.no_3d:
return None, uv_mesh
viewer = UVMeshViewer(
uv_mesh,
max_magnitude=args.magnitude_max,
threshold=args.depth_threshold if args.mesh_map == "depth" else args.flow_threshold,
color_mode=args.mesh_map,
)
print(
f"3D mesh viewer: {len(uv_mesh.rest_vertices)} vertices, "
f"{len(uv_mesh.triangles)} triangles, {args.mesh_map} surface mode"
)
return viewer, uv_mesh
def main():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="UV-unroll finger tactile pipeline with 3D mesh viewer" description="UV-unroll finger tactile pipeline with unified Qt dashboard"
) )
parser.add_argument( parser.add_argument("--video", "-v", default="0", help="Camera index or video file path")
"--export-dir",
default=DEFAULT_FINGER_EXPORT_DIR,
help="Blender export folder containing mesh.npz, camera_intrinsics.npz, world_to_cam.npy",
)
parser.add_argument("--map_cache", default=DEFAULT_FINGER_MAP_CACHE, help="Cached UV domain .npz")
parser.add_argument("--camera_id", type=int, default=0)
parser.add_argument("--video", help="Video file path (overrides camera)")
parser.add_argument("--config", "-c", default="./config/dd02-ov.json", help="Orisys SDK config") parser.add_argument("--config", "-c", default="./config/dd02-ov.json", help="Orisys SDK config")
parser.add_argument( parser.add_argument("--cal", default="", help="Optional calibration file path")
"--source_scale",
type=float,
default=None,
help="Resize live source before UV remap (default: camera.source_scale in config)",
)
parser.add_argument("--no_flip_source_vertical", action="store_true", help="Skip vertical flip before UV remap")
parser.add_argument("--show", action="store_true", help="Show 2D unrolled image window with flow arrows")
parser.add_argument("--verbose", action="store_true", help="SDK verbose logs") parser.add_argument("--verbose", action="store_true", help="SDK verbose logs")
parser.add_argument(
"--magnitude-max",
type=float,
default=None,
help="Upper optical-flow magnitude for 3D surface heat-map scaling; default uses frame percentile.",
)
parser.add_argument( parser.add_argument(
"--mesh-map", "--mesh-map",
choices=["flow", "depth"], choices=["flow", "depth"],
default="depth", default="depth",
help="Color the 3D mesh from optical-flow magnitude or the contact depth/divergence map.", help="Color the 3D mesh from flow magnitude or contact depth.",
)
parser.add_argument("--threshold", type=float, default=None, help="Mesh/arrow threshold")
parser.add_argument("--no-3d", action="store_true", help="Disable 3D mesh view")
parser.add_argument(
"--viz-pose",
default=DEFAULT_FINGER_VIZ_POSE,
help="world_to_cam.npy for the default view; pass 'none' to skip",
) )
parser.add_argument( parser.add_argument(
"--flow-threshold", "--cutoff",
type=float, type=float,
default=0.4, default=5.0,
help="Minimum optical-flow magnitude to color 3D surface / draw 2D arrows (matches draw_arrows).", help="Low-pass cutoff for force curves (Hz)",
) )
parser.add_argument( return parser.parse_args()
"--depth-threshold",
type=float,
default=0.0,
help="Minimum depth/divergence value to color on the 3D mesh.",
)
parser.add_argument("--no-3d", action="store_true", help="Disable Open3D mesh viewer")
args = parser.parse_args()
video = args.camera_id
if args.video:
video = args.video
try: def main():
runtime = FingerRuntime.open( args = _parse_args()
export_dir=args.export_dir, config = PipelineConfig.from_args(args)
config=args.config, sys.exit(run_dashboard(config))
video=video,
verbose=args.verbose,
)
except RuntimeError as exc:
print(str(exc), file=sys.stderr)
sys.exit(1)
if args.verbose:
geometry = runtime.geometry
print(
f"UV geometry: native={geometry.native_width}x{geometry.native_height}, "
f"source={geometry.source_width}x{geometry.source_height} "
f"(scale={geometry.source_scale:g}), output={geometry.out_width}x{geometry.out_height}, "
f"map_cache={geometry.map_cache}"
)
mesh_viewer, _uv_mesh = open_mesh_viewer(runtime, args)
runtime.warmup(color=False)
tactile = runtime.tactile
print("Press 'q' to quit, 'r' to reset tracker. Close the 3D window to stop.")
try:
while True:
ok, unrolled = runtime.grab_and_process(color=False)
if not ok:
break
tactile.compute_deformation(decompose=True, check_motion=True, threshold=0)
tactile.compute_contact()
fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = tactile.read_info(
tactile.info.FPS,
tactile.info.VRAW,
tactile.info.VNORMAL,
tactile.info.IMG,
tactile.info.CONTOUR,
tactile.info.CENTROID,
tactile.info.DEPTH,
tactile.info.FNORMAL,
tactile.info.FSHEARX,
tactile.info.FSHEARY,
)
flow_mag = float((flow[:, :, 0] ** 2 + flow[:, :, 1] ** 2).mean() ** 0.5) if flow.size else 0.0
print(
f"FPS={fps:.2f} flow_rms={flow_mag:.2f} "
f"mask={tactile.mask_stats['coverage']:.2f} normal={fnormal:.2f} shear=({fshearx:.2f}, {fsheary:.2f})",
end="\r",
)
if mesh_viewer is not None:
if not mesh_viewer.update(flow=flow, scalar_map=depth_map):
print("\n3D viewer closed.")
break
if args.show:
show_finger_uv_panels(
unrolled,
vnormal,
depth_map,
flow_threshold=args.flow_threshold,
)
key = cv2.waitKey(1) & 0xFF
if key == ord("q"):
print("\nExiting.")
break
if key == ord("r"):
tactile.reset()
print("\nTracker reset.")
finally:
runtime.close()
if mesh_viewer is not None:
mesh_viewer.close()
cv2.destroyAllWindows()
if __name__ == "__main__": if __name__ == "__main__":
-101
View File
@@ -1,101 +0,0 @@
"""
Finger UV pipeline with real-time force charts and OpenCV panels.
Demonstrates:
1. UV-unrolled deformation arrows
2. Contact region overlay
3. PyQtGraph normal/shear force plots
4. Optional raw-frame recording (s/e keys)
"""
import argparse
import os
import sys
import time
from pathlib import Path
os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0"
import cv2
import orisys
from orisys.finger import FingerRuntime
_EXAMPLES_DIR = Path(__file__).resolve().parent
if str(_EXAMPLES_DIR) not in sys.path:
sys.path.insert(0, str(_EXAMPLES_DIR))
from viz import ForceChartWindow, FrameRecorder, lowpass_filter
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--video", "-v", type=str, default="0", help="视频源:摄像头编号或视频文件路径")
parser.add_argument("--config", "-c", type=str, default="./config/dd02-ov.json", help="配置文件名称或路径")
parser.add_argument("--verbose", "-verbose", type=bool, default=True, help="是否输出详细日志")
args = parser.parse_args()
charts = ForceChartWindow()
recorder = FrameRecorder()
runtime = FingerRuntime.open(
config=args.config,
video=args.video,
verbose=args.verbose,
)
runtime.warmup(color=True)
tactile = runtime.tactile
start_time = time.time()
print("\n按键说明:'q' 退出程序,'s' 开始录像,'e' 结束录像。")
try:
while True:
ok, unrolled = runtime.grab_and_process(color=True)
if not ok:
break
recorder.write_if_recording(tactile.frame)
tactile.compute_deformation(check_motion=True, threshold=0)
tactile.compute_contact()
fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = tactile.read_info(
tactile.info.FPS,
tactile.info.VRAW,
tactile.info.VNORMAL,
tactile.info.IMG,
tactile.info.CONTOUR,
tactile.info.CENTROID,
tactile.info.DEPTH,
tactile.info.FNORMAL,
tactile.info.FSHEARX,
tactile.info.FSHEARY,
)
print(f"FPS={fps:.2f}, 法向力={fnormal:.4f}, 切向力X={fshearx:.4f}, 切向力Y={fsheary:.4f}")
arrows = orisys.util.draw_arrows(unrolled, flow, threshold=3, grid_spacing=10, arrow_scale=1.0)
cv2.imshow("arrows", arrows)
cv2.imshow("contact", orisys.util.draw_contact(img, contour, centroid))
charts.push(time.time() - start_time, fnormal, fshearx, fsheary)
display = charts.data.copy()
for channel in range(3):
display[:, 1, channel] = lowpass_filter(
charts.data[:, 1, channel],
cutoff_hz=5,
sample_rate_hz=30,
)
charts.refresh(display)
key = cv2.waitKey(1)
if key & 0xFF == ord("q"):
print("\n正在退出示例程序...")
break
if key == ord("s"):
recorder.start(tactile.frame)
if key == ord("e"):
recorder.stop()
finally:
runtime.close()
if __name__ == "__main__":
main()
+30
View File
@@ -0,0 +1,30 @@
"""Public GUI entry points for finger examples."""
from .dashboard import FingerDashboardWindow, run_dashboard
from .layout import (
DashboardLayout,
LAYOUT,
LOG_PANEL,
PANEL_SETUP,
TOP_ROW,
PanelRow,
PanelSetup,
PanelSlot,
top_row_splitter_sizes,
top_row_slot_index,
)
__all__ = [
"DashboardLayout",
"FingerDashboardWindow",
"LAYOUT",
"LOG_PANEL",
"PANEL_SETUP",
"TOP_ROW",
"PanelRow",
"PanelSetup",
"PanelSlot",
"run_dashboard",
"top_row_splitter_sizes",
"top_row_slot_index",
]
+34
View File
@@ -0,0 +1,34 @@
"""Qt application bootstrap for Orisys example dashboards."""
from __future__ import annotations
import os
import sys
def fix_qt_plugin_path() -> None:
"""OpenCV redirects Qt to cv2/qt/plugins; use PyQt5's plugins instead."""
try:
from PyQt5.QtCore import QLibraryInfo
os.environ["QT_QPA_PLATFORM_PLUGIN_PATH"] = QLibraryInfo.location(
QLibraryInfo.PluginsPath
)
except Exception:
path = os.environ.get("QT_QPA_PLATFORM_PLUGIN_PATH", "")
if "cv2" in path.replace("\\", "/"):
os.environ.pop("QT_QPA_PLATFORM_PLUGIN_PATH", None)
def get_or_create_qapp(app_name: str = "Orisys Finger Dashboard"):
fix_qt_plugin_path()
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QApplication
QApplication.setAttribute(Qt.AA_EnableHighDpiScaling, True)
QApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True)
app = QApplication.instance()
if app is None:
app = QApplication(sys.argv)
app.setApplicationName(app_name)
return app
+333
View File
@@ -0,0 +1,333 @@
"""Unified Qt dashboard for the finger tactile example."""
from __future__ import annotations
import sys
from pathlib import Path
from PyQt5.QtCore import Qt, QTimer
from PyQt5.QtGui import QIcon
from PyQt5.QtWidgets import (
QApplication,
QFileDialog,
QHBoxLayout,
QLineEdit,
QMainWindow,
QPushButton,
QSplitter,
QVBoxLayout,
QWidget,
)
from pipeline.config import PipelineConfig
from pipeline.service import FingerPipeline
from .layout import (
DashboardLayout,
LAYOUT,
PANEL_SETUP,
PanelSetup,
top_row_splitter_sizes,
)
from .styles import STYLE
from .widgets import (
ArrowPanelWidget,
ForceChartWidget,
MeshViewportWidget,
TeeStream,
WidgetLogHandler,
make_log_widget,
)
class FingerDashboardWindow(QMainWindow):
"""Single-window finger demo: 3D mesh | UV arrows | force charts."""
def __init__(
self,
config: PipelineConfig,
*,
layout: DashboardLayout = LAYOUT,
log_file=None,
):
super().__init__()
self.config = config
self.layout = layout
self.pipeline = FingerPipeline(config)
self._running = False
self.setWindowTitle(layout.window_title)
self.setStyleSheet(STYLE)
logo_path = Path(__file__).resolve().parents[2] / "logo.png"
if logo_path.is_file():
self.setWindowIcon(QIcon(str(logo_path)))
self._setup_ui()
self._wire_logging(log_file)
self.timer = QTimer()
self.timer.timeout.connect(self._on_timer)
self.camera_edit.setText(str(config.video))
self.config_edit.setText(config.config)
self.cal_edit.setText(config.calib or "")
def _setup_ui(self) -> None:
layout = self.layout
central = QWidget()
self.setCentralWidget(central)
root_layout = QVBoxLayout(central)
root_layout.setContentsMargins(layout.root_margin_px, layout.root_margin_px, layout.root_margin_px, layout.root_margin_px)
root_layout.setSpacing(layout.root_spacing_px)
bar = QHBoxLayout()
bar.setSpacing(layout.toolbar_spacing_px)
self.btn_camera = QPushButton("摄像机")
self.btn_camera.clicked.connect(self._select_camera)
bar.addWidget(self.btn_camera)
self.camera_edit = QLineEdit("0")
self.camera_edit.setPlaceholderText("摄像头编号 或 视频路径")
bar.addWidget(self.camera_edit, 1)
self.btn_config = QPushButton("配置")
self.btn_config.clicked.connect(self._select_config)
bar.addWidget(self.btn_config)
self.config_edit = QLineEdit("./config/dd02-ov.json")
bar.addWidget(self.config_edit, 1)
self.cal_edit = QLineEdit("")
bar.addWidget(self.cal_edit, 1)
self.btn_reset = QPushButton("重置")
self.btn_reset.setEnabled(False)
self.btn_reset.clicked.connect(self._on_reset)
bar.addWidget(self.btn_reset)
self.btn_start = QPushButton("启动")
self.btn_start.setObjectName("btnStart")
self.btn_start.clicked.connect(self._on_start)
bar.addWidget(self.btn_start)
self.btn_stop = QPushButton("暂停")
self.btn_stop.setObjectName("btnStop")
self.btn_stop.setEnabled(False)
self.btn_stop.clicked.connect(self._on_stop)
bar.addWidget(self.btn_stop)
root_layout.addLayout(bar)
setup = layout.panel_setup
self._panel_widgets: dict[str, QWidget] = {}
self.mesh_viewport = MeshViewportWidget(layout=layout)
self.arrow_panel = ArrowPanelWidget(flow_threshold=layout.arrow_flow_threshold, layout=layout)
self.force_charts = ForceChartWidget(layout=layout, lowpass_cutoff_hz=self.config.lowpass_cutoff_hz)
self.log_widget = make_log_widget()
self._panel_widgets["mesh"] = self.mesh_viewport
self._panel_widgets["arrows"] = self.arrow_panel
self._panel_widgets["charts"] = self.force_charts.widget
self._panel_widgets["log"] = self.log_widget
# Top row: mesh | arrows | charts (order from layout.PANEL_SETUP.top_row)
h_splitter = QSplitter(Qt.Horizontal)
for slot in setup.top_row.slots:
if not slot.enabled:
continue
h_splitter.addWidget(self._panel_widgets[slot.id])
h_splitter.setStretchFactor(h_splitter.count() - 1, slot.stretch)
h_splitter.setChildrenCollapsible(setup.top_row.collapsible)
self.h_splitter = h_splitter
# Vertical: top panel | log panel
v_splitter = QSplitter(Qt.Vertical)
v_splitter.addWidget(h_splitter)
if setup.log_panel.enabled:
v_splitter.addWidget(self.log_widget)
v_splitter.setStretchFactor(0, setup.top_stretch)
v_splitter.setStretchFactor(1, setup.log_stretch)
else:
v_splitter.setStretchFactor(0, 1)
self.v_splitter = v_splitter
root_layout.addWidget(v_splitter)
self.resize(layout.window_width, layout.window_height)
QTimer.singleShot(0, self._apply_panel_layout)
def _apply_panel_layout(self) -> None:
setup = self.layout.panel_setup
for slot in setup.top_row.slots:
if not slot.enabled:
continue
if slot.id in ("mesh", "arrows"):
self._panel_widgets[slot.id].set_fixed_size(slot.width_px, slot.height_px)
elif slot.id == "charts":
self.force_charts.set_panel_size(slot.width_px, slot.height_px)
if setup.log_panel.enabled and setup.log_panel.height_px > 0:
self.log_widget.setFixedHeight(setup.log_panel.height_px)
self.h_splitter.setSizes(list(top_row_splitter_sizes(self.layout)))
def resizeEvent(self, event):
super().resizeEvent(event)
QTimer.singleShot(0, self._apply_panel_layout)
def showEvent(self, event):
super().showEvent(event)
QTimer.singleShot(0, self._apply_panel_layout)
def _wire_logging(self, log_file) -> None:
streams = []
if log_file is not None:
streams.append(log_file)
self.log_handler = WidgetLogHandler(self.log_widget)
streams.append(self.log_handler)
sys.stdout = TeeStream(*streams)
sys.stderr = sys.stdout
def _on_start(self) -> None:
if self._running:
return
self.btn_start.setText("启动中...")
self.btn_start.setEnabled(False)
QApplication.processEvents()
self.config.video = self.camera_edit.text().strip()
self.config.config = self.config_edit.text().strip()
cal = self.cal_edit.text().strip()
self.config.calib = cal or None
try:
self.pipeline = FingerPipeline(self.config)
self.pipeline.open()
self.mesh_viewport.configure(self.pipeline)
self.force_charts.reset()
QTimer.singleShot(0, self._apply_panel_layout)
except Exception as exc:
self.mesh_viewport.close_renderer()
self.btn_start.setText("启动")
self.btn_start.setEnabled(True)
print(f"启动失败: {exc}")
return
self._running = True
self.timer.start(self.layout.timer_interval_ms)
self.btn_start.setText("启动")
self.btn_stop.setEnabled(True)
self.btn_reset.setEnabled(True)
print(
f"▶ 启动 — 摄像机={self.config.video}, 配置={self.config.config}, "
f"标定={self.config.calib or '(none)'}"
)
def _on_stop(self) -> None:
if not self._running:
return
self._running = False
self.timer.stop()
self.btn_stop.setText("停止中...")
self.btn_stop.setEnabled(False)
QApplication.processEvents()
self.mesh_viewport.close_renderer()
self.pipeline.close()
self.btn_start.setEnabled(True)
self.btn_stop.setText("暂停")
self.btn_reset.setEnabled(False)
print("⏸ 已暂停")
def _on_reset(self) -> None:
if not self._running:
print("请先启动传感器后再重置追踪器。")
return
self.pipeline.reset()
print("追踪器已重置。")
def _on_timer(self) -> None:
if not self._running:
return
frame = self.pipeline.tick()
if frame is None:
print("采集结束或读取失败。")
self._on_stop()
return
flow_mag = 0.0
if frame.flow.size:
flow_mag = float((frame.flow[:, :, 0] ** 2 + frame.flow[:, :, 1] ** 2).mean() ** 0.5)
print(
f"FPS={frame.fps:.2f} flow_rms={flow_mag:.2f} "
f"mask={frame.mask_coverage:.2f} normal={frame.fnormal:.2f} "
f"shear=({frame.fshearx:.2f}, {frame.fsheary:.2f})",
end="\r",
)
self.mesh_viewport.update_frame(frame)
self.arrow_panel.update_frame(frame)
self.force_charts.update_frame(frame)
def _select_camera(self) -> None:
path, _ = QFileDialog.getOpenFileName(
self,
"选择视频文件",
"data",
"视频 (*.mp4 *.avi *.mov);;所有文件 (*)",
)
if path:
self.camera_edit.setText(path)
def _select_config(self) -> None:
path, _ = QFileDialog.getOpenFileName(
self,
"选择配置文件",
"config",
"JSON (*.json);;所有文件 (*)",
)
if path:
self.config_edit.setText(path)
def keyPressEvent(self, event):
key = event.key()
if key == Qt.Key_Escape:
if self.isFullScreen():
self.showNormal()
else:
self.showFullScreen()
elif key == Qt.Key_Q:
print("\n正在退出...")
self.close()
elif key == Qt.Key_S:
if self.pipeline.start_recording():
print("开始录像。")
elif key == Qt.Key_E:
self.pipeline.stop_recording()
print("录像结束。")
elif key == Qt.Key_R:
self._on_reset()
else:
super().keyPressEvent(event)
def closeEvent(self, event):
self.timer.stop()
self.mesh_viewport.close_renderer()
self.pipeline.close()
event.accept()
def run_dashboard(config: PipelineConfig, *, layout: DashboardLayout = LAYOUT) -> int:
"""Launch the unified finger dashboard and block until the window closes."""
from .app import get_or_create_qapp
print("\n按键说明:Q 退出 | R 重置追踪器 | S 开始录像 | E 结束录像 | ESC 全屏")
app = get_or_create_qapp()
window = FingerDashboardWindow(config, layout=layout)
window.show()
QTimer.singleShot(0, window._apply_panel_layout)
return app.exec_()
+141
View File
@@ -0,0 +1,141 @@
"""
Finger dashboard layout — edit this file to define the panel hierarchy and sizes.
Structure:
┌─ toolbar ─────────────────────────────────────────────┐
├─ TOP (stretch: top_stretch) ──────────────────────────┤
│ ┌ mesh ─┬─ arrows ─┬─ charts ─┐ (left → right) │
│ └───────┴───────────┴─────────┘ │
├─ LOG (stretch: log_stretch) ──────────────────────────┤
└───────────────────────────────────────────────────────┘
Each panel uses explicit width_px and height_px.
top_stretch / log_stretch control how extra vertical space is shared on resize.
Edit TOP_ROW.slots to reorder or resize panels.
Edit PANEL_SETUP to change top vs log proportions.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Literal
PanelId = Literal["mesh", "arrows", "charts", "log"]
@dataclass(frozen=True)
class PanelSlot:
"""One panel in the dashboard."""
id: PanelId
title: str
width_px: int
height_px: int
stretch: int = 0
enabled: bool = True
@dataclass(frozen=True)
class PanelRow:
"""Horizontal row of panels (the top visual strip)."""
slots: tuple[PanelSlot, ...]
collapsible: bool = False
@dataclass(frozen=True)
class PanelSetup:
"""Full panel hierarchy: top row + bottom log."""
top_row: PanelRow
log_panel: PanelSlot
top_stretch: int = 5
log_stretch: int = 1
# --- Top strip: mesh | arrows | charts (edit order and sizes here) ---
TOP_ROW = PanelRow(
slots=(
PanelSlot("mesh", "3D mesh", width_px=500, height_px=500),
PanelSlot("arrows", "UV Unrolled", width_px=500, height_px=500),
PanelSlot("charts", "Force charts", width_px=500, height_px=500),
),
)
LOG_PANEL = PanelSlot("log", "Log", width_px=0, height_px=100)
PANEL_SETUP = PanelSetup(
top_row=TOP_ROW,
log_panel=LOG_PANEL,
top_stretch=5,
log_stretch=1,
)
@dataclass(frozen=True)
class DashboardLayout:
"""Window chrome, chart styling, and panel hierarchy."""
panel_setup: PanelSetup = PANEL_SETUP
window_title: str = "Orisys Finger Dashboard"
window_width: int = 1500
window_height: int = 600
root_margin_px: int = 6
root_spacing_px: int = 6
toolbar_spacing_px: int = 8
plot_buffer_points: int = 120
plot_sample_rate_hz: float = 30.0
chart_title_font_pt: int = 11
chart_axis_label_font_pt: int = 9
chart_tick_font_pt: int = 9
y_axis_min_span: float = 20000.0
y_range_smooth_alpha: float = 0.5
curve_color: str = "#00E5FF"
# UV arrow panel: min flow magnitude to draw an arrow (display only; not dd02-ov.json contact.flow_threshold)
arrow_flow_threshold: float = 1.0
timer_interval_ms: int = 10
def slot(self, panel_id: PanelId) -> PanelSlot | None:
if self.panel_setup.log_panel.id == panel_id:
return self.panel_setup.log_panel
for slot in self.panel_setup.top_row.slots:
if slot.id == panel_id:
return slot
return None
def top_slot(self, panel_id: PanelId) -> PanelSlot:
for slot in self.panel_setup.top_row.slots:
if slot.id == panel_id:
return slot
raise KeyError(f"panel {panel_id!r} not in top row")
@property
def mesh_panel_title(self) -> str:
return self.top_slot("mesh").title
@property
def arrows_panel_title(self) -> str:
return self.top_slot("arrows").title
LAYOUT = DashboardLayout()
def top_row_splitter_sizes(layout: DashboardLayout = LAYOUT) -> tuple[int, ...]:
"""Horizontal splitter widths from panel setup."""
return tuple(slot.width_px for slot in layout.panel_setup.top_row.slots if slot.enabled)
def top_row_slot_index(panel_id: PanelId, layout: DashboardLayout = LAYOUT) -> int:
"""Index of a panel in the horizontal top splitter."""
for i, slot in enumerate(layout.panel_setup.top_row.slots):
if slot.id == panel_id:
return i
raise KeyError(f"panel {panel_id!r} not in top row")
+114
View File
@@ -0,0 +1,114 @@
"""Qt stylesheet for the finger dashboard."""
STYLE = """
QMainWindow {
background-color: #000000;
}
QLabel#panelLabel, QLabel#meshViewport {
border: 1px solid #262626;
border-radius: 4px;
background-color: #0b0b0b;
}
QPushButton {
background-color: #1a1a1a;
color: #999999;
border: 1px solid #333333;
border-radius: 4px;
padding: 6px 16px;
font-family: "SimHei";
font-size: 13px;
}
QPushButton:hover {
background-color: #262626;
color: #cccccc;
border: 1px solid #555555;
}
QPushButton#btnStart {
background-color: #0d2818;
color: #4caf84;
border: 1px solid #1a4d30;
}
QPushButton#btnStart:hover {
background-color: #143d24;
color: #6fd4a0;
}
QPushButton#btnStart:disabled {
background-color: #0f0f0f;
color: #333333;
border: 1px solid #222222;
}
QPushButton#btnStop {
background-color: #2b1010;
color: #e05555;
border: 1px solid #4d1a1a;
}
QPushButton#btnStop:hover {
background-color: #3d1818;
color: #f07070;
}
QPushButton#btnStop:disabled {
background-color: #0f0f0f;
color: #333333;
border: 1px solid #222222;
}
QLineEdit {
background-color: #0d0d0d;
color: #999999;
border: 1px solid #333333;
border-radius: 4px;
padding: 5px 10px;
font-size: 13px;
}
GraphicsLayoutWidget {
border: 1px solid #262626;
border-radius: 4px;
}
QPlainTextEdit {
background-color: #0b0b0b;
color: #777777;
border: 1px solid #262626;
border-radius: 4px;
font-family: "Cascadia Code", "Consolas", "Courier New", monospace;
font-size: 13px;
padding: 8px;
selection-background-color: #1a3a4a;
selection-color: #cccccc;
}
QPlainTextEdit QScrollBar:vertical {
background-color: #0b0b0b;
width: 6px;
margin: 1px;
border-radius: 3px;
}
QPlainTextEdit QScrollBar::handle:vertical {
background-color: #262626;
border-radius: 3px;
min-height: 30px;
}
QPlainTextEdit QScrollBar::handle:vertical:hover {
background-color: #3a3a3a;
}
QPlainTextEdit QScrollBar::add-line:vertical,
QPlainTextEdit QScrollBar::sub-line:vertical {
height: 0px;
}
QPlainTextEdit QScrollBar::add-page:vertical,
QPlainTextEdit QScrollBar::sub-page:vertical {
background: none;
}
QSplitter::handle {
background-color: #1a1a1a;
}
QSplitter::handle:vertical {
height: 1px;
}
QSplitter::handle:horizontal {
width: 1px;
}
"""
+20
View File
@@ -0,0 +1,20 @@
"""Qt widgets for the finger dashboard."""
from .force_charts import ForceChartWidget
from .log_panel import TeeStream, WidgetLogHandler, make_log_widget
from .mesh_viewport import MeshViewportWidget
from .arrow_image import ArrowPanelWidget
from .qt_pixmap import ndarray_to_pixmap, pixmap_keep_aspect
from .image_panel import ImagePanel
__all__ = [
"ArrowPanelWidget",
"ForceChartWidget",
"MeshViewportWidget",
"ImagePanel",
"TeeStream",
"WidgetLogHandler",
"make_log_widget",
"ndarray_to_pixmap",
"pixmap_keep_aspect",
]
+44
View File
@@ -0,0 +1,44 @@
"""UV flow-arrow panel for the Qt dashboard."""
from __future__ import annotations
import cv2
from pipeline.frame import FingerFrameView
from viz.render_2d import render_uv_arrows
from ..layout import DashboardLayout, LAYOUT
from .image_panel import ImagePanel
from .qt_pixmap import ndarray_to_pixmap
class ArrowPanelWidget(ImagePanel):
"""UV unrolled flow-arrow panel."""
def __init__(
self,
*,
flow_threshold: float = 0.5,
layout: DashboardLayout = LAYOUT,
parent=None,
):
slot = layout.top_slot("arrows")
super().__init__(
title=layout.arrows_panel_title,
object_name="panelLabel",
default_side=min(slot.width_px, slot.height_px),
parent=parent,
)
self.flow_threshold = flow_threshold
def set_flow_threshold(self, threshold: float) -> None:
self.flow_threshold = threshold
def update_frame(self, frame: FingerFrameView) -> None:
arrows = render_uv_arrows(
frame.unrolled,
frame.vnormal,
flow_threshold=self.flow_threshold,
)
arrows = cv2.rotate(arrows, cv2.ROTATE_180)
self.set_source_pixmap(ndarray_to_pixmap(arrows, bgr=True))
+202
View File
@@ -0,0 +1,202 @@
"""Embedded PyQtGraph force charts for the Qt dashboard."""
from __future__ import annotations
import numpy as np
from PyQt5.QtGui import QFont
from PyQt5.QtWidgets import QSizePolicy
from gui.layout import DashboardLayout, LAYOUT
from pipeline.frame import FingerFrameView
from viz.signal_filters import lowpass_filter
def _require_pyqtgraph():
try:
import pyqtgraph as pg
except ImportError as exc:
raise ImportError(
"PyQtGraph is not installed. Install with: pip install pyqt5 pyqtgraph"
) from exc
return pg
class ForceChartWidget:
"""Rolling normal/shear force charts as an embeddable QWidget."""
def __init__(
self,
*,
layout: DashboardLayout = LAYOUT,
lowpass_cutoff_hz: float = 5.0,
parent=None,
):
pg = _require_pyqtgraph()
pg.setConfigOption("background", "#000000")
pg.setConfigOption("foreground", "#777777")
pg.setConfigOptions(antialias=True)
self._layout = layout
self._pg = pg
self.npoints = layout.plot_buffer_points
self.sample_rate_hz = layout.plot_sample_rate_hz
self.lowpass_cutoff_hz = lowpass_cutoff_hz
self._plot_window_sec = self.npoints / self.sample_rate_hz
self._plot_sample_count = 0
self._y_range_alpha = layout.y_range_smooth_alpha
self._y_range_smooth: list[tuple[float, float] | None] = [None, None, None]
self.widget = pg.GraphicsLayoutWidget(parent=parent)
charts = layout.top_slot("charts")
self.set_panel_size(charts.width_px, charts.height_px)
self._init_plot_buffer()
self._create_plots()
def set_panel_size(self, width: int, height: int) -> None:
"""Apply layout width/height; height 0 means fill top-row height."""
width = max(int(width), 64)
self.widget.setFixedWidth(width)
if height > 0:
self.widget.setFixedHeight(height)
self.widget.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
else:
self.widget.setMaximumHeight(16777215)
self.widget.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Expanding)
def _init_plot_buffer(self) -> None:
self.data = np.full((self.npoints, 2, 3), np.nan, dtype=np.float64)
self._plot_sample_count = 0
self._y_range_smooth = [None, None, None]
def _create_plots(self) -> None:
pg = self._pg
layout = self._layout
titles = ["法向力", "切向力 X", "切向力 Y"]
self.plots = []
self.curves = []
self.dot_glows = []
self.dot_cores = []
axis_pen = pg.mkPen(color=(38, 38, 38), width=1)
label_font = QFont("SimHei", layout.chart_axis_label_font_pt)
title_pt = layout.chart_title_font_pt
for i, title in enumerate(titles):
plot = self.widget.addPlot(title=None)
plot.setTitle(title, color="#999999", size=f"{title_pt}pt")
plot.titleLabel.setFont(QFont("SimHei", title_pt))
plot.setLabel("bottom", "时间", units="s")
plot.getAxis("bottom").label.setFont(label_font)
plot.showAxis("top", False)
plot.showAxis("right", False)
plot.getAxis("left").setPen(axis_pen)
plot.getAxis("bottom").setPen(axis_pen)
plot.showGrid(x=False, y=True, alpha=0.12)
tick_font = QFont("Consolas", layout.chart_tick_font_pt)
tick_font.setStyleHint(QFont.Monospace)
plot.getAxis("left").setTickFont(tick_font)
plot.getAxis("bottom").setTickFont(tick_font)
line_pen = pg.mkPen(color=layout.curve_color, width=2)
curve = plot.plot(pen=line_pen, fillLevel=0)
dot_glow = plot.plot(
[],
[],
pen=None,
symbol="o",
symbolSize=8,
symbolPen=None,
symbolBrush=(0, 229, 255, 76),
)
dot_core = plot.plot(
[],
[],
pen=None,
symbol="o",
symbolSize=4,
symbolPen=None,
symbolBrush=(255, 255, 255, 255),
)
plot.setXRange(0, self._plot_window_sec, padding=0)
plot.enableAutoRange(x=False, y=False)
self.plots.append(plot)
self.curves.append(curve)
self.dot_glows.append(dot_glow)
self.dot_cores.append(dot_core)
if i < 2:
self.widget.nextRow()
def reset(self) -> None:
self._init_plot_buffer()
for plot in self.plots:
plot.setXRange(0, self._plot_window_sec, padding=0)
def update_frame(self, frame: FingerFrameView) -> None:
if self._plot_sample_count < self.npoints:
idx = self._plot_sample_count
self.data[idx, 0, :] = frame.timestamp
self.data[idx, 1, 0] = frame.fnormal
self.data[idx, 1, 1] = frame.fshearx
self.data[idx, 1, 2] = frame.fsheary
self._plot_sample_count += 1
else:
for channel in range(3):
self.data[:-1, 0, channel] = self.data[1:, 0, channel]
self.data[:-1, 1, channel] = self.data[1:, 1, channel]
self.data[-1, 0, :] = frame.timestamp
self.data[-1, 1, 0] = frame.fnormal
self.data[-1, 1, 1] = frame.fshearx
self.data[-1, 1, 2] = frame.fsheary
n = self._plot_sample_count
if n == 0:
return
layout = self._layout
data_plot = self.data[:n].copy()
times = data_plot[:, 0, 0] - float(data_plot[0, 0, 0])
for channel in range(3):
data_plot[:, 1, channel] = lowpass_filter(
data_plot[:, 1, channel],
self.lowpass_cutoff_hz,
sample_rate_hz=self.sample_rate_hz,
)
x_end = float(times[-1])
x_start = max(0.0, x_end - self._plot_window_sec)
if x_end - x_start < self._plot_window_sec:
x_start = 0.0
x_end = max(self._plot_window_sec, x_end)
for i, curve in enumerate(self.curves):
y = data_plot[:, 1, i]
curve.setData(times, y)
finite = np.isfinite(y)
if not np.any(finite):
continue
ymin, ymax = float(np.min(y[finite])), float(np.max(y[finite]))
if self._y_range_smooth[i] is None:
self._y_range_smooth[i] = (ymin, ymax)
else:
prev_min, prev_max = self._y_range_smooth[i]
alpha = self._y_range_alpha
ymin = alpha * ymin + (1 - alpha) * prev_min
ymax = alpha * ymax + (1 - alpha) * prev_max
span = ymax - ymin
if span < layout.y_axis_min_span:
mid = (ymin + ymax) / 2
half = layout.y_axis_min_span / 2
ymin = mid - half
ymax = mid + half
self._y_range_smooth[i] = (ymin, ymax)
pad = max((ymax - ymin) * 0.12, 0.01)
self.plots[i].setYRange(ymin - pad, ymax + pad)
last_x = times[-1:]
last_y = y[-1:]
self.dot_glows[i].setData(last_x, last_y)
self.dot_cores[i].setData(last_x, last_y)
for plot in self.plots:
plot.setXRange(x_start, x_end, padding=0)
+71
View File
@@ -0,0 +1,71 @@
"""Square 1:1 image panel host for dashboard viewports."""
from __future__ import annotations
from PyQt5.QtCore import Qt
from PyQt5.QtGui import QPixmap
from PyQt5.QtWidgets import QLabel, QSizePolicy, QVBoxLayout, QWidget
from .qt_pixmap import pixmap_keep_aspect
class ImagePanel(QWidget):
"""Fixed square widget that displays an image at 1:1 aspect ratio."""
def __init__(
self,
*,
title: str = "",
object_name: str = "panelLabel",
default_side: int = 256,
min_side: int = 64,
parent=None,
):
super().__init__(parent)
self._side = default_side
self._min_side = min_side
self._source_pixmap: QPixmap | None = None
self.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
layout = QVBoxLayout(self)
layout.setContentsMargins(0, 0, 0, 0)
self.label = QLabel(title)
self.label.setObjectName(object_name)
self.label.setAlignment(Qt.AlignCenter)
layout.addWidget(self.label)
self.set_side_length(self._side)
def set_side_length(self, side: int) -> None:
self.set_fixed_size(side, side)
def set_fixed_size(self, width: int, height: int) -> None:
width = max(int(width), self._min_side)
height = max(int(height), self._min_side)
if width == self.width() and height == self.height() and width == self._side:
return
self._side = width
self.setFixedSize(width, height)
self.label.setFixedSize(width, height)
self.on_side_length_changed(min(width, height))
self._refresh_pixmap()
@property
def side_length(self) -> int:
return self._side
def on_side_length_changed(self, side: int) -> None:
"""Hook for subclasses (e.g. resize an offscreen renderer)."""
def set_source_pixmap(self, pixmap: QPixmap | None) -> None:
self._source_pixmap = pixmap
self._refresh_pixmap()
def _refresh_pixmap(self) -> None:
if self._source_pixmap is None or self._source_pixmap.isNull():
return
self.label.setPixmap(pixmap_keep_aspect(self._source_pixmap, self.label.size()))
def resizeEvent(self, event):
super().resizeEvent(event)
self._refresh_pixmap()
+51
View File
@@ -0,0 +1,51 @@
"""Logging helpers for the finger dashboard."""
from __future__ import annotations
from PyQt5.QtGui import QFont
from PyQt5.QtWidgets import QPlainTextEdit
class TeeStream:
"""Write to multiple text streams."""
def __init__(self, *streams):
self.streams = streams
def write(self, text):
for stream in self.streams:
stream.write(text)
def flush(self):
for stream in self.streams:
stream.flush()
class WidgetLogHandler:
"""Mirror print output into a QPlainTextEdit."""
def __init__(self, widget: QPlainTextEdit):
self.widget = widget
self.buf = ""
def write(self, text: str):
self.buf += text
while "\n" in self.buf:
line, self.buf = self.buf.split("\n", 1)
if line:
self.widget.appendPlainText(line)
def flush(self):
if self.buf:
self.widget.appendPlainText(self.buf)
self.buf = ""
def make_log_widget() -> QPlainTextEdit:
widget = QPlainTextEdit()
widget.setReadOnly(True)
widget.setMaximumBlockCount(2000)
font = QFont("Consolas", 12)
font.setStyleHint(QFont.Monospace)
widget.setFont(font)
return widget
+79
View File
@@ -0,0 +1,79 @@
"""Embedded offscreen 3D mesh viewport for the Qt dashboard."""
from __future__ import annotations
from pipeline.frame import FingerFrameView
from viz.offscreen_mesh import OffscreenFingerMeshRenderer
from ..layout import DashboardLayout, LAYOUT
from .qt_pixmap import ndarray_to_pixmap
from .image_panel import ImagePanel
class MeshViewportWidget(ImagePanel):
"""Render the finger mesh offscreen in a square 1:1 viewport."""
def __init__(self, parent=None, *, layout: DashboardLayout = LAYOUT):
self._layout = layout
self._renderer: OffscreenFingerMeshRenderer | None = None
self._mesh_map = "depth"
self._threshold = 0.0
self._enable_3d = True
slot = layout.top_slot("mesh")
super().__init__(
title=layout.mesh_panel_title,
object_name="meshViewport",
default_side=min(slot.width_px, slot.height_px),
parent=parent,
)
def configure(self, pipeline) -> None:
self._mesh_map = pipeline.config.mesh_map
self._threshold = pipeline.config.flow_threshold
self._enable_3d = pipeline.config.enable_3d and pipeline.uv_mesh is not None
if self._renderer is not None:
self._renderer.close()
self._renderer = None
if not self._enable_3d:
self.label.setText(
"3D mesh disabled" if not pipeline.config.enable_3d else "3D mesh unavailable"
)
self.set_source_pixmap(None)
return
self.label.setText("")
self._init_renderer(pipeline)
def _init_renderer(self, pipeline) -> None:
if not self._enable_3d or pipeline.uv_mesh is None:
return
side = self.side_length
self._renderer = OffscreenFingerMeshRenderer(
pipeline.uv_mesh,
width=side,
height=side,
threshold=self._threshold,
color_mode=self._mesh_map,
world_to_cam=pipeline.world_to_cam,
lookat=pipeline.mesh_lookat,
)
def on_side_length_changed(self, side: int) -> None:
if self._renderer is not None:
self._renderer.resize(side, side)
def update_frame(self, frame: FingerFrameView) -> None:
if not self._enable_3d or self._renderer is None:
return
rgb = self._renderer.render(
flow=frame.flow,
scalar_map=frame.depth_map,
centroid=frame.centroid,
centroid_found=frame.centroid_found,
fnormal=frame.fnormal,
)
self.set_source_pixmap(ndarray_to_pixmap(rgb, bgr=False))
def close_renderer(self) -> None:
if self._renderer is not None:
self._renderer.close()
self._renderer = None
+32
View File
@@ -0,0 +1,32 @@
"""Qt pixmap conversion helpers (no widget dependencies)."""
from __future__ import annotations
import cv2
import numpy as np
from PyQt5.QtCore import Qt
from PyQt5.QtGui import QImage, QPixmap
def ndarray_to_pixmap(image: np.ndarray, *, bgr: bool = True) -> QPixmap:
"""Convert a numpy image to QPixmap for QLabel display."""
arr = np.ascontiguousarray(image)
if arr.ndim == 2:
arr = cv2.cvtColor(arr, cv2.COLOR_GRAY2BGR)
bgr = True
if arr.ndim == 3 and arr.shape[2] == 4:
arr = arr[:, :, :3]
if bgr:
rgb = cv2.cvtColor(arr, cv2.COLOR_BGR2RGB)
else:
rgb = arr
h, w, ch = rgb.shape
qimg = QImage(rgb.data, w, h, ch * w, QImage.Format_RGB888)
return QPixmap.fromImage(qimg.copy())
def pixmap_keep_aspect(pixmap: QPixmap, size) -> QPixmap:
"""Scale a pixmap to fit ``size`` without changing aspect ratio."""
if size.width() <= 0 or size.height() <= 0:
return pixmap
return pixmap.scaled(size, Qt.KeepAspectRatio, Qt.SmoothTransformation)
+12
View File
@@ -0,0 +1,12 @@
"""Finger example pipeline: runtime acquisition without GUI dependencies."""
from .config import PipelineConfig, mesh_threshold
from .frame import FingerFrameView
from .service import FingerPipeline
__all__ = [
"FingerFrameView",
"FingerPipeline",
"PipelineConfig",
"mesh_threshold",
]
+46
View File
@@ -0,0 +1,46 @@
"""Configuration for the finger tactile example pipeline."""
from __future__ import annotations
from dataclasses import dataclass
from viz.mesh_camera import DEFAULT_FINGER_VIZ_POSE
def mesh_threshold(mesh_map: str, threshold: float | None) -> float:
if threshold is not None:
return threshold
return 0.0 if mesh_map == "depth" else 0.4
@dataclass
class PipelineConfig:
video: str = "0"
config: str = "./config/dd02-ov.json"
calib: str | None = None
verbose: bool = False
mesh_map: str = "depth"
threshold: float | None = None
viz_pose: str = DEFAULT_FINGER_VIZ_POSE
enable_3d: bool = True
use_color: bool = True
lowpass_cutoff_hz: float = 5.0
force_sample_rate_hz: float = 30.0
@classmethod
def from_args(cls, args) -> "PipelineConfig":
return cls(
video=args.video,
config=args.config,
calib=getattr(args, "cal", None) or None,
verbose=args.verbose,
mesh_map=args.mesh_map,
threshold=args.threshold,
viz_pose=args.viz_pose,
enable_3d=not args.no_3d,
lowpass_cutoff_hz=getattr(args, "cutoff", 5.0),
)
@property
def flow_threshold(self) -> float:
return mesh_threshold(self.mesh_map, self.threshold)
+26
View File
@@ -0,0 +1,26 @@
"""Display snapshot produced by one finger pipeline tick."""
from __future__ import annotations
from dataclasses import dataclass
import numpy as np
@dataclass
class FingerFrameView:
timestamp: float
fps: float
unrolled: np.ndarray
flow: np.ndarray
vnormal: np.ndarray
depth_map: np.ndarray
img: np.ndarray
contour: np.ndarray
centroid: np.ndarray
fnormal: float
fshearx: float
fsheary: float
centroid_found: bool
mask_coverage: float
raw_frame: np.ndarray | None = None
+123
View File
@@ -0,0 +1,123 @@
"""Finger tactile acquisition loop for example dashboards."""
from __future__ import annotations
import time
import numpy as np
from orisys.finger import FingerRuntime
from viz import FrameRecorder, build_uv_surface_mesh, resolve_viz_pose_path
from .config import PipelineConfig
from .frame import FingerFrameView
class FingerPipeline:
"""Owns FingerRuntime and produces FingerFrameView snapshots per tick."""
def __init__(self, config: PipelineConfig):
self.config = config
self.runtime: FingerRuntime | None = None
self.recorder = FrameRecorder()
self._start_time: float | None = None
self.uv_mesh = None
self.world_to_cam: np.ndarray | None = None
self.mesh_lookat: np.ndarray | None = None
@property
def is_open(self) -> bool:
return self.runtime is not None
@property
def tactile(self):
if self.runtime is None:
raise RuntimeError("FingerPipeline is not open")
return self.runtime.tactile
def open(self) -> None:
if self.runtime is not None:
return
self.runtime = FingerRuntime.open(
config=self.config.config,
calib=self.config.calib,
video=self.config.video,
verbose=self.config.verbose,
)
self._prepare_mesh_assets()
self.runtime.warmup(color=self.config.use_color)
self._start_time = time.time()
def _prepare_mesh_assets(self) -> None:
if self.runtime is None:
return
domain = self.runtime.domain
if domain.surface is None or getattr(domain.surface, "world_xyz", None) is None:
return
self.uv_mesh = build_uv_surface_mesh(domain.surface.world_xyz, domain.valid_mask)
if self.config.viz_pose.lower() != "none":
self.world_to_cam = np.load(resolve_viz_pose_path(self.config.viz_pose))
self.mesh_lookat = self.uv_mesh.rest_vertices.mean(axis=0)
def close(self) -> None:
if self.runtime is not None:
self.runtime.close()
self.runtime = None
self._start_time = None
def reset(self) -> None:
if self.runtime is not None:
self.runtime.tactile.reset()
def tick(self) -> FingerFrameView | None:
if self.runtime is None or self._start_time is None:
return None
ok, unrolled = self.runtime.grab_and_process(color=self.config.use_color)
if not ok:
return None
tactile = self.runtime.tactile
self.recorder.write_if_recording(tactile.frame)
tactile.compute_deformation(decompose=True, check_motion=True, threshold=0)
tactile.compute_contact()
fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = tactile.read_info(
tactile.info.FPS,
tactile.info.VRAW,
tactile.info.VNORMAL,
tactile.info.IMG,
tactile.info.CONTOUR,
tactile.info.CENTROID,
tactile.info.DEPTH,
tactile.info.FNORMAL,
tactile.info.FSHEARX,
tactile.info.FSHEARY,
)
return FingerFrameView(
timestamp=time.time() - self._start_time,
fps=float(fps),
unrolled=unrolled,
flow=flow,
vnormal=vnormal,
depth_map=depth_map,
img=img,
contour=contour,
centroid=centroid,
fnormal=float(fnormal),
fshearx=float(fshearx),
fsheary=float(fsheary),
centroid_found=bool(tactile.raw_sensor.centroid_found),
mask_coverage=float(tactile.mask_stats["coverage"]),
raw_frame=tactile.frame,
)
def start_recording(self) -> bool:
if self.runtime is None:
return False
return self.recorder.start(self.runtime.tactile.frame)
def stop_recording(self) -> None:
self.recorder.stop()
+20 -2
View File
@@ -1,20 +1,38 @@
"""Shared visualization helpers for Orisys example scripts.""" """Shared visualization helpers for Orisys example scripts."""
from .finger_panels import show_finger_uv_panels
from .force_charts import ForceChartWindow from .force_charts import ForceChartWindow
from .mesh_camera import (
DEFAULT_FINGER_VIZ_POSE,
apply_mesh_lighting,
apply_view_from_world_to_cam,
camera_look_at_from_world_to_cam,
resolve_viz_pose_path,
)
from .offscreen_mesh import OffscreenFingerMeshRenderer
from .render_2d import render_contact_panel, render_depth_panel, render_uv_arrows
from .finger_mesh_viewer import FingerMeshViewer
from .mesh_viewer import UVMeshViewer, build_uv_surface_mesh from .mesh_viewer import UVMeshViewer, build_uv_surface_mesh
from .opencv_maps import depth_colormap, draw_magnitude_map, frame_to_bgr from .opencv_maps import depth_colormap, draw_magnitude_map, frame_to_bgr
from .recording import FrameRecorder from .recording import FrameRecorder
from .signal_filters import lowpass_filter from .signal_filters import lowpass_filter
__all__ = [ __all__ = [
"DEFAULT_FINGER_VIZ_POSE",
"FingerMeshViewer",
"ForceChartWindow", "ForceChartWindow",
"FrameRecorder", "FrameRecorder",
"UVMeshViewer", "UVMeshViewer",
"apply_mesh_lighting",
"apply_view_from_world_to_cam",
"build_uv_surface_mesh", "build_uv_surface_mesh",
"camera_look_at_from_world_to_cam",
"depth_colormap", "depth_colormap",
"draw_magnitude_map", "draw_magnitude_map",
"frame_to_bgr", "frame_to_bgr",
"lowpass_filter", "lowpass_filter",
"show_finger_uv_panels", "OffscreenFingerMeshRenderer",
"render_contact_panel",
"render_depth_panel",
"render_uv_arrows",
"resolve_viz_pose_path",
] ]
+223
View File
@@ -0,0 +1,223 @@
"""Contact normal-force arrow overlay for finger 3D mesh examples."""
from __future__ import annotations
import numpy as np
from orisys.finger.mesh import UVSurfaceMesh, compute_uv_surface_frames, mesh_uv_to_world_scale
DEFAULT_NORMAL_FORCE_SCALE = 1.0
MIN_ARROW_LENGTH_FRACTION = 0.02
MAX_ARROW_LENGTH_FRACTION = 0.30
REFERENCE_NORMAL_FORCE = 10000.0
MIN_ARROW_THICKNESS_SCALE = 0.20
MAX_ARROW_THICKNESS_SCALE = 1.0
CONTACT_ARROW_COLOR = (1.0, 0.92, 0.10)
def _mesh_span(vertices: np.ndarray) -> float:
extent = np.ptp(vertices, axis=0)
return float(max(np.max(extent), 1e-6))
def _rotation_matrix_z_to(direction: np.ndarray) -> np.ndarray:
"""Rotation matrix mapping unit +Z to ``direction``."""
direction = np.asarray(direction, dtype=np.float64).reshape(3)
norm = np.linalg.norm(direction)
if norm < 1e-8:
return np.eye(3, dtype=np.float64)
direction = direction / norm
z_axis = np.array([0.0, 0.0, 1.0], dtype=np.float64)
cross = np.cross(z_axis, direction)
dot = float(np.dot(z_axis, direction))
if dot > 0.9999:
return np.eye(3, dtype=np.float64)
if dot < -0.9999:
return np.diag([1.0, -1.0, -1.0]).astype(np.float64)
skew = np.array(
[
[0.0, -cross[2], cross[1]],
[cross[2], 0.0, -cross[0]],
[-cross[1], cross[0], 0.0],
],
dtype=np.float64,
)
return np.eye(3, dtype=np.float64) + skew + skew @ skew * (1.0 / (1.0 + dot))
def thickness_scale_from_force(force: float) -> float:
"""Map normal force to a radial scale factor (thin at low force, thicker at high force)."""
if force <= 0.0:
return 0.0
t = force / REFERENCE_NORMAL_FORCE
return float(
np.clip(
MIN_ARROW_THICKNESS_SCALE + (MAX_ARROW_THICKNESS_SCALE - MIN_ARROW_THICKNESS_SCALE) * t,
MIN_ARROW_THICKNESS_SCALE,
MAX_ARROW_THICKNESS_SCALE,
)
)
def _scale_arrow_template(
template_vertices: np.ndarray,
*,
length_scale: float,
thickness_scale: float,
) -> np.ndarray:
"""Scale a +Z arrow template axially and radially."""
scaled = np.asarray(template_vertices, dtype=np.float64).copy()
scaled[:, 0] *= thickness_scale
scaled[:, 1] *= thickness_scale
scaled[:, 2] *= length_scale
return scaled
def create_arrow_template(o3d):
"""Unit-height arrow mesh along +Z; radial size is scaled per-frame from force."""
arrow = o3d.geometry.TriangleMesh.create_arrow(
cylinder_radius=0.03,
cone_radius=0.06,
cylinder_height=0.72,
cone_height=0.28,
)
arrow.compute_vertex_normals()
return (
np.asarray(arrow.vertices, dtype=np.float64),
np.asarray(arrow.triangles, dtype=np.int32),
)
def build_contact_normal_arrow_mesh(
origin: np.ndarray,
normal: np.ndarray,
length: float,
force: float,
*,
mesh_span: float,
template_vertices: np.ndarray,
template_triangles: np.ndarray,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Build a yellow 3D arrow mesh pointing opposite the surface normal."""
origin = np.asarray(origin, dtype=np.float64).reshape(3)
normal = np.asarray(normal, dtype=np.float64).reshape(3)
norm = np.linalg.norm(normal)
color = np.tile(np.asarray(CONTACT_ARROW_COLOR, dtype=np.float64), (len(template_vertices), 1))
radial_scale = thickness_scale_from_force(force)
if norm < 1e-8 or length <= 0.0 or radial_scale <= 0.0 or not np.all(np.isfinite(origin)):
return np.zeros_like(template_vertices), template_triangles, color
direction = -normal / norm
length = float(np.clip(length, mesh_span * MIN_ARROW_LENGTH_FRACTION, mesh_span * MAX_ARROW_LENGTH_FRACTION))
if length <= 0.0:
return np.zeros_like(template_vertices), template_triangles, color
rotation = _rotation_matrix_z_to(direction)
scaled_template = _scale_arrow_template(
template_vertices,
length_scale=length,
thickness_scale=radial_scale,
)
vertices = scaled_template @ rotation.T + origin
return vertices, template_triangles, color
def create_arrow_mesh_geometry(o3d, template_vertices: np.ndarray, template_triangles: np.ndarray):
"""Create a persistent Open3D mesh for the contact arrow overlay."""
arrow_mesh = o3d.geometry.TriangleMesh()
arrow_mesh.vertices = o3d.utility.Vector3dVector(np.zeros_like(template_vertices))
arrow_mesh.triangles = o3d.utility.Vector3iVector(template_triangles)
arrow_mesh.vertex_colors = o3d.utility.Vector3dVector(
np.tile(np.asarray(CONTACT_ARROW_COLOR, dtype=np.float64), (len(template_vertices), 1))
)
arrow_mesh.compute_vertex_normals()
return arrow_mesh
class ContactArrowOverlay:
"""Example-layer contact arrow driven by centroid and normal force."""
def __init__(
self,
mesh: UVSurfaceMesh,
*,
normal_force_scale: float = DEFAULT_NORMAL_FORCE_SCALE,
enabled: bool = True,
):
import open3d as o3d
self.mesh = mesh
self.normal_force_scale = float(normal_force_scale)
self.enabled = enabled
self._mesh_span = _mesh_span(mesh.rest_vertices)
self._pixel_to_world = mesh_uv_to_world_scale(mesh.world_xyz, mesh.valid_mask)
_, _, self._surface_normals = compute_uv_surface_frames(mesh.world_xyz, mesh.valid_mask)
self._template_vertices, self._template_triangles = (
create_arrow_template(o3d) if enabled else (None, None)
)
self.arrow_mesh = (
create_arrow_mesh_geometry(o3d, self._template_vertices, self._template_triangles)
if enabled
else None
)
def compute(
self,
centroid,
centroid_found: bool,
fnormal: float,
) -> tuple[np.ndarray, np.ndarray, np.ndarray] | None:
if not self.enabled or self.arrow_mesh is None:
return None
template_vertices = self._template_vertices
template_triangles = self._template_triangles
color = np.tile(
np.asarray(CONTACT_ARROW_COLOR, dtype=np.float64),
(len(template_vertices), 1),
)
hidden = (np.zeros_like(template_vertices), template_triangles, color)
if not centroid_found or centroid is None:
return hidden
centroid = np.asarray(centroid, dtype=np.float64).reshape(-1)
if centroid.size < 2:
return hidden
cx, cy = int(round(centroid[0])), int(round(centroid[1]))
h, w = self.mesh.shape
if not (0 <= cy < h and 0 <= cx < w):
return hidden
if self.mesh.index_map[cy, cx] < 0:
return hidden
origin = self.mesh.world_xyz[cy, cx]
normal = self._surface_normals[cy, cx]
if not np.all(np.isfinite(origin)) or not np.all(np.isfinite(normal)):
return hidden
force = max(float(fnormal), 0.0)
if force <= 0.0:
return hidden
length = force * self.normal_force_scale * self._pixel_to_world
return build_contact_normal_arrow_mesh(
origin,
normal,
length,
force,
mesh_span=self._mesh_span,
template_vertices=template_vertices,
template_triangles=template_triangles,
)
def apply(self, arrow_vertices: np.ndarray, arrow_colors: np.ndarray) -> None:
import open3d as o3d
self.arrow_mesh.vertices = o3d.utility.Vector3dVector(arrow_vertices.astype(np.float64))
self.arrow_mesh.vertex_colors = o3d.utility.Vector3dVector(arrow_colors.astype(np.float64))
self.arrow_mesh.compute_vertex_normals()
+66
View File
@@ -0,0 +1,66 @@
"""Finger 3D mesh viewer with example-only contact arrow overlay."""
from __future__ import annotations
from orisys.finger.viewer import UVMeshViewer
from .contact_arrow import ContactArrowOverlay, DEFAULT_NORMAL_FORCE_SCALE
class FingerMeshViewer(UVMeshViewer):
"""UV mesh viewer with a force-scaled contact normal arrow."""
def __init__(
self,
mesh,
*,
max_magnitude=None,
threshold: float = 0.0,
color_mode: str = "flow",
value_min=None,
value_max=None,
window_width: int = 1024,
window_height: int = 1024,
normal_force_scale: float = DEFAULT_NORMAL_FORCE_SCALE,
show_contact_arrow: bool = True,
):
super().__init__(
mesh,
max_magnitude=max_magnitude,
threshold=threshold,
color_mode=color_mode,
value_min=value_min,
value_max=value_max,
window_width=window_width,
window_height=window_height,
)
self._contact_arrow = ContactArrowOverlay(
mesh,
normal_force_scale=normal_force_scale,
enabled=show_contact_arrow,
)
def _extra_geometries_for_init(self):
if self._contact_arrow.arrow_mesh is None:
return []
return [self._contact_arrow.arrow_mesh]
def _extra_geometries_for_update(self):
if self._contact_arrow.arrow_mesh is None:
return []
return [self._contact_arrow.arrow_mesh]
def update(
self,
flow=None,
scalar_map=None,
*,
centroid=None,
centroid_found: bool = False,
fnormal: float = 0.0,
):
arrow = self._contact_arrow.compute(centroid, centroid_found, fnormal)
if arrow is not None and self._contact_arrow.arrow_mesh is not None:
arrow_vertices, _, arrow_colors = arrow
self._contact_arrow.apply(arrow_vertices, arrow_colors)
return super().update(flow=flow, scalar_map=scalar_map)
-29
View File
@@ -1,29 +0,0 @@
"""Finger-specific OpenCV panels for example scripts."""
from __future__ import annotations
import cv2
import orisys
from .opencv_maps import depth_colormap
def show_finger_uv_panels(
unrolled,
vnormal,
depth_map,
*,
flow_threshold: float = 0.4,
grid_spacing: int = 20,
arrow_scale: float = 1.0,
) -> None:
"""Show depth colormap and UV unrolled flow arrows."""
cv2.imshow("Depth", depth_colormap(depth_map))
arrows = orisys.util.draw_arrows(
unrolled,
vnormal,
threshold=flow_threshold,
grid_spacing=grid_spacing,
arrow_scale=arrow_scale,
)
cv2.imshow("UV Unrolled", arrows)
+4 -3
View File
@@ -27,8 +27,8 @@ class ForceChartWindow:
app_name: str = "Orisys Plot Example", app_name: str = "Orisys Plot Example",
) -> None: ) -> None:
pg = _require_pyqtgraph() pg = _require_pyqtgraph()
pg.setConfigOption("background", "w") pg.setConfigOption("background", "k")
pg.setConfigOption("foreground", "k") pg.setConfigOption("foreground", "w")
self._pg = pg self._pg = pg
self.npoints = npoints self.npoints = npoints
@@ -36,6 +36,7 @@ class ForceChartWindow:
self.app = pg.mkQApp(app_name) self.app = pg.mkQApp(app_name)
self.win = pg.GraphicsLayoutWidget(show=True, title=title) self.win = pg.GraphicsLayoutWidget(show=True, title=title)
self.win.setWindowTitle(title) self.win.setWindowTitle(title)
self.pen = pg.mkPen(color="w")
self.data = np.zeros((npoints, 2, 3), dtype=np.float64) self.data = np.zeros((npoints, 2, 3), dtype=np.float64)
for channel in range(3): for channel in range(3):
@@ -51,7 +52,7 @@ class ForceChartWindow:
p3.setLabel("bottom", "时间", units="s") p3.setLabel("bottom", "时间", units="s")
self._plots = (p1, p2, p3) self._plots = (p1, p2, p3)
self._curves = (p1.plot(pen="k"), p2.plot(pen="k"), p3.plot(pen="k")) self._curves = (p1.plot(pen=self.pen ), p2.plot(pen=self.pen ), p3.plot(pen=self.pen ))
def push(self, timestamp: float, fn: float, fx: float, fy: float) -> None: def push(self, timestamp: float, fn: float, fx: float, fy: float) -> None:
for channel in range(3): for channel in range(3):
+110
View File
@@ -0,0 +1,110 @@
"""Open3D camera helpers for finger mesh examples."""
from __future__ import annotations
from pathlib import Path
import numpy as np
DEFAULT_FINGER_VIZ_POSE = "assets/finger/visualization/world_to_cam.npy"
_CAMERA_X_ROT_90 = np.array(
[
[1.0, 0.0, 0.0],
[0.0, 0.0, -1.0],
[0.0, -1.0, 0.0],
],
dtype=np.float64,
)
_CAMERA_Z_ROT_180 = np.array(
[
[1.0, 0.0, 0.0],
[0.0, 1.0, -1],
[0.0, 0.0, -1],
],
dtype=np.float64,
)
def resolve_viz_pose_path(path: str | Path | None = None) -> Path:
"""Resolve world_to_cam.npy from cwd or packaged asset layout."""
raw = Path(path or DEFAULT_FINGER_VIZ_POSE).expanduser()
if raw.is_file():
return raw.resolve()
here = Path(__file__).resolve()
candidates = [
Path.cwd() / raw,
here.parents[2] / raw,
]
for candidate in candidates:
if candidate.is_file():
return candidate.resolve()
checked = ", ".join(str(p) for p in candidates)
raise FileNotFoundError(f"Visualization pose file not found: {raw}. Checked: {checked}")
def camera_look_at_from_world_to_cam(
world_to_cam: np.ndarray,
*,
lookat: np.ndarray,
distance_scale: float = 2.5,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Derive Open3D look_at(center, eye, up) from a Blender world_to_cam matrix."""
world_to_cam = np.asarray(world_to_cam, dtype=np.float64)
cam_to_world = np.linalg.inv(world_to_cam)
rotation = cam_to_world[:3, :3] @ _CAMERA_X_ROT_90 @ _CAMERA_Z_ROT_180
front = rotation @ np.array([0.0, 0.0, -1.0])
up = rotation @ np.array([0.0, 1.0, 0.0])
front_norm = np.linalg.norm(front)
up_norm = np.linalg.norm(up)
if front_norm > 0:
front = front / front_norm
if up_norm > 0:
up = up / up_norm
lookat = np.asarray(lookat, dtype=np.float64).reshape(3)
extent = float(np.linalg.norm(lookat - cam_to_world[:3, 3]))
if extent < 1e-6:
extent = 1.0
eye = lookat - front * (extent * distance_scale)
return lookat, eye, up
def apply_view_from_world_to_cam(vis, world_to_cam: np.ndarray, *, lookat: np.ndarray) -> None:
"""Apply Blender-exported world_to_cam as the Open3D default view."""
world_to_cam = np.asarray(world_to_cam, dtype=np.float64)
cam_to_world = np.linalg.inv(world_to_cam)
rotation = cam_to_world[:3, :3] @ _CAMERA_X_ROT_90 @ _CAMERA_Z_ROT_180
front = rotation @ np.array([0.0, 0.0, -1.0])
up = rotation @ np.array([0.0, 1.0, 0.0])
front_norm = np.linalg.norm(front)
up_norm = np.linalg.norm(up)
if front_norm > 0:
front = front / front_norm
if up_norm > 0:
up = up / up_norm
view_control = vis.get_view_control()
view_control.set_lookat(lookat.astype(np.float64))
view_control.set_front(front.astype(np.float64))
view_control.set_up(up.astype(np.float64))
vis.update_renderer()
def apply_mesh_lighting(vis) -> None:
"""Enable shaded lighting so the tactile mesh reads as a 3D surface."""
import open3d as o3d
render_option = vis.get_render_option()
if hasattr(render_option, "light_on"):
render_option.light_on = True
if hasattr(render_option, "mesh_color_option"):
render_option.mesh_color_option = o3d.visualization.MeshColorOption.Color
if hasattr(render_option, "mesh_shade_option"):
render_option.mesh_shade_option = o3d.visualization.MeshShadeOption.Color
vis.update_renderer()
+2 -2
View File
@@ -1,4 +1,4 @@
"""Re-exports for finger 3D mesh examples; implementation lives in orisys.finger.""" """Re-exports for finger 3D mesh examples; core viewer lives in orisys.finger."""
from orisys.finger.mesh import * # noqa: F403 from orisys.finger.mesh import * # noqa: F403
from orisys.finger.viewer import * # noqa: F403 from orisys.finger.viewer import UVMeshViewer, init_uv_mesh_viewer, update_uv_mesh_viewer # noqa: F401
+357
View File
@@ -0,0 +1,357 @@
"""Embedded Open3D mesh renderer for Qt dashboards (offscreen or hidden-window capture)."""
from __future__ import annotations
import sys
import numpy as np
from orisys.finger.mesh import UVSurfaceMesh
from orisys.finger.viewer import flow_magnitude_vertex_colors, scalar_map_vertex_colors
from .contact_arrow import ContactArrowOverlay
from .mesh_camera import apply_view_from_world_to_cam, camera_look_at_from_world_to_cam
def _numpy_o3d_image_to_rgb(image) -> np.ndarray:
arr = np.asarray(image)
if arr.ndim == 3 and arr.shape[2] >= 3:
if arr.dtype == np.float32 or arr.dtype == np.float64:
return (np.clip(arr[:, :, :3], 0.0, 1.0) * 255.0).astype(np.uint8)
return arr[:, :, :3].astype(np.uint8)
return arr.astype(np.uint8)
def _make_unlit_material(o3d):
material = o3d.visualization.rendering.MaterialRecord()
material.shader = "defaultUnlit"
material.base_color = (1.0, 1.0, 1.0, 1.0)
return material
def _mesh_to_o3d(o3d, vertices: np.ndarray, triangles: np.ndarray, colors: np.ndarray):
mesh = o3d.geometry.TriangleMesh()
mesh.vertices = o3d.utility.Vector3dVector(vertices.astype(np.float64))
mesh.triangles = o3d.utility.Vector3iVector(triangles.astype(np.int32))
mesh.vertex_colors = o3d.utility.Vector3dVector(colors.astype(np.float64))
mesh.compute_vertex_normals()
return mesh
def _vertex_colors(mesh, *, color_mode, threshold, value_min, value_max, flow=None, scalar_map=None):
if color_mode == "depth":
return scalar_map_vertex_colors(
scalar_map,
mesh,
value_min=value_min,
value_max=value_max,
threshold=threshold,
)
return flow_magnitude_vertex_colors(
flow,
mesh,
threshold=threshold,
)
class _FilamentOffscreenBackend:
"""Filament OffscreenRenderer (Linux headless / platforms with EGL)."""
def __init__(
self,
mesh: UVSurfaceMesh,
*,
width: int,
height: int,
threshold: float,
color_mode: str,
value_min,
value_max,
world_to_cam: np.ndarray | None,
lookat: np.ndarray,
show_contact_arrow: bool,
):
import open3d as o3d
self._o3d = o3d
self.mesh = mesh
self.width = width
self.height = height
self.threshold = threshold
self.color_mode = color_mode
self.value_min = value_min
self.value_max = value_max
self.world_to_cam = world_to_cam
self.lookat = lookat
self._contact_arrow = ContactArrowOverlay(mesh, enabled=show_contact_arrow)
self._renderer = o3d.visualization.rendering.OffscreenRenderer(self.width, self.height)
self._renderer.scene.set_background([0.05, 0.05, 0.08, 1.0])
self._material = _make_unlit_material(o3d)
self._mesh_added = False
self._arrow_added = False
self._apply_camera()
def _apply_camera(self) -> None:
if self.world_to_cam is not None:
center, eye, up = camera_look_at_from_world_to_cam(self.world_to_cam, lookat=self.lookat)
else:
center = self.lookat.astype(np.float64)
extent = np.ptp(self.mesh.rest_vertices, axis=0)
span = float(np.max(extent))
eye = center + np.array([0.0, 0.0, span * 2.0], dtype=np.float64)
up = np.array([0.0, 1.0, 0.0], dtype=np.float64)
self._renderer.scene.camera.look_at(center, eye, up)
def resize(self, width: int, height: int) -> None:
width = max(int(width), 64)
height = max(int(height), 64)
if width == self.width and height == self.height:
return
self.width = width
self.height = height
self._renderer = self._o3d.visualization.rendering.OffscreenRenderer(self.width, self.height)
self._renderer.scene.set_background([0.05, 0.05, 0.08, 1.0])
self._mesh_added = False
self._arrow_added = False
self._apply_camera()
def render(self, *, flow=None, scalar_map=None, centroid=None, centroid_found=False, fnormal=0.0):
o3d = self._o3d
colors = _vertex_colors(
self.mesh,
color_mode=self.color_mode,
threshold=self.threshold,
value_min=self.value_min,
value_max=self.value_max,
flow=flow,
scalar_map=scalar_map,
)
body_mesh = _mesh_to_o3d(o3d, self.mesh.rest_vertices, self.mesh.triangles, colors)
if not self._mesh_added:
self._renderer.scene.add_geometry("finger_mesh", body_mesh, self._material)
self._mesh_added = True
else:
self._renderer.scene.remove_geometry("finger_mesh")
self._renderer.scene.add_geometry("finger_mesh", body_mesh, self._material)
arrow_result = self._contact_arrow.compute(centroid, centroid_found, fnormal)
if arrow_result is not None and self._contact_arrow.arrow_mesh is not None:
arrow_vertices, arrow_triangles, arrow_colors = arrow_result
if np.any(arrow_vertices):
arrow_mesh = _mesh_to_o3d(o3d, arrow_vertices, arrow_triangles, arrow_colors)
if not self._arrow_added:
self._renderer.scene.add_geometry("contact_arrow", arrow_mesh, self._material)
self._arrow_added = True
else:
self._renderer.scene.remove_geometry("contact_arrow")
self._renderer.scene.add_geometry("contact_arrow", arrow_mesh, self._material)
elif self._arrow_added:
self._renderer.scene.remove_geometry("contact_arrow")
self._arrow_added = False
return _numpy_o3d_image_to_rgb(self._renderer.render_to_image())
def close(self) -> None:
self._renderer = None
class _HiddenVisualizerBackend:
"""Invisible Open3D Visualizer + framebuffer capture (Windows-friendly)."""
def __init__(
self,
mesh: UVSurfaceMesh,
*,
width: int,
height: int,
threshold: float,
color_mode: str,
value_min,
value_max,
world_to_cam: np.ndarray | None,
lookat: np.ndarray,
show_contact_arrow: bool,
):
import open3d as o3d
self._o3d = o3d
self.mesh = mesh
self.width = width
self.height = height
self.threshold = threshold
self.color_mode = color_mode
self.value_min = value_min
self.value_max = value_max
self.world_to_cam = world_to_cam
self.lookat = lookat
self._contact_arrow = ContactArrowOverlay(mesh, enabled=show_contact_arrow)
self.vis = None
self.tri_mesh = None
self._create_window()
def _create_window(self) -> None:
o3d = self._o3d
if self.vis is not None:
self.vis.destroy_window()
colors = _vertex_colors(
self.mesh,
color_mode=self.color_mode,
threshold=self.threshold,
value_min=self.value_min,
value_max=self.value_max,
)
self.vis = o3d.visualization.Visualizer()
self.vis.create_window(
window_name="Orisys Finger Mesh (embedded)",
width=self.width,
height=self.height,
visible=False,
)
self.tri_mesh = o3d.geometry.TriangleMesh()
self.tri_mesh.vertices = o3d.utility.Vector3dVector(self.mesh.rest_vertices.astype(np.float64))
self.tri_mesh.triangles = o3d.utility.Vector3iVector(self.mesh.triangles.astype(np.int32))
self.tri_mesh.vertex_colors = o3d.utility.Vector3dVector(colors.astype(np.float64))
self.tri_mesh.compute_vertex_normals()
self.vis.add_geometry(self.tri_mesh)
if self._contact_arrow.arrow_mesh is not None:
self.vis.add_geometry(self._contact_arrow.arrow_mesh)
render_option = self.vis.get_render_option()
render_option.mesh_show_back_face = True
if hasattr(render_option, "light_on"):
render_option.light_on = False
render_option.background_color = np.array([0.05, 0.05, 0.08])
if self.world_to_cam is not None:
apply_view_from_world_to_cam(self.vis, self.world_to_cam, lookat=self.lookat)
else:
view_control = self.vis.get_view_control()
center = self.lookat.astype(np.float64)
extent = np.ptp(self.mesh.rest_vertices, axis=0)
span = float(np.max(extent))
view_control.set_lookat(center)
view_control.set_front(np.array([0.0, 0.0, -1.0]))
view_control.set_up(np.array([0.0, 1.0, 0.0]))
view_control.set_zoom(0.5 if span > 0 else 0.7)
self.vis.poll_events()
self.vis.update_renderer()
def resize(self, width: int, height: int) -> None:
width = max(int(width), 64)
height = max(int(height), 64)
if width == self.width and height == self.height:
return
self.width = width
self.height = height
self._create_window()
def render(self, *, flow=None, scalar_map=None, centroid=None, centroid_found=False, fnormal=0.0):
o3d = self._o3d
colors = _vertex_colors(
self.mesh,
color_mode=self.color_mode,
threshold=self.threshold,
value_min=self.value_min,
value_max=self.value_max,
flow=flow,
scalar_map=scalar_map,
)
vertices = self.mesh.rest_vertices
self.tri_mesh.vertices = o3d.utility.Vector3dVector(vertices.astype(np.float64))
self.tri_mesh.vertex_colors = o3d.utility.Vector3dVector(colors.astype(np.float64))
self.tri_mesh.compute_vertex_normals()
self.vis.update_geometry(self.tri_mesh)
arrow_result = self._contact_arrow.compute(centroid, centroid_found, fnormal)
if arrow_result is not None and self._contact_arrow.arrow_mesh is not None:
arrow_vertices, _, arrow_colors = arrow_result
self._contact_arrow.apply(arrow_vertices, arrow_colors)
self.vis.update_geometry(self._contact_arrow.arrow_mesh)
self.vis.poll_events()
self.vis.update_renderer()
return _numpy_o3d_image_to_rgb(self.vis.capture_screen_float_buffer(do_render=False))
def close(self) -> None:
if self.vis is not None:
self.vis.destroy_window()
self.vis = None
def _create_backend(mesh: UVSurfaceMesh, **kwargs):
# Filament OffscreenRenderer needs EGL headless (not available on Windows).
if sys.platform == "win32":
return _HiddenVisualizerBackend(mesh, **kwargs)
try:
return _FilamentOffscreenBackend(mesh, **kwargs)
except RuntimeError:
return _HiddenVisualizerBackend(mesh, **kwargs)
class OffscreenFingerMeshRenderer:
"""Render finger UV mesh + contact arrow to an RGB image for Qt embedding."""
def __init__(
self,
mesh: UVSurfaceMesh,
*,
width: int = 512,
height: int = 512,
threshold: float = 0.0,
color_mode: str = "depth",
value_min=None,
value_max=None,
world_to_cam: np.ndarray | None = None,
lookat: np.ndarray | None = None,
show_contact_arrow: bool = True,
):
lookat = lookat if lookat is not None else mesh.rest_vertices.mean(axis=0)
self._backend = _create_backend(
mesh,
width=int(width),
height=int(height),
threshold=threshold,
color_mode=color_mode,
value_min=value_min,
value_max=value_max,
world_to_cam=world_to_cam,
lookat=lookat,
show_contact_arrow=show_contact_arrow,
)
@property
def width(self) -> int:
return self._backend.width
@property
def height(self) -> int:
return self._backend.height
def resize(self, width: int, height: int) -> None:
self._backend.resize(width, height)
def render(
self,
*,
flow=None,
scalar_map=None,
centroid=None,
centroid_found: bool = False,
fnormal: float = 0.0,
) -> np.ndarray:
return self._backend.render(
flow=flow,
scalar_map=scalar_map,
centroid=centroid,
centroid_found=centroid_found,
fnormal=fnormal,
)
def close(self) -> None:
self._backend.close()
+35
View File
@@ -0,0 +1,35 @@
"""Pure numpy→BGR render helpers for finger 2D panels (no window I/O)."""
from __future__ import annotations
import orisys
from .opencv_maps import depth_colormap
def render_depth_panel(depth_map):
"""Return a BGR depth colormap image."""
return depth_colormap(depth_map)
def render_uv_arrows(
unrolled,
vnormal,
*,
flow_threshold: float = 0.4,
grid_spacing: int = 20,
arrow_scale: float = 1.0,
):
"""Return a BGR image with UV unrolled flow arrows."""
return orisys.util.draw_arrows(
unrolled,
vnormal,
threshold=flow_threshold,
grid_spacing=grid_spacing,
arrow_scale=arrow_scale,
)
def render_contact_panel(img, contour, centroid):
"""Return a BGR contact overlay on the raw sensor image."""
return orisys.util.draw_contact(img, contour, centroid)
+7 -2
View File
@@ -20,8 +20,13 @@ def lowpass_filter(data, cutoff_hz, sample_rate_hz=30, order=2) -> np.ndarray:
) )
b, a = butter(order, cutoff_hz / nyquist, btype="low") b, a = butter(order, cutoff_hz / nyquist, btype="low")
padlen = 3 * (max(len(b), len(a)) - 1) taplen = max(len(a), len(b))
# scipy.signal.filtfilt requires len(x) > padlen (default 3 * taplen for recent scipy).
padlen = 3 * taplen
if y.size <= padlen: if y.size <= padlen:
return y.copy() return y.copy()
return filtfilt(b, a, y) try:
return filtfilt(b, a, y)
except ValueError:
return y.copy()
+2 -1
View File
@@ -21,6 +21,7 @@ cuda12 = ["cupy-cuda12x>=12.3.0"]
cuda13 = ["cupy-cuda13x>=13.0.0"] cuda13 = ["cupy-cuda13x>=13.0.0"]
# Backward compatibility: 'cuda' defaults to CUDA 12.x # Backward compatibility: 'cuda' defaults to CUDA 12.x
cuda = ["cupy-cuda12x>=12.3.0"] cuda = ["cupy-cuda12x>=12.3.0"]
gui = ["pyqt5>=5.15", "pyqtgraph>=0.13"]
[tool.setuptools] [tool.setuptools]
package-dir = { "" = "src" } package-dir = { "" = "src" }
@@ -50,7 +51,7 @@ dev_install = true
name = "examples" name = "examples"
source = "examples" source = "examples"
target = "examples" target = "examples"
patterns = ["*.py", "README.md", "viz/*.py"] patterns = ["*.py", "README.md", "viz/*.py", "pipeline/*.py", "gui/*.py", "gui/widgets/*.py"]
preserve_tree = true preserve_tree = true
[[tool.orisys.bundle.items]] [[tool.orisys.bundle.items]]
+30 -839
View File
@@ -1,858 +1,49 @@
""" #!/usr/bin/env python3
Orisys SDK 实时监控 — Qt 统一界面 """Shim for the Orisys finger Qt dashboard (see examples/gui/)."""
将所有可视化和曲线整合到一个窗口中,适合销售演示使用。
SDK version: 0.3.1
"""
import os import os
import sys import sys
os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" from pathlib import Path
import cv2
def _fix_qt_plugin_path():
"""OpenCV redirects Qt to cv2/qt/plugins; use PyQt5's plugins instead."""
try:
from PyQt5.QtCore import QLibraryInfo
os.environ["QT_QPA_PLATFORM_PLUGIN_PATH"] = QLibraryInfo.location(
QLibraryInfo.PluginsPath
)
except Exception:
path = os.environ.get("QT_QPA_PLATFORM_PLUGIN_PATH", "")
if "cv2" in path.replace("\\", "/"):
os.environ.pop("QT_QPA_PLATFORM_PLUGIN_PATH", None)
_fix_qt_plugin_path()
import orisys
from orisys.finger import FingerRuntime, UVMeshViewer, build_uv_surface_mesh
import numpy as np
import time
import argparse
from datetime import datetime
from scipy.signal import butter, filtfilt
# Force curve sampling rate (Hz), matches typical sensor loop ~30 FPS. os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0")
FORCE_SAMPLE_RATE_HZ = 30.0
PLOT_BUFFER_POINTS = 120
PLOT_WINDOW_SEC = PLOT_BUFFER_POINTS / FORCE_SAMPLE_RATE_HZ # 4 s window
# Minimum y-axis span so autoscale does not over-zoom on small fluctuations.
Y_AXIS_MIN_SPAN = 20000
CURVE_THEME_COLOR = '#00E5FF'
NORMAL_FORCE_CAL_SLOPE = 3.92685554668795e-05
NORMAL_FORCE_CAL_OFFSET = 0.0217596967278535
from PyQt5.QtWidgets import ( _EXAMPLES_DIR = Path(__file__).resolve().parents[1] / "examples"
QMainWindow, QWidget, QHBoxLayout, QVBoxLayout, if str(_EXAMPLES_DIR) not in sys.path:
QLabel, QApplication, QSplitter, QPlainTextEdit, sys.path.insert(0, str(_EXAMPLES_DIR))
QPushButton, QFileDialog, QLineEdit,
)
from PyQt5.QtCore import Qt, QTimer
from PyQt5 import QtGui
from PyQt5.QtGui import QImage, QPixmap, QFont, QIcon
try: from argparse import Namespace
import pyqtgraph as pg
except Exception:
print("PyQtGraph is not installed. Please install it using 'pip install pyqt5 pyqtgraph'.")
sys.exit(1)
# --------------------------------------------------------------------------- from gui import run_dashboard
# 工具 from pipeline import PipelineConfig
# ---------------------------------------------------------------------------
def cv2_to_pixmap(img_bgr: np.ndarray) -> QPixmap:
rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
h, w, ch = rgb.shape
qimg = QImage(rgb.data, w, h, ch * w, QImage.Format_RGB888)
return QPixmap.fromImage(qimg)
def draw_magnitude_map(vfield, threshold=3, colormap=cv2.COLORMAP_JET) -> np.ndarray:
flow_field = vfield
magnitude = np.sqrt(flow_field[:, :, 0] ** 2 + flow_field[:, :, 1] ** 2)
downsample_factor = 10
h, w = magnitude.shape
magnitude_downsampled = magnitude[::downsample_factor, ::downsample_factor]
magnitude_smoothed = cv2.resize(magnitude_downsampled, (w, h), interpolation=cv2.INTER_CUBIC)
magnitude_thresholded = magnitude_smoothed.copy()
magnitude_thresholded[magnitude_smoothed < threshold] = 0
mag_min, mag_max = magnitude_thresholded.min(), magnitude_thresholded.max()
if mag_max > mag_min:
magnitude_norm = ((magnitude_thresholded - mag_min) / (mag_max - mag_min) * 255).astype(np.uint8)
else:
magnitude_norm = np.zeros_like(magnitude_thresholded, dtype=np.uint8)
magnitude_colored = cv2.applyColorMap(magnitude_norm, colormap)
out_img = cv2.resize(magnitude_colored, (400, 400), interpolation=cv2.INTER_CUBIC)
return out_img
def lowpass_filter(data, cutoff_hz, sample_rate_hz=FORCE_SAMPLE_RATE_HZ, order=2):
"""
Zero-phase Butterworth low-pass filter for uniformly sampled 1-D data.
Parameters
----------
data : array-like
Input samples acquired at `sample_rate_hz` (default 30 Hz).
cutoff_hz : float
-3 dB cutoff frequency in Hz; use 0 to skip filtering (return raw data).
Otherwise must satisfy 0 < cutoff_hz < sample_rate_hz / 2.
sample_rate_hz : float
Sampling rate of `data` in Hz.
order : int
Butterworth filter order.
Returns
-------
np.ndarray
Filtered 1-D array with the same length as `data`.
"""
y = np.asarray(data, dtype=np.float64)
if cutoff_hz == 0:
return y.copy()
if y.size == 0:
return y
nyquist = 0.5 * sample_rate_hz
if cutoff_hz < 0 or cutoff_hz >= nyquist:
raise ValueError(
f"cutoff_hz must be in (0, {nyquist}) for sample_rate_hz={sample_rate_hz}"
)
b, a = butter(order, cutoff_hz / nyquist, btype="low")
padlen = 3 * (max(len(b), len(a)) - 1)
if y.size <= padlen:
return y.copy()
return filtfilt(b, a, y)
def calibrate_normal_force(raw_value):
"""Convert raw normal-force reading to Newtons using y = mx + c."""
return NORMAL_FORCE_CAL_SLOPE * raw_value + NORMAL_FORCE_CAL_OFFSET
class TeeStream:
"""同时写入多个流"""
def __init__(self, *streams):
self.streams = streams
def write(self, text):
for s in self.streams:
s.write(text)
def flush(self):
for s in self.streams:
s.flush()
class WidgetLogHandler:
"""将 print 输出实时显示到 QPlainTextEdit"""
def __init__(self, widget: QPlainTextEdit):
self.widget = widget
self.buf = ""
def write(self, text: str):
self.buf += text
while "\n" in self.buf:
line, self.buf = self.buf.split("\n", 1)
if line:
self.widget.appendPlainText(line)
def flush(self):
if self.buf:
self.widget.appendPlainText(self.buf)
self.buf = ""
# ---------------------------------------------------------------------------
# 样式表
# ---------------------------------------------------------------------------
STYLE = """
QMainWindow {
background-color: #000000;
}
/* ---- 图像面板 ---- */
QLabel#arrowsLabel {
border: 1px solid #262626;
border-radius: 4px;
background-color: #0b0b0b;
}
/* ---- 顶栏按钮 & 输入 - — */
QPushButton {
background-color: #1a1a1a;
color: #999999;
border: 1px solid #333333;
border-radius: 4px;
padding: 6px 16px;
font-family: "SimHei";
font-size: 13px;
}
QPushButton:hover {
background-color: #262626;
color: #cccccc;
border: 1px solid #555555;
}
QPushButton#btnStart {
background-color: #0d2818;
color: #4caf84;
border: 1px solid #1a4d30;
}
QPushButton#btnStart:hover {
background-color: #143d24;
color: #6fd4a0;
}
QPushButton#btnStart:disabled {
background-color: #0f0f0f;
color: #333333;
border: 1px solid #222222;
}
QPushButton#btnStop {
background-color: #2b1010;
color: #e05555;
border: 1px solid #4d1a1a;
}
QPushButton#btnStop:hover {
background-color: #3d1818;
color: #f07070;
}
QPushButton#btnStop:disabled {
background-color: #0f0f0f;
color: #333333;
border: 1px solid #222222;
}
QLineEdit {
background-color: #0d0d0d;
color: #999999;
border: 1px solid #333333;
border-radius: 4px;
padding: 5px 10px;
font-size: 13px;
}
/* ---- 力曲线容器 ---- */
GraphicsLayoutWidget {
border: 1px solid #262626;
border-radius: 4px;
}
/* ---- 日志面板 ---- */
QPlainTextEdit {
background-color: #0b0b0b;
color: #777777;
border: 1px solid #262626;
border-radius: 4px;
font-family: "Cascadia Code", "Consolas", "Courier New", monospace;
font-size: 13px;
padding: 8px;
selection-background-color: #1a3a4a;
selection-color: #cccccc;
}
QPlainTextEdit QScrollBar:vertical {
background-color: #0b0b0b;
width: 6px;
margin: 1px;
border-radius: 3px;
}
QPlainTextEdit QScrollBar::handle:vertical {
background-color: #262626;
border-radius: 3px;
min-height: 30px;
}
QPlainTextEdit QScrollBar::handle:vertical:hover {
background-color: #3a3a3a;
}
QPlainTextEdit QScrollBar::add-line:vertical,
QPlainTextEdit QScrollBar::sub-line:vertical {
height: 0px;
}
QPlainTextEdit QScrollBar::add-page:vertical,
QPlainTextEdit QScrollBar::sub-page:vertical {
background: none;
}
QSplitter::handle {
background-color: #1a1a1a;
}
QSplitter::handle:vertical {
height: 1px;
}
QSplitter::handle:horizontal {
width: 1px;
}
"""
# ---------------------------------------------------------------------------
# 主窗口
# ---------------------------------------------------------------------------
class MainWindow(QMainWindow):
def __init__(self, args, log_file):
super().__init__()
self.setWindowTitle("Orisys SDK")
self.setStyleSheet(STYLE)
logo_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "logo.png")
if os.path.exists(logo_path):
self.setWindowIcon(QIcon(logo_path))
self.sensor = None
self.mesh_viewer = None
self.start_time = None
self.args = args
# 录像状态
self.is_save = False
self.frame_count = 0
self.writer = None
self.save_path = None
# 曲线数据缓冲(先填满 npoints 再滚动;120 点 @ 30 Hz = 4 s)
self.npoints = PLOT_BUFFER_POINTS
self.lowpass_cutoff_hz = getattr(args, "cutoff", 5.0)
self._plot_sample_count = 0
self._init_plot_buffer()
self._setup_ui()
self._create_plots()
# 同步参数到 UI
self.camera_edit.setText(args.video)
self.config_edit.setText(args.config)
self.cal_edit.setText(getattr(args, "cal", ""))
# stdout → 日志文件 + 界面日志面板
self.log_handler = WidgetLogHandler(self.log_widget)
sys.stdout = TeeStream(log_file, self.log_handler)
sys.stderr = sys.stdout
self._y_range_alpha = 0.5
self.timer = QTimer()
self.timer.timeout.connect(self._on_timer)
self._running = False
def _init_plot_buffer(self):
"""Empty buffer; x-axis grows until npoints real samples, then scrolls."""
self.data = np.full((self.npoints, 2, 3), np.nan, dtype=np.float64)
self._plot_sample_count = 0
self._y_range_smooth = [None, None, None]
# ---- UI 构建 ----
def _setup_ui(self):
central = QWidget()
self.setCentralWidget(central)
root_layout = QVBoxLayout(central)
root_layout.setContentsMargins(6, 6, 6, 6)
root_layout.setSpacing(6)
# ---- 顶栏 ----
bar = QHBoxLayout()
bar.setSpacing(8)
self.btn_camera = QPushButton("摄像机")
self.btn_camera.setObjectName("btnCamera")
self.btn_camera.clicked.connect(self._select_camera)
bar.addWidget(self.btn_camera)
self.camera_edit = QLineEdit("1")
self.camera_edit.setPlaceholderText("摄像头编号 或 视频路径")
bar.addWidget(self.camera_edit, 1)
self.btn_config = QPushButton("配置")
self.btn_config.setObjectName("btnConfig")
self.btn_config.clicked.connect(self._select_config)
bar.addWidget(self.btn_config)
self.config_edit = QLineEdit("./config/ddjx01.json")
bar.addWidget(self.config_edit, 1)
self.btn_reset = QPushButton("重置")
self.btn_reset.setObjectName("btnReset")
self.btn_reset.setEnabled(False)
self.btn_reset.clicked.connect(self._on_reset)
bar.addWidget(self.btn_reset)
self.cal_edit = QLineEdit("./config/ddjx01.npy")
bar.addWidget(self.cal_edit, 1)
self.btn_start = QPushButton("启动")
self.btn_start.setObjectName("btnStart")
self.btn_start.clicked.connect(self._on_start)
bar.addWidget(self.btn_start)
self.btn_stop = QPushButton("暂停")
self.btn_stop.setObjectName("btnStop")
self.btn_stop.setEnabled(False)
self.btn_stop.clicked.connect(self._on_stop)
bar.addWidget(self.btn_stop)
root_layout.addLayout(bar)
# ---- 外层纵向 splitter:主区域 / 日志 ----
v_splitter = QSplitter(Qt.Vertical)
# ---- 主区域:图像 (左) | 曲线 (右) ----
h_splitter = QSplitter(Qt.Horizontal)
# 左侧图像面板
left = QWidget()
left_layout = QVBoxLayout(left)
left_layout.setContentsMargins(0, 0, 0, 0)
left_layout.setSpacing(6)
self.arrows_label = QLabel()
self.arrows_label.setObjectName("arrowsLabel")
self.arrows_label.setMinimumSize(160, 160)
self.arrows_label.setAlignment(Qt.AlignCenter)
self.arrows_label.setWordWrap(True)
self.arrows_label.setText(self._mesh_viewer_status_text())
left_layout.addWidget(self.arrows_label)
h_splitter.addWidget(left)
# 右侧力曲线
pg.setConfigOption('background', '#000000')
pg.setConfigOption('foreground', '#777777')
pg.setConfigOptions(antialias=True)
self.graph_widget = pg.GraphicsLayoutWidget()
self.graph_widget.setMinimumWidth(300)
h_splitter.addWidget(self.graph_widget)
self.h_splitter = h_splitter
v_splitter.addWidget(h_splitter)
# ---- 底部日志面板 ----
self.log_widget = QPlainTextEdit()
self.log_widget.setReadOnly(True)
self.log_widget.setMaximumBlockCount(2000)
font = QFont("Consolas", 12)
font.setStyleHint(QFont.Monospace)
self.log_widget.setFont(font)
v_splitter.addWidget(self.log_widget)
v_splitter.setStretchFactor(0, 4)
v_splitter.setStretchFactor(1, 1)
root_layout.addWidget(v_splitter)
def _create_plots(self):
titles = ["法向力 (N)", "切向力 X", "切向力 Y"]
self.plots = []
self.curves = []
self.dot_glows = []
self.dot_cores = []
axis_pen = pg.mkPen(color=(38, 38, 38), width=1) # #262626
label_font = QFont("SimHei", 9)
for i, title in enumerate(titles):
p = self.graph_widget.addPlot(title=None)
# 标题:黑体
p.setTitle(title, color='#999999', size='14pt')
p.titleLabel.setFont(QFont("SimHei", 14))
# X 轴标签
p.setLabel('bottom', '时间', units='s')
p.getAxis('bottom').label.setFont(label_font)
# 仅保留左 + 下轴线,隐藏右 + 上边框
p.showAxis('top', False)
p.showAxis('right', False)
p.getAxis('left').setPen(axis_pen)
p.getAxis('bottom').setPen(axis_pen)
# 仅横向网格,极淡
p.showGrid(x=False, y=True, alpha=0.12)
# 等宽刻度字体
tick_font = QFont("Consolas", 11)
tick_font.setStyleHint(QFont.Monospace)
p.getAxis('left').setTickFont(tick_font)
p.getAxis('bottom').setTickFont(tick_font)
# 曲线: 青色描边 + 向下渐变填充(与 demo_wuxi 一致)
line_pen = pg.mkPen(color=CURVE_THEME_COLOR, width=2)
# grad_brush = make_gradient_brush(CURVE_THEME_COLOR, max_alpha=120)
curve = p.plot(pen=line_pen, fillLevel=0)
# 终点光晕: #00e5ff 30% 透明,半径 8
dot_glow = p.plot([], [], pen=None, symbol='o', symbolSize=8,
symbolPen=None, symbolBrush=(0, 229, 255, 76))
# 终点核心: 纯白,半径 4
dot_core = p.plot([], [], pen=None, symbol='o', symbolSize=4,
symbolPen=None, symbolBrush=(255, 255, 255, 255))
p.setXRange(0, PLOT_WINDOW_SEC, padding=0)
p.enableAutoRange(x=False, y=False)
self.plots.append(p)
self.curves.append(curve)
self.dot_glows.append(dot_glow)
self.dot_cores.append(dot_core)
if i < 2:
self.graph_widget.nextRow()
def adjust_square_images(self):
"""根据窗口高度调整左侧面板宽度,使左侧图像保持正方形"""
top_bar_h = 48
log_h = self.log_widget.height()
spacing = 20
available = self.height() - top_bar_h - log_h - spacing
image_w = max(available, 160)
self.h_splitter.setSizes([image_w, self.width() - image_w])
def _mesh_viewer_status_text(self):
if self.mesh_viewer is None:
return "3D mesh viewer will open in a separate window\nwhen acquisition starts."
return "3D mesh viewer is running in a separate window."
def _update_mesh_status_label(self):
self.arrows_label.setText(self._mesh_viewer_status_text())
def _init_mesh_viewer(self):
if self.sensor is None:
self._update_mesh_status_label()
return
domain = getattr(self.sensor, "domain", None)
surface = getattr(domain, "surface", None)
if surface is None or getattr(surface, "world_xyz", None) is None:
self.arrows_label.setText("3D mesh viewer unavailable.\nUV domain has no surface map.")
return
try:
uv_mesh = build_uv_surface_mesh(domain.surface.world_xyz, domain.valid_mask)
self.mesh_viewer = UVMeshViewer(
uv_mesh,
color_mode="flow",
threshold=2.0,
)
self._update_mesh_status_label()
except Exception as exc:
self.mesh_viewer = None
self.arrows_label.setText(f"3D mesh viewer init failed.\n{exc}")
def _close_mesh_viewer(self):
if self.mesh_viewer is not None:
self.mesh_viewer.close()
self.mesh_viewer = None
self._update_mesh_status_label()
# ---- 帧处理 ----
def _on_timer(self):
if self.sensor is None:
return
runtime = self.sensor
tactile = runtime.tactile
# isimg = sensor.get_img()
ok, unrolled = runtime.grab_and_process(color=True)
if not ok:
print("Failed to get image")
return
# 旋转原始图像,使力方向与实际物理方向对齐
# sensor.img = cv2.rotate(sensor.img, cv2.ROTATE_90_COUNTERCLOCKWISE)
# if sensor.frame is not None:
# sensor.frame = cv2.rotate(sensor.frame, cv2.ROTATE_90_COUNTERCLOCKWISE)
# 录像写入
if self.is_save and self.writer is not None and tactile.frame is not None:
self.frame_count += 1
save_img = tactile.frame
if len(save_img.shape) == 2 or (len(save_img.shape) == 3 and save_img.shape[2] == 1):
if len(save_img.shape) == 3:
save_img = save_img.squeeze(2)
save_img = cv2.cvtColor(save_img, cv2.COLOR_GRAY2BGR)
self.writer.write(save_img)
tactile.compute_deformation(check_motion=True, threshold=1)
tactile.compute_contact()
fps, fn_raw, fx, fy, flow, vnormal, vshear, _ = tactile.read_info(
tactile.info.FPS, tactile.info.FNORMAL,
tactile.info.FSHEARX, tactile.info.FSHEARY,
tactile.info.VRAW, tactile.info.VNORMAL,
tactile.info.VSHEAR,
tactile.info.DEPTH
)
fn = calibrate_normal_force(fn_raw)
print(f"FPS={fps:.2f}, 法向力={fn:.4f}, 切向力X={fx:.4f}, 切向力Y={fy:.4f}")
# print(f"CUPY Available: {sensor.CUPY_AVAILABLE}")
if self.mesh_viewer is not None:
if not self.mesh_viewer.update(flow=flow):
self.mesh_viewer.close()
self.mesh_viewer = None
self.arrows_label.setText("3D mesh viewer closed.\nRestart acquisition to reopen it.")
# 曲线:先顺序写入 npoints 个真实样本,满后再左移滚动
current_time = time.time() - self.start_time
if self._plot_sample_count < self.npoints:
idx = self._plot_sample_count
self.data[idx, 0, :] = current_time
self.data[idx, 1, 0] = fn
self.data[idx, 1, 1] = fx
self.data[idx, 1, 2] = fy
self._plot_sample_count += 1
else:
for i in range(3):
self.data[:-1, 0, i] = self.data[1:, 0, i]
self.data[:-1, 1, i] = self.data[1:, 1, i]
self.data[-1, 0, :] = current_time
self.data[-1, 1, 0] = fn
self.data[-1, 1, 1] = fx
self.data[-1, 1, 2] = fy
n = self._plot_sample_count
if n == 0:
return
data_plot = self.data[:n].copy()
for i in range(3):
data_plot[:, 1, i] = lowpass_filter(
data_plot[:, 1, i],
self.lowpass_cutoff_hz,
sample_rate_hz=FORCE_SAMPLE_RATE_HZ,
)
for i, curve in enumerate(self.curves):
x = data_plot[:, 0, i]
y = data_plot[:, 1, i]
curve.setData(x, y)
ymin, ymax = y.min(), y.max()
if self._y_range_smooth[i] is None:
self._y_range_smooth[i] = (ymin, ymax)
else:
prev_min, prev_max = self._y_range_smooth[i]
alpha = self._y_range_alpha
ymin = alpha * ymin + (1 - alpha) * prev_min
ymax = alpha * ymax + (1 - alpha) * prev_max
span = ymax - ymin
if span < Y_AXIS_MIN_SPAN:
mid = (ymin + ymax) / 2
half = Y_AXIS_MIN_SPAN / 2
ymin = mid - half
ymax = mid + half
self._y_range_smooth[i] = (ymin, ymax)
pad = max((ymax - ymin) * 0.12, 0.01)
self.plots[i].setYRange(ymin - pad, ymax + pad)
last_x = x[-1:]
last_y = y[-1:]
self.dot_glows[i].setData(last_x, last_y)
self.dot_cores[i].setData(last_x, last_y)
if n < self.npoints:
for p in self.plots:
p.setXRange(0, PLOT_WINDOW_SEC, padding=0)
else:
x0 = float(data_plot[0, 0, 0])
x1 = float(data_plot[-1, 0, 0])
if x1 <= x0:
x1 = x0 + PLOT_WINDOW_SEC
for p in self.plots:
p.setXRange(x0, x1, padding=0)
# ---- 键盘 & 录像 ----
def _on_start(self):
if self._running:
return
self.btn_start.setText("启动中...")
self.btn_start.setEnabled(False)
QApplication.processEvents()
try:
src = self.camera_edit.text().strip()
cfg = self.config_edit.text()
cal = self.cal_edit.text().strip()
verbose = getattr(self.args, 'verbose', True)
try:
video = int(src)
except ValueError:
video = src
# self.sensor = orisys.Sensor(src, config_name=cfg, cal_path=cal, verbose=verbose)
self.sensor = FingerRuntime.open(
config=cfg,
calib=cal or None,
source_scale=None,
video=video,
verbose=verbose,
)
self.sensor.warmup(color=True)
self._init_mesh_viewer()
except Exception as e:
self._close_mesh_viewer()
self.btn_start.setText("启动")
self.btn_start.setEnabled(True)
print(f"启动失败: {e}")
return
self.start_time = time.time()
self._init_plot_buffer()
self._running = True
self.timer.start(10)
self.btn_start.setText("启动")
self.btn_stop.setEnabled(True)
self.btn_reset.setEnabled(True)
print(f"▶ 启动 — 摄像机={src}, 配置={cfg}, 标定={cal}")
def _on_stop(self):
if not self._running:
return
self._running = False
self.timer.stop()
self.btn_stop.setText("停止中...")
self.btn_stop.setEnabled(False)
QApplication.processEvents()
if self.sensor is not None:
self.sensor.close()
self.sensor = None
self._close_mesh_viewer()
self.btn_start.setEnabled(True)
self.btn_stop.setText("暂停")
self.btn_reset.setEnabled(False)
print("⏸ 已暂停")
def _select_camera(self):
text = self.camera_edit.text()
path, _ = QFileDialog.getOpenFileName(
self, "选择视频文件", "data", "视频 (*.mp4 *.avi *.mov);;所有文件 (*)",
)
if path:
self.camera_edit.setText(path)
def _select_config(self):
path, _ = QFileDialog.getOpenFileName(
self, "选择配置文件", "config", "JSON (*.json);;所有文件 (*)",
)
if path:
self.config_edit.setText(path)
def _on_reset(self):
"""Reset optical flow tracker when drift occurs (same as basic_pipeline 'r' key)."""
if self.sensor is None or not self._running:
print("请先启动传感器后再重置追踪器。")
return
self.sensor.tactile.reset()
print("追踪器已重置。")
def keyPressEvent(self, event):
key = event.key()
if key == Qt.Key_Escape:
if self.isFullScreen():
self.showNormal()
else:
self.showFullScreen()
elif key == Qt.Key_Q:
print("\n正在退出示例程序...")
self.close()
elif key == Qt.Key_S:
self._start_recording()
elif key == Qt.Key_E:
self._stop_recording()
elif key == Qt.Key_R:
self._on_reset()
else:
super().keyPressEvent(event)
def _start_recording(self):
if self.sensor is None:
print("请先启动采集")
return
self.save_path = "./data/pulse_test.mp4"
os.makedirs(os.path.dirname(self.save_path), exist_ok=True)
save_img = self.sensor.tactile.frame
if save_img is None:
print("当前没有可用帧,无法确定录像尺寸。")
return
temp_img = save_img.copy()
if len(temp_img.shape) == 2 or (len(temp_img.shape) == 3 and temp_img.shape[2] == 1):
if len(temp_img.shape) == 3:
temp_img = temp_img.squeeze(2)
temp_img = cv2.cvtColor(temp_img, cv2.COLOR_GRAY2BGR)
fh, fw = temp_img.shape[:2]
self.writer = cv2.VideoWriter(
self.save_path, cv2.VideoWriter_fourcc(*'mp4v'), 30, (fw, fh),
)
if self.writer.isOpened():
self.is_save = True
print(f"开始录像:{self.save_path}")
else:
print(f"无法创建录像文件:{self.save_path}")
self.writer = None
def _stop_recording(self):
self.is_save = False
if self.writer is not None:
self.writer.release()
self.writer = None
if self.save_path is not None:
print(f"录像结束:{self.save_path},总帧数:{self.frame_count}")
else:
print(f"录像结束,总帧数:{self.frame_count}")
self.frame_count = 0
def closeEvent(self, event):
self.timer.stop()
if self.writer is not None:
self.writer.release()
self._close_mesh_viewer()
if self.sensor is not None:
self.sensor.close()
event.accept()
# ---------------------------------------------------------------------------
# 入口
# ---------------------------------------------------------------------------
def main(): def main():
script_dir = os.path.dirname(os.path.abspath(__file__)) import argparse
log_dir = os.path.join(script_dir, "logs")
os.makedirs(log_dir, exist_ok=True)
log_path = os.path.join(log_dir, f"qt_viewer_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log")
log_file = open(log_path, 'w', encoding='utf-8')
# stdout 到文件的临时重定向(创建 MainWindow 后会加上日志面板)
sys.stdout = log_file
sys.stderr = sys.stdout
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser(description="Orisys finger Qt dashboard")
parser.add_argument("--video", "-v", type=str, default="0") parser.add_argument("--video", "-v", type=str, default="0")
parser.add_argument("--config", "-c", type=str, default="./config/dd02-ov.json") parser.add_argument("--config", "-c", type=str, default="./config/dd02-ov.json")
parser.add_argument("--cal", type=str, default="") parser.add_argument("--cal", type=str, default="")
parser.add_argument("--verbose", "-verbose", type=bool, default=True) parser.add_argument("--verbose", action="store_true")
parser.add_argument( parser.add_argument("--cutoff", type=float, default=10.0)
"--cutoff", parser.add_argument("--no-3d", action="store_true")
type=float,
default=10.0,
help="Low-pass cutoff for force curves (Hz), sample rate 30 Hz",
)
args = parser.parse_args() args = parser.parse_args()
print("\n按键说明:Q 退出 | R 重置追踪器 | S 开始录像 | E 结束录像 | ESC 全屏") config = PipelineConfig.from_args(
Namespace(
video=args.video,
config=args.config,
cal=args.cal,
verbose=args.verbose,
mesh_map="depth",
threshold=None,
viz_pose="assets/finger/visualization/world_to_cam.npy",
no_3d=args.no_3d,
cutoff=args.cutoff,
)
)
sys.exit(run_dashboard(config))
# 高 DPI 适配(笔记本高分屏)
QApplication.setAttribute(Qt.AA_EnableHighDpiScaling, True)
QApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True)
app = QApplication(sys.argv)
window = MainWindow(args, log_file)
window.showFullScreen()
QApplication.processEvents()
window.adjust_square_images()
sys.exit(app.exec_()) if __name__ == "__main__":
if __name__ == '__main__':
main() main()
+40 -12
View File
@@ -16,6 +16,10 @@ def init_uv_mesh_viewer(
vertices: np.ndarray | None = None, vertices: np.ndarray | None = None,
vertex_colors: np.ndarray | None = None, vertex_colors: np.ndarray | None = None,
window_name: str = "UV Finger Mesh 3D", window_name: str = "UV Finger Mesh 3D",
*,
window_width: int = 1280,
window_height: int = 720,
extra_geometries: list | None = None,
): ):
"""Create Open3D viewer with an unlit triangle mesh.""" """Create Open3D viewer with an unlit triangle mesh."""
import open3d as o3d import open3d as o3d
@@ -26,7 +30,7 @@ def init_uv_mesh_viewer(
vertex_colors = mesh.base_colors vertex_colors = mesh.base_colors
vis = o3d.visualization.Visualizer() vis = o3d.visualization.Visualizer()
vis.create_window(window_name=window_name, width=1280, height=720) vis.create_window(window_name=window_name, width=window_width, height=window_height)
tri_mesh = o3d.geometry.TriangleMesh() tri_mesh = o3d.geometry.TriangleMesh()
tri_mesh.vertices = o3d.utility.Vector3dVector(vertices.astype(np.float64)) tri_mesh.vertices = o3d.utility.Vector3dVector(vertices.astype(np.float64))
@@ -35,6 +39,9 @@ def init_uv_mesh_viewer(
tri_mesh.compute_vertex_normals() tri_mesh.compute_vertex_normals()
vis.add_geometry(tri_mesh) vis.add_geometry(tri_mesh)
for geometry in extra_geometries or []:
vis.add_geometry(geometry)
render_option = vis.get_render_option() render_option = vis.get_render_option()
render_option.mesh_show_back_face = True render_option.mesh_show_back_face = True
if hasattr(render_option, "light_on"): if hasattr(render_option, "light_on"):
@@ -51,13 +58,18 @@ def update_uv_mesh_viewer(
o3d, o3d,
vertices: np.ndarray, vertices: np.ndarray,
vertex_colors: np.ndarray, vertex_colors: np.ndarray,
*,
extra_geometries: list | None = None,
): ):
"""Update mesh vertices/colors.""" """Update mesh vertices/colors and optional extra geometries."""
tri_mesh.vertices = o3d.utility.Vector3dVector(vertices.astype(np.float64)) tri_mesh.vertices = o3d.utility.Vector3dVector(vertices.astype(np.float64))
tri_mesh.vertex_colors = o3d.utility.Vector3dVector(vertex_colors.astype(np.float64)) tri_mesh.vertex_colors = o3d.utility.Vector3dVector(vertex_colors.astype(np.float64))
tri_mesh.compute_vertex_normals() tri_mesh.compute_vertex_normals()
vis.update_geometry(tri_mesh) vis.update_geometry(tri_mesh)
for geometry in extra_geometries or []:
vis.update_geometry(geometry)
vis.poll_events() vis.poll_events()
vis.update_renderer() vis.update_renderer()
@@ -74,6 +86,8 @@ class UVMeshViewer:
color_mode: str = "flow", color_mode: str = "flow",
value_min: float | None = None, value_min: float | None = None,
value_max: float | None = None, value_max: float | None = None,
window_width: int = 1024,
window_height: int = 1024,
): ):
self.mesh = mesh self.mesh = mesh
self.max_magnitude = max_magnitude self.max_magnitude = max_magnitude
@@ -81,27 +95,37 @@ class UVMeshViewer:
self.color_mode = color_mode self.color_mode = color_mode
self.value_min = value_min self.value_min = value_min
self.value_max = value_max self.value_max = value_max
self.window_width = window_width
self.window_height = window_height
self.vis = None self.vis = None
self.tri_mesh = None self.tri_mesh = None
self._o3d = None self._o3d = None
def update(self, flow=None, scalar_map=None): def _vertex_colors(self, flow=None, scalar_map=None):
"""Update viewer from UV-space maps. Returns False if window closed."""
if self.color_mode == "depth": if self.color_mode == "depth":
colors = scalar_map_vertex_colors( return scalar_map_vertex_colors(
scalar_map, scalar_map,
self.mesh, self.mesh,
value_min=self.value_min, value_min=self.value_min,
value_max=self.value_max, value_max=self.value_max,
threshold=self.threshold, threshold=self.threshold,
) )
else: return flow_magnitude_vertex_colors(
colors = flow_magnitude_vertex_colors( flow,
flow, self.mesh,
self.mesh, max_magnitude=self.max_magnitude,
max_magnitude=self.max_magnitude, threshold=self.threshold,
threshold=self.threshold, )
)
def _extra_geometries_for_init(self):
return []
def _extra_geometries_for_update(self):
return []
def update(self, flow=None, scalar_map=None):
"""Update viewer from UV-space maps. Returns False if window closed."""
colors = self._vertex_colors(flow=flow, scalar_map=scalar_map)
vertices = self.mesh.rest_vertices vertices = self.mesh.rest_vertices
if self.vis is None: if self.vis is None:
@@ -109,6 +133,9 @@ class UVMeshViewer:
self.mesh, self.mesh,
vertices=vertices, vertices=vertices,
vertex_colors=colors, vertex_colors=colors,
window_width=self.window_width,
window_height=self.window_height,
extra_geometries=self._extra_geometries_for_init(),
) )
return True return True
@@ -121,6 +148,7 @@ class UVMeshViewer:
self._o3d, self._o3d,
vertices, vertices,
colors, colors,
extra_geometries=self._extra_geometries_for_update(),
) )
return True return True
Binary file not shown.
Binary file not shown.
BIN
View File
Binary file not shown.