1. tuned arrow mesh drawing

2. centralixzed arrow config in layout.py
3. seperate mesh render and display resolution to speed up (~40 fps)
This commit is contained in:
ykcheungab
2026-06-19 14:28:28 +08:00
parent 3fd9b839a9
commit 2e5c047d1e
19 changed files with 980 additions and 147 deletions
+14 -2
View File
@@ -2,23 +2,35 @@
from .dashboard import FingerDashboardWindow, run_dashboard
from .layout import (
DashboardLayout,
LAYOUT,
LOG_CONFIG,
LOG_PANEL,
LAYOUT,
MESH_CONTACT_ARROW,
MESH_LIGHTING,
PANEL_SETUP,
TOP_ROW,
DashboardLayout,
LogConfig,
MeshLighting,
PanelRow,
PanelSetup,
PanelSlot,
top_row_splitter_sizes,
top_row_slot_index,
)
from viz.contact_arrow import ContactArrowStyle
__all__ = [
"DashboardLayout",
"FingerDashboardWindow",
"LAYOUT",
"LOG_CONFIG",
"LOG_PANEL",
"LogConfig",
"MESH_CONTACT_ARROW",
"MESH_LIGHTING",
"ContactArrowStyle",
"MeshLighting",
"PANEL_SETUP",
"TOP_ROW",
"PanelRow",
+66 -11
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import sys
import time
from pathlib import Path
from PyQt5.QtCore import Qt, QTimer
@@ -55,6 +56,9 @@ class FingerDashboardWindow(QMainWindow):
self.layout = layout
self.pipeline = FingerPipeline(config)
self._running = False
self._frame_index = 0
self._wall_fps = 0.0
self._wall_fps_alpha = 0.15
self.setWindowTitle(layout.window_title)
self.setStyleSheet(STYLE)
@@ -124,7 +128,7 @@ class FingerDashboardWindow(QMainWindow):
self.mesh_viewport = MeshViewportWidget(layout=layout)
self.arrow_panel = ArrowPanelWidget(flow_threshold=layout.arrow_flow_threshold, layout=layout)
self.force_charts = ForceChartWidget(layout=layout, lowpass_cutoff_hz=self.config.lowpass_cutoff_hz)
self.log_widget = make_log_widget()
self.log_widget = make_log_widget(layout.log_config)
self._panel_widgets["mesh"] = self.mesh_viewport
self._panel_widgets["arrows"] = self.arrow_panel
@@ -216,6 +220,8 @@ class FingerDashboardWindow(QMainWindow):
return
self._running = True
self._frame_index = 0
self._wall_fps = 0.0
self.timer.start(self.layout.timer_interval_ms)
self.btn_start.setText("启动")
self.btn_stop.setEnabled(True)
@@ -230,6 +236,7 @@ class FingerDashboardWindow(QMainWindow):
return
self._running = False
self.timer.stop()
self.log_handler.clear_status_line()
self.btn_stop.setText("停止中...")
self.btn_stop.setEnabled(False)
QApplication.processEvents()
@@ -252,25 +259,70 @@ class FingerDashboardWindow(QMainWindow):
def _on_timer(self) -> None:
if not self._running:
return
frame = self.pipeline.tick()
frame_t0 = time.perf_counter()
frame, timings = self.pipeline.tick_timed()
if frame is None:
print("采集结束或读取失败。")
self._on_stop()
return
self._frame_index += 1
layout = self.layout
profile = self.pipeline.profiler
mesh_ms = 0.0
if layout.mesh_render_stride <= 1 or self._frame_index % layout.mesh_render_stride == 0:
mesh_ms = self.mesh_viewport.update_frame(frame)
if profile is not None and mesh_ms > 0:
profile.record("mesh", mesh_ms)
arrows_ms = 0.0
t0 = time.perf_counter()
self.arrow_panel.update_frame(frame)
arrows_ms = (time.perf_counter() - t0) * 1000.0
if profile is not None:
profile.record("arrows", arrows_ms)
charts_ms = 0.0
if layout.chart_update_stride <= 1 or self._frame_index % layout.chart_update_stride == 0:
t0 = time.perf_counter()
self.force_charts.update_frame(frame)
charts_ms = (time.perf_counter() - t0) * 1000.0
if profile is not None:
profile.record("charts", charts_ms)
total_ms = (time.perf_counter() - frame_t0) * 1000.0
instant_fps = 1000.0 / total_ms if total_ms > 0 else 0.0
if self._wall_fps <= 0:
self._wall_fps = instant_fps
else:
self._wall_fps = (
self._wall_fps_alpha * instant_fps
+ (1.0 - self._wall_fps_alpha) * self._wall_fps
)
if profile is not None:
profile.record("total", total_ms)
self.pipeline.record_total_frame(total_ms)
flow_mag = 0.0
if frame.flow.size:
flow_mag = float((frame.flow[:, :, 0] ** 2 + frame.flow[:, :, 1] ** 2).mean() ** 0.5)
print(
f"FPS={frame.fps:.2f} flow_rms={flow_mag:.2f} "
f"mask={frame.mask_coverage:.2f} normal={frame.fnormal:.2f} "
f"shear=({frame.fshearx:.2f}, {frame.fsheary:.2f})",
end="\r",
)
self.mesh_viewport.update_frame(frame)
self.arrow_panel.update_frame(frame)
self.force_charts.update_frame(frame)
log_cfg = layout.log_config
if log_cfg.status_enabled:
self.log_handler.set_status_line(
log_cfg.format_status(
fps=frame.fps,
fnormal=frame.fnormal,
fshearx=frame.fshearx,
fsheary=frame.fsheary,
flow_rms=flow_mag,
mask_coverage=frame.mask_coverage,
wall_fps=self._wall_fps,
)
)
def _select_camera(self) -> None:
path, _ = QFileDialog.getOpenFileName(
@@ -325,6 +377,9 @@ def run_dashboard(config: PipelineConfig, *, layout: DashboardLayout = LAYOUT) -
from .app import get_or_create_qapp
print("\n按键说明:Q 退出 | R 重置追踪器 | S 开始录像 | E 结束录像 | ESC 全屏")
if config.profile:
print(f"Profiling enabled (report every {config.profile_every} frames).")
print("Log FPS= is end-to-end dashboard rate; sdk= is Sensor internal rate.")
app = get_or_create_qapp()
window = FingerDashboardWindow(config, layout=layout)
+131 -1
View File
@@ -13,15 +13,40 @@ Structure:
Each panel uses explicit width_px and height_px.
top_stretch / log_stretch control how extra vertical space is shared on resize.
MeshLighting (below) controls the 3D mesh panel sun light:
azimuth_deg — horizontal angle in degrees (0° = +X, 90° = +Y)
elevation_deg — vertical angle in degrees (positive = from above)
intensity — Filament sun strength (typical 30000–120000)
ambient/diffuse — Windows viewer shading weights (0–1)
MESH_CONTACT_ARROW (below) controls the yellow 3D contact arrow on the mesh panel:
reference_normal_force — Fn at which arrow reaches full size (log/power/linear curve)
force_strength_mode — "log" | "power" | "linear"
min/max_length_fraction, min/max_thickness_scale, color
Mesh panel has two sizes (edit MESH_RENDER_* and TOP_ROW mesh slot):
display (TOP_ROW mesh width_px/height_px) — Qt widget size on screen
render (MESH_RENDER_WIDTH/HEIGHT) — Open3D offscreen resolution (upscaled for display)
Performance (3D dashboard; edit DashboardLayout defaults):
timer_interval_ms — Qt timer period (16 ≈ 60 Hz tick rate)
mesh_render_stride — render 3D mesh every N frames (2 ≈ half mesh cost)
chart_update_stride — update force charts every N frames
mesh_async_render — background mesh render (Linux only; Windows uses sync)
Edit TOP_ROW.slots to reorder or resize panels.
Edit PANEL_SETUP to change top vs log proportions.
Edit LOG_CONFIG for live status line and log panel appearance.
"""
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import Literal
from viz.contact_arrow import ContactArrowStyle, DEFAULT_CONTACT_ARROW_STYLE
PanelId = Literal["mesh", "arrows", "charts", "log"]
@@ -73,6 +98,91 @@ PANEL_SETUP = PanelSetup(
log_stretch=1,
)
# Open3D offscreen resolution for the mesh panel (independent of on-screen widget size).
MESH_RENDER_WIDTH_PX = 256
MESH_RENDER_HEIGHT_PX = 256
@dataclass(frozen=True)
class MeshLighting:
"""3D mesh panel light — edit MESH_LIGHTING or DashboardLayout.mesh_lighting."""
azimuth_deg: float = -45.0 # finger outline
elevation_deg: float = -35.0 # tip to root
intensity: float = 75000.0
ambient: float = 0.35
diffuse: float = 0.65
@property
def direction(self) -> tuple[float, float, float]:
"""Unit light direction derived from azimuth / elevation (degrees)."""
az = math.radians(self.azimuth_deg)
el = math.radians(self.elevation_deg)
cos_el = math.cos(el)
return (cos_el * math.cos(az), cos_el * math.sin(az), math.sin(el))
MESH_LIGHTING = MeshLighting(
azimuth_deg=-45.0,
elevation_deg=0,
intensity=75000.0,
ambient=0.5,
diffuse=0.5,
)
# 3D mesh contact arrow — edit here for dashboard tuning (see ContactArrowStyle in viz/contact_arrow.py).
MESH_CONTACT_ARROW = ContactArrowStyle(
reference_normal_force=10000.0,
force_strength_mode="log",
force_strength_gamma=0.5,
min_length_fraction=0.02,
max_length_fraction=0.30,
min_thickness_scale=0.20,
max_thickness_scale=1.0,
normal_force_scale=1.0,
color=(1.0, 0.92, 0.10),
)
@dataclass(frozen=True)
class LogConfig:
"""Log panel — edit LOG_CONFIG or DashboardLayout.log_config."""
status_enabled: bool = True
max_lines: int = 2000
font_family: str = "Consolas"
font_pt: int = 12
show_flow: bool = True
show_mask: bool = True
def format_status(
self,
*,
fps: float,
fnormal: float,
fshearx: float,
fsheary: float,
flow_rms: float = 0.0,
mask_coverage: float = 0.0,
wall_fps: float | None = None,
) -> str:
shear_mag = float((fshearx**2 + fsheary**2) ** 0.5)
if wall_fps is not None:
parts = [f"FPS={wall_fps:.1f}", f"sdk={fps:.1f}"]
else:
parts = [f"FPS={fps:.1f}"]
parts.extend(
[
f"Fn={fnormal:.1f}",
f"|Fs|={shear_mag:.1f}",
f"Fs=({fshearx:.1f}, {fsheary:.1f})",
]
)
return " ".join(parts)
LOG_CONFIG = LogConfig()
@dataclass(frozen=True)
class DashboardLayout:
@@ -100,7 +210,27 @@ class DashboardLayout:
# UV arrow panel: min flow magnitude to draw an arrow (display only; not dd02-ov.json contact.flow_threshold)
arrow_flow_threshold: float = 1.0
timer_interval_ms: int = 10
mesh_lighting: MeshLighting = MESH_LIGHTING
mesh_contact_arrow: ContactArrowStyle = MESH_CONTACT_ARROW
log_config: LogConfig = LOG_CONFIG
mesh_render_width_px: int = MESH_RENDER_WIDTH_PX
mesh_render_height_px: int = MESH_RENDER_HEIGHT_PX
timer_interval_ms: int = 16
mesh_async_render: bool = True
mesh_render_stride: int = 2
chart_update_stride: int = 2
def mesh_display_size(self) -> tuple[int, int]:
"""On-screen mesh panel size from TOP_ROW mesh slot."""
slot = self.top_slot("mesh")
return slot.width_px, slot.height_px
def mesh_render_size(self) -> tuple[int, int]:
"""Open3D offscreen render target size (may differ from display size)."""
return self.mesh_render_width_px, self.mesh_render_height_px
def slot(self, panel_id: PanelId) -> PanelSlot | None:
if self.panel_setup.log_panel.id == panel_id:
+74
View File
@@ -0,0 +1,74 @@
"""Background Open3D mesh rendering for the Qt dashboard."""
from __future__ import annotations
import numpy as np
from PyQt5.QtCore import QMutex, QMutexLocker, QThread, pyqtSignal
class MeshRenderWorker(QThread):
"""Render finger mesh off the GUI thread; drops stale frames."""
rgb_ready = pyqtSignal(object)
def __init__(self, parent=None):
super().__init__(parent)
self._mutex = QMutex()
self._stop = False
self._wake = False
self._renderer = None
self._pending: dict | None = None
def set_renderer(self, renderer) -> None:
with QMutexLocker(self._mutex):
self._renderer = renderer
def clear_renderer(self) -> None:
with QMutexLocker(self._mutex):
self._renderer = None
self._pending = None
def submit(self, *, flow, scalar_map, centroid, centroid_found, fnormal) -> None:
with QMutexLocker(self._mutex):
self._pending = {
"flow": flow,
"scalar_map": scalar_map,
"centroid": centroid,
"centroid_found": centroid_found,
"fnormal": fnormal,
}
self._wake = True
if not self.isRunning():
self.start()
def stop_worker(self) -> None:
with QMutexLocker(self._mutex):
self._stop = True
self._wake = True
self.wait(2000)
def run(self) -> None:
while True:
with QMutexLocker(self._mutex):
if self._stop:
break
if not self._wake or self._renderer is None or self._pending is None:
self.msleep(1)
continue
job = self._pending
self._pending = None
self._wake = False
renderer = self._renderer
rgb = renderer.render(
flow=job["flow"],
scalar_map=job["scalar_map"],
centroid=job["centroid"],
centroid_found=job["centroid_found"],
fnormal=job["fnormal"],
)
self.rgb_ready.emit(np.ascontiguousarray(rgb))
with QMutexLocker(self._mutex):
if self._pending is not None:
self._wake = True
+24 -22
View File
@@ -66,6 +66,7 @@ class ForceChartWidget:
def _init_plot_buffer(self) -> None:
self.data = np.full((self.npoints, 2, 3), np.nan, dtype=np.float64)
self._plot_sample_count = 0
self._time_origin: float | None = None
self._y_range_smooth = [None, None, None]
def _create_plots(self) -> None:
@@ -134,6 +135,9 @@ class ForceChartWidget:
plot.setXRange(0, self._plot_window_sec, padding=0)
def update_frame(self, frame: FingerFrameView) -> None:
if self._time_origin is None:
self._time_origin = frame.timestamp
if self._plot_sample_count < self.npoints:
idx = self._plot_sample_count
self.data[idx, 0, :] = frame.timestamp
@@ -151,36 +155,36 @@ class ForceChartWidget:
self.data[-1, 1, 2] = frame.fsheary
n = self._plot_sample_count
if n == 0:
if n == 0 or self._time_origin is None:
return
layout = self._layout
data_plot = self.data[:n].copy()
times = data_plot[:, 0, 0] - float(data_plot[0, 0, 0])
data_plot = self.data[:n]
times = data_plot[:, 0, 0] - self._time_origin
x_end = float(times[-1])
if x_end < self._plot_window_sec:
x_start = 0.0
x_end = self._plot_window_sec
else:
x_start = x_end - self._plot_window_sec
for channel in range(3):
data_plot[:, 1, channel] = lowpass_filter(
data_plot[:, 1, channel],
y = lowpass_filter(
data_plot[:, 1, channel].copy(),
self.lowpass_cutoff_hz,
sample_rate_hz=self.sample_rate_hz,
)
x_end = float(times[-1])
x_start = max(0.0, x_end - self._plot_window_sec)
if x_end - x_start < self._plot_window_sec:
x_start = 0.0
x_end = max(self._plot_window_sec, x_end)
for i, curve in enumerate(self.curves):
y = data_plot[:, 1, i]
curve = self.curves[channel]
curve.setData(times, y)
finite = np.isfinite(y)
if not np.any(finite):
continue
ymin, ymax = float(np.min(y[finite])), float(np.max(y[finite]))
if self._y_range_smooth[i] is None:
self._y_range_smooth[i] = (ymin, ymax)
if self._y_range_smooth[channel] is None:
self._y_range_smooth[channel] = (ymin, ymax)
else:
prev_min, prev_max = self._y_range_smooth[i]
prev_min, prev_max = self._y_range_smooth[channel]
alpha = self._y_range_alpha
ymin = alpha * ymin + (1 - alpha) * prev_min
ymax = alpha * ymax + (1 - alpha) * prev_max
@@ -190,13 +194,11 @@ class ForceChartWidget:
half = layout.y_axis_min_span / 2
ymin = mid - half
ymax = mid + half
self._y_range_smooth[i] = (ymin, ymax)
self._y_range_smooth[channel] = (ymin, ymax)
pad = max((ymax - ymin) * 0.12, 0.01)
self.plots[i].setYRange(ymin - pad, ymax + pad)
last_x = times[-1:]
last_y = y[-1:]
self.dot_glows[i].setData(last_x, last_y)
self.dot_cores[i].setData(last_x, last_y)
self.plots[channel].setYRange(ymin - pad, ymax + pad)
self.dot_glows[channel].setData(times[-1:], y[-1:])
self.dot_cores[channel].setData(times[-1:], y[-1:])
for plot in self.plots:
plot.setXRange(x_start, x_end, padding=0)
+45 -7
View File
@@ -2,7 +2,7 @@
from __future__ import annotations
from PyQt5.QtGui import QFont
from PyQt5.QtGui import QFont, QTextCursor
from PyQt5.QtWidgets import QPlainTextEdit
@@ -22,30 +22,68 @@ class TeeStream:
class WidgetLogHandler:
"""Mirror print output into a QPlainTextEdit."""
"""Mirror print output into a QPlainTextEdit with a live status line at the bottom."""
def __init__(self, widget: QPlainTextEdit):
self.widget = widget
self.buf = ""
self._has_status_line = False
def write(self, text: str):
self.buf += text
while "\n" in self.buf:
line, self.buf = self.buf.split("\n", 1)
if line:
self.widget.appendPlainText(line)
self._append_log_line(line)
def flush(self):
if self.buf:
self.widget.appendPlainText(self.buf)
self._append_log_line(self.buf)
self.buf = ""
def set_status_line(self, text: str) -> None:
"""Update the pinned bottom line (FPS / forces) without scrolling the log."""
text = text.rstrip("\n")
if self._has_status_line:
cursor = QTextCursor(self.widget.document().lastBlock())
cursor.select(QTextCursor.BlockUnderCursor)
cursor.insertText(text)
else:
self.widget.appendPlainText(text)
self._has_status_line = True
scrollbar = self.widget.verticalScrollBar()
scrollbar.setValue(scrollbar.maximum())
def make_log_widget() -> QPlainTextEdit:
def clear_status_line(self) -> None:
if not self._has_status_line:
return
cursor = QTextCursor(self.widget.document().lastBlock())
cursor.select(QTextCursor.BlockUnderCursor)
cursor.removeSelectedText()
self._has_status_line = False
def _append_log_line(self, line: str) -> None:
status = None
if self._has_status_line:
status = self.widget.document().lastBlock().text()
cursor = QTextCursor(self.widget.document().lastBlock())
cursor.select(QTextCursor.BlockUnderCursor)
cursor.removeSelectedText()
self._has_status_line = False
self.widget.appendPlainText(line)
if status is not None:
self.set_status_line(status)
def make_log_widget(config=None) -> QPlainTextEdit:
if config is None:
from ..layout import LOG_CONFIG
config = LOG_CONFIG
widget = QPlainTextEdit()
widget.setReadOnly(True)
widget.setMaximumBlockCount(2000)
font = QFont("Consolas", 12)
widget.setMaximumBlockCount(config.max_lines)
font = QFont(config.font_family, config.font_pt)
font.setStyleHint(QFont.Monospace)
widget.setFont(font)
return widget
+47 -10
View File
@@ -2,10 +2,13 @@
from __future__ import annotations
import sys
from pipeline.frame import FingerFrameView
from viz.offscreen_mesh import OffscreenFingerMeshRenderer
from ..layout import DashboardLayout, LAYOUT
from ..mesh_render_worker import MeshRenderWorker
from .qt_pixmap import ndarray_to_pixmap
from .image_panel import ImagePanel
@@ -16,14 +19,15 @@ class MeshViewportWidget(ImagePanel):
def __init__(self, parent=None, *, layout: DashboardLayout = LAYOUT):
self._layout = layout
self._renderer: OffscreenFingerMeshRenderer | None = None
self._mesh_worker: MeshRenderWorker | None = None
self._mesh_map = "depth"
self._threshold = 0.0
self._enable_3d = True
slot = layout.top_slot("mesh")
display_w, display_h = layout.mesh_display_size()
super().__init__(
title=layout.mesh_panel_title,
object_name="meshViewport",
default_side=min(slot.width_px, slot.height_px),
default_side=min(display_w, display_h),
parent=parent,
)
@@ -31,6 +35,7 @@ class MeshViewportWidget(ImagePanel):
self._mesh_map = pipeline.config.mesh_map
self._threshold = pipeline.config.flow_threshold
self._enable_3d = pipeline.config.enable_3d and pipeline.uv_mesh is not None
self._stop_worker()
if self._renderer is not None:
self._renderer.close()
self._renderer = None
@@ -42,28 +47,58 @@ class MeshViewportWidget(ImagePanel):
return
self.label.setText("")
self._init_renderer(pipeline)
if self._layout.mesh_async_render and sys.platform != "win32":
self._mesh_worker = MeshRenderWorker(self)
self._mesh_worker.rgb_ready.connect(self._on_rgb_ready)
self._mesh_worker.set_renderer(self._renderer)
def _init_renderer(self, pipeline) -> None:
if not self._enable_3d or pipeline.uv_mesh is None:
return
side = self.side_length
render_w, render_h = self._layout.mesh_render_size()
self._renderer = OffscreenFingerMeshRenderer(
pipeline.uv_mesh,
width=side,
height=side,
width=render_w,
height=render_h,
threshold=self._threshold,
color_mode=self._mesh_map,
world_to_cam=pipeline.world_to_cam,
lookat=pipeline.mesh_lookat,
mesh_lighting=self._layout.mesh_lighting,
contact_arrow_style=self._layout.mesh_contact_arrow,
)
def on_side_length_changed(self, side: int) -> None:
if self._renderer is not None:
self._renderer.resize(side, side)
def _stop_worker(self) -> None:
if self._mesh_worker is not None:
self._mesh_worker.stop_worker()
self._mesh_worker = None
def update_frame(self, frame: FingerFrameView) -> None:
def on_side_length_changed(self, side: int) -> None:
"""Display widget resized; offscreen render size stays on layout.mesh_render_*."""
del side # display size only; ImagePanel upscales the pixmap to fit the label
def _on_rgb_ready(self, rgb) -> None:
self.set_source_pixmap(ndarray_to_pixmap(rgb, bgr=False))
def update_frame(self, frame: FingerFrameView, *, async_render: bool | None = None) -> float:
"""Update mesh colors. Returns render wall time (0 when async submit only)."""
if not self._enable_3d or self._renderer is None:
return
return 0.0
use_async = self._layout.mesh_async_render if async_render is None else async_render
if use_async and self._mesh_worker is not None:
self._mesh_worker.submit(
flow=frame.flow,
scalar_map=frame.depth_map,
centroid=frame.centroid,
centroid_found=frame.centroid_found,
fnormal=frame.fnormal,
)
return 0.0
import time
t0 = time.perf_counter()
rgb = self._renderer.render(
flow=frame.flow,
scalar_map=frame.depth_map,
@@ -72,8 +107,10 @@ class MeshViewportWidget(ImagePanel):
fnormal=frame.fnormal,
)
self.set_source_pixmap(ndarray_to_pixmap(rgb, bgr=False))
return (time.perf_counter() - t0) * 1000.0
def close_renderer(self) -> None:
self._stop_worker()
if self._renderer is not None:
self._renderer.close()
self._renderer = None