diff --git a/.gitignore b/.gitignore index 0db79d4..dd9df22 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,12 @@ __pycache__/ # Packaging / build outputs /build/nuitka/ -/build/lib/orisys/ +/build/build/ +/build/bdist.*/ +/build/dist/ +/build/lib/ /build/pyinstaller_work/ +/build/*.spec /dist/ /dist_obfuscated/ /dist_pyd/ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..2a0ae0d --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ +# Non-Python files needed for sdist -> wheel builds from an unpacked sdist. +include pyproject.toml +include setup.py +include README.md +recursive-include config *.json +recursive-include sensors * +recursive-include build/common *.py +recursive-include build/wheel *.py diff --git a/README.md b/README.md index e21d31b..4875b4a 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,13 @@ Orisys SDK is a Python toolkit for optical tactile sensing. It provides image ca stitching, dense optical flow, Helmholtz-Hodge decomposition, contact/depth estimation, and several runnable examples for camera and video-based workflows. +PyPI: [https://pypi.org/project/orisys/](https://pypi.org/project/orisys/) + ## Repository Layout - `src/orisys/`: installable SDK package - `config/`: default and lab/runtime JSON configurations -- `examples/`: user-facing example scripts +- `examples/`: runnable demos (`basic_pipeline.py`, `mini_pipeline.py`, `finger_pipeline*.py`) plus shared `gui/`, `pipeline/`, `viz/` - `scripts/`: maintenance, calibration, packaging, and demo scripts - `tools/`: non-package tooling such as patch-generation utilities - `tests/`: automated pytest suite @@ -49,38 +51,67 @@ pytest ### 4. Run an example +ORY-BOX: + ```bash -python examples/basic_pipeline.py -v 0 -c ./config/ddjx01.json +python examples/basic_pipeline.py -v 0 -c box ``` +ORY-MINI: + +```bash +python examples/mini_pipeline.py -v 0 -c mini +``` + +ORY-FINGER: + +```bash +python examples/finger_pipeline.py --camera_id 0 --config finger +``` + +Run commands from the repository root. + ## Configuration The canonical configuration files now live in the repo-level `config/` folder. The package loader `orisys.configs` resolves configs from: 1. an explicit JSON file path -2. a config file stem under `config/` -3. a backward-compatible JSON `"name"` match, such as `"prototype"` +2. a config file stem under `config/` (for example `box`, `mini`, `finger`) +3. a product alias such as `dd02` -> `dd02-ov` +4. a backward-compatible JSON `"name"` match, such as `"prototype"` Examples: ```python import orisys -sensor = orisys.Sensor(0, config_name="ddjx01") -cfg = orisys.export_config("ddjx01", "./config/exported_config.json") +sensor = orisys.Sensor(0, config_name="box") +cfg = orisys.export_config("box", "./config/exported_config.json") ``` ## Examples and Tools - Example entry points: `examples/README.md` - User documentation: `docs/get_started.md` -- Packaging helper: `build/build_nuitka.py` (compiled SDK wheel) -- GUI standalone folder: `build/build_gui_win.bat` → `dist/OrisysDashboard/` +- Build launcher: `python build/run.py` (see `build/README.md`) +- Manylinux / PyPI binary wheel: `docs/publishing.md` - Specialized utilities: `tools/` +## Publishing (Linux manylinux) + +To build PyPI-compliant CPython 3.10/3.11/3.12 Linux wheels from Windows Docker Desktop: + +```powershell +python build/run.py manylinux +``` + +Upload candidates land in `dist/repaired/`. Full TestPyPI → PyPI steps: +`docs/publishing.md`. + ## Notes for Collaborators - Generated build outputs such as `build/`, `dist/`, and `SDK/` are intentionally ignored. - Large or generated tooling outputs should stay under ignored output folders rather than be committed. - If you add new images or arrays that should be versioned, review `.gitignore` first. +- Never commit PyPI / TestPyPI API tokens. diff --git a/assets/finger/30fps-0616/mesh.npz b/assets/finger/30fps-0616/mesh.npz deleted file mode 100644 index ecb8980..0000000 Binary files a/assets/finger/30fps-0616/mesh.npz and /dev/null differ diff --git a/assets/finger/30fps-0616/unroll_map_uv_s0.4.npz b/assets/finger/30fps-0616/unroll_map_uv_s0.4.npz deleted file mode 100644 index e9a8bf1..0000000 Binary files a/assets/finger/30fps-0616/unroll_map_uv_s0.4.npz and /dev/null differ diff --git a/assets/finger/30fps-0616/world_to_cam.npy b/assets/finger/30fps-0616/world_to_cam.npy deleted file mode 100644 index 1923088..0000000 Binary files a/assets/finger/30fps-0616/world_to_cam.npy and /dev/null differ diff --git a/assets/finger/30fps/camera_intrinsics.npz b/assets/finger/30fps/camera_intrinsics.npz deleted file mode 100644 index 59ac687..0000000 Binary files a/assets/finger/30fps/camera_intrinsics.npz and /dev/null differ diff --git a/assets/finger/30fps/unroll_map_uv.npz b/assets/finger/30fps/unroll_map_uv.npz deleted file mode 100644 index 8415977..0000000 Binary files a/assets/finger/30fps/unroll_map_uv.npz and /dev/null differ diff --git a/assets/finger/30fps/unroll_map_uv_s0.4.npz b/assets/finger/30fps/unroll_map_uv_s0.4.npz deleted file mode 100644 index aafa344..0000000 Binary files a/assets/finger/30fps/unroll_map_uv_s0.4.npz and /dev/null differ diff --git a/assets/finger/visualization/flatten_remap.npz b/assets/finger/visualization/flatten_remap.npz deleted file mode 100644 index 02ca287..0000000 Binary files a/assets/finger/visualization/flatten_remap.npz and /dev/null differ diff --git a/assets/finger/visualization/mesh.npz b/assets/finger/visualization/mesh.npz deleted file mode 100644 index ad81542..0000000 Binary files a/assets/finger/visualization/mesh.npz and /dev/null differ diff --git a/docs/images/orisys_logo.png b/assets/orisys_logo.png similarity index 100% rename from docs/images/orisys_logo.png rename to assets/orisys_logo.png diff --git a/docs/images/orisys_logo_notext.png b/assets/orisys_logo_notext.png similarity index 100% rename from docs/images/orisys_logo_notext.png rename to assets/orisys_logo_notext.png diff --git a/build/OrisysDashboard.spec b/build/OrisysDashboard.spec deleted file mode 100644 index 47bae68..0000000 --- a/build/OrisysDashboard.spec +++ /dev/null @@ -1,44 +0,0 @@ -# -*- mode: python ; coding: utf-8 -*- - - -a = Analysis( - ['D:\\repo\\orisyssdk_git\\examples\\finger_pipeline_3dview.py'], - pathex=[], - binaries=[], - datas=[('D:\\repo\\orisyssdk_git\\config', 'config'), ('D:\\repo\\orisyssdk_git\\assets', 'assets')], - hiddenimports=['PyQt5', 'pyqtgraph', 'open3d', 'cv2', 'orisys', 'gui', 'pipeline', 'viz', 'app_paths', 'plotly', '_plotly_utils', 'open3d.cpu.pybind'], - hookspath=[], - hooksconfig={}, - runtime_hooks=[], - excludes=['cupy', 'cupyx', 'pyopencl', 'tkinter', 'pip', 'wheel', 'matplotlib'], - noarchive=False, - optimize=0, -) -pyz = PYZ(a.pure) - -exe = EXE( - pyz, - a.scripts, - [], - exclude_binaries=True, - name='OrisysDashboard', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=True, - console=False, - disable_windowed_traceback=False, - argv_emulation=False, - target_arch=None, - codesign_identity=None, - entitlements_file=None, -) -coll = COLLECT( - exe, - a.binaries, - a.datas, - strip=False, - upx=True, - upx_exclude=[], - name='OrisysDashboard', -) diff --git a/build/README.md b/build/README.md new file mode 100644 index 0000000..fce906d --- /dev/null +++ b/build/README.md @@ -0,0 +1,65 @@ +# Build tooling + +Use one launcher from the repo root: + +```bash +python build/run.py +``` + +```text +build/ + run.py << single entrypoint + README.md + common/ shared helpers (paths, pyproject, verify) + wheel/ SDK .whl (Nuitka + manylinux Docker) + gui/ + common/ shared PyInstaller driver + ory_finger/ ORY-FINGER dashboard + ory_mini/ ORY-MINI internal GUI (scaffold) + ory_box/ ORY-BOX internal GUI (scaffold) +``` + +## Commands + +### SDK wheel + +```bash +python build/run.py wheel-win +python build/run.py wheel-manylinux + +# Single Version +python build/run.py wheel +# Inside Docker +python build/run.py wheel --target pypi-linux --verify +``` + +Direct module paths (same behavior): + +```bash +python build/wheel/build.py +build\wheel\win.bat +.\build\wheel\docker\run.ps1 +``` + +### GUI + +```bash +python build/run.py gui finger +python build/run.py gui mini # scaffold / NotImplemented +python build/run.py gui box # scaffold / NotImplemented +``` + +### Verify + +```bash +python build/run.py verify path\to\orisys.whl +python build/run.py verify --gui-dist --product ory_finger +``` + +## Outputs + +- Wheels: `SDK//`, `dist/repaired/` +- Finger GUI: `dist/OrisysDashboard/` +- Future mini/box: `dist/OryMini/`, `dist/OryBox/` + +Generated work files under `build/nuitka/` and `build/pyinstaller_work/` are gitignored. diff --git a/build/build_gui_win_pyinstaller.py b/build/build_gui_win_pyinstaller.py deleted file mode 100644 index 9ca0284..0000000 --- a/build/build_gui_win_pyinstaller.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Build a CPU-only standalone Windows folder for Orisys 3D dashboard using PyInstaller. - -Usage: - conda activate py310 # or a clean virtualenv with only required packages - pip install pyinstaller - python build/build_gui_win_pyinstaller.py - -Output: - dist/OrisysDashboard/OrisysDashboard.exe (+ bundled Python, DLLs, config/, assets/) -""" - -from __future__ import annotations - -import os -import shutil -import subprocess -import sys -from pathlib import Path - -# ---------- Configuration ---------- -ENTRY_SCRIPT = Path("examples/finger_pipeline_3dview.py").resolve() -EXE_NAME = "OrisysDashboard.exe" -DIST_NAME = "OrisysDashboard" # final folder name -repo_root = Path(__file__).parent.parent - -# Packages you want to force-include (PyInstaller often auto-detects them, but explicit avoids misses) -HIDDEN_IMPORTS = [ - "PyQt5", - "pyqtgraph", - "open3d", - "cv2", - "orisys", - "gui", - "pipeline", - "viz", - "app_paths", - "plotly", - "_plotly_utils", - # sometimes needed for open3d: - "open3d.cpu.pybind", # adjust to actual module if needed -] - -# Modules to completely exclude (they won't be imported anyway if absent in venv) -EXCLUDE_MODULES = [ - "cupy", - "cupyx", - "pyopencl", - "tkinter", - # "unittest", - # "email", - # "http", - # "xmlrpc", - # "pkg_resources", - # "setuptools", - "pip", - "wheel", - "matplotlib", # unless you use it - # "scipy", # unless you use it -] - -# Data directories: (source relative to repo root, destination in bundle) -DATA_DIRS = [ - ("config", "config"), - ("assets", "assets"), -] - -# Optional: UPX directory (download from https://upx.github.io) -UPX_DIR = r"C:\tools\upx" # set to None if not available -# ---------- End Configuration ---------- - - -def clean_dist(dist_path: Path) -> None: - if dist_path.exists(): - shutil.rmtree(dist_path) - dist_path.mkdir(parents=True) - - -def build_with_pyinstaller(): - # Ensure we are in a clean, minimal venv - print("Using Python:", sys.executable) - - # Output folder for PyInstaller work files - work_dir = Path("build/pyinstaller_work") - dist_dir = Path("dist") / DIST_NAME - clean_dist(work_dir) - clean_dist(dist_dir) - - # Build PyInstaller command - cmd = [ - sys.executable, "-m", "PyInstaller", - "--onedir", # produce a folder, not a single exe - "--name", DIST_NAME, - f"--distpath={dist_dir.parent}", - f"--workpath={work_dir}", - f"--specpath=build", - "--noconfirm", - "--clean", - "--windowed", # or "--console" if you need terminal - ] - - # Add data files - for src_rel, dest_rel in DATA_DIRS: - src = Path(src_rel).resolve() - if not src.is_dir(): - raise FileNotFoundError(f"Data directory missing: {src}") - cmd.append(f"--add-data={src};{dest_rel}") - - # Logo file if exists - logo = Path("logo.png") - if logo.exists(): - cmd.append(f"--add-data={logo.resolve()};.") - - # Hidden imports - for mod in HIDDEN_IMPORTS: - cmd.append(f"--hidden-import={mod}") - - # Exclude modules - for mod in EXCLUDE_MODULES: - cmd.append(f"--exclude-module={mod}") - - # UPX if available - if UPX_DIR and Path(UPX_DIR).is_dir(): - cmd.append(f"--upx-dir={UPX_DIR}") - - # Entry point - cmd.append(str(ENTRY_SCRIPT)) - - # Environment (same as Nuitka script) - env = os.environ.copy() - env["PYTHONPATH"] = str(Path("src").resolve()) + os.pathsep + str(Path("examples").resolve()) + os.pathsep + env.get("PYTHONPATH", "") - - print("\n=== Running PyInstaller ===") - print("Command:", " ".join(cmd)) - subprocess.check_call(cmd, env=env) - - # After build, copy data directories explicitly - # PyInstaller output folder - dist_dir = Path("dist") / DIST_NAME - final_exe = dist_dir / EXE_NAME - if not final_exe.is_file(): - raise RuntimeError(f"Executable not found at {final_exe}") - print("\n=== Copying data directories ===") - for src_rel, dest_rel in DATA_DIRS: - src = repo_root / src_rel # use an absolute reference to repo root - dest = dist_dir / dest_rel - if not src.is_dir(): - raise FileNotFoundError(f"Data directory not found: {src}") - if dest.exists(): - shutil.rmtree(dest) - shutil.copytree(src, dest, dirs_exist_ok=True) # Python 3.8+ use dirs_exist_ok=True - print(f" {src} -> {dest}") - - # Copy logo - logo_src = repo_root / "logo.png" - if logo_src.is_file(): - shutil.copy2(logo_src, dist_dir / "logo.png") - print(f" Logo copied") - - # Write README (same as Nuitka script) - readme = dist_dir / "README.txt" - readme.write_text( - "\n".join([ - "Orisys Finger Dashboard (standalone)", - "", - "Run:", - f" {EXE_NAME}", - f" {EXE_NAME} --video path\\to\\video.mp4", - f" {EXE_NAME} --config config\\dd02-ov.json", - "", - "Bundled folders:", - " config/ sensor JSON configs", - " assets/ finger mesh and visualization data", - "", - "CPU-only build (no CUDA/CuPy).", - ]), - encoding="utf-8", - ) - - print(f"\n[SUCCESS] Standalone GUI folder: {dist_dir}") - print(f"Executable: {final_exe}") - - # Optional smoke test (same as Nuitka script) - try: - from verify import verify_gui_dist - print("\n=== Smoke test ===") - verify_gui_dist(dist_dir) - except Exception as exc: - print(f"\n[WARNING] Smoke test failed: {exc}") - - return dist_dir - - -if __name__ == "__main__": - build_with_pyinstaller() \ No newline at end of file diff --git a/build/build_nuitka.py b/build/build_nuitka.py deleted file mode 100644 index 23249bf..0000000 --- a/build/build_nuitka.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Build a Nuitka-compiled orisys wheel and publish it to SDK/{version}/. - -Pipeline: - 1. Compile src/orisys → .pyd - 2. Stage wheel tree (compiled module + bundled files from pyproject.toml) - 3. Run bdist_wheel and move output to SDK/{version}/ - 4. Smoke-test imports in a temporary venv - -Run from repo root: python build/build_nuitka.py -""" - -from __future__ import annotations - -import os -import shutil -import subprocess - -from nuitka_compile import compile_orisys, copy_compiled_orisys, install_nuitka -from paths import NUITKA_COMPILE_DIR, NUITKA_WHEEL_DIR, REPO_ROOT, repo_path -from pyproject_utils import bundle_items, extras_require, get_version, install_requires -from verify import verify_built_wheel -from wheel_build import publish_wheel_to_sdk, run_bdist_wheel, write_wheel_setup -from wheel_layout import copy_bundle_items, package_data_from_bundle - - -def _reset_dir(path) -> None: - """Remove and recreate a build output directory.""" - if path.exists(): - shutil.rmtree(path) - path.mkdir(parents=True) - - -def build() -> None: - # --- 1. Compile --- - install_nuitka() - - version = get_version(REPO_ROOT) - _reset_dir(NUITKA_COMPILE_DIR) - _reset_dir(NUITKA_WHEEL_DIR) - - print("\n=== Compiling with Nuitka ===") - env = os.environ.copy() - src_dir = repo_path("src") - env["PYTHONPATH"] = src_dir.as_posix() + os.pathsep + env.get("PYTHONPATH", "") - ext_file = compile_orisys(NUITKA_COMPILE_DIR, env) - - # --- 2. Stage wheel contents --- - print("\n=== Preparing Wheel Structure ===") - pkg_dir = NUITKA_WHEEL_DIR / "wheel_root" / "orisys" - pkg_dir.mkdir(parents=True) - - ext_suffix = copy_compiled_orisys(pkg_dir, ext_file) - - # Bundle rules live in pyproject.toml → [[tool.orisys.bundle.items]] - bundle = bundle_items(REPO_ROOT) - copied_counts = copy_bundle_items(REPO_ROOT, pkg_dir, bundle) - print(f"Copied {copied_counts['configs']} JSON config files") - print(f"Copied {copied_counts['examples']} example files to orisys/examples/") - print(f"Copied {copied_counts['docs']} doc files to orisys/docs/") - print(f"Copied {copied_counts['assets']} finger asset files to orisys/assets/") - - write_wheel_setup( - NUITKA_WHEEL_DIR, - version=version, - package_data=package_data_from_bundle(ext_suffix, bundle), - install_requires=install_requires(REPO_ROOT), - extras_require=extras_require(REPO_ROOT), - ) - - # --- 3. Build wheel and publish to SDK/ --- - wheel_path = run_bdist_wheel(NUITKA_WHEEL_DIR) - if wheel_path is None: - print("\n[ERROR] Wheel not found.") - return - - final_wheel = publish_wheel_to_sdk(wheel_path, version) - print(f"\n[SUCCESS] Wheel created and moved to: {final_wheel}") - - # # --- 4. Verify (isolated venv; does not touch the active environment) --- - # print("\n=== Verifying finger mesh/viewer imports ===") - # try: - # verify_built_wheel(final_wheel) - # except subprocess.CalledProcessError as exc: - # print(f"\n[WARNING] Wheel verify failed: {exc}") - # print("The wheel was built successfully; verification runs in an isolated temp venv.") - - -if __name__ == "__main__": - build() diff --git a/build/common/__init__.py b/build/common/__init__.py new file mode 100644 index 0000000..0404422 --- /dev/null +++ b/build/common/__init__.py @@ -0,0 +1 @@ +"""Shared build utilities used by wheel and GUI packaging targets.""" diff --git a/build/common/bootstrap.py b/build/common/bootstrap.py new file mode 100644 index 0000000..7606ab0 --- /dev/null +++ b/build/common/bootstrap.py @@ -0,0 +1,15 @@ +"""Ensure ``build/`` is importable as the root for ``common``, ``wheel``, and ``gui``.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[1] + + +def ensure_build_root_on_path() -> Path: + root = str(_BUILD_ROOT) + if root not in sys.path: + sys.path.insert(0, root) + return _BUILD_ROOT diff --git a/build/common/paths.py b/build/common/paths.py new file mode 100644 index 0000000..d5163d5 --- /dev/null +++ b/build/common/paths.py @@ -0,0 +1,31 @@ +"""Repository and packaging output paths.""" + +from __future__ import annotations + +from pathlib import Path + +# build/common/paths.py → parents: common, build, repo +REPO_ROOT = Path(__file__).resolve().parents[2] + +NUITKA_COMPILE_DIR = REPO_ROOT / "build" / "nuitka" / "compile" +NUITKA_WHEEL_DIR = REPO_ROOT / "build" / "nuitka" / "wheel" +NUITKA_GUI_DIR = REPO_ROOT / "build" / "nuitka" / "gui" +GUI_DIST_DIR = REPO_ROOT / "dist" / "OrisysDashboard" +PYINSTALLER_WORK_DIR = REPO_ROOT / "build" / "pyinstaller_work" + + +def repo_path(*parts: str) -> Path: + return REPO_ROOT.joinpath(*parts) + + +def sdk_dir(version: str) -> Path: + return REPO_ROOT / "SDK" / version + + +def repaired_wheel_dir() -> Path: + """Directory for auditwheel-repaired wheels under the Nuitka wheel build tree.""" + return NUITKA_WHEEL_DIR / "repaired" + + +def gui_dist_dir(dist_name: str) -> Path: + return REPO_ROOT / "dist" / dist_name diff --git a/build/pyproject_utils.py b/build/common/pyproject_utils.py similarity index 51% rename from build/pyproject_utils.py rename to build/common/pyproject_utils.py index dc1f73f..c46ff34 100644 --- a/build/pyproject_utils.py +++ b/build/common/pyproject_utils.py @@ -2,11 +2,25 @@ from __future__ import annotations -import importlib +import ast +import importlib.util +import shutil import subprocess import sys from pathlib import Path +_BUILD_ROOT = Path(__file__).resolve().parents[1] + + +def _load_wheel_layout_module(): + layout_path = _BUILD_ROOT / "wheel" / "layout.py" + spec = importlib.util.spec_from_file_location("_orisys_build_wheel_layout", layout_path) + if spec is None or spec.loader is None: + raise ImportError(f"Cannot load wheel layout helper from {layout_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + def _load_toml(text: str) -> dict: if sys.version_info >= (3, 11): @@ -41,17 +55,29 @@ def extras_require(repo_root: Path) -> dict[str, list[str]]: def _resolve_version_attr(repo_root: Path, attr: str) -> str: """Resolve a setuptools dynamic version attr (e.g. orisys.__version__).""" - src_dir = str(repo_root / "src") - if src_dir not in sys.path: - sys.path.insert(0, src_dir) module_name, _, attr_name = attr.partition(".") if not attr_name: raise RuntimeError(f"Invalid dynamic version attr: {attr!r}") - module = importlib.import_module(module_name) - value = module - for name in attr_name.split("."): - value = getattr(value, name) - return str(value) + + # Avoid importing package modules here: __init__ may import heavy runtime deps + # (e.g., cv2/numpy) that are not required just to read __version__. + rel_module = Path(*module_name.split(".")) + module_file = repo_root / "src" / rel_module.with_suffix(".py") + if not module_file.is_file(): + module_file = repo_root / "src" / rel_module / "__init__.py" + if not module_file.is_file(): + raise RuntimeError(f"Cannot resolve module file for dynamic version attr: {attr!r}") + + tree = ast.parse(module_file.read_text(encoding="utf-8"), filename=str(module_file)) + target_name = attr_name.split(".")[0] + for node in tree.body: + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == target_name: + value = ast.literal_eval(node.value) + return str(value) + + raise RuntimeError(f"Attribute {target_name!r} not found in {module_file}") def get_version(repo_root: Path) -> str: @@ -77,8 +103,12 @@ def bundle_items(repo_root: Path) -> list[dict]: def bundle_data_files(repo_root: Path) -> list[tuple[str, list[str]]]: - """Map bundle items marked dev_install=true to setuptools data_files entries.""" - from wheel_layout import resolve_bundle_source_files + """Map bundle items marked ``dev_install=true`` to install destinations. + + Kept for tests / tooling. Prefer :func:`stage_dev_bundle_into` for pip + installs so files land inside ``site-packages/orisys/``. + """ + resolve_bundle_source_files = _load_wheel_layout_module().resolve_bundle_source_files grouped: dict[str, list[str]] = {} for item in bundle_items(repo_root): @@ -98,3 +128,33 @@ def bundle_data_files(repo_root: Path) -> list[tuple[str, list[str]]]: grouped.setdefault(dest, []).append(rel_src) return [(dest, sorted(files)) for dest, files in sorted(grouped.items())] + + +def stage_dev_bundle_into(repo_root: Path, pkg_dir: Path) -> list[str]: + """Copy ``dev_install`` bundle files under ``pkg_dir`` (the orisys package). + + Returns package-relative paths that were written (e.g. ``sensors/box/...``). + """ + resolve_bundle_source_files = _load_wheel_layout_module().resolve_bundle_source_files + packaged: list[str] = [] + + for item in bundle_items(repo_root): + if not item.get("dev_install", False): + continue + + src_dir = repo_root / item["source"] + dst_dir = pkg_dir / item["target"] + dst_dir.mkdir(parents=True, exist_ok=True) + + for src_file in resolve_bundle_source_files(repo_root, item): + if item.get("preserve_tree", False): + rel = src_file.relative_to(src_dir) + dst_file = dst_dir / rel + else: + rel = Path(src_file.name) + dst_file = dst_dir / src_file.name + dst_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_file, dst_file) + packaged.append(f"{item['target']}/{rel.as_posix()}") + + return packaged diff --git a/build/common/verify/__init__.py b/build/common/verify/__init__.py new file mode 100644 index 0000000..f9d7fa5 --- /dev/null +++ b/build/common/verify/__init__.py @@ -0,0 +1,6 @@ +"""Post-build smoke tests for wheels and product GUI folders.""" + +from common.verify.gui import GUI_PRODUCTS, verify_gui_dist +from common.verify.wheel import verify_built_wheel + +__all__ = ["GUI_PRODUCTS", "verify_built_wheel", "verify_gui_dist"] diff --git a/build/common/verify/cli.py b/build/common/verify/cli.py new file mode 100644 index 0000000..acf7f29 --- /dev/null +++ b/build/common/verify/cli.py @@ -0,0 +1,34 @@ +"""CLI for packaged artifact verification.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from common.verify import GUI_PRODUCTS, verify_built_wheel, verify_gui_dist + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Verify packaged Orisys artifacts.") + parser.add_argument("--gui-dist", action="store_true", help="Verify a GUI dist folder") + parser.add_argument( + "--product", + choices=sorted(GUI_PRODUCTS), + default="ory_finger", + help="GUI product profile (default: ory_finger)", + ) + parser.add_argument("--dist-dir", type=Path, default=None, help="Override GUI dist folder") + parser.add_argument("wheel", nargs="?", type=Path, help="Wheel path for import smoke test") + args = parser.parse_args(argv) + + if args.gui_dist: + verify_gui_dist(args.dist_dir, product=args.product) + return 0 + if args.wheel is None: + parser.error("Provide a wheel path or pass --gui-dist") + verify_built_wheel(args.wheel) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/build/common/verify/gui.py b/build/common/verify/gui.py new file mode 100644 index 0000000..5306b6b --- /dev/null +++ b/build/common/verify/gui.py @@ -0,0 +1,68 @@ +"""Smoke tests for standalone GUI product folders.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[2] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import GUI_DIST_DIR, gui_dist_dir + +# Product profiles for GUI packaging verification. +GUI_PRODUCTS: dict[str, dict] = { + "ory_finger": { + "dist_name": "OrisysDashboard", + "exe_name": "OrisysDashboard.exe", + "required_relpaths": [ + "config/dd02-ov.json", + "sensors/finger/assets/30fps-0703/mesh.npz", + "sensors/finger/assets/30fps-0703/world_to_cam.npy", + "sensors/finger/assets/visualization/world_to_cam.npy", + ], + }, + "ory_mini": { + "dist_name": "OryMini", + "exe_name": "OryMini.exe", + "required_relpaths": [ + "config/mini.json", + ], + }, + "ory_box": { + "dist_name": "OryBox", + "exe_name": "OryBox.exe", + "required_relpaths": [ + "config/ddjx01.json", + ], + }, +} + + +def verify_gui_dist( + dist_dir: Path | None = None, + *, + product: str = "ory_finger", +) -> None: + """Smoke-test a standalone GUI folder for the given product.""" + if product not in GUI_PRODUCTS: + raise ValueError(f"Unknown GUI product {product!r}; choose from {sorted(GUI_PRODUCTS)}") + + profile = GUI_PRODUCTS[product] + if dist_dir is None: + dist_dir = gui_dist_dir(profile["dist_name"]) if product != "ory_finger" else GUI_DIST_DIR + dist_dir = dist_dir.resolve() + + exe = dist_dir / profile["exe_name"] + if not exe.is_file(): + raise FileNotFoundError(f"GUI executable not found: {exe}") + + required = [dist_dir / rel for rel in profile["required_relpaths"]] + missing = [str(path.relative_to(dist_dir)) for path in required if not path.is_file()] + if missing: + raise FileNotFoundError(f"Missing bundled files in {dist_dir}: {', '.join(missing)}") + + subprocess.check_call([str(exe), "--help"], cwd=dist_dir) + print(f"GUI dist smoke test OK ({product}): {exe}") diff --git a/build/common/verify/wheel.py b/build/common/verify/wheel.py new file mode 100644 index 0000000..a8742f9 --- /dev/null +++ b/build/common/verify/wheel.py @@ -0,0 +1,62 @@ +"""Isolated venv smoke tests for packaged orisys wheels.""" + +from __future__ import annotations + +import subprocess +import sys +import tempfile +import venv +from pathlib import Path + +# Public SDK surface that must import from a released wheel. +_SMOKE_IMPORT_SCRIPT = """ +import orisys +from orisys import Sensor, list_config, export_config, util, finger +from orisys.finger import UVMeshViewer, build_uv_surface_mesh, FingerRuntime +from orisys.finger.mesh import UVSurfaceMesh +from orisys.finger.viewer import init_uv_mesh_viewer +from orisys.hhd import nHHD +from orisys.tracker import Tracker +from orisys import capture +from orisys.Stitcher import Stitcher + +assert orisys.__version__ +assert Sensor is not None +assert callable(list_config) +assert callable(export_config) +assert util is not None +assert finger is not None +assert UVMeshViewer is not None +assert build_uv_surface_mesh is not None +assert FingerRuntime is not None +assert UVSurfaceMesh is not None +assert init_uv_mesh_viewer is not None +assert nHHD is not None +assert Tracker is not None +assert capture is not None +assert Stitcher is not None +print(f"orisys {orisys.__version__} public API imports OK") +""" + + +def _venv_python(venv_dir: Path) -> Path: + if sys.platform == "win32": + return venv_dir / "Scripts" / "python.exe" + return venv_dir / "bin" / "python" + + +def verify_built_wheel(wheel_path: Path) -> None: + """Install the wheel in a temporary venv and smoke-test public imports.""" + wheel_path = wheel_path.resolve() + with tempfile.TemporaryDirectory(prefix="orisys-wheel-verify-") as tmp: + venv_dir = Path(tmp) / "venv" + venv.create(venv_dir, with_pip=True) + python = _venv_python(venv_dir) + + subprocess.check_call( + [python, "-m", "pip", "install", "-q", "--upgrade", "pip"], + ) + subprocess.check_call( + [python, "-m", "pip", "install", "-q", str(wheel_path)], + ) + subprocess.check_call([python, "-c", _SMOKE_IMPORT_SCRIPT]) diff --git a/build/gui/__init__.py b/build/gui/__init__.py new file mode 100644 index 0000000..a293b27 --- /dev/null +++ b/build/gui/__init__.py @@ -0,0 +1 @@ +"""Standalone GUI packaging targets (PyInstaller).""" diff --git a/build/gui/common/__init__.py b/build/gui/common/__init__.py new file mode 100644 index 0000000..4c9ff42 --- /dev/null +++ b/build/gui/common/__init__.py @@ -0,0 +1 @@ +"""Shared helpers for product GUI builds.""" diff --git a/build/gui/common/pyinstaller_driver.py b/build/gui/common/pyinstaller_driver.py new file mode 100644 index 0000000..ac8dddd --- /dev/null +++ b/build/gui/common/pyinstaller_driver.py @@ -0,0 +1,141 @@ +"""Shared PyInstaller driver for product GUI standalone folders.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable + +_BUILD_ROOT = Path(__file__).resolve().parents[2] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import PYINSTALLER_WORK_DIR, REPO_ROOT, gui_dist_dir + + +@dataclass +class GuiBuildConfig: + product: str + entry_script: Path + exe_name: str + dist_name: str + hidden_imports: list[str] = field(default_factory=list) + exclude_modules: list[str] = field(default_factory=list) + data_dirs: list[tuple[str, str]] = field(default_factory=list) + readme_lines: list[str] = field(default_factory=list) + windowed: bool = True + upx_dir: str | None = r"C:\tools\upx" + logo_name: str = "logo.png" + work_dir: Path = PYINSTALLER_WORK_DIR + spec_dir: Path = REPO_ROOT / "build" / "pyinstaller_work" + + +def clean_dir(path: Path) -> None: + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True) + + +def build_with_pyinstaller( + config: GuiBuildConfig, + *, + verify_fn: Callable[[Path], None] | None = None, +) -> Path: + """Run PyInstaller for one GUI product and copy bundled data folders.""" + print("Using Python:", sys.executable) + print(f"Building GUI product: {config.product}") + + dist_dir = gui_dist_dir(config.dist_name) + clean_dir(config.work_dir) + clean_dir(dist_dir) + config.spec_dir.mkdir(parents=True, exist_ok=True) + + entry = config.entry_script + if not entry.is_file(): + raise FileNotFoundError(f"GUI entry script not found: {entry}") + + cmd = [ + sys.executable, + "-m", + "PyInstaller", + "--onedir", + "--name", + config.dist_name, + f"--distpath={dist_dir.parent}", + f"--workpath={config.work_dir}", + f"--specpath={config.spec_dir}", + "--noconfirm", + "--clean", + "--windowed" if config.windowed else "--console", + ] + + for src_rel, dest_rel in config.data_dirs: + src = (REPO_ROOT / src_rel).resolve() + if not src.is_dir(): + raise FileNotFoundError(f"Data directory missing: {src}") + cmd.append(f"--add-data={src};{dest_rel}") + + logo = REPO_ROOT / config.logo_name + if logo.is_file(): + cmd.append(f"--add-data={logo.resolve()};.") + + for mod in config.hidden_imports: + cmd.append(f"--hidden-import={mod}") + for mod in config.exclude_modules: + cmd.append(f"--exclude-module={mod}") + + if config.upx_dir and Path(config.upx_dir).is_dir(): + cmd.append(f"--upx-dir={config.upx_dir}") + + cmd.append(str(entry.resolve())) + + env = os.environ.copy() + env["PYTHONPATH"] = ( + str((REPO_ROOT / "src").resolve()) + + os.pathsep + + str((REPO_ROOT / "examples").resolve()) + + os.pathsep + + env.get("PYTHONPATH", "") + ) + + print("\n=== Running PyInstaller ===") + print("Command:", " ".join(cmd)) + subprocess.check_call(cmd, env=env, cwd=REPO_ROOT) + + final_exe = dist_dir / config.exe_name + if not final_exe.is_file(): + raise RuntimeError(f"Executable not found at {final_exe}") + + print("\n=== Copying data directories ===") + for src_rel, dest_rel in config.data_dirs: + src = REPO_ROOT / src_rel + dest = dist_dir / dest_rel + if not src.is_dir(): + raise FileNotFoundError(f"Data directory not found: {src}") + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree(src, dest, dirs_exist_ok=True) + print(f" {src} -> {dest}") + + if logo.is_file(): + shutil.copy2(logo, dist_dir / config.logo_name) + print(" Logo copied") + + if config.readme_lines: + (dist_dir / "README.txt").write_text("\n".join(config.readme_lines) + "\n", encoding="utf-8") + + print(f"\n[SUCCESS] Standalone GUI folder: {dist_dir}") + print(f"Executable: {final_exe}") + + if verify_fn is not None: + try: + print("\n=== Smoke test ===") + verify_fn(dist_dir) + except Exception as exc: + print(f"\n[WARNING] Smoke test failed: {exc}") + + return dist_dir diff --git a/build/gui/ory_box/__init__.py b/build/gui/ory_box/__init__.py new file mode 100644 index 0000000..ed836af --- /dev/null +++ b/build/gui/ory_box/__init__.py @@ -0,0 +1 @@ +"""ORY-BOX internal calibration/testing GUI packaging (scaffold).""" diff --git a/build/gui/ory_box/build.py b/build/gui/ory_box/build.py new file mode 100644 index 0000000..3b1233f --- /dev/null +++ b/build/gui/ory_box/build.py @@ -0,0 +1,30 @@ +"""Scaffold for ORY-BOX internal GUI packaging. + +Not implemented yet. When ready, wire a dedicated calibration/testing entry +script and call ``gui.common.pyinstaller_driver.build_with_pyinstaller``. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[2] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from gui.ory_box.config import DIST_NAME, ENTRY_SCRIPT, make_config + + +def main() -> None: + config = make_config() + raise NotImplementedError( + "ORY-BOX GUI packaging is a scaffold only. " + f"Planned dist={DIST_NAME!r}, entry={ENTRY_SCRIPT}. " + "Implement the internal calibration/testing UI entry, then call " + "gui.common.pyinstaller_driver.build_with_pyinstaller(config)." + ) + + +if __name__ == "__main__": + main() diff --git a/build/gui/ory_box/config.py b/build/gui/ory_box/config.py new file mode 100644 index 0000000..10eeaff --- /dev/null +++ b/build/gui/ory_box/config.py @@ -0,0 +1,63 @@ +"""Product configuration scaffold for the ORY-BOX internal GUI.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[2] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import REPO_ROOT +from gui.common.pyinstaller_driver import GuiBuildConfig + +ENTRY_SCRIPT = REPO_ROOT / "examples" / "basic_pipeline.py" +EXE_NAME = "OryBox.exe" +DIST_NAME = "OryBox" + +HIDDEN_IMPORTS = [ + "PyQt5", + "pyqtgraph", + "cv2", + "orisys", + "gui", + "pipeline", + "viz", +] + +EXCLUDE_MODULES = [ + "cupy", + "cupyx", + "pyopencl", + "open3d", + "tkinter", + "pip", + "wheel", +] + +DATA_DIRS = [ + ("config", "config"), +] + +README_LINES = [ + "ORY-BOX internal calibration/testing GUI (scaffold)", + "", + "This product packaging target is reserved for a future internal tool.", + "Entry point placeholder:", + f" {ENTRY_SCRIPT.as_posix()}", +] + + +def make_config() -> GuiBuildConfig: + return GuiBuildConfig( + product="ory_box", + entry_script=ENTRY_SCRIPT, + exe_name=EXE_NAME, + dist_name=DIST_NAME, + hidden_imports=list(HIDDEN_IMPORTS), + exclude_modules=list(EXCLUDE_MODULES), + data_dirs=list(DATA_DIRS), + readme_lines=list(README_LINES), + windowed=False, + ) diff --git a/build/gui/ory_finger/__init__.py b/build/gui/ory_finger/__init__.py new file mode 100644 index 0000000..8061dac --- /dev/null +++ b/build/gui/ory_finger/__init__.py @@ -0,0 +1 @@ +"""ORY-FINGER standalone GUI packaging.""" diff --git a/build/gui/ory_finger/build.py b/build/gui/ory_finger/build.py new file mode 100644 index 0000000..46c32fc --- /dev/null +++ b/build/gui/ory_finger/build.py @@ -0,0 +1,31 @@ +"""Build the ORY-FINGER standalone Windows GUI folder with PyInstaller. + +Usage: + python build/run.py gui finger + python build/gui/ory_finger/build.py +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[2] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.verify.gui import verify_gui_dist +from gui.common.pyinstaller_driver import build_with_pyinstaller +from gui.ory_finger.config import make_config + + +def main() -> None: + config = make_config() + build_with_pyinstaller( + config, + verify_fn=lambda dist_dir: verify_gui_dist(dist_dir, product="ory_finger"), + ) + + +if __name__ == "__main__": + main() diff --git a/build/gui/ory_finger/config.py b/build/gui/ory_finger/config.py new file mode 100644 index 0000000..39f6ecd --- /dev/null +++ b/build/gui/ory_finger/config.py @@ -0,0 +1,75 @@ +"""Product configuration for the ORY-FINGER Windows dashboard.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[2] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import REPO_ROOT +from gui.common.pyinstaller_driver import GuiBuildConfig + +ENTRY_SCRIPT = REPO_ROOT / "examples" / "finger_pipeline_3dview.py" +EXE_NAME = "OrisysDashboard.exe" +DIST_NAME = "OrisysDashboard" + +HIDDEN_IMPORTS = [ + "PyQt5", + "pyqtgraph", + "open3d", + "cv2", + "orisys", + "gui", + "pipeline", + "viz", + "gui.app_paths", + "plotly", + "_plotly_utils", + "open3d.cpu.pybind", +] + +EXCLUDE_MODULES = [ + "cupy", + "cupyx", + "pyopencl", + "tkinter", + "pip", + "wheel", + "matplotlib", +] + +DATA_DIRS = [ + ("config", "config"), + ("sensors", "sensors"), +] + +README_LINES = [ + "Orisys Finger Dashboard (standalone)", + "", + "Run:", + f" {EXE_NAME}", + f" {EXE_NAME} --video path\\to\\video.mp4", + f" {EXE_NAME} --config config\\dd02-ov.json", + "", + "Bundled folders:", + " config/ sensor JSON configs", + " sensors/ per-sensor bundles (finger mesh and visualization data)", + "", + "CPU-only build (no CUDA/CuPy).", +] + + +def make_config() -> GuiBuildConfig: + return GuiBuildConfig( + product="ory_finger", + entry_script=ENTRY_SCRIPT, + exe_name=EXE_NAME, + dist_name=DIST_NAME, + hidden_imports=list(HIDDEN_IMPORTS), + exclude_modules=list(EXCLUDE_MODULES), + data_dirs=list(DATA_DIRS), + readme_lines=list(README_LINES), + ) diff --git a/build/gui/ory_mini/__init__.py b/build/gui/ory_mini/__init__.py new file mode 100644 index 0000000..86780ca --- /dev/null +++ b/build/gui/ory_mini/__init__.py @@ -0,0 +1 @@ +"""ORY-MINI internal calibration/testing GUI packaging (scaffold).""" diff --git a/build/gui/ory_mini/build.py b/build/gui/ory_mini/build.py new file mode 100644 index 0000000..9f1a3c6 --- /dev/null +++ b/build/gui/ory_mini/build.py @@ -0,0 +1,30 @@ +"""Scaffold for ORY-MINI internal GUI packaging. + +Not implemented yet. When ready, wire a dedicated calibration/testing entry +script and call ``gui.common.pyinstaller_driver.build_with_pyinstaller``. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[2] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from gui.ory_mini.config import DIST_NAME, ENTRY_SCRIPT, make_config + + +def main() -> None: + config = make_config() + raise NotImplementedError( + "ORY-MINI GUI packaging is a scaffold only. " + f"Planned dist={DIST_NAME!r}, entry={ENTRY_SCRIPT}. " + "Implement the internal calibration/testing UI entry, then call " + "gui.common.pyinstaller_driver.build_with_pyinstaller(config)." + ) + + +if __name__ == "__main__": + main() diff --git a/build/gui/ory_mini/config.py b/build/gui/ory_mini/config.py new file mode 100644 index 0000000..151aef1 --- /dev/null +++ b/build/gui/ory_mini/config.py @@ -0,0 +1,63 @@ +"""Product configuration scaffold for the ORY-MINI internal GUI.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[2] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import REPO_ROOT +from gui.common.pyinstaller_driver import GuiBuildConfig + +ENTRY_SCRIPT = REPO_ROOT / "examples" / "mini_pipeline.py" +EXE_NAME = "OryMini.exe" +DIST_NAME = "OryMini" + +HIDDEN_IMPORTS = [ + "PyQt5", + "pyqtgraph", + "cv2", + "orisys", + "gui", + "pipeline", + "viz", +] + +EXCLUDE_MODULES = [ + "cupy", + "cupyx", + "pyopencl", + "open3d", + "tkinter", + "pip", + "wheel", +] + +DATA_DIRS = [ + ("config", "config"), +] + +README_LINES = [ + "ORY-MINI internal calibration/testing GUI (scaffold)", + "", + "This product packaging target is reserved for a future internal tool.", + "Entry point placeholder:", + f" {ENTRY_SCRIPT.as_posix()}", +] + + +def make_config() -> GuiBuildConfig: + return GuiBuildConfig( + product="ory_mini", + entry_script=ENTRY_SCRIPT, + exe_name=EXE_NAME, + dist_name=DIST_NAME, + hidden_imports=list(HIDDEN_IMPORTS), + exclude_modules=list(EXCLUDE_MODULES), + data_dirs=list(DATA_DIRS), + readme_lines=list(README_LINES), + windowed=False, + ) diff --git a/build/paths.py b/build/paths.py deleted file mode 100644 index 817ac98..0000000 --- a/build/paths.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Repository and Nuitka build output paths.""" - -from __future__ import annotations - -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parent.parent - -NUITKA_COMPILE_DIR = REPO_ROOT / "build" / "nuitka" / "compile" -NUITKA_WHEEL_DIR = REPO_ROOT / "build" / "nuitka" / "wheel" -NUITKA_GUI_DIR = REPO_ROOT / "build" / "nuitka" / "gui" -GUI_DIST_DIR = REPO_ROOT / "dist" / "OrisysDashboard" - - -def repo_path(*parts: str) -> Path: - return REPO_ROOT.joinpath(*parts) - - -def sdk_dir(version: str) -> Path: - return REPO_ROOT / "SDK" / version diff --git a/build/run.py b/build/run.py new file mode 100644 index 0000000..092c06f --- /dev/null +++ b/build/run.py @@ -0,0 +1,104 @@ +"""Unified Orisys packaging entrypoint. + +Usage (from repo root): + + python build/run.py wheel [--target internal|pypi-linux] [--verify] + python build/run.py wheel-win + python build/run.py wheel-manylinux [PowerShell args...] + python build/run.py gui finger|mini|box + python build/run.py verify [wheel.whl] + python build/run.py verify --gui-dist [--product ory_finger] + +See build/README.md for details. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parent +_REPO_ROOT = _BUILD_ROOT.parent +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + + +def _run_python(script: Path, argv: list[str]) -> int: + cmd = [sys.executable, str(script), *argv] + return subprocess.call(cmd, cwd=_REPO_ROOT) + + +def _cmd_wheel(argv: list[str]) -> int: + return _run_python(_BUILD_ROOT / "wheel" / "build.py", argv) + + +def _cmd_wheel_win(argv: list[str]) -> int: + bat = _BUILD_ROOT / "wheel" / "win.bat" + return subprocess.call(["cmd", "/c", str(bat), *argv], cwd=_REPO_ROOT) + + +def _cmd_manylinux(argv: list[str]) -> int: + ps1 = _BUILD_ROOT / "wheel" / "docker" / "run.ps1" + cmd = [ + "powershell", + "-ExecutionPolicy", + "Bypass", + "-File", + str(ps1), + *argv, + ] + return subprocess.call(cmd, cwd=_REPO_ROOT) + + +def _cmd_gui(argv: list[str]) -> int: + parser = argparse.ArgumentParser(prog="build/run.py gui") + parser.add_argument( + "product", + choices=("finger", "mini", "box", "ory_finger", "ory_mini", "ory_box"), + help="GUI product to package", + ) + args, rest = parser.parse_known_args(argv) + product = { + "finger": "ory_finger", + "mini": "ory_mini", + "box": "ory_box", + }.get(args.product, args.product) + return _run_python(_BUILD_ROOT / "gui" / product / "build.py", rest) + + +def _cmd_verify(argv: list[str]) -> int: + from common.verify.cli import main as verify_main + + return verify_main(argv) + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if not argv or argv[0] in ("-h", "--help"): + print(__doc__) + print("Commands: wheel | wheel-win | manylinux | gui | verify") + print("Examples:") + print(" python build/run.py wheel --help") + print(" python build/run.py gui finger") + print(" python build/run.py manylinux -PythonVersions 3.10") + return 0 + + command, rest = argv[0], argv[1:] + dispatch = { + "wheel": _cmd_wheel, + "wheel-win": _cmd_wheel_win, + "wheel-manylinux": _cmd_manylinux, + "gui": _cmd_gui, + "verify": _cmd_verify, + } + if command not in dispatch: + print(f"Unknown command: {command!r}", file=sys.stderr) + print("Choose from: wheel, wheel-win, manylinux, gui, verify", file=sys.stderr) + return 2 + return dispatch[command](rest) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/build/verify.py b/build/verify.py deleted file mode 100644 index b4aa4ca..0000000 --- a/build/verify.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Post-build smoke tests for packaged wheels and standalone GUI builds.""" - -from __future__ import annotations - -import argparse -import subprocess -import sys -import tempfile -import venv -from pathlib import Path - -from paths import GUI_DIST_DIR - - -def _venv_python(venv_dir: Path) -> Path: - if sys.platform == "win32": - return venv_dir / "Scripts" / "python.exe" - return venv_dir / "bin" / "python" - - -def verify_built_wheel(wheel_path: Path) -> None: - """Install the wheel in a temporary venv and smoke-test finger imports.""" - wheel_path = wheel_path.resolve() - with tempfile.TemporaryDirectory(prefix="orisys-wheel-verify-") as tmp: - venv_dir = Path(tmp) / "venv" - venv.create(venv_dir, with_pip=True) - python = _venv_python(venv_dir) - - subprocess.check_call( - [python, "-m", "pip", "install", "-q", str(wheel_path)], - ) - subprocess.check_call( - [ - python, - "-c", - ( - "from orisys.finger import UVMeshViewer, build_uv_surface_mesh, FingerRuntime; " - "from orisys.finger.mesh import UVSurfaceMesh; " - "from orisys.finger.viewer import init_uv_mesh_viewer; " - "print('finger mesh/viewer imports OK')" - ), - ], - ) - - -def verify_gui_dist(dist_dir: Path | None = None) -> None: - """Smoke-test a standalone GUI folder produced by build_gui_win.py.""" - dist_dir = (dist_dir or GUI_DIST_DIR).resolve() - exe = dist_dir / "OrisysDashboard.exe" - if not exe.is_file(): - raise FileNotFoundError(f"GUI executable not found: {exe}") - - required = [ - dist_dir / "config" / "dd02-ov.json", - dist_dir / "assets" / "finger" / "visualization" / "world_to_cam.npy", - dist_dir / "assets" / "finger" / "30fps" / "mesh.npz", - ] - missing = [str(path.relative_to(dist_dir)) for path in required if not path.is_file()] - if missing: - raise FileNotFoundError(f"Missing bundled files in {dist_dir}: {', '.join(missing)}") - - subprocess.check_call([str(exe), "--help"], cwd=dist_dir) - print(f"GUI dist smoke test OK: {exe}") - - -def main() -> None: - parser = argparse.ArgumentParser(description="Verify packaged Orisys artifacts.") - parser.add_argument("--gui-dist", action="store_true", help="Verify dist/OrisysDashboard/") - parser.add_argument("--dist-dir", type=Path, default=None, help="Override GUI dist folder") - parser.add_argument("wheel", nargs="?", type=Path, help="Wheel path for import smoke test") - args = parser.parse_args() - - if args.gui_dist: - verify_gui_dist(args.dist_dir) - return - if args.wheel is None: - parser.error("Provide a wheel path or pass --gui-dist") - verify_built_wheel(args.wheel) - - -if __name__ == "__main__": - main() diff --git a/build/wheel/__init__.py b/build/wheel/__init__.py new file mode 100644 index 0000000..a7580e0 --- /dev/null +++ b/build/wheel/__init__.py @@ -0,0 +1 @@ +"""Nuitka-compiled SDK wheel packaging.""" diff --git a/build/wheel/build.py b/build/wheel/build.py new file mode 100644 index 0000000..ea2dab0 --- /dev/null +++ b/build/wheel/build.py @@ -0,0 +1,203 @@ +"""Build a Nuitka-compiled orisys wheel and publish it to SDK/{version}/. + +Pipeline: + 1. Compile src/orisys → extension module (.pyd / .so) + 2. Stage wheel tree (compiled module + bundled files from pyproject.toml) + 3. Run bdist_wheel; on Linux, auditwheel-repair to a manylinux tag + 4. Smoke-test imports in a temporary venv (required for --target pypi-linux) + +Targets: + internal – local/SDK delivery (default). Linux still repairs with auditwheel. + pypi-linux – PyPI binary wheel; requires Linux + CPython 3.10/3.11/3.12 + compliant tag. + +Run from repo root: + python build/run.py wheel + python build/run.py wheel --target pypi-linux + python build/wheel/build.py --target pypi-linux +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[1] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import NUITKA_COMPILE_DIR, NUITKA_WHEEL_DIR, REPO_ROOT, repaired_wheel_dir, repo_path +from common.pyproject_utils import bundle_items, extras_require, get_version, install_requires +from common.verify.wheel import verify_built_wheel +from wheel.compile import compile_orisys, copy_compiled_orisys, install_nuitka +from wheel.layout import ( + copy_bundle_items, + copy_orisys_blend, + package_data_from_bundle, +) +from wheel.package import ( + is_compliant_manylinux_wheel, + publish_wheel_to_sdk, + repair_linux_wheel, + run_bdist_wheel, + write_wheel_setup, +) + +VALID_TARGETS = ("internal", "pypi-linux") +# ABI tags that manylinux images ship under /opt/python/cpXYY-cpXYY. +PYPI_LINUX_PYTHON_VERSIONS = ((3, 10), (3, 11), (3, 12)) + + +def _reset_dir(path: Path) -> None: + """Remove and recreate a build output directory.""" + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True) + + +def _validate_pypi_linux_environment() -> None: + if sys.platform != "linux": + raise SystemExit( + "[ERROR] --target pypi-linux must run inside a Linux manylinux container " + "(for example quay.io/pypa/manylinux_2_28_x86_64 via Docker Desktop)." + ) + if sys.version_info[:2] not in PYPI_LINUX_PYTHON_VERSIONS: + supported = ", ".join(f"{major}.{minor}" for major, minor in PYPI_LINUX_PYTHON_VERSIONS) + raise SystemExit( + f"[ERROR] --target pypi-linux requires CPython {supported}, got " + f"{sys.version_info.major}.{sys.version_info.minor}. " + "Use /opt/python/cp310-cp310 (or cp311/cp312) in the manylinux_2_28 image." + ) + + +def build(*, target: str = "internal", verify: bool | None = None) -> Path: + if target not in VALID_TARGETS: + raise SystemExit(f"[ERROR] Unknown target {target!r}; choose from {VALID_TARGETS}") + + if target == "pypi-linux": + _validate_pypi_linux_environment() + + # pypi-linux always verifies; internal opts in with --verify. + if verify is None: + verify = target == "pypi-linux" + + # --- 1. Compile --- + install_nuitka() + + version = get_version(REPO_ROOT) + _reset_dir(NUITKA_COMPILE_DIR) + _reset_dir(NUITKA_WHEEL_DIR) + + print(f"\n=== Compiling with Nuitka (target={target}, version={version}) ===") + env = os.environ.copy() + src_dir = repo_path("src") + env["PYTHONPATH"] = src_dir.as_posix() + os.pathsep + env.get("PYTHONPATH", "") + ext_file = compile_orisys(NUITKA_COMPILE_DIR, env) + + # --- 2. Stage wheel contents --- + print("\n=== Preparing Wheel Structure ===") + pkg_dir = NUITKA_WHEEL_DIR / "wheel_root" / "orisys" + pkg_dir.mkdir(parents=True) + + ext_suffix = copy_compiled_orisys(pkg_dir, ext_file) + + wheel_root = NUITKA_WHEEL_DIR / "wheel_root" + blend_count = copy_orisys_blend(REPO_ROOT, wheel_root) + print(f"Staged orisys_blend Python modules: {blend_count}") + + # Bundle rules live in pyproject.toml → [[tool.orisys.bundle.items]] + bundle = bundle_items(REPO_ROOT) + copied_counts = copy_bundle_items(REPO_ROOT, pkg_dir, bundle) + print(f"Copied {copied_counts['configs']} JSON config files") + print(f"Copied {copied_counts['examples']} example files to orisys/examples/") + print(f"Copied {copied_counts['docs']} doc files to orisys/docs/") + print(f"Copied {copied_counts['sensors']} sensor bundle files to orisys/sensors/") + + write_wheel_setup( + NUITKA_WHEEL_DIR, + version=version, + package_data=package_data_from_bundle(ext_suffix, bundle), + install_requires=install_requires(REPO_ROOT), + extras_require=extras_require(REPO_ROOT), + ) + + # --- 3. Build wheel, repair on Linux, publish to SDK/ --- + wheel_path = run_bdist_wheel(NUITKA_WHEEL_DIR) + if wheel_path is None: + raise SystemExit("\n[ERROR] Wheel not found after bdist_wheel.") + + repair_dir = repaired_wheel_dir() + final_candidate = repair_linux_wheel(wheel_path, repair_dir) + + if target == "pypi-linux" and not is_compliant_manylinux_wheel(final_candidate): + raise SystemExit( + f"[ERROR] Wheel is not PyPI-compliant: {final_candidate.name}. " + "Expected a manylinux_2_17_x86_64 (or newer) platform tag." + ) + + # Keep a durable copy under dist/repaired for upload tooling. + dist_repaired = REPO_ROOT / "dist" / "repaired" + dist_repaired.mkdir(parents=True, exist_ok=True) + dist_copy = dist_repaired / final_candidate.name + if dist_copy.resolve() != final_candidate.resolve(): + shutil.copy2(final_candidate, dist_copy) + + final_wheel = publish_wheel_to_sdk(final_candidate, version) + print(f"\n[SUCCESS] Wheel created and copied to: {final_wheel}") + print(f"[SUCCESS] Upload candidate also at: {dist_copy}") + + # --- 4. Verify (isolated venv; does not touch the active environment) --- + if verify: + print("\n=== Verifying wheel imports in isolated venv ===") + try: + verify_built_wheel(final_wheel) + print("[SUCCESS] Wheel smoke test passed.") + except subprocess.CalledProcessError as exc: + if target == "pypi-linux": + raise SystemExit( + f"[ERROR] Wheel verify failed (required for pypi-linux): {exc}" + ) from exc + print(f"\n[WARNING] Wheel verify failed: {exc}") + print("The wheel was built successfully; verification runs in an isolated temp venv.") + else: + print("\n[INFO] Skipping wheel verify (pass --verify to enable).") + + return final_wheel + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description="Build a Nuitka-compiled orisys wheel.") + parser.add_argument( + "--target", + choices=VALID_TARGETS, + default="internal", + help="internal: SDK delivery; pypi-linux: manylinux PyPI binary wheel", + ) + parser.add_argument( + "--verify", + action="store_true", + help="Run isolated venv smoke test (always on for --target pypi-linux)", + ) + parser.add_argument( + "--skip-verify", + action="store_true", + help="Skip smoke test when using --target internal", + ) + args = parser.parse_args(argv) + verify: bool | None + if args.target == "pypi-linux": + verify = True + elif args.verify: + verify = True + elif args.skip_verify: + verify = False + else: + verify = None + build(target=args.target, verify=verify) + + +if __name__ == "__main__": + main() diff --git a/build/nuitka_compile.py b/build/wheel/compile.py similarity index 85% rename from build/nuitka_compile.py rename to build/wheel/compile.py index cd87e0d..6b4d525 100644 --- a/build/nuitka_compile.py +++ b/build/wheel/compile.py @@ -8,21 +8,36 @@ import sys from importlib.machinery import EXTENSION_SUFFIXES from pathlib import Path -from paths import repo_path +_BUILD_ROOT = Path(__file__).resolve().parents[1] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import repo_path NUITKA_INCLUDE_PACKAGES = ( "orisys", + "orisys.capture", "orisys.finger", + "orisys.stitch", + "orisys.stitch.backends", "orisys.unroll", "orisys.hhd", "orisys.hhd.solvers", ) NUITKA_NOFOLLOW_IMPORTS = ( + # Keep as runtime deps (shipped via wheel install_requires / extras). + "numpy", + "scipy", + "cv2", + "numba", + "orisys_blend", "cupy", "cupyx", "pyopencl", "open3d", + "PyQt5", + "pyqtgraph", ) diff --git a/build/wheel/docker/Dockerfile.manylinux b/build/wheel/docker/Dockerfile.manylinux new file mode 100644 index 0000000..fdc6ca0 --- /dev/null +++ b/build/wheel/docker/Dockerfile.manylinux @@ -0,0 +1,26 @@ +# manylinux_2_28: modern glibc baseline with binary wheels for numpy/scipy/opencv. +# Prefer this over manylinux2014 — recent NumPy requires GCC >= 10.3 and will try +# (and fail) to build from source on the older image. +FROM quay.io/pypa/manylinux_2_28_x86_64:latest + +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_ROOT_USER_ACTION=ignore + +# Only install compile/repair tooling here. Runtime deps (numpy/scipy/opencv/numba) +# are declared in the wheel metadata and installed by pip when users install orisys. +# Nuitka is configured with --nofollow-import-to for those packages. +RUN for PYBIN in \ + /opt/python/cp310-cp310/bin \ + /opt/python/cp311-cp311/bin \ + /opt/python/cp312-cp312/bin; do \ + echo "=== Installing build tooling into ${PYBIN} ===" && \ + "${PYBIN}/python" -m pip install --upgrade pip setuptools wheel && \ + "${PYBIN}/python" -m pip install \ + "nuitka>=2.0" \ + ordered-set \ + zstandard \ + auditwheel; \ + done + +WORKDIR /src +ENTRYPOINT ["/bin/bash", "/src/build/wheel/docker/entrypoint.sh"] diff --git a/build/wheel/docker/entrypoint.sh b/build/wheel/docker/entrypoint.sh new file mode 100644 index 0000000..470b754 --- /dev/null +++ b/build/wheel/docker/entrypoint.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Runs inside quay.io/pypa/manylinux_2_28_x86_64 with the repo mounted at /src. +# Builds one Nuitka wheel per CPython ABI listed in ORISYS_PYTHON_TAGS. +set -euo pipefail + +TARGET="${ORISYS_BUILD_TARGET:-pypi-linux}" +# Comma-separated manylinux tags, e.g. "cp310-cp310,cp311-cp311,cp312-cp312" +PYTHON_TAGS="${ORISYS_PYTHON_TAGS:-cp310-cp310,cp311-cp311,cp312-cp312}" + +cd /src + +echo "=== manylinux entrypoint ===" +echo "Target: ${TARGET}" +echo "Python tags: ${PYTHON_TAGS}" +uname -a + +IFS=',' read -r -a TAG_ARRAY <<< "${PYTHON_TAGS}" +FAILED=0 +BUILT=() + +for tag in "${TAG_ARRAY[@]}"; do + tag="$(echo "${tag}" | tr -d '[:space:]')" + if [[ -z "${tag}" ]]; then + continue + fi + + PYBIN="/opt/python/${tag}/bin" + if [[ ! -x "${PYBIN}/python" ]]; then + echo "[ERROR] Python not found for tag ${tag}: ${PYBIN}/python" >&2 + FAILED=1 + continue + fi + + echo "" + echo "========================================" + echo "Building orisys wheel: ${tag}" + echo "Python: $("${PYBIN}/python" -V)" + echo "========================================" + + export PATH="${PYBIN}:${PATH}" + export PYTHONPATH="/src/src${PYTHONPATH:+:${PYTHONPATH}}" + + "${PYBIN}/python" -m pip install -q --upgrade pip setuptools wheel auditwheel + + if "${PYBIN}/python" build/wheel/build.py --target "${TARGET}"; then + BUILT+=("${tag}") + else + echo "[ERROR] Build failed for ${tag}" >&2 + FAILED=1 + fi +done + +echo "" +echo "=== Build artifacts ===" +ls -la /src/dist/repaired/ || true +ls -la /src/SDK/*/ || true + +if [[ ${#BUILT[@]} -gt 0 ]]; then + echo "Built tags: ${BUILT[*]}" +fi + +if [[ "${FAILED}" -ne 0 ]]; then + echo "[ERROR] One or more Python ABI builds failed." >&2 + exit 1 +fi + +echo "[SUCCESS] All requested manylinux wheels built." diff --git a/build/wheel/docker/run.ps1 b/build/wheel/docker/run.ps1 new file mode 100644 index 0000000..219674e --- /dev/null +++ b/build/wheel/docker/run.ps1 @@ -0,0 +1,100 @@ +# Build PyPI-compliant manylinux wheels for CPython 3.10/3.11/3.12 using Docker Desktop. +# +# Usage (from repo root, PowerShell): +# .\build\wheel\docker\run.ps1 +# .\build\build_manylinux_docker.ps1 # compatibility wrapper +# +# From cmd.exe: +# powershell -ExecutionPolicy Bypass -File .\build\build_manylinux_docker.ps1 + +param( + [ValidateSet("pypi-linux", "internal")] + [string]$Target = "pypi-linux", + [string]$PythonVersions = "3.10,3.11,3.12", + [switch]$SkipImageBuild, + [string]$ImageName = "orisys-manylinux" +) + +$ErrorActionPreference = "Stop" + +function Convert-ToManylinuxTag { + param([string]$Version) + $v = $Version.Trim() + switch ($v) { + "3.10" { return "cp310-cp310" } + "3.11" { return "cp311-cp311" } + "3.12" { return "cp312-cp312" } + "cp310-cp310" { return "cp310-cp310" } + "cp311-cp311" { return "cp311-cp311" } + "cp312-cp312" { return "cp312-cp312" } + default { + throw "Unsupported Python version '$Version'. Use 3.10, 3.11, and/or 3.12." + } + } +} + +$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..\..")).Path +Set-Location $RepoRoot + +$tags = @() +foreach ($part in ($PythonVersions -split ",")) { + if ([string]::IsNullOrWhiteSpace($part)) { continue } + $tags += (Convert-ToManylinuxTag $part) +} +if ($tags.Count -eq 0) { + throw "No Python versions specified." +} +$pythonTags = [string]::Join(",", $tags) + +Write-Host "=== Orisys manylinux Docker build ===" +Write-Host "Repo: $RepoRoot" +Write-Host "Target: $Target" +Write-Host "Image: $ImageName" +Write-Host "Python tags: $pythonTags" + +# Docker Desktop must be in Linux container mode for manylinux. +$dockerInfo = docker version --format "{{.Server.Os}}" 2>$null +if ($LASTEXITCODE -ne 0) { + throw "Docker is not available. Start Docker Desktop and retry." +} +if ($dockerInfo -ne "linux") { + throw "Docker server OS is '$dockerInfo'. Switch Docker Desktop to Linux containers." +} + +$entrypoint = Join-Path $RepoRoot "build\wheel\docker\entrypoint.sh" +# Normalize CRLF -> LF so bash in the container can execute the script. +$raw = [System.IO.File]::ReadAllText($entrypoint) +$normalized = $raw -replace "`r`n", "`n" -replace "`r", "`n" +[System.IO.File]::WriteAllText($entrypoint, $normalized) + +$dockerfile = "build/wheel/docker/Dockerfile.manylinux" + +if (-not $SkipImageBuild) { + Write-Host "`n=== Building Docker image ===" + docker build -f $dockerfile -t $ImageName . + if ($LASTEXITCODE -ne 0) { + throw "docker build failed" + } +} + +Write-Host "`n=== Running manylinux container ===" +# Mount the repo so artifacts land under dist/repaired and SDK// on the host. +docker run --rm ` + -e "ORISYS_BUILD_TARGET=$Target" ` + -e "ORISYS_PYTHON_TAGS=$pythonTags" ` + -v "${RepoRoot}:/src" ` + $ImageName + +if ($LASTEXITCODE -ne 0) { + throw "manylinux container build failed" +} + +Write-Host "`n=== Host artifacts ===" +$repaired = Join-Path $RepoRoot "dist\repaired" +if (Test-Path $repaired) { + Get-ChildItem $repaired -Filter *.whl | ForEach-Object { Write-Host $_.FullName } +} else { + Write-Warning "No dist/repaired directory found." +} + +Write-Host "`nNext steps: see docs/publishing.md (TestPyPI then PyPI)." diff --git a/build/wheel_layout.py b/build/wheel/layout.py similarity index 54% rename from build/wheel_layout.py rename to build/wheel/layout.py index 00f2ec9..c5141e3 100644 --- a/build/wheel_layout.py +++ b/build/wheel/layout.py @@ -5,6 +5,21 @@ from __future__ import annotations import shutil from pathlib import Path +# Required files that must ship with the wheel for ORY-FINGER runtime/GUI. +# Accept both the flat layout and a nested assets/finger/ checkout tree. +REQUIRED_ASSET_RELPATHS = ( + "finger/assets/30fps-0703/mesh.npz", + "finger/assets/30fps-0703/camera_intrinsics.npz", + "finger/assets/30fps-0703/world_to_cam.npy", + "finger/assets/visualization/world_to_cam.npy", +) +REQUIRED_ASSET_RELPATH_ALTERNATES = { + "finger/assets/30fps-0703/mesh.npz": "finger/assets/finger/30fps-0703/mesh.npz", + "finger/assets/30fps-0703/camera_intrinsics.npz": "finger/assets/finger/30fps-0703/camera_intrinsics.npz", + "finger/assets/30fps-0703/world_to_cam.npy": "finger/assets/finger/30fps-0703/world_to_cam.npy", + "finger/assets/visualization/world_to_cam.npy": "finger/assets/finger/visualization/world_to_cam.npy", +} + def resolve_bundle_source_files(repo_root: Path, item: dict) -> list[Path]: """Resolve explicit files and/or glob patterns for one bundle item.""" @@ -37,6 +52,36 @@ def resolve_bundle_source_files(repo_root: Path, item: dict) -> list[Path]: return resolved +def verify_required_assets(repo_root: Path, items: list[dict]) -> None: + """Fail early when required finger assets are missing from the bundle.""" + asset_items = [item for item in items if item.get("name") == "sensors"] + if not asset_items: + return + + resolved: set[str] = set() + for item in asset_items: + src_dir = (repo_root / item["source"]).resolve() + for path in resolve_bundle_source_files(repo_root, item): + try: + rel = path.resolve().relative_to(src_dir).as_posix() + except ValueError: + continue + resolved.add(rel) + + missing = [] + for rel in REQUIRED_ASSET_RELPATHS: + alt = REQUIRED_ASSET_RELPATH_ALTERNATES.get(rel) + if rel not in resolved and (alt is None or alt not in resolved): + missing.append(rel) + if missing: + joined = ", ".join(missing) + raise FileNotFoundError( + "Required finger assets are missing from the wheel bundle. " + f"Missing: {joined}. Update [[tool.orisys.bundle.items]] patterns " + "or restore assets under sensors/finger/assets/ before building." + ) + + def package_data_paths_for_item(item: dict) -> list[str]: """Return setuptools package_data paths relative to the orisys package.""" target = str(item["target"]).replace("\\", "/") @@ -54,6 +99,7 @@ def data_file_entries_for_item(item: dict) -> list[str]: def copy_bundle_items(repo_root: Path, pkg_dir: Path, items: list[dict]) -> dict[str, int]: + verify_required_assets(repo_root, items) copied_counts: dict[str, int] = {} for item in items: name = item["name"] @@ -82,10 +128,23 @@ def copy_bundle_items(repo_root: Path, pkg_dir: Path, items: list[dict]) -> dict def package_data_from_bundle(ext_suffix: str, items: list[dict]) -> dict[str, list[str]]: package_data: dict[str, list[str]] = { "orisys": [f"*{ext_suffix}"], - "orisys.examples": ["*.py", "README.md", "app_paths.py", "viz/**/*.py", "pipeline/**/*.py", "gui/**/*.py"], + "orisys.examples": ["*.py", "README.md", "viz/**/*.py", "pipeline/**/*.py", "gui/**/*.py"], + "orisys_blend": ["*.py"], } for item in items: if item["name"] == "examples": continue package_data["orisys"].extend(package_data_paths_for_item(item)) return package_data + + +def copy_orisys_blend(repo_root: Path, wheel_root: Path) -> int: + """Copy pure-Python orisys_blend (Numba kernels) next to the Nuitka orisys .pyd.""" + src = repo_root / "src" / "orisys_blend" + dst = wheel_root / "orisys_blend" + if dst.exists(): + shutil.rmtree(dst) + shutil.copytree(src, dst) + count = sum(1 for p in dst.rglob("*.py") if p.is_file()) + print(f"Copied orisys_blend ({count} .py files) to {dst}") + return count diff --git a/build/wheel/package.py b/build/wheel/package.py new file mode 100644 index 0000000..9bcf495 --- /dev/null +++ b/build/wheel/package.py @@ -0,0 +1,139 @@ +"""Assemble and build the Nuitka wheel with setuptools.""" + +from __future__ import annotations + +import re +import shutil +import subprocess +import sys +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[1] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import sdk_dir + +# PyPI rejects bare tags like "manylinux_x86_64"; require a glibc ABI version. +_MANYLINUX_TAG_RE = re.compile( + r"(?:manylinux2014|manylinux_2_(?:17|24|27|28|31|34|35))_x86_64" +) + + +def write_wheel_setup( + wheel_dir: Path, + *, + version: str, + package_data: dict[str, list[str]], + install_requires: list[str], + extras_require: dict[str, list[str]], +) -> Path: + setup_py_path = wheel_dir / "setup.py" + setup_content = f""" +from setuptools import setup, Distribution + +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return True + +setup( + name="orisys", + version="{version}", + description="Orisys - Optical Tactile Sensor SDK (Nuitka Compiled)", + author="Orisys", + packages=['orisys', 'orisys.examples', 'orisys_blend'], + package_dir={{'': 'wheel_root'}}, + package_data={package_data}, + include_package_data=True, + install_requires={install_requires!r}, + extras_require={extras_require!r}, + distclass=BinaryDistribution, + zip_safe=False, +) +""" + setup_py_path.write_text(setup_content, encoding="utf-8") + return setup_py_path + + +def run_bdist_wheel(wheel_dir: Path) -> Path | None: + print("\n=== Building Wheel ===") + subprocess.check_call([sys.executable, "setup.py", "bdist_wheel"], cwd=wheel_dir) + wheels = sorted((wheel_dir / "dist").glob("*.whl")) + return wheels[0] if wheels else None + + +def _ensure_auditwheel() -> None: + try: + subprocess.check_call( + [sys.executable, "-m", "auditwheel", "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except (subprocess.CalledProcessError, FileNotFoundError): + print("Installing auditwheel...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "auditwheel", "-q"]) + + +def is_compliant_manylinux_wheel(wheel_path: Path) -> bool: + """True if the wheel filename uses a PyPI-accepted manylinux x86_64 tag.""" + return bool(_MANYLINUX_TAG_RE.search(wheel_path.name)) + + +def repair_linux_wheel(wheel_path: Path, repair_dir: Path) -> Path: + """Run auditwheel show/repair and return a PyPI-compliant manylinux wheel. + + Raises RuntimeError if repair fails or the result still lacks a valid tag. + On non-Linux platforms the input wheel is returned unchanged. + """ + if sys.platform != "linux": + print("Skipping auditwheel repair (not running on Linux).") + return wheel_path + + _ensure_auditwheel() + repair_dir = repair_dir.resolve() + if repair_dir.exists(): + shutil.rmtree(repair_dir) + repair_dir.mkdir(parents=True) + + print("\n=== auditwheel show ===") + subprocess.check_call([sys.executable, "-m", "auditwheel", "show", str(wheel_path)]) + + print("\n=== auditwheel repair ===") + subprocess.check_call( + [ + sys.executable, + "-m", + "auditwheel", + "repair", + str(wheel_path), + "--wheel-dir", + str(repair_dir), + ] + ) + + repaired = sorted(repair_dir.glob("*.whl")) + if not repaired: + raise RuntimeError(f"auditwheel repair produced no wheels in {repair_dir}") + + compliant = [path for path in repaired if is_compliant_manylinux_wheel(path)] + if not compliant: + names = ", ".join(path.name for path in repaired) + raise RuntimeError( + "auditwheel repair did not produce a PyPI-compliant manylinux tag. " + f"Got: {names}. Rebuild inside quay.io/pypa/manylinux_2_28_x86_64 " + "(or manylinux2014_x86_64) instead of a generic Linux host." + ) + + final = compliant[0] + print(f"Repaired wheel: {final.name}") + return final + + +def publish_wheel_to_sdk(wheel_path: Path, version: str) -> Path: + sdk_version_dir = sdk_dir(version) + sdk_version_dir.mkdir(parents=True, exist_ok=True) + dest_wheel = sdk_version_dir / wheel_path.name + if dest_wheel.exists(): + dest_wheel.unlink() + shutil.copy2(str(wheel_path), str(dest_wheel)) + return dest_wheel diff --git a/build/build_nuitka_win.bat b/build/wheel/win.bat similarity index 92% rename from build/build_nuitka_win.bat rename to build/wheel/win.bat index fb9f9d1..6a45b5e 100644 --- a/build/build_nuitka_win.bat +++ b/build/wheel/win.bat @@ -3,7 +3,7 @@ REM Build Nuitka wheels for Python 3.10, 3.11, and 3.12. REM Expects conda envs named py310, py311, py312 (or set CONDA_ENVS_ROOT). setlocal enabledelayedexpansion -cd /d "%~dp0\.." +cd /d "%~dp0\..\.." if errorlevel 1 ( echo ERROR: Could not change to repository root. exit /b 1 @@ -32,7 +32,7 @@ for %%E in (py310 py311 py312) do ( set "OK=0" if !HAS_CONDA! equ 1 ( - call conda run --no-capture-output -n %%E python build\build_nuitka.py + call conda run --no-capture-output -n %%E python build\wheel\build.py if not errorlevel 1 set "OK=1" ) @@ -44,7 +44,7 @@ for %%E in (py310 py311 py312) do ( if not exist "!PYEXE!" ( echo ERROR: Python not found: !PYEXE! ) else ( - "!PYEXE!" build\build_nuitka.py + "!PYEXE!" build\wheel\build.py if not errorlevel 1 set "OK=1" ) ) diff --git a/build/wheel_build.py b/build/wheel_build.py deleted file mode 100644 index e23edaa..0000000 --- a/build/wheel_build.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Assemble and build the Nuitka wheel with setuptools.""" - -from __future__ import annotations - -import shutil -import subprocess -import sys -from pathlib import Path - -from paths import sdk_dir - - -def write_wheel_setup( - wheel_dir: Path, - *, - version: str, - package_data: dict[str, list[str]], - install_requires: list[str], - extras_require: dict[str, list[str]], -) -> Path: - setup_py_path = wheel_dir / "setup.py" - setup_content = f""" -from setuptools import setup, Distribution - -class BinaryDistribution(Distribution): - def has_ext_modules(self): - return True - -setup( - name="orisys", - version="{version}", - description="Orisys - Optical Tactile Sensor SDK (Nuitka Compiled)", - author="Orisys", - packages=['orisys', 'orisys.examples'], - package_dir={{'': 'wheel_root'}}, - package_data={package_data}, - include_package_data=True, - install_requires={install_requires!r}, - extras_require={extras_require!r}, - distclass=BinaryDistribution, - zip_safe=False, -) -""" - setup_py_path.write_text(setup_content, encoding="utf-8") - return setup_py_path - - -def run_bdist_wheel(wheel_dir: Path) -> Path | None: - print("\n=== Building Wheel ===") - subprocess.check_call([sys.executable, "setup.py", "bdist_wheel"], cwd=wheel_dir) - wheels = sorted((wheel_dir / "dist").glob("*.whl")) - return wheels[0] if wheels else None - - -def publish_wheel_to_sdk(wheel_path: Path, version: str) -> Path: - sdk_version_dir = sdk_dir(version) - sdk_version_dir.mkdir(parents=True, exist_ok=True) - dest_wheel = sdk_version_dir / wheel_path.name - if dest_wheel.exists(): - dest_wheel.unlink() - shutil.move(str(wheel_path), str(dest_wheel)) - return dest_wheel diff --git a/config/X-ov.json b/config/X-ov.json new file mode 100644 index 0000000..4c966cd --- /dev/null +++ b/config/X-ov.json @@ -0,0 +1,101 @@ +{ + "name": "dd06-ov", + "type": "single_camera", + "version": "1.0", + "camera": { + "resolution_raw": [ + 1280, + 720 + ], + "fps": 30, + "resolution_stitch": [ + 195, + 275 + ], + "source_scale": 0.4, + "old": [ + 672, + 728, + 1, + 0.5 + ], + "work": [ + 504, + 546, + 1 + ], + "work2": [ + 420, + 450, + 0.8 + ], + "work3": [ + 350, + 375, + 4, + 0.7 + ], + "work4": [ + 280, + 300, + 4, + 0.4, + 0.2 + ] + }, + "tracker": { + "adaptive": true, + "lower_threshold": 12.0, + "upper_threshold": 18.0, + "loop_closure_interval": 1, + "algorithm": "dis", + "dis_preset": "medium", + "dis_mode": 3, + "algorithm_list": [ + "raft", + "dis" + ], + "hhd_downsample": 3 + }, + "contact": { + "min_contour_area": 3000, + "flow_threshold": 0.2, + "div_threshold": 0.022, + "expansion_threshold": 0.01, + "kernel_size": [ + 5, + 5 + ], + "denoise_sigma": 12 + }, + "roi": { + "type": "polygon", + "image_size": [ + 1280, + 720 + ], + "created_at": "2026-07-26T11:56:15", + "points": [ + [ + 242, + 1 + ], + [ + 402, + 597 + ], + [ + 799, + 606 + ], + [ + 976, + 2 + ] + ], + "mask_path": "dd06-ov_mask.png", + "metadata": { + "source": "./data/dd06/0726.mp4" + } + } +} diff --git a/config/prototype_of3.json b/config/box.json similarity index 77% rename from config/prototype_of3.json rename to config/box.json index 8d94471..af88413 100644 --- a/config/prototype_of3.json +++ b/config/box.json @@ -11,14 +11,16 @@ }, "stitcher": { "min_area_frac": 0.01, - "max_area_frac": 0.04, - "max_aspect_ratio": 100, + "max_area_frac": 0.05, + "max_aspect_ratio": 50, "index": [1,2,3,6,7,8,11,12,13], "margin": 10, "nominal_ov_pos": 100, "n": [3,3], "row_tol": 0.6, - "box_size": [220, 220] + "box_size": [250, 250], + "blend":"numba", + "blend_list": ["numpy", "numba","none"] }, "tracker": { "adaptive": true, @@ -33,11 +35,11 @@ }, "contact": { "min_contour_area": 3000, - "flow_threshold":1, - "div_threshold":0.02, + "flow_threshold":5, + "div_threshold":0.05, "expansion_threshold": 0.01, "kernel_size": [5,5], - "denoise_sigma": 15 + "denoise_sigma": 12 } } diff --git a/config/dd06-ov-0810.json b/config/dd06-ov-0810.json new file mode 100644 index 0000000..209d274 --- /dev/null +++ b/config/dd06-ov-0810.json @@ -0,0 +1,101 @@ +{ + "name": "dd06-ov", + "type": "single_camera", + "version": "1.0", + "camera": { + "resolution_raw": [ + 1280, + 720 + ], + "fps": 30, + "resolution_stitch": [ + 195, + 275 + ], + "source_scale": 0.4, + "old": [ + 672, + 728, + 1, + 0.5 + ], + "work": [ + 504, + 546, + 1 + ], + "work2": [ + 420, + 450, + 0.8 + ], + "work3": [ + 350, + 375, + 4, + 0.7 + ], + "work4": [ + 280, + 300, + 4, + 0.4, + 0.2 + ] + }, + "tracker": { + "adaptive": true, + "lower_threshold": 12.0, + "upper_threshold": 18.0, + "loop_closure_interval": 1, + "algorithm": "dis", + "dis_preset": "medium", + "dis_mode": 3, + "algorithm_list": [ + "raft", + "dis" + ], + "hhd_downsample": 3 + }, + "contact": { + "min_contour_area": 3000, + "flow_threshold": 0.2, + "div_threshold": 0.022, + "expansion_threshold": 0.01, + "kernel_size": [ + 5, + 5 + ], + "denoise_sigma": 12 + }, + "roi": { + "type": "polygon", + "image_size": [ + 1280, + 720 + ], + "points": [ + [ + 628, + 532 + ], + [ + 1145, + 694 + ], + [ + 1161, + 8 + ], + [ + 630, + 126 + ] + ], + "created_at": "2026-08-10T14:26:31", + "mask_path": "dd06-ov-0810_mask.png", + "metadata": { + "source": ".\\data\\dd06\\2026-08-10-142316.webm" + } + } +} diff --git a/config/dd06-ov.json b/config/dd06-ov.json new file mode 100644 index 0000000..4c966cd --- /dev/null +++ b/config/dd06-ov.json @@ -0,0 +1,101 @@ +{ + "name": "dd06-ov", + "type": "single_camera", + "version": "1.0", + "camera": { + "resolution_raw": [ + 1280, + 720 + ], + "fps": 30, + "resolution_stitch": [ + 195, + 275 + ], + "source_scale": 0.4, + "old": [ + 672, + 728, + 1, + 0.5 + ], + "work": [ + 504, + 546, + 1 + ], + "work2": [ + 420, + 450, + 0.8 + ], + "work3": [ + 350, + 375, + 4, + 0.7 + ], + "work4": [ + 280, + 300, + 4, + 0.4, + 0.2 + ] + }, + "tracker": { + "adaptive": true, + "lower_threshold": 12.0, + "upper_threshold": 18.0, + "loop_closure_interval": 1, + "algorithm": "dis", + "dis_preset": "medium", + "dis_mode": 3, + "algorithm_list": [ + "raft", + "dis" + ], + "hhd_downsample": 3 + }, + "contact": { + "min_contour_area": 3000, + "flow_threshold": 0.2, + "div_threshold": 0.022, + "expansion_threshold": 0.01, + "kernel_size": [ + 5, + 5 + ], + "denoise_sigma": 12 + }, + "roi": { + "type": "polygon", + "image_size": [ + 1280, + 720 + ], + "created_at": "2026-07-26T11:56:15", + "points": [ + [ + 242, + 1 + ], + [ + 402, + 597 + ], + [ + 799, + 606 + ], + [ + 976, + 2 + ] + ], + "mask_path": "dd06-ov_mask.png", + "metadata": { + "source": "./data/dd06/0726.mp4" + } + } +} diff --git a/config/ddjx01.json b/config/ddjx01.json index 21b5e96..6b66456 100644 --- a/config/ddjx01.json +++ b/config/ddjx01.json @@ -18,7 +18,9 @@ "nominal_ov_pos": 100, "n": [3,3], "row_tol": 0.6, - "box_size": [250, 250] + "box_size": [250, 250], + "blend":"numba", + "blend_list": ["numpy", "numba","none"] }, "tracker": { "adaptive": true, @@ -29,7 +31,7 @@ "dis_preset": "medium", "dis_mode": 3, "algorithm_list": ["raft", "dis"], - "hhd_downsample": 3 + "hhd_downsample": 1 }, "contact": { "min_contour_area": 3000, diff --git a/config/ddjx01_480p.json b/config/ddjx01_480p.json index 6083c90..e58f998 100644 --- a/config/ddjx01_480p.json +++ b/config/ddjx01_480p.json @@ -18,7 +18,9 @@ "nominal_ov_pos": 44, "n": [3,3], "row_tol": 0.6, - "box_size": [98, 98] + "box_size": [98, 98], + "blend":"numpy", + "blend_list": ["numpy", "numba","none"] }, "tracker": { "adaptive": true, diff --git a/config/ddjx01_q6a.json b/config/ddjx01_q6a.json index f267045..08c7f59 100644 --- a/config/ddjx01_q6a.json +++ b/config/ddjx01_q6a.json @@ -18,7 +18,9 @@ "nominal_ov_pos": 67, "n": [3,3], "row_tol": 0.6, - "box_size": [180, 180] + "box_size": [180, 180], + "blend":"numpy", + "blend_list": ["numpy", "numba","none"] }, "tracker": { "adaptive": true, @@ -29,7 +31,7 @@ "dis_preset": "medium", "dis_mode": 3, "algorithm_list": ["raft", "dis"], - "hhd_downsample": 3, + "hhd_downsample": 1, "backend": "cpu" }, "contact": { diff --git a/config/ddjx01_wuxi.json b/config/ddjx01_wuxi.json deleted file mode 100644 index 0f0a874..0000000 --- a/config/ddjx01_wuxi.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "name": "prototype", - "version": "1.0", - "camera": { - "resolution_raw": [1280, 720], - "fps": 15, - "units": 15, - "units_used": 9, - "resolution_stitch": [263, 263], - "stitch_res_list": [365,263,123,63] - }, - "stitcher": { - "min_area_frac": 0.01, - "max_area_frac": 0.04, - "max_aspect_ratio": 25, - "index": [1,2,3,6,7,8,11,12,13], - "margin": 10, - "nominal_ov_pos": 67, - "n": [3,3], - "row_tol": 0.6, - "box_size": [180, 180] - }, - "tracker": { - "adaptive": true, - "lower_threshold": 12.0, - "upper_threshold": 18.0, - "loop_closure_interval": 1, - "algorithm": "dis", - "dis_preset": "medium", - "dis_mode": 3, - "algorithm_list": ["raft", "dis"], - "hhd_downsample": 3, - "backend": "cpu" - }, - "contact": { - "min_contour_area": 3000, - "flow_threshold":1, - "div_threshold":0.02, - "expansion_threshold": 0.01, - "kernel_size": [5,5], - "denoise_sigma": 10 - } -} - - diff --git a/config/finger.json b/config/finger.json new file mode 100644 index 0000000..d5f6264 --- /dev/null +++ b/config/finger.json @@ -0,0 +1,35 @@ +{ + "name": "dd02-ov", + "type": "single_camera", + "version": "1.0", + "camera": { + "resolution_raw": [1280, 720], + "fps": 30, + "resolution_stitch": [280, 300], + "source_scale": 0.4, + "old":[672, 728, 1,0.5], + "work": [504, 546,1], + "work2": [420, 450,0.8], + "work3": [350, 375,4,0.7], + "work4": [280, 300,4,0.4,0.2] + }, + "tracker": { + "adaptive": true, + "lower_threshold": 12.0, + "upper_threshold": 18.0, + "loop_closure_interval": 1, + "algorithm": "dis", + "dis_preset": "medium", + "dis_mode": 1, + "algorithm_list": ["raft", "dis"], + "hhd_downsample": 5 + }, + "contact": { + "min_contour_area": 3000, + "flow_threshold": 0.2, + "div_threshold": 0.022, + "expansion_threshold": 0.01, + "kernel_size": [5, 5], + "denoise_sigma": 12 + } +} diff --git a/config/mini-fast.json b/config/mini-fast.json new file mode 100644 index 0000000..cc5868f --- /dev/null +++ b/config/mini-fast.json @@ -0,0 +1,47 @@ +{ + "name": "prototype", + "version": "1.0", + "camera": { + "resolution_raw": [1280, 720], + "fps": 30, + "units": 15, + "units_used": 9, + "resolution_stitch": [263, 263], + "stitch_res_list": [365,263,123,63] + }, + "stitcher": { + "min_area_frac": 0.01, + "max_area_frac": 0.04, + "max_aspect_ratio": 25, + "index": [1,2,3,6,7,8,11,12,13], + "margin": 10, + "nominal_ov_pos": 67, + "n": [3,3], + "row_tol": 0.6, + "box_size": [180, 180], + "blend":"numpy", + "blend_list": ["numpy", "numba","none"] + }, + "tracker": { + "adaptive": true, + "lower_threshold": 12.0, + "upper_threshold": 18.0, + "loop_closure_interval": 1, + "algorithm": "dis", + "dis_preset": "medium", + "dis_mode": 3, + "algorithm_list": ["raft", "dis"], + "hhd_downsample": 3 + }, + "contact": { + "min_contour_area": 3000, + "flow_threshold":1, + "div_threshold":0.02, + "expansion_threshold": 0.01, + "kernel_size": [5,5], + "denoise_sigma": 10 + } + } + + + \ No newline at end of file diff --git a/config/prototype_raft.json b/config/mini.json similarity index 67% rename from config/prototype_raft.json rename to config/mini.json index 480995b..af88413 100644 --- a/config/prototype_raft.json +++ b/config/mini.json @@ -6,25 +6,28 @@ "fps": 30, "units": 15, "units_used": 9, - "resolution_stitch": [384, 384] + "resolution_stitch": [365, 365], + "stitch_res_list": [365,263,123,63] }, "stitcher": { "min_area_frac": 0.01, - "max_area_frac": 0.04, - "max_aspect_ratio": 100, + "max_area_frac": 0.05, + "max_aspect_ratio": 50, "index": [1,2,3,6,7,8,11,12,13], "margin": 10, "nominal_ov_pos": 100, "n": [3,3], "row_tol": 0.6, - "box_size": [220, 220] + "box_size": [250, 250], + "blend":"numba", + "blend_list": ["numpy", "numba","none"] }, "tracker": { "adaptive": true, "lower_threshold": 12.0, "upper_threshold": 18.0, "loop_closure_interval": 1, - "algorithm": "raft", + "algorithm": "dis", "dis_preset": "medium", "dis_mode": 3, "algorithm_list": ["raft", "dis"], @@ -32,11 +35,11 @@ }, "contact": { "min_contour_area": 3000, - "flow_threshold":1.5, - "div_threshold":0.08, + "flow_threshold":5, + "div_threshold":0.05, "expansion_threshold": 0.01, "kernel_size": [5,5], - "denoise_sigma": 15 + "denoise_sigma": 12 } } diff --git a/data/jx01/jx01_hex1.3_arrows.mp4 b/data/jx01/jx01_hex1.3_arrows.mp4 deleted file mode 100644 index 4e3969e..0000000 Binary files a/data/jx01/jx01_hex1.3_arrows.mp4 and /dev/null differ diff --git a/data/jx01/jx01_hex1.3_depth.mp4 b/data/jx01/jx01_hex1.3_depth.mp4 deleted file mode 100644 index 4fd91e7..0000000 Binary files a/data/jx01/jx01_hex1.3_depth.mp4 and /dev/null differ diff --git a/data/jx01/jx01_hex1.3_pyqt.mp4 b/data/jx01/jx01_hex1.3_pyqt.mp4 deleted file mode 100644 index 4d4fea9..0000000 Binary files a/data/jx01/jx01_hex1.3_pyqt.mp4 and /dev/null differ diff --git a/data/jx01/jx01_hex1.5_2_arrows.mp4 b/data/jx01/jx01_hex1.5_2_arrows.mp4 deleted file mode 100644 index bf92053..0000000 Binary files a/data/jx01/jx01_hex1.5_2_arrows.mp4 and /dev/null differ diff --git a/data/jx01/jx01_hex1.5_2_depth.mp4 b/data/jx01/jx01_hex1.5_2_depth.mp4 deleted file mode 100644 index a09711a..0000000 Binary files a/data/jx01/jx01_hex1.5_2_depth.mp4 and /dev/null differ diff --git a/data/jx01/jx01_hex1.5_2_pyqt.mp4 b/data/jx01/jx01_hex1.5_2_pyqt.mp4 deleted file mode 100644 index daf398a..0000000 Binary files a/data/jx01/jx01_hex1.5_2_pyqt.mp4 and /dev/null differ diff --git a/docs/api.md b/docs/api.md index e07c23f..5afc69b 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,86 +1,30 @@ -# orisys API 文档 +--- +title: API Reference +description: Orisys SDK Python API 中文参考 +--- -本文件提供了 `orisys` 套件的中文文档 +# API Reference -## 准备 Python 开发环境 -推荐下载 Anaconda，并使用Python 3.10 -``` -# 创建并激活虚拟环境 -conda create -n orisys python=3.10 -conda activate orisys -``` -## 安装 CUDA Toolkit 和 cuDNN -在官方链接下载和安装：https://developer.nvidia.com/cuda/toolkit -或 -在Anaconda环境中安装（例子使用cuda12.9）： -https://anaconda.org/channels/nvidia/packages/cuda/overview -``` -conda install nvidia/label/cuda-12.9.0::cuda-toolkit nvidia::cudnn -``` -## SDK安装方法 -1. 根据系统（win / linux）,平台(amd64, aarch64), python(如p38、p39、p310) 版本下载对应的`.whl`文件 -2. 运行pip命令安装： -```bash -pip install orisys-{version}-{python-version}-{python-version}-{os}_{machine}.whl -``` -如 -```bash -pip install orisys-0.4.0-cp310-cp310-win_amd64.whl -``` +本文件提供 `orisys` 套件的 Python API 中文参考。安装、CUDA、CuPy 与运行示例的基础步骤请参考 [快速开始](./get_started.md)。 -3. **安装 CuPy（根据您的 CUDA 版本选择）：** - - **方法 1：使用 extras（推荐）** - ```bash - # 对于 CUDA 12.x - pip install orisys[cuda12] - - # 对于 CUDA 13.x - pip install orisys[cuda13] - ``` - - **方法 2：手动安装** - ```bash - # 对于 CUDA 12.x - pip install cupy-cuda12x>=12.3.0 - - # 对于 CUDA 13.x - pip install cupy-cuda13x>=13.0.0 - ``` +## Sensor API -4. 测试导入： -```python -import orisys -print(orisys.__version__) -``` - -5 安装opencl（实验）: - -在不带nvidia显卡的设备，可选择通过opencl进行gpu 计算加速，需要确保opencl的库或驱动已经安装，然后在conda环境中安装依赖： - -``` bash -pip install pyopencl -conda install conda-forge::pyvkfft -``` - -*** - -## `Sensor` 类别 -### 描述 +### `Sensor` 类别 +#### 描述 `orisys`套件的核心类，用于从视频文件或实时摄像头中读取传感器图像，创建一个具体的传感器对象（实例）。并完成形变、光流及力信息的计算与管理。 -### 输入参数 +#### 输入参数 * **vid_src** (`int | str`, 必填): 传感器 ID、相机编号或视频路径。默认为 `0`。 -* **config_name** (`str`，可选): 传感器配置文件名称或 `json` 文件路径，用于加载对应的参数配置；例如 `ddjx01` 或 `./config/ddjx01.json`。 +* **config_name** (`str`，可选): 传感器配置文件名称或 `json` 文件路径，用于加载对应的参数配置；例如 `box` 或 `./config/box.json`。 * **cal_path** (`str`，可选，已弃用): 该参数仅为兼容旧代码保留，当前版本中不再使用，传入后也会被忽略。 * **backend** (`str`，可选): 光流场解耦计算后端，默认为 `auto`，可选`cupy`,`opencl`,`cpu`。 * **verbose** (`bool`，可选): 是否输出初始化及运行过程中的详细日志信息，默认为 `False`。 -### 返回 +#### 返回 * `Sensor` 实例，用于后续图像采集、形变计算和数据读取操作。 -### 示例 +#### 示例 ```python # Example 1： 用相机编号创建 @@ -91,63 +35,62 @@ sensor = Sensor(0) sensor = Sensor("/data/data.mp4") ``` -*** +### `Sensor.get_img` 方法 -## 2. `Sensor.get_img` 方法 - -### 描述 +#### 描述 采集并预处理一帧图像并存到`Sensor` 类别中。 -### 返回 +#### 返回 * `bool`：是否成功读取图像。 -### 状态更新 +#### 状态更新 * 成功时更新`sensor`类里面的`sensor.img`。处理后的BGR格式图像(`uint8`，尺寸为json文件中`resolution_stitch`对应分辨率)。 -### 示例 +#### 示例 ```python sensor.get_img() ``` -*** +### `Sensor.compute_deformation` 方法 -## 3. `Sensor.compute_deformation` 方法 - -### 描述 +#### 描述 基于当前帧图像计算光流/形变向量场，并通过 HHD 分解将形变场分解为法向/切向分量，同时估计法向力、切向力与扭矩等标量输出。该方法会更新 `sensor`的形变场与力相关状态，并返回当前帧的光流场。 调用前置条件：需要先通过`get_img()`成功更新图像。 -### 输入参数 -* **check_motion**(`bool`，可选): 为 `True` 时先根据光流判断是否有运动，有运动才做 HHD 分解；为 `False` 时每帧都做 HHD。默认 `True`。 -* **threshold**(`float`，可选): **check_motion** 功能的阈值，默认为`6`。 -* **tail_frames** (`int`，可选): 光流变化低于阈值后继续解算多少帧，确保传感器回复，，默认为 `10`. +#### 输入参数 +* **decompose** (`bool`，可选): 是否在计算光流后继续执行 HHD 分解，默认 `True`。 +* **check_motion** (`bool`，可选): 是否启用运动检测门控。当前不推荐依赖该功能；新代码建议显式传入 `False`，让每帧都执行 HHD 分解。为兼容旧代码，函数默认值仍为 `True`。 -### 返回 +#### 兼容参数（暂不推荐） -* `bool`：是否接触到物体，如果**check_motion**为`False`时返回`None`。 +以下参数只在 `check_motion=True` 时生效。由于运动检测门控目前不稳定，建议在新代码中暂时不要使用： -### 示例 +* **threshold** (`float`，可选): 运动检测阈值，默认 `6`。 +* **tail_frames** (`int`，可选): 运动低于阈值后继续解算多少帧，默认 `10`。 + +#### 返回 + +* `None`: 当 `check_motion=False` 时返回。推荐用法下不要依赖该返回值判断接触状态。 +* `bool`: 仅当 `check_motion=True` 时返回运动检测结果；该模式目前暂不推荐。 + +#### 示例 ```python sensor.get_img() - -is_touched = sensor.compute_deformation(check_motion=True, threshold=6) -# 步骤 2.3: 计算触碰点 -if is_touched: - sensor.compute_contact() +sensor.compute_deformation(check_motion=False) +sensor.compute_contact() ``` -*** -## 4. `Sensor.compute_contact` 方法 +### `Sensor.compute_contact` 方法 -### 描述 +#### 描述 基于已计算的法向形变场进行接触区域分析，输出接触质心与轮廓等信息，并生成接触深度图，默认配置在`json`文件中，可按需调节。 调用前置条件：需先执行`compute_deformation()`。 -### 返回 +#### 返回 `dict`为接触分析结果字典，至少包含以下键： * **expansion_mask**: 扩张区域掩码（`np.ndarray`）。 @@ -157,12 +100,12 @@ if is_touched: * **centroid_found**: 是否检测到接触区域（`bool`）。 * **contour**: 最大轮廓或轮廓列表（可选字段，存在时返回）。 -### 状态更新 +#### 状态更新 * **sensor.centroid**: 接触质心（单点为 `(x, y)`，多点为列表）。 * **sensor.centroid_found**: 是否找到质心（`bool`）。 -### 示例 +#### 示例 ```python sensor.compute_contact() centroid, contour, depth = sensor.read_info( @@ -171,19 +114,18 @@ centroid, contour, depth = sensor.read_info( sensor.info.DEPTH, ) ``` -*** -## 4. `Sensor.read_info` 方法 +### `Sensor.read_info` 方法 -### 描述 +#### 描述 按指定的信息类型（`sensor.info`枚举值）批量读取传感器状态数据，并按传入顺序返回对应值的元组。 该方法通常用于在完成形变计算后，读取光流场、分量场、力、帧率等信息。 -### 输入参数 +#### 输入参数 * **args**: 任意数量的 `Sensor.info` 枚举，用于指定需要获取的数据类型。支持的枚举值及对应数据如下： -### 支持的枚 info keys： +#### 支持的 info keys 向量/图像类 @@ -197,6 +139,9 @@ centroid, contour, depth = sensor.read_info( * FNORMAL: 法向力，`float`。 * FSHEARX: X切向力，`float`。 * FSHEARY: Y切向力，`float`。 +* FNORMAL_RAW: 未经过低通滤波的原始法向力，`float`。 +* FSHEARX_RAW: 未经过低通滤波的原始 X 切向力，`float`。 +* FSHEARY_RAW: 未经过低通滤波的原始 Y 切向力，`float`。 * FPS: 输出帧率，`float`。 * CENTROID： 触碰点的坐标，(x, y)。 * CONTOUR： 触碰区域轮廓，`np.ndarray`, shape=(H, W), 未生成时返回`np.empty` @@ -207,7 +152,7 @@ centroid, contour, depth = sensor.read_info( 注：H/W 由当前`json`文件中的`resolution_stitch`配置分辨率决定。 -### 返回 +#### 返回 `Tuple` @@ -219,7 +164,7 @@ centroid, contour, depth = sensor.read_info( - 标量类数据：`float` - 时间戳：`str`（`TIMESTAMP`，格式为 `HH:MM:SS`） -### 示例 +#### 示例 ```python # Example 1：读取标量数据 fps, fn, fx, fy = sensor.read_info( @@ -232,23 +177,352 @@ fps, fn, fx, fy = sensor.read_info( # Example 2： 读取单个向量场（注意解包方式） flow = sensor.read_info(sensor.info.VRAW) ``` -*** +### `Sensor.disconnect` 方法 -## 5. `Sensor.disconnect` 方法 - -### 描述 +#### 描述 释放传感器相关资源并停止后台异步线程。 调用后将关闭视频/摄像头输入源，并销毁所有 OpenCV 窗口。 建议在程序退出前调用以确保资源被正确释放。 -*** -*** -# 可视化相关 +### `Sensor.configure` 低通滤波设置 -## 1. `util.draw_arrows` 方法 +#### 描述 -### 输入参数 +运行时更新传感器设置，无需重新连接相机。常用于开启力信号低通滤波，降低 `FNORMAL`、`FSHEARX`、`FSHEARY` 的高频噪声。 + +默认采样率按 30 fps 计算，推荐低通截止频率为 5 Hz： + +```python +sensor.configure(lowpass_cutoff_hz=5.0, lowpass_sample_rate_hz=30.0) +``` + +注意：`lowpass_cutoff_hz` 必须小于 Nyquist 频率，即 `lowpass_sample_rate_hz / 2`。例如 30 fps 时，截止频率必须小于 15 Hz。 + +#### 输入参数 + +* **lowpass_cutoff_hz** (`float`，可选): 力信号低通截止频率，单位 Hz。`0` 表示关闭滤波；正数默认开启滤波。 +* **lowpass_sample_rate_hz** (`float`，可选): 力信号采样率，默认 `30.0`。 +* **lowpass_order** (`int`，可选): Butterworth 滤波器阶数，默认 `2`。 +* **lowpass_enabled** (`bool`，可选): 显式开启或关闭低通滤波。 +* **hhd_downsample** (`int`，可选): HHD 解算下采样倍数，需为 `>= 1` 的整数，配置默认 `3`。 + +#### 读取结果 + +开启低通滤波后，`read_info()` 读取的力输出会被滤波： + +```python +fn, fx, fy = sensor.read_info( + sensor.info.FNORMAL, + sensor.info.FSHEARX, + sensor.info.FSHEARY, +) +``` + +原始力输出仍可通过 `FNORMAL_RAW`、`FSHEARX_RAW`、`FSHEARY_RAW` 读取，便于调试或对比。 + +## Signal Processing API + +### `orisys.signal_processing.lowpass_filter` 函数 + +#### 描述 + +对一维、等间隔采样数据执行零相位 Butterworth 低通滤波，适合示例程序或图表中对历史数据做平滑处理。 + +默认参数为 `cutoff_hz=5.0`、`sample_rate_hz=30.0`、`order=2`。这表示在 30 fps 数据上使用 5 Hz 低通滤波。 + +#### 输入参数 + +* **data**: 一维数据序列。 +* **cutoff_hz** (`float`，可选): 截止频率，默认 `5.0`。传入 `0` 时不做滤波并返回数据副本。 +* **sample_rate_hz** (`float`，可选): 采样率，默认 `30.0`。 +* **order** (`int`，可选): Butterworth 滤波器阶数，默认 `2`。 + +#### 返回 + +* `np.ndarray`: 滤波后的 `float64` 数组。 + +#### 示例 + +```python +from orisys.signal_processing import lowpass_filter + +filtered_force = lowpass_filter(force_values, cutoff_hz=5.0, sample_rate_hz=30.0) +``` + +### `orisys.signal_processing.LowpassFilter` 类别 + +#### 描述 + +有状态的因果 Butterworth 低通滤波器，适合逐帧处理实时数据。它会保留上一帧的滤波状态，并通过 `process()` 一次处理一个多通道样本。 + +`Sensor.configure()` 内部使用该类来滤波力信号。构造时默认 `cutoff_hz=0.0`，表示关闭滤波；若要按 30 fps 使用推荐滤波，设置 `cutoff_hz=5.0`、`sample_rate_hz=30.0`。 + +#### 输入参数 + +* **cutoff_hz** (`float`，可选): 截止频率，默认 `0.0`，表示关闭滤波。 +* **sample_rate_hz** (`float`，可选): 采样率，默认 `30.0`。 +* **order** (`int`，可选): Butterworth 滤波器阶数，默认 `2`。 +* **n_channels** (`int`，可选): 每个样本的通道数，默认 `3`。 + +#### 常用方法 + +* **configure(...)**: 运行时更新 `cutoff_hz`、`sample_rate_hz` 或 `order`。 +* **reset(x0=None)**: 清除滤波器状态；传入 `x0` 时会用该样本初始化稳态。 +* **process(values)**: 滤波一个多通道样本，返回 shape 为 `(n_channels,)` 的 `np.ndarray`。 + +#### 示例 + +```python +from orisys.signal_processing import LowpassFilter + +filt = LowpassFilter(cutoff_hz=5.0, sample_rate_hz=30.0, n_channels=3) +filtered = filt.process([fnormal, fshearx, fsheary]) +``` + +## Finger Pipeline API + +### `orisys.finger` 手指 UV pipeline + +`examples/finger_pipeline.py` 展示了一个更适合手指/单相机场景的高层 API： + +1. 使用 `FingerRuntime.open()` 打开视频源、加载 Blender 导出的资产，并构建 UV 展开映射。 +2. 用 `grab_and_process()` 或 `read_source() + process_frame()` 获取一帧展开后的触觉图像。 +3. 通过 `runtime.tactile`（`TactileProcessor`）调用 `compute_deformation()`、`compute_contact()`、`read_info()`。 +4. 退出前调用 `runtime.close()` 释放视频源和内部 `Sensor` 资源。 + +相比直接使用 `Sensor`，`FingerRuntime` 已封装了视频读取、相机尺寸设置、UV 展开和有效区域掩码传递，更适合作为手指 2D/3D 示例与上层应用的入口。 + +### `FingerRuntime.open` 方法 + +#### 描述 + +创建一个可复用的手指 UV runtime。该方法会完成以下初始化： + +- 解析 `export_dir` 下的 `mesh.npz`、`camera_intrinsics.npz`、`world_to_cam.npy` +- 根据配置和标定信息构建/加载 `unroll_map_uv.npz` +- 打开相机或视频文件 +- 创建内部 `Sensor("external", ...)` +- 创建 `FrameUnwarper` 与 `TactileProcessor` + +#### 输入参数 + +* **export_dir** (`str`，可选): 手指导出资产目录，默认值来自 `orisys.finger.DEFAULT_FINGER_EXPORT_DIR`。目录下通常需要包含 `mesh.npz`、`camera_intrinsics.npz`、`world_to_cam.npy`。 +* **config** (`str`，可选): Orisys 配置文件名称或路径，默认 `finger`。 +* **map_cache** (`str`，可选): UV 展开缓存文件路径，默认值来自 `orisys.finger.DEFAULT_FINGER_MAP_CACHE`。 +* **source_scale** (`float`，可选): 输入相机分辨率缩放比例；未提供时根据配置自动推断。 +* **calib** (`str | None`，可选): 外部相机标定文件路径；未提供时优先使用导出资产中的内参。 +* **video** (`int | str`，可选): 相机编号、数字字符串或视频路径，默认 `0`。 +* **camera_id** (`int | None`，可选): `video` 的别名；若提供则优先使用。 +* **flip_source_vertical** (`bool`，可选): 是否在 UV 展开前垂直翻转输入源，默认 `True`。 +* **verbose** (`bool`，可选): 是否输出初始化日志。 +* **mirror** (`bool`，可选): 是否对内部触觉图再做镜像，默认 `False`。 + +#### 返回 + +* `FingerRuntime` 实例。 + +#### 示例 + +```python +from orisys.finger import FingerRuntime + +runtime = FingerRuntime.open( + config="finger", + video=0, + verbose=True, +) +``` + +### `FingerRuntime` 常用属性 + +* **runtime.geometry**: `FingerGeometry`，包含输入/输出分辨率、导出资产路径、相机内参、缓存路径等一次性初始化结果。 +* **runtime.domain**: UV 展开域对象，内部包含 remap、valid mask 以及可选 surface 数据。 +* **runtime.tactile**: `TactileProcessor`，用于执行形变、接触和数据读取。 +* **runtime.capture**: OpenCV `VideoCapture` 对象。 +* **runtime.valid_mask**: 当前 UV 域有效像素掩码。 + +### `FingerRuntime.read_source` 方法 + +#### 描述 + +仅读取一帧原始视频源，等价于对内部 `cv2.VideoCapture` 调用 `read()`。适合需要把采集和 UV 展开分别计时的场景。 + +#### 返回 + +* `(ok, frame)`: + * **ok** (`bool`): 是否成功读取。 + * **frame** (`np.ndarray | None`): 原始 BGR 帧。 + +### `FingerRuntime.process_frame` 方法 + +#### 描述 + +对一帧原始输入执行源图预处理、UV 展开，并把展开后的结果送入内部 `TactileProcessor`。这是 `examples/finger_pipeline.py` 中 profile 模式使用的方法。 + +#### 输入参数 + +* **frame** (`np.ndarray`，必填): 原始输入帧。 +* **color** (`bool`，可选): 为 `False` 时走灰度展开流程；为 `True` 时保留 BGR 颜色展开。默认 `False`。 + +#### 返回 + +* `np.ndarray`: 展开后的触觉图像。`color=False` 时通常为单通道灰度图；`color=True` 时为 BGR 图。 + +### `FingerRuntime.grab_and_process` 方法 + +#### 描述 + +先读取一帧视频源，再调用 `process_frame()` 完成 UV 展开并推入内部触觉处理器。非 profile 模式下，`examples/finger_pipeline.py` 使用该方法作为主循环入口。 + +#### 输入参数 + +* **color** (`bool`，可选): 是否保留颜色，默认 `False`。 + +#### 返回 + +* `(ok, unrolled)`: + * **ok** (`bool`): 是否成功读取并处理。 + * **unrolled** (`np.ndarray | None`): 展开后的触觉图像。 + +### `FingerRuntime.get_img` 方法 + +#### 描述 + +`Sensor.get_img()` 风格的别名：内部执行一次采集和 UV 展开，并在成功时返回当前展开图，否则返回 `None`。 + +#### 输入参数 + +* **color** (`bool`，可选): 是否保留颜色，默认 `False`。 + +#### 返回 + +* `np.ndarray | None` + +### `FingerRuntime.warmup` 方法 + +#### 描述 + +先执行一次采集与 UV 展开，然后调用 `runtime.tactile.warmup()` 预热内部光流/HHD 相关缓存。`examples/finger_pipeline.py` 在进入主循环前会先执行一次 warmup。 + +#### 输入参数 + +* **color** (`bool`，可选): 是否保留颜色，默认 `False`。 + +#### 返回 + +* `(ok, unrolled)`: + * **ok** (`bool`): 是否成功完成 warmup 前的采集。 + * **unrolled** (`np.ndarray | None`): warmup 时得到的展开图。 + +### `FingerRuntime.close` 方法 + +#### 描述 + +释放内部 `VideoCapture` 并调用 `runtime.tactile.disconnect()` 清理 `Sensor` 资源。程序退出前应调用。 + +### `TactileProcessor` 类别 + +#### 描述 + +`TactileProcessor` 是对内部 `Sensor("external", ...)` 的窄封装。它保留了 `compute_deformation()`、`compute_contact()`、`read_info()`、`reset()` 等常用接口，同时屏蔽了视频采集与 UV 展开细节。 + +#### 常用属性 + +* **raw_sensor**: 底层 `Sensor` 对象。 +* **info**: 与 `Sensor.info` 相同的枚举入口。 +* **img / flow / vnormal / vshear / depth_map**: 当前帧处理结果。 +* **mask_stats**: 有效 UV 区域统计信息；`examples/finger_pipeline.py` 会读取 `mask_stats["coverage"]` 输出当前掩码覆盖率。 +* **valid_mask**: 当前有效区域掩码。 + +#### 说明 + +除 `push_frame()` / `warmup()` 之外，`TactileProcessor` 的 `compute_deformation()`、`compute_contact()`、`read_info()`、`reset()` 调用方式与 `Sensor` 基本一致，因此可直接复用前文 `Sensor` 章节中的参数说明。 + +### 最小手指 pipeline 示例 + +下面的流程与 `examples/finger_pipeline.py` 一致： + +```python +import time +import cv2 +import orisys +from orisys.finger import FingerRuntime + +runtime = FingerRuntime.open( + config="finger", + video=0, + verbose=True, +) + +runtime.warmup(color=False) +tactile = runtime.tactile + +try: + while True: + ok, unrolled = runtime.grab_and_process(color=False) + if not ok: + break + + tactile.compute_deformation(check_motion=True, threshold=0) + tactile.compute_contact() + + fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = tactile.read_info( + tactile.info.FPS, + tactile.info.VRAW, + tactile.info.VNORMAL, + tactile.info.IMG, + tactile.info.CONTOUR, + tactile.info.CENTROID, + tactile.info.DEPTH, + tactile.info.FNORMAL, + tactile.info.FSHEARX, + tactile.info.FSHEARY, + ) + + arrows = orisys.util.draw_arrows( + unrolled, + flow, + threshold=0.4, + grid_spacing=10, + arrow_scale=1.0, + ) + cv2.imshow("UV Unrolled", arrows) + + print( + f"FPS={fps:.2f} normal={fnormal:.2f} " + f"shear=({fshearx:.2f}, {fsheary:.2f}) " + f"mask={tactile.mask_stats['coverage']:.2f}", + end="\r", + ) + + key = cv2.waitKey(1) & 0xFF + if key == ord("q"): + break + if key == ord("r"): + tactile.reset() +finally: + runtime.close() + cv2.destroyAllWindows() +``` + +### 关于 profile 模式 + +`examples/finger_pipeline.py` 中的 `--profile` / `--profile-every` 是示例脚本参数，不属于 `orisys` SDK 核心 API。本质上它只是把主循环拆成： + +1. `runtime.read_source()` +2. `runtime.process_frame(...)` +3. `tactile.compute_deformation(...)` +4. `tactile.compute_contact()` +5. `tactile.read_info(...)` + +并对各阶段做耗时统计。因此在业务代码中，也可以按同样方式自行插入计时逻辑。 + +## 可视化相关 + +### `util.draw_arrows` 方法 + +#### 输入参数 * **frame**：背景图像 * **optical_flow**：光流场 @@ -260,12 +534,12 @@ flow = sensor.read_info(sensor.info.VRAW) * **below_threshold_color**：RGB格式的`uint8`颜色，如(0, 0, 255) * **edge_margin**: 不显示箭头边距（pixel） -### 返回 +#### 返回 `np.ndarray` ：箭头光流可视化图像 -## 2. `util.draw_contact` 方法 +### `util.draw_contact` 方法 -### 输入参数 +#### 输入参数 * **image**：背景图像 * **contour**：轮廓 @@ -273,42 +547,36 @@ flow = sensor.read_info(sensor.info.VRAW) * **point_color**：触碰点的颜色 (BGR) * **region_color**：轮廓的颜色 (BGR) -### 返回 +#### 返回 `np.ndarray` ：箭头光流可视化图像 -*** +## 配置标定相关 -# 配置标定相关 +### `list_config` 方法 -## 1. `list_config` 方法 - -### 描述 +#### 描述 列出内置传感器配置参数字典。 -### 返回 +#### 返回 * `list`: 内置传感器配置参数字典的名字列表。 -*** -## 2. `export_config` 方法 +### `export_config` 方法 -### 描述 +#### 描述 导出当前传感器实例使用的配置参数字典。 -### 输入参数 +#### 输入参数 * **config_name**: 需要导出的配置参数字典名字，`str` * **path**: 导出的路径，默认`"./config/exported_config.json"` -### 返回 +#### 返回 * `dict`: 导出的配置参数字典。 -### 保存 +#### 保存 * 导出内置的`config_name`配置参数字典至`path` -*** -*** - ## 常见问题解答 (FAQ) ### Q1:为什么必须先调用`get_img()`，再调用`compute_deformation()`？ A：`compute_deformation()`依赖最近一次获取并拼接的图像数据进行形变与力计算。 @@ -328,9 +596,5 @@ A：当需要接触相关信息（如接触质心、轮廓或深度/散度图） A：`read_info()`按传入的`sensor.info`参数顺序返回一个元组。 返回值顺序与参数顺序一一对应，与内部存储顺序无关。 -### Q5:如果没有标定文件，或拼接结果不正确，会发生什么？ -A：若未指定或未找到标定文件，系统会在需要时创建新的拼接标定数据并保存到默认路径。 -拼接结果异常通常与标定数据、拼接配置或输入图像质量有关，建议重新生成拼接标定并检查配置参数。 - -### Q6: 运行报错：“AttributeError: 'Stitcher' object has no attribute 'pos_default'” +### Q5: 运行报错：“AttributeError: 'Stitcher' object has no attribute 'pos_default'” A: 请检查视频源的编号是否正确，以及是否能正常打开视频源。在linux 使用`v4l2-ctl --list-devices` 或 Windows使用`Get-PnpDevice -PresentOnly | Where-Object { $_.Class -match '^Camera' }` 列出摄像头设备。 \ No newline at end of file diff --git a/docs/get_started.md b/docs/get_started.md index cc11ec2..67b517c 100644 --- a/docs/get_started.md +++ b/docs/get_started.md @@ -1,80 +1,106 @@ -叠动科技软件使用手册 -版本： 0.4.0 -更新日期： 2026年4月16日 +--- +title: 快速开始 +description: 安装并运行 Orisys SDK 的基础步骤 +--- -# 简介 +# 快速开始 + +版本：0.5.0 + +## 简介 本文件介绍如何上手和运行叠动视触觉传感器配套的开发套件。本套件是配合叠动视触觉传感器进行图像处理，计算形变、力以及接触信息。它可以从摄像头或视频文件中读取图像，处理后打印和展示处理后信息。 支持操作系统：Windows、Ubuntu -建议Python 版本: 3.10 +建议Python 版本: 3.10, 3.11, 3.12 -# 基础环境设置 -## 安装 Python 与 Anaconda -1. 访问 [Anaconda 官网](https://www.anaconda.com/download)下载适合您操作系统的安装包（Windows、Linux）。 -2. 按照提示完成安装。（Widnows：建议在安装过程中勾选“将 Anaconda 添加到系统环境变量”选项，方便后续使用）。 -3. 安装完成后，打开命令行终端（Windows 用户可在”Anaconda Navigator”打开“Anaconda Prompt”），输入 `conda info` ，若能显示 conda 的信息，则说明安装成功。 +项目链接： -## 创建conda虚拟环境 +- PyPI 安装页：[https://pypi.org/project/orisys/](https://pypi.org/project/orisys/) +- 项目文档/代码：[https://github.com/Orisys-Tech/orisys-sdk](https://github.com/Orisys-Tech/orisys-sdk) + +## 基础环境设置 +### 安装 Python 与 Anaconda +1. 访问 [Anaconda 官网](https://www.anaconda.com/download) 下载适合您操作系统的安装包（Windows、Linux）。 +2. 按照提示完成安装。 +3. 安装完成后，打开命令行终端（Windows 用户可在 “Anaconda Navigator” 打开 “Anaconda Prompt”），输入 `conda info`，若能显示 conda 的信息，则说明安装成功。 + +### 创建 conda 虚拟环境 虚拟环境相当于一个独立的 Python 工作空间，可以避免不同项目所需的库版本冲突。 -打开命令行终端（或Anaconda Prompt），依次输入以下命令(以 python 3.10 为例)： -``` bash +打开命令行终端（或 Anaconda Prompt），依次输入以下命令（以 Python 3.10 为例）： +```bash conda create -n orisys python=3.10 conda activate orisys ``` 激活后，命令行提示符前会出现 `(orisys)` 字样，表示当前已进入虚拟环境。 -## 创建工作区文件夹 -“Anaconda Prompt”启动后的默认工作路径在用户根文件夹中，建议创建工作区文件夹方便管理，可以手动创建或者运行`mkdir `命令：, 例如 +### 创建工作区文件夹 +“Anaconda Prompt” 启动后的默认工作路径在用户根文件夹中，建议创建工作区文件夹方便管理。可以手动创建文件夹，也可以运行 `mkdir ` 命令。例如： -``` bash +```bash mkdir orisys_ws ``` -创建后如果要更换工作区，使用 `cd `命令，例如： +创建后如果要更换工作区，使用 `cd ` 命令，例如： -``` bash +```bash cd "C:\Users\orisys\orisys_ws" ``` -
- -## 安装 CUDA 和 cuDNN（可选） +### 安装 CUDA 和 cuDNN（可选） 如果您拥有 NVIDIA 显卡，并希望利用 GPU 加速计算，则需要安装 CUDA 工具包和 cuDNN。 1. 方法一（官网下载）： -访问 [NVIDIA CUDA Toolkit官网](https://developer.nvidia.com/cuda/toolkit)下载对应系统的 CUDA Toolkit 安装包，并按照提示安装, 注意12.X 和 13.X 的版本对应不同的 CuPy 库版本。 +访问 [NVIDIA CUDA Toolkit 官网](https://developer.nvidia.com/cuda/toolkit) 下载对应系统的 CUDA Toolkit 安装包，并按照提示安装。注意 12.x 和 13.x 的版本对应不同的 CuPy 库版本。 2. 方法二（使用 Anaconda 安装，推荐）： 在已激活的 orisys 环境中直接运行以下命令（以 CUDA 12.9 为例）： -`conda install nvidia/label/cuda-12.9.0::cuda-toolkit nvidia::cudnn` + +```bash +conda install nvidia/label/cuda-12.9.0::cuda-toolkit nvidia::cudnn +``` + 该方法会自动配置好 CUDA 和 cuDNN，无需手动下载。 注意：如果您的计算机没有 NVIDIA 显卡，或者不想使用 GPU 加速，可以跳过此步骤。套件也支持纯 CPU 运行（但速度可能较慢）。 -# 安装 orisys SDK +## 安装 orisys SDK -orisys SDK 以 .whl 文件形式提供，您需要根据操作系统、Python 版本和硬件平台选择正确的文件。 +推荐优先从 PyPI 安装。PyPI 会自动选择适合当前 Python 版本和操作系统的安装包，适合大多数用户。 -## 下载正确的 .whl 文件 +### 使用 PyPI 安装（推荐） +确认已经激活 `orisys` 虚拟环境后，运行： + +```bash +pip install orisys +``` + +如果需要升级到最新版本，运行： + +```bash +pip install --upgrade orisys +``` + +如果当前网络无法访问 PyPI，或需要安装客服提供的指定版本，可以使用下方 `.whl` 文件安装方式。 + +### 使用 .whl 文件安装（离线或指定版本） 从官方渠道获取 .whl 文件，文件名格式通常为： orisys-{版本}-{Python版本}-{Python版本}-{操作系统}_{硬件架构}.whl -例如：orisys-0.4.0-cp310-cp310-win_amd64.whl 表示： +例如：orisys-0.5.0-cp310-cp310-win_amd64.whl 表示： -版本：0.4.0 +版本：0.5.0 Python 版本：3.10（cp310） 操作系统：Windows（win） 硬件架构：64位（amd64） -请根据您的系统选择对应的文件，或者咨询客服索取： +请根据您的系统选择对应的文件，或者咨询客服索取。 -## 安装 .whl 文件 -将下载好的 .whl 文件放在一个文件夹中（如上述`orisys_ws`文件夹），打开命令行（确保已激活 orisys 环境或使用"Anaconda Prompt"），使用 `cd` 命令切换到该文件夹，然后运行： +将下载好的 `.whl` 文件放在一个文件夹中（如上述 `orisys_ws` 文件夹），打开命令行（确保已激活 `orisys` 环境或使用 “Anaconda Prompt”），使用 `cd` 命令切换到该文件夹，然后运行： -``` bash +```bash # 请将文件名替换为您实际下载的文件名 -pip install orisys-0.4.0-cp310-cp310-win_amd64.whl +pip install orisys-0.5.0-cp310-cp310-win_amd64.whl ``` -## 安装 CuPy（可选） +### 安装 CuPy（可选） 注意：如果您没有安装 CUDA，或者使用 CPU 运行，可以跳过这一步。 CuPy 是一个与 NumPy 类似但支持 GPU 加速的库，您需要根据您安装的 CUDA 版本选择对应的 CuPy 版本。 @@ -84,55 +110,117 @@ CuPy 是一个与 NumPy 类似但支持 GPU 加速的库，您需要根据您安 ```bash # 如果您的 CUDA 版本是 12.x -pip install orisys[cuda12] +pip install "orisys[cuda12]" # 如果您的 CUDA 版本是 13.x -pip install orisys[cuda13] +pip install "orisys[cuda13]" ``` 如果之前已经安装了 orisys，可以单独安装 CuPy： 方法二（手动安装）： -``` bash +```bash # CUDA 12.x -pip install cupy-cuda12x>=12.3.0 +pip install "cupy-cuda12x>=12.3.0" # CUDA 13.x -pip install cupy-cuda13x>=13.0.0 +pip install "cupy-cuda13x>=13.0.0" ``` -## 测试安装 -安装完成后，在命令行输入” python“命令进入 Python 环境， 并导入 orisys 库并查看版本号，验证是否成功： -``` bash +### 测试安装 +安装完成后，在命令行输入 `python` 命令进入 Python 环境，并导入 `orisys` 库查看版本号，验证是否成功： +```bash python +``` + +进入 Python 后继续输入： + +```python import orisys print(orisys.__version__) ``` -如果输出版本号（例如 0.4.0），则说明安装成功。 +如果输出版本号（例如 0.5.0），则说明安装成功。 -# 使用方法 -## 运行示例程序 -套件里面包含了例子展示用法，运行步骤如下： +## 使用方法 +### 运行示例程序 +套件里面包含了多个例子。运行示例前，请先进入 SDK 示例所在的文件夹，建议在仓库根目录运行（即 `config/`、`examples/`、`assets/` 同级目录）。如果您是通过 PyPI 安装并且本地没有示例文件，可以从上方项目链接下载示例。按产品入口如下： -### Windows: -1. 检查是否有其他正在连接的USB摄像头设备：打开powershell, 运行命令`Get-PnpDevice -PresentOnly | Where-Object { $_.Class -match '^Camera' }`，如果没有的话不会有输出。 -2. 把传感器的USB头与电脑连接，再次运行步骤一的命令，会新增一行输出 -3. 如果第一步没有输出，传感器的ID就是“0”，如果第一步有一个设备，传感器的ID就是“1”，如此类推。 -4. 在确定了传感器的ID（下方例子假设为“0”）后，打开Anaconda Prompt, 切换到工作路径，然后在命令行中输入： - ``` bash - python examples/basic_pipeline.py -v 0 +1. `examples/basic_pipeline.py`：ORY-BOX 复眼/拼接传感器基础 OpenCV 示例（默认配置 `box`） +2. `examples/mini_pipeline.py`：ORY-MINI 基础 OpenCV 示例（默认配置 `mini`） +3. `examples/finger_pipeline.py`：ORY-FINGER 2D 显示最小流程（默认配置 `finger`） +4. `examples/finger_pipeline_3dview.py`：ORY-FINGER Qt 3D 可视化界面（默认配置 `finger`） + +`finger` 是当前 ORY-FINGER 的产品配置名。 + +下面先介绍最基础的 ORY-BOX 运行方式。 + +#### Windows +1. 检查是否有其他正在连接的 USB 摄像头设备：打开 PowerShell，运行命令 `Get-PnpDevice -PresentOnly | Where-Object { $_.Class -match '^Camera' }`，如果没有的话不会有输出。 +2. 把传感器的 USB 头与电脑连接，再次运行步骤一的命令，会新增一行输出。 +3. 如果第一步没有输出，传感器的 ID 就是 `0`；如果第一步有一个设备，传感器的 ID 就是 `1`，如此类推。 +4. 在确定了传感器的 ID（下方例子假设为 `0`）后，打开 Anaconda Prompt，切换到工作路径，然后在命令行中输入： + ```bash + python examples/basic_pipeline.py -v 0 -c box ``` -5. 程序会运行，并自动打开可视化窗口和打印资料，在窗口上按键盘”q”键可以停止运行，按”r”键可以重设数据 +5. 程序会运行，并自动打开可视化窗口和打印资料。在窗口上按键盘 `q` 键可以停止运行，按 `r` 键可以重设数据。 -### Ubuntu: -1. 检查是否有其他正在连接的USB摄像头设备：打开终端, 运行命令`v4l2 –list-devices`，如果没有的话不会有输出。 -2. 把传感器的USB头与电脑连接，再次运行步骤一的命令，会新增输出 -3. 根据新增的输出，例如”/dev/video0”，确定传感器的ID为“0”, 如果是”/dev/video3”的话ID为“3” ，如此类推。 -4. 在确定了传感器的ID（下方例子假设为“0”）后，打开终端, 激活相应的python环境，切换到工作路径，然后在命令行中输入： - ``` bash - python examples/basic_pipeline.py -v 0 +#### Ubuntu +1. 检查是否有其他正在连接的 USB 摄像头设备：打开终端，运行命令 `v4l2 --list-devices`，如果没有的话不会有输出。 +2. 把传感器的 USB 头与电脑连接，再次运行步骤一的命令，会新增输出。 +3. 根据新增的输出，例如 `/dev/video0`，确定传感器的 ID 为 `0`；如果是 `/dev/video3`，ID 为 `3`，如此类推。 +4. 在确定了传感器的 ID（下方例子假设为 `0`）后，打开终端，激活相应的 Python 环境，切换到工作路径，然后在命令行中输入： + ```bash + python examples/basic_pipeline.py -v 0 -c box ``` -5. 程序会运行，并自动打开可视化窗口和打印资料，在窗口上按键盘”q”键可以停止运行，按”r”键可以重设数据 +5. 程序会运行，并自动打开可视化窗口和打印资料。在窗口上按键盘 `q` 键可以停止运行，按 `r` 键可以重设数据。 -## 开发套件使用方法 +#### ORY-FINGER 示例 + +如果您使用的是 ORY-FINGER（配置名 `finger`），可运行： + +```bash +python examples/finger_pipeline.py --camera_id 0 --config finger +``` + +如果输入来自视频文件，也可以改用： + +```bash +python examples/finger_pipeline.py --video ./data/sample.mp4 --config finger +``` + +该示例会： + +- 读取相机或视频源 +- 将图像展开到平面 +- 计算光流、法向/切向力和接触区域 +- 在 OpenCV 窗口中显示展开后的光流箭头图 + +常用参数： + +- `--verbose`：输出更详细的初始化日志 + +例如： + +```bash +python examples/finger_pipeline.py --camera_id 0 --config finger +``` + +#### ORY-FINGER 3D 可视化界面 + +如果要使用 GUI，先安装 GUI 依赖： +```bash +pip install "orisys[gui]" +``` +然后运行： +```bash +python examples/finger_pipeline_3dview.py --video 0 --config finger +``` + +该界面会提供： + +- 手指平面化光流箭头显示 +- 力曲线 +- 3D 视图 + +### 开发套件使用方法 参考开发套件附带的开发文档。 \ No newline at end of file diff --git a/examples/README.md b/examples/README.md index 4ee9a3d..d4d974a 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,43 +1,62 @@ # Examples -Top-level demo scripts under `examples/`. Run them from the repository root. +Runnable demos live at the `examples/` root. Shared helpers stay in +`pipeline/`, `viz/`, and `gui/`. Run commands from the repository root. -## Top-Level Files +## Product Entry Points -- `README.md` — this overview of the root-level example entry points -- `app_paths.py` — path helpers used by the packaged / frozen dashboard entry point -- `basic_pipeline.py` — basic single-sensor OpenCV demo for the compound-eye pipeline -- `plot_data.py` — single-sensor demo with OpenCV views plus PyQtGraph force charts -- `two_sensor_pipeline.py` — two-sensor multiprocessing demo for parallel acquisition / display -- `finger_pipeline.py` — minimal finger UV-unroll pipeline with optional profiling -- `finger_pipeline_3dview.py` — main finger dashboard entry point with the unified Qt UI +| Product | Entry | Default config | +|---------|-------|----------------| +| ORY-BOX | `basic_pipeline.py` | `box` | +| ORY-MINI | `mini_pipeline.py` | `mini` | +| ORY-FINGER (2D) | `finger_pipeline.py` | `finger` | +| ORY-FINGER (3D Qt) | `finger_pipeline_3dview.py` | `finger` | + +`finger` is the public ORY-FINGER config stem. `dd02` remains a compatibility alias for `dd02-ov`. ## Typical Usage -For the Qt dashboard: +ORY-BOX: + +```bash +python examples/basic_pipeline.py -v 0 -c box +``` + +ORY-MINI: + +```bash +python examples/mini_pipeline.py -v 0 -c mini +``` + +ORY-FINGER 2D: + +```bash +python examples/finger_pipeline.py --camera_id 0 --config finger +``` + +ORY-FINGER video file: + +```bash +python examples/finger_pipeline.py --video ./data/sample.mp4 --config finger +``` + +ORY-FINGER 3D Qt dashboard: ```bash pip install -e ".[gui]" -python examples/finger_pipeline_3dview.py -v 0 -c ./config/dd02-ov.json +python examples/finger_pipeline_3dview.py --video 0 --config finger ``` -For the minimal finger pipeline: +## Other Root Examples -```bash -python examples/finger_pipeline.py --camera_id 0 --config ./config/dd02-ov.json -``` - -For the basic compound-eye pipeline: - -```bash -python examples/basic_pipeline.py -v 0 -c ./config/ddjx01.json -``` +- `plot_data.py` — single-sensor OpenCV views plus PyQtGraph force charts +- `two_sensor_pipeline.py` — two-sensor multiprocessing demo ## Requirements - SDK installed from the repository root, for example `pip install -e .` - camera index or test video path -- matching config files under `./config/` +- matching config names under `./config/` (or packaged `orisys/configs/`) Qt / PyQtGraph examples additionally require: @@ -47,6 +66,10 @@ pip install -e ".[gui]" ## Notes -- `finger_pipeline_3dview.py` is the main interactive demo. It uses `app_paths.py` so the same entry point works in editable and packaged runs. -- Dashboard controls: click **启动** to start acquisition; `Q` quits, `R` resets, `S` starts recording, `E` stops recording, and `Esc` toggles fullscreen. -- `finger_pipeline.py` is the leanest finger example if you want to validate UV unroll + force/contact processing without the full dashboard. +- `finger_pipeline_3dview.py` is the main interactive finger demo. + It uses `gui/app_paths.py` so the same entry works in editable and packaged runs. +- Dashboard controls: click **启动** to start acquisition; use **Record CSV / Stop CSV** + and the CSV save-directory browser to log contact/force data; `Q` quits, `R` resets, + `S` starts video recording, `E` stops video recording, and `Esc` toggles fullscreen. +- `finger_pipeline.py` is the leanest finger example if you want to + validate UV unroll + force/contact processing without the full dashboard. diff --git a/examples/basic_pipeline.py b/examples/basic_pipeline.py index 7404637..03ea21c 100644 --- a/examples/basic_pipeline.py +++ b/examples/basic_pipeline.py @@ -1,5 +1,5 @@ """ -示例 1：单传感器基础流程 +ORY-BOX：单传感器基础流程 本示例演示 Orisys SDK 的标准处理流程： 1. 初始化传感器（摄像头或视频文件） @@ -7,7 +7,7 @@ 3. 按需计算接触区域 4. 读取并可视化结果 -SDK version: 0.3.1 +SDK version: 0.5.0 """ import os os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" @@ -19,7 +19,7 @@ import argparse def main(): parser = argparse.ArgumentParser() parser.add_argument("--video","-v", type=str, default="0", help="视频源：摄像头编号或视频文件路径") - parser.add_argument("--config","-c", type=str, default="./config/ddjx01.json", help="配置文件名称或路径") + parser.add_argument("--config","-c", type=str, default="box", help="配置文件名称或路径") parser.add_argument("--verbose","-verbose", type=bool, default=True, help="是否输出详细日志") args = parser.parse_args() @@ -28,12 +28,9 @@ def main(): # cal_path 为标定文件路径；每台设备建议使用独立的标定文件 # 输入既可以是摄像头编号，也可以是视频文件路径 - try: - input = int(args.video) - is_camera = True - sensor = orisys.Sensor(input, config_name=args.config, verbose=args.verbose) - except: - is_camera = False + if args.video.isdigit(): + sensor = orisys.Sensor(int(args.video), config_name=args.config, verbose=args.verbose) + else: sensor = orisys.Sensor(args.video, config_name=args.config, verbose=args.verbose) print("\n按键说明：'q' 退出程序，'r' 重置追踪器。") @@ -43,14 +40,29 @@ def main(): # 步骤 2.1：获取并拼接图像（必需） sensor.get_img() # 步骤 2.2：计算形变场（必需）；返回值表示当前帧是否检测到接触/运动 - is_contact_now = sensor.compute_deformation(check_motion=True, threshold=0) + is_contact_now = sensor.compute_deformation() # 步骤 2.3：仅在当前帧检测到接触/运动时计算接触区域 # if is_contact_now: sensor.compute_contact() - # 步骤 2.4：读取结果 - fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = sensor.read_info( + # 步骤 2.4：读取结果（FNORMAL/FSHEAR* 在有 sensors/box/force_calibration.json 时为标定后牛顿力； + # *_RAW 始终为原始积分量） + ( + fps, + flow, + vnormal, + img, + contour, + centroid, + depth_map, + fnormal, + fshearx, + fsheary, + fnormal_raw, + fshearx_raw, + fsheary_raw, + ) = sensor.read_info( sensor.info.FPS, sensor.info.VRAW, sensor.info.VNORMAL, @@ -60,7 +72,10 @@ def main(): sensor.info.DEPTH, sensor.info.FNORMAL, sensor.info.FSHEARX, - sensor.info.FSHEARY + sensor.info.FSHEARY, + sensor.info.FNORMAL_RAW, + sensor.info.FSHEARX_RAW, + sensor.info.FSHEARY_RAW, ) # 步骤 3：可视化显示 @@ -72,11 +87,11 @@ def main(): grid_spacing=20, arrow_scale=1.0 ) - cv2.imshow("形变矢量场", arrows) + cv2.imshow("arrow", arrows) # 步骤 3.2：显示接触区域和质心 image_with_foe = orisys.util.draw_contact(img, contour, centroid) - cv2.imshow("接触区域与质心", image_with_foe) + cv2.imshow("contact", image_with_foe) # 步骤 3.3：显示深度图 # 将深度值归一化到 0-255 范围 @@ -87,12 +102,15 @@ def main(): div_normalized = np.zeros_like(div_abs, dtype=np.uint8) # 应用伪彩色映射 - cv2.imshow("深度图", cv2.applyColorMap(div_normalized, cv2.COLORMAP_JET)) + cv2.imshow("depth", cv2.applyColorMap(div_normalized, cv2.COLORMAP_JET)) - # 步骤 3.4：打印关键结果 + # 步骤 3.4：打印关键结果（标定力 + 原始力） print( - f"FPS={fps:.2f}, 法向力={fnormal:.4f}, 切向力X={fshearx:.4f}, " - f"切向力Y={fsheary:.4f}, 深度图尺寸={depth_map.shape}, 原始分辨率={sensor.img_size_raw}" + f"FPS={fps:.2f}, " + f"法向力={fnormal:.4f} N (raw={fnormal_raw:.4f}), " + f"切向力X={fshearx:.4f} N (raw={fshearx_raw:.4f}), " + f"切向力Y={fsheary:.4f} N (raw={fsheary_raw:.4f}), " + f"深度图尺寸={depth_map.shape}, 原始分辨率={sensor.img_size_raw}" ) # 步骤 3.5：键盘控制 diff --git a/examples/finger_pipeline.py b/examples/finger_pipeline.py index f359976..fc92862 100644 --- a/examples/finger_pipeline.py +++ b/examples/finger_pipeline.py @@ -1,18 +1,19 @@ #!/usr/bin/env python3 """ -Minimum UV-unroll finger tactile pipeline. +ORY-FINGER：Minimum UV-unroll finger tactile pipeline. Offline: load Blender export -> build_or_load_domain -> unroll_map_uv.npz -Runtime: camera frame -> UVUnrollFrameProcessor -> TactileProcessor -> forces/contact +Runtime: camera frame -> FrameUnwarper -> TactileProcessor -> forces/contact -For the full demo (UV debug panels, dashboard, benchmarks) see: - tools/3d_patch_generation_blender/unroll_realtime.py +Recommended entry: + python examples/finger_pipeline.py --camera_id 0 --config finger + +Current product config: ``finger`` (``dd02`` still resolves to ``dd02-ov``). """ import argparse import os import sys -import time os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0") @@ -20,38 +21,14 @@ import cv2 import orisys from orisys.finger import FingerRuntime -from pipeline.profiling import StageProfiler, run_deformation_timed - -_FINGER_PIPELINE_STAGES = ( - "capture", - "uv_unroll", - "flow", - "hhd", - "contact", - "read_info", - "display", - "total", -) - def main(): parser = argparse.ArgumentParser(description="Minimum UV-unroll finger tactile pipeline") parser.add_argument("--camera_id", type=int, default=0) - parser.add_argument("--video", help="Video file path (overrides camera)") - parser.add_argument("--config", "-c", default="./config/dd02-ov.json", help="Orisys SDK config") + parser.add_argument("--video","-v", help="Video file path (overrides camera)") + parser.add_argument("--config", "-c", default="finger", help="Orisys SDK config name or path") parser.add_argument("--show", action="store_true", default=True, help="Show unrolled image window") parser.add_argument("--verbose", action="store_true", help="SDK verbose logs") - parser.add_argument( - "--profile", - action="store_true", - help="Print per-stage timing (capture, UV, flow, HHD, contact, read_info, display)", - ) - parser.add_argument( - "--profile-every", - type=int, - default=30, - help="When --profile is set, print rolling averages every N frames (default: 30)", - ) args = parser.parse_args() try: @@ -77,48 +54,17 @@ def main(): runtime.warmup(color=False) tactile = runtime.tactile - profiler = ( - StageProfiler(report_every=args.profile_every, stage_order=_FINGER_PIPELINE_STAGES) - if args.profile - else None - ) print("Press 'q' to quit, 'r' to reset tracker.") - if args.profile: - print(f"Profiling enabled (report every {args.profile_every} frames).") try: while True: - frame_t0 = time.perf_counter() + ok, unrolled = runtime.grab_and_process(color=False) + if not ok: + break - if profiler is not None: - t0 = time.perf_counter() - ok, frame = runtime.read_source() - profiler.record("capture", (time.perf_counter() - t0) * 1000.0) - if not ok: - break - t0 = time.perf_counter() - unrolled = runtime.process_frame(frame, color=False) - profiler.record("uv_unroll", (time.perf_counter() - t0) * 1000.0) - else: - ok, unrolled = runtime.grab_and_process(color=False) - if not ok: - break - - if profiler is not None: - flow_ms, hhd_ms = run_deformation_timed( - tactile, check_motion=True, threshold=0 - ) - profiler.record("flow", flow_ms) - profiler.record("hhd", hhd_ms) - else: - tactile.compute_deformation(check_motion=True, threshold=0) - - t0 = time.perf_counter() + tactile.compute_deformation() tactile.compute_contact() - if profiler is not None: - profiler.record("contact", (time.perf_counter() - t0) * 1000.0) - t0 = time.perf_counter() fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = tactile.read_info( tactile.info.FPS, tactile.info.VRAW, @@ -131,8 +77,6 @@ def main(): tactile.info.FSHEARX, tactile.info.FSHEARY, ) - if profiler is not None: - profiler.record("read_info", (time.perf_counter() - t0) * 1000.0) status = ( f"FPS={fps:.2f} normal={fnormal:.2f} shear=({fshearx:.2f}, {fsheary:.2f}) " @@ -140,26 +84,16 @@ def main(): ) if args.show: - t0 = time.perf_counter() arrows = orisys.util.draw_arrows( unrolled, flow, threshold=0.4, grid_spacing=10, arrow_scale=1.0, + valid_mask=tactile.valid_mask, ) cv2.imshow("UV Unrolled", arrows) - if profiler is not None: - profiler.record("display", (time.perf_counter() - t0) * 1000.0) - if profiler is not None: - total_ms = (time.perf_counter() - frame_t0) * 1000.0 - profiler.record("total", total_ms) - status += ( - f" | wall={total_ms:.1f}ms flow={profiler.last.get('flow', 0):.1f}ms" - f" hhd={profiler.last.get('hhd', 0):.1f}ms" - ) - profiler.tick() print(status, end="\r") key = cv2.waitKey(1) & 0xFF @@ -170,8 +104,6 @@ def main(): tactile.reset() print("\nTracker reset.") finally: - if profiler is not None and profiler.frame_count: - profiler.print_report(prefix="[profile final]") runtime.close() cv2.destroyAllWindows() diff --git a/examples/finger_pipeline_3dview.py b/examples/finger_pipeline_3dview.py index 0567d93..8d9561d 100644 --- a/examples/finger_pipeline_3dview.py +++ b/examples/finger_pipeline_3dview.py @@ -1,24 +1,39 @@ #!/usr/bin/env python3 """ -UV-unroll finger tactile pipeline with unified Qt dashboard. +ORY-FINGER：UV-unroll finger tactile pipeline with unified Qt dashboard. Where to customize the GUI: - Layout (sizes, proportions) -> examples/gui/layout.py <-- edit here - Colors / QSS -> examples/gui/styles.py - Widget rendering logic -> examples/gui/widgets/ - Window behavior -> examples/gui/dashboard.py + Layout (sizes, proportions) -> examples/gui/viewer3d/layout.py + Colors / QSS -> examples/gui/viewer3d/styles.py + Widget rendering logic -> examples/gui/viewer3d/widgets/ + Window behavior -> examples/gui/viewer3d/dashboard.py + +Recommended entry: + python examples/finger_pipeline_3dview.py --video 0 --config finger """ import argparse import os import sys +from pathlib import Path os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0") -from app_paths import configure_frozen_runtime, default_config_path, ensure_examples_import_path +# Make shared packages importable when launched as a file path. +_EXAMPLES_DIR = Path(__file__).resolve().parent +if str(_EXAMPLES_DIR) not in sys.path: + sys.path.insert(0, str(_EXAMPLES_DIR)) + +from gui.app_paths import ( + configure_frozen_runtime, + default_config_path, + ensure_examples_import_path, + require_gui_extras, +) configure_frozen_runtime() ensure_examples_import_path() +require_gui_extras() from gui import run_dashboard from pipeline import PipelineConfig diff --git a/examples/gui/__init__.py b/examples/gui/__init__.py index 102a142..562677c 100644 --- a/examples/gui/__init__.py +++ b/examples/gui/__init__.py @@ -1,20 +1,29 @@ -"""Public GUI entry points for finger examples.""" +"""Public GUI entry points for Orisys example apps. -from .dashboard import FingerDashboardWindow, run_dashboard -from .layout import ( - LOG_CONFIG, - LOG_PANEL, +Two apps live under ``examples/gui``: + +- Finger 3D dashboard: ``from gui import run_dashboard`` (also ``gui.viewer3d``) +- 2D sensor monitor: ``from gui.viewer2d import run_viewer`` + +Acquisition adapters (no widgets) live in ``gui.sensors``. +Shared bootstrap: ``gui.app``, ``gui.app_paths``, ``gui.log_config``, ``gui.widgets``. +""" + +from .log_config import LOG_CONFIG, LogConfig +from .viewer3d import ( + FingerDashboardWindow, LAYOUT, + LOG_PANEL, MESH_CONTACT_ARROW, MESH_LIGHTING, PANEL_SETUP, TOP_ROW, DashboardLayout, - LogConfig, MeshLighting, PanelRow, PanelSetup, PanelSlot, + run_dashboard, top_row_splitter_sizes, top_row_slot_index, ) diff --git a/examples/app_paths.py b/examples/gui/app_paths.py similarity index 63% rename from examples/app_paths.py rename to examples/gui/app_paths.py index 3136362..667bba7 100644 --- a/examples/app_paths.py +++ b/examples/gui/app_paths.py @@ -1,4 +1,4 @@ -"""Resolve bundled resource paths for dev runs and Nuitka standalone builds.""" +"""Resolve bundled resource paths for dev runs and standalone builds.""" from __future__ import annotations @@ -6,7 +6,8 @@ import os import sys from pathlib import Path -_EXAMPLES_DIR = Path(__file__).resolve().parent +_GUI_DIR = Path(__file__).resolve().parent +_EXAMPLES_DIR = _GUI_DIR.parent _REPO_ROOT = _EXAMPLES_DIR.parent @@ -26,7 +27,17 @@ def resource_path(*parts: str) -> Path: def default_config_path() -> str: - return str(resource_path("config", "dd02-ov.json")) + """Default ORY-FINGER config name (packaged stem ``finger``).""" + return "finger" + + +def require_gui_extras() -> None: + try: + import open3d # noqa: F401 + from PyQt5 import QtWidgets # noqa: F401 + import pyqtgraph # noqa: F401 + except ImportError as exc: + raise SystemExit('GUI example requires: pip install "orisys[gui]"') from exc def configure_frozen_runtime() -> None: diff --git a/examples/gui/log_config.py b/examples/gui/log_config.py new file mode 100644 index 0000000..a0c0a40 --- /dev/null +++ b/examples/gui/log_config.py @@ -0,0 +1,52 @@ +"""Shared log-panel configuration for Qt viewers (2D and 3D).""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class LogConfig: + """Log panel — edit LOG_CONFIG or DashboardLayout.log_config.""" + + status_enabled: bool = True + max_lines: int = 2000 + font_family: str = "Cascadia Mono" + font_pt: int = 12 + show_flow: bool = True + show_mask: bool = True + + def format_status( + self, + *, + fps: float, + fnormal: float, + fshearx: float, + fsheary: float, + flow_rms: float = 0.0, + mask_coverage: float = 0.0, + wall_fps: float | None = None, + contact_xyz_mm: tuple[float, float, float] | None = None, + ) -> str: + shear_mag = float((fshearx**2 + fsheary**2) ** 0.5) + if wall_fps is not None: + parts = [f"FPS={wall_fps:.1f}", f"sdk={fps:.1f}"] + else: + parts = [f"FPS={fps:.1f}"] + parts.extend( + [ + f"Fn={fnormal:.1f}", + f"|(Fx,Fy)|={shear_mag:.1f}", + f"Fx, Fy=({fshearx:.1f}, {fsheary:.1f})", + ] + ) + force_line = " ".join(parts) + if contact_xyz_mm is None: + contact_line = "contact=---" + else: + cx, cy, cz = contact_xyz_mm + contact_line = f"contact=({cx:.1f}, {cy:.1f}, {cz:.1f}) mm" + return f"{force_line}\n{contact_line}" + + +LOG_CONFIG = LogConfig() diff --git a/examples/gui/preference.json b/examples/gui/preference.json new file mode 100644 index 0000000..331fe3a --- /dev/null +++ b/examples/gui/preference.json @@ -0,0 +1,11 @@ +{ + "video_source": "D:/repo/orisys-sdk/data/dd02-0703/1.mp4", + "config_path": "finger", + "finger_assets_dir": "sensors/finger/assets/30fps-0703", + "csv_save_dir": "./outputs/csv", + "show_log": true, + "show_flow_arrows": false, + "arrow_background": "unrolled", + "free_rotate": true, + "sensor_bundle_dir": "sensors/finger" +} diff --git a/examples/gui/sensors/__init__.py b/examples/gui/sensors/__init__.py new file mode 100644 index 0000000..e275bd6 --- /dev/null +++ b/examples/gui/sensors/__init__.py @@ -0,0 +1,23 @@ +"""Sensor acquisition adapters for Qt / multi-sensor viewers (no Qt widgets).""" + +from .adapter import ( + Dd06RoiAdapter, + FrameSnapshot, + OrisysSensorAdapter, + SensorAdapter, + calibrate_normal_force, + create_sensor_adapter, + _snapshot_from_sensor, +) +from . import roi_geometry + +__all__ = [ + "Dd06RoiAdapter", + "FrameSnapshot", + "OrisysSensorAdapter", + "SensorAdapter", + "calibrate_normal_force", + "create_sensor_adapter", + "roi_geometry", + "_snapshot_from_sensor", +] diff --git a/examples/gui/sensors/adapter.py b/examples/gui/sensors/adapter.py new file mode 100644 index 0000000..8386494 --- /dev/null +++ b/examples/gui/sensors/adapter.py @@ -0,0 +1,330 @@ +"""Sensor adapters that normalize frames for Qt / multi-sensor viewers.""" + +from __future__ import annotations + +import time +from abc import ABC, abstractmethod +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +import cv2 +import numpy as np +import orisys + +from .roi_geometry import ( + build_roi_warp, + load_json_config, + load_roi_polygon, + open_video_source, + rectified_output_size, + warp_roi_to_rectangle, +) + + +NORMAL_FORCE_CAL_SLOPE = 3.92685554668795e-05 +NORMAL_FORCE_CAL_OFFSET = 0.0217596967278535 + + +@dataclass +class FrameSnapshot: + """Normalized per-frame payload consumed by the Qt viewer.""" + + image: Optional[np.ndarray] = None + flow: Optional[np.ndarray] = None + depth: Optional[np.ndarray] = None + contour: Optional[np.ndarray] = None + centroid: Any = None + # Raw SDK force channels used for curves (same units for FN/FX/FY). + normal_force: float = 0.0 + shear_x: float = 0.0 + shear_y: float = 0.0 + # Optional calibrated FN in Newtons for export/display overlays. + normal_force_calibrated: float = 0.0 + fps: float = 0.0 + timestamp: float = 0.0 + raw_image: Optional[np.ndarray] = None + processed_image: Optional[np.ndarray] = None + flow_overlay: Optional[np.ndarray] = None + record_frame: Optional[np.ndarray] = None + + +def calibrate_normal_force(raw_value: float) -> float: + """Convert raw normal-force reading to Newtons using y = mx + c.""" + return NORMAL_FORCE_CAL_SLOPE * float(raw_value) + NORMAL_FORCE_CAL_OFFSET + + +class SensorAdapter(ABC): + """Common interface between Qt and concrete sensor backends.""" + + @abstractmethod + def start(self) -> None: + raise NotImplementedError + + @abstractmethod + def read(self) -> Optional[FrameSnapshot]: + raise NotImplementedError + + @abstractmethod + def reset(self) -> None: + raise NotImplementedError + + @abstractmethod + def close(self) -> None: + raise NotImplementedError + + +def _bgr_from_gray_or_bgr(img: np.ndarray) -> np.ndarray: + if img is None: + return None + if img.ndim == 2: + return cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + if img.ndim == 3 and img.shape[2] == 1: + return cv2.cvtColor(img[:, :, 0], cv2.COLOR_GRAY2BGR) + return img + + +def _snapshot_from_sensor( + sensor, + *, + raw_image=None, + processed_image=None, +) -> Optional[FrameSnapshot]: + """Build a FrameSnapshot with HHD/forces refreshed every frame.""" + if sensor is None or sensor.img is None: + return None + + # Always decompose so FN/FX/FY update every frame (motion gating freezes forces). + sensor.compute_deformation(decompose=True, check_motion=False) + sensor.compute_contact() + + fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = ( + sensor.read_info( + sensor.info.FPS, + sensor.info.VRAW, + sensor.info.VNORMAL, + sensor.info.IMG, + sensor.info.CONTOUR, + sensor.info.CENTROID, + sensor.info.DEPTH, + sensor.info.FNORMAL, + sensor.info.FSHEARX, + sensor.info.FSHEARY, + ) + ) + + image_bgr = _bgr_from_gray_or_bgr(img) + processed = processed_image if processed_image is not None else image_bgr + record = None + if getattr(sensor, "frame", None) is not None: + record = _bgr_from_gray_or_bgr(sensor.frame) + elif processed is not None: + record = processed + + flow_overlay = None + if image_bgr is not None and flow is not None: + flow_overlay = orisys.util.draw_arrows( + image_bgr, + flow, + threshold=2, + grid_spacing=20, + arrow_scale=0.5, + below_threshold_color=(160, 160, 160), + ) + + fn_raw = float(fnormal) + return FrameSnapshot( + image=image_bgr, + flow=flow, + depth=depth_map, + contour=contour, + centroid=centroid, + normal_force=fn_raw, + shear_x=float(fshearx), + shear_y=float(fsheary), + normal_force_calibrated=calibrate_normal_force(fn_raw), + fps=float(fps), + timestamp=time.time(), + raw_image=raw_image, + processed_image=processed, + flow_overlay=flow_overlay, + record_frame=record, + ) + + +class OrisysSensorAdapter(SensorAdapter): + """Direct orisys.Sensor capture path (compound-eye / single-camera).""" + + def __init__( + self, + video, + config_name, + cal_path=None, + verbose=True, + *, + rotate_ccw90=False, + motion_threshold=1, + ): + self.video = video + self.config_name = config_name + self.cal_path = cal_path + self.verbose = verbose + self.rotate_ccw90 = bool(rotate_ccw90) + self.motion_threshold = motion_threshold + self.sensor = None + + def start(self) -> None: + src = self.video + try: + src = int(str(src).strip()) + except (TypeError, ValueError): + pass + self.sensor = orisys.Sensor( + src, + config_name=self.config_name, + cal_path=self.cal_path, + verbose=self.verbose, + ) + + def read(self) -> Optional[FrameSnapshot]: + if self.sensor is None: + return None + img = self.sensor.get_img() + if img is None: + return None + + if self.rotate_ccw90: + self.sensor.img = cv2.rotate(self.sensor.img, cv2.ROTATE_90_COUNTERCLOCKWISE) + if self.sensor.frame is not None: + self.sensor.frame = cv2.rotate( + self.sensor.frame, cv2.ROTATE_90_COUNTERCLOCKWISE + ) + + return _snapshot_from_sensor( + self.sensor, + raw_image=_bgr_from_gray_or_bgr(self.sensor.frame) + if self.sensor.frame is not None + else None, + processed_image=_bgr_from_gray_or_bgr(self.sensor.img), + ) + + def reset(self) -> None: + if self.sensor is not None: + self.sensor.reset() + + def close(self) -> None: + if self.sensor is not None: + self.sensor.disconnect() + self.sensor = None + + +class Dd06RoiAdapter(SensorAdapter): + """DD06 path: external capture + polygon ROI rectify + already_processed Sensor.""" + + def __init__(self, video, config_name, verbose=True, *, motion_threshold=0): + self.video = video + self.config_name = config_name + self.verbose = verbose + self.motion_threshold = motion_threshold + self.sensor = None + self.cap = None + self.roi_points = None + self.roi_image_size = None + self.out_size = None + self.warp_matrix = None + self.warp_src = None + self.last_frame_size = None + self._source = None + + def start(self) -> None: + config = load_json_config(self.config_name) + self.roi_points, self.roi_image_size = load_roi_polygon(config) + self.out_size = rectified_output_size(config, self.roi_points) + self.sensor = orisys.Sensor( + "external", + config_name=self.config_name, + verbose=self.verbose, + ) + self.cap, self._source = open_video_source(self.video) + self.warp_matrix = None + self.last_frame_size = None + if self.verbose: + print( + f"DD06 adapter: ROI points={len(self.roi_points)} " + f"source_size={self.roi_image_size} output={self.out_size} " + f"video={self._source}" + ) + + def read(self) -> Optional[FrameSnapshot]: + if self.sensor is None or self.cap is None: + return None + + ok, frame = self.cap.read() + if not ok or frame is None: + return None + + frame_h, frame_w = frame.shape[:2] + frame_size = (frame_w, frame_h) + if self.warp_matrix is None or self.last_frame_size != frame_size: + self.warp_matrix, self.warp_src, _ = build_roi_warp( + self.roi_points, + self.roi_image_size, + frame_size, + self.out_size, + ) + self.last_frame_size = frame_size + if self.verbose: + print(f"Built ROI warp for frame {frame_w}x{frame_h}") + + rectified = warp_roi_to_rectangle(frame, self.warp_matrix, self.out_size) + self.sensor.get_img(frame=rectified, already_processed=True) + return _snapshot_from_sensor( + self.sensor, + raw_image=frame, + processed_image=rectified, + ) + + def reset(self) -> None: + if self.sensor is not None: + self.sensor.reset() + + def close(self) -> None: + if self.cap is not None: + self.cap.release() + self.cap = None + if self.sensor is not None: + self.sensor.disconnect() + self.sensor = None + + +def create_sensor_adapter( + sensor_type: str, + *, + video, + config_name, + cal_path=None, + verbose=True, +) -> SensorAdapter: + """Factory for Qt / demo viewers.""" + kind = str(sensor_type or "orisys").strip().lower() + if kind in ("orisys", "direct", "default", "compound_eye", "single_camera"): + # Preserve historical Wuxi viewer rotation for compound-eye demos. + rotate = kind in ("orisys", "direct", "default", "compound_eye") + return OrisysSensorAdapter( + video, + config_name, + cal_path=cal_path, + verbose=verbose, + rotate_ccw90=rotate, + motion_threshold=1, + ) + if kind in ("dd06", "dd06-ov", "roi", "polygon_roi"): + return Dd06RoiAdapter( + video, + config_name, + verbose=verbose, + motion_threshold=0, + ) + raise ValueError( + f"Unknown sensor_type={sensor_type!r}; expected 'orisys' or 'dd06'" + ) diff --git a/examples/gui/sensors/roi_geometry.py b/examples/gui/sensors/roi_geometry.py new file mode 100644 index 0000000..2cff48c --- /dev/null +++ b/examples/gui/sensors/roi_geometry.py @@ -0,0 +1,140 @@ +"""Polygon ROI geometry helpers for rectifying tactile sensor frames.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import cv2 +import numpy as np + + +def load_json_config(path): + path = Path(path) + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def load_roi_polygon(config): + """Return (points Nx2 float32, image_size [w,h]) from nested or legacy ROI fields.""" + roi = config.get("roi") if isinstance(config.get("roi"), dict) else config + if not isinstance(roi, dict): + raise ValueError("Config has no usable ROI section") + + image_size = roi.get("image_size") or config.get("camera", {}).get("resolution_raw") + if image_size is None: + raise ValueError("ROI image_size is missing") + + points = roi.get("points") or roi.get("polygon") or roi.get("roi_polygon") + if points is None and roi.get("xyxy") is not None: + x1, y1, x2, y2 = [int(v) for v in roi["xyxy"]] + points = [[x1, y1], [x2, y1], [x2, y2], [x1, y2]] + if not points: + raise ValueError("ROI points are missing") + + pts = np.asarray(points, dtype=np.float32).reshape(-1, 2) + if pts.shape[0] < 4: + raise ValueError(f"ROI needs at least 4 points, got {pts.shape[0]}") + return pts, [int(image_size[0]), int(image_size[1])] + + +def scale_points(points, source_size, target_size): + source_w, source_h = float(source_size[0]), float(source_size[1]) + target_w, target_h = float(target_size[0]), float(target_size[1]) + pts = np.asarray(points, dtype=np.float32).reshape(-1, 2).copy() + if source_w > 0 and source_h > 0 and (source_w, source_h) != (target_w, target_h): + pts[:, 0] *= target_w / source_w + pts[:, 1] *= target_h / source_h + return pts + + +def order_quad_points(points): + """Order 4 points as TL, TR, BR, BL for perspective transforms.""" + pts = np.asarray(points, dtype=np.float32).reshape(-1, 2) + if pts.shape[0] != 4: + hull = cv2.convexHull(pts.astype(np.float32)) + rect = cv2.minAreaRect(hull) + pts = cv2.boxPoints(rect).astype(np.float32) + + s = pts.sum(axis=1) + diff = np.diff(pts, axis=1).reshape(-1) + ordered = np.zeros((4, 2), dtype=np.float32) + ordered[0] = pts[np.argmin(s)] # top-left + ordered[2] = pts[np.argmax(s)] # bottom-right + ordered[1] = pts[np.argmin(diff)] # top-right + ordered[3] = pts[np.argmax(diff)] # bottom-left + return ordered + + +def rectified_output_size(config, points=None): + """Prefer camera.resolution_stitch; fall back to ROI bounding box size.""" + stitch = config.get("camera", {}).get("resolution_stitch") + if stitch is not None and len(stitch) >= 2: + return int(stitch[0]), int(stitch[1]) + if points is not None: + pts = np.asarray(points, dtype=np.float32).reshape(-1, 2) + width = max( + int(np.linalg.norm(pts[1] - pts[0])), + int(np.linalg.norm(pts[2] - pts[3])), + 1, + ) + height = max( + int(np.linalg.norm(pts[3] - pts[0])), + int(np.linalg.norm(pts[2] - pts[1])), + 1, + ) + return width, height + raise ValueError("Cannot determine rectified output size") + + +def build_roi_warp(points, source_image_size, frame_size, output_size): + """ + Build a perspective transform that maps the ROI polygon in frame coords + onto a rectangle of ``output_size`` (width, height). + """ + frame_w, frame_h = int(frame_size[0]), int(frame_size[1]) + scaled = scale_points(points, source_image_size, (frame_w, frame_h)) + src = order_quad_points(scaled) + out_w, out_h = int(output_size[0]), int(output_size[1]) + dst = np.array( + [ + [0.0, 0.0], + [out_w - 1.0, 0.0], + [out_w - 1.0, out_h - 1.0], + [0.0, out_h - 1.0], + ], + dtype=np.float32, + ) + matrix = cv2.getPerspectiveTransform(src, dst) + return matrix, src, (out_w, out_h) + + +def warp_roi_to_rectangle(frame, matrix, output_size): + out_w, out_h = int(output_size[0]), int(output_size[1]) + return cv2.warpPerspective( + frame, + matrix, + (out_w, out_h), + flags=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_REPLICATE, + ) + + +def open_video_source(video): + """Open a camera index or video file. Returns (cap, resolved_source).""" + text = str(video).strip() + if text.isdigit(): + source = int(text) + cap = cv2.VideoCapture(source, cv2.CAP_DSHOW) + if not cap.isOpened(): + cap.release() + cap = cv2.VideoCapture(source) + else: + path = Path(text) + if not path.exists(): + raise FileNotFoundError(f"Video not found: {path}") + source = str(path) + cap = cv2.VideoCapture(source) + if not cap.isOpened(): + raise RuntimeError(f"Failed to open video source: {video}") + return cap, source diff --git a/examples/gui/viewer2d/__init__.py b/examples/gui/viewer2d/__init__.py new file mode 100644 index 0000000..a55b3cf --- /dev/null +++ b/examples/gui/viewer2d/__init__.py @@ -0,0 +1,30 @@ +"""2D Qt sensor monitor (image + force curves; no finger 3D mesh).""" + +from .backend import ( + PURE_COLOR_BG_BGR, + ArrowBackground, + ForceHistory, + ForcePlotState, + QtViewerBackend, + ViewerFrame, + build_display_image, + image_shape_meta, + panel_width_for_aspect, + select_display_image, +) +from .window import QtViewerWindow, run_viewer + +__all__ = [ + "PURE_COLOR_BG_BGR", + "ArrowBackground", + "ForceHistory", + "ForcePlotState", + "QtViewerBackend", + "QtViewerWindow", + "ViewerFrame", + "build_display_image", + "image_shape_meta", + "panel_width_for_aspect", + "run_viewer", + "select_display_image", +] diff --git a/examples/gui/viewer2d/backend.py b/examples/gui/viewer2d/backend.py new file mode 100644 index 0000000..06d92c1 --- /dev/null +++ b/examples/gui/viewer2d/backend.py @@ -0,0 +1,400 @@ +"""Qt-facing display backend for sensor viewer snapshots. + +Separates acquisition (`gui.sensors.FrameSnapshot`) from Qt display state: +normalized images, force history / filtering, and aspect-ratio metadata. +Does not import camera, ROI, or `orisys.Sensor`. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal, Optional, Sequence, Tuple + +import cv2 +import numpy as np +import orisys +from scipy.signal import butter, filtfilt + +FORCE_SAMPLE_RATE_HZ = 30.0 +PLOT_BUFFER_POINTS = 120 +PLOT_WINDOW_SEC = PLOT_BUFFER_POINTS / FORCE_SAMPLE_RATE_HZ +Y_AXIS_MIN_SPAN = 20000.0 + +ArrowBackground = Literal["pure_color", "captured"] +# #0B1016 as BGR (matches demo_wuxi / promote_video theme) +PURE_COLOR_BG_BGR = (22, 16, 11) + +_ARROW_DRAW_KW = dict( + threshold=2, + grid_spacing=20, + arrow_scale=0.5, + below_threshold_color=(160, 160, 160), +) + + +@dataclass +class ViewerFrame: + """Normalized display payload for one UI tick.""" + + display_image: np.ndarray + image_height: int + image_width: int + aspect_ratio: float # width / height + depth: Optional[np.ndarray] + normal_force: float + shear_x: float + shear_y: float + normal_force_calibrated: float + fps: float + timestamp: float + record_frame: Optional[np.ndarray] + + +@dataclass +class ForcePlotState: + """Filtered force series ready for pyqtgraph curves.""" + + times: np.ndarray # shape (n,) + values: np.ndarray # shape (n, 3) FN, FX, FY + sample_count: int + x_range: Tuple[float, float] + y_ranges: Tuple[Tuple[float, float], Tuple[float, float], Tuple[float, float]] + + +def lowpass_filter( + data, + cutoff_hz: float, + sample_rate_hz: float = FORCE_SAMPLE_RATE_HZ, + order: int = 2, +) -> np.ndarray: + """Zero-phase Butterworth low-pass for uniformly sampled 1-D data.""" + y = np.asarray(data, dtype=np.float64) + if cutoff_hz == 0: + return y.copy() + if y.size == 0: + return y + + nyquist = 0.5 * sample_rate_hz + if cutoff_hz < 0 or cutoff_hz >= nyquist: + raise ValueError( + f"cutoff_hz must be in (0, {nyquist}) for sample_rate_hz={sample_rate_hz}" + ) + + b, a = butter(order, cutoff_hz / nyquist, btype="low") + padlen = 3 * (max(len(b), len(a)) - 1) + if y.size <= padlen: + return y.copy() + return filtfilt(b, a, y) + + +def ensure_bgr(img: Optional[np.ndarray]) -> Optional[np.ndarray]: + if img is None: + return None + if img.ndim == 2: + return cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + if img.ndim == 3 and img.shape[2] == 1: + return cv2.cvtColor(img[:, :, 0], cv2.COLOR_GRAY2BGR) + return img + + +def image_shape_meta(img: np.ndarray) -> Tuple[int, int, float]: + """Return (height, width, width/height) from a display image.""" + h, w = int(img.shape[0]), int(img.shape[1]) + aspect = float(w) / float(h) if h > 0 else 1.0 + return h, w, aspect + + +def _base_bgr(snapshot) -> Optional[np.ndarray]: + for attr in ("image", "processed_image", "raw_image"): + out = ensure_bgr(getattr(snapshot, attr, None)) + if out is not None: + return out + return None + + +def select_display_image(snapshot) -> np.ndarray: + """Pick flow overlay / processed / raw image for the left panel.""" + arrows = getattr(snapshot, "flow_overlay", None) + if arrows is not None: + out = ensure_bgr(arrows) + if out is not None: + return out + + for attr in ("processed_image", "image", "raw_image"): + base = getattr(snapshot, attr, None) + out = ensure_bgr(base) + if out is not None: + return out + + return np.zeros((400, 400, 3), dtype=np.uint8) + + +def build_display_image( + snapshot, + background: ArrowBackground = "pure_color", +) -> np.ndarray: + """Build left-panel image: arrows on pure color or on the captured frame.""" + flow = getattr(snapshot, "flow", None) + base = _base_bgr(snapshot) + + if flow is not None and base is not None: + if background == "pure_color": + h, w = base.shape[:2] + canvas = np.empty((h, w, 3), dtype=np.uint8) + canvas[:, :] = PURE_COLOR_BG_BGR + return orisys.util.draw_arrows(canvas, flow, **_ARROW_DRAW_KW) + + overlay = ensure_bgr(getattr(snapshot, "flow_overlay", None)) + if overlay is not None: + return overlay + return orisys.util.draw_arrows(base, flow, **_ARROW_DRAW_KW) + + return select_display_image(snapshot) + + +def bgr_to_rgb_bytes(img_bgr: np.ndarray) -> Tuple[np.ndarray, int, int]: + """Convert BGR uint8 image to contiguous RGB for QImage.""" + rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB) + if not rgb.flags["C_CONTIGUOUS"]: + rgb = np.ascontiguousarray(rgb) + h, w = rgb.shape[:2] + return rgb, h, w + + +def scale_bgr_to_fit( + img_bgr: np.ndarray, + max_w: int, + max_h: int, +) -> np.ndarray: + """Resize preserving aspect ratio to fit inside (max_w, max_h).""" + if max_w <= 0 or max_h <= 0: + return img_bgr + h, w = img_bgr.shape[:2] + if h <= 0 or w <= 0: + return img_bgr + scale = min(max_w / float(w), max_h / float(h)) + if abs(scale - 1.0) < 1e-6: + return img_bgr + nw = max(1, int(round(w * scale))) + nh = max(1, int(round(h * scale))) + return cv2.resize(img_bgr, (nw, nh), interpolation=cv2.INTER_AREA) + + +def panel_width_for_aspect( + available_height: int, + aspect_ratio: float, + *, + min_size: int = 160, +) -> int: + """Preferred left-panel width given available height and image aspect.""" + h = max(int(available_height), min_size) + w = int(round(h * float(aspect_ratio))) + return max(w, min_size) + + +class ForceHistory: + """Ring-style force history buffer with filtering and y-range smoothing.""" + + def __init__( + self, + npoints: int = PLOT_BUFFER_POINTS, + *, + sample_rate_hz: float = FORCE_SAMPLE_RATE_HZ, + lowpass_cutoff_hz: float = 10.0, + y_axis_min_span: float = Y_AXIS_MIN_SPAN, + y_range_alpha: float = 0.5, + ): + self.npoints = int(npoints) + self.sample_rate_hz = float(sample_rate_hz) + self.lowpass_cutoff_hz = float(lowpass_cutoff_hz) + self.y_axis_min_span = float(y_axis_min_span) + self.y_range_alpha = float(y_range_alpha) + self.window_sec = self.npoints / self.sample_rate_hz + self.reset() + + def reset(self) -> None: + self.data = np.full((self.npoints, 2, 3), np.nan, dtype=np.float64) + self.sample_count = 0 + self._y_range_smooth = [None, None, None] + + def push(self, t: float, fn: float, fx: float, fy: float) -> None: + values = (float(fn), float(fx), float(fy)) + if self.sample_count < self.npoints: + idx = self.sample_count + self.data[idx, 0, :] = t + self.data[idx, 1, :] = values + self.sample_count += 1 + return + + self.data[:-1, 0, :] = self.data[1:, 0, :] + self.data[:-1, 1, :] = self.data[1:, 1, :] + self.data[-1, 0, :] = t + self.data[-1, 1, :] = values + + def _smooth_y_range(self, channel: int, ymin: float, ymax: float) -> Tuple[float, float]: + if self._y_range_smooth[channel] is None: + smoothed = (ymin, ymax) + else: + prev_min, prev_max = self._y_range_smooth[channel] + alpha = self.y_range_alpha + ymin = alpha * ymin + (1.0 - alpha) * prev_min + ymax = alpha * ymax + (1.0 - alpha) * prev_max + smoothed = (ymin, ymax) + + span = smoothed[1] - smoothed[0] + if span < self.y_axis_min_span: + mid = 0.5 * (smoothed[0] + smoothed[1]) + half = self.y_axis_min_span / 2.0 + smoothed = (mid - half, mid + half) + + self._y_range_smooth[channel] = smoothed + pad = max((smoothed[1] - smoothed[0]) * 0.12, 0.01) + return smoothed[0] - pad, smoothed[1] + pad + + def plot_state(self) -> Optional[ForcePlotState]: + n = self.sample_count + if n == 0: + return None + + data_plot = self.data[:n].copy() + for i in range(3): + data_plot[:, 1, i] = lowpass_filter( + data_plot[:, 1, i], + self.lowpass_cutoff_hz, + sample_rate_hz=self.sample_rate_hz, + ) + + times = data_plot[:, 0, 0].copy() + values = data_plot[:, 1, :].copy() + y_ranges = tuple( + self._smooth_y_range(i, float(values[:, i].min()), float(values[:, i].max())) + for i in range(3) + ) + + if n < self.npoints: + x_range = (0.0, self.window_sec) + else: + x0 = float(times[0]) + x1 = float(times[-1]) + if x1 <= x0: + x1 = x0 + self.window_sec + x_range = (x0, x1) + + return ForcePlotState( + times=times, + values=values, + sample_count=n, + x_range=x_range, + y_ranges=y_ranges, # type: ignore[arg-type] + ) + + +class QtViewerBackend: + """Normalize FrameSnapshot into display images + force history.""" + + def __init__( + self, + *, + npoints: int = PLOT_BUFFER_POINTS, + lowpass_cutoff_hz: float = 10.0, + y_axis_min_span: float = Y_AXIS_MIN_SPAN, + arrow_background: ArrowBackground = "pure_color", + ): + self.force_history = ForceHistory( + npoints=npoints, + lowpass_cutoff_hz=lowpass_cutoff_hz, + y_axis_min_span=y_axis_min_span, + ) + self._start_time: Optional[float] = None + self.last_viewer_frame: Optional[ViewerFrame] = None + self.arrow_background: ArrowBackground = arrow_background + + def set_arrow_background(self, background: ArrowBackground) -> None: + self.arrow_background = background + + def rebuild_display(self, snapshot) -> ViewerFrame: + """Re-render display image without advancing force history.""" + display = build_display_image(snapshot, self.arrow_background) + h, w, aspect = image_shape_meta(display) + prev = self.last_viewer_frame + if prev is not None: + frame = ViewerFrame( + display_image=display, + image_height=h, + image_width=w, + aspect_ratio=aspect, + depth=prev.depth, + normal_force=prev.normal_force, + shear_x=prev.shear_x, + shear_y=prev.shear_y, + normal_force_calibrated=prev.normal_force_calibrated, + fps=prev.fps, + timestamp=prev.timestamp, + record_frame=prev.record_frame, + ) + else: + frame = ViewerFrame( + display_image=display, + image_height=h, + image_width=w, + aspect_ratio=aspect, + depth=getattr(snapshot, "depth", None), + normal_force=float(getattr(snapshot, "normal_force", 0.0)), + shear_x=float(getattr(snapshot, "shear_x", 0.0)), + shear_y=float(getattr(snapshot, "shear_y", 0.0)), + normal_force_calibrated=float( + getattr(snapshot, "normal_force_calibrated", 0.0) + ), + fps=float(getattr(snapshot, "fps", 0.0)), + timestamp=float(getattr(snapshot, "timestamp", 0.0) or 0.0), + record_frame=ensure_bgr(getattr(snapshot, "record_frame", None)), + ) + self.last_viewer_frame = frame + return frame + + def reset(self) -> None: + self.force_history.reset() + self._start_time = None + self.last_viewer_frame = None + + def ingest(self, snapshot, *, now: Optional[float] = None) -> ViewerFrame: + """Convert a FrameSnapshot into ViewerFrame and update force history.""" + display = build_display_image(snapshot, self.arrow_background) + h, w, aspect = image_shape_meta(display) + + fn = float(getattr(snapshot, "normal_force", 0.0)) + fx = float(getattr(snapshot, "shear_x", 0.0)) + fy = float(getattr(snapshot, "shear_y", 0.0)) + fn_cal = float(getattr(snapshot, "normal_force_calibrated", 0.0)) + fps = float(getattr(snapshot, "fps", 0.0)) + ts = float(getattr(snapshot, "timestamp", 0.0) or 0.0) + if now is None: + now = ts if ts > 0 else 0.0 + + if self._start_time is None: + self._start_time = now + t = now - self._start_time + self.force_history.push(t, fn, fx, fy) + + record = ensure_bgr(getattr(snapshot, "record_frame", None)) + depth = getattr(snapshot, "depth", None) + + frame = ViewerFrame( + display_image=display, + image_height=h, + image_width=w, + aspect_ratio=aspect, + depth=depth, + normal_force=fn, + shear_x=fx, + shear_y=fy, + normal_force_calibrated=fn_cal, + fps=fps, + timestamp=ts, + record_frame=record, + ) + self.last_viewer_frame = frame + return frame + + def plot_state(self) -> Optional[ForcePlotState]: + return self.force_history.plot_state() diff --git a/examples/gui/viewer2d/styles.py b/examples/gui/viewer2d/styles.py new file mode 100644 index 0000000..bc44ec4 --- /dev/null +++ b/examples/gui/viewer2d/styles.py @@ -0,0 +1,154 @@ +"""Styles for the 2D Qt sensor monitor.""" + +CURVE_THEME_COLOR = "#00E5FF" + +STYLE = """ +QMainWindow { + background-color: #000000; +} + +/* ---- 图像面板 ---- */ +QLabel#arrowsLabel { + border: 1px solid #262626; + border-radius: 4px; + background-color: #0b0b0b; +} + +/* ---- 顶栏按钮 & 输入 ---- */ +QPushButton { + background-color: #1a1a1a; + color: #999999; + border: 1px solid #333333; + border-radius: 4px; + padding: 6px 16px; + font-family: "SimHei"; + font-size: 13px; +} +QPushButton:hover { + background-color: #262626; + color: #cccccc; + border: 1px solid #555555; +} +QPushButton#btnStart { + background-color: #0d2818; + color: #4caf84; + border: 1px solid #1a4d30; +} +QPushButton#btnStart:hover { + background-color: #143d24; + color: #6fd4a0; +} +QPushButton#btnStart:disabled { + background-color: #0f0f0f; + color: #333333; + border: 1px solid #222222; +} +QPushButton#btnStop { + background-color: #2b1010; + color: #e05555; + border: 1px solid #4d1a1a; +} +QPushButton#btnStop:hover { + background-color: #3d1818; + color: #f07070; +} +QPushButton#btnStop:disabled { + background-color: #0f0f0f; + color: #333333; + border: 1px solid #222222; +} +QLineEdit { + background-color: #0d0d0d; + color: #999999; + border: 1px solid #333333; + border-radius: 4px; + padding: 5px 10px; + font-size: 13px; +} + +QLabel#toolbarCaption { + color: #777777; + font-family: "SimHei"; + font-size: 13px; + padding-left: 4px; +} + +QComboBox { + background-color: #0d0d0d; + color: #999999; + border: 1px solid #333333; + border-radius: 4px; + padding: 5px 10px; + font-family: "SimHei"; + font-size: 13px; + min-width: 96px; +} +QComboBox:hover { + background-color: #1a1a1a; + color: #cccccc; + border: 1px solid #555555; +} +QComboBox::drop-down { + border: none; + width: 20px; +} +QComboBox QAbstractItemView { + background-color: #0d0d0d; + color: #999999; + border: 1px solid #333333; + selection-background-color: #1a3a4a; + selection-color: #cccccc; +} + +/* ---- 力曲线容器 ---- */ +GraphicsLayoutWidget { + border: 1px solid #262626; + border-radius: 4px; +} + +/* ---- 日志面板 ---- */ +QPlainTextEdit { + background-color: #0b0b0b; + color: #777777; + border: 1px solid #262626; + border-radius: 4px; + font-family: "Cascadia Code", "Consolas", "Courier New", monospace; + font-size: 13px; + padding: 8px; + selection-background-color: #1a3a4a; + selection-color: #cccccc; +} + +QPlainTextEdit QScrollBar:vertical { + background-color: #0b0b0b; + width: 6px; + margin: 1px; + border-radius: 3px; +} +QPlainTextEdit QScrollBar::handle:vertical { + background-color: #262626; + border-radius: 3px; + min-height: 30px; +} +QPlainTextEdit QScrollBar::handle:vertical:hover { + background-color: #3a3a3a; +} +QPlainTextEdit QScrollBar::add-line:vertical, +QPlainTextEdit QScrollBar::sub-line:vertical { + height: 0px; +} +QPlainTextEdit QScrollBar::add-page:vertical, +QPlainTextEdit QScrollBar::sub-page:vertical { + background: none; +} + +QSplitter::handle { + background-color: #1a1a1a; +} +QSplitter::handle:vertical { + height: 1px; +} +QSplitter::handle:horizontal { + width: 1px; +} +""" diff --git a/examples/gui/viewer2d/window.py b/examples/gui/viewer2d/window.py new file mode 100644 index 0000000..fa95e2c --- /dev/null +++ b/examples/gui/viewer2d/window.py @@ -0,0 +1,525 @@ +"""2D Qt sensor monitor window (image + force curves).""" + +from __future__ import annotations + +import os +import sys +import time +from datetime import datetime +from pathlib import Path + +import cv2 +from PyQt5.QtCore import Qt, QTimer +from PyQt5.QtGui import QFont, QIcon +from PyQt5.QtWidgets import ( + QApplication, + QComboBox, + QFileDialog, + QHBoxLayout, + QLabel, + QLineEdit, + QMainWindow, + QPushButton, + QSplitter, + QVBoxLayout, + QWidget, +) + +try: + import pyqtgraph as pg +except Exception as exc: # pragma: no cover + raise ImportError( + "PyQtGraph is not installed. Please install it using 'pip install pyqt5 pyqtgraph'." + ) from exc + +from gui.sensors import create_sensor_adapter +from gui.widgets.log_panel import TeeStream, WidgetLogHandler, make_log_widget +from gui.widgets.qt_pixmap import ndarray_to_pixmap, pixmap_keep_aspect + +from .backend import ( + PLOT_BUFFER_POINTS, + PLOT_WINDOW_SEC, + QtViewerBackend, + panel_width_for_aspect, +) +from .styles import CURVE_THEME_COLOR, STYLE + + +class QtViewerWindow(QMainWindow): + """2D monitor: flow/image panel + FN/FX/FY curves driven by QtViewerBackend.""" + + def __init__(self, args, log_file): + super().__init__() + self.setWindowTitle("Orisys SDK") + self.setStyleSheet(STYLE) + + logo_path = Path(__file__).resolve().parents[3] / "logo.png" + if logo_path.exists(): + self.setWindowIcon(QIcon(str(logo_path))) + + self.sensor_adapter = None + self._last_snapshot = None + self._last_viewer_frame = None + self._display_aspect = 1.0 + self._source_pixmap = None + self.args = args + + self.is_save = False + self.frame_count = 0 + self.writer = None + self.save_path = None + + self.viewer_backend = QtViewerBackend( + npoints=PLOT_BUFFER_POINTS, + lowpass_cutoff_hz=getattr(args, "cutoff", 10.0), + ) + + self._setup_ui() + self._create_plots() + + self.camera_edit.setText(args.video) + self.config_edit.setText(args.config) + self.cal_edit.setText(args.cal) + + self.log_handler = WidgetLogHandler(self.log_widget) + sys.stdout = TeeStream(log_file, self.log_handler) + sys.stderr = sys.stdout + + self.timer = QTimer() + self.timer.timeout.connect(self._on_timer) + self._running = False + + def _setup_ui(self): + central = QWidget() + self.setCentralWidget(central) + root_layout = QVBoxLayout(central) + root_layout.setContentsMargins(6, 6, 6, 6) + root_layout.setSpacing(6) + + bar = QHBoxLayout() + bar.setSpacing(8) + + self.btn_camera = QPushButton("摄像机") + self.btn_camera.setObjectName("btnCamera") + self.btn_camera.clicked.connect(self._select_camera) + bar.addWidget(self.btn_camera) + self.camera_edit = QLineEdit("1") + self.camera_edit.setPlaceholderText("摄像头编号 或 视频路径") + bar.addWidget(self.camera_edit, 1) + + self.btn_config = QPushButton("配置") + self.btn_config.setObjectName("btnConfig") + self.btn_config.clicked.connect(self._select_config) + bar.addWidget(self.btn_config) + self.config_edit = QLineEdit("box") + bar.addWidget(self.config_edit, 1) + + self.btn_reset = QPushButton("重置") + self.btn_reset.setObjectName("btnReset") + self.btn_reset.setEnabled(False) + self.btn_reset.clicked.connect(self._on_reset) + bar.addWidget(self.btn_reset) + self.cal_edit = QLineEdit("./config/ddjx01.npy") + bar.addWidget(self.cal_edit, 1) + + self.btn_start = QPushButton("启动") + self.btn_start.setObjectName("btnStart") + self.btn_start.clicked.connect(self._on_start) + bar.addWidget(self.btn_start) + + self.btn_stop = QPushButton("暂停") + self.btn_stop.setObjectName("btnStop") + self.btn_stop.setEnabled(False) + self.btn_stop.clicked.connect(self._on_stop) + bar.addWidget(self.btn_stop) + + bg_caption = QLabel("背景") + bg_caption.setObjectName("toolbarCaption") + bar.addWidget(bg_caption) + self.bg_combo = QComboBox() + self.bg_combo.setObjectName("bgCombo") + self.bg_combo.addItem("纯色背景", "pure_color") + self.bg_combo.addItem("采集图像", "captured") + self.bg_combo.setCurrentIndex(0) + self.bg_combo.currentIndexChanged.connect(self._on_arrow_background_changed) + bar.addWidget(self.bg_combo) + + root_layout.addLayout(bar) + + v_splitter = QSplitter(Qt.Vertical) + h_splitter = QSplitter(Qt.Horizontal) + + left = QWidget() + left_layout = QVBoxLayout(left) + left_layout.setContentsMargins(0, 0, 0, 0) + left_layout.setSpacing(6) + + self.arrows_label = QLabel() + self.arrows_label.setObjectName("arrowsLabel") + self.arrows_label.setMinimumSize(160, 160) + self.arrows_label.setScaledContents(False) + self.arrows_label.setAlignment(Qt.AlignCenter) + left_layout.addWidget(self.arrows_label) + + h_splitter.addWidget(left) + + pg.setConfigOption("background", "#000000") + pg.setConfigOption("foreground", "#777777") + pg.setConfigOptions(antialias=True) + self.graph_widget = pg.GraphicsLayoutWidget() + self.graph_widget.setMinimumWidth(300) + h_splitter.addWidget(self.graph_widget) + + self.h_splitter = h_splitter + v_splitter.addWidget(h_splitter) + + from gui.log_config import LOG_CONFIG + + self.log_widget = make_log_widget(LOG_CONFIG) + v_splitter.addWidget(self.log_widget) + + v_splitter.setStretchFactor(0, 4) + v_splitter.setStretchFactor(1, 1) + root_layout.addWidget(v_splitter) + + def _create_plots(self): + titles = ["法向力 (raw)", "切向力 X", "切向力 Y"] + self.plots = [] + self.curves = [] + self.dot_glows = [] + self.dot_cores = [] + + axis_pen = pg.mkPen(color=(38, 38, 38), width=1) + label_font = QFont("SimHei", 9) + + for i, title in enumerate(titles): + p = self.graph_widget.addPlot(title=None) + p.setTitle(title, color="#999999", size="14pt") + p.titleLabel.setFont(QFont("SimHei", 14)) + p.setLabel("bottom", "时间", units="s") + p.getAxis("bottom").label.setFont(label_font) + p.showAxis("top", False) + p.showAxis("right", False) + p.getAxis("left").setPen(axis_pen) + p.getAxis("bottom").setPen(axis_pen) + p.showGrid(x=False, y=True, alpha=0.12) + + tick_font = QFont("Consolas", 11) + tick_font.setStyleHint(QFont.Monospace) + p.getAxis("left").setTickFont(tick_font) + p.getAxis("bottom").setTickFont(tick_font) + + line_pen = pg.mkPen(color=CURVE_THEME_COLOR, width=2) + curve = p.plot(pen=line_pen, fillLevel=0) + dot_glow = p.plot( + [], + [], + pen=None, + symbol="o", + symbolSize=8, + symbolPen=None, + symbolBrush=(0, 229, 255, 76), + ) + dot_core = p.plot( + [], + [], + pen=None, + symbol="o", + symbolSize=4, + symbolPen=None, + symbolBrush=(255, 255, 255, 255), + ) + + p.setXRange(0, PLOT_WINDOW_SEC, padding=0) + p.enableAutoRange(x=False, y=False) + + self.plots.append(p) + self.curves.append(curve) + self.dot_glows.append(dot_glow) + self.dot_cores.append(dot_core) + + if i < 2: + self.graph_widget.nextRow() + + def adjust_image_panel(self): + """Size the left panel from the current image aspect ratio.""" + top_bar_h = 48 + log_h = self.log_widget.height() + spacing = 20 + available = max(self.height() - top_bar_h - log_h - spacing, 160) + image_w = panel_width_for_aspect(available, self._display_aspect, min_size=160) + max_w = max(self.width() // 2, 160) + image_w = min(image_w, max_w) + self.h_splitter.setSizes([image_w, max(self.width() - image_w, 200)]) + self._render_display_pixmap() + + def resizeEvent(self, event): + super().resizeEvent(event) + self._render_display_pixmap() + + def _render_display_pixmap(self): + if self._source_pixmap is None or self._source_pixmap.isNull(): + return + target = self.arrows_label.size() + if target.width() < 2 or target.height() < 2: + return + self.arrows_label.setPixmap(pixmap_keep_aspect(self._source_pixmap, target)) + + def _on_timer(self): + if self.sensor_adapter is None: + return + + snapshot = self.sensor_adapter.read() + if snapshot is None: + print("Failed to get image") + return + self._last_snapshot = snapshot + + viewer_frame = self.viewer_backend.ingest(snapshot, now=time.time()) + self._last_viewer_frame = viewer_frame + first_image = self._source_pixmap is None + aspect_changed = abs(viewer_frame.aspect_ratio - self._display_aspect) > 1e-3 + self._display_aspect = viewer_frame.aspect_ratio + + if self.is_save and self.writer is not None and snapshot.record_frame is not None: + self.frame_count += 1 + save_img = snapshot.record_frame + if len(save_img.shape) == 2 or ( + len(save_img.shape) == 3 and save_img.shape[2] == 1 + ): + if len(save_img.shape) == 3: + save_img = save_img.squeeze(2) + save_img = cv2.cvtColor(save_img, cv2.COLOR_GRAY2BGR) + self.writer.write(save_img) + + print( + f"FPS={viewer_frame.fps:.2f}, " + f"法向力(raw)={viewer_frame.normal_force:.4f}, " + f"切向力X={viewer_frame.shear_x:.4f}, " + f"切向力Y={viewer_frame.shear_y:.4f}, " + f"法向力(N)={viewer_frame.normal_force_calibrated:.6f}" + ) + + self._source_pixmap = ndarray_to_pixmap(viewer_frame.display_image, bgr=True) + if first_image or aspect_changed: + self.adjust_image_panel() + else: + self._render_display_pixmap() + + plot_state = self.viewer_backend.plot_state() + if plot_state is None: + return + + for i, curve in enumerate(self.curves): + x = plot_state.times + y = plot_state.values[:, i] + curve.setData(x, y) + ymin, ymax = plot_state.y_ranges[i] + self.plots[i].setYRange(ymin, ymax) + self.dot_glows[i].setData(x[-1:], y[-1:]) + self.dot_cores[i].setData(x[-1:], y[-1:]) + + x0, x1 = plot_state.x_range + for p in self.plots: + p.setXRange(x0, x1, padding=0) + + def _on_start(self): + if self._running: + return + self.btn_start.setText("启动中...") + self.btn_start.setEnabled(False) + QApplication.processEvents() + + try: + src = self.camera_edit.text() + cfg = self.config_edit.text() + cal = self.cal_edit.text() + verbose = getattr(self.args, "verbose", True) + sensor_type = getattr(self.args, "sensor_type", "orisys") + self.sensor_adapter = create_sensor_adapter( + sensor_type, + video=src, + config_name=cfg, + cal_path=cal, + verbose=verbose, + ) + self.sensor_adapter.start() + except Exception as e: + self.btn_start.setText("启动") + self.btn_start.setEnabled(True) + self.sensor_adapter = None + print(f"启动失败: {e}") + return + + self.viewer_backend.reset() + self._running = True + self.timer.start(10) + self.btn_start.setText("启动") + self.btn_stop.setEnabled(True) + self.btn_reset.setEnabled(True) + print( + f"▶ 启动 — sensor_type={getattr(self.args, 'sensor_type', 'orisys')}, " + f"摄像机={src}, 配置={cfg}, 标定={cal}" + ) + + def _on_stop(self): + if not self._running: + return + self._running = False + self.timer.stop() + self.btn_stop.setText("停止中...") + self.btn_stop.setEnabled(False) + QApplication.processEvents() + + if self.sensor_adapter is not None: + self.sensor_adapter.close() + self.sensor_adapter = None + self._last_snapshot = None + + self.btn_start.setEnabled(True) + self.btn_stop.setText("暂停") + self.btn_reset.setEnabled(False) + print("⏸ 已暂停") + + def _select_camera(self): + path, _ = QFileDialog.getOpenFileName( + self, + "选择视频文件", + "data", + "视频 (*.mp4 *.avi *.mov);;所有文件 (*)", + ) + if path: + self.camera_edit.setText(path) + + def _select_config(self): + path, _ = QFileDialog.getOpenFileName( + self, + "选择配置文件", + "config", + "JSON (*.json);;所有文件 (*)", + ) + if path: + self.config_edit.setText(path) + + def _on_reset(self): + if self.sensor_adapter is None or not self._running: + print("请先启动传感器后再重置追踪器。") + return + self.sensor_adapter.reset() + print("追踪器已重置。") + + def _on_arrow_background_changed(self, _index: int) -> None: + value = self.bg_combo.currentData() + if value not in ("pure_color", "captured"): + return + self.viewer_backend.set_arrow_background(value) + if self._last_snapshot is None: + return + viewer_frame = self.viewer_backend.rebuild_display(self._last_snapshot) + self._last_viewer_frame = viewer_frame + aspect_changed = abs(viewer_frame.aspect_ratio - self._display_aspect) > 1e-3 + self._display_aspect = viewer_frame.aspect_ratio + self._source_pixmap = ndarray_to_pixmap(viewer_frame.display_image, bgr=True) + if aspect_changed: + self.adjust_image_panel() + else: + self._render_display_pixmap() + + def keyPressEvent(self, event): + key = event.key() + if key == Qt.Key_Escape: + if self.isFullScreen(): + self.showNormal() + else: + self.showFullScreen() + elif key == Qt.Key_Q: + print("\n正在退出示例程序...") + self.close() + elif key == Qt.Key_S: + self._start_recording() + elif key == Qt.Key_E: + self._stop_recording() + elif key == Qt.Key_R: + self._on_reset() + else: + super().keyPressEvent(event) + + def _start_recording(self): + if self.sensor_adapter is None: + print("请先启动采集") + return + self.save_path = "./data/pulse_test.mp4" + os.makedirs(os.path.dirname(self.save_path), exist_ok=True) + + save_img = None + if self._last_snapshot is not None: + save_img = self._last_snapshot.record_frame + if save_img is None: + print("当前没有可用帧，无法确定录像尺寸。") + return + + temp_img = save_img.copy() + if len(temp_img.shape) == 2 or ( + len(temp_img.shape) == 3 and temp_img.shape[2] == 1 + ): + if len(temp_img.shape) == 3: + temp_img = temp_img.squeeze(2) + temp_img = cv2.cvtColor(temp_img, cv2.COLOR_GRAY2BGR) + + fh, fw = temp_img.shape[:2] + self.writer = cv2.VideoWriter( + self.save_path, + cv2.VideoWriter_fourcc(*"mp4v"), + 30, + (fw, fh), + ) + if self.writer.isOpened(): + self.is_save = True + print(f"开始录像：{self.save_path}") + else: + print(f"无法创建录像文件：{self.save_path}") + self.writer = None + + def _stop_recording(self): + self.is_save = False + if self.writer is not None: + self.writer.release() + self.writer = None + if self.save_path is not None: + print(f"录像结束：{self.save_path}，总帧数：{self.frame_count}") + else: + print(f"录像结束，总帧数：{self.frame_count}") + self.frame_count = 0 + + def closeEvent(self, event): + self.timer.stop() + if self.writer is not None: + self.writer.release() + if self.sensor_adapter is not None: + self.sensor_adapter.close() + self.sensor_adapter = None + event.accept() + + +def run_viewer(args, *, log_dir: str | Path | None = None) -> int: + """Launch the 2D Qt viewer and block until the window closes.""" + from gui.app import get_or_create_qapp + + if log_dir is None: + log_dir = Path(__file__).resolve().parents[3] / "scripts" / "logs" + else: + log_dir = Path(log_dir) + log_dir.mkdir(parents=True, exist_ok=True) + log_path = log_dir / f"qt_viewer_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" + log_file = open(log_path, "w", encoding="utf-8") + sys.stdout = log_file + sys.stderr = sys.stdout + + print("\n按键说明：Q 退出 | R 重置追踪器 | S 开始录像 | E 结束录像 | ESC 全屏") + + app = get_or_create_qapp("Orisys SDK") + window = QtViewerWindow(args, log_file) + window.showFullScreen() + QApplication.processEvents() + window.adjust_image_panel() + return app.exec_() diff --git a/examples/gui/viewer3d/__init__.py b/examples/gui/viewer3d/__init__.py new file mode 100644 index 0000000..90ae850 --- /dev/null +++ b/examples/gui/viewer3d/__init__.py @@ -0,0 +1,36 @@ +"""Finger 3D Qt dashboard (mesh + UV arrows + force charts).""" + +from .dashboard import FingerDashboardWindow, run_dashboard +from .layout import ( + LOG_PANEL, + LAYOUT, + MESH_CONTACT_ARROW, + MESH_LIGHTING, + PANEL_SETUP, + TOP_ROW, + DashboardLayout, + MeshLighting, + PanelRow, + PanelSetup, + PanelSlot, + top_row_splitter_sizes, + top_row_slot_index, +) + +__all__ = [ + "DashboardLayout", + "FingerDashboardWindow", + "LAYOUT", + "LOG_PANEL", + "MESH_CONTACT_ARROW", + "MESH_LIGHTING", + "MeshLighting", + "PANEL_SETUP", + "TOP_ROW", + "PanelRow", + "PanelSetup", + "PanelSlot", + "run_dashboard", + "top_row_splitter_sizes", + "top_row_slot_index", +] diff --git a/examples/gui/viewer3d/camera_discovery.py b/examples/gui/viewer3d/camera_discovery.py new file mode 100644 index 0000000..b161abe --- /dev/null +++ b/examples/gui/viewer3d/camera_discovery.py @@ -0,0 +1,139 @@ +"""Discover USB cameras for the dashboard (Windows DSHOW/MSMF, Linux V4L2).""" + +from __future__ import annotations + +import contextlib +import os +import sys +from dataclasses import dataclass + +import cv2 + +DEFAULT_MAX_CAMERA_INDEX = 10 + + +@dataclass(frozen=True) +class CameraInfo: + index: int + backend_name: str + backend_api: int + width: int + height: int + + @property + def source(self) -> str: + return str(self.index) + + def label(self) -> str: + return f"{self.index} ({self.width}x{self.height}) [{self.backend_name}]" + + +@contextlib.contextmanager +def _suppress_opencv_noise(): + """Hide OpenCV VideoCapture probe warnings (stderr + OpenCV log).""" + prev_level = None + set_level = None + try: + logging = getattr(getattr(cv2, "utils", None), "logging", None) + if logging is not None: + set_level = getattr(logging, "setLogLevel", None) + get_level = getattr(logging, "getLogLevel", None) + silent = getattr(logging, "LOG_LEVEL_SILENT", 0) + if set_level is not None: + if get_level is not None: + prev_level = get_level() + set_level(silent) + except Exception: + prev_level = None + + devnull = open(os.devnull, "w") + try: + with contextlib.redirect_stderr(devnull): + stderr_fd = sys.stderr.fileno() + saved_fd = os.dup(stderr_fd) + try: + os.dup2(devnull.fileno(), stderr_fd) + yield + finally: + os.dup2(saved_fd, stderr_fd) + os.close(saved_fd) + finally: + devnull.close() + if set_level is not None and prev_level is not None: + try: + set_level(prev_level) + except Exception: + pass + + +def _backend_candidates() -> list[tuple[str, int]]: + candidates: list[tuple[str, int]] = [] + if sys.platform == "win32": + if hasattr(cv2, "CAP_DSHOW"): + candidates.append(("DSHOW", cv2.CAP_DSHOW)) + if hasattr(cv2, "CAP_MSMF"): + candidates.append(("MSMF", cv2.CAP_MSMF)) + elif sys.platform.startswith("linux"): + # Prefer v4l2 for USB camera indices on Linux. + if hasattr(cv2, "CAP_V4L2"): + candidates.append(("V4L2", cv2.CAP_V4L2)) + if not candidates: + candidates.append(("ANY", cv2.CAP_ANY)) + return candidates + + +def _try_open(index: int, backend_name: str, backend_api: int) -> CameraInfo | None: + cap = cv2.VideoCapture(index, backend_api) + if not cap.isOpened(): + cap.release() + return None + ok, frame = cap.read() + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) + if not ok or frame is None: + if width <= 0 or height <= 0: + cap.release() + return None + else: + h, w = frame.shape[:2] + width = width or int(w) + height = height or int(h) + cap.release() + return CameraInfo( + index=index, + backend_name=backend_name, + backend_api=backend_api, + width=width, + height=height, + ) + + +def list_cameras(max_index: int = DEFAULT_MAX_CAMERA_INDEX) -> list[CameraInfo]: + """Probe indices ``0 .. max_index`` and return working cameras.""" + found: list[CameraInfo] = [] + seen_indices: set[int] = set() + backends = _backend_candidates() + + with _suppress_opencv_noise(): + for index in range(0, max_index + 1): + for backend_name, backend_api in backends: + info = _try_open(index, backend_name, backend_api) + if info is None: + continue + if index in seen_indices: + break + found.append(info) + seen_indices.add(index) + break + return found + + +def pick_default_camera(cameras: list[CameraInfo], preferred: str | None = None) -> str | None: + if not cameras: + return None + preferred = (preferred or "").strip() + if preferred: + for cam in cameras: + if cam.source == preferred: + return cam.source + return cameras[0].source diff --git a/examples/gui/dashboard.py b/examples/gui/viewer3d/dashboard.py similarity index 64% rename from examples/gui/dashboard.py rename to examples/gui/viewer3d/dashboard.py index 5b79021..b95e623 100644 --- a/examples/gui/dashboard.py +++ b/examples/gui/viewer3d/dashboard.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math import sys import time @@ -17,9 +18,12 @@ from PyQt5.QtWidgets import ( ) from pipeline.config import PipelineConfig +from pipeline.csv_recorder import CsvRecorder from pipeline.service import FingerPipeline -from app_paths import resource_path +from gui.app_paths import resource_path +from .camera_discovery import list_cameras, pick_default_camera +from .preferences import get_preferred_video_source, load_preferences, save_preferences from .layout import ( DashboardLayout, LAYOUT, @@ -29,14 +33,12 @@ from .layout import ( ) from .mesh_view_state import MeshViewState from .styles import STYLE +from gui.widgets import TeeStream, WidgetLogHandler, make_log_widget from .widgets import ( ArrowPanelWidget, ForceChartWidget, MeshViewportWidget, SettingsPanelWidget, - TeeStream, - WidgetLogHandler, - make_log_widget, ) @@ -54,6 +56,7 @@ class FingerDashboardWindow(QMainWindow): self.config = config self.layout = layout self.pipeline = FingerPipeline(config) + self.csv_recorder = CsvRecorder(config.csv_save_dir) self._running = False self._frame_index = 0 self._wall_fps = 0.0 @@ -99,22 +102,45 @@ class FingerDashboardWindow(QMainWindow): mesh_state=self.mesh_state, show_log=self._log_panel_visible, ) + # CLI / pipeline defaults first, then overlay saved user preferences. self.settings_panel.set_video_source(str(self.config.video)) self.settings_panel.set_config_path(self.config.config) self.settings_panel.set_calibration_path(self.config.calib or "") self.settings_panel.set_finger_export_dir(self.config.export_dir) + self.settings_panel.set_csv_save_dir(self.config.csv_save_dir) - self.settings_panel.log_visibility_changed.connect(self._set_log_panel_visible) - self.settings_panel.free_rotation_changed.connect(self.mesh_viewport.set_free_rotation) + self._preferences = load_preferences() + self.settings_panel.apply_preference_state(self._preferences) + self._log_panel_visible = bool(self.settings_panel.show_log_check.isChecked()) + self.mesh_viewport.set_free_rotation(bool(self.settings_panel.free_rotate_check.isChecked())) if self.arrow_panel is not None: - self.settings_panel.arrow_background_changed.connect(self.arrow_panel.set_background) + self.arrow_panel.set_background(self.settings_panel.arrow_background()) + # Sync pipeline config fields from restored panel values. + self.config.video = self.settings_panel.video_source() or self.config.video + self.config.config = self.settings_panel.config_path() or self.config.config + self.config.export_dir = self.settings_panel.finger_export_dir() or self.config.export_dir + self.config.csv_save_dir = self.settings_panel.csv_save_dir() or self.config.csv_save_dir + + self.settings_panel.log_visibility_changed.connect(self._on_log_visibility_changed) + self.settings_panel.free_rotation_changed.connect(self._on_free_rotation_changed) + if self.arrow_panel is not None: + self.settings_panel.arrow_background_changed.connect(self._on_arrow_background_changed) + else: + self.settings_panel.arrow_background_changed.connect(lambda _v: self._save_preferences()) self.settings_panel.view_reset_requested.connect(self.mesh_viewport.reset_view) self.settings_panel.browse_video_requested.connect(self._select_camera) + self.settings_panel.camera_refresh_requested.connect(self._refresh_cameras) + self.settings_panel.camera_selection_changed.connect(self._on_camera_selection_changed) self.settings_panel.browse_config_requested.connect(self._select_config) self.settings_panel.browse_finger_assets_requested.connect(self._select_finger_assets) + self.settings_panel.browse_csv_dir_requested.connect(self._select_csv_dir) + self.settings_panel.csv_record_toggled.connect(self._on_csv_record_toggled) + self.settings_panel.show_flow_arrows_check.toggled.connect(lambda _v: self._save_preferences()) self.settings_panel.btn_play_pause.clicked.connect(self._on_play_pause_clicked) self.settings_panel.btn_reset_tracker.clicked.connect(self._on_reset) self.settings_panel.set_play_pause_state("run") + self.settings_panel.set_csv_controls_for_pipeline(False) + self._refresh_cameras(announce=True) self._panel_widgets: dict[str, QWidget] = {} self._panel_widgets["mesh"] = self.mesh_viewport @@ -155,6 +181,37 @@ class FingerDashboardWindow(QMainWindow): self.log_widget.setVisible(self._log_panel_visible) QTimer.singleShot(0, self._apply_panel_layout) + def _on_log_visibility_changed(self, visible: bool) -> None: + self._set_log_panel_visible(visible) + self._save_preferences() + + def _on_free_rotation_changed(self, enabled: bool) -> None: + self.mesh_viewport.set_free_rotation(enabled) + self._save_preferences() + + def _on_arrow_background_changed(self, value: str) -> None: + if self.arrow_panel is not None: + self.arrow_panel.set_background(value) + self._save_preferences() + + def _save_preferences(self) -> None: + prefs = self.settings_panel.preference_state() + self._preferences = prefs + save_preferences(prefs) + # Keep in-memory pipeline config aligned with panel values. + video = prefs.get("video_source") + if isinstance(video, str) and video.strip(): + self.config.video = video.strip() + config_path = prefs.get("config_path") + if isinstance(config_path, str) and config_path.strip(): + self.config.config = config_path.strip() + assets = prefs.get("finger_assets_dir") + if isinstance(assets, str) and assets.strip(): + self.config.export_dir = assets.strip() + csv_dir = prefs.get("csv_save_dir") + if isinstance(csv_dir, str) and csv_dir.strip(): + self.config.csv_save_dir = csv_dir.strip() + def _apply_panel_layout(self) -> None: setup = self.layout.panel_setup scale = self._layout_scale_factor() @@ -314,6 +371,15 @@ class FingerDashboardWindow(QMainWindow): cal = self.settings_panel.calibration_path() self.config.calib = cal or None self.config.export_dir = self.settings_panel.finger_export_dir() + self.config.csv_save_dir = self.settings_panel.csv_save_dir() or "./outputs/csv" + + if not self.config.video: + self._set_play_pause_button("run") + self.settings_panel.set_finger_export_dir_enabled(True) + print("请先选择摄像机或视频文件。") + return + + self._save_preferences() try: self.pipeline = FingerPipeline(self.config) @@ -334,6 +400,8 @@ class FingerDashboardWindow(QMainWindow): self.timer.start(self.layout.timer_interval_ms) self._set_play_pause_button("pause") self.settings_panel.btn_reset_tracker.setEnabled(True) + self.settings_panel.set_camera_controls_enabled(False) + self.settings_panel.set_csv_controls_for_pipeline(True) print( f"▶ 启动 — 摄像机={self.config.video}, 配置={self.config.config}, " f"标定={self.config.calib or '(none)'}, 资产={self.config.export_dir}" @@ -342,6 +410,7 @@ class FingerDashboardWindow(QMainWindow): def _on_stop(self) -> None: if not self._running: return + self._stop_csv_recording(announce=True) self._running = False self.timer.stop() self.log_handler.clear_status_line() @@ -354,6 +423,9 @@ class FingerDashboardWindow(QMainWindow): self._set_play_pause_button("run") self.settings_panel.btn_reset_tracker.setEnabled(False) self.settings_panel.set_finger_export_dir_enabled(True) + self.settings_panel.set_camera_controls_enabled(True) + self.settings_panel.set_csv_controls_for_pipeline(False) + self._update_play_enabled_for_source() def _on_reset(self) -> None: if not self._running: @@ -413,10 +485,26 @@ class FingerDashboardWindow(QMainWindow): profile.record("total", total_ms) self.pipeline.record_total_frame(total_ms) + measurement = self.pipeline.measure_for_csv(frame, self._frame_index) + if self.csv_recorder.is_recording: + self.csv_recorder.write(measurement) + flow_mag = 0.0 if frame.flow.size: flow_mag = float((frame.flow[:, :, 0] ** 2 + frame.flow[:, :, 1] ** 2).mean() ** 0.5) + contact_xyz_mm = None + if ( + math.isfinite(measurement.contact_x) + and math.isfinite(measurement.contact_y) + and math.isfinite(measurement.contact_z) + ): + contact_xyz_mm = ( + measurement.contact_x, + measurement.contact_y, + measurement.contact_z, + ) + log_cfg = layout.log_config if log_cfg.status_enabled: self.log_handler.set_status_line( @@ -428,9 +516,66 @@ class FingerDashboardWindow(QMainWindow): flow_rms=flow_mag, mask_coverage=frame.mask_coverage, wall_fps=self._wall_fps, + contact_xyz_mm=contact_xyz_mm, ) ) + def _refresh_cameras(self, announce: bool = True) -> None: + """Scan cameras and populate the settings dropdown.""" + keep_file = self.settings_panel.is_file_source() + existing_file = self.settings_panel.video_source() if keep_file else "" + + preferred = None + current = self.settings_panel.selected_camera_source() + saved = get_preferred_video_source(self._preferences) + cli_video = str(self.config.video).strip() + if current.isdigit(): + preferred = current + elif saved and str(saved).isdigit(): + preferred = str(saved) + elif cli_video.isdigit(): + preferred = cli_video + + cameras = list_cameras() + selected = pick_default_camera(cameras, preferred) + + self.settings_panel.set_camera_options(cameras, selected) + if keep_file and existing_file: + self.settings_panel.set_video_source(existing_file) + elif selected: + self.settings_panel.set_video_source(selected) + self.config.video = selected + elif not keep_file: + self.config.video = "" + + self._update_play_enabled_for_source() + self._save_preferences() + if announce: + if cameras: + names = ", ".join(cam.label() for cam in cameras) + print(f"已发现 {len(cameras)} 路摄像机：{names}") + if selected and not keep_file: + print(f"当前摄像机：{selected}") + else: + print("未发现可用摄像机。可点击 Refresh 重扫，或通过 … 选择视频文件。") + + def _update_play_enabled_for_source(self) -> None: + if self._running: + return + usable = self.settings_panel.has_usable_source() + self.settings_panel.set_play_pause_state("run") + self.settings_panel.btn_play_pause.setEnabled(usable) + + def _on_camera_selection_changed(self, camera_id: str) -> None: + camera_id = str(camera_id).strip() + if not camera_id: + return + # Explicit camera pick clears sticky file mode and persists the live source. + self.settings_panel.set_video_source(camera_id) + self.config.video = camera_id + self._save_preferences() + self._update_play_enabled_for_source() + def _select_camera(self) -> None: path, _ = QFileDialog.getOpenFileName( self, @@ -440,25 +585,99 @@ class FingerDashboardWindow(QMainWindow): ) if path: self.settings_panel.set_video_source(path) + self.config.video = path + self._save_preferences() + self._update_play_enabled_for_source() def _select_config(self) -> None: + from .preferences import ( + default_finger_config, + normalize_config_preference, + packaged_finger_bundle_root, + ) + + start = packaged_finger_bundle_root() + start_dir = str(start) if start is not None else "sensors/finger" path, _ = QFileDialog.getOpenFileName( self, "选择配置文件", - "config", + start_dir, "JSON (*.json);;所有文件 (*)", ) if path: - self.settings_panel.set_config_path(path) + config = normalize_config_preference(path) or default_finger_config() + self.settings_panel.set_config_path(config) + self.config.config = config + self._save_preferences() def _select_finger_assets(self) -> None: + from .preferences import ( + default_finger_assets_dir, + normalize_assets_preference, + packaged_finger_assets_root, + ) + + start = packaged_finger_assets_root() + start_dir = ( + str(start) + if start is not None + else (self.settings_panel.finger_export_dir() or default_finger_assets_dir()) + ) path = QFileDialog.getExistingDirectory( self, "选择 finger 资产目录", - self.settings_panel.finger_export_dir() or "assets/finger", + start_dir, ) if path: - self.settings_panel.set_finger_export_dir(path) + assets = normalize_assets_preference(path) + self.settings_panel.set_finger_export_dir(assets) + self.config.export_dir = assets + self._save_preferences() + + def _select_csv_dir(self) -> None: + path = QFileDialog.getExistingDirectory( + self, + "选择 CSV 保存目录", + self.settings_panel.csv_save_dir() or "./outputs/csv", + ) + if path: + self.settings_panel.set_csv_save_dir(path) + self.config.csv_save_dir = path + self._save_preferences() + + def _on_csv_record_toggled(self, start: bool) -> None: + if start: + self._start_csv_recording() + else: + self._stop_csv_recording(announce=True) + + def _start_csv_recording(self) -> None: + if not self._running: + print("请先启动传感器后再记录 CSV。") + self.settings_panel.set_csv_recording(False) + return + save_dir = self.settings_panel.csv_save_dir() or "./outputs/csv" + self.config.csv_save_dir = save_dir + try: + path = self.csv_recorder.start(save_dir) + except OSError as exc: + print(f"无法开始 CSV 记录：{exc}") + self.settings_panel.set_csv_recording(False) + return + self.settings_panel.set_csv_recording(True) + print(f"开始 CSV 记录：{path}") + + def _stop_csv_recording(self, *, announce: bool = False) -> None: + if not self.csv_recorder.is_recording: + self.settings_panel.set_csv_recording(False) + return + path, count = self.csv_recorder.stop() + self.settings_panel.set_csv_recording(False) + if announce: + if path is not None: + print(f"CSV 记录结束：{path}，总行数：{count}") + else: + print(f"CSV 记录结束，总行数：{count}") def keyPressEvent(self, event): key = event.key() @@ -483,6 +702,8 @@ class FingerDashboardWindow(QMainWindow): def closeEvent(self, event): self.timer.stop() + self._stop_csv_recording(announce=True) + self._save_preferences() self.mesh_viewport.close_renderer() self.pipeline.close() event.accept() @@ -490,9 +711,10 @@ class FingerDashboardWindow(QMainWindow): def run_dashboard(config: PipelineConfig, *, layout: DashboardLayout = LAYOUT) -> int: """Launch the unified finger dashboard and block until the window closes.""" - from .app import get_or_create_qapp + from gui.app import get_or_create_qapp print("\n按键说明：Q 退出 | R 重置追踪器 | S 开始录像 | E 结束录像 | ESC 全屏") + print("GUI：Record CSV / Stop CSV 写入接触点与力到所选目录。") if config.profile: print(f"Profiling enabled (report every {config.profile_every} frames).") print("Log FPS= is end-to-end dashboard rate; sdk= is Sensor internal rate.") diff --git a/examples/gui/layout.py b/examples/gui/viewer3d/layout.py similarity index 89% rename from examples/gui/layout.py rename to examples/gui/viewer3d/layout.py index 85139b0..1e74181 100644 --- a/examples/gui/layout.py +++ b/examples/gui/viewer3d/layout.py @@ -19,6 +19,9 @@ MeshLighting (below) controls the 3D mesh panel sun light: MESH_BACKGROUND (below) sets the 3D mesh panel clear color (RGB 0–255). +MESH_AXIS_OVERLAY (below) controls the fixed top-right pre-rendered 3D axis triad: + enabled, padding_px, size_px, opacity + MESH_CONTACT_ARROW (below) controls the yellow 3D contact arrow on the mesh panel: reference_normal_force — Fn at which arrow reaches full size (log/power/linear curve) force_strength_mode — "log" | "power" | "linear" @@ -44,6 +47,7 @@ import math from dataclasses import dataclass from typing import Literal +from gui.log_config import LOG_CONFIG, LogConfig from viz.contact_arrow import ContactArrowStyle, DEFAULT_CONTACT_ARROW_STYLE from viz.flow_arrow import FlowArrowStyle from viz.render_2d import ArrowBackground, ArrowCanvasColors @@ -87,7 +91,7 @@ TOP_ROW = PanelRow( PanelSlot("mesh", "3D mesh", width_px=400, height_px=400), PanelSlot("arrows", "UV Unrolled", width_px=400, height_px=400, enabled=True), PanelSlot("charts", "Force charts", width_px=300, height_px=400), - PanelSlot("settings", "Settings", width_px=200, height_px=0), + PanelSlot("settings", "Settings", width_px=250, height_px=0), ), ) @@ -142,6 +146,20 @@ class MeshBackground: MESH_BACKGROUND = MeshBackground(color=(0, 0, 0)) + +@dataclass(frozen=True) +class MeshAxisOverlay: + """Fixed pre-rendered XYZ triad HUD on the 3D mesh RGB output.""" + + enabled: bool = True + padding_px: int = 12 + size_px: int = 96 + opacity: float = 1.0 + + +MESH_AXIS_OVERLAY = MeshAxisOverlay() + + # 3D mesh contact arrow — edit here for dashboard tuning (see ContactArrowStyle in viz/contact_arrow.py). MESH_CONTACT_ARROW = ContactArrowStyle( reference_normal_force=10000.0, @@ -176,46 +194,6 @@ ARROW_CANVAS_COLORS = ArrowCanvasColors( ) -@dataclass(frozen=True) -class LogConfig: - """Log panel — edit LOG_CONFIG or DashboardLayout.log_config.""" - - status_enabled: bool = True - max_lines: int = 2000 - font_family: str = "Cascadia Mono" - font_pt: int = 12 - show_flow: bool = True - show_mask: bool = True - - def format_status( - self, - *, - fps: float, - fnormal: float, - fshearx: float, - fsheary: float, - flow_rms: float = 0.0, - mask_coverage: float = 0.0, - wall_fps: float | None = None, - ) -> str: - shear_mag = float((fshearx**2 + fsheary**2) ** 0.5) - if wall_fps is not None: - parts = [f"FPS={wall_fps:.1f}", f"sdk={fps:.1f}"] - else: - parts = [f"FPS={fps:.1f}"] - parts.extend( - [ - f"Fn={fnormal:.1f}", - f"|Fs|={shear_mag:.1f}", - f"Fs=({fshearx:.1f}, {fsheary:.1f})", - ] - ) - return " ".join(parts) - - -LOG_CONFIG = LogConfig() - - @dataclass(frozen=True) class DashboardLayout: """Window chrome, chart styling, and panel hierarchy.""" @@ -224,7 +202,7 @@ class DashboardLayout: window_title: str = "Orisys Finger Dashboard" window_width: int = 1300 - window_height: int = 520 + window_height: int = 560 root_margin_px: int = 6 root_spacing_px: int = 6 @@ -249,6 +227,7 @@ class DashboardLayout: mesh_background: MeshBackground = MESH_BACKGROUND mesh_contact_arrow: ContactArrowStyle = MESH_CONTACT_ARROW mesh_flow_arrow: FlowArrowStyle = MESH_FLOW_ARROW + mesh_axis_overlay: MeshAxisOverlay = MESH_AXIS_OVERLAY log_config: LogConfig = LOG_CONFIG diff --git a/examples/gui/mesh_render_worker.py b/examples/gui/viewer3d/mesh_render_worker.py similarity index 100% rename from examples/gui/mesh_render_worker.py rename to examples/gui/viewer3d/mesh_render_worker.py diff --git a/examples/gui/mesh_view_state.py b/examples/gui/viewer3d/mesh_view_state.py similarity index 100% rename from examples/gui/mesh_view_state.py rename to examples/gui/viewer3d/mesh_view_state.py diff --git a/examples/gui/viewer3d/preferences.py b/examples/gui/viewer3d/preferences.py new file mode 100644 index 0000000..3d57f11 --- /dev/null +++ b/examples/gui/viewer3d/preferences.py @@ -0,0 +1,165 @@ +"""Load/save dashboard user preferences for the finger 3D viewer.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from gui.app_paths import resource_path + +PREFERENCE_FILE = "preference.json" +LEGACY_CAMERA_SELECTION_FILE = "camera_selection.json" +# Keep preference.json at examples/gui/ for backward compatibility. +_GUI_DIR = Path(__file__).resolve().parents[1] + +# Old lab / repo-relative paths that predate packaged sensors//. +_LEGACY_CONFIG_PATHS = { + "dd02", + "dd02-ov", + "config/dd02-ov.json", + "./config/dd02-ov.json", + "config/finger.json", + "./config/finger.json", + "sensors/finger/sensor.json", +} + + +def preference_path() -> Path: + # Keep preference file at examples/gui/preference.json + return _GUI_DIR / PREFERENCE_FILE + + +def legacy_camera_selection_path() -> Path: + return resource_path(LEGACY_CAMERA_SELECTION_FILE) + + +def default_finger_config() -> str: + """Public product config stem resolved via packaged ``sensors/finger``.""" + return "finger" + + +def default_finger_assets_dir() -> str: + try: + from orisys.finger import DEFAULT_FINGER_EXPORT_DIR + + return DEFAULT_FINGER_EXPORT_DIR + except ImportError: + return "sensors/finger/assets/30fps-0703" + + +def packaged_finger_bundle_root() -> Path | None: + """Filesystem root of the installed ``sensors/finger`` bundle when present.""" + try: + from orisys.configs import resolve_sensor_bundle + except ImportError: + return None + bundle = resolve_sensor_bundle("finger") + return bundle.root if bundle is not None else None + + +def packaged_finger_assets_root() -> Path | None: + try: + from orisys.configs import resolve_sensor_asset_dir + except ImportError: + return None + return resolve_sensor_asset_dir("finger") + + +def normalize_config_preference(value: str | None) -> str: + """Map legacy config paths to the packaged ``finger`` sensor name.""" + if value is None or not str(value).strip(): + return default_finger_config() + text = str(value).strip().replace("\\", "/") + if text in _LEGACY_CONFIG_PATHS or text.endswith("/dd02-ov.json"): + return default_finger_config() + + root = packaged_finger_bundle_root() + if root is not None: + try: + if Path(value).resolve() == (root / "sensor.json").resolve(): + return default_finger_config() + except OSError: + pass + return text + + +def normalize_assets_preference(value: str | None) -> str: + """Prefer packaged ``sensors/finger/assets/...`` when saved paths are stale.""" + default = default_finger_assets_dir() + if value is None or not str(value).strip(): + return default + text = str(value).strip() + posix = text.replace("\\", "/") + if posix == default or posix.endswith("/sensors/finger/assets/30fps-0703"): + return default + + path = Path(text) + if path.is_dir(): + packaged = packaged_finger_assets_root() + if packaged is not None: + try: + if path.resolve() == packaged.resolve(): + return default + except OSError: + pass + # Keep a user-selected custom directory that still exists. + if "sensors/finger" in posix or path.name == "30fps-0703": + return text if path.is_absolute() else default + return text + + # Missing old absolute/repo paths (assets/finger, ./config era) -> packaged default. + return default + + +def _normalize_preferences(prefs: dict[str, Any]) -> dict[str, Any]: + prefs = dict(prefs) + if "config_path" in prefs or prefs: + prefs["config_path"] = normalize_config_preference(prefs.get("config_path")) + if "finger_assets_dir" in prefs or prefs: + prefs["finger_assets_dir"] = normalize_assets_preference(prefs.get("finger_assets_dir")) + prefs.setdefault("sensor_bundle_dir", "sensors/finger") + return prefs + + +def _read_json(path: Path) -> dict[str, Any]: + if not path.is_file(): + return {} + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + return data if isinstance(data, dict) else {} + + +def load_preferences() -> dict[str, Any]: + """Load preferences, migrating camera_id / legacy config paths when needed.""" + prefs = _read_json(preference_path()) + if "video_source" not in prefs: + legacy = _read_json(legacy_camera_selection_path()) + camera_id = legacy.get("camera_id") + if camera_id is not None and str(camera_id).strip(): + prefs["video_source"] = str(camera_id).strip() + return _normalize_preferences(prefs) + + +def save_preferences(prefs: dict[str, Any]) -> None: + """Write preferences atomically enough for dashboard use.""" + if not isinstance(prefs, dict): + return + path = preference_path() + normalized = _normalize_preferences(prefs) + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(normalized, indent=2) + "\n", encoding="utf-8") + except OSError: + pass + + +def get_preferred_video_source(prefs: dict[str, Any] | None = None) -> str | None: + data = prefs if prefs is not None else load_preferences() + value = data.get("video_source") + if value is None: + return None + text = str(value).strip() + return text or None diff --git a/examples/gui/viewer3d/presets.py b/examples/gui/viewer3d/presets.py new file mode 100644 index 0000000..712c037 --- /dev/null +++ b/examples/gui/viewer3d/presets.py @@ -0,0 +1,143 @@ +"""Finger-compatible sensor-bundle presets for the 3D dashboard.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path + +from orisys.finger import DEFAULT_FINGER_EXPORT_DIR + + +def _repo_root_candidates() -> list[Path]: + here = Path(__file__).resolve() + return [ + Path.cwd(), + here.parents[3], # examples/gui/viewer3d -> repo root + ] + + +@dataclass(frozen=True) +class SensorBundlePreset: + label: str + bundle_dir: str + config: str + export_dir: str + + +def _packaged_finger_preset() -> SensorBundlePreset | None: + """Use logical names that resolve through the installed wheel sensors/ layout.""" + try: + from orisys.configs import resolve_sensor_bundle + except ImportError: + return None + bundle = resolve_sensor_bundle("finger") + if bundle is None: + return None + return SensorBundlePreset( + label="ORY-FINGER", + bundle_dir="sensors/finger", + config="finger", + export_dir=DEFAULT_FINGER_EXPORT_DIR, + ) + + +def _load_manifest(bundle_root: Path) -> dict: + path = bundle_root / "manifest.json" + if not path.is_file(): + return {} + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + + +def _bundle_from_root(bundle_root: Path, *, label: str | None = None) -> SensorBundlePreset | None: + sensor_json = bundle_root / "sensor.json" + if not sensor_json.is_file(): + return None + manifest = _load_manifest(bundle_root) + assets_dir = manifest.get("assets_dir", "assets/30fps-0703") + if not isinstance(assets_dir, str) or not assets_dir.strip(): + assets_dir = "assets/30fps-0703" + rel_bundle = bundle_root.as_posix() + for root in _repo_root_candidates(): + try: + rel_bundle = bundle_root.relative_to(root).as_posix() + break + except ValueError: + continue + # Packaged / checkout sensors/finger: keep resolvable product names, not absolute paths. + if rel_bundle.replace("\\", "/").endswith("sensors/finger") or bundle_root.name == "finger": + return SensorBundlePreset( + label=label or "ORY-FINGER", + bundle_dir="sensors/finger", + config="finger", + export_dir=DEFAULT_FINGER_EXPORT_DIR, + ) + return SensorBundlePreset( + label=label or "ORY-FINGER", + bundle_dir=rel_bundle, + config=f"{rel_bundle}/sensor.json", + export_dir=f"{rel_bundle}/{assets_dir}".replace("\\", "/"), + ) + + +def default_finger_bundle() -> SensorBundlePreset: + packaged = _packaged_finger_preset() + if packaged is not None: + return packaged + for root in _repo_root_candidates(): + preset = _bundle_from_root(root / "sensors" / "finger", label="ORY-FINGER") + if preset is not None: + return preset + return SensorBundlePreset( + label="ORY-FINGER", + bundle_dir="sensors/finger", + config="finger", + export_dir=DEFAULT_FINGER_EXPORT_DIR, + ) + + +def discover_sensor_bundle_presets() -> list[SensorBundlePreset]: + """Return finger-compatible sensor bundle presets.""" + presets = [default_finger_bundle()] + known_dirs = {presets[0].bundle_dir.replace("\\", "/")} + + for root in _repo_root_candidates(): + sensors_root = root / "sensors" + if not sensors_root.is_dir(): + continue + for child in sorted(sensors_root.iterdir()): + if not child.is_dir(): + continue + if child.name != "finger": + continue + preset = _bundle_from_root(child, label="ORY-FINGER") + if preset is None: + continue + key = preset.bundle_dir.replace("\\", "/") + if key in known_dirs: + continue + presets.append(preset) + known_dirs.add(key) + break + + return presets + + +def sensor_bundle_preset_index( + config: str, + export_dir: str, + presets: list[SensorBundlePreset], +) -> int: + config = (config or "").strip().replace("\\", "/") + export_dir = (export_dir or "").strip().replace("\\", "/") + for idx, preset in enumerate(presets): + if config == preset.config and export_dir == preset.export_dir: + return idx + if config in ("finger", "dd02", "dd02-ov") and preset.config == "finger": + return idx + if config.endswith("/sensor.json") and export_dir.endswith("/" + Path(preset.export_dir).name): + return idx + return -1 diff --git a/examples/gui/styles.py b/examples/gui/viewer3d/styles.py similarity index 100% rename from examples/gui/styles.py rename to examples/gui/viewer3d/styles.py diff --git a/examples/gui/transport_icons.py b/examples/gui/viewer3d/transport_icons.py similarity index 100% rename from examples/gui/transport_icons.py rename to examples/gui/viewer3d/transport_icons.py diff --git a/examples/gui/viewer3d/widgets/__init__.py b/examples/gui/viewer3d/widgets/__init__.py new file mode 100644 index 0000000..4a58e76 --- /dev/null +++ b/examples/gui/viewer3d/widgets/__init__.py @@ -0,0 +1,15 @@ +"""Finger-specific Qt widgets for the 3D dashboard.""" + +from .arrow_image import ArrowPanelWidget +from .force_charts import ForceChartWidget +from .image_panel import ImagePanel +from .mesh_viewport import MeshViewportWidget +from .settings_panel import SettingsPanelWidget + +__all__ = [ + "ArrowPanelWidget", + "ForceChartWidget", + "ImagePanel", + "MeshViewportWidget", + "SettingsPanelWidget", +] diff --git a/examples/gui/widgets/arrow_image.py b/examples/gui/viewer3d/widgets/arrow_image.py similarity index 96% rename from examples/gui/widgets/arrow_image.py rename to examples/gui/viewer3d/widgets/arrow_image.py index 6885b31..968afd5 100644 --- a/examples/gui/widgets/arrow_image.py +++ b/examples/gui/viewer3d/widgets/arrow_image.py @@ -9,7 +9,7 @@ from viz.render_2d import ArrowBackground, ArrowCanvasColors, render_uv_arrows from ..layout import DashboardLayout, LAYOUT from .image_panel import ImagePanel -from .qt_pixmap import ndarray_to_pixmap +from gui.widgets.qt_pixmap import ndarray_to_pixmap class ArrowPanelWidget(ImagePanel): diff --git a/examples/gui/widgets/force_charts.py b/examples/gui/viewer3d/widgets/force_charts.py similarity index 98% rename from examples/gui/widgets/force_charts.py rename to examples/gui/viewer3d/widgets/force_charts.py index b1d605d..3557355 100644 --- a/examples/gui/widgets/force_charts.py +++ b/examples/gui/viewer3d/widgets/force_charts.py @@ -6,8 +6,8 @@ import numpy as np from PyQt5.QtGui import QFont from PyQt5.QtWidgets import QSizePolicy -from gui.layout import DashboardLayout, LAYOUT -from gui.styles import LOG_FONT_PRIMARY, UI_FONT_PRIMARY +from ..layout import DashboardLayout, LAYOUT +from ..styles import LOG_FONT_PRIMARY, UI_FONT_PRIMARY from pipeline.frame import FingerFrameView from viz.signal_filters import lowpass_filter diff --git a/examples/gui/widgets/image_panel.py b/examples/gui/viewer3d/widgets/image_panel.py similarity index 97% rename from examples/gui/widgets/image_panel.py rename to examples/gui/viewer3d/widgets/image_panel.py index 5102540..56abb84 100644 --- a/examples/gui/widgets/image_panel.py +++ b/examples/gui/viewer3d/widgets/image_panel.py @@ -6,7 +6,7 @@ from PyQt5.QtCore import Qt from PyQt5.QtGui import QPixmap from PyQt5.QtWidgets import QLabel, QSizePolicy, QVBoxLayout, QWidget -from .qt_pixmap import pixmap_keep_aspect +from gui.widgets.qt_pixmap import pixmap_keep_aspect class ImagePanel(QWidget): diff --git a/examples/gui/widgets/mesh_viewport.py b/examples/gui/viewer3d/widgets/mesh_viewport.py similarity index 98% rename from examples/gui/widgets/mesh_viewport.py rename to examples/gui/viewer3d/widgets/mesh_viewport.py index a3907b7..14672ee 100644 --- a/examples/gui/widgets/mesh_viewport.py +++ b/examples/gui/viewer3d/widgets/mesh_viewport.py @@ -11,7 +11,7 @@ from viz.offscreen_mesh import OffscreenFingerMeshRenderer from ..layout import DashboardLayout, LAYOUT from ..mesh_view_state import MeshViewState from ..mesh_render_worker import MeshRenderWorker -from .qt_pixmap import ndarray_to_pixmap +from gui.widgets.qt_pixmap import ndarray_to_pixmap from .image_panel import ImagePanel _VIEW_ROTATE_SENSITIVITY = 0.35 @@ -120,6 +120,7 @@ class MeshViewportWidget(ImagePanel): mesh_background=self._layout.mesh_background.color, contact_arrow_style=self._layout.mesh_contact_arrow, flow_arrow_style=self._layout.mesh_flow_arrow, + axis_overlay=self._layout.mesh_axis_overlay, ) self._renderer.flow_arrow.set_enabled(self._mesh_state.flow_arrows_enabled) diff --git a/examples/gui/viewer3d/widgets/settings_panel.py b/examples/gui/viewer3d/widgets/settings_panel.py new file mode 100644 index 0000000..094651f --- /dev/null +++ b/examples/gui/viewer3d/widgets/settings_panel.py @@ -0,0 +1,476 @@ +"""Settings side panel for the finger dashboard.""" + +from __future__ import annotations + +from PyQt5.QtCore import QSize, Qt, pyqtSignal +from PyQt5.QtWidgets import ( + QCheckBox, + QComboBox, + QHBoxLayout, + QLabel, + QLineEdit, + QPushButton, + QSizePolicy, + QVBoxLayout, + QWidget, +) + +from orisys.finger import DEFAULT_FINGER_EXPORT_DIR +from viz.render_2d import ArrowBackground + +from ..layout import DashboardLayout, LAYOUT +from ..mesh_view_state import MeshViewState +from ..transport_icons import reset_icon, transport_icon + +_PLAY_PAUSE_BUTTON_NAME = "btnPlayPause" +_PLAY_PAUSE_ICON_PX = 18 +_PLAY_PAUSE_ICON_COLORS = { + "run": "#4caf84", + "starting": "#555555", + "pause": "#ff9800", +} + + +class SettingsPanelWidget(QWidget): + """Right-side settings: log, sources, and 3D view controls.""" + + log_visibility_changed = pyqtSignal(bool) + free_rotation_changed = pyqtSignal(bool) + arrow_background_changed = pyqtSignal(str) + view_reset_requested = pyqtSignal() + browse_video_requested = pyqtSignal() + browse_config_requested = pyqtSignal() + browse_finger_assets_requested = pyqtSignal() + browse_csv_dir_requested = pyqtSignal() + csv_record_toggled = pyqtSignal(bool) + camera_refresh_requested = pyqtSignal() + camera_selection_changed = pyqtSignal(str) + + def __init__( + self, + parent=None, + *, + layout: DashboardLayout = LAYOUT, + mesh_state: MeshViewState | None = None, + show_log: bool = True, + ): + super().__init__(parent) + self._layout = layout + self._mesh_state = mesh_state or MeshViewState.from_layout(layout) + self.setObjectName("settingsPanel") + self.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) + + root = QVBoxLayout(self) + root.setContentsMargins(8, 8, 8, 8) + root.setSpacing(6) + + title = QLabel(layout.settings_panel_title) + title.setObjectName("settingsTitle") + root.addWidget(title) + + control_row = QHBoxLayout() + control_row.setSpacing(8) + + self.btn_play_pause = QPushButton() + self.btn_play_pause.setObjectName(_PLAY_PAUSE_BUTTON_NAME) + self.btn_play_pause.setFixedSize(36, 32) + self.btn_play_pause.setIconSize(QSize(_PLAY_PAUSE_ICON_PX, _PLAY_PAUSE_ICON_PX)) + control_row.addWidget(self.btn_play_pause) + + self.btn_reset_tracker = QPushButton("Reset Sensor") + self.btn_reset_tracker.setToolTip("Reset tracker") + self.btn_reset_tracker.setEnabled(False) + control_row.addWidget(self.btn_reset_tracker, 1) + root.addLayout(control_row) + + self.show_log_check = QCheckBox("Show log panel") + self.show_log_check.setChecked(show_log) + self.show_log_check.toggled.connect(self.log_visibility_changed.emit) + root.addWidget(self.show_log_check) + + self.show_flow_arrows_check = QCheckBox("Show displacement arrows") + self.show_flow_arrows_check.setChecked(self._mesh_state.flow_arrows_enabled) + self.show_flow_arrows_check.toggled.connect(self._mesh_state.set_flow_arrows_enabled) + root.addWidget(self.show_flow_arrows_check) + + root.addWidget(self._caption("UV arrow background")) + self.arrow_background_combo = QComboBox() + for label, value in ( + ("Unrolled", "unrolled"), + ("Pure color", "pure_color"), + ): + self.arrow_background_combo.addItem(label, value) + self.set_arrow_background(layout.arrow_background) + self.arrow_background_combo.currentIndexChanged.connect(self._emit_arrow_background_changed) + root.addWidget(self.arrow_background_combo) + + root.addWidget(self._caption("Video source")) + video_row = QHBoxLayout() + video_row.setSpacing(6) + self.camera_combo = QComboBox() + self.camera_combo.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) + self.camera_combo.setToolTip("Select an available camera") + # activated fires on user pick even when the index is unchanged (file → live override). + self.camera_combo.activated.connect(self._on_camera_combo_changed) + self.camera_combo.currentIndexChanged.connect(self._on_camera_combo_changed) + video_row.addWidget(self.camera_combo, 1) + self.btn_refresh_cameras = QPushButton("Refresh") + self.btn_refresh_cameras.setFixedWidth(64) + self.btn_refresh_cameras.setToolTip("Re-scan available cameras") + self.btn_refresh_cameras.clicked.connect(self.camera_refresh_requested.emit) + video_row.addWidget(self.btn_refresh_cameras) + self.btn_browse_video = QPushButton("…") + self.btn_browse_video.setFixedWidth(32) + self.btn_browse_video.setToolTip("Browse video file") + self.btn_browse_video.clicked.connect(self.browse_video_requested.emit) + video_row.addWidget(self.btn_browse_video) + root.addLayout(video_row) + + self.video_file_label = QLabel("") + self.video_file_label.setObjectName("settingsCaption") + self.video_file_label.setWordWrap(True) + self.video_file_label.hide() + root.addWidget(self.video_file_label) + + self._source_mode = "camera" # "camera" | "file" + self._video_file_path = "" + self._updating_camera_options = False + + root.addWidget(self._caption("Config")) + config_row = QHBoxLayout() + config_row.setSpacing(6) + self.config_edit = QLineEdit("finger") + config_row.addWidget(self.config_edit, 1) + self.btn_browse_config = QPushButton("…") + self.btn_browse_config.setFixedWidth(32) + self.btn_browse_config.setToolTip("Browse config JSON") + self.btn_browse_config.clicked.connect(self.browse_config_requested.emit) + config_row.addWidget(self.btn_browse_config) + root.addLayout(config_row) + + # Optional calibration path is kept for future re-enablement but hidden for now. + self.cal_edit = QLineEdit("") + self.cal_edit.setPlaceholderText("Optional calibration path") + self.cal_edit.hide() + + root.addWidget(self._caption("Finger assets")) + finger_assets_row = QHBoxLayout() + finger_assets_row.setSpacing(6) + self.finger_assets_edit = QLineEdit(DEFAULT_FINGER_EXPORT_DIR) + self.finger_assets_edit.setPlaceholderText("Blender export bundle directory") + finger_assets_row.addWidget(self.finger_assets_edit, 1) + self.btn_browse_finger_assets = QPushButton("…") + self.btn_browse_finger_assets.setFixedWidth(32) + self.btn_browse_finger_assets.setToolTip("Browse finger asset directory") + self.btn_browse_finger_assets.clicked.connect(self.browse_finger_assets_requested.emit) + finger_assets_row.addWidget(self.btn_browse_finger_assets) + root.addLayout(finger_assets_row) + + root.addWidget(self._caption("CSV save directory")) + csv_dir_row = QHBoxLayout() + csv_dir_row.setSpacing(6) + self.csv_dir_edit = QLineEdit("./outputs/csv") + self.csv_dir_edit.setPlaceholderText("Directory for CSV recordings") + csv_dir_row.addWidget(self.csv_dir_edit, 1) + self.btn_browse_csv_dir = QPushButton("…") + self.btn_browse_csv_dir.setFixedWidth(32) + self.btn_browse_csv_dir.setToolTip("Browse CSV save directory") + self.btn_browse_csv_dir.clicked.connect(self.browse_csv_dir_requested.emit) + csv_dir_row.addWidget(self.btn_browse_csv_dir) + root.addLayout(csv_dir_row) + + self.btn_csv_record = QPushButton("Record CSV") + self.btn_csv_record.setToolTip("Start writing contact/force CSV") + self.btn_csv_record.setEnabled(False) + self.btn_csv_record.clicked.connect(self._on_csv_toggle_clicked) + root.addWidget(self.btn_csv_record) + self._csv_recording = False + self._pipeline_running = False + + root.addSpacing(4) + rotate_row = QHBoxLayout() + rotate_row.setSpacing(6) + self.free_rotate_check = QCheckBox("Free rotate 3D view") + self.free_rotate_check.toggled.connect(self.free_rotation_changed.emit) + rotate_row.addWidget(self.free_rotate_check, 1) + + self.btn_reset_view = QPushButton() + self.btn_reset_view.setObjectName("btnResetView") + self.btn_reset_view.setToolTip("Reset 3D mesh camera") + self.btn_reset_view.setIcon(reset_icon("#999999")) + self.btn_reset_view.setIconSize(QSize(14, 14)) + self.btn_reset_view.setFixedSize(28, 28) + self.btn_reset_view.clicked.connect(self.view_reset_requested.emit) + rotate_row.addWidget(self.btn_reset_view) + root.addLayout(rotate_row) + + root.addStretch() + + @staticmethod + def _caption(text: str) -> QLabel: + label = QLabel(text) + label.setObjectName("settingsCaption") + return label + + def set_panel_size(self, width: int, height: int) -> None: + self.setFixedSize(max(int(width), 64), max(int(height), 64)) + + def set_play_pause_state(self, state: str) -> None: + """``run`` = green play, ``starting`` = grey disabled play, ``pause`` = orange pause.""" + btn = self.btn_play_pause + if state == "run": + btn.setToolTip("Run") + btn.setEnabled(True) + elif state == "starting": + btn.setToolTip("Starting…") + btn.setEnabled(False) + elif state == "pause": + btn.setToolTip("Pause") + btn.setEnabled(True) + else: + raise ValueError(f"unknown play/pause state: {state!r}") + + kind = "pause" if state == "pause" else "play" + btn.setText("") + btn.setIcon(transport_icon(kind, _PLAY_PAUSE_ICON_COLORS[state])) + btn.setProperty("playState", state) + style = btn.style() + style.unpolish(btn) + style.polish(btn) + btn.update() + + def set_camera_options(self, cameras, selected_value: str | None = None) -> None: + """Populate camera combo from discovered cameras. + + ``cameras`` items may be objects with ``.source`` / ``.label()`` or + ``(source, label)`` tuples. + """ + self._updating_camera_options = True + self.camera_combo.blockSignals(True) + self.camera_combo.clear() + + sources: list[str] = [] + for cam in cameras: + if hasattr(cam, "source") and callable(getattr(cam, "label", None)): + source = str(cam.source) + label = str(cam.label()) + else: + source, label = cam + source = str(source) + label = str(label) + self.camera_combo.addItem(label, source) + sources.append(source) + + preferred = (selected_value or "").strip() + idx = self.camera_combo.findData(preferred) if preferred else -1 + if idx < 0 and self.camera_combo.count() > 0: + idx = 0 + if idx >= 0: + self.camera_combo.setCurrentIndex(idx) + # File mode is cleared only when we intentionally select a camera. + if self._source_mode != "file": + self._source_mode = "camera" + self._video_file_path = "" + self.video_file_label.hide() + self.camera_combo.setEnabled(self.camera_combo.count() > 0) + self.camera_combo.blockSignals(False) + self._updating_camera_options = False + + def set_video_source(self, value: str) -> None: + """Set current video source (camera index string or file path).""" + value = (value or "").strip() + if not value: + return + # Numeric camera index → select in combo if present; else keep as preferred text via adding. + if value.isdigit() or (value.startswith("-") and value[1:].isdigit()): + idx = self.camera_combo.findData(value) + if idx >= 0: + self._source_mode = "camera" + self._video_file_path = "" + self.video_file_label.hide() + self.camera_combo.blockSignals(True) + self.camera_combo.setCurrentIndex(idx) + self.camera_combo.blockSignals(False) + else: + # Not yet scanned / not present: remember as pending camera selection. + self._source_mode = "camera" + self._video_file_path = "" + self.video_file_label.hide() + return + + # Treat as video file path. + self._source_mode = "file" + self._video_file_path = value + self.video_file_label.setText(f"File: {value}") + self.video_file_label.show() + + def selected_camera_source(self) -> str: + data = self.camera_combo.currentData() + return str(data) if data is not None else "" + + def video_source(self) -> str: + if self._source_mode == "file" and self._video_file_path: + return self._video_file_path + return self.selected_camera_source() + + def has_usable_source(self) -> bool: + if self._source_mode == "file" and self._video_file_path: + return True + return bool(self.selected_camera_source()) + + def is_file_source(self) -> bool: + return self._source_mode == "file" and bool(self._video_file_path) + + def set_camera_controls_enabled(self, enabled: bool) -> None: + self.btn_refresh_cameras.setEnabled(enabled) + self.btn_browse_video.setEnabled(enabled) + self.camera_combo.setEnabled(enabled and self.camera_combo.count() > 0) + + def _on_camera_combo_changed(self, _index: int) -> None: + if self._updating_camera_options: + return + source = self.selected_camera_source() + if not source: + return + self._source_mode = "camera" + self._video_file_path = "" + self.video_file_label.hide() + self.camera_selection_changed.emit(source) + + def set_config_path(self, value: str) -> None: + self.config_edit.setText(value) + + def set_calibration_path(self, value: str) -> None: + self.cal_edit.setText(value or "") + + def set_finger_export_dir(self, value: str) -> None: + self.finger_assets_edit.setText(value) + + def set_finger_export_dir_enabled(self, enabled: bool) -> None: + self.finger_assets_edit.setEnabled(enabled) + self.btn_browse_finger_assets.setEnabled(enabled) + + def set_csv_save_dir(self, value: str) -> None: + self.csv_dir_edit.setText(value) + + def set_csv_controls_for_pipeline(self, running: bool) -> None: + """Update CSV controls from pipeline start/stop.""" + self._pipeline_running = bool(running) + if not self._pipeline_running: + self._csv_recording = False + self._refresh_csv_controls() + + def set_csv_recording(self, recording: bool) -> None: + """Update CSV controls after a successful start/stop of recording.""" + self._csv_recording = bool(recording) and self._pipeline_running + self._refresh_csv_controls() + + def _refresh_csv_controls(self) -> None: + self.btn_csv_record.setEnabled(self._pipeline_running) + if self._csv_recording: + self.btn_csv_record.setText("Stop CSV") + self.btn_csv_record.setToolTip("Stop CSV recording") + self.btn_csv_record.setProperty("recording", True) + else: + self.btn_csv_record.setText("Record CSV") + self.btn_csv_record.setToolTip("Start writing contact/force CSV") + self.btn_csv_record.setProperty("recording", False) + style = self.btn_csv_record.style() + style.unpolish(self.btn_csv_record) + style.polish(self.btn_csv_record) + self.btn_csv_record.update() + + dir_enabled = not self._csv_recording + self.csv_dir_edit.setEnabled(dir_enabled) + self.btn_browse_csv_dir.setEnabled(dir_enabled) + + def _on_csv_toggle_clicked(self) -> None: + if not self._pipeline_running: + return + self.csv_record_toggled.emit(not self._csv_recording) + + def _emit_arrow_background_changed(self, _index: int) -> None: + value = self.arrow_background_combo.currentData() + if value: + self.arrow_background_changed.emit(value) + + def set_arrow_background(self, value: ArrowBackground) -> None: + idx = self.arrow_background_combo.findData(value) + if idx < 0: + return + self.arrow_background_combo.blockSignals(True) + self.arrow_background_combo.setCurrentIndex(idx) + self.arrow_background_combo.blockSignals(False) + + def arrow_background(self) -> ArrowBackground: + value = self.arrow_background_combo.currentData() + return value if value else "unrolled" + + def config_path(self) -> str: + return self.config_edit.text().strip() + + def calibration_path(self) -> str: + return self.cal_edit.text().strip() + + def finger_export_dir(self) -> str: + return self.finger_assets_edit.text().strip() + + def csv_save_dir(self) -> str: + return self.csv_dir_edit.text().strip() + + def preference_state(self) -> dict: + """Serialize user-facing settings suitable for preference.json.""" + return { + "video_source": self.video_source(), + "config_path": self.config_path(), + "finger_assets_dir": self.finger_export_dir(), + "csv_save_dir": self.csv_save_dir(), + "show_log": bool(self.show_log_check.isChecked()), + "show_flow_arrows": bool(self.show_flow_arrows_check.isChecked()), + "arrow_background": self.arrow_background(), + "free_rotate": bool(self.free_rotate_check.isChecked()), + } + + def apply_preference_state(self, state: dict | None) -> None: + """Restore user-facing settings from preference.json.""" + if not isinstance(state, dict) or not state: + return + + show_log = state.get("show_log") + if isinstance(show_log, bool): + self.show_log_check.blockSignals(True) + self.show_log_check.setChecked(show_log) + self.show_log_check.blockSignals(False) + + show_flow = state.get("show_flow_arrows") + if isinstance(show_flow, bool): + self.show_flow_arrows_check.blockSignals(True) + self.show_flow_arrows_check.setChecked(show_flow) + self.show_flow_arrows_check.blockSignals(False) + self._mesh_state.set_flow_arrows_enabled(show_flow) + + arrow_bg = state.get("arrow_background") + if isinstance(arrow_bg, str) and arrow_bg: + self.set_arrow_background(arrow_bg) # type: ignore[arg-type] + + free_rotate = state.get("free_rotate") + if isinstance(free_rotate, bool): + self.free_rotate_check.blockSignals(True) + self.free_rotate_check.setChecked(free_rotate) + self.free_rotate_check.blockSignals(False) + + config_path = state.get("config_path") + if isinstance(config_path, str) and config_path.strip(): + self.set_config_path(config_path.strip()) + + assets = state.get("finger_assets_dir") + if isinstance(assets, str) and assets.strip(): + self.set_finger_export_dir(assets.strip()) + + csv_dir = state.get("csv_save_dir") + if isinstance(csv_dir, str) and csv_dir.strip(): + self.set_csv_save_dir(csv_dir.strip()) + + video = state.get("video_source") + if isinstance(video, str) and video.strip(): + self.set_video_source(video.strip()) diff --git a/examples/gui/widgets/__init__.py b/examples/gui/widgets/__init__.py index 6ab2fa6..ca4dad6 100644 --- a/examples/gui/widgets/__init__.py +++ b/examples/gui/widgets/__init__.py @@ -1,19 +1,9 @@ -"""Qt widgets for the finger dashboard.""" +"""Shared Qt widgets used by 2D and 3D viewers.""" -from .settings_panel import SettingsPanelWidget -from .force_charts import ForceChartWidget from .log_panel import TeeStream, WidgetLogHandler, make_log_widget -from .mesh_viewport import MeshViewportWidget -from .arrow_image import ArrowPanelWidget from .qt_pixmap import ndarray_to_pixmap, pixmap_keep_aspect -from .image_panel import ImagePanel __all__ = [ - "ArrowPanelWidget", - "SettingsPanelWidget", - "ForceChartWidget", - "MeshViewportWidget", - "ImagePanel", "TeeStream", "WidgetLogHandler", "make_log_widget", diff --git a/examples/gui/widgets/log_panel.py b/examples/gui/widgets/log_panel.py index 1431f9f..6587f35 100644 --- a/examples/gui/widgets/log_panel.py +++ b/examples/gui/widgets/log_panel.py @@ -22,12 +22,13 @@ class TeeStream: class WidgetLogHandler: - """Mirror print output into a QPlainTextEdit with a live status line at the bottom.""" + """Mirror print output into a QPlainTextEdit with pinned status lines at the bottom.""" def __init__(self, widget: QPlainTextEdit): self.widget = widget self.buf = "" - self._has_status_line = False + self._status_text: str | None = None + self._status_block_count = 0 def write(self, text: str): self.buf += text @@ -42,34 +43,42 @@ class WidgetLogHandler: self.buf = "" def set_status_line(self, text: str) -> None: - """Update the pinned bottom line (FPS / forces) without scrolling the log.""" + """Update pinned bottom status (supports multi-line) without scrolling away.""" text = text.rstrip("\n") - if self._has_status_line: - cursor = QTextCursor(self.widget.document().lastBlock()) - cursor.select(QTextCursor.BlockUnderCursor) - cursor.insertText(text) - else: - self.widget.appendPlainText(text) - self._has_status_line = True + self._remove_status_blocks() + self.widget.appendPlainText(text) + self._status_text = text + self._status_block_count = max(1, text.count("\n") + 1) scrollbar = self.widget.verticalScrollBar() scrollbar.setValue(scrollbar.maximum()) def clear_status_line(self) -> None: - if not self._has_status_line: + self._remove_status_blocks() + self._status_text = None + self._status_block_count = 0 + + def _remove_status_blocks(self) -> None: + if self._status_block_count <= 0: return - cursor = QTextCursor(self.widget.document().lastBlock()) - cursor.select(QTextCursor.BlockUnderCursor) + doc = self.widget.document() + end = QTextCursor(doc) + end.movePosition(QTextCursor.End) + start = QTextCursor(doc) + start.movePosition(QTextCursor.End) + for _ in range(self._status_block_count): + start.movePosition(QTextCursor.StartOfBlock) + if start.position() > 0: + start.movePosition(QTextCursor.Left) + cursor = QTextCursor(doc) + cursor.setPosition(start.position()) + cursor.setPosition(end.position(), QTextCursor.KeepAnchor) cursor.removeSelectedText() - self._has_status_line = False + self._status_block_count = 0 def _append_log_line(self, line: str) -> None: - status = None - if self._has_status_line: - status = self.widget.document().lastBlock().text() - cursor = QTextCursor(self.widget.document().lastBlock()) - cursor.select(QTextCursor.BlockUnderCursor) - cursor.removeSelectedText() - self._has_status_line = False + status = self._status_text + if status is not None: + self.clear_status_line() self.widget.appendPlainText(line) if status is not None: self.set_status_line(status) @@ -77,7 +86,7 @@ class WidgetLogHandler: def make_log_widget(config=None) -> QPlainTextEdit: if config is None: - from ..layout import LOG_CONFIG + from ..log_config import LOG_CONFIG config = LOG_CONFIG widget = QPlainTextEdit() diff --git a/examples/gui/widgets/settings_panel.py b/examples/gui/widgets/settings_panel.py deleted file mode 100644 index 22fb18f..0000000 --- a/examples/gui/widgets/settings_panel.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Settings side panel for the finger dashboard.""" - -from __future__ import annotations - -from PyQt5.QtCore import QSize, Qt, pyqtSignal -from PyQt5.QtWidgets import ( - QCheckBox, - QComboBox, - QHBoxLayout, - QLabel, - QLineEdit, - QPushButton, - QSizePolicy, - QVBoxLayout, - QWidget, -) - -from orisys.finger import DEFAULT_FINGER_EXPORT_DIR -from viz.render_2d import ArrowBackground - -from ..layout import DashboardLayout, LAYOUT -from ..mesh_view_state import MeshViewState -from ..transport_icons import reset_icon, transport_icon - -_PLAY_PAUSE_BUTTON_NAME = "btnPlayPause" -_PLAY_PAUSE_ICON_PX = 18 -_PLAY_PAUSE_ICON_COLORS = { - "run": "#4caf84", - "starting": "#555555", - "pause": "#ff9800", -} - - -class SettingsPanelWidget(QWidget): - """Right-side settings: log, sources, and 3D view controls.""" - - log_visibility_changed = pyqtSignal(bool) - free_rotation_changed = pyqtSignal(bool) - arrow_background_changed = pyqtSignal(str) - view_reset_requested = pyqtSignal() - browse_video_requested = pyqtSignal() - browse_config_requested = pyqtSignal() - browse_finger_assets_requested = pyqtSignal() - - def __init__( - self, - parent=None, - *, - layout: DashboardLayout = LAYOUT, - mesh_state: MeshViewState | None = None, - show_log: bool = True, - ): - super().__init__(parent) - self._layout = layout - self._mesh_state = mesh_state or MeshViewState.from_layout(layout) - self.setObjectName("settingsPanel") - self.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) - - root = QVBoxLayout(self) - root.setContentsMargins(10, 10, 10, 10) - root.setSpacing(10) - - title = QLabel(layout.settings_panel_title) - title.setObjectName("settingsTitle") - root.addWidget(title) - - control_row = QHBoxLayout() - control_row.setSpacing(8) - - self.btn_play_pause = QPushButton() - self.btn_play_pause.setObjectName(_PLAY_PAUSE_BUTTON_NAME) - self.btn_play_pause.setFixedSize(36, 32) - self.btn_play_pause.setIconSize(QSize(_PLAY_PAUSE_ICON_PX, _PLAY_PAUSE_ICON_PX)) - control_row.addWidget(self.btn_play_pause) - - self.btn_reset_tracker = QPushButton("Reset Sensor") - self.btn_reset_tracker.setToolTip("Reset tracker") - self.btn_reset_tracker.setEnabled(False) - control_row.addWidget(self.btn_reset_tracker, 1) - root.addLayout(control_row) - - self.show_log_check = QCheckBox("Show log panel") - self.show_log_check.setChecked(show_log) - self.show_log_check.toggled.connect(self.log_visibility_changed.emit) - root.addWidget(self.show_log_check) - - self.show_flow_arrows_check = QCheckBox("Show displacement arrows") - self.show_flow_arrows_check.setChecked(self._mesh_state.flow_arrows_enabled) - self.show_flow_arrows_check.toggled.connect(self._mesh_state.set_flow_arrows_enabled) - root.addWidget(self.show_flow_arrows_check) - - root.addWidget(self._caption("UV arrow background")) - self.arrow_background_combo = QComboBox() - for label, value in ( - ("Unrolled", "unrolled"), - ("Pure color", "pure_color"), - ): - self.arrow_background_combo.addItem(label, value) - self.set_arrow_background(layout.arrow_background) - self.arrow_background_combo.currentIndexChanged.connect(self._emit_arrow_background_changed) - root.addWidget(self.arrow_background_combo) - - root.addWidget(self._caption("Video source")) - video_row = QHBoxLayout() - video_row.setSpacing(6) - self.video_edit = QLineEdit("0") - self.video_edit.setPlaceholderText("Camera ID or video path") - video_row.addWidget(self.video_edit, 1) - self.btn_browse_video = QPushButton("…") - self.btn_browse_video.setFixedWidth(32) - self.btn_browse_video.setToolTip("Browse video file") - self.btn_browse_video.clicked.connect(self.browse_video_requested.emit) - video_row.addWidget(self.btn_browse_video) - root.addLayout(video_row) - - root.addWidget(self._caption("Config")) - config_row = QHBoxLayout() - config_row.setSpacing(6) - self.config_edit = QLineEdit("./config/dd02-ov.json") - config_row.addWidget(self.config_edit, 1) - self.btn_browse_config = QPushButton("…") - self.btn_browse_config.setFixedWidth(32) - self.btn_browse_config.setToolTip("Browse config JSON") - self.btn_browse_config.clicked.connect(self.browse_config_requested.emit) - config_row.addWidget(self.btn_browse_config) - root.addLayout(config_row) - - self.cal_edit = QLineEdit("") - self.cal_edit.setPlaceholderText("Optional calibration path") - root.addWidget(self._caption("Calibration")) - root.addWidget(self.cal_edit) - - root.addWidget(self._caption("Finger assets")) - finger_assets_row = QHBoxLayout() - finger_assets_row.setSpacing(6) - self.finger_assets_edit = QLineEdit(DEFAULT_FINGER_EXPORT_DIR) - self.finger_assets_edit.setPlaceholderText("Blender export bundle directory") - finger_assets_row.addWidget(self.finger_assets_edit, 1) - self.btn_browse_finger_assets = QPushButton("…") - self.btn_browse_finger_assets.setFixedWidth(32) - self.btn_browse_finger_assets.setToolTip("Browse finger asset directory") - self.btn_browse_finger_assets.clicked.connect(self.browse_finger_assets_requested.emit) - finger_assets_row.addWidget(self.btn_browse_finger_assets) - root.addLayout(finger_assets_row) - - root.addSpacing(8) - rotate_row = QHBoxLayout() - rotate_row.setSpacing(6) - self.free_rotate_check = QCheckBox("Free rotate 3D view") - self.free_rotate_check.toggled.connect(self.free_rotation_changed.emit) - rotate_row.addWidget(self.free_rotate_check, 1) - - self.btn_reset_view = QPushButton() - self.btn_reset_view.setObjectName("btnResetView") - self.btn_reset_view.setToolTip("Reset 3D mesh camera") - self.btn_reset_view.setIcon(reset_icon("#999999")) - self.btn_reset_view.setIconSize(QSize(14, 14)) - self.btn_reset_view.setFixedSize(28, 28) - self.btn_reset_view.clicked.connect(self.view_reset_requested.emit) - rotate_row.addWidget(self.btn_reset_view) - root.addLayout(rotate_row) - - root.addStretch() - - @staticmethod - def _caption(text: str) -> QLabel: - label = QLabel(text) - label.setObjectName("settingsCaption") - return label - - def set_panel_size(self, width: int, height: int) -> None: - self.setFixedSize(max(int(width), 64), max(int(height), 64)) - - def set_play_pause_state(self, state: str) -> None: - """``run`` = green play, ``starting`` = grey disabled play, ``pause`` = orange pause.""" - btn = self.btn_play_pause - if state == "run": - btn.setToolTip("Run") - btn.setEnabled(True) - elif state == "starting": - btn.setToolTip("Starting…") - btn.setEnabled(False) - elif state == "pause": - btn.setToolTip("Pause") - btn.setEnabled(True) - else: - raise ValueError(f"unknown play/pause state: {state!r}") - - kind = "pause" if state == "pause" else "play" - btn.setText("") - btn.setIcon(transport_icon(kind, _PLAY_PAUSE_ICON_COLORS[state])) - btn.setProperty("playState", state) - style = btn.style() - style.unpolish(btn) - style.polish(btn) - btn.update() - - def set_video_source(self, value: str) -> None: - self.video_edit.setText(value) - - def set_config_path(self, value: str) -> None: - self.config_edit.setText(value) - - def set_calibration_path(self, value: str) -> None: - self.cal_edit.setText(value or "") - - def set_finger_export_dir(self, value: str) -> None: - self.finger_assets_edit.setText(value) - - def set_finger_export_dir_enabled(self, enabled: bool) -> None: - self.finger_assets_edit.setEnabled(enabled) - self.btn_browse_finger_assets.setEnabled(enabled) - - def _emit_arrow_background_changed(self, _index: int) -> None: - value = self.arrow_background_combo.currentData() - if value: - self.arrow_background_changed.emit(value) - - def set_arrow_background(self, value: ArrowBackground) -> None: - idx = self.arrow_background_combo.findData(value) - if idx < 0: - return - self.arrow_background_combo.blockSignals(True) - self.arrow_background_combo.setCurrentIndex(idx) - self.arrow_background_combo.blockSignals(False) - - def arrow_background(self) -> ArrowBackground: - value = self.arrow_background_combo.currentData() - return value if value else "unrolled" - - def video_source(self) -> str: - return self.video_edit.text().strip() - - def config_path(self) -> str: - return self.config_edit.text().strip() - - def calibration_path(self) -> str: - return self.cal_edit.text().strip() - - def finger_export_dir(self) -> str: - return self.finger_assets_edit.text().strip() diff --git a/examples/mini_pipeline.py b/examples/mini_pipeline.py new file mode 100644 index 0000000..3247752 --- /dev/null +++ b/examples/mini_pipeline.py @@ -0,0 +1,115 @@ +""" +ORY-MINI：单传感器基础流程 + +本示例演示 Orisys SDK 的标准处理流程： +1. 初始化传感器（摄像头或视频文件） +2. 获取图像并计算形变 +3. 按需计算接触区域 +4. 读取并可视化结果 + +SDK version: 0.5.0 +""" +import os +os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" +import cv2 +import orisys +import numpy as np +import argparse + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--video","-v", type=str, default="0", help="视频源：摄像头编号或视频文件路径") + parser.add_argument("--config","-c", type=str, default="mini", help="配置文件名称或路径") + parser.add_argument("--verbose","-verbose", type=bool, default=True, help="是否输出详细日志") + args = parser.parse_args() + + # 步骤 1：创建传感器对象（必需） + # config_name 可以是内置配置名称，也可以是自定义配置文件路径 + # cal_path 为标定文件路径；每台设备建议使用独立的标定文件 + + # 输入既可以是摄像头编号，也可以是视频文件路径 + try: + input = int(args.video) + is_camera = True + sensor = orisys.Sensor(input, config_name=args.config, verbose=args.verbose) + except: + is_camera = False + sensor = orisys.Sensor(args.video, config_name=args.config, verbose=args.verbose) + + print("\n按键说明：'q' 退出程序，'r' 重置追踪器。") + + while True: + # 步骤 2：数据处理 + # 步骤 2.1：获取并拼接图像（必需） + sensor.get_img() + # 步骤 2.2：计算形变场（必需）；返回值表示当前帧是否检测到接触/运动 + is_contact_now = sensor.compute_deformation() + # 步骤 2.3：仅在当前帧检测到接触/运动时计算接触区域 + # if is_contact_now: + sensor.compute_contact() + + + # 步骤 2.4：读取结果 + fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = sensor.read_info( + sensor.info.FPS, + sensor.info.VRAW, + sensor.info.VNORMAL, + sensor.info.IMG, + sensor.info.CONTOUR, + sensor.info.CENTROID, + sensor.info.DEPTH, + sensor.info.FNORMAL, + sensor.info.FSHEARX, + sensor.info.FSHEARY + ) + + # 步骤 3：可视化显示 + # 步骤 3.1：显示形变矢量场 + arrows = orisys.util.draw_arrows( + img, + flow, + threshold=2, + grid_spacing=20, + arrow_scale=1.0 + ) + cv2.imshow("arrows", arrows) + + # 步骤 3.2：显示接触区域和质心 + image_with_foe = orisys.util.draw_contact(img, contour, centroid) + cv2.imshow("contact", image_with_foe) + + # 步骤 3.3：显示深度图 + # 将深度值归一化到 0-255 范围 + div_abs = depth_map + if div_abs.max() > div_abs.min(): + div_normalized = ((div_abs - div_abs.min()) / (div_abs.max() - div_abs.min()) * 255).astype(np.uint8) + else: + div_normalized = np.zeros_like(div_abs, dtype=np.uint8) + + # 应用伪彩色映射 + cv2.imshow("depth", cv2.applyColorMap(div_normalized, cv2.COLORMAP_JET)) + + # 步骤 3.4：打印关键结果 + print( + f"FPS={fps:.2f}, 法向力={fnormal:.4f}, 切向力X={fshearx:.4f}, " + f"切向力Y={fsheary:.4f}, 深度图尺寸={depth_map.shape}, 原始分辨率={sensor.img_size_raw}" + ) + + # 步骤 3.5：键盘控制 + key = cv2.waitKey(1) & 0xFF + if key == ord("q"): + print("\n正在退出示例程序...") + break + if key == ord("r"): + # 当光流追踪出现漂移时，可手动重置追踪器 + sensor.reset() + print("追踪器已重置。") + + # ========================================================================= + # 4. 释放资源 + # ========================================================================= + sensor.disconnect() # 释放视频源及相关资源 + + +if __name__ == '__main__': + main() diff --git a/examples/pipeline/__init__.py b/examples/pipeline/__init__.py index 6c20f30..ec58516 100644 --- a/examples/pipeline/__init__.py +++ b/examples/pipeline/__init__.py @@ -1,10 +1,14 @@ """Finger example pipeline: runtime acquisition without GUI dependencies.""" from .config import PipelineConfig, mesh_threshold +from .csv_recorder import CSV_COLUMNS, CsvMeasurement, CsvRecorder from .frame import FingerFrameView from .service import FingerPipeline __all__ = [ + "CSV_COLUMNS", + "CsvMeasurement", + "CsvRecorder", "FingerFrameView", "FingerPipeline", "PipelineConfig", diff --git a/examples/pipeline/config.py b/examples/pipeline/config.py index a44baf5..9806cb6 100644 --- a/examples/pipeline/config.py +++ b/examples/pipeline/config.py @@ -17,9 +17,10 @@ def mesh_threshold(mesh_map: str, threshold: float | None) -> float: @dataclass class PipelineConfig: video: str = "0" - config: str = "./config/dd02-ov.json" + config: str = "finger" calib: str | None = None export_dir: str = DEFAULT_FINGER_EXPORT_DIR + csv_save_dir: str = "./outputs/csv" verbose: bool = False mesh_map: str = "depth" threshold: float | None = None @@ -45,6 +46,7 @@ class PipelineConfig: lowpass_cutoff_hz=getattr(args, "cutoff", 5.0), profile=getattr(args, "profile", False), profile_every=getattr(args, "profile_every", 30), + csv_save_dir=getattr(args, "csv_save_dir", "./outputs/csv"), ) @property diff --git a/examples/pipeline/csv_recorder.py b/examples/pipeline/csv_recorder.py new file mode 100644 index 0000000..a8d961c --- /dev/null +++ b/examples/pipeline/csv_recorder.py @@ -0,0 +1,114 @@ +"""CSV recorder for DD02 contact / force logging.""" + +from __future__ import annotations + +import csv +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + + +CSV_COLUMNS = ( + "frame_index", + "timestamp_s", + "contact_x_mm", + "contact_y_mm", + "contact_z_mm", + "normal_x", + "normal_y", + "normal_z", + "force_local_fn", + "force_local_fx", + "force_local_fy", +) + + +@dataclass +class CsvMeasurement: + frame_index: int + timestamp_s: float + contact_x: float + contact_y: float + contact_z: float + normal_x: float + normal_y: float + normal_z: float + force_local_fn: float + force_local_fx: float + force_local_fy: float + + +def measurement_to_csv_row(row: CsvMeasurement) -> dict[str, float | int]: + return { + "frame_index": row.frame_index, + "timestamp_s": row.timestamp_s, + "contact_x_mm": row.contact_x, + "contact_y_mm": row.contact_y, + "contact_z_mm": row.contact_z, + "normal_x": row.normal_x, + "normal_y": row.normal_y, + "normal_z": row.normal_z, + "force_local_fn": row.force_local_fn, + "force_local_fx": row.force_local_fx, + "force_local_fy": row.force_local_fy, + } + + +class CsvRecorder: + """Write one contact/force measurement row per frame.""" + + def __init__(self, save_dir: str | Path = "./outputs/csv") -> None: + self.save_dir = Path(save_dir) + self._file = None + self._writer: csv.DictWriter | None = None + self._path: Path | None = None + self._row_count = 0 + + @property + def is_recording(self) -> bool: + return self._writer is not None + + @property + def path(self) -> Path | None: + return self._path + + @property + def row_count(self) -> int: + return self._row_count + + def start(self, save_dir: str | Path | None = None) -> Path: + if self.is_recording: + self.stop() + + if save_dir is not None: + self.save_dir = Path(save_dir) + self.save_dir.mkdir(parents=True, exist_ok=True) + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + self._path = self.save_dir / f"dd02_record_{stamp}.csv" + self._file = self._path.open("w", newline="", encoding="utf-8") + self._writer = csv.DictWriter(self._file, fieldnames=CSV_COLUMNS) + self._writer.writeheader() + self._file.flush() + self._row_count = 0 + return self._path + + def write(self, row: CsvMeasurement) -> None: + if self._writer is None: + return + self._writer.writerow(measurement_to_csv_row(row)) + self._row_count += 1 + if self._file is not None and self._row_count % 30 == 0: + self._file.flush() + + def stop(self) -> tuple[Path | None, int]: + count = self._row_count + saved_path = self._path + if self._file is not None: + self._file.flush() + self._file.close() + self._file = None + self._writer = None + self._path = None + self._row_count = 0 + return saved_path, count diff --git a/examples/pipeline/service.py b/examples/pipeline/service.py index 2da397d..677e732 100644 --- a/examples/pipeline/service.py +++ b/examples/pipeline/service.py @@ -6,11 +6,12 @@ import time from dataclasses import dataclass import numpy as np -from orisys.finger import FingerRuntime +from orisys.finger import FingerRuntime, compute_uv_surface_frames from viz import FrameRecorder, build_uv_surface_mesh, resolve_viz_pose_path from .config import PipelineConfig +from .csv_recorder import CsvMeasurement from .frame import FingerFrameView from .profiling import StageProfiler, run_deformation_timed @@ -40,6 +41,12 @@ class FingerPipeline: self.profiler: StageProfiler | None = ( StageProfiler(report_every=config.profile_every) if config.profile else None ) + self._world_xyz: np.ndarray | None = None + self._surface_mask: np.ndarray | None = None + self._geometric_center: np.ndarray | None = None + self._tangent_x: np.ndarray | None = None + self._tangent_y: np.ndarray | None = None + self._normals: np.ndarray | None = None @property def is_open(self) -> bool: @@ -72,10 +79,26 @@ class FingerPipeline: if domain.surface is None or getattr(domain.surface, "world_xyz", None) is None: return self.uv_mesh = build_uv_surface_mesh(domain.surface.world_xyz, domain.valid_mask) + self._cache_surface_geometry(domain.surface.world_xyz, domain.valid_mask) if self.config.viz_pose.lower() != "none": self.world_to_cam = np.load(resolve_viz_pose_path(self.config.viz_pose)) self.mesh_lookat = self.uv_mesh.rest_vertices.mean(axis=0) + def _cache_surface_geometry(self, world_xyz: np.ndarray, valid_mask: np.ndarray) -> None: + world_xyz = np.asarray(world_xyz, dtype=np.float64) + valid_mask = np.asarray(valid_mask, dtype=bool) + finite = np.all(np.isfinite(world_xyz), axis=2) + mask = valid_mask & finite + self._world_xyz = world_xyz + self._surface_mask = mask + points = world_xyz[mask] + self._geometric_center = ( + points.mean(axis=0) if len(points) else np.zeros(3, dtype=np.float64) + ) + self._tangent_x, self._tangent_y, self._normals = compute_uv_surface_frames( + world_xyz, mask + ) + def close(self) -> None: if self.profiler is not None and self.profiler.frame_count: self.profiler.print_report(prefix="[profile final]") @@ -83,6 +106,12 @@ class FingerPipeline: self.runtime.close() self.runtime = None self._start_time = None + self._world_xyz = None + self._surface_mask = None + self._geometric_center = None + self._tangent_x = None + self._tangent_y = None + self._normals = None def reset(self) -> None: if self.runtime is not None: @@ -165,3 +194,53 @@ class FingerPipeline: def stop_recording(self) -> None: self.recorder.stop() + + def measure_for_csv(self, frame: FingerFrameView, frame_index: int) -> CsvMeasurement: + """Lift contact + local forces into global-frame measurements for CSV export.""" + nan = float("nan") + contact = (nan, nan, nan) + normal = (nan, nan, nan) + + fn = float(frame.fnormal) + fx = float(frame.fshearx) + fy = float(frame.fsheary) + + if ( + frame.centroid_found + and frame.centroid is not None + and np.asarray(frame.centroid).size >= 2 + and self._world_xyz is not None + and self._geometric_center is not None + and self._normals is not None + and self._tangent_x is not None + and self._tangent_y is not None + ): + cx = int(round(float(np.asarray(frame.centroid).reshape(-1)[0]))) + cy = int(round(float(np.asarray(frame.centroid).reshape(-1)[1]))) + h, w = self._world_xyz.shape[:2] + if 0 <= cy < h and 0 <= cx < w: + if self._surface_mask is None or bool(self._surface_mask[cy, cx]): + world = self._world_xyz[cy, cx] + if np.all(np.isfinite(world)): + centered = world - self._geometric_center + contact = (float(centered[0]), float(centered[1]), float(centered[2])) + + n = self._normals[cy, cx] + n_norm = float(np.linalg.norm(n)) + if n_norm > 1e-8: + n_u = n / n_norm + normal = (float(n_u[0]), float(n_u[1]), float(n_u[2])) + + return CsvMeasurement( + frame_index=int(frame_index), + timestamp_s=float(frame.timestamp), + contact_x=contact[0], + contact_y=contact[1], + contact_z=contact[2], + normal_x=normal[0], + normal_y=normal[1], + normal_z=normal[2], + force_local_fn=fn, + force_local_fx=fx, + force_local_fy=fy, + ) diff --git a/examples/plot_data.py b/examples/plot_data.py index 9e3cb7f..cdbb002 100644 --- a/examples/plot_data.py +++ b/examples/plot_data.py @@ -1,29 +1,24 @@ """ -示例 3：实时曲线与幅值图显示 +示例 3：实时曲线与深度图显示 本示例演示如何在读取传感器结果的同时： 1. 显示形变矢量场 -2. 显示流场幅值热力图 +2. 显示接触深度图 3. 使用 PyQtGraph 实时绘制法向力与切向力曲线 4. 按键控制录像开始与结束 """ import argparse import os -import sys import time -from pathlib import Path os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" import cv2 +import numpy as np import orisys -_EXAMPLES_DIR = Path(__file__).resolve().parent -if str(_EXAMPLES_DIR) not in sys.path: - sys.path.insert(0, str(_EXAMPLES_DIR)) - -from viz import ForceChartWindow, FrameRecorder, draw_magnitude_map, lowpass_filter +from viz import ForceChartWindow, FrameRecorder def open_sensor(video: str, config: str, cal: str, verbose: bool): @@ -37,14 +32,21 @@ def open_sensor(video: str, config: str, cal: str, verbose: bool): def main(): parser = argparse.ArgumentParser() parser.add_argument("--video", "-v", type=str, default="0", help="视频源：摄像头编号或视频文件路径") - parser.add_argument("--config", "-c", type=str, default="./config/ddjx01.json", help="配置文件名称或路径") - parser.add_argument("--cal", "-cal", type=str, default="./config/ddjx01.npy", help="标定文件路径") + parser.add_argument("--config", "-c", type=str, default="box", help="配置文件名称或路径") + parser.add_argument("--cal", "-cal", type=str, default="", help="标定文件路径") + parser.add_argument( + "--cutoff", + type=float, + default=5.0, + help="力曲线低通滤波截止频率(Hz)，30 Hz 采样时必须小于 15；设为 0 可关闭滤波", + ) parser.add_argument("--verbose", "-verbose", type=bool, default=True, help="是否输出详细日志") args = parser.parse_args() charts = ForceChartWindow() recorder = FrameRecorder() sensor = open_sensor(args.video, args.config, args.cal, args.verbose) + sensor.configure(lowpass_cutoff_hz=args.cutoff, lowpass_sample_rate_hz=30.0) start_time = time.time() print("\n按键说明：'q' 退出程序，'s' 开始录像，'e' 结束录像。") @@ -53,31 +55,30 @@ def main(): sensor.get_img() recorder.write_if_recording(sensor.frame) - sensor.compute_deformation(check_motion=True, threshold=0) - fps, fn, fx, fy, flow, vnormal, vshear = sensor.read_info( + sensor.compute_deformation() + sensor.compute_contact() + fps, fn, fx, fy, flow, depth_map = sensor.read_info( sensor.info.FPS, sensor.info.FNORMAL, sensor.info.FSHEARX, sensor.info.FSHEARY, sensor.info.VRAW, - sensor.info.VNORMAL, - sensor.info.VSHEAR, + sensor.info.DEPTH, ) print(f"FPS={fps:.2f}, 法向力={fn:.4f}, 切向力X={fx:.4f}, 切向力Y={fy:.4f}") arrows = orisys.util.draw_arrows(sensor.img, flow, threshold=5, grid_spacing=10, arrow_scale=1.0) cv2.imshow("形变矢量场", arrows) - cv2.imshow("流场幅值图", draw_magnitude_map(flow)) + if depth_map.max() > depth_map.min(): + depth_normalized = ( + (depth_map - depth_map.min()) / (depth_map.max() - depth_map.min()) * 255 + ).astype(np.uint8) + else: + depth_normalized = np.zeros_like(depth_map, dtype=np.uint8) + cv2.imshow("depth", cv2.applyColorMap(depth_normalized, cv2.COLORMAP_JET)) charts.push(time.time() - start_time, fn, fx, fy) - display = charts.data.copy() - for channel in range(3): - display[:, 1, channel] = lowpass_filter( - charts.data[:, 1, channel], - cutoff_hz=5, - sample_rate_hz=30, - ) - charts.refresh(display) + charts.refresh() key = cv2.waitKey(1) if key & 0xFF == ord("q"): diff --git a/examples/two_sensor_pipeline.py b/examples/two_sensor_pipeline.py index 0bf7c5a..1c5cc67 100644 --- a/examples/two_sensor_pipeline.py +++ b/examples/two_sensor_pipeline.py @@ -18,7 +18,7 @@ - Windows 下必须在 if __name__ == '__main__' 中调用 mp.freeze_support() - 每个传感器进程都会占用独立的 GPU 显存 -SDK version: 0.3.1 +SDK version: 0.5.0 """ import os os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" @@ -138,12 +138,12 @@ def main(): { 'sensor_id': 0, 'vid_src': args.video1, - 'config_name': './config/ddjx01.json' + 'config_name': 'box' }, { 'sensor_id': 1, 'vid_src': args.video2, # 可替换为其他视频源或摄像头索引 - 'config_name': './config/ddjx01.json' + 'config_name': 'box' }, ] diff --git a/examples/viz/mesh_camera.py b/examples/viz/mesh_camera.py index 6dc4f2d..49ad899 100644 --- a/examples/viz/mesh_camera.py +++ b/examples/viz/mesh_camera.py @@ -6,7 +6,38 @@ from pathlib import Path import numpy as np -DEFAULT_FINGER_VIZ_POSE = "assets/finger/visualization/world_to_cam.npy" +DEFAULT_FINGER_VIZ_POSE = "sensors/finger/assets/visualization/world_to_cam.npy" +_LEGACY_FINGER_VIZ_POSES = { + "assets/finger/visualization/world_to_cam.npy", + DEFAULT_FINGER_VIZ_POSE, +} + + +def _finger_bundle_viz_pose_candidates() -> list[Path]: + """Resolve viz pose paths from the packaged sensors/finger bundle when present.""" + try: + from orisys.configs import resolve_sensor_bundle, resolve_sensor_resource + except ImportError: + return [] + + found: list[Path] = [] + bundle = resolve_sensor_bundle("finger") + if bundle is not None and bundle.manifest_json is not None: + try: + import json + + manifest = json.loads(bundle.manifest_json.read_text(encoding="utf-8")) + except Exception: + manifest = {} + rel = manifest.get("viz_pose") + if isinstance(rel, str) and rel.strip(): + found.append((bundle.root / rel).resolve()) + + packaged = resolve_sensor_resource("finger", "assets", "visualization", "world_to_cam.npy") + if packaged is not None: + found.append(packaged.resolve()) + return found + _CAMERA_X_ROT_90 = np.array( [ @@ -28,27 +59,43 @@ _CAMERA_Z_ROT_180 = np.array( def resolve_viz_pose_path(path: str | Path | None = None) -> Path: - """Resolve world_to_cam.npy from cwd or packaged asset layout.""" + """Resolve world_to_cam.npy from sensor bundle, cwd, or packaged layout.""" raw = Path(path or DEFAULT_FINGER_VIZ_POSE).expanduser() if raw.is_file(): return raw.resolve() here = Path(__file__).resolve() - candidates = [ - Path.cwd() / raw, - here.parents[2] / raw, - ] - try: - from app_paths import app_root + candidates: list[Path] = [] - candidates.insert(0, app_root() / raw) + # Prefer the sensors/finger bundle for default / legacy relative paths. + posix = raw.as_posix() + if path is None or posix in _LEGACY_FINGER_VIZ_POSES or not raw.is_absolute(): + candidates.extend(_finger_bundle_viz_pose_candidates()) + + try: + from gui.app_paths import app_root + + candidates.append(app_root() / raw) except ImportError: pass - for candidate in candidates: - if candidate.is_file(): - return candidate.resolve() + candidates.extend( + [ + Path.cwd() / raw, + here.parents[2] / raw, + ] + ) - checked = ", ".join(str(p) for p in candidates) + unique: list[Path] = [] + for candidate in candidates: + resolved = candidate.resolve() + if resolved not in unique: + unique.append(resolved) + + for candidate in unique: + if candidate.is_file(): + return candidate + + checked = ", ".join(str(p) for p in unique) raise FileNotFoundError(f"Visualization pose file not found: {raw}. Checked: {checked}") diff --git a/examples/viz/offscreen_mesh.py b/examples/viz/offscreen_mesh.py index ac7a5de..a7ff151 100644 --- a/examples/viz/offscreen_mesh.py +++ b/examples/viz/offscreen_mesh.py @@ -84,6 +84,168 @@ def _background_rgb01(color: tuple[int, int, int]) -> np.ndarray: return np.asarray(_background_rgba01(color)[:3], dtype=np.float64) +# Cached pre-rendered mesh-axis triads keyed by size_px. +_AXIS_TRIAD_CACHE: dict[int, np.ndarray] = {} + + +def _project_mesh_axis_to_screen(point_xyz: np.ndarray, *, size_px: int) -> tuple[int, int, float]: + """Project mesh coords to HUD pixels. + + Screen mapping for the fixed mesh-axis reference: + +X -> down, +Y -> left, +Z -> inward (into the page). + + Mild foreshortening keeps +Z visible as a short into-page axis. + Returns (px, py, depth) where larger depth is farther from the viewer. + """ + x, y, z = float(point_xyz[0]), float(point_xyz[1]), float(point_xyz[2]) + # Camera looks predominantly along +Z (into page): u right, v down. + # Soft shear so Z has a short foreshortened stub. + shear = 0.34 + u = -y - shear * z + v = x - shear * z + scale = size_px * 0.30 + # Origin slightly lower-right-of-center so labeled tips stay in-frame. + cx = size_px * 0.58 + cy = size_px * 0.48 + return int(round(cx + u * scale)), int(round(cy + v * scale)), z + + +def _render_axis_triad_rgba(size_px: int) -> np.ndarray: + """Build (and cache) a transparent RGBA 3D-looking XYZ triad.""" + import cv2 + + size_px = max(48, int(size_px)) + cached = _AXIS_TRIAD_CACHE.get(size_px) + if cached is not None: + return cached + + rgba = np.zeros((size_px, size_px, 4), dtype=np.uint8) + rgb = rgba[:, :, :3] + alpha = rgba[:, :, 3] + + # Soft circular plate for contrast on bright contact heatmaps. + plate = np.zeros((size_px, size_px), dtype=np.uint8) + center = (size_px // 2, size_px // 2) + radius = int(size_px * 0.46) + cv2.circle(plate, center, radius, 255, thickness=-1, lineType=cv2.LINE_AA) + rgba[:, :, 0:3][plate > 0] = (18, 18, 18) + alpha[plate > 0] = 140 + + thickness = max(2, size_px // 42) + font = cv2.FONT_HERSHEY_SIMPLEX + font_scale = max(0.35, size_px / 220.0) + origin = np.zeros(3, dtype=np.float64) + o_xy = _project_mesh_axis_to_screen(origin, size_px=size_px)[:2] + + axes = ( + ("X", np.array([1.0, 0.0, 0.0]), (230, 70, 70)), + ("Y", np.array([0.0, 1.0, 0.0]), (70, 210, 95)), + ("Z", np.array([0.0, 0.0, 1.0]), (70, 140, 255)), + ) + # Paint far axes first so nearer shafts occlude correctly. + painted = [] + for label, direction, color in axes: + tip = direction * 1.0 + px, py, depth = _project_mesh_axis_to_screen(tip, size_px=size_px) + painted.append((depth, label, color, (px, py))) + painted.sort(key=lambda item: item[0], reverse=True) + + layer = np.zeros((size_px, size_px, 3), dtype=np.uint8) + for _, label, color, tip_xy in painted: + cv2.arrowedLine( + layer, + o_xy, + tip_xy, + color, + thickness, + line_type=cv2.LINE_AA, + tipLength=0.28, + ) + (tw, th), _ = cv2.getTextSize(label, font, font_scale, max(1, thickness - 1)) + # Offset label slightly past the arrow tip, away from origin. + dx = tip_xy[0] - o_xy[0] + dy = tip_xy[1] - o_xy[1] + norm = max(1.0, float(np.hypot(dx, dy))) + lx = int(tip_xy[0] + 8 * dx / norm - tw // 2) + ly = int(tip_xy[1] + 8 * dy / norm + th // 2) + lx = min(max(1, lx), size_px - tw - 1) + ly = min(max(th + 1, ly), size_px - 2) + cv2.putText( + layer, + label, + (lx, ly), + font, + font_scale, + color, + max(1, thickness - 1), + cv2.LINE_AA, + ) + # Origin hub on the same contiguous buffer. + cv2.circle(layer, o_xy, max(2, thickness), (235, 235, 235), -1, lineType=cv2.LINE_AA) + + mask = np.any(layer > 0, axis=2) + rgb[mask] = layer[mask] + alpha[mask] = 255 + + _AXIS_TRIAD_CACHE[size_px] = rgba + return rgba + + +def draw_axis_overlay( + rgb: np.ndarray, + *, + enabled: bool = True, + padding_px: int = 12, + size_px: int = 96, + opacity: float = 1.0, +) -> np.ndarray: + """Composite a fixed top-right mesh-axis triad onto an RGB frame. + + The triad is pre-rendered and cached; orientation is locked to true mesh + axes (+X down, +Y left, +Z inward) and does not follow free-rotate. + """ + if not enabled or rgb is None or rgb.size == 0: + return rgb + + out = np.ascontiguousarray(rgb) + if out.ndim != 3 or out.shape[2] < 3: + return out + + triad = _render_axis_triad_rgba(size_px) + th, tw = triad.shape[:2] + pad = max(0, int(padding_px)) + h, w = out.shape[:2] + x0 = max(0, w - tw - pad) + y0 = max(0, pad) + x1 = min(w, x0 + tw) + y1 = min(h, y0 + th) + if x1 <= x0 or y1 <= y0: + return out + + roi = out[y0:y1, x0:x1] + overlay = triad[: y1 - y0, : x1 - x0] + alpha = (overlay[:, :, 3].astype(np.float32) / 255.0) * float(np.clip(opacity, 0.0, 1.0)) + if alpha.ndim == 2: + alpha = alpha[:, :, None] + rgb_o = overlay[:, :, :3].astype(np.float32) + base = roi.astype(np.float32) + blended = rgb_o * alpha + base * (1.0 - alpha) + roi[:] = np.clip(blended, 0, 255).astype(np.uint8) + return out + + +def _apply_axis_overlay(rgb: np.ndarray, axis_overlay) -> np.ndarray: + if axis_overlay is None: + return draw_axis_overlay(rgb) + return draw_axis_overlay( + rgb, + enabled=bool(getattr(axis_overlay, "enabled", True)), + padding_px=int(getattr(axis_overlay, "padding_px", 12)), + size_px=int(getattr(axis_overlay, "size_px", 96)), + opacity=float(getattr(axis_overlay, "opacity", 1.0)), + ) + + class _FilamentOffscreenBackend: """Filament OffscreenRenderer (Linux headless / platforms with EGL).""" @@ -104,6 +266,7 @@ class _FilamentOffscreenBackend: mesh_background: tuple[int, int, int] = (0, 0, 0), contact_arrow_style=None, flow_arrow_style=None, + axis_overlay=None, ): import open3d as o3d @@ -119,6 +282,7 @@ class _FilamentOffscreenBackend: self.lookat = lookat self.mesh_lighting = mesh_lighting self.mesh_background = mesh_background + self.axis_overlay = axis_overlay self._shade_factors = ( precompute_shade_factors(mesh.rest_vertices, mesh.triangles, mesh_lighting) if mesh_lighting is not None @@ -257,7 +421,8 @@ class _FilamentOffscreenBackend: mesh_to_o3d=lambda v, t, c: _mesh_to_o3d(o3d, v, t, c), ) - return _numpy_o3d_image_to_rgb(self._renderer.render_to_image()) + rgb = _numpy_o3d_image_to_rgb(self._renderer.render_to_image()) + return _apply_axis_overlay(rgb, self.axis_overlay) def close(self) -> None: self._renderer = None @@ -283,6 +448,7 @@ class _HiddenVisualizerBackend: mesh_background: tuple[int, int, int] = (0, 0, 0), contact_arrow_style=None, flow_arrow_style=None, + axis_overlay=None, ): import open3d as o3d @@ -298,6 +464,7 @@ class _HiddenVisualizerBackend: self.lookat = lookat self.mesh_lighting = mesh_lighting self.mesh_background = mesh_background + self.axis_overlay = axis_overlay self._shade_factors = ( precompute_shade_factors(mesh.rest_vertices, mesh.triangles, mesh_lighting) if mesh_lighting is not None @@ -441,7 +608,8 @@ class _HiddenVisualizerBackend: self.vis.poll_events() self.vis.update_renderer() - return _numpy_o3d_image_to_rgb(self.vis.capture_screen_float_buffer(do_render=False)) + rgb = _numpy_o3d_image_to_rgb(self.vis.capture_screen_float_buffer(do_render=False)) + return _apply_axis_overlay(rgb, self.axis_overlay) def close(self) -> None: if self.vis is not None: @@ -478,6 +646,7 @@ class OffscreenFingerMeshRenderer: mesh_background: tuple[int, int, int] = (0, 0, 0), contact_arrow_style=None, flow_arrow_style=None, + axis_overlay=None, ): lookat = lookat if lookat is not None else mesh.rest_vertices.mean(axis=0) self._backend = _create_backend( @@ -495,6 +664,7 @@ class OffscreenFingerMeshRenderer: mesh_background=mesh_background, contact_arrow_style=contact_arrow_style, flow_arrow_style=flow_arrow_style, + axis_overlay=axis_overlay, ) @property diff --git a/pyproject.toml b/pyproject.toml index a3a204a..acbd128 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,27 +1,55 @@ [build-system] -requires = ["setuptools>=61.0", "wheel"] +requires = ["setuptools>=61.0,<80", "wheel==0.45.1"] build-backend = "setuptools.build_meta" [project] name = "orisys" -description = "Orisys - Vision Based Tactile Sensor SDK" +description = "SDK for Orisys Technology Visuotactile Sensors" readme = "README.md" requires-python = ">=3.10" -authors = [{ name = "Orisys" }] +license = { text = "Proprietary" } +authors = [{ name = "Orisys-Tech" }] +maintainers = [ + {name = "Yijian Zhang", email = "zhangyijian@orisys-tech.cn"} +] dynamic = ["version"] dependencies = [ "numpy>=1.24.4", "opencv-python>=4.10.0.84", "scipy>=1.10.1", - "open3d>=0.17.0", + "numba>=0.59.0", ] +classifiers = [ + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 4 - Beta", + + # Indicate who your project is intended for + "Intended Audience :: Developers", + "Topic :: Software Development :: Libraries :: Python Modules", + "License :: Other/Proprietary License", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + + # Specify the Python versions you support here. + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] + +[project.urls] +Homepage = "https://www.orisys-tech.cn" +Documentation = "https://github.com/Orisys-Tech/orisys-sdk" +Issues = "https://github.com/Orisys-Tech/orisys-sdk/issues" [project.optional-dependencies] cuda12 = ["cupy-cuda12x>=12.3.0"] cuda13 = ["cupy-cuda13x>=13.0.0"] +gui = ["open3d>=0.17.0","pyqt5>=5.15", "pyqtgraph>=0.13"] # Backward compatibility: 'cuda' defaults to CUDA 12.x cuda = ["cupy-cuda12x>=12.3.0"] -gui = ["pyqt5>=5.15", "pyqtgraph>=0.13"] [tool.setuptools] package-dir = { "" = "src" } @@ -35,7 +63,7 @@ version = { attr = "orisys.__version__" } # Single source of truth for bundled non-code files. # - dev_install = true -> pip install / editable install (via setup.py) -# - all items -> Nuitka wheel (build/wheel_layout.py) +# - all items -> Nuitka wheel (build/wheel/layout.py) [tool.orisys.bundle] # Use patterns (globs) and/or files (explicit names under source/). @@ -47,11 +75,28 @@ patterns = ["*.json"] # files = ["dd02-ov.json", "ddjx01.json"] dev_install = true +[[tool.orisys.bundle.items]] +name = "sensors" +source = "sensors" +target = "sensors" +# JSON bundles (sensor.json, force_calibration.json) plus finger mesh/viz assets. +# Prefer the flat layout sensors/finger/assets/{30fps-*,visualization}/; +# also accept a nested assets/finger/ tree if present in a checkout. +patterns = [ + "**/*.json", + "finger/assets/30fps*/*", + "finger/assets/visualization/*", + "finger/assets/finger/30fps*/*", + "finger/assets/finger/visualization/*", +] +preserve_tree = true +dev_install = true + [[tool.orisys.bundle.items]] name = "examples" source = "examples" target = "examples" -patterns = ["*.py", "README.md", "app_paths.py", "viz/**/*.py", "pipeline/**/*.py", "gui/**/*.py"] +patterns = ["*.py", "README.md", "viz/**/*.py", "pipeline/**/*.py", "gui/**/*.py"] preserve_tree = true [[tool.orisys.bundle.items]] @@ -60,11 +105,3 @@ source = "docs" target = "docs" patterns = ["*.md", "*.pdf"] -[[tool.orisys.bundle.items]] -name = "assets" -source = "assets" -target = "assets" -patterns = ["finger/30fps/*", "finger/visualization/*"] -preserve_tree = true -dev_install = true - diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..029fccc --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,14 @@ +# CuPy - Install the appropriate version based on your CUDA version: +# For CUDA 12.x: pip install cupy-cuda12x>=12.3.0 +# For CUDA 13.x: pip install cupy-cuda13x>=13.0.0 +# Or use extras: pip install orisys[cuda12] or pip install orisys[cuda13] + +numpy>=1.24.4 +opencv_python>=4.10.0.84 +scipy==1.13 +setuptools>=45.2.0 +av +numba +pyqt5 +pyqtgraph +nuitka \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e95cff8..bb5f7e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,27 @@ -# CuPy - Install the appropriate version based on your CUDA version: -# For CUDA 12.x: pip install cupy-cuda12x>=12.3.0 -# For CUDA 13.x: pip install cupy-cuda13x>=13.0.0 -# Or use extras: pip install orisys[cuda12] or pip install orisys[cuda13] +# Runtime dependencies for editable / source installs. +# Prefer extras from pyproject.toml: +# pip install -e . +# pip install -e ".[gui]" +# pip install -e ".[cuda12]" +# pip install -e ".[cuda13]" +# pip install -e ".[gui,cuda12]" +# PyPI users: pip install orisys / "orisys[gui]" / "orisys[cuda12]" +# Core (same as [project.dependencies]) numpy>=1.24.4 -opencv_python>=4.10.0.84 -scipy==1.13 -setuptools>=45.2.0 -av -numba +opencv-python>=4.10.0.84 +scipy>=1.10.1 +numba>=0.59.0 +# GUI extras: pip install -e ".[gui]" +# open3d>=0.17.0 +# pyqt5>=5.15 +# pyqtgraph>=0.13 + +# CUDA extras (pick one CUDA major version) +# CUDA 12.x: pip install -e ".[cuda12]" -> cupy-cuda12x>=12.3.0 +# CUDA 13.x: pip install -e ".[cuda13]" -> cupy-cuda13x>=13.0.0 +# Alias: pip install -e ".[cuda]" -> same as cuda12 + +# Optional video tooling (not an orisys extra) +# av diff --git a/scripts/calibration_script.py b/scripts/calibration_script.py index f8d85ac..87955b3 100644 --- a/scripts/calibration_script.py +++ b/scripts/calibration_script.py @@ -115,7 +115,7 @@ def main(): writer.write(save_img) - sensor1.compute_deformation(check_motion=False) # 计算形变和分解 + sensor1.compute_deformation() # 计算形变和分解 # 提取信息： # 向量场: VRAW, VNORMAL, VSHEAR: 原始，法向，切向 # 帧率，时间戳: FPS, TIMESTAMP diff --git a/scripts/demo_wuxi.py b/scripts/demo_wuxi.py deleted file mode 100644 index d6cfe1c..0000000 --- a/scripts/demo_wuxi.py +++ /dev/null @@ -1,961 +0,0 @@ -import os -os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" -# Force pyqtgraph to bind to PyQt5 when multiple Qt bindings are installed. -os.environ.setdefault("PYQTGRAPH_QT_LIB", "PyQt5") -import cv2 -import orisys -import numpy as np -import time -import argparse -from datetime import datetime -import pyqtgraph as pg -from PyQt5.QtCore import Qt, QSize, QEvent -from PyQt5.QtGui import QImage, QFont, QPixmap, QKeySequence, QCursor -from PyQt5.QtWidgets import QLabel, QHBoxLayout, QVBoxLayout, QWidget, QSizePolicy, QShortcut, QApplication -from pyqtgraph.Qt import QtGui - - -# Final desired appearance on the mirrored 1872x1560 (6:5) display. -# The desktop is first shown on a 1920x1080 (16:9) screen, then stretched by -# the OS onto the 6:5 mirror. We therefore pre-distort the source layout so -# the mirrored output looks correct. -TARGET_SINGLE_PANEL_W = 936 -TARGET_ASPECT_W = TARGET_SINGLE_PANEL_W * 2 -TARGET_VIDEO_FRAME_H = 1053 -TARGET_BOTTOM_PANEL_H = 507 -TARGET_ASPECT_H_TOTAL = TARGET_VIDEO_FRAME_H + TARGET_BOTTOM_PANEL_H -TARGET_MIRROR_W = 1872 -TARGET_MIRROR_H = 1560 - -SINGLE_PANEL_W = TARGET_SINGLE_PANEL_W -ASPECT_W = TARGET_ASPECT_W -BOTTOM_PANEL_H = TARGET_BOTTOM_PANEL_H -VIDEO_FRAME_W = TARGET_ASPECT_W -VIDEO_FRAME_H = TARGET_VIDEO_FRAME_H -ASPECT_H_TOTAL = TARGET_ASPECT_H_TOTAL -UI_SCALE = 1.0 -ARROW_STRETCH = 507 -PLOT_STRETCH = 429 -VIDEO_LOOP_PATH = "./叠动机械爪_成片.m4v" -npoints = 120 # 图表数据长度 -dispaly_angle = -90 # 箭头图显示角度 -touch_threshold = 5 # 触碰反应阈值 -frame_to_recover = 10 # 触碰后等待回复帧数 - -# Opneing Hour for keep reading image -open_hour = 9 -close_hour = 17 -manual_open_enable = True # 控制是否可以手动切换营业状态 （按键G) -is_open = True - -# Match arrow panel / Qt style (not pure black) -PLOT_THEME_BG = "#0B1016" -PLOT_THEME_FG = "#A0A0A0" - -# OS title bar cannot be tinted to #0B1016; frameless removes it (use Alt+drag to move). -USE_FRAMELESS_WINDOW = False - - -def scale_ui_px(px, floor=1): - return max(floor, int(round(px * UI_SCALE))) - - -def compute_source_layout(screen_w, screen_h): - """Pre-distort the app on the source screen so the mirrored monitor is correct.""" - target_video_ratio = TARGET_VIDEO_FRAME_H / float(TARGET_ASPECT_H_TOTAL) - target_bottom_ratio = TARGET_BOTTOM_PANEL_H / float(TARGET_ASPECT_H_TOTAL) - - return { - "window_w": max(320, int(screen_w)), - "window_h": max(240, int(screen_h)), - "video_w": max(320, int(screen_w)), - "video_h": max(1, int(round(screen_h * target_video_ratio))), - "bottom_h": max(1, int(round(screen_h * target_bottom_ratio))), - "single_panel_w": max(1, int(round(screen_w / 2.0))), - } - - -class CombinedTactileWindow(QWidget): - """ - Resizable window that keeps width:height = ASPECT_W : ASPECT_H_TOTAL. - Dragging the width or height edge updates the other dimension accordingly. - """ - - RESIZE_MARGIN = 8 - - def __init__(self): - super().__init__() - self._aspect_hw = ASPECT_H_TOTAL / float(ASPECT_W) - self._resize_guard = False - self.user_closed = False - self._drag_anchor = None - self._frameless_resize_edge = None - self._frameless_resize_start_global = None - self._frameless_resize_start_geom = None - - def _min_client_size(self): - w = max(320, self.minimumSizeHint().width()) - h = int(round(w * self._aspect_hw)) - return w, h - - def _hit_test_resize(self, global_pos): - if not USE_FRAMELESS_WINDOW: - return None - p = self.mapFromGlobal(global_pos) - w, h = self.width(), self.height() - m = self.RESIZE_MARGIN - left = p.x() <= m - right = p.x() >= w - m - top = p.y() <= m - bottom = p.y() >= h - m - if top and left: - return "tl" - if top and right: - return "tr" - if bottom and left: - return "bl" - if bottom and right: - return "br" - if top: - return "t" - if bottom: - return "b" - if left: - return "l" - if right: - return "r" - return None - - def _cursor_for_edge(self, edge): - return { - "tl": Qt.SizeFDiagCursor, - "br": Qt.SizeFDiagCursor, - "tr": Qt.SizeBDiagCursor, - "bl": Qt.SizeBDiagCursor, - "l": Qt.SizeHorCursor, - "r": Qt.SizeHorCursor, - "t": Qt.SizeVerCursor, - "b": Qt.SizeVerCursor, - }.get(edge, Qt.ArrowCursor) - - def _apply_frameless_resize(self, dg): - edge = self._frameless_resize_edge - g0 = self._frameless_resize_start_geom - min_w, min_h = self._min_client_size() - dx, dy = dg.x(), dg.y() - x0, y0, w0, h0 = g0.x(), g0.y(), g0.width(), g0.height() - - if edge == "br": - nw = max(min_w, w0 + dx) - nh = int(round(nw * self._aspect_hw)) - self.setGeometry(x0, y0, nw, nh) - elif edge == "r": - nw = max(min_w, w0 + dx) - nh = int(round(nw * self._aspect_hw)) - self.setGeometry(x0, y0, nw, nh) - elif edge == "b": - nh = max(min_h, h0 + dy) - nw = int(round(nh / self._aspect_hw)) - self.setGeometry(x0, y0, nw, nh) - elif edge == "l": - nw = max(min_w, w0 - dx) - nh = int(round(nw * self._aspect_hw)) - nx = g0.right() - nw - self.setGeometry(nx, y0, nw, nh) - elif edge == "t": - nh = max(min_h, h0 - dy) - nw = int(round(nh / self._aspect_hw)) - ny = g0.bottom() - nh - self.setGeometry(x0, ny, nw, nh) - elif edge == "tr": - nw = max(min_w, w0 + dx) - nh = int(round(nw * self._aspect_hw)) - ny = g0.bottom() - nh - self.setGeometry(x0, ny, nw, nh) - elif edge == "bl": - nw = max(min_w, w0 - dx) - nh = int(round(nw * self._aspect_hw)) - nx = g0.right() - nw - self.setGeometry(nx, y0, nw, nh) - elif edge == "tl": - nw = max(min_w, w0 - dx) - nh = int(round(nw * self._aspect_hw)) - nx = g0.right() - nw - ny = g0.bottom() - nh - self.setGeometry(nx, ny, nw, nh) - - def install_drag_filters(self): - """Alt+left-drag to move when frameless; grabs go to this widget.""" - for w in self.findChildren(QWidget): - if w is not self: - w.installEventFilter(self) - - def eventFilter(self, watched, event): - if not USE_FRAMELESS_WINDOW: - return super().eventFilter(watched, event) - et = event.type() - if et == QEvent.MouseButtonPress and event.button() == Qt.LeftButton: - edge = self._hit_test_resize(event.globalPos()) - if edge is not None: - self._frameless_resize_edge = edge - self._frameless_resize_start_global = event.globalPos() - self._frameless_resize_start_geom = self.geometry() - self.grabMouse() - return True - if QApplication.keyboardModifiers() & Qt.AltModifier: - self._drag_anchor = event.globalPos() - self.frameGeometry().topLeft() - self.grabMouse() - return True - if et == QEvent.MouseMove and not event.buttons(): - edge = self._hit_test_resize(event.globalPos()) - watched.setCursor(QCursor(self._cursor_for_edge(edge))) - return super().eventFilter(watched, event) - - def mouseMoveEvent(self, event): - if USE_FRAMELESS_WINDOW and self._frameless_resize_edge and event.buttons() & Qt.LeftButton: - dg = event.globalPos() - self._frameless_resize_start_global - self._apply_frameless_resize(dg) - event.accept() - return - if ( - USE_FRAMELESS_WINDOW - and self._drag_anchor is not None - and event.buttons() & Qt.LeftButton - ): - self.move(event.globalPos() - self._drag_anchor) - event.accept() - return - super().mouseMoveEvent(event) - - def mouseReleaseEvent(self, event): - if USE_FRAMELESS_WINDOW and self._frameless_resize_edge and event.button() == Qt.LeftButton: - self.releaseMouse() - self._frameless_resize_edge = None - self._frameless_resize_start_global = None - self._frameless_resize_start_geom = None - event.accept() - return - if USE_FRAMELESS_WINDOW and self._drag_anchor is not None and event.button() == Qt.LeftButton: - self.releaseMouse() - self._drag_anchor = None - event.accept() - return - super().mouseReleaseEvent(event) - - def closeEvent(self, event): - self.user_closed = True - event.accept() - - def minimumSizeHint(self): - w = 320 - return QSize(w, int(round(w * self._aspect_hw))) - - def resizeEvent(self, event): - if self._resize_guard: - super().resizeEvent(event) - return - super().resizeEvent(event) - w, h = self.width(), self.height() - if w < 8 or h < 8: - return - old = event.oldSize() - self._resize_guard = True - try: - if old.isValid() and old.width() > 0 and old.height() > 0: - dw = abs(w - old.width()) - dh = abs(h - old.height()) - if dw >= dh: - new_w, new_h = w, int(round(w * self._aspect_hw)) - else: - new_w, new_h = int(round(h / self._aspect_hw)), h - else: - new_w, new_h = w, int(round(w * self._aspect_hw)) - if new_w != w or new_h != h: - self.resize(new_w, new_h) - finally: - self._resize_guard = False - - -def compute_arrow_canvas_side(panel_widget): - """ - Pixel size of the square arrow canvas from the window client area and the - fixed horizontal split ARROW_STRETCH : PLOT_STRETCH (matches QHBoxLayout stretch). - """ - w = panel_widget.width() - h = panel_widget.height() - if w < 16 or h < 16: - return max(32, ASPECT_H_TOTAL) - w_arrow = int(round(w * ARROW_STRETCH / float(ARROW_STRETCH + PLOT_STRETCH))) - # Keep the rendered arrow image square inside the left panel. - return max(32, min(h, w_arrow)) - - -def draw_arrows_wuxi( - frame, - optical_flow, - fps=True, - threshold=5, - grid_spacing=40, - arrow_scale=1.0, - margin=0.8, - colormap=cv2.COLORMAP_PARULA, - below_threshold_color=(0, 0, 255), - edge_margin=15, - target_size=936, -): - """ - Draw colored optical flow arrows on the image by magnitude (JET colormap). - Resizes frame and flow to target_size x target_size; scales flow vectors and - grid_spacing / margins / threshold so arrow density matches the original resolution. - """ - image = frame.copy() - if len(image.shape) == 2 or image.shape[2] == 1: - image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) - - h0, w0 = image.shape[:2] - if h0 != target_size or w0 != target_size: - sx = target_size / float(w0) - sy = target_size / float(h0) - image = cv2.resize(image, (target_size, target_size), interpolation=cv2.INTER_LINEAR) - u = cv2.resize(optical_flow[:, :, 0], (target_size, target_size), interpolation=cv2.INTER_LINEAR) * sx - v = cv2.resize(optical_flow[:, :, 1], (target_size, target_size), interpolation=cv2.INTER_LINEAR) * sy - optical_flow = np.stack([u, v], axis=2) - scale = max(sx, sy) - grid_spacing = max(4, int(round(grid_spacing * scale))) - edge_margin = max(1, int(round(edge_margin * scale))) - threshold = float(threshold) * scale - - h, w = image.shape[:2] - # Arrow grid in pixel coordinates (assume frame and flow same size) - rr = np.arange(grid_spacing // 2, h, grid_spacing) - cc = np.arange(grid_spacing // 2, w, grid_spacing) - rr, cc = np.meshgrid(rr, cc, indexing="ij") - rr, cc = rr.reshape(-1), cc.reshape(-1) - - # Displacement at each grid point (flow stored as half-displacement) - disp = 2.0 * optical_flow[rr, cc, :] - mag = np.linalg.norm(disp, axis=1) - dx = disp[:, 0] / arrow_scale - dy = disp[:, 1] / arrow_scale - length = np.sqrt(dx * dx + dy * dy) - # Cap arrow length at grid_spacing - scale = np.where(length > grid_spacing*margin, grid_spacing / np.maximum(length, 1e-9), 1.0)*margin - dx, dy = dx * scale, dy * scale - end_x = (cc + dx).astype(np.int32) - end_y = (rr + dy).astype(np.int32) - - valid = ( - (cc >= edge_margin) & (cc < w - edge_margin) & (rr >= edge_margin) & (rr < h - edge_margin) - & (end_x >= 0) & (end_x < w) & (end_y >= 0) & (end_y < h) - ) - mag_range = mag.max() - mag.min() - norm_mag = ((mag - mag.min()) / mag_range * 255).astype(np.uint8) if mag_range > 0 else np.zeros_like(mag, dtype=np.uint8) - norm_mag[mag < threshold] = 0 - colors = cv2.applyColorMap(norm_mag.reshape(-1, 1), colormap) - - for i in np.where(valid)[0]: - pt1, pt2 = (int(cc[i]), int(rr[i])), (int(end_x[i]), int(end_y[i])) - color = tuple(int(c) for c in colors[i, 0]) - if mag[i] < threshold: - cv2.circle(image, pt1, radius=1, color=below_threshold_color, thickness=-1) - else: - # thickness = max(1, int(2 * mag[i] / mag.max())) if mag.max() > 0 else 2 - thickness = 1 - cv2.arrowedLine(image, pt1, pt2, color, thickness=thickness, tipLength=0.2) - return image - - -def set_arrow_label_bgr(label, bgr_image): - """Show a BGR OpenCV image in a QLabel using source-screen pre-distortion.""" - rgb = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) - rgb = np.ascontiguousarray(rgb) - h, w, ch = rgb.shape - qimg = QImage(rgb.data, w, h, ch * w, QImage.Format_RGB888).copy() - pm = QPixmap.fromImage(qimg) - lw, lh = label.width(), label.height() - if lw > 0 and lh > 0 and (pm.width() != lw or pm.height() != lh): - pm = pm.scaled(lw, lh, Qt.IgnoreAspectRatio, Qt.SmoothTransformation) - label.setPixmap(pm) - -def set_video_label_bgr(label, bgr_image): - """Show loop video in the top frame using the fitted display size.""" - rgb = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) - rgb = np.ascontiguousarray(rgb) - h, w, ch = rgb.shape - qimg = QImage(rgb.data, w, h, ch * w, QImage.Format_RGB888).copy() - pm = QPixmap.fromImage(qimg) - lw, lh = label.width(), label.height() - if lw > 0 and lh > 0 and (pm.width() != lw or pm.height() != lh): - pm = pm.scaled(lw, lh, Qt.IgnoreAspectRatio, Qt.FastTransformation) - label.setPixmap(pm) - - -def read_loop_video_frame_realtime(video_cap, video_clock): - """Read loop-video frame paced to wall clock; skip frames when main loop is slow.""" - if video_cap is None or not video_cap.isOpened(): - return False, None - - now = time.perf_counter() - fps = max(1.0, float(video_clock.get("fps", 30.0))) - frame_dt = 1.0 / fps - next_ts = video_clock.get("next_ts") - if next_ts is None: - next_ts = now - - frames_due = 1 - if now > next_ts + frame_dt: - frames_due += int((now - next_ts) / frame_dt) - # Cap skip burst to avoid long catch-up loops after pauses. - frames_due = min(frames_due, int(max(2.0, fps))) - - for _ in range(frames_due - 1): - if not video_cap.grab(): - video_cap.set(cv2.CAP_PROP_POS_FRAMES, 0) - if not video_cap.grab(): - return False, None - - vret, vframe = video_cap.read() - if not vret: - video_cap.set(cv2.CAP_PROP_POS_FRAMES, 0) - vret, vframe = video_cap.read() - if not vret: - return False, None - - next_ts = next_ts + frames_due * frame_dt - if next_ts < now - 2.0 * frame_dt: - next_ts = now - video_clock["next_ts"] = next_ts - return True, vframe - - -def make_gradient_brush(color_hex, max_alpha=150): - """ - 创建一个从上到下渐变的笔刷 (兼容 PyQt5 和 PyQt6) - """ - c_top = QtGui.QColor(color_hex) - c_top.setAlpha(max_alpha) # 顶部半透明 - - c_bot = QtGui.QColor(color_hex) - c_bot.setAlpha(0) # 底部全透明 - - # 创建线性渐变：(0,0)到(0,1) 代表从控件顶部到底部 - grad = QtGui.QLinearGradient(0, 0, 0, 1) - - if hasattr(QtGui.QGradient, 'CoordinateMode'): - grad.setCoordinateMode(QtGui.QGradient.CoordinateMode.ObjectBoundingMode) - else: - grad.setCoordinateMode(QtGui.QGradient.ObjectBoundingMode) - # ============================================== - - grad.setColorAt(0, c_top) # 0.0 (顶端) 颜色 - grad.setColorAt(1, c_bot) # 1.0 (底端) 颜色 - - return QtGui.QBrush(grad) - -def create_plot(plot_container): - pg.setConfigOption("background", PLOT_THEME_BG) - pg.setConfigOption("foreground", PLOT_THEME_FG) - pg.setConfigOptions(antialias=True) # 抗锯齿 - theme_color = '#00E5FF' # '#00E5FF' - line_pen = pg.mkPen(color=theme_color, width=2) - grad_brush = make_gradient_brush(theme_color, max_alpha=120) - - axis_font = QFont("Segoe UI", scale_ui_px(11, floor=8)) - p1 = plot_container.addPlot() - p1.setLabel('left', 'N') - p1.getAxis('left').label.setFont(axis_font) - p1.getAxis('left').label.setRotation(dispaly_angle) - p1.showGrid(x=True, y=True, alpha=0.3) - plot_container.nextRow() - p2 = plot_container.addPlot() - p2.setLabel('left', 'X') - p2.getAxis('left').label.setFont(axis_font) - p2.getAxis('left').label.setRotation(dispaly_angle) - p2.showGrid(x=True, y=True, alpha=0.3) - plot_container.nextRow() - p3 = plot_container.addPlot() - p3.setLabel('left', 'Y') - - # p3.setLabel('bottom', 'Time', units='s') - p3.getAxis('left').label.setFont(axis_font) - p3.getAxis('left').label.setRotation(dispaly_angle) - p3.getAxis('bottom').label.setFont(axis_font) - p3.showGrid(x=True, y=True, alpha=0.3) - _plot_bg = pg.mkColor(PLOT_THEME_BG) - for p in (p1, p2, p3): - p.getAxis("bottom").setStyle(showValues=False) - p.getAxis("left").setStyle(showValues=False) - # Keep title as "N" / "X" / "Y" — no PyQtGraph "(x0.001)" suffix on small ranges. - p.getAxis("left").enableAutoSIPrefix(False) - # PlotItem.setBackground exists only in newer pyqtgraph; ViewBox works on older installs. - p.vb.setBackgroundColor(_plot_bg) - plot_af = p1.plot(pen=line_pen, fillLevel=0, fillBrush=grad_brush) - plot_tfx = p2.plot(pen=line_pen, fillLevel=0, fillBrush=grad_brush) - plot_tfy = p3.plot(pen=line_pen, fillLevel=0, fillBrush=grad_brush) - return p1, p2, p3, plot_af, plot_tfx, plot_tfy - -def _plot_y_range(y): - """Avoid degenerate Y span; explicit x/y in setData so older PyQtGraph never plots time as y.""" - mn = float(np.min(y)) - mx = float(np.max(y)) - if not (np.isfinite(mn) and np.isfinite(mx)): - return 0.0, 1.0 - if mx - mn <= 1e-15: - mid = 0.5 * (mn + mx) - return mid - 0.5, mid + 0.5 - pad = (mx - mn) * 0.05 - return mn - pad, mx + pad - - -def update(plot_af, plot_tfx, plot_tfy, data, p1, p2, p3): - # global plot_af, plot_tfx, plot_tfy, data, p1, p2, p3 - plot_af.setData(data[:, 0, 0], data[:, 1, 0]) - p1.setYRange(*_plot_y_range(data[:, 1, 0])) - plot_tfx.setData(data[:, 0, 1], data[:, 1, 1]) - p2.setYRange(*_plot_y_range(data[:, 1, 1])) - plot_tfy.setData(data[:, 0, 2], data[:, 1, 2]) - p3.setYRange(*_plot_y_range(data[:, 1, 2])) - - -def build_sensor_panel(parent_layout): - panel = QWidget() - panel_layout = QHBoxLayout(panel) - panel_layout.setContentsMargins(0, 0, 0, 0) - panel_layout.setSpacing(0) - - # Ignored: do not let pixmap/content drive QLabel size — layout stretch sets arrow vs plot widths. - _policy_layout = QSizePolicy(QSizePolicy.Ignored, QSizePolicy.Ignored) - - arrow_label = QLabel() - arrow_label.setAlignment(Qt.AlignHCenter | Qt.AlignVCenter) - arrow_label.setStyleSheet( - "background-color: #0B1016; margin: 0; padding: 0; border: none;" - ) - arrow_label.setSizePolicy(_policy_layout) - arrow_label.setMinimumSize(0, 0) - arrow_label.setScaledContents(False) - panel_layout.addWidget(arrow_label, stretch=ARROW_STRETCH) - - plot_widget = pg.GraphicsLayoutWidget() - plot_widget.setBackground(PLOT_THEME_BG) - plot_widget.setSizePolicy(_policy_layout) - plot_widget.setMinimumSize(0, 0) - panel_layout.addWidget(plot_widget, stretch=PLOT_STRETCH) - - disconnected_label = QLabel("已断开") - disconnected_label.setAlignment(Qt.AlignCenter) - disconnected_label.setStyleSheet( - f"background-color: #0B1016; color: #FF4D4D; font-size: {scale_ui_px(54, floor=24)}px; " - "font-weight: bold; letter-spacing: 2px; margin: 0; padding: 0; border: none;" - ) - disconnected_label.hide() - panel_layout.addWidget(disconnected_label, stretch=SINGLE_PANEL_W) - - parent_layout.addWidget(panel, stretch=1) - - p1, p2, p3, plot_af, plot_tfx, plot_tfy = create_plot(plot_widget) - data = np.zeros([npoints, 2, 3]) - for i in range(3): - data[:, 0, i] = np.linspace(0.0, npoints / 30.0, npoints) - - return { - "panel": panel, - "arrow_label": arrow_label, - "plot_widget": plot_widget, - "disconnected_label": disconnected_label, - "p1": p1, - "p2": p2, - "p3": p3, - "plot_af": plot_af, - "plot_tfx": plot_tfx, - "plot_tfy": plot_tfy, - "data": data, - } - - -# 实时图表：单窗口，两套并排（左 sensor1，右 sensor2） -pg.setConfigOption("background", PLOT_THEME_BG) -pg.setConfigOption("foreground", PLOT_THEME_FG) -app = pg.mkQApp("Wuxi Demo") -screen = app.primaryScreen() -screen_geometry = screen.availableGeometry() if screen is not None else None -screen_w = screen_geometry.width() if screen_geometry is not None else 1920 -screen_h = screen_geometry.height() if screen_geometry is not None else 1080 -source_layout = compute_source_layout(screen_w, screen_h) -ASPECT_W = source_layout["window_w"] -ASPECT_H_TOTAL = source_layout["window_h"] -VIDEO_FRAME_W = source_layout["video_w"] -VIDEO_FRAME_H = source_layout["video_h"] -BOTTOM_PANEL_H = source_layout["bottom_h"] -SINGLE_PANEL_W = source_layout["single_panel_w"] -UI_SCALE = max(0.1, screen_h / float(TARGET_ASPECT_H_TOTAL)) - -combined_win = CombinedTactileWindow() -combined_win.setWindowTitle("Wuxi tactile — dual flow + forces") -main_layout = QVBoxLayout(combined_win) -main_layout.setContentsMargins(0, 0, 0, 0) -main_layout.setSpacing(0) - -video_label = QLabel() -video_label.setAlignment(Qt.AlignCenter) -video_label.setStyleSheet("background-color: #000000; margin: 0; padding: 0; border: none;") -video_label.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding) -video_label.setMinimumSize(0, 0) -main_layout.addWidget(video_label, stretch=VIDEO_FRAME_H) - -bottom_container = QWidget() -bottom_layout = QHBoxLayout(bottom_container) -bottom_layout.setContentsMargins(0, 0, 0, 0) -bottom_layout.setSpacing(0) -main_layout.addWidget(bottom_container, stretch=BOTTOM_PANEL_H) - -panel1_ui = build_sensor_panel(bottom_layout) -panel2_ui = build_sensor_panel(bottom_layout) - -if USE_FRAMELESS_WINDOW: - combined_win.setWindowFlags(Qt.Window | Qt.FramelessWindowHint) -# Client area matches theme (no bright border flash on frameless) -combined_win.setStyleSheet(f"background-color: {PLOT_THEME_BG};") -combined_win.resize(ASPECT_W, ASPECT_H_TOTAL) -combined_win.install_drag_filters() -combined_win.showFullScreen() -closed_hour_overlay = QLabel(combined_win) -closed_hour_overlay.setAlignment(Qt.AlignCenter) -closed_hour_overlay.setStyleSheet( - f"background-color: {PLOT_THEME_BG}; color: #A0A0A0; " - f"font-size: {scale_ui_px(72, floor=32)}px; font-weight: bold; " - "margin: 0; padding: 0; border: none;" -) -closed_hour_overlay.setText("Closed hour\n非营业时间") -closed_hour_overlay.hide() -start_time = None # Will be set when sensor starts -fps = 30.0 - - -def outside_opening_hours(now): - return now.hour < open_hour or now.hour >= close_hour - - - -def main(): - global start_time - - parser = argparse.ArgumentParser() - parser.add_argument("--video1","-v1", type=str, default="0", help="视频文件路径") - parser.add_argument("--video2","-v2", type=str, default="0", help="视频文件路径") - args = parser.parse_args() - - session_start = datetime.now() - print( - f"[session] START {session_start:%Y-%m-%d %H:%M:%S} (long-run / crash log marker)", - flush=True, - ) - if USE_FRAMELESS_WINDOW: - print( - "[window] Frameless: resize = drag window edges/corners; move = Alt+left-drag; quit = Q.", - flush=True, - ) - - sensor1 = None - sensor2 = None - video_cap = None - try: - input1 = int(args.video1) - input2 = int(args.video2) - is_camera = True - except: - is_camera = False - try: - if is_camera: - sensor1 = orisys.Sensor( - input1, - config_name="./config/ddjx01_wuxi.json", - cal_path="./config/dd01a_0306.npy", - isstitch=True, - cuda=False, - verbose=False, - ) - - sensor2 = orisys.Sensor( - input2, - config_name="./config/ddjx01_wuxi.json", - cal_path="./config/dd01a_0306.npy", - isstitch=True, - cuda=False, - verbose=False, - ) - - else: - sensor1 = orisys.Sensor( - args.video1, - config_name="./config/ddjx01_wuxi.json", - cal_path="./config/dd01a_0306.npy", - isstitch=True, - cuda=False, - verbose=False, - ) - sensor2 = orisys.Sensor( - args.video2, - config_name="./config/ddjx01_wuxi.json", - cal_path="./config/dd01a_0306.npy", - isstitch=True, - cuda=False, - verbose=False, - ) - - def _shortcut_reset_tracker(): - sensor1.reset() - sensor2.reset() - print("传感器已重置 (快捷键 R)", flush=True) - - def _shortcut_quit(): - combined_win.close() - def _shortcut_toggle_open(): - global is_open - is_open = not is_open - print(f"营业状态: {'营业' if is_open else '非营业'}", flush=True) - - _sc_r = QShortcut(QKeySequence(Qt.Key_R), combined_win) - _sc_r.activated.connect(_shortcut_reset_tracker) - if manual_open_enable: - _sc_g = QShortcut(QKeySequence(Qt.Key_G), combined_win) - _sc_g.activated.connect(_shortcut_toggle_open) - _sc_q = QShortcut(QKeySequence(Qt.Key_Q), combined_win) - _sc_q.activated.connect(_shortcut_quit) - - video_cap = cv2.VideoCapture(VIDEO_LOOP_PATH) - video_clock = {"fps": 30.0, "next_ts": None} - if not video_cap.isOpened(): - print(f"[video] Failed to open loop video: {VIDEO_LOOP_PATH}", flush=True) - else: - src_fps = float(video_cap.get(cv2.CAP_PROP_FPS) or 0.0) - if np.isfinite(src_fps) and src_fps >= 1.0: - video_clock["fps"] = src_fps - - start_time = time.time() - is_save = False - frame_count = 0 - writer = None - sensor1_active = True - sensor2_active = True - # Local to main(): assignments inside the loop make this local; must exist - # before the first iteration that skips the closed branch. - start_open = False - while True: - now = datetime.now() - if outside_opening_hours(now) or not is_open: - start_open = False - closed_hour_overlay.setGeometry(0, 0, combined_win.width(), combined_win.height()) - closed_hour_overlay.raise_() - closed_hour_overlay.show() - app.processEvents() - if combined_win.user_closed or not combined_win.isVisible(): - print("\n窗口已关闭，退出程序") - if writer is not None: - writer.release() - writer = None - break - time.sleep(1) - continue - - if not start_open: - start_open = True - sensor1.reset() - sensor2.reset() - print(f"传感器已重置: {datetime.now():%Y-%m-%d %H:%M:%S}", flush=True) - closed_hour_overlay.hide() - - if video_cap is not None and video_cap.isOpened(): - vret, vframe = read_loop_video_frame_realtime(video_cap, video_clock) - if vret: - if vframe.shape[1] != VIDEO_FRAME_W or vframe.shape[0] != VIDEO_FRAME_H: - vframe = cv2.resize(vframe, (VIDEO_FRAME_W, VIDEO_FRAME_H), interpolation=cv2.INTER_LINEAR) - set_video_label_bgr(video_label, vframe) - - if sensor1_active: - img1_valid = sensor1.get_img() - if img1_valid is None: - panel1_ui["arrow_label"].hide() - panel1_ui["plot_widget"].hide() - panel1_ui["disconnected_label"].show() - sensor1_active = False - print(f"Sensor1 disconnected. Showing DISCONNECTED screen. {datetime.now():%Y-%m-%d %H:%M:%S}", flush=True) - else: - sensor1.compute_deformation(check_motion=True, tail_frames=frame_to_recover, threshold=touch_threshold) - _, fn, fx, fy, flow = sensor1.read_info( - sensor1.info.FPS, - sensor1.info.FNORMAL, - sensor1.info.FSHEARX, - sensor1.info.FSHEARY, - sensor1.info.VRAW, - ) - _bg_bgr = (22, 16, 11) - _img = sensor1.img - _h, _w = _img.shape[:2] - black_img = np.empty((_h, _w, 3), dtype=_img.dtype) - black_img[:, :, 0] = _bg_bgr[0] - black_img[:, :, 1] = _bg_bgr[1] - black_img[:, :, 2] = _bg_bgr[2] - lw, lh = panel1_ui["arrow_label"].width(), panel1_ui["arrow_label"].height() - if lw > 16 and lh > 16: - side = max(32, min(lw, lh)) - else: - side = compute_arrow_canvas_side(panel1_ui["panel"]) - arrows = draw_arrows_wuxi( - black_img, - flow, - threshold=2, - grid_spacing=20, - arrow_scale=0.5, - below_threshold_color=(160, 160, 160), - target_size=int(side), - ) - set_arrow_label_bgr(panel1_ui["arrow_label"], arrows) - data1 = panel1_ui["data"] - for i in range(3): - data1[:-1, 0, i] = data1[1:, 0, i] - data1[:-1, 1, i] = data1[1:, 1, i] - data1[-1, 0, :] = frame_count / 15.0 - data1[-1, 1, 0] = fn - data1[-1, 1, 1] = abs(fx) - data1[-1, 1, 2] = abs(fy) - update( - panel1_ui["plot_af"], - panel1_ui["plot_tfx"], - panel1_ui["plot_tfy"], - data1, - panel1_ui["p1"], - panel1_ui["p2"], - panel1_ui["p3"], - ) - - if sensor2_active: - img2_valid = sensor2.get_img() - if img2_valid is None: - panel2_ui["arrow_label"].hide() - panel2_ui["plot_widget"].hide() - panel2_ui["disconnected_label"].show() - sensor2_active = False - print(f"Sensor2 disconnected. Showing DISCONNECTED screen. {datetime.now():%Y-%m-%d %H:%M:%S}", flush=True) - else: - sensor2.compute_deformation(check_motion=True, tail_frames=frame_to_recover, threshold=touch_threshold) - _, fn2, fx2, fy2, flow2 = sensor2.read_info( - sensor2.info.FPS, - sensor2.info.FNORMAL, - sensor2.info.FSHEARX, - sensor2.info.FSHEARY, - sensor2.info.VRAW, - ) - _bg_bgr2 = (22, 16, 11) - _img2 = sensor2.img - _h2, _w2 = _img2.shape[:2] - black_img2 = np.empty((_h2, _w2, 3), dtype=_img2.dtype) - black_img2[:, :, 0] = _bg_bgr2[0] - black_img2[:, :, 1] = _bg_bgr2[1] - black_img2[:, :, 2] = _bg_bgr2[2] - lw2, lh2 = panel2_ui["arrow_label"].width(), panel2_ui["arrow_label"].height() - if lw2 > 16 and lh2 > 16: - side2 = max(32, min(lw2, lh2)) - else: - side2 = compute_arrow_canvas_side(panel2_ui["panel"]) - arrows2 = draw_arrows_wuxi( - black_img2, - flow2, - threshold=2, - grid_spacing=20, - arrow_scale=0.5, - below_threshold_color=(160, 160, 160), - target_size=int(side2), - ) - set_arrow_label_bgr(panel2_ui["arrow_label"], arrows2) - data2 = panel2_ui["data"] - for i in range(3): - data2[:-1, 0, i] = data2[1:, 0, i] - data2[:-1, 1, i] = data2[1:, 1, i] - data2[-1, 0, :] = frame_count / 15.0 - data2[-1, 1, 0] = fn2 - data2[-1, 1, 1] = abs(fx2) - data2[-1, 1, 2] = abs(fy2) - update( - panel2_ui["plot_af"], - panel2_ui["plot_tfx"], - panel2_ui["plot_tfy"], - data2, - panel2_ui["p1"], - panel2_ui["p2"], - panel2_ui["p3"], - ) - - app.processEvents() - now = datetime.now() - if now.hour == 9 and now.minute == 0 and now.second == 0: - if sensor1_active: - sensor1.reset() - if sensor2_active: - sensor2.reset() - print( - f"[session] 传感器已重置: {now:%Y-%m-%d %H:%M:%S} ", - flush=True, - ) - - if not sensor1_active and not sensor2_active: - while not combined_win.user_closed and combined_win.isVisible(): - app.processEvents() - time.sleep(0.05) - break - - frame_count += 1 - app.processEvents() - - if combined_win.user_closed or not combined_win.isVisible(): - print("\n窗口已关闭，退出程序") - if writer is not None: - writer.release() - writer = None - break - - # Convert combined window to image (arrows + curves) - pixmap = combined_win.grab() - # Convert QPixmap to QImage - qimg = pixmap.toImage() - # Convert QImage to numpy array (RGB format) - qimg = qimg.convertToFormat(QImage.Format_RGB888) - width = qimg.width() - height = qimg.height() - # Convert QImage to numpy array - ptr = qimg.bits() - ptr.setsize(height * width * 3) - arr = np.array(ptr, dtype=np.uint8).reshape((height, width, 3)) - - except BaseException as exc: - session_abnormal = datetime.now() - print( - f"[session] ABNORMAL at {session_abnormal:%Y-%m-%d %H:%M:%S}: " - f"{type(exc).__name__}: {exc}", - flush=True, - ) - raise - finally: - session_end = datetime.now() - elapsed = session_end - session_start - print( - f"[session] END {session_end:%Y-%m-%d %H:%M:%S} " - f"(elapsed {elapsed})", - flush=True, - ) - if sensor1 is not None: - try: - sensor1.disconnect() - except Exception as disconnect_err: - print(f"[session] disconnect error: {disconnect_err} {datetime.now():%Y-%m-%d %H:%M:%S}", flush=True) - if 'sensor2' in locals() and sensor2 is not None: - try: - sensor2.disconnect() - except Exception as disconnect_err: - print(f"[session] disconnect error (sensor2): {disconnect_err} {datetime.now():%Y-%m-%d %H:%M:%S}", flush=True) - if video_cap is not None: - video_cap.release() - - -if __name__ == '__main__': - main() diff --git a/scripts/example_uart.py b/scripts/example_uart.py new file mode 100644 index 0000000..77f5c82 --- /dev/null +++ b/scripts/example_uart.py @@ -0,0 +1,461 @@ +#!/usr/bin/env python3 +"""UART finger controller for Radxa / Dragon Q6A (sensor-side receiver). + +Combines ORY-FINGER processing (same path as ``examples/finger_pipeline.py``) +with a line-based UART command interface. A host (or ``uart_send.py``-style +transmitter) sends commands; this script replies and streams forces. + +Dependencies (beyond orisys): + pip install pyserial + +Default UART (Dragon Q6A): + port=/dev/ttyHS1 baud=115200 8N1 + +Commands (one line, case-insensitive): + search List cameras (/dev/video*) via orisys.capture.list_cameras + connect [source] Open FingerRuntime (camera index, /dev/videoN, or video file) + run Stream FNORMAL / FSHEARX / FSHEARY until stop (or EOF) + stop Stop force streaming; keep source connected + quit | exit Close runtime and serial, then exit + +Replies: + camera,,, + OK search + OK connect + OK run + force,,, + OK eof Video file ended (when --no-loop-video) + OK stop + OK quit + ERR + +Examples: + # Live camera over UART + python scripts/example_uart.py --port /dev/ttyHS1 --config finger + + # Offline debug from a recorded file (no camera needed) + python scripts/example_uart.py --port /dev/ttyHS1 --video /path/to/clip.mp4 + # then UART: connect (uses --video) or connect /path/to/clip.mp4 + # then UART: run +""" + +from __future__ import annotations + +import argparse +import os +import shlex +import sys +from pathlib import Path +from typing import Optional, Union + +os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0") + +try: + import serial +except ImportError as exc: # pragma: no cover + raise SystemExit( + "pyserial is required for this script.\n" + " pip install pyserial\n" + f"Import error: {exc}" + ) from exc + +import cv2 +from orisys.capture import is_live_camera, list_cameras +from orisys.finger import FingerRuntime + +Source = Union[int, str] + + +def _parse_source(token: str) -> Optional[Source]: + """Accept camera index int, numeric string, V4L path, or filesystem path.""" + text = str(token).strip().strip('"').strip("'") + if not text: + return None + if text.isdigit() or (text.startswith("-") and text[1:].isdigit()): + return int(text) + return text + + +def _is_video_file(source: Source) -> bool: + """True when source is a filesystem video path (not a live camera).""" + if is_live_camera(source): + return False + if isinstance(source, int): + return False + path = Path(str(source)) + return path.is_file() or path.suffix.lower() in { + ".mp4", + ".avi", + ".mkv", + ".mov", + ".m4v", + ".wmv", + ".webm", + } + + +class UartFingerController: + """Serial command dispatcher around a single FingerRuntime.""" + + def __init__( + self, + ser: serial.Serial, + *, + config: str = "finger", + default_source: Source = 0, + max_camera_index: int = 10, + loop_video: bool = True, + show: bool = False, + verbose: bool = False, + ): + self.ser = ser + self.config = config + self.default_source = default_source + self.max_camera_index = max_camera_index + self.loop_video = loop_video + self.show = show + self.verbose = verbose + + self.runtime: Optional[FingerRuntime] = None + self._connected_source: Optional[Source] = None + self._last_search_source: Optional[Source] = None + self._streaming = False + self._should_quit = False + + def reply(self, line: str) -> None: + text = line.rstrip("\r\n") + "\n" + self.ser.write(text.encode("utf-8")) + self.ser.flush() + if self.verbose: + print(f"<< {text.rstrip()}", flush=True) + + def close_runtime(self) -> None: + self._streaming = False + if self.runtime is not None: + try: + self.runtime.close() + except Exception: # noqa: BLE001 + pass + self.runtime = None + self._connected_source = None + if self.show: + try: + cv2.destroyAllWindows() + except Exception: # noqa: BLE001 + pass + + def _resolve_connect_source(self, args: list[str]) -> Source: + if args: + # Join remaining tokens so unquoted paths with spaces still work. + joined = " ".join(args).strip() + parsed = _parse_source(joined) + if parsed is None: + raise ValueError("empty connect source") + return parsed + # Prefer --video default over last search so offline debug stays on file. + if isinstance(self.default_source, str) and not is_live_camera(self.default_source): + return self.default_source + if self._last_search_source is not None: + return self._last_search_source + return self.default_source + + def cmd_search(self, _args: list[str]) -> None: + try: + cameras = list_cameras(max_index=self.max_camera_index) + except Exception as exc: # noqa: BLE001 + self.reply(f"ERR search {exc}") + return + + self._last_search_source = None + for cam in cameras: + path = (cam.path or f"/dev/video{cam.index}").strip() or f"/dev/video{cam.index}" + size = f"{cam.width}x{cam.height}" + self.reply(f"camera,{cam.index},{path},{size}") + if self._last_search_source is None: + self._last_search_source = cam.index + self.reply(f"OK search {len(cameras)}") + + def cmd_connect(self, args: list[str]) -> None: + try: + source = self._resolve_connect_source(args) + except ValueError as exc: + self.reply(f"ERR connect {exc}") + return + + if isinstance(source, str) and not is_live_camera(source): + path = Path(source) + if not path.is_file(): + self.reply(f"ERR connect file not found: {source}") + return + + self._streaming = False + self.close_runtime() + try: + runtime = FingerRuntime.open( + config=self.config, + video=source, + verbose=self.verbose, + ) + runtime.warmup(color=False) + except Exception as exc: # noqa: BLE001 + self.reply(f"ERR connect {exc}") + return + + self.runtime = runtime + self._connected_source = source + kind = "file" if _is_video_file(source) else "camera" + self.reply(f"OK connect {source} ({kind})") + + def _rewind_video(self) -> bool: + """Seek connected file source back to frame 0. Return True on success.""" + if self.runtime is None or self._connected_source is None: + return False + if not _is_video_file(self._connected_source): + return False + cap = self.runtime.capture + if cap is None: + return False + try: + return bool(cap.set(cv2.CAP_PROP_POS_FRAMES, 0)) + except Exception: # noqa: BLE001 + return False + + def _poll_command_line(self, *, blocking: bool = True) -> Optional[str]: + """Read one command line if available; return None on timeout/empty.""" + if not blocking and self.ser.in_waiting <= 0: + return None + raw = self.ser.readline() + if not raw: + return None + try: + line = raw.decode("utf-8", errors="replace").strip() + except Exception: # noqa: BLE001 + return None + return line or None + + def cmd_run(self, _args: list[str]) -> None: + if self.runtime is None: + self.reply("ERR run not connected") + return + + self._streaming = True + self.reply("OK run") + tactile = self.runtime.tactile + is_file = self._connected_source is not None and _is_video_file(self._connected_source) + + while self._streaming and not self._should_quit: + ok, unrolled = self.runtime.grab_and_process(color=False) + if not ok: + if is_file and self.loop_video and self._rewind_video(): + if self.verbose: + print("video EOF: looping", flush=True) + continue + if is_file: + self.reply("OK eof") + else: + self.reply("ERR run grab failed") + self._streaming = False + break + + tactile.compute_deformation() + tactile.compute_contact() + fnormal, fshearx, fsheary = tactile.read_info( + tactile.info.FNORMAL, + tactile.info.FSHEARX, + tactile.info.FSHEARY, + ) + self.reply(f"force,{float(fnormal):.6f},{float(fshearx):.6f},{float(fsheary):.6f}") + + if self.show: + try: + import orisys + + arrows = orisys.util.draw_arrows( + unrolled, + tactile.flow, + threshold=0.4, + grid_spacing=10, + arrow_scale=1.0, + valid_mask=tactile.valid_mask, + ) + cv2.imshow("UV Unrolled", arrows) + key = cv2.waitKey(1) & 0xFF + if key == ord("q"): + self._should_quit = True + break + if key == ord("r"): + tactile.reset() + except Exception as exc: # noqa: BLE001 + if self.verbose: + print(f"show error: {exc}", file=sys.stderr) + + # Non-blocking command poll so force streaming is not delayed by UART timeout. + pending = self._poll_command_line(blocking=False) + if pending is not None: + if self.verbose: + print(f">> {pending}", flush=True) + if not self._dispatch(pending, allow_run=False): + break + + def cmd_stop(self, _args: list[str]) -> None: + self._streaming = False + self.reply("OK stop") + + def cmd_quit(self, _args: list[str]) -> None: + self._streaming = False + self._should_quit = True + self.reply("OK quit") + + def _dispatch(self, line: str, *, allow_run: bool = True) -> bool: + """Dispatch one command line. Return False if the main loop should exit.""" + try: + parts = shlex.split(line) + except ValueError: + parts = line.split() + if not parts: + return True + cmd = parts[0].lower() + args = parts[1:] + + if cmd == "search": + self.cmd_search(args) + elif cmd == "connect": + self.cmd_connect(args) + elif cmd == "run": + if not allow_run: + self.reply("ERR run already running") + else: + self.cmd_run(args) + elif cmd == "stop": + self.cmd_stop(args) + elif cmd in ("quit", "exit"): + self.cmd_quit(args) + return False + else: + self.reply(f"ERR unknown {cmd}") + return not self._should_quit + + def serve(self) -> None: + self.reply("OK ready") + while not self._should_quit: + line = self._poll_command_line() + if line is None: + continue + if self.verbose: + print(f">> {line}", flush=True) + if not self._dispatch(line): + break + + +def open_serial(port: str, baud: int, timeout: float) -> serial.Serial: + try: + return serial.Serial( + port=port, + baudrate=baud, + bytesize=serial.EIGHTBITS, + parity=serial.PARITY_NONE, + stopbits=serial.STOPBITS_ONE, + timeout=timeout, + ) + except serial.SerialException as exc: + print(f"Serial error: {exc}", file=sys.stderr) + print("Check:", file=sys.stderr) + print(f" 1. Device node {port} exists", file=sys.stderr) + print(" 2. User has access (sudo or dialout group)", file=sys.stderr) + print(" 3. Wiring / USB-TTL connection", file=sys.stderr) + raise SystemExit(1) from exc + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="UART finger controller (search/connect/run/stop force stream)" + ) + parser.add_argument("--port", default="/dev/ttyHS1", help="Serial device (default: /dev/ttyHS1)") + parser.add_argument("--baud", type=int, default=115200, help="Baud rate (default: 115200)") + parser.add_argument( + "--timeout", + type=float, + default=0.1, + help="Serial readline timeout seconds (default: 0.1)", + ) + parser.add_argument("--config", "-c", default="finger", help="Orisys config name or path") + parser.add_argument( + "--camera-id", + type=int, + default=0, + help="Default camera index when connect has no argument (ignored if --video is set)", + ) + parser.add_argument( + "--video", + default=None, + help="Default video file for connect (offline debug without a camera)", + ) + parser.add_argument( + "--max-camera-index", + type=int, + default=10, + help="Highest index probed by search", + ) + parser.add_argument( + "--loop-video", + dest="loop_video", + action="store_true", + default=True, + help="Loop video file on EOF during run (default: on)", + ) + parser.add_argument( + "--no-loop-video", + dest="loop_video", + action="store_false", + help="Stop run with OK eof when video file ends", + ) + parser.add_argument( + "--show", + action="store_true", + help="Show UV unrolled OpenCV window (local debug only)", + ) + parser.add_argument("--verbose", action="store_true", help="Echo UART traffic and SDK logs") + args = parser.parse_args(argv) + + if args.video: + video_path = Path(args.video) + if not video_path.is_file(): + print(f"Video file not found: {args.video}", file=sys.stderr) + return 1 + default_source: Source = str(video_path.resolve()) + else: + default_source = args.camera_id + + ser = open_serial(args.port, args.baud, args.timeout) + controller = UartFingerController( + ser, + config=args.config, + default_source=default_source, + max_camera_index=args.max_camera_index, + loop_video=args.loop_video, + show=args.show, + verbose=args.verbose, + ) + + print(f"UART finger controller on {args.port} @ {args.baud} bps", flush=True) + print(f"Default source: {default_source}", flush=True) + print("Commands: search | connect [source|file] | run | stop | quit", flush=True) + + try: + controller.serve() + except KeyboardInterrupt: + print("\nInterrupted.", flush=True) + try: + controller.reply("OK quit") + except Exception: # noqa: BLE001 + pass + finally: + controller.close_runtime() + if ser.is_open: + ser.close() + print("Serial closed.", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/qt_viewer_2.py b/scripts/qt_viewer_2.py index f918a6a..aa34ec9 100644 --- a/scripts/qt_viewer_2.py +++ b/scripts/qt_viewer_2.py @@ -1,794 +1,31 @@ -""" -Orisys SDK 实时监控 — Qt 统一界面 - -将所有可视化和曲线整合到一个窗口中，适合销售演示使用。 -SDK version: 0.3.1 -""" +#!/usr/bin/env python3 +"""Shim for the Orisys 2D Qt sensor monitor (see examples/gui/viewer2d/).""" import os import sys -os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" -import cv2 -def _fix_qt_plugin_path(): - """OpenCV redirects Qt to cv2/qt/plugins; use PyQt5's plugins instead.""" - try: - from PyQt5.QtCore import QLibraryInfo - os.environ["QT_QPA_PLATFORM_PLUGIN_PATH"] = QLibraryInfo.location( - QLibraryInfo.PluginsPath - ) - except Exception: - path = os.environ.get("QT_QPA_PLATFORM_PLUGIN_PATH", "") - if "cv2" in path.replace("\\", "/"): - os.environ.pop("QT_QPA_PLATFORM_PLUGIN_PATH", None) -_fix_qt_plugin_path() -import orisys -import numpy as np -import time -import argparse -from datetime import datetime -from scipy.signal import butter, filtfilt +from pathlib import Path -# Force curve sampling rate (Hz), matches typical sensor loop ~30 FPS. -FORCE_SAMPLE_RATE_HZ = 30.0 -PLOT_BUFFER_POINTS = 120 -PLOT_WINDOW_SEC = PLOT_BUFFER_POINTS / FORCE_SAMPLE_RATE_HZ # 4 s window -# Minimum y-axis span so autoscale does not over-zoom on small fluctuations. -Y_AXIS_MIN_SPAN = 20000 -CURVE_THEME_COLOR = '#00E5FF' -NORMAL_FORCE_CAL_SLOPE = 3.92685554668795e-05 -NORMAL_FORCE_CAL_OFFSET = 0.0217596967278535 +os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0") -from PyQt5.QtWidgets import ( - QMainWindow, QWidget, QHBoxLayout, QVBoxLayout, - QLabel, QApplication, QSplitter, QPlainTextEdit, - QPushButton, QFileDialog, QLineEdit, -) -from PyQt5.QtCore import Qt, QTimer -from PyQt5 import QtGui -from PyQt5.QtGui import QImage, QPixmap, QFont, QIcon +_EXAMPLES_DIR = Path(__file__).resolve().parents[1] / "examples" +if str(_EXAMPLES_DIR) not in sys.path: + sys.path.insert(0, str(_EXAMPLES_DIR)) -try: - import pyqtgraph as pg -except Exception: - print("PyQtGraph is not installed. Please install it using 'pip install pyqt5 pyqtgraph'.") - sys.exit(1) +from gui.viewer2d import run_viewer -# --------------------------------------------------------------------------- -# 工具 -# --------------------------------------------------------------------------- - -def cv2_to_pixmap(img_bgr: np.ndarray) -> QPixmap: - rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB) - h, w, ch = rgb.shape - qimg = QImage(rgb.data, w, h, ch * w, QImage.Format_RGB888) - return QPixmap.fromImage(qimg) - -def draw_magnitude_map(vfield, threshold=3, colormap=cv2.COLORMAP_JET) -> np.ndarray: - flow_field = vfield - magnitude = np.sqrt(flow_field[:, :, 0] ** 2 + flow_field[:, :, 1] ** 2) - - downsample_factor = 10 - h, w = magnitude.shape - magnitude_downsampled = magnitude[::downsample_factor, ::downsample_factor] - magnitude_smoothed = cv2.resize(magnitude_downsampled, (w, h), interpolation=cv2.INTER_CUBIC) - - magnitude_thresholded = magnitude_smoothed.copy() - magnitude_thresholded[magnitude_smoothed < threshold] = 0 - - mag_min, mag_max = magnitude_thresholded.min(), magnitude_thresholded.max() - if mag_max > mag_min: - magnitude_norm = ((magnitude_thresholded - mag_min) / (mag_max - mag_min) * 255).astype(np.uint8) - else: - magnitude_norm = np.zeros_like(magnitude_thresholded, dtype=np.uint8) - - magnitude_colored = cv2.applyColorMap(magnitude_norm, colormap) - out_img = cv2.resize(magnitude_colored, (400, 400), interpolation=cv2.INTER_CUBIC) - return out_img - - -def lowpass_filter(data, cutoff_hz, sample_rate_hz=FORCE_SAMPLE_RATE_HZ, order=2): - """ - Zero-phase Butterworth low-pass filter for uniformly sampled 1-D data. - - Parameters - ---------- - data : array-like - Input samples acquired at `sample_rate_hz` (default 30 Hz). - cutoff_hz : float - -3 dB cutoff frequency in Hz; use 0 to skip filtering (return raw data). - Otherwise must satisfy 0 < cutoff_hz < sample_rate_hz / 2. - sample_rate_hz : float - Sampling rate of `data` in Hz. - order : int - Butterworth filter order. - - Returns - ------- - np.ndarray - Filtered 1-D array with the same length as `data`. - """ - y = np.asarray(data, dtype=np.float64) - if cutoff_hz == 0: - return y.copy() - if y.size == 0: - return y - - nyquist = 0.5 * sample_rate_hz - if cutoff_hz < 0 or cutoff_hz >= nyquist: - raise ValueError( - f"cutoff_hz must be in (0, {nyquist}) for sample_rate_hz={sample_rate_hz}" - ) - - b, a = butter(order, cutoff_hz / nyquist, btype="low") - padlen = 3 * (max(len(b), len(a)) - 1) - if y.size <= padlen: - return y.copy() - - return filtfilt(b, a, y) - - -def calibrate_normal_force(raw_value): - """Convert raw normal-force reading to Newtons using y = mx + c.""" - return NORMAL_FORCE_CAL_SLOPE * raw_value + NORMAL_FORCE_CAL_OFFSET - - -class TeeStream: - """同时写入多个流""" - def __init__(self, *streams): - self.streams = streams - - def write(self, text): - for s in self.streams: - s.write(text) - - def flush(self): - for s in self.streams: - s.flush() - - -class WidgetLogHandler: - """将 print 输出实时显示到 QPlainTextEdit""" - def __init__(self, widget: QPlainTextEdit): - self.widget = widget - self.buf = "" - - def write(self, text: str): - self.buf += text - while "\n" in self.buf: - line, self.buf = self.buf.split("\n", 1) - if line: - self.widget.appendPlainText(line) - - def flush(self): - if self.buf: - self.widget.appendPlainText(self.buf) - self.buf = "" - -# --------------------------------------------------------------------------- -# 样式表 -# --------------------------------------------------------------------------- - -STYLE = """ -QMainWindow { - background-color: #000000; -} - -/* ---- 图像面板 ---- */ -QLabel#arrowsLabel { - border: 1px solid #262626; - border-radius: 4px; - background-color: #0b0b0b; -} - -/* ---- 顶栏按钮 & 输入 - — */ -QPushButton { - background-color: #1a1a1a; - color: #999999; - border: 1px solid #333333; - border-radius: 4px; - padding: 6px 16px; - font-family: "SimHei"; - font-size: 13px; -} -QPushButton:hover { - background-color: #262626; - color: #cccccc; - border: 1px solid #555555; -} -QPushButton#btnStart { - background-color: #0d2818; - color: #4caf84; - border: 1px solid #1a4d30; -} -QPushButton#btnStart:hover { - background-color: #143d24; - color: #6fd4a0; -} -QPushButton#btnStart:disabled { - background-color: #0f0f0f; - color: #333333; - border: 1px solid #222222; -} -QPushButton#btnStop { - background-color: #2b1010; - color: #e05555; - border: 1px solid #4d1a1a; -} -QPushButton#btnStop:hover { - background-color: #3d1818; - color: #f07070; -} -QPushButton#btnStop:disabled { - background-color: #0f0f0f; - color: #333333; - border: 1px solid #222222; -} -QLineEdit { - background-color: #0d0d0d; - color: #999999; - border: 1px solid #333333; - border-radius: 4px; - padding: 5px 10px; - font-size: 13px; -} - -/* ---- 力曲线容器 ---- */ -GraphicsLayoutWidget { - border: 1px solid #262626; - border-radius: 4px; -} - -/* ---- 日志面板 ---- */ -QPlainTextEdit { - background-color: #0b0b0b; - color: #777777; - border: 1px solid #262626; - border-radius: 4px; - font-family: "Cascadia Code", "Consolas", "Courier New", monospace; - font-size: 13px; - padding: 8px; - selection-background-color: #1a3a4a; - selection-color: #cccccc; -} - -QPlainTextEdit QScrollBar:vertical { - background-color: #0b0b0b; - width: 6px; - margin: 1px; - border-radius: 3px; -} -QPlainTextEdit QScrollBar::handle:vertical { - background-color: #262626; - border-radius: 3px; - min-height: 30px; -} -QPlainTextEdit QScrollBar::handle:vertical:hover { - background-color: #3a3a3a; -} -QPlainTextEdit QScrollBar::add-line:vertical, -QPlainTextEdit QScrollBar::sub-line:vertical { - height: 0px; -} -QPlainTextEdit QScrollBar::add-page:vertical, -QPlainTextEdit QScrollBar::sub-page:vertical { - background: none; -} - -QSplitter::handle { - background-color: #1a1a1a; -} -QSplitter::handle:vertical { - height: 1px; -} -QSplitter::handle:horizontal { - width: 1px; -} -""" - -# --------------------------------------------------------------------------- -# 主窗口 -# --------------------------------------------------------------------------- - -class MainWindow(QMainWindow): - def __init__(self, args, log_file): - super().__init__() - self.setWindowTitle("Orisys SDK") - self.setStyleSheet(STYLE) - - logo_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "logo.png") - if os.path.exists(logo_path): - self.setWindowIcon(QIcon(logo_path)) - - self.sensor = None - self.start_time = None - self.args = args - - # 录像状态 - self.is_save = False - self.frame_count = 0 - self.writer = None - self.save_path = None - - # 曲线数据缓冲（先填满 npoints 再滚动；120 点 @ 30 Hz = 4 s） - self.npoints = PLOT_BUFFER_POINTS - self.lowpass_cutoff_hz = getattr(args, "cutoff", 5.0) - self._plot_sample_count = 0 - self._init_plot_buffer() - - self._setup_ui() - self._create_plots() - - # 同步参数到 UI - self.camera_edit.setText(args.video) - self.config_edit.setText(args.config) - self.cal_edit.setText(args.cal) - - # stdout → 日志文件 + 界面日志面板 - self.log_handler = WidgetLogHandler(self.log_widget) - sys.stdout = TeeStream(log_file, self.log_handler) - sys.stderr = sys.stdout - - self._y_range_alpha = 0.5 - - self.timer = QTimer() - self.timer.timeout.connect(self._on_timer) - self._running = False - - def _init_plot_buffer(self): - """Empty buffer; x-axis grows until npoints real samples, then scrolls.""" - self.data = np.full((self.npoints, 2, 3), np.nan, dtype=np.float64) - self._plot_sample_count = 0 - self._y_range_smooth = [None, None, None] - - # ---- UI 构建 ---- - - def _setup_ui(self): - central = QWidget() - self.setCentralWidget(central) - root_layout = QVBoxLayout(central) - root_layout.setContentsMargins(6, 6, 6, 6) - root_layout.setSpacing(6) - - # ---- 顶栏 ---- - bar = QHBoxLayout() - bar.setSpacing(8) - - self.btn_camera = QPushButton("摄像机") - self.btn_camera.setObjectName("btnCamera") - self.btn_camera.clicked.connect(self._select_camera) - bar.addWidget(self.btn_camera) - self.camera_edit = QLineEdit("1") - self.camera_edit.setPlaceholderText("摄像头编号 或 视频路径") - bar.addWidget(self.camera_edit, 1) - - self.btn_config = QPushButton("配置") - self.btn_config.setObjectName("btnConfig") - self.btn_config.clicked.connect(self._select_config) - bar.addWidget(self.btn_config) - self.config_edit = QLineEdit("./config/ddjx01.json") - bar.addWidget(self.config_edit, 1) - - self.btn_reset = QPushButton("重置") - self.btn_reset.setObjectName("btnReset") - self.btn_reset.setEnabled(False) - self.btn_reset.clicked.connect(self._on_reset) - bar.addWidget(self.btn_reset) - self.cal_edit = QLineEdit("./config/ddjx01.npy") - bar.addWidget(self.cal_edit, 1) - - self.btn_start = QPushButton("启动") - self.btn_start.setObjectName("btnStart") - self.btn_start.clicked.connect(self._on_start) - bar.addWidget(self.btn_start) - - self.btn_stop = QPushButton("暂停") - self.btn_stop.setObjectName("btnStop") - self.btn_stop.setEnabled(False) - self.btn_stop.clicked.connect(self._on_stop) - bar.addWidget(self.btn_stop) - - root_layout.addLayout(bar) - - # ---- 外层纵向 splitter：主区域 / 日志 ---- - v_splitter = QSplitter(Qt.Vertical) - - # ---- 主区域：图像 (左) | 曲线 (右) ---- - h_splitter = QSplitter(Qt.Horizontal) - - # 左侧图像面板 - left = QWidget() - left_layout = QVBoxLayout(left) - left_layout.setContentsMargins(0, 0, 0, 0) - left_layout.setSpacing(6) - - self.arrows_label = QLabel() - self.arrows_label.setObjectName("arrowsLabel") - self.arrows_label.setMinimumSize(160, 160) - self.arrows_label.setScaledContents(True) - self.arrows_label.setAlignment(Qt.AlignCenter) - left_layout.addWidget(self.arrows_label) - - h_splitter.addWidget(left) - - # 右侧力曲线 - pg.setConfigOption('background', '#000000') - pg.setConfigOption('foreground', '#777777') - pg.setConfigOptions(antialias=True) - self.graph_widget = pg.GraphicsLayoutWidget() - self.graph_widget.setMinimumWidth(300) - h_splitter.addWidget(self.graph_widget) - - self.h_splitter = h_splitter - - v_splitter.addWidget(h_splitter) - - # ---- 底部日志面板 ---- - self.log_widget = QPlainTextEdit() - self.log_widget.setReadOnly(True) - self.log_widget.setMaximumBlockCount(2000) - font = QFont("Consolas", 12) - font.setStyleHint(QFont.Monospace) - self.log_widget.setFont(font) - v_splitter.addWidget(self.log_widget) - - v_splitter.setStretchFactor(0, 4) - v_splitter.setStretchFactor(1, 1) - - root_layout.addWidget(v_splitter) - - def _create_plots(self): - titles = ["法向力 (N)", "切向力 X", "切向力 Y"] - self.plots = [] - self.curves = [] - self.dot_glows = [] - self.dot_cores = [] - - axis_pen = pg.mkPen(color=(38, 38, 38), width=1) # #262626 - label_font = QFont("SimHei", 9) - - for i, title in enumerate(titles): - p = self.graph_widget.addPlot(title=None) - - # 标题：黑体 - p.setTitle(title, color='#999999', size='14pt') - p.titleLabel.setFont(QFont("SimHei", 14)) - - # X 轴标签 - p.setLabel('bottom', '时间', units='s') - p.getAxis('bottom').label.setFont(label_font) - - # 仅保留左 + 下轴线，隐藏右 + 上边框 - p.showAxis('top', False) - p.showAxis('right', False) - p.getAxis('left').setPen(axis_pen) - p.getAxis('bottom').setPen(axis_pen) - - # 仅横向网格，极淡 - p.showGrid(x=False, y=True, alpha=0.12) - - # 等宽刻度字体 - tick_font = QFont("Consolas", 11) - tick_font.setStyleHint(QFont.Monospace) - p.getAxis('left').setTickFont(tick_font) - p.getAxis('bottom').setTickFont(tick_font) - - # 曲线: 青色描边 + 向下渐变填充（与 demo_wuxi 一致） - line_pen = pg.mkPen(color=CURVE_THEME_COLOR, width=2) - # grad_brush = make_gradient_brush(CURVE_THEME_COLOR, max_alpha=120) - curve = p.plot(pen=line_pen, fillLevel=0) - - # 终点光晕: #00e5ff 30% 透明，半径 8 - dot_glow = p.plot([], [], pen=None, symbol='o', symbolSize=8, - symbolPen=None, symbolBrush=(0, 229, 255, 76)) - # 终点核心: 纯白，半径 4 - dot_core = p.plot([], [], pen=None, symbol='o', symbolSize=4, - symbolPen=None, symbolBrush=(255, 255, 255, 255)) - - p.setXRange(0, PLOT_WINDOW_SEC, padding=0) - p.enableAutoRange(x=False, y=False) - - self.plots.append(p) - self.curves.append(curve) - self.dot_glows.append(dot_glow) - self.dot_cores.append(dot_core) - - if i < 2: - self.graph_widget.nextRow() - - def adjust_square_images(self): - """根据窗口高度调整左侧面板宽度，使左侧图像保持正方形""" - top_bar_h = 48 - log_h = self.log_widget.height() - spacing = 20 - available = self.height() - top_bar_h - log_h - spacing - image_w = max(available, 160) - self.h_splitter.setSizes([image_w, self.width() - image_w]) - - # ---- 帧处理 ---- - - def _on_timer(self): - if self.sensor is None: - return - sensor = self.sensor - - isimg = sensor.get_img() - if isimg is None: - print("Failed to get image") - return - - # 旋转原始图像，使力方向与实际物理方向对齐 - sensor.img = cv2.rotate(sensor.img, cv2.ROTATE_90_COUNTERCLOCKWISE) - if sensor.frame is not None: - sensor.frame = cv2.rotate(sensor.frame, cv2.ROTATE_90_COUNTERCLOCKWISE) - - # 录像写入 - if self.is_save and self.writer is not None and sensor.frame is not None: - self.frame_count += 1 - save_img = sensor.frame - if len(save_img.shape) == 2 or (len(save_img.shape) == 3 and save_img.shape[2] == 1): - if len(save_img.shape) == 3: - save_img = save_img.squeeze(2) - save_img = cv2.cvtColor(save_img, cv2.COLOR_GRAY2BGR) - self.writer.write(save_img) - - is_touched = sensor.compute_deformation(check_motion=True, threshold=1) - sensor.compute_contact() - - fps, fn_raw, fx, fy, flow, vnormal, vshear, _ = sensor.read_info( - sensor.info.FPS, sensor.info.FNORMAL, - sensor.info.FSHEARX, sensor.info.FSHEARY, - sensor.info.VRAW, sensor.info.VNORMAL, - sensor.info.VSHEAR, - sensor.info.DEPTH - ) - fn = calibrate_normal_force(fn_raw) - - - print(f"FPS={fps:.2f}, 法向力={fn:.4f}, 切向力X={fx:.4f}, 切向力Y={fy:.4f}") - # print(f"CUPY Available: {sensor.CUPY_AVAILABLE}") - - # 图像 - _bg_bgr = (22, 16, 11) - _img = sensor.img - _h, _w = _img.shape[:2] - black_img = np.empty((_h, _w, 3), dtype=_img.dtype) - black_img[:, :, 0] = _bg_bgr[0] - black_img[:, :, 1] = _bg_bgr[1] - black_img[:, :, 2] = _bg_bgr[2] - - arrows = orisys.util.draw_arrows( - black_img, flow, - threshold=2, grid_spacing=20, - arrow_scale=0.5, - below_threshold_color=(160, 160, 160), - ) - arrows = cv2.resize(arrows, (400, 400), interpolation=cv2.INTER_CUBIC) - self.arrows_label.setPixmap(cv2_to_pixmap(arrows)) - - # 曲线：先顺序写入 npoints 个真实样本，满后再左移滚动 - current_time = time.time() - self.start_time - if self._plot_sample_count < self.npoints: - idx = self._plot_sample_count - self.data[idx, 0, :] = current_time - self.data[idx, 1, 0] = fn - self.data[idx, 1, 1] = fx - self.data[idx, 1, 2] = fy - self._plot_sample_count += 1 - else: - for i in range(3): - self.data[:-1, 0, i] = self.data[1:, 0, i] - self.data[:-1, 1, i] = self.data[1:, 1, i] - self.data[-1, 0, :] = current_time - self.data[-1, 1, 0] = fn - self.data[-1, 1, 1] = fx - self.data[-1, 1, 2] = fy - - n = self._plot_sample_count - if n == 0: - return - - data_plot = self.data[:n].copy() - for i in range(3): - data_plot[:, 1, i] = lowpass_filter( - data_plot[:, 1, i], - self.lowpass_cutoff_hz, - sample_rate_hz=FORCE_SAMPLE_RATE_HZ, - ) - - for i, curve in enumerate(self.curves): - x = data_plot[:, 0, i] - y = data_plot[:, 1, i] - curve.setData(x, y) - ymin, ymax = y.min(), y.max() - if self._y_range_smooth[i] is None: - self._y_range_smooth[i] = (ymin, ymax) - else: - prev_min, prev_max = self._y_range_smooth[i] - alpha = self._y_range_alpha - ymin = alpha * ymin + (1 - alpha) * prev_min - ymax = alpha * ymax + (1 - alpha) * prev_max - span = ymax - ymin - if span < Y_AXIS_MIN_SPAN: - mid = (ymin + ymax) / 2 - half = Y_AXIS_MIN_SPAN / 2 - ymin = mid - half - ymax = mid + half - self._y_range_smooth[i] = (ymin, ymax) - pad = max((ymax - ymin) * 0.12, 0.01) - self.plots[i].setYRange(ymin - pad, ymax + pad) - last_x = x[-1:] - last_y = y[-1:] - self.dot_glows[i].setData(last_x, last_y) - self.dot_cores[i].setData(last_x, last_y) - - if n < self.npoints: - for p in self.plots: - p.setXRange(0, PLOT_WINDOW_SEC, padding=0) - else: - x0 = float(data_plot[0, 0, 0]) - x1 = float(data_plot[-1, 0, 0]) - if x1 <= x0: - x1 = x0 + PLOT_WINDOW_SEC - for p in self.plots: - p.setXRange(x0, x1, padding=0) - - # ---- 键盘 & 录像 ---- - - def _on_start(self): - if self._running: - return - self.btn_start.setText("启动中...") - self.btn_start.setEnabled(False) - QApplication.processEvents() - - try: - src = self.camera_edit.text() - cfg = self.config_edit.text() - cal = self.cal_edit.text() - verbose = getattr(self.args, 'verbose', True) - try: - src = int(src) - except ValueError: - pass - self.sensor = orisys.Sensor(src, config_name=cfg, cal_path=cal, verbose=verbose) - except Exception as e: - self.btn_start.setText("启动") - self.btn_start.setEnabled(True) - print(f"启动失败: {e}") - return - - self.start_time = time.time() - self._init_plot_buffer() - self._running = True - self.timer.start(10) - self.btn_start.setText("启动") - self.btn_stop.setEnabled(True) - self.btn_reset.setEnabled(True) - print(f"▶ 启动 — 摄像机={src}, 配置={cfg}, 标定={cal}") - - def _on_stop(self): - if not self._running: - return - self._running = False - self.timer.stop() - self.btn_stop.setText("停止中...") - self.btn_stop.setEnabled(False) - QApplication.processEvents() - - if self.sensor is not None: - self.sensor.disconnect() - self.sensor = None - - self.btn_start.setEnabled(True) - self.btn_stop.setText("暂停") - self.btn_reset.setEnabled(False) - print("⏸ 已暂停") - - def _select_camera(self): - text = self.camera_edit.text() - path, _ = QFileDialog.getOpenFileName( - self, "选择视频文件", "data", "视频 (*.mp4 *.avi *.mov);;所有文件 (*)", - ) - if path: - self.camera_edit.setText(path) - - def _select_config(self): - path, _ = QFileDialog.getOpenFileName( - self, "选择配置文件", "config", "JSON (*.json);;所有文件 (*)", - ) - if path: - self.config_edit.setText(path) - - def _on_reset(self): - """Reset optical flow tracker when drift occurs (same as basic_pipeline 'r' key).""" - if self.sensor is None or not self._running: - print("请先启动传感器后再重置追踪器。") - return - self.sensor.reset() - print("追踪器已重置。") - - def keyPressEvent(self, event): - key = event.key() - if key == Qt.Key_Escape: - if self.isFullScreen(): - self.showNormal() - else: - self.showFullScreen() - elif key == Qt.Key_Q: - print("\n正在退出示例程序...") - self.close() - elif key == Qt.Key_S: - self._start_recording() - elif key == Qt.Key_E: - self._stop_recording() - elif key == Qt.Key_R: - self._on_reset() - else: - super().keyPressEvent(event) - - def _start_recording(self): - if self.sensor is None: - print("请先启动采集") - return - self.save_path = "./data/pulse_test.mp4" - os.makedirs(os.path.dirname(self.save_path), exist_ok=True) - - save_img = self.sensor.frame - if save_img is None: - print("当前没有可用帧，无法确定录像尺寸。") - return - - temp_img = save_img.copy() - if len(temp_img.shape) == 2 or (len(temp_img.shape) == 3 and temp_img.shape[2] == 1): - if len(temp_img.shape) == 3: - temp_img = temp_img.squeeze(2) - temp_img = cv2.cvtColor(temp_img, cv2.COLOR_GRAY2BGR) - - fh, fw = temp_img.shape[:2] - self.writer = cv2.VideoWriter( - self.save_path, cv2.VideoWriter_fourcc(*'mp4v'), 30, (fw, fh), - ) - if self.writer.isOpened(): - self.is_save = True - print(f"开始录像：{self.save_path}") - else: - print(f"无法创建录像文件：{self.save_path}") - self.writer = None - - def _stop_recording(self): - self.is_save = False - if self.writer is not None: - self.writer.release() - self.writer = None - if self.save_path is not None: - print(f"录像结束：{self.save_path}，总帧数：{self.frame_count}") - else: - print(f"录像结束，总帧数：{self.frame_count}") - self.frame_count = 0 - - def closeEvent(self, event): - self.timer.stop() - if self.writer is not None: - self.writer.release() - if self.sensor is not None: - self.sensor.disconnect() - event.accept() - -# --------------------------------------------------------------------------- -# 入口 -# --------------------------------------------------------------------------- def main(): - script_dir = os.path.dirname(os.path.abspath(__file__)) - log_dir = os.path.join(script_dir, "logs") - os.makedirs(log_dir, exist_ok=True) - log_path = os.path.join(log_dir, f"qt_viewer_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log") - log_file = open(log_path, 'w', encoding='utf-8') - # stdout 到文件的临时重定向（创建 MainWindow 后会加上日志面板） - sys.stdout = log_file - sys.stderr = sys.stdout + import argparse - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(description="Orisys 2D Qt sensor monitor") + parser.add_argument( + "--sensor-type", + "-s", + type=str, + default="orisys", + choices=("orisys", "dd06"), + help="Sensor adapter: orisys (direct Sensor) or dd06 (polygon ROI rectify)", + ) parser.add_argument("--video", "-v", type=str, default="0") parser.add_argument("--config", "-c", type=str, default="./config/ddjx01.json") parser.add_argument("--cal", "-cal", type=str, default="./config/ddjx01.npy") @@ -801,18 +38,15 @@ def main(): ) args = parser.parse_args() - print("\n按键说明：Q 退出 | R 重置追踪器 | S 开始录像 | E 结束录像 | ESC 全屏") + # Sensible DD06 defaults when user only switches sensor type. + if args.sensor_type == "dd06": + if args.video in ("0", "1") and args.config.endswith("ddjx01.json"): + args.video = "./data/dd06/0726.mp4" + args.config = "./config/dd06-ov.json" - # 高 DPI 适配（笔记本高分屏） - QApplication.setAttribute(Qt.AA_EnableHighDpiScaling, True) - QApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True) - app = QApplication(sys.argv) - window = MainWindow(args, log_file) - window.showFullScreen() - QApplication.processEvents() - window.adjust_square_images() + log_dir = Path(__file__).resolve().parent / "logs" + sys.exit(run_viewer(args, log_dir=log_dir)) - sys.exit(app.exec_()) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/scripts/qt_viewer_finger.py b/scripts/qt_viewer_finger.py deleted file mode 100644 index 7165a35..0000000 --- a/scripts/qt_viewer_finger.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 -"""Shim for the Orisys finger Qt dashboard (see examples/gui/).""" - -import os -import sys -from pathlib import Path - -os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0") - -_EXAMPLES_DIR = Path(__file__).resolve().parents[1] / "examples" -if str(_EXAMPLES_DIR) not in sys.path: - sys.path.insert(0, str(_EXAMPLES_DIR)) - -from argparse import Namespace - -from gui import run_dashboard -from pipeline import PipelineConfig - - -def main(): - import argparse - - parser = argparse.ArgumentParser(description="Orisys finger Qt dashboard") - parser.add_argument("--video", "-v", type=str, default="0") - parser.add_argument("--config", "-c", type=str, default="./config/dd02-ov.json") - parser.add_argument("--cal", type=str, default="") - parser.add_argument("--verbose", action="store_true") - parser.add_argument("--cutoff", type=float, default=10.0) - parser.add_argument("--no-3d", action="store_true") - args = parser.parse_args() - - config = PipelineConfig.from_args( - Namespace( - video=args.video, - config=args.config, - cal=args.cal, - verbose=args.verbose, - mesh_map="depth", - threshold=None, - viz_pose="assets/finger/visualization/world_to_cam.npy", - no_3d=args.no_3d, - cutoff=args.cutoff, - ) - ) - sys.exit(run_dashboard(config)) - - -if __name__ == "__main__": - main() diff --git a/scripts/test_ffmpeg.py b/scripts/test_ffmpeg.py deleted file mode 100644 index 1c3396f..0000000 --- a/scripts/test_ffmpeg.py +++ /dev/null @@ -1,159 +0,0 @@ -import argparse -import av -import cv2 -import platform -import subprocess -import sys -import time - -TARGET_FPS = 30 -WIDTH = 1920 -HEIGHT = 1080 - - -def list_windows_dshow_devices(): - """ - Print DirectShow devices using ffmpeg, if available. - """ - cmd = ["ffmpeg", "-hide_banner", "-list_devices", "true", "-f", "dshow", "-i", "dummy"] - print("Listing Windows DirectShow devices via ffmpeg...") - try: - result = subprocess.run(cmd, capture_output=True, text=True, check=False) - except FileNotFoundError as exc: - raise RuntimeError("ffmpeg was not found in PATH; cannot list DirectShow devices.") from exc - - output = (result.stderr or "") + (result.stdout or "") - print(output.strip()) - - -def open_pyav_capture(system_name, device, pixel_format): - """ - Open a camera device via PyAV on Linux or Windows. - """ - if system_name == "linux": - options = { - "video_size": f"{WIDTH}x{HEIGHT}", - "framerate": str(TARGET_FPS), - "input_format": pixel_format or "mjpeg", - } - container = av.open(device or "/dev/video4", format="v4l2", options=options) - elif system_name == "windows": - if not device: - raise RuntimeError( - "On Windows, PyAV/dshow requires a camera name, for example " - '--device "USB Camera". Use --list-devices first.' - ) - options = { - "video_size": f"{WIDTH}x{HEIGHT}", - "framerate": str(TARGET_FPS), - } - if pixel_format: - options["pixel_format"] = pixel_format - container = av.open(f"video={device}", format="dshow", options=options) - else: - raise RuntimeError( - f"Unsupported OS: {platform.system()}. Only Linux and Windows are supported." - ) - - stream = container.streams.video[0] - return container, stream - - -def main(): - parser = argparse.ArgumentParser( - description="Capture from camera via PyAV on Linux or Windows, optionally save to MP4." - ) - parser.add_argument( - "-d", - "--device", - type=str, - default="", - help=( - "Linux: V4L2 device path like /dev/video4. " - "Windows: DirectShow camera name like 'USB Camera'. " - "If empty, Linux uses /dev/video4; Windows requires --device." - ), - ) - parser.add_argument( - "--pixel-format", - type=str, - default="", - help=( - "Optional input pixel format / format hint. " - "Examples: Linux 'mjpeg'; Windows may accept values such as 'mjpeg' if supported." - ), - ) - parser.add_argument( - "--list-devices", - action="store_true", - help="On Windows, list DirectShow device names via ffmpeg and exit.", - ) - parser.add_argument( - "-o", - "--output", - type=str, - default="", - help="Output MP4 path (e.g. capture.mp4). If empty, only preview, no file.", - ) - args = parser.parse_args() - - system_name = platform.system().lower() - if args.list_devices: - if system_name != "windows": - raise RuntimeError("--list-devices is currently only implemented for Windows.") - list_windows_dshow_devices() - return - - container, stream = open_pyav_capture(system_name, args.device, args.pixel_format) - print( - f"Using PyAV backend on {platform.system()} " - f"with device={args.device or '/dev/video4'}" - ) - - frame_count = 0 - t_start = time.perf_counter() - - writer = None - if args.output: - fourcc = cv2.VideoWriter_fourcc(*"mp4v") - writer = cv2.VideoWriter( - args.output, fourcc, float(TARGET_FPS), (WIDTH, HEIGHT) - ) - if not writer.isOpened(): - container.close() - raise RuntimeError(f"Could not open VideoWriter for {args.output}") - print(f"Recording to {args.output} @ {TARGET_FPS} fps, {WIDTH}x{HEIGHT}") - - try: - for frame in container.decode(stream): - t0 = time.perf_counter() - img = frame.to_ndarray(format="bgr24") # OpenCV-compatible - t1 = time.perf_counter() - - if writer is not None: - writer.write(img) - - cv2.imshow("pyav_cam", img) - key = cv2.waitKey(1) & 0xFF - t2 = time.perf_counter() - - frame_count += 1 - if frame_count % 30 == 0: - elapsed = t2 - t_start - print( - f"Frame {frame_count}: convert={1000*(t1-t0):.1f}ms, " - f"imshow+waitKey={1000*(t2-t1):.1f}ms, avg_fps={frame_count/elapsed:.1f}" - ) - - if key == ord("q"): - break - finally: - container.close() - if writer is not None: - writer.release() - print(f"Saved {frame_count} frames to {args.output}") - cv2.destroyAllWindows() - - -if __name__ == "__main__": - main() diff --git a/sensors/box/force_calibration.json b/sensors/box/force_calibration.json new file mode 100644 index 0000000..015fc07 --- /dev/null +++ b/sensors/box/force_calibration.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "unit": "N", + "channels": { + "normal": { + "method": "piecewise_linear", + "raw_key": "fnormal", + "extrapolate": true, + "clamp_output_min": 0.0, + "points": [ + [1000.0, 0.0], + [10000.0, 0.377], + [30000.0, 1.377], + [62000.0, 2.377] + ] + }, + "shear": { + "method": "piecewise_linear", + "input": "components", + "signed": true, + "extrapolate": true, + "clamp_output_min": 0.0, + "points": [ + [1000.0, 0.0], + [36000.0, 1.19], + [68000.0, 2.06] + ] + } + } +} diff --git a/config/prototype_of3_720p.json b/sensors/box/sensor.json similarity index 65% rename from config/prototype_of3_720p.json rename to sensors/box/sensor.json index 185ccbf..af88413 100644 --- a/config/prototype_of3_720p.json +++ b/sensors/box/sensor.json @@ -2,23 +2,25 @@ "name": "prototype", "version": "1.0", "camera": { - "resolution_raw": [1280, 720], + "resolution_raw": [1920, 1080], "fps": 30, "units": 15, "units_used": 9, - "resolution_stitch": [263, 263], + "resolution_stitch": [365, 365], "stitch_res_list": [365,263,123,63] }, "stitcher": { "min_area_frac": 0.01, - "max_area_frac": 0.04, - "max_aspect_ratio": 25, + "max_area_frac": 0.05, + "max_aspect_ratio": 50, "index": [1,2,3,6,7,8,11,12,13], "margin": 10, - "nominal_ov_pos": 67, + "nominal_ov_pos": 100, "n": [3,3], "row_tol": 0.6, - "box_size": [180, 180] + "box_size": [250, 250], + "blend":"numba", + "blend_list": ["numpy", "numba","none"] }, "tracker": { "adaptive": true, @@ -27,17 +29,17 @@ "loop_closure_interval": 1, "algorithm": "dis", "dis_preset": "medium", - "dis_mode": 2, + "dis_mode": 3, "algorithm_list": ["raft", "dis"], "hhd_downsample": 3 }, "contact": { "min_contour_area": 3000, - "flow_threshold":1, - "div_threshold":0.02, + "flow_threshold":5, + "div_threshold":0.05, "expansion_threshold": 0.01, "kernel_size": [5,5], - "denoise_sigma": 10 + "denoise_sigma": 12 } } diff --git a/sensors/finger/assets/30fps-0703/README.md b/sensors/finger/assets/30fps-0703/README.md new file mode 100644 index 0000000..24a039e --- /dev/null +++ b/sensors/finger/assets/30fps-0703/README.md @@ -0,0 +1,9 @@ +Place the ORY-FINGER runtime asset bundle files in this directory before +building release wheels: + +- `mesh.npz` +- `camera_intrinsics.npz` +- `world_to_cam.npy` + +These binary assets are validated by the wheel build and are intentionally not +generated by the package build step. diff --git a/assets/finger/30fps-0616/camera_intrinsics.npz b/sensors/finger/assets/30fps-0703/camera_intrinsics.npz similarity index 100% rename from assets/finger/30fps-0616/camera_intrinsics.npz rename to sensors/finger/assets/30fps-0703/camera_intrinsics.npz diff --git a/sensors/finger/assets/30fps-0703/mesh.npz b/sensors/finger/assets/30fps-0703/mesh.npz new file mode 100644 index 0000000..4da6bb1 Binary files /dev/null and b/sensors/finger/assets/30fps-0703/mesh.npz differ diff --git a/assets/finger/30fps-0616/pixel_to_xyz.npz b/sensors/finger/assets/30fps-0703/pixel_to_xyz.npz similarity index 50% rename from assets/finger/30fps-0616/pixel_to_xyz.npz rename to sensors/finger/assets/30fps-0703/pixel_to_xyz.npz index c15f54f..15eee73 100644 Binary files a/assets/finger/30fps-0616/pixel_to_xyz.npz and b/sensors/finger/assets/30fps-0703/pixel_to_xyz.npz differ diff --git a/sensors/finger/assets/30fps-0703/unroll_map_uv_s0.4.npz b/sensors/finger/assets/30fps-0703/unroll_map_uv_s0.4.npz new file mode 100644 index 0000000..995cee0 Binary files /dev/null and b/sensors/finger/assets/30fps-0703/unroll_map_uv_s0.4.npz differ diff --git a/assets/finger/visualization/world_to_cam.npy b/sensors/finger/assets/30fps-0703/world_to_cam.npy similarity index 66% rename from assets/finger/visualization/world_to_cam.npy rename to sensors/finger/assets/30fps-0703/world_to_cam.npy index 2d5e411..3d6f897 100644 Binary files a/assets/finger/visualization/world_to_cam.npy and b/sensors/finger/assets/30fps-0703/world_to_cam.npy differ diff --git a/sensors/finger/assets/visualization/README.md b/sensors/finger/assets/visualization/README.md new file mode 100644 index 0000000..fe08404 --- /dev/null +++ b/sensors/finger/assets/visualization/README.md @@ -0,0 +1,6 @@ +Place visualization camera assets in this directory before building release +wheels: + +- `world_to_cam.npy` + +The 3D dashboard uses this pose as the default view preset when available. diff --git a/assets/finger/visualization/camera_intrinsics.npz b/sensors/finger/assets/visualization/camera_intrinsics.npz similarity index 100% rename from assets/finger/visualization/camera_intrinsics.npz rename to sensors/finger/assets/visualization/camera_intrinsics.npz diff --git a/assets/finger/30fps/flatten_remap.npz b/sensors/finger/assets/visualization/flatten_remap.npz similarity index 100% rename from assets/finger/30fps/flatten_remap.npz rename to sensors/finger/assets/visualization/flatten_remap.npz diff --git a/assets/finger/30fps/mesh.npz b/sensors/finger/assets/visualization/mesh.npz similarity index 100% rename from assets/finger/30fps/mesh.npz rename to sensors/finger/assets/visualization/mesh.npz diff --git a/assets/finger/visualization/pixel_to_xyz.npz b/sensors/finger/assets/visualization/pixel_to_xyz.npz similarity index 100% rename from assets/finger/visualization/pixel_to_xyz.npz rename to sensors/finger/assets/visualization/pixel_to_xyz.npz diff --git a/assets/finger/30fps/world_to_cam.npy b/sensors/finger/assets/visualization/world_to_cam.npy similarity index 100% rename from assets/finger/30fps/world_to_cam.npy rename to sensors/finger/assets/visualization/world_to_cam.npy diff --git a/sensors/finger/manifest.json b/sensors/finger/manifest.json new file mode 100644 index 0000000..27f92c8 --- /dev/null +++ b/sensors/finger/manifest.json @@ -0,0 +1,6 @@ +{ + "name": "finger", + "sensor_json": "sensor.json", + "assets_dir": "assets/30fps-0703", + "viz_pose": "assets/visualization/world_to_cam.npy" +} diff --git a/sensors/finger/sensor.json b/sensors/finger/sensor.json new file mode 100644 index 0000000..930caeb --- /dev/null +++ b/sensors/finger/sensor.json @@ -0,0 +1,35 @@ +{ + "name": "dd02-ov", + "type": "single_camera", + "version": "1.0", + "camera": { + "resolution_raw": [1280, 720], + "fps": 30, + "resolution_stitch": [280, 300], + "source_scale": 0.4, + "old":[672, 728, 1,0.5], + "work": [504, 546,1], + "work2": [420, 450,0.8], + "work3": [350, 375,4,0.7], + "work4": [280, 300,4,0.4,0.2] + }, + "tracker": { + "adaptive": true, + "lower_threshold": 12.0, + "upper_threshold": 18.0, + "loop_closure_interval": 1, + "algorithm": "dis", + "dis_preset": "medium", + "dis_mode": 1, + "algorithm_list": ["raft", "dis"], + "hhd_downsample": 1 + }, + "contact": { + "min_contour_area": 3000, + "flow_threshold": 0.2, + "div_threshold": 0.022, + "expansion_threshold": 0.01, + "kernel_size": [5, 5], + "denoise_sigma": 12 + } +} diff --git a/config/prototype_of3_480p.json b/sensors/mini/sensor.json similarity index 63% rename from config/prototype_of3_480p.json rename to sensors/mini/sensor.json index 5c321a3..af88413 100644 --- a/config/prototype_of3_480p.json +++ b/sensors/mini/sensor.json @@ -2,23 +2,25 @@ "name": "prototype", "version": "1.0", "camera": { - "resolution_raw": [854, 480], + "resolution_raw": [1920, 1080], "fps": 30, "units": 15, "units_used": 9, - "resolution_stitch": [123, 123], + "resolution_stitch": [365, 365], "stitch_res_list": [365,263,123,63] }, "stitcher": { "min_area_frac": 0.01, - "max_area_frac": 0.06, - "max_aspect_ratio": 100, + "max_area_frac": 0.05, + "max_aspect_ratio": 50, "index": [1,2,3,6,7,8,11,12,13], - "margin": 4, - "nominal_ov_pos": 44, + "margin": 10, + "nominal_ov_pos": 100, "n": [3,3], "row_tol": 0.6, - "box_size": [98, 98] + "box_size": [250, 250], + "blend":"numba", + "blend_list": ["numpy", "numba","none"] }, "tracker": { "adaptive": true, @@ -27,17 +29,17 @@ "loop_closure_interval": 1, "algorithm": "dis", "dis_preset": "medium", - "dis_mode": 2, + "dis_mode": 3, "algorithm_list": ["raft", "dis"], "hhd_downsample": 3 }, "contact": { "min_contour_area": 3000, - "flow_threshold":1, - "div_threshold":0.02, + "flow_threshold":5, + "div_threshold":0.05, "expansion_threshold": 0.01, "kernel_size": [5,5], - "denoise_sigma": 6 + "denoise_sigma": 12 } } diff --git a/setup.py b/setup.py index 6008341..a7d79dc 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,42 @@ -"""Setuptools entry point; bundle rules are read from pyproject.toml.""" +"""Setuptools entry point; bundle rules are read from pyproject.toml. + +``dev_install`` bundle items (configs/sensors) are staged into the ``orisys`` +package during build so ``pip install .`` places them at +``site-packages/orisys/...`` (importlib / configs resolvers), not under +``sys.prefix/orisys`` via deprecated ``data_files``. +""" + +from __future__ import annotations -from pathlib import Path import sys +from pathlib import Path from setuptools import setup +from setuptools.command.build_py import build_py as _build_py REPO_ROOT = Path(__file__).resolve().parent -sys.path.insert(0, str(REPO_ROOT / "build")) +sys.path.insert(0, str(REPO_ROOT / "build" / "common")) -from pyproject_utils import bundle_data_files +from pyproject_utils import stage_dev_bundle_into # noqa: E402 -setup(data_files=bundle_data_files(REPO_ROOT)) + +class build_py(_build_py): + def run(self) -> None: + super().run() + # After pure modules are copied, stage configs/sensors into build/lib/orisys. + pkg_dir = Path(self.build_lib) / "orisys" + pkg_dir.mkdir(parents=True, exist_ok=True) + stage_dev_bundle_into(REPO_ROOT, pkg_dir) + + +setup( + cmdclass={"build_py": build_py}, + # Declared so wheels record the staged files; actual copy happens in build_py. + package_data={ + "orisys": [ + "configs/*.json", + "sensors/**/*", + ], + }, + include_package_data=True, +) diff --git a/src/orisys/Stitcher.py b/src/orisys/Stitcher.py index f76c925..f3444d7 100644 --- a/src/orisys/Stitcher.py +++ b/src/orisys/Stitcher.py @@ -1,1206 +1,21 @@ -import os -import numpy as np -import scipy.fft -from scipy import ndimage -import cv2 -# from numba import njit +"""Compatibility shim: prefer ``orisys.stitch.Stitcher``. -# @njit(cache=False) -def _blend_kernel(blend_patch, wsum_patch, cov_patch, src_patch, w_patch): - """Fused incremental weighted-average composite (numba-accelerated).""" - sh = src_patch.shape[0] - sw = src_patch.shape[1] - for y in range(sh): - for x in range(sw): - w = w_patch[y, x] - s0 = np.float32(src_patch[y, x, 0]) - s1 = np.float32(src_patch[y, x, 1]) - s2 = np.float32(src_patch[y, x, 2]) - if cov_patch[y, x] == 0: - blend_patch[y, x, 0] = s0 - blend_patch[y, x, 1] = s1 - blend_patch[y, x, 2] = s2 - wsum_patch[y, x] = w - else: - ws = wsum_patch[y, x] + w - if ws < 1e-6: - ws = 1e-6 - a = w / ws - blend_patch[y, x, 0] += (s0 - blend_patch[y, x, 0]) * a - blend_patch[y, x, 1] += (s1 - blend_patch[y, x, 1]) * a - blend_patch[y, x, 2] += (s2 - blend_patch[y, x, 2]) * a - wsum_patch[y, x] = ws - cov_patch[y, x] = 1 +Historical import path ``from orisys.Stitcher import Stitcher`` remains supported. +""" +from .stitch import Stitcher +from .stitch.backends.numpy_backend import blend_kernel_numpy as _blend_kernel_numpy +from .stitch.config import BLEND_MODES as _BLEND_MODES +from .stitch.config import DEFAULT_BLEND_MODE as _DEFAULT_BLEND_MODE +from .stitch.config import numba_blend_available, resolve_blend_mode as _resolve_blend_mode -class Stitcher: +# Backward-compatible aliases used by older call sites / tests. +_blend_kernel = _blend_kernel_numpy +_NUMBA_AVAILABLE = numba_blend_available() - def __init__(self,isstitch=False, config_stitcher=None,verbose=False): +try: + from orisys_blend.kernel import blend_kernel as _blend_kernel_numba +except Exception: + _blend_kernel_numba = None - # Settings - self.isstitch = isstitch - self.config_stitcher = config_stitcher - self.verbose = verbose - # Find Boxes Related - self.min_area_frac = config_stitcher["min_area_frac"] - self.max_area_frac = config_stitcher["max_area_frac"] - self.max_aspect_ratio = config_stitcher["max_aspect_ratio"] - - # stitch related - self.stitch_valid = False - self.boxes = self.units = None - self.index = config_stitcher["index"] - self.stitch_calibration = self.tile_list = [] - self.ncol = config_stitcher["n"][0] - self.nrow = config_stitcher["n"][1] - self.nominal_ov_pos=config_stitcher["nominal_ov_pos"] - self.margin=config_stitcher["margin"] - self.bound=self.nominal_ov_pos - self.margin - self.row_tol = config_stitcher["row_tol"] - self.box_size = config_stitcher["box_size"] - - # stich related (BF-matching) - self._blend_coef_cache = {} - self._feature_stitch_cache = { - "ref_idx": None, - "moving_indices": None, - "tile_shapes": None, - "transforms": None, - "canvas": {}, - } - self._feature_stitch_frame_idx = 0 - self._feature_stitch_print_every = 30 - self._feature_stitch_pad = 5 - self._feature_stitch_match_brightness = False - - if self.verbose: - print(f"Stitcher initialized with config: {self.config_stitcher}") - - def configure_feature_stitch(self, *, match_brightness=None, pad=None, print_every=None): - """Update runtime options for feature-based stitching.""" - if match_brightness is not None: - self._feature_stitch_match_brightness = bool(match_brightness) - if pad is not None: - self._feature_stitch_pad = max(0, int(pad)) - if print_every is not None: - self._feature_stitch_print_every = max(1, int(print_every)) - - def _get_blend_coef(self, ov_pos, dim, ndim): - key = (ov_pos, dim, ndim) - coef = self._blend_coef_cache.get(key) - if coef is not None: - return coef - blend_coef = np.linspace(1.0, 0.0, ov_pos, dtype=np.float32) - if dim == 1: - blend_coef = blend_coef.reshape(1, ov_pos, 1) if ndim == 3 else blend_coef.reshape(1, ov_pos) - else: - blend_coef = blend_coef.reshape(ov_pos, 1, 1) if ndim == 3 else blend_coef.reshape(ov_pos, 1) - self._blend_coef_cache[key] = blend_coef - return blend_coef - - def stitch_rt_optimized(self, change_sv=False, s_mean=28., v_mean=180., blend=True): - feature_stitched = self.stitch_feature_optimized( - blend=blend, - match_brightness=self._feature_stitch_match_brightness, - pad=self._feature_stitch_pad, - print_every=self._feature_stitch_print_every, - ) - if feature_stitched is not None: - return feature_stitched - - # tile_list = self.tile_list - # stitch_calibration = self.stitch_calibration - # ncol = self.ncol - # nrow = self.nrow - - # img_stitched_height = None - - # for i in range(nrow): - # # Start with first tile of the row - # img_stitched_width = tile_list[i * ncol] - - # # Stitch horizontally within the row - # for j in range(1, ncol): - # img_1 = img_stitched_width - # img_2 = tile_list[i * ncol + j] - - # ov_pos = int(stitch_calibration[i * ncol + j - 1][1]) - # ov_pos, ov1, ov2 = self.__find_ov(img_1, img_2, ov_pos, dim=1) - - # img_stitched_width = self.__stitch_optimized( - # img_1, img_2, ov_pos, ov1, ov2, dim=1, blend=blend - # ) - - # # Stitch rows vertically - # if i == 0: - # img_stitched_height = img_stitched_width - # else: - # width_h = img_stitched_height.shape[1] - # width_w = img_stitched_width.shape[1] - # wdiff = width_h - width_w - - # if wdiff != 0: - # if wdiff > 0: - # padding = ( - # ((0, 0), (0, wdiff), (0, 0)) - # if len(img_stitched_width.shape) == 3 - # else ((0, 0), (0, wdiff)) - # ) - # img_stitched_width = np.pad(img_stitched_width, padding, mode='edge') - # else: - # img_stitched_width = img_stitched_width[:, :width_h] - - # ov_pos_h = int(stitch_calibration[(i + 1) * nrow - 1][1]) - # ov_pos_h, ov_h1, ov_h2 = self.__find_ov( - # img_stitched_height, img_stitched_width, ov_pos_h, dim=0 - # ) - - # img_stitched_height = self.__stitch_optimized( - # img_stitched_height, img_stitched_width, ov_pos_h, ov_h1, ov_h2, dim=0, blend=blend - # ) - - # return img_stitched_height - - def stitch_feature_optimized(self, blend=True, match_brightness=False, pad=5, print_every=30): - """ - Stitch the current tile grid by estimating tile-to-reference transforms once and - reusing them across frames. Falls back to overlap stitching when feature matching - cannot build a valid layout. - """ - plan = self._get_feature_stitch_plan() - if plan is None: - return None - - ref_idx, t_ref, moving_indices, moving_tiles, selected_positions = plan - self._feature_stitch_frame_idx += 1 - frame_idx = self._feature_stitch_frame_idx - tile_shapes = tuple(self.tile_list[pos].shape[:2] for pos in selected_positions) - cache = self._feature_stitch_cache - cache_changed = ( - cache["ref_idx"] != ref_idx - or cache["moving_indices"] != moving_indices - or cache["tile_shapes"] != tile_shapes - ) - - if cache_changed: - cache["ref_idx"] = ref_idx - cache["moving_indices"] = moving_indices - cache["tile_shapes"] = tile_shapes - cache["transforms"] = None - cache["canvas"] = {} - if self.verbose: - print( - "Feature stitch tile selection: " - f"ref_idx={ref_idx}, moving_indices={list(moving_indices)}" - ) - - H_use_list = cache["transforms"] - if H_use_list is None: - do_verbose_first = self.verbose and frame_idx % print_every == 0 - H_use_list = [] - for j, tile in enumerate(moving_tiles): - H_use_j, _ = self._estimate_bfmatcher_stitch_transform( - t_ref, - tile, - verbose=(do_verbose_first and j == 0), - ) - H_use_list.append(H_use_j) - cache["transforms"] = H_use_list - - if not any(H is not None for H in H_use_list): - return None - - return self._render_feature_tiles( - t_ref, - moving_tiles, - H_use_list, - pad=pad, - frame_idx=frame_idx, - print_every=print_every, - verbose=self.verbose, - match_brightness=match_brightness, - canvas_cache=cache["canvas"], - blend=blend, - ) - - # def __find_ov(self, img1, img2, ov_pos, dim=1): - - # _dim = len(img1.shape) - - # if dim == 1: - # if _dim == 3: - # # RGB case - # ov_1 = img1[:, -ov_pos:, :] - # ov_2 = img2[:, :ov_pos, :] - # else: - # # gray scale - # ov_1 = img1[:, -ov_pos:] - # ov_2 = img2[:, :ov_pos] - # elif dim == 0: - # if _dim == 3: - # ov_1 = img1[-ov_pos:, :, :] - # ov_2 = img2[:ov_pos, :, :] - # else: - # # gray scale - # ov_1 = img1[-ov_pos:, :] - # ov_2 = img2[:ov_pos, :] - - # return ov_pos, ov_1, ov_2 - - def _get_feature_stitch_plan(self): - if not self.isstitch: - return None - - if isinstance(self.index, int): - configured_indices = (self.index,) - elif isinstance(self.index, np.ndarray): - configured_indices = tuple(int(i) for i in self.index.tolist()) - else: - configured_indices = tuple(int(i) for i in self.index) - - total_tiles = len(configured_indices) - expected_tiles = self.nrow * self.ncol - if total_tiles <= 1 or total_tiles != expected_tiles or len(self.tile_list) < total_tiles: - return None - - # Use the configured grid order from self.index and keep the same reference tile - # location as the original stitch setup. - ref_pos = (self.nrow // 2) * self.ncol + (self.ncol // 2) - if ref_pos >= total_tiles: - return None - - selected_positions = tuple(range(total_tiles)) - ref_idx = configured_indices[ref_pos] - moving_positions = tuple(pos for pos in selected_positions if pos != ref_pos) - moving_indices = tuple(configured_indices[pos] for pos in moving_positions) - moving_tiles = [self.tile_list[pos] for pos in moving_positions] - return ref_idx, self.tile_list[ref_pos], moving_indices, moving_tiles, selected_positions - - def _tile_to_bgr(self, img): - if img.ndim == 2: - return cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) - return img - - def _tile_to_gray(self, bgr): - if bgr.ndim == 2: - return bgr - return cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY) - - def _ensure_u8_gray(self, gray): - if gray.dtype == np.uint8: - return gray - return cv2.normalize(gray, None, 0, 255, cv2.NORM_MINMAX).astype(np.uint8) - - def _preprocess_gray_for_orb(self, g0, g1): - g0 = cv2.GaussianBlur(g0, (5, 5), 0) - g1 = cv2.GaussianBlur(g1, (5, 5), 0) - g0 = cv2.equalizeHist(g0) - g1 = cv2.equalizeHist(g1) - return g0, g1 - - def _estimate_bfmatcher_stitch_transform( - self, - t0, - t1, - *, - ransac_reproj_thresh=5.0, - ratio=0.9, - orb_nfeatures=5000, - verbose=True, - ): - """ - Estimate a 3x3 transform mapping tile coordinates into the reference tile frame. - """ - t0_bgr = self._tile_to_bgr(t0) - t1_bgr = self._tile_to_bgr(t1) - g0 = self._ensure_u8_gray(self._tile_to_gray(t0_bgr)) - g1 = self._ensure_u8_gray(self._tile_to_gray(t1_bgr)) - - if verbose: - print( - f"BF stitcher debug: t0 dtype={t0.dtype}, t1 dtype={t1.dtype}, " - f"g0 dtype={g0.dtype}, g0 range=({np.min(g0)}, {np.max(g0)}), " - f"g1 dtype={g1.dtype}, g1 range=({np.min(g1)}, {np.max(g1)})" - ) - - g0, g1 = self._preprocess_gray_for_orb(g0, g1) - orb = cv2.ORB_create( - nfeatures=orb_nfeatures, - scaleFactor=1.2, - nlevels=8, - edgeThreshold=15, - fastThreshold=5, - ) - k0, d0 = orb.detectAndCompute(g0, None) - k1, d1 = orb.detectAndCompute(g1, None) - - info = { - "ok": False, - "method": None, - "inliers": 0, - "mean_err": None, - "good_matches": 0, - "k0": k0, - "k1": k1, - "t0_bgr": t0_bgr, - "t1_bgr": t1_bgr, - } - if d0 is None or d1 is None or len(k0) < 4 or len(k1) < 4: - if verbose: - print( - "OpenCV BFMatcher stitcher: not enough features.", - f"k0={0 if k0 is None else len(k0)}", - f"k1={0 if k1 is None else len(k1)}", - f"d0_is_none={d0 is None}", - f"d1_is_none={d1 is None}", - ) - return None, info - - matches = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=False).knnMatch(d0, d1, k=2) - good = [] - for m_n in matches: - if len(m_n) != 2: - continue - m, n = m_n - if m.distance < ratio * n.distance: - good.append(m) - - info["good_matches"] = len(good) - if verbose: - print(f"BFMatcher good matches: {len(good)}") - if len(good) < 4: - return None, info - - pts0 = np.float32([k0[m.queryIdx].pt for m in good]).reshape(-1, 1, 2) - pts1 = np.float32([k1[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) - method = "affine_partial2d" - M, inlier_mask = cv2.estimateAffinePartial2D( - pts1, - pts0, - method=cv2.RANSAC, - ransacReprojThreshold=ransac_reproj_thresh, - ) - if M is not None: - H = np.vstack([M, [0, 0, 1]]).astype(np.float64) - else: - method = "homography" - H, inlier_mask = cv2.findHomography(pts1, pts0, cv2.RANSAC, ransac_reproj_thresh) - - info["method"] = method - info["inliers"] = int(inlier_mask.sum()) if inlier_mask is not None else 0 - if verbose: - print(f"Estimated {method} inliers: {info['inliers']}") - if H is None: - return None, info - - if abs(H[2, 2]) > 1e-12: - H = H / H[2, 2] - - mean_err = 1e9 - if inlier_mask is not None: - inlier_bool = inlier_mask.ravel().astype(bool) - if np.any(inlier_bool): - projected = cv2.perspectiveTransform(pts1[inlier_bool], H) - err = np.linalg.norm( - projected.reshape(-1, 2) - pts0[inlier_bool].reshape(-1, 2), - axis=1, - ) - mean_err = float(err.mean()) if err.size else 1e9 - info["mean_err"] = mean_err - info["ok"] = True - return H, info - - def _translation_from_H_to_ref(self, H): - if H is None: - return 0.0, 0.0 - H = np.asarray(H, dtype=np.float64) - if abs(H[2, 2]) > 1e-12: - H = H / H[2, 2] - pt = H @ np.array([[0.0], [0.0], [1.0]], dtype=np.float64) - return float(pt[0, 0]), float(pt[1, 0]) - - def _tile_weight_map(self, h, w, margin=16): - """ - Linear ramp weight that is 0 at tile edges and 1.0 at margin pixels inward. - Used for feathered blending at tile overlaps. - """ - wy = np.ones(h, dtype=np.float32) - wx = np.ones(w, dtype=np.float32) - m = min(margin, h // 2, w // 2) - if m > 0: - ramp = np.linspace(0.0, 1.0, m + 1, dtype=np.float32)[1:] - wy[:m] = ramp - wy[-m:] = ramp[::-1] - wx[:m] = ramp - wx[-m:] = ramp[::-1] - return wy[:, None] * wx[None, :] - - def _blend_tile_inplace(self, blend, wsum, cov, src, weight, y0, x0, scratch=None): - """Composite a tile onto the canvas using numba-fused weighted average.""" - h, w = src.shape[:2] - H, W = blend.shape[:2] - y0d, x0d = max(0, y0), max(0, x0) - y1d, x1d = min(H, y0 + h), min(W, x0 + w) - if y0d >= y1d or x0d >= x1d: - return - - sy0, sx0 = y0d - y0, x0d - x0 - sh, sw = y1d - y0d, x1d - x0d - - src_f = src[sy0:sy0+sh, sx0:sx0+sw].astype(np.float32, copy=False) - _blend_kernel( - blend[y0d:y1d, x0d:x1d], - wsum[y0d:y1d, x0d:x1d], - cov[y0d:y1d, x0d:x1d], - src_f, - weight[sy0:sy0+sh, sx0:sx0+sw], - ) - - def _mean_gray_bgr(self, bgr): - ch = cv2.mean(bgr) - return (ch[0] + ch[1] + ch[2]) / 3.0 - - def _match_bgr_brightness(self, tile_bgr, ref_mean_gray): - delta = int(round(ref_mean_gray - self._mean_gray_bgr(tile_bgr))) - if delta == 0: - return tile_bgr - lut = np.clip(np.arange(256, dtype=np.int16) + delta, 0, 255).astype(np.uint8) - return cv2.LUT(tile_bgr, lut) - - def _largest_fully_covered_rect(self, cov): - if cov.size == 0: - return None - - mask = cov > 0 - if not np.any(mask): - return None - - h, w = mask.shape - heights = np.zeros(w, dtype=np.int32) - best_area = 0 - best_rect = None - for y in range(h): - heights = np.where(mask[y], heights + 1, 0) - stack = [] - for x in range(w + 1): - cur_h = heights[x] if x < w else 0 - start = x - while stack and stack[-1][1] > cur_h: - start0, h0 = stack.pop() - area = h0 * (x - start0) - if area > best_area: - best_area = area - best_rect = (y - h0 + 1, y + 1, start0, x) - start = start0 - if not stack or stack[-1][1] < cur_h: - stack.append((start, cur_h)) - return best_rect - - def _compute_final_crop_rect(self, cov): - valid = cov > 0 - if not np.any(valid): - return None - - ys, xs = np.where(valid) - by0, by1 = int(ys.min()), int(ys.max()) + 1 - bx0, bx1 = int(xs.min()), int(xs.max()) + 1 - rect2 = self._largest_fully_covered_rect(cov[by0:by1, bx0:bx1]) - if rect2 is None: - return by0, by1, bx0, bx1 - - y0, y1, x0, x1 = rect2 - return by0 + y0, by0 + y1, bx0 + x0, bx0 + x1 - - def _geom_cache_key(self, t_ref_bgr, tiles_bgr_c, pad): - ref_h, ref_w = t_ref_bgr.shape[:2] - shapes = tuple(tb.shape[:2] for tb in tiles_bgr_c) - return ref_h, ref_w, shapes, pad - - def _render_feature_tiles( - self, - t_ref_bgr, - tiles_bgr, - H_use_list, - *, - pad=5, - frame_idx=0, - print_every=30, - verbose=False, - match_brightness=False, - canvas_cache=None, - blend=True, - ): - """ - Composite multiple tiles into one canvas using translation only, where each tile - origin maps into the reference tile coordinates. - """ - if H_use_list is None: - return None - - if canvas_cache is None: - canvas_cache = {} - do_profile = verbose and frame_idx % print_every == 0 - t_profile0 = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 - - if t_ref_bgr.ndim == 2: - t_ref_bgr = cv2.cvtColor(t_ref_bgr, cv2.COLOR_GRAY2BGR) - h0c, w0c = t_ref_bgr.shape[:2] - if match_brightness: - ref_mean_gray = self._mean_gray_bgr(t_ref_bgr) - tiles_bgr_c = [ - self._match_bgr_brightness( - tb if tb.ndim == 3 else cv2.cvtColor(tb, cv2.COLOR_GRAY2BGR), - ref_mean_gray, - ) - for tb in tiles_bgr - ] - elif any(tb.ndim == 2 for tb in tiles_bgr): - tiles_bgr_c = [tb if tb.ndim == 3 else cv2.cvtColor(tb, cv2.COLOR_GRAY2BGR) for tb in tiles_bgr] - else: - tiles_bgr_c = tiles_bgr - t_after_brightness = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 - - geom_key = self._geom_cache_key(t_ref_bgr, tiles_bgr_c, pad) - geom_cache_hit = canvas_cache.get("geom_key") == geom_key - if not geom_cache_hit: - ref_corners = np.array([[0, 0], [w0c, 0], [w0c, h0c], [0, h0c]], dtype=np.float64) - pts = [ref_corners] - for tile_bgr, H_use in zip(tiles_bgr_c, H_use_list): - if H_use is None: - continue - hi, wi = tile_bgr.shape[:2] - tx, ty = self._translation_from_H_to_ref(H_use) - pts.append( - np.array( - [[tx, ty], [tx + wi, ty], [tx + wi, ty + hi], [tx, ty + hi]], - dtype=np.float64, - ) - ) - - all_xy = np.concatenate(pts, axis=0) - x_min_f = float(all_xy[:, 0].min()) - pad - y_min_f = float(all_xy[:, 1].min()) - pad - x_max_f = float(all_xy[:, 0].max()) + pad - y_max_f = float(all_xy[:, 1].max()) + pad - out_w = max(1, int(np.ceil(x_max_f - x_min_f))) - out_h = max(1, int(np.ceil(y_max_f - y_min_f))) - ref_y0 = int(np.floor(-y_min_f)) - ref_x0 = int(np.floor(-x_min_f)) - - pastes = [] - for H_use in H_use_list: - if H_use is None: - pastes.append(None) - continue - tx, ty = self._translation_from_H_to_ref(H_use) - pastes.append((int(np.floor(ty - y_min_f)), int(np.floor(tx - x_min_f)))) - - _dr_y0 = max(0, ref_y0) - _dr_x0 = max(0, ref_x0) - _dr_y1 = min(out_h, ref_y0 + h0c) - _dr_x1 = min(out_w, ref_x0 + w0c) - for _tb, _pa in zip(tiles_bgr_c, pastes): - if _pa is None: - continue - _py, _px = _pa - _th, _tw = _tb.shape[:2] - _dr_y0 = min(_dr_y0, max(0, _py)) - _dr_x0 = min(_dr_x0, max(0, _px)) - _dr_y1 = max(_dr_y1, min(out_h, _py + _th)) - _dr_x1 = max(_dr_x1, min(out_w, _px + _tw)) - - canvas_cache["geom_key"] = geom_key - canvas_cache["out_w"] = out_w - canvas_cache["out_h"] = out_h - canvas_cache["ref_y0"] = ref_y0 - canvas_cache["ref_x0"] = ref_x0 - canvas_cache["pastes"] = pastes - canvas_cache["x_min_f"] = x_min_f - canvas_cache["y_min_f"] = y_min_f - canvas_cache["crop_rect"] = None - canvas_cache["dirty_rect"] = (_dr_y0, _dr_y1, _dr_x0, _dr_x1) - canvas_cache["blend_buf"] = np.zeros((out_h, out_w, 3), dtype=np.float32) - canvas_cache["wsum_buf"] = np.zeros((out_h, out_w), dtype=np.float32) - canvas_cache["cov_buf"] = np.zeros((out_h, out_w), dtype=np.uint8) - canvas_cache["weight_maps"] = {} - else: - out_w = canvas_cache["out_w"] - out_h = canvas_cache["out_h"] - ref_y0 = canvas_cache["ref_y0"] - ref_x0 = canvas_cache["ref_x0"] - pastes = canvas_cache["pastes"] - t_after_geom = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 - - blend_buf = canvas_cache["blend_buf"] - wsum = canvas_cache["wsum_buf"] - cov = canvas_cache["cov_buf"] - weight_maps = canvas_cache["weight_maps"] - if geom_cache_hit: - _dr = canvas_cache.get("dirty_rect") - if _dr is not None: - _dy0, _dy1, _dx0, _dx1 = _dr - blend_buf[_dy0:_dy1, _dx0:_dx1] = 0 - wsum[_dy0:_dy1, _dx0:_dx1] = 0 - cov[_dy0:_dy1, _dx0:_dx1] = 0 - else: - blend_buf.fill(0) - wsum.fill(0) - cov.fill(0) - t_after_reset = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 - - blend_margin = 16 if blend else 0 - - blended_tiles = 0 - for tile_bgr, H_use, paste in zip(tiles_bgr_c, H_use_list, pastes): - if H_use is None or paste is None: - continue - y0, x0 = paste - th, tw = tile_bgr.shape[:2] - wkey = (th, tw, blend_margin) - if wkey not in weight_maps: - weight_maps[wkey] = self._tile_weight_map(th, tw, blend_margin) - self._blend_tile_inplace(blend_buf, wsum, cov, tile_bgr, weight_maps[wkey], y0, x0) - blended_tiles += 1 - - ref_wkey = (h0c, w0c, blend_margin) - if ref_wkey not in weight_maps: - weight_maps[ref_wkey] = self._tile_weight_map(h0c, w0c, blend_margin) - self._blend_tile_inplace(blend_buf, wsum, cov, t_ref_bgr, weight_maps[ref_wkey], ref_y0, ref_x0) - t_after_blend = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 - - crop_rect = canvas_cache.get("crop_rect") - if crop_rect is None: - crop_rect = self._compute_final_crop_rect(cov) - canvas_cache["crop_rect"] = crop_rect - if crop_rect is not None: - y0, y1, x0, x1 = crop_rect - roi = blend_buf[y0:y1, x0:x1] - else: - roi = blend_buf - rh, rw = roi.shape[:2] - stitch_buf = canvas_cache.get("stitch_buf") - if stitch_buf is None or stitch_buf.shape[0] != rh or stitch_buf.shape[1] != rw: - stitch_buf = np.empty((rh, rw, 3), dtype=np.uint8) - canvas_cache["stitch_buf"] = stitch_buf - cv2.convertScaleAbs(roi, dst=stitch_buf) - stitched = stitch_buf - t_after_finalize = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 - - if do_profile: - xmf = canvas_cache.get("x_min_f", 0.0) - ymf = canvas_cache.get("y_min_f", 0.0) - print( - f"stitch_feature_optimized: canvas out_w={out_w}, out_h={out_h}, " - f"x_min={xmf:.2f}, y_min={ymf:.2f}" - ) - print( - "stitch_feature_optimized steps: " - f"brightness={t_after_brightness - t_profile0:.6f}s, " - f"geom={t_after_geom - t_after_brightness:.6f}s ({'hit' if geom_cache_hit else 'miss'}), " - f"reset={t_after_reset - t_after_geom:.6f}s, " - f"blend={t_after_blend - t_after_reset:.6f}s ({blended_tiles + 1} tiles incl ref), " - f"finalize={t_after_finalize - t_after_blend:.6f}s, " - f"total={t_after_finalize - t_profile0:.6f}s" - ) - - return stitched - - def find_stitch_boxes(self,img_src,show = False): - - img_corr = self.__correct_edge_bloom(img_src) - self.__detect_nonblack_regions(img_corr,show = show, row_tol = self.row_tol) - return len(self.boxes), self.boxes - - # return self.boxes - - - def _copy_into_tile_buffer(self, src_view, dst, flip_x=True, flip_y=True): - if flip_x and flip_y: - cv2.flip(src_view, -1, dst) - elif flip_x: - cv2.flip(src_view, 1, dst) - elif flip_y: - cv2.flip(src_view, 0, dst) - else: - np.copyto(dst, src_view) - - def get_stitch_unit(self, img, flip_x=True, flip_y=True,index = np.arange(15)): - """ - Extract each tile given by boxes (x0,y0,x1,y1), default flip both X and Y due to inverting lens. - - img: source image (use corrected image if you prefer) - - boxes: list of (x0,y0,x1,y1) - - flip_x, flip_y: booleans (performed sequentially: X then Y) - - overlay_original: if True paste onto original image copy, otherwise onto blank background - - bg_color: background color for blank canvas (B,G,R) - Returns: canvas image and list of flipped tiles as (x0,y0,flipped_tile) - """ - h, w = img.shape[:2] - active_index = index if self.units is None else self.index - if isinstance(active_index, int): - selected_index = {active_index} - else: - selected_index = set(active_index) - flipped_units = [] if self.units is None else self.units - - for i, (x0, y0, x1, y1) in enumerate(self.boxes): - if i not in selected_index: - continue - - x0 = max(0, int(x0)); y0 = max(0, int(y0)) - x1 = min(w, int(x1)); y1 = min(h, int(y1)) - if x1 <= x0 or y1 <= y0: - continue - - src_view = img[y0:y1, x0:x1] - if self.units is None: - tile = np.empty_like(src_view) - self._copy_into_tile_buffer(src_view, tile, flip_x=flip_x, flip_y=flip_y) - flipped_units.append((x0, y0, tile)) - else: - _, _, tile = flipped_units[i] - if tile.shape != src_view.shape or tile.dtype != src_view.dtype: - tile = np.empty_like(src_view) - self._copy_into_tile_buffer(src_view, tile, flip_x=flip_x, flip_y=flip_y) - flipped_units[i] = (x0, y0, tile) - - self.units = flipped_units - - return True - - def extract_tiles(self): - if not self.isstitch: - # tile_idx = self.__get_tile_index_by_position(self.pos_default[7][0], self.pos_default[7][1]) - tile_idx = 7 - # if tile_idx is None: - # tile_idx = 7 - # Ensure index is always iterable (list or array) - self.index = [tile_idx] if isinstance(tile_idx, int) else tile_idx - self.ncol = 1 - self.nrow = 1 - - tile_list = self.__get_tile_by_index(self.index) - self.tile_list = tile_list - - return tile_list - - # def stitch_cal(self): - - # # init - # tile_list = self.tile_list - # img_stitched_height = None - # stitch_calibration = [] - # # Flip col and row for porting - # nrow, ncol = self.ncol, self.nrow - # if not self.isstitch: - # return self.tile_list[0], None - - # for i in range(ncol): - # img_stitched_width = None - # for j in range(nrow): - # if j == 0: - # img_j0 = tile_list[i*ncol] - # img_stitched_width = img_j0 - # else: - # img_1 = tile_list[i*ncol + j-1] - # img_2 = tile_list[i*ncol + j] - - # ov_pos, ov1, ov2, c_corr = self.__get_ov(img_1.astype(np.float32), img_2.astype(np.float32), dim=1) - - # stitch_calibration.append(["hor {}".format(j-1),ov_pos]) - - # # stitch images with stacking - # img_stitched_width = self.__stitch_optimized(img_stitched_width, img_2, ov_pos, ov1, ov2, blend=True, dim=1) - - # if i == 0: - # img_stitched_height = img_stitched_width - # stitch_calibration.append(["ver {}".format(i),0]) - # pass - # else: - # width = img_stitched_height.shape[1] - # width_new = img_stitched_width.shape[1] - # wdiff = width - width_new - # if wdiff > 0: - # img_stitched_width = np.hstack((img_stitched_width, img_stitched_width[:, -wdiff:])) - # elif wdiff < 0: - # img_stitched_width = img_stitched_width[:, :wdiff] - - # ov_pos_h, ov_h1, ov_h2, c_corr = self.__get_ov(img_stitched_height, img_stitched_width, dim=0) - - # stitch_calibration.append(["ver {}".format(i),ov_pos_h]) - - # img_stitched_height = self.__stitch_optimized(img_stitched_height, img_stitched_width, ov_pos_h, ov_h1, ov_h2, blend=True, dim=0) - - # self.stitch_calibration = stitch_calibration - - # return img_stitched_height, stitch_calibration - - # def __stitch_optimized(self, img1, img2, ov_pos, ov1, ov2, blend=True, dim=1): - # _dim = len(img1.shape) - - # # Clamp overlap for safety - # if dim == 1: - # ov_pos = int(max(0, min(ov_pos, img1.shape[1], img2.shape[1]))) - # else: - # ov_pos = int(max(0, min(ov_pos, img1.shape[0], img2.shape[0]))) - - # # Fast blend path with cached coefficient tensors - # if not blend or ov_pos <= 1: - # ov = ov1 - # else: - # blend_coef = self._get_blend_coef(ov_pos, dim, _dim) - # ov = ( - # ov1.astype(np.float32, copy=False) * blend_coef - # + ov2.astype(np.float32, copy=False) * (1.0 - blend_coef) - # ).astype(np.uint8) - - # # Build stitched output with one target allocation and slice writes - # if dim == 1: - # left_w = img1.shape[1] - ov_pos - # right_w = img2.shape[1] - ov_pos - # out_w = left_w + ov_pos + right_w - # if _dim == 3: - # stitched_img = np.empty((img1.shape[0], out_w, img1.shape[2]), dtype=img1.dtype) - # if left_w > 0: - # stitched_img[:, :left_w, :] = img1[:, :left_w, :] - # if ov_pos > 0: - # stitched_img[:, left_w:left_w + ov_pos, :] = ov - # if right_w > 0: - # stitched_img[:, left_w + ov_pos:, :] = img2[:, ov_pos:, :] - # else: - # stitched_img = np.empty((img1.shape[0], out_w), dtype=img1.dtype) - # if left_w > 0: - # stitched_img[:, :left_w] = img1[:, :left_w] - # if ov_pos > 0: - # stitched_img[:, left_w:left_w + ov_pos] = ov - # if right_w > 0: - # stitched_img[:, left_w + ov_pos:] = img2[:, ov_pos:] - # else: - # top_h = img1.shape[0] - ov_pos - # bottom_h = img2.shape[0] - ov_pos - # out_h = top_h + ov_pos + bottom_h - # if _dim == 3: - # stitched_img = np.empty((out_h, img1.shape[1], img1.shape[2]), dtype=img1.dtype) - # if top_h > 0: - # stitched_img[:top_h, :, :] = img1[:top_h, :, :] - # if ov_pos > 0: - # stitched_img[top_h:top_h + ov_pos, :, :] = ov - # if bottom_h > 0: - # stitched_img[top_h + ov_pos:, :, :] = img2[ov_pos:, :, :] - # else: - # stitched_img = np.empty((out_h, img1.shape[1]), dtype=img1.dtype) - # if top_h > 0: - # stitched_img[:top_h, :] = img1[:top_h, :] - # if ov_pos > 0: - # stitched_img[top_h:top_h + ov_pos, :] = ov - # if bottom_h > 0: - # stitched_img[top_h + ov_pos:, :] = img2[ov_pos:, :] - - # return stitched_img - - # def __get_ov(self, img1, img2, dim=1): - # ''' - # default dimension is along axis 1 (width of the images) - # ''' - # # pass parameters - # nominal_ov = self.nominal_ov_pos - # bound_ov = self.bound - - # cross_corr = [] - # h1, w1 = img1.shape[:2] - # h2, w2 = img2.shape[:2] - # # clamp tested overlap to valid range (1..min dimension) - # max_possible = w1 if dim == 1 else h1 - # max_possible = min(max_possible, w2 if dim == 1 else h2) - # lo = max(1, nominal_ov - bound_ov) - # hi = min(max_possible, nominal_ov + bound_ov) - # ov_space = range(lo, hi + 1) - - # for ov in ov_space: - # if dim == 1: - # ov_area1 = img1[:, max(0, w1 - ov):, :] - # ov_area2 = img2[:, :min(ov, w2), :] - # elif dim == 0: - # ov_area1 = img1[max(0, h1 - ov):, :, :] - # ov_area2 = img2[:min(ov, h2), :, :] - - # # require exact shape match (detect_nonblack_regions guarantees uniform tile size) - # if ov_area1.size == 0 or ov_area2.size == 0 or ov_area1.shape != ov_area2.shape: - # cross_corr.append(-1.0) - # continue - - # # multi-channel cross correlation (average over channels) - # corr = 0.0 - # for i in range(min(3, ov_area1.shape[2])): - # a = ov_area1[:, :, i].astype(np.float32) - # b = ov_area2[:, :, i].astype(np.float32) - # denom = np.sqrt(np.sum(a * a) * np.sum(b * b)) + 1e-12 - # corr += float(np.sum(a * b) / denom) - # corr = corr / float(min(3, ov_area1.shape[2])) - # cross_corr.append(corr) - - # # pick best overlap (ignore invalid entries) - # best_idx = int(np.argmax(cross_corr)) - # ov_pos = list(ov_space)[best_idx] - - # # extract the overlap regions (assume shapes match) - # if dim == 1: - # ov_1 = img1[:, max(0, w1 - ov_pos):, :].copy() - # ov_2 = img2[:, :min(ov_pos, w2), :].copy() - # else: - # ov_1 = img1[max(0, h1 - ov_pos):, :, :].copy() - # ov_2 = img2[:min(ov_pos, h2), :, :].copy() - # return ov_pos, ov_1, ov_2, cross_corr - - -# Helper functions for stitching - def __get_tile_index_by_position(self, x, y, tol=20): - """ - Find flipped tile whose top-left matches (x,y) within tol pixels. - flipped_units: list of (x,y,tile) - """ - i= 0 - for xi, yi, tile in self.units: - if abs(int(x) - int(xi)) <= tol and abs(int(y) - int(yi)) <= tol: - # return tile - return i - i += 1 - return None - - def __get_tile_by_index(self, idx): - """ - Retrieve tile by index - """ - if isinstance(idx, int): - idx = [idx] - - if len(idx) > len(self.units): - raise ValueError("Index exceeds number of tiles.") - - tile_list = [] - for i in range(len(idx)): - [xi, yi, tile] = self.units[idx[i]] - tile_list.append(tile) - - return tile_list - - def __correct_edge_bloom(self, img, method="divide", kernel_size=None, clip=True): - """ - Simple illumination correction to reduce edge bloom / vignetting. Only for getting tile boxes. - - method: "divide" (default) or "subtract" - * divide: img_corrected = (img / (bg + eps)) * gain -- preserves colors - * subtract: img_corrected = img - (bg - mean_bg) -- removes low-freq background - - kernel_size: size for large-kernel blur used as background estimate (odd int). - If None, chosen from image size. - Returns corrected uint8 image. - """ - if img is None: - raise ValueError("Input image is None.") - - h, w = img.shape[:2] - if kernel_size is None: - # choose large kernel relative to image size, ensure odd - k = max(31, min(h, w) // 2) - if k % 2 == 0: - k += 1 - kernel_size = k - # convert to float32 for stable ops - img_f = img.astype(np.float32) - # estimate background by large Gaussian blur (works well for smooth bloom) - bg = cv2.GaussianBlur(img_f, (kernel_size, kernel_size), 0) - if method == "subtract": - # remove low-frequency background but keep overall brightness - mean_bg = bg.mean(axis=(0, 1), keepdims=True) - corr = img_f - (bg - mean_bg) - else: - # divide: normalized gain correction - eps = 1e-6 - # compute gain so that mid-level stays near 128 - mean_bg = bg.mean(axis=(0, 1), keepdims=True) + eps - gain = (mean_bg * 0.5) # target scale; 0.5 keeps values reasonable - corr = (img_f / (bg + eps)) * gain - if clip: - corr = np.clip(corr, 0, 255).astype(np.uint8) - else: - corr = corr.astype(np.uint8) - return corr - - def __get_unit_pos(self,idx): - - if isinstance(idx, int): - idx = [idx] - - if len(idx) > len(self.units): - raise ValueError("Index exceeds number of vision units.") - - pos_list = [] - for i in range(len(idx)): - [xi, yi, tile] = self.units[idx[i]] - pos_list.append((xi, yi)) - - return pos_list - - # Computation Functions - def __detect_nonblack_regions(self, img, morph_k=3, n_boxes_target=15, bk_thresh_step=10, bk_thresh_max=100,show = False, row_tol = 0.4): - """ - Return binary mask of non-black regions and list of bounding boxes (x,y,w,h). - Uses adaptive thresholding on grayscale + small morphology to be robust to illumination. - - New behavior: after finding candidate boxes, center-crop all boxes to the same - width/height (smallest found) so returned boxes share a common size. - """ - if img is None: - raise ValueError("Input image is None.") - - h, w = img.shape[:2] - n_boxes = 0 - bk_thresh = 0 # initial black threshold - bk_thresh_step = 5 - bk_thresh_max = 150 - - # Also include a low global V threshold to capture colored but dark tiles - hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) - v = hsv[:,:,2] - ww = hh = 0 - while n_boxes != n_boxes_target and bk_thresh <= bk_thresh_max: - _, v_mask = cv2.threshold(v, bk_thresh, 255, cv2.THRESH_BINARY) # tune 50 if needed - - mask = v_mask.copy() - # Exclude left and right boundary (20 pixels each) from detection - edge = 20 - if w > edge * 2: - mask[:, :edge] = 0 - mask[:, -edge:] = 0 - - # Morphology: remove small specks, fill small holes - k = max(1, morph_k) - kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (k, k)) - mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel, iterations=1) - mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel, iterations=2) - # cv2.imshow("mask", mask) - # cv2.waitKey(0) - # find contours of candidate regions - contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - - boxes = [] - for c in contours: - x,y,ww,hh = cv2.boundingRect(c) - boxes.append((x,y,ww,hh)) - - # filter boxes by size (ignore too small or too large tiles) - boxes = self.__filter_boxes_by_size(boxes, (h, w)) - n_boxes = len(boxes) - bk_thresh += bk_thresh_step - - print("Detected {} candidate boxes with bk_thresh={}".format(n_boxes, bk_thresh - bk_thresh_step)) - if n_boxes == 0: - self.boxes = [] - return n_boxes - # compute common center-crop size: use minimum width/height among boxes - widths = [b[2] for b in boxes] - heights = [b[3] for b in boxes] - # target_w = max(1,int(min(widths))) - # target_h = max(1,int(min(heights))) - target_w = self.box_size[0] - target_h = self.box_size[1] - - # center-crop each box to (target_w, target_h) inside original box bounds - cropped_boxes = [] - for (x, y, ww, hh) in boxes: - cx = x + ww // 2 - cy = y + hh // 2 - nx0 = int(cx - target_w // 2) - ny0 = int(cy - target_h // 2) - # clamp to image bounds - nx0 = max(0, min(nx0, w - target_w)) - ny0 = max(0, min(ny0, h - target_h)) - # Convert to (x0, y0, x1, y1) format for direct image region retrieval - nx1 = nx0 + target_w - ny1 = ny0 + target_h - cropped_boxes.append((nx0, ny0, nx1, ny1)) - - # sort boxes left-to-right, top-to-bottom (useful for grid) - # Use quantized y-coordinates to group boxes into rows, making sorting robust to noise - if len(cropped_boxes) > 0: - # Calculate row tolerance based on typical box height (use median height) - heights = [b[3] - b[1] for b in cropped_boxes] - median_height = np.median(heights) if heights else target_h - # Use a fraction of the box height as tolerance (e.g., 20% of box height) - row_tolerance = max(1, int(median_height * row_tol)) - - # Quantize y-coordinates to group boxes into rows - def get_row_key(box): - y = box[1] # y0 coordinate - # Round y to nearest multiple of row_tolerance to group into rows - quantized_y = (y // row_tolerance) * row_tolerance - return (quantized_y, box[0]) # Sort by quantized row, then by x - - cropped_boxes = sorted(cropped_boxes, key=get_row_key) - else: - cropped_boxes = sorted(cropped_boxes, key=lambda b: (b[1], b[0])) - - self.boxes = cropped_boxes - img_with_boxes = self.draw_boxes(img= img) - if show: - cv2.imshow("Detected Boxes", img_with_boxes) - cv2.waitKey(0) - - img_path = "./config/compound_eye.png" - if os.path.exists(img_path): - cv2.imwrite(img_path, img_with_boxes) - else: - os.makedirs(os.path.dirname(img_path), exist_ok=True) - cv2.imwrite(img_path, img_with_boxes) - # print("Check if compound eye detection image valid: {}".format(img_path)) - return cropped_boxes - - def draw_boxes(self, img, thickness=2): - """ - Draw bounding boxes on the image. - - img: input image (will be copied, not modified in-place) - - thickness: line thickness - Returns: image with drawn boxes - """ - if self.boxes is None or len(self.boxes) == 0: - print("No boxes to draw") - return img - - img_draw = img.copy() - color=(0, 255, 0) - - for i, (x0, y0, x1, y1) in enumerate(self.boxes): - # Draw rectangle - cv2.rectangle(img_draw, (x0, y0), (x1, y1), color, thickness) - - # Draw index - text_size = cv2.getTextSize(str(i), cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2)[0] - text_pos = (x0 + 5, y0 + text_size[1] + 5) - # Draw text background and text - cv2.rectangle(img_draw, (text_pos[0] - 2, text_pos[1] - text_size[1] - 2), - (text_pos[0] + text_size[0] + 2, text_pos[1] + 2), (0, 0, 0), -1) - cv2.putText(img_draw, str(i), text_pos, cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) - - return img_draw - - - - def __filter_boxes_by_size(self, boxes, img_shape): - """ - Remove boxes that are too small or too large relative to the image. - - boxes: list of (x,y,w,h) - - img_shape: (height, width) - - min_area_frac / max_area_frac: allowed area fraction of image - Returns filtered list of boxes. - """ - h, w = img_shape - img_area = h * w - min_area = img_area * float(self.min_area_frac) - max_area = img_area * float(self.max_area_frac) - filtered = [] - for (x,y,ww,hh) in boxes: - area = ww * hh - if area < min_area or area > max_area or (abs(ww - hh) > self.max_aspect_ratio): - continue - filtered.append((x,y,ww,hh)) - return filtered - - \ No newline at end of file +__all__ = ["Stitcher"] diff --git a/src/orisys/__init__.py b/src/orisys/__init__.py index 0b7eb55..8871750 100644 --- a/src/orisys/__init__.py +++ b/src/orisys/__init__.py @@ -5,13 +5,16 @@ A Python library for processing and analyzing optical tactile sensor data, including image stitching, optical flow tracking, and deformation decomposition. """ -__version__ = "0.4.0" +__version__ = "0.5.0" __author__ = "Orisys" from .sensor import Sensor from . import util from . import configs from . import finger +from . import signal_processing +from . import capture +from .frame_processors import FFCFrameProcessor CompoundEyeSensor = Sensor CameraSensor = Sensor @@ -32,7 +35,16 @@ __all__ = [ # finger runtime helpers 'finger', + # signal helpers + 'signal_processing', + + # capture helpers + 'capture', + # configs utilities functions 'list_config', - 'export_config' + 'export_config', + + # test + 'FFCFrameProcessor' ] diff --git a/src/orisys/capture/__init__.py b/src/orisys/capture/__init__.py new file mode 100644 index 0000000..a2f8148 --- /dev/null +++ b/src/orisys/capture/__init__.py @@ -0,0 +1,45 @@ +"""Capture subsystem: OpenCV source, optional background thread, discovery.""" + +from .config import ( + MAX_SIMULTANEOUS_CAPTURES, + CaptureConfig, + CaptureHealth, + CaptureStatus, + ReconnectPolicy, +) +from .discovery import ( + DEFAULT_LAPTOP_WEBCAM_PATTERNS, + DEFAULT_MAX_CAMERA_INDEX, + CameraFilter, + CameraInfo, + list_cameras, + pick_default_camera, +) +from .source import ( + CaptureSource, + CaptureWorker, + is_live_camera, + normalize_source, + open_capture, + open_opencv_capture, +) + +__all__ = [ + "MAX_SIMULTANEOUS_CAPTURES", + "DEFAULT_MAX_CAMERA_INDEX", + "DEFAULT_LAPTOP_WEBCAM_PATTERNS", + "CaptureConfig", + "CaptureHealth", + "CaptureStatus", + "ReconnectPolicy", + "CaptureSource", + "CaptureWorker", + "CameraInfo", + "CameraFilter", + "list_cameras", + "pick_default_camera", + "normalize_source", + "is_live_camera", + "open_opencv_capture", + "open_capture", +] diff --git a/src/orisys/capture/config.py b/src/orisys/capture/config.py new file mode 100644 index 0000000..536c00d --- /dev/null +++ b/src/orisys/capture/config.py @@ -0,0 +1,131 @@ +"""Capture configuration, health state, and reconnect policy.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Optional, Sequence, Tuple, Union + +MAX_SIMULTANEOUS_CAPTURES = 10 + + +class CaptureHealth(str, Enum): + """Lifecycle / health state for a capture source.""" + + CLOSED = "closed" + CONNECTING = "connecting" + CONNECTED = "connected" + RECONNECTING = "reconnecting" + FAILED = "failed" + + +SourceType = Union[int, str] + + +@dataclass(frozen=True) +class ReconnectPolicy: + """Auto-reconnect settings for live cameras. + + Parameters + ---------- + enabled: + When False, failed reads do not attempt reopen. + max_attempts: + Maximum reopen attempts per disconnect episode. ``None`` means unlimited. + initial_backoff_s: + First wait before a reconnect attempt. + max_backoff_s: + Upper bound for exponential backoff. + fail_threshold: + Consecutive failed reads that trigger reconnect. + stale_timeout_s: + If no frame arrives for this many seconds while connected, treat as stale + and reconnect. ``None`` disables stale detection. + """ + + enabled: bool = False + max_attempts: Optional[int] = None + initial_backoff_s: float = 0.25 + max_backoff_s: float = 5.0 + fail_threshold: int = 5 + stale_timeout_s: Optional[float] = 2.0 + + def next_backoff(self, attempt: int) -> float: + """Exponential backoff capped at ``max_backoff_s`` (attempt is 0-based).""" + delay = self.initial_backoff_s * (2 ** max(0, attempt)) + return min(delay, self.max_backoff_s) + + +@dataclass +class CaptureConfig: + """Configuration for opening and running a capture source.""" + + source: SourceType + backend: str = "auto" + width: Optional[int] = None + height: Optional[int] = None + fps: Optional[float] = None + fourcc: Optional[str] = "MJPG" + background: bool = True + reconnect: ReconnectPolicy = field(default_factory=ReconnectPolicy) + verbose: bool = False + # Optional discovery filters (used by list/open helpers, not open itself). + name_include: Optional[Sequence[str]] = None + name_exclude: Optional[Sequence[str]] = None + vid_pid_allowlist: Optional[Sequence[Tuple[int, int]]] = None + min_width: Optional[int] = None + min_height: Optional[int] = None + + def __post_init__(self) -> None: + if not isinstance(self.reconnect, ReconnectPolicy): + raise TypeError("reconnect must be a ReconnectPolicy") + + @classmethod + def from_camera_dict( + cls, + source: SourceType, + camera: dict[str, Any], + *, + reconnect: Optional[ReconnectPolicy] = None, + verbose: bool = False, + **kwargs: Any, + ) -> "CaptureConfig": + """Build config from a sensor JSON ``camera`` section.""" + # Accept deprecated queue/worker kwargs from older callers without error. + kwargs.pop("mode", None) + kwargs.pop("queue_size", None) + kwargs.pop("overflow", None) + if "start_worker" in kwargs: + kwargs.setdefault("background", bool(kwargs.pop("start_worker"))) + res = camera.get("resolution_raw") or [None, None] + width = int(res[0]) if res and res[0] else None + height = int(res[1]) if res and res[1] else None + fps = camera.get("fps") + return cls( + source=source, + width=width, + height=height, + fps=float(fps) if fps is not None else None, + reconnect=reconnect or ReconnectPolicy(), + verbose=verbose, + **kwargs, + ) + + +@dataclass +class CaptureStatus: + """Snapshot of capture health for diagnostics / multi-camera UIs.""" + + health: CaptureHealth = CaptureHealth.CLOSED + source: Optional[SourceType] = None + last_error: Optional[str] = None + last_frame_time: Optional[float] = None + last_seq: int = 0 + dropped_frames: int = 0 + reconnect_attempts: int = 0 + consecutive_failures: int = 0 + width: int = 0 + height: int = 0 + fps: float = 0.0 + backend_name: str = "" + fourcc: str = "" diff --git a/src/orisys/capture/discovery.py b/src/orisys/capture/discovery.py new file mode 100644 index 0000000..0ae7285 --- /dev/null +++ b/src/orisys/capture/discovery.py @@ -0,0 +1,334 @@ +"""Discover cameras with optional platform metadata and filters.""" + +from __future__ import annotations + +import contextlib +import os +import re +import sys +from dataclasses import dataclass, field +from typing import Iterable, Optional, Sequence, Tuple + +import cv2 + +from .config import MAX_SIMULTANEOUS_CAPTURES +from .source import _backend_candidates + +DEFAULT_MAX_CAMERA_INDEX = MAX_SIMULTANEOUS_CAPTURES + +# Common built-in laptop webcam name fragments (case-insensitive). +DEFAULT_LAPTOP_WEBCAM_PATTERNS: Tuple[str, ...] = ( + r"integrated", + r"built[- ]?in", + r"laptop", + r"facetime", + r"ir\s*camera", + r"rgb\s*camera", + r"hd\s*webcam", + r"hd\s*camera", + r"user[- ]facing", + r"front\s*camera", + r"dell\s*webcam", + r"lenovo\s*easycamera", + r"hp\s*(truevision|webcam|hd)", + r"microsoft\s*life(cam|studio)", + r"realtek", + r"chicony", + r"sonix", +) + + +@dataclass(frozen=True) +class CameraInfo: + """Description of a probed camera device.""" + + index: int + backend_name: str + backend_api: int + width: int + height: int + name: str = "" + path: str = "" + vendor_id: Optional[int] = None + product_id: Optional[int] = None + bus: str = "" + + @property + def source(self) -> str: + return str(self.index) + + def label(self) -> str: + name = self.name.strip() + if name: + return f"{self.index}: {name} ({self.width}x{self.height}) [{self.backend_name}]" + return f"{self.index} ({self.width}x{self.height}) [{self.backend_name}]" + + @property + def vid_pid(self) -> Optional[Tuple[int, int]]: + if self.vendor_id is None or self.product_id is None: + return None + return (int(self.vendor_id), int(self.product_id)) + + +@dataclass(frozen=True) +class CameraFilter: + """Optional filters for ``list_cameras`` / ``pick_default_camera``.""" + + name_include: Optional[Sequence[str]] = None + name_exclude: Optional[Sequence[str]] = None + exclude_laptop_webcams: bool = False + laptop_patterns: Sequence[str] = field(default_factory=lambda: DEFAULT_LAPTOP_WEBCAM_PATTERNS) + vid_pid_allowlist: Optional[Sequence[Tuple[int, int]]] = None + min_width: Optional[int] = None + min_height: Optional[int] = None + prefer_named: bool = True + + def matches(self, cam: CameraInfo) -> bool: + name = (cam.name or "").lower() + if self.min_width is not None and cam.width < int(self.min_width): + return False + if self.min_height is not None and cam.height < int(self.min_height): + return False + if self.vid_pid_allowlist: + allowed = {(int(v), int(p)) for v, p in self.vid_pid_allowlist} + if cam.vid_pid is None or cam.vid_pid not in allowed: + return False + if self.name_include: + if not any(re.search(pat, name, re.I) for pat in self.name_include): + # If name unknown, keep the device unless prefer_named forces drop. + if name: + return False + if self.prefer_named: + return False + if self.name_exclude: + if name and any(re.search(pat, name, re.I) for pat in self.name_exclude): + return False + if self.exclude_laptop_webcams and name: + if any(re.search(pat, name, re.I) for pat in self.laptop_patterns): + return False + return True + + +@contextlib.contextmanager +def _suppress_opencv_noise(): + """Hide OpenCV VideoCapture probe warnings (stderr + OpenCV log).""" + prev_level = None + set_level = None + try: + logging = getattr(getattr(cv2, "utils", None), "logging", None) + if logging is not None: + set_level = getattr(logging, "setLogLevel", None) + get_level = getattr(logging, "getLogLevel", None) + silent = getattr(logging, "LOG_LEVEL_SILENT", 0) + if set_level is not None: + if get_level is not None: + prev_level = get_level() + set_level(silent) + except Exception: + prev_level = None + + devnull = open(os.devnull, "w") + try: + with contextlib.redirect_stderr(devnull): + stderr_fd = sys.stderr.fileno() + saved_fd = os.dup(stderr_fd) + try: + os.dup2(devnull.fileno(), stderr_fd) + yield + finally: + os.dup2(saved_fd, stderr_fd) + os.close(saved_fd) + finally: + devnull.close() + if set_level is not None and prev_level is not None: + try: + set_level(prev_level) + except Exception: + pass + + +def _linux_device_metadata(index: int) -> dict: + """Best-effort V4L2 name / USB VID:PID from sysfs.""" + meta = {"name": "", "path": f"/dev/video{index}", "vendor_id": None, "product_id": None, "bus": ""} + name_path = f"/sys/class/video4linux/video{index}/name" + try: + with open(name_path, "r", encoding="utf-8", errors="ignore") as fh: + meta["name"] = fh.read().strip() + except OSError: + pass + + # Walk up to a USB device directory containing idVendor/idProduct. + device_link = f"/sys/class/video4linux/video{index}/device" + try: + cur = os.path.realpath(device_link) + for _ in range(8): + vend = os.path.join(cur, "idVendor") + prod = os.path.join(cur, "idProduct") + if os.path.isfile(vend) and os.path.isfile(prod): + with open(vend, "r", encoding="utf-8") as fh: + meta["vendor_id"] = int(fh.read().strip(), 16) + with open(prod, "r", encoding="utf-8") as fh: + meta["product_id"] = int(fh.read().strip(), 16) + meta["bus"] = "usb" + break + parent = os.path.dirname(cur) + if parent == cur: + break + cur = parent + except (OSError, ValueError): + pass + return meta + + +def _windows_device_names() -> dict[int, dict]: + """Best-effort mapping of camera index → name / hardware id via PowerShell. + + Index order is not guaranteed to match OpenCV; names are matched later by + probing order best-effort. Returns ``{ordinal: meta}`` in enumeration order. + """ + if sys.platform != "win32": + return {} + try: + import json + import subprocess + + # Camera devices from PnP; keep this soft-fail for machines without cameras. + cmd = [ + "powershell", + "-NoProfile", + "-Command", + ( + "Get-PnpDevice -Class Camera,Image -Status OK -ErrorAction SilentlyContinue | " + "Select-Object FriendlyName, InstanceId | ConvertTo-Json -Compress" + ), + ] + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=5, check=False) + if proc.returncode != 0 or not proc.stdout.strip(): + return {} + data = json.loads(proc.stdout) + if isinstance(data, dict): + data = [data] + out: dict[int, dict] = {} + for i, item in enumerate(data): + name = str(item.get("FriendlyName") or "").strip() + instance = str(item.get("InstanceId") or "") + vid = pid = None + m = re.search(r"VID_([0-9A-Fa-f]{4}).*PID_([0-9A-Fa-f]{4})", instance) + if m: + vid = int(m.group(1), 16) + pid = int(m.group(2), 16) + out[i] = { + "name": name, + "path": "", + "vendor_id": vid, + "product_id": pid, + "bus": "usb" if vid is not None else "", + } + return out + except Exception: + return {} + + +def _device_metadata(index: int, windows_names: Optional[dict[int, dict]] = None) -> dict: + if sys.platform.startswith("linux"): + return _linux_device_metadata(index) + if sys.platform == "win32" and windows_names is not None: + # Best-effort: OpenCV index often matches enumeration order, but not always. + return windows_names.get(index, {"name": "", "path": "", "vendor_id": None, "product_id": None, "bus": ""}) + return {"name": "", "path": "", "vendor_id": None, "product_id": None, "bus": ""} + + +def _try_open(index: int, backend_name: str, backend_api: int, meta: dict) -> Optional[CameraInfo]: + cap = cv2.VideoCapture(index, backend_api) + if not cap.isOpened(): + cap.release() + return None + ok, frame = cap.read() + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) + if not ok or frame is None: + if width <= 0 or height <= 0: + cap.release() + return None + else: + h, w = frame.shape[:2] + width = width or int(w) + height = height or int(h) + cap.release() + return CameraInfo( + index=index, + backend_name=backend_name, + backend_api=backend_api, + width=width, + height=height, + name=str(meta.get("name") or ""), + path=str(meta.get("path") or ""), + vendor_id=meta.get("vendor_id"), + product_id=meta.get("product_id"), + bus=str(meta.get("bus") or ""), + ) + + +def list_cameras( + max_index: int = DEFAULT_MAX_CAMERA_INDEX, + *, + camera_filter: Optional[CameraFilter] = None, + exclude_laptop_webcams: bool = False, + name_include: Optional[Sequence[str]] = None, + name_exclude: Optional[Sequence[str]] = None, + vid_pid_allowlist: Optional[Sequence[Tuple[int, int]]] = None, + min_width: Optional[int] = None, + min_height: Optional[int] = None, + backend: str = "auto", +) -> list[CameraInfo]: + """Probe indices ``0 .. max_index`` and return working cameras. + + Laptop webcam filtering is best-effort and requires device names from the + OS. When names are unavailable, devices are kept unless other filters reject + them. Pass ``exclude_laptop_webcams=True`` or a ``CameraFilter`` to opt in. + """ + filt = camera_filter or CameraFilter( + name_include=name_include, + name_exclude=name_exclude, + exclude_laptop_webcams=exclude_laptop_webcams, + vid_pid_allowlist=vid_pid_allowlist, + min_width=min_width, + min_height=min_height, + ) + + found: list[CameraInfo] = [] + seen_indices: set[int] = set() + backends = _backend_candidates(backend) + windows_names = _windows_device_names() if sys.platform == "win32" else None + + with _suppress_opencv_noise(): + for index in range(0, max_index + 1): + meta = _device_metadata(index, windows_names) + for backend_name, backend_api in backends: + info = _try_open(index, backend_name, backend_api, meta) + if info is None: + continue + if index in seen_indices: + break + if not filt.matches(info): + seen_indices.add(index) + break + found.append(info) + seen_indices.add(index) + break + return found + + +def pick_default_camera( + cameras: Iterable[CameraInfo], + preferred: str | None = None, +) -> str | None: + cameras = list(cameras) + if not cameras: + return None + preferred = (preferred or "").strip() + if preferred: + for cam in cameras: + if cam.source == preferred: + return cam.source + return cameras[0].source diff --git a/src/orisys/capture/source.py b/src/orisys/capture/source.py new file mode 100644 index 0000000..6e16983 --- /dev/null +++ b/src/orisys/capture/source.py @@ -0,0 +1,552 @@ +"""OpenCV-backed capture source with optional background thread and reconnect.""" + +from __future__ import annotations + +import os +import sys +import threading +import time +from typing import Any, Optional, Protocol, runtime_checkable + +import cv2 +import numpy as np + +from .config import ( + CaptureConfig, + CaptureHealth, + CaptureStatus, + ReconnectPolicy, + SourceType, +) + + +@runtime_checkable +class VideoCaptureLike(Protocol): + """Minimal protocol so tests can inject fake captures.""" + + def isOpened(self) -> bool: ... + def read(self) -> tuple[bool, Any]: ... + def release(self) -> None: ... + def get(self, propId: int) -> float: ... + def set(self, propId: int, value: float) -> bool: ... + def grab(self) -> bool: ... + + +def normalize_source(source: SourceType | None) -> SourceType: + """Accept camera index, numeric string, V4L path, or filesystem path.""" + if source is None: + return 0 + if isinstance(source, str): + stripped = source.strip() + if not stripped: + return 0 + if stripped.isdigit(): + return int(stripped) + return stripped.rstrip("/\\") or stripped + return source + + +def is_live_camera(source: SourceType) -> bool: + """True for camera indices and Linux V4L2 device paths.""" + if isinstance(source, int): + return True + if isinstance(source, str) and source.startswith("/dev/video"): + return True + return False + + +def _backend_candidates(preferred: str = "auto") -> list[tuple[str, int]]: + preferred = (preferred or "auto").lower() + candidates: list[tuple[str, int]] = [] + + def add(name: str, attr: str) -> None: + if hasattr(cv2, attr): + candidates.append((name, getattr(cv2, attr))) + + if preferred in ("dshow", "directshow") and sys.platform == "win32": + add("DSHOW", "CAP_DSHOW") + elif preferred in ("msmf",) and sys.platform == "win32": + add("MSMF", "CAP_MSMF") + elif preferred in ("v4l2", "v4l") and sys.platform.startswith("linux"): + add("V4L2", "CAP_V4L2") + elif preferred not in ("auto", "any", ""): + pass + + if not candidates: + if sys.platform == "win32": + add("DSHOW", "CAP_DSHOW") + add("MSMF", "CAP_MSMF") + elif sys.platform.startswith("linux"): + add("V4L2", "CAP_V4L2") + if not candidates: + candidates.append(("ANY", cv2.CAP_ANY)) + return candidates + + +def _fourcc_code(fourcc: Optional[str]) -> Optional[int]: + if not fourcc: + return None + text = str(fourcc).strip() + if len(text) != 4: + return None + return cv2.VideoWriter.fourcc(*text) + + +def _decode_fourcc(value: float) -> str: + fourcc_int = int(value) + if fourcc_int <= 0: + return "N/A" + bytes_ = [(fourcc_int >> (8 * i)) & 0xFF for i in range(4)] + return "".join(chr(b) if 32 <= b <= 126 else "." for b in bytes_) + + +def _read_stream_meta(cap: VideoCaptureLike) -> tuple[int, int, float, str]: + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) + fps = float(cap.get(cv2.CAP_PROP_FPS) or 0.0) + fourcc = _decode_fourcc(cap.get(cv2.CAP_PROP_FOURCC) or 0) + return width, height, fps, fourcc + + +def _apply_stream_props( + cap: VideoCaptureLike, + *, + width: Optional[int], + height: Optional[int], + fps: Optional[float], + fourcc: Optional[str], +) -> None: + """Apply live-camera stream properties (width/height before fourcc on DSHOW).""" + code = _fourcc_code(fourcc) + if width is not None: + cap.set(cv2.CAP_PROP_FRAME_WIDTH, float(width)) + if height is not None: + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, float(height)) + if code is not None: + cap.set(cv2.CAP_PROP_FOURCC, code) + if fps is not None: + cap.set(cv2.CAP_PROP_FPS, float(fps)) + # DSHOW often resets fourcc after size negotiation. + if code is not None: + cap.set(cv2.CAP_PROP_FOURCC, code) + + +def _warmup_capture(cap: VideoCaptureLike, *, grabs: int = 3) -> None: + for _ in range(max(0, grabs)): + try: + cap.grab() + except Exception: + break + + +def open_opencv_capture( + source: SourceType, + *, + backend: str = "auto", + width: Optional[int] = None, + height: Optional[int] = None, + fps: Optional[float] = None, + fourcc: Optional[str] = "MJPG", + apply_stream_props: bool = True, +) -> tuple[VideoCaptureLike, str]: + """Open a ``cv2.VideoCapture`` trying preferred backends in order. + + Returns ``(capture, backend_name)``. + """ + if sys.platform == "win32": + os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0") + + source = normalize_source(source) + live = is_live_camera(source) + last_error = "Failed to open video source." + requested_fourcc = (fourcc or "").strip().upper() + + backends = _backend_candidates(backend) if live else [("ANY", cv2.CAP_ANY)] + for backend_name, backend_api in backends: + try: + if live and backend_api != cv2.CAP_ANY: + cap = cv2.VideoCapture(source, backend_api) + else: + cap = cv2.VideoCapture(source) + except Exception as exc: # pragma: no cover - OpenCV quirks + last_error = str(exc) + continue + if not cap.isOpened(): + cap.release() + last_error = f"Backend {backend_name} failed to open {source!r}" + continue + if live and apply_stream_props: + _apply_stream_props( + cap, + width=width, + height=height, + fps=fps, + fourcc=fourcc, + ) + _warmup_capture(cap) + _, _, _, negotiated = _read_stream_meta(cap) + if requested_fourcc and negotiated.upper() != requested_fourcc: + # Retry once: some drivers need a second fourcc set after warmup. + code = _fourcc_code(fourcc) + if code is not None: + cap.set(cv2.CAP_PROP_FOURCC, code) + _warmup_capture(cap, grabs=2) + return cap, backend_name + + raise RuntimeError(last_error) + + +class CaptureSource: + """Owns a video capture handle, optional background reader, and reconnect logic. + + Optionally accepts an injected ``factory`` for tests:: + + CaptureSource(config, factory=lambda cfg: (FakeCap(), "FAKE")) + """ + + def __init__( + self, + config: CaptureConfig, + *, + factory=None, + ): + self.config = config + self._factory = factory or self._default_factory + self._cap: Optional[VideoCaptureLike] = None + self._backend_name = "" + self._status = CaptureStatus(source=normalize_source(config.source)) + self._reconnect_attempt = 0 + self._lock = threading.Lock() + self._frame_lock = threading.Lock() + self._frame_cond = threading.Condition(self._frame_lock) + self._latest_frame: Optional[np.ndarray] = None + self._latest_seq = 0 + self._thread: Optional[threading.Thread] = None + self._background_running = False + + @staticmethod + def _default_factory(config: CaptureConfig) -> tuple[VideoCaptureLike, str]: + return open_opencv_capture( + config.source, + backend=config.backend, + width=config.width, + height=config.height, + fps=config.fps, + fourcc=config.fourcc, + apply_stream_props=is_live_camera(normalize_source(config.source)), + ) + + # Compatibility with older CaptureWorker API. + @property + def source(self) -> "CaptureSource": + return self + + @property + def running(self) -> bool: + return self._background_running + + @property + def status(self) -> CaptureStatus: + with self._lock: + return CaptureStatus( + health=self._status.health, + source=self._status.source, + last_error=self._status.last_error, + last_frame_time=self._status.last_frame_time, + last_seq=self._status.last_seq, + dropped_frames=self._status.dropped_frames, + reconnect_attempts=self._status.reconnect_attempts, + consecutive_failures=self._status.consecutive_failures, + width=self._status.width, + height=self._status.height, + fps=self._status.fps, + backend_name=self._status.backend_name, + fourcc=self._status.fourcc, + ) + + @property + def cap(self) -> Optional[VideoCaptureLike]: + """Raw capture handle (compat with Sensor.cap).""" + return self._cap + + def is_open(self) -> bool: + cap = self._cap + return cap is not None and cap.isOpened() + + def open(self) -> None: + with self._lock: + self._status.health = CaptureHealth.CONNECTING + self._status.last_error = None + try: + cap, backend_name = self._factory(self.config) + except Exception as exc: + with self._lock: + self._status.health = CaptureHealth.FAILED + self._status.last_error = str(exc) + raise + with self._lock: + self._release_unlocked() + self._cap = cap + self._backend_name = backend_name + self._refresh_stream_meta_unlocked() + self._status.health = CaptureHealth.CONNECTED + self._status.backend_name = backend_name + self._status.consecutive_failures = 0 + self._reconnect_attempt = 0 + if self.config.verbose: + st = self.status + print( + f"OpenCV negotiated stream: FOURCC='{st.fourcc}', " + f"size={st.width}x{st.height}, fps={st.fps:.2f}, backend={st.backend_name}" + ) + requested = (self.config.fourcc or "").strip().upper() + if ( + requested + and is_live_camera(normalize_source(self.config.source)) + and st.fourcc.upper() != requested + ): + print( + f"Warning: requested FOURCC '{self.config.fourcc}' but camera " + f"negotiated '{st.fourcc}'." + ) + + def release(self) -> None: + self.stop_background() + with self._lock: + self._release_unlocked() + self._status.health = CaptureHealth.CLOSED + + def close(self) -> None: + """Stop background thread and release the capture handle.""" + self.release() + + def _release_unlocked(self) -> None: + if self._cap is not None: + try: + self._cap.release() + except Exception: + pass + self._cap = None + with self._frame_cond: + self._latest_frame = None + self._latest_seq = 0 + + def _refresh_stream_meta_unlocked(self) -> None: + cap = self._cap + if cap is None: + return + try: + width, height, fps, fourcc = _read_stream_meta(cap) + self._status.width = width + self._status.height = height + self._status.fps = fps + self._status.fourcc = fourcc + except Exception: + pass + + def warmup_frames(self, count: int = 3) -> None: + """Discard frames so the driver/format stabilizes (sync reads only).""" + if self._background_running: + return + for _ in range(max(0, int(count))): + self.read_raw() + + def read_raw(self) -> tuple[bool, Optional[np.ndarray]]: + """Read one frame directly from the underlying capture (no reconnect).""" + cap = self._cap + if cap is None: + return False, None + try: + ok, frame = cap.read() + except Exception as exc: + with self._lock: + self._status.consecutive_failures += 1 + self._status.last_error = str(exc) + return False, None + if not ok or frame is None: + with self._lock: + self._status.consecutive_failures += 1 + return False, None + with self._lock: + self._status.consecutive_failures = 0 + self._status.last_frame_time = time.perf_counter() + if self._status.health != CaptureHealth.CONNECTED: + self._status.health = CaptureHealth.CONNECTED + return True, frame + + def read_latest( + self, + *, + copy: bool = True, + wait_s: float = 0.0, + ) -> tuple[bool, Optional[np.ndarray]]: + """Return the newest frame from the background thread, or read sync.""" + if not self._background_running: + ok, frame = self.read_raw() + if ok and copy and frame is not None: + return ok, frame.copy() + return ok, frame + + deadline = time.perf_counter() + max(0.0, wait_s) + with self._frame_cond: + while self._latest_frame is None: + remaining = deadline - time.perf_counter() + if remaining <= 0: + return False, None + self._frame_cond.wait(timeout=remaining) + frame = self._latest_frame.copy() if copy else self._latest_frame + return True, frame + + def read(self, *, copy: bool = True, wait_s: float = 1.0) -> tuple[bool, Optional[np.ndarray]]: + """Compat alias used by finger runtime and older worker callers.""" + wait = wait_s if wait_s > 0 else 1.0 + return self.read_latest(copy=copy, wait_s=wait) + + def start_background(self) -> None: + """Start a daemon thread that keeps only the latest frame.""" + if self._thread is not None: + return + if not self.is_open(): + self.open() + self._background_running = True + self._thread = threading.Thread( + target=self._background_loop, + daemon=True, + name="orisys-capture", + ) + self._thread.start() + if self.config.verbose: + print("Capture background thread started.") + + def start(self) -> None: + """Compat alias for ``start_background``.""" + self.start_background() + + def stop_background(self, timeout: float = 1.0) -> None: + self._background_running = False + if self._thread is not None: + self._thread.join(timeout=timeout) + self._thread = None + with self._frame_cond: + self._latest_frame = None + + def stop(self, timeout: float = 1.0) -> None: + """Compat alias for ``stop_background``.""" + self.stop_background(timeout=timeout) + + def _background_loop(self) -> None: + while self._background_running: + if self.should_reconnect(): + if not self.reconnect(): + time.sleep(0.05) + if self.status.health == CaptureHealth.FAILED: + time.sleep(0.2) + continue + + ok, frame = self.read_raw() + if not ok or frame is None: + if self.should_reconnect(): + continue + time.sleep(0.001) + continue + + with self._frame_cond: + if self._latest_frame is not None: + with self._lock: + self._status.dropped_frames += 1 + self._latest_seq += 1 + self._latest_frame = frame + with self._lock: + self._status.last_seq = self._latest_seq + self._frame_cond.notify_all() + + def note_published(self, seq: int) -> None: + with self._lock: + self._status.last_seq = int(seq) + + def note_dropped(self, n: int = 1) -> None: + with self._lock: + self._status.dropped_frames += int(n) + + def should_reconnect(self) -> bool: + policy = self.config.reconnect + if not policy.enabled: + return False + with self._lock: + if self._status.health == CaptureHealth.CLOSED: + return False + if not self.is_open(): + return True + if self._status.consecutive_failures >= max(1, policy.fail_threshold): + return True + if policy.stale_timeout_s is not None and self._status.last_frame_time is not None: + age = time.perf_counter() - self._status.last_frame_time + if age >= policy.stale_timeout_s and self._status.health == CaptureHealth.CONNECTED: + return True + return False + + def reconnect(self) -> bool: + """Attempt to reopen the capture. Returns True on success.""" + policy: ReconnectPolicy = self.config.reconnect + if not policy.enabled: + return False + if policy.max_attempts is not None and self._reconnect_attempt >= policy.max_attempts: + with self._lock: + self._status.health = CaptureHealth.FAILED + self._status.last_error = "Reconnect attempts exhausted" + return False + + with self._lock: + self._status.health = CaptureHealth.RECONNECTING + self._status.reconnect_attempts += 1 + attempt = self._reconnect_attempt + self._reconnect_attempt += 1 + self._release_unlocked() + + delay = policy.next_backoff(attempt) + if self.config.verbose: + print( + f"Reconnecting to {self.config.source!r} " + f"(attempt {attempt + 1}, wait {delay:.2f}s)..." + ) + time.sleep(delay) + try: + self.open() + return True + except Exception as exc: + with self._lock: + self._status.last_error = str(exc) + if policy.max_attempts is not None and self._reconnect_attempt >= policy.max_attempts: + self._status.health = CaptureHealth.FAILED + else: + self._status.health = CaptureHealth.RECONNECTING + return False + + +# Backward-compatible alias for one release cycle. +CaptureWorker = CaptureSource + + +def open_capture( + config: CaptureConfig, + *, + factory=None, + start_background: Optional[bool] = None, + start_worker: Optional[bool] = None, +) -> CaptureSource: + """Open a capture source and optionally start its background thread. + + For live cameras, the background thread starts by default (``config.background``). + File/video sources stay synchronous unless ``start_background=True``. + """ + source = CaptureSource(config, factory=factory) + source.open() + should_start = config.background + if start_background is not None: + should_start = start_background + elif start_worker is not None: + should_start = start_worker + live = is_live_camera(normalize_source(config.source)) + if should_start and live: + source.start_background() + return source diff --git a/src/orisys/configs.py b/src/orisys/configs.py index ebdfad8..27bed95 100644 --- a/src/orisys/configs.py +++ b/src/orisys/configs.py @@ -1,6 +1,28 @@ import json +from contextlib import ExitStack +from dataclasses import dataclass +from importlib.resources import as_file, files from pathlib import Path +# Product-facing names that map to on-disk config stems. +# Public product stems are ``box.json`` and ``finger.json``. +# Keep older lab names working: ``dd02`` still resolves to ``dd02-ov``. +CONFIG_ALIASES = { + "dd02": "dd02-ov", +} + +PRODUCT_BUNDLES = ("box", "mini", "finger") +_RESOURCE_STACK = ExitStack() + + +def _resource_path(*parts: str) -> Path | None: + """Resolve a package resource to a filesystem path for wheel-safe loaders.""" + try: + resource = files("orisys").joinpath(*parts) + return _RESOURCE_STACK.enter_context(as_file(resource)) + except Exception: + return None + def _candidate_config_dirs() -> list[Path]: """ @@ -21,6 +43,26 @@ def _candidate_config_dirs() -> list[Path]: return unique +def _candidate_sensor_roots() -> list[Path]: + """Candidate roots that contain per-sensor bundle folders.""" + here = Path(__file__).resolve().parent + candidates = [] + packaged = _resource_path("sensors") + if packaged is not None: + candidates.append(packaged) # packaged wheel layout: orisys/sensors/ + candidates.extend([ + here / "sensors", + here.parent.parent / "sensors", # editable install: repo_root/sensors/ + here.parent / "sensors", + Path.cwd() / "sensors", + ]) + unique: list[Path] = [] + for path in candidates: + if path not in unique: + unique.append(path) + return unique + + def _get_config_dir() -> Path: for candidate in _candidate_config_dirs(): if candidate.exists() and any(candidate.glob("*.json")): @@ -43,14 +85,96 @@ def _load_json(path: Path) -> dict: return json.load(f) +@dataclass(frozen=True) +class SensorBundle: + """Resolved paths for one additive per-sensor package folder.""" + + name: str + root: Path + sensor_json: Path + force_calibration_json: Path | None = None + manifest_json: Path | None = None + assets_dir: Path | None = None + + +def resolve_sensor_bundle(name: str) -> SensorBundle | None: + """Resolve ``sensors//`` when present; otherwise return ``None``.""" + requested = CONFIG_ALIASES.get(str(name), str(name)) + for root in _candidate_sensor_roots(): + bundle_root = root / requested + sensor_json = bundle_root / "sensor.json" + if not sensor_json.is_file(): + continue + force_path = bundle_root / "force_calibration.json" + manifest_path = bundle_root / "manifest.json" + assets_dir = None + if manifest_path.is_file(): + try: + manifest = _load_json(manifest_path) + except Exception: + manifest = {} + rel_assets = manifest.get("assets_dir") + if isinstance(rel_assets, str) and rel_assets.strip(): + candidate = (bundle_root / rel_assets).resolve() + if candidate.is_dir(): + assets_dir = candidate + return SensorBundle( + name=requested, + root=bundle_root.resolve(), + sensor_json=sensor_json.resolve(), + force_calibration_json=force_path.resolve() if force_path.is_file() else None, + manifest_json=manifest_path.resolve() if manifest_path.is_file() else None, + assets_dir=assets_dir, + ) + return None + + +def resolve_sensor_config_path(name: str) -> Path | None: + """Return ``sensor.json`` for a resolved sensor bundle.""" + bundle = resolve_sensor_bundle(name) + return bundle.sensor_json if bundle is not None else None + + +def resolve_sensor_asset_dir(name: str) -> Path | None: + """Return the default asset directory for a resolved sensor bundle.""" + bundle = resolve_sensor_bundle(name) + return bundle.assets_dir if bundle is not None else None + + +def resolve_sensor_resource(name: str, *relative_parts: str) -> Path | None: + """Return a path inside one sensor bundle when present.""" + bundle = resolve_sensor_bundle(name) + if bundle is None: + return None + return bundle.root.joinpath(*relative_parts) + + +def list_sensor_bundles() -> list[str]: + """Return product names that have an additive ``sensors//`` folder.""" + found: list[str] = [] + for name in PRODUCT_BUNDLES: + if resolve_sensor_bundle(name) is not None: + found.append(name) + return found + + +def load_force_calibration(name: str) -> dict | None: + """Load ``force_calibration.json`` from a sensor bundle when available.""" + bundle = resolve_sensor_bundle(name) + if bundle is None or bundle.force_calibration_json is None: + return None + return _load_json(bundle.force_calibration_json) + + def load_config(config_name_or_path): """ Load sensor configuration from a JSON file path or config name. Config lookup order: 1. Exact file path (absolute or relative) - 2. File stem under the config directory (e.g. ``ddjx01`` -> ``config/ddjx01.json``) - 3. JSON field ``"name"`` matching the requested name (keeps ``prototype`` compatible) + 2. Additive sensor bundle ``sensors//sensor.json`` + 3. File stem under the config directory (e.g. ``box`` -> ``config/box.json``) + 4. JSON field ``"name"`` matching the requested name (keeps ``prototype`` compatible) """ config_path = Path(config_name_or_path) @@ -62,9 +186,21 @@ def load_config(config_name_or_path): if config_path.is_absolute(): raise FileNotFoundError(f"Config file not found at absolute path: {config_path}") - direct = CONFIG_DIR / f"{config_name_or_path}.json" - if direct.exists(): - return _load_json(direct) + requested = str(config_name_or_path) + candidates = [requested] + alias = CONFIG_ALIASES.get(requested) + if alias and alias not in candidates: + candidates.append(alias) + + for name in candidates: + bundle = resolve_sensor_bundle(name) + if bundle is not None: + return _load_json(bundle.sensor_json) + + for name in candidates: + direct = CONFIG_DIR / f"{name}.json" + if direct.exists(): + return _load_json(direct) # Backward-compatible alias resolution: support old names via JSON "name" field. for candidate in _config_files(): @@ -72,7 +208,7 @@ def load_config(config_name_or_path): data = _load_json(candidate) except Exception: continue - if data.get("name") == config_name_or_path: + if data.get("name") in candidates: return data available = [f.stem for f in _config_files()] @@ -80,7 +216,7 @@ def load_config(config_name_or_path): raise FileNotFoundError( f"Input config file not found with name or path: {config_name_or_path}\n" f" Tried as file path: {config_path} (not found)\n" - f" Tried as config file: {direct} (not found)\n" + f" Tried as config files: {', '.join(candidates)} (not found)\n" f" Available config files: {available_str}" ) @@ -89,7 +225,11 @@ def list_config(): """ List all available JSON configuration files by file stem. """ - return [f.stem for f in _config_files()] + stems = [f.stem for f in _config_files()] + for name in list_sensor_bundles(): + if name not in stems: + stems.append(name) + return sorted(stems) def export_config(config_name=None, path="./config/exported_config.json"): diff --git a/src/orisys/finger/exports.py b/src/orisys/finger/exports.py index 7f39d22..82b7e8a 100644 --- a/src/orisys/finger/exports.py +++ b/src/orisys/finger/exports.py @@ -4,6 +4,7 @@ import sys from dataclasses import dataclass from pathlib import Path +from ..configs import resolve_sensor_asset_dir from ..unroll import ( compute_uv_distortion_metrics, load_exported_data, @@ -11,7 +12,7 @@ from ..unroll import ( scaled_map_cache_path, ) -DEFAULT_FINGER_EXPORT_DIR = "assets/finger/30fps" +DEFAULT_FINGER_EXPORT_DIR = "sensors/finger/assets/30fps-0703" DEFAULT_FINGER_MAP_CACHE = f"{DEFAULT_FINGER_EXPORT_DIR}/unroll_map_uv.npz" @@ -44,11 +45,15 @@ def _candidate_export_dirs(export_dir) -> list[Path]: return [raw] here = Path(__file__).resolve() - candidates = [ + candidates = [] + bundled_assets = resolve_sensor_asset_dir("finger") + if bundled_assets is not None and raw.as_posix() == DEFAULT_FINGER_EXPORT_DIR: + candidates.append(bundled_assets) + candidates.extend([ Path.cwd() / raw, - here.parents[1] / raw, # packaged wheel: orisys/assets/... + here.parents[1] / raw, # packaged wheel filesystem layout here.parents[3] / raw, # editable install: repo/assets/... - ] + ]) if getattr(sys, "frozen", False): candidates.insert(0, Path(sys.executable).resolve().parent / raw) unique: list[Path] = [] diff --git a/src/orisys/finger/runtime.py b/src/orisys/finger/runtime.py index 7aaccc2..80be9f6 100644 --- a/src/orisys/finger/runtime.py +++ b/src/orisys/finger/runtime.py @@ -2,11 +2,18 @@ from __future__ import annotations from dataclasses import dataclass -import cv2 import numpy as np +from ..capture import ( + CaptureConfig, + CaptureSource, + ReconnectPolicy, + is_live_camera, + normalize_source, + open_capture, +) from ..configs import load_config -from ..frame_processors import UVUnrollFrameProcessor +from ..frame_processors import FrameUnwarper from ..sensor import Sensor from ..unroll import ( build_or_load_domain, @@ -24,25 +31,26 @@ from .exports import ( def normalize_video_source(video): """Accept camera index, numeric string, or filesystem video path.""" - if video is None: - return 0 - if isinstance(video, str): - stripped = video.strip().strip("/\\") - if not stripped: - return 0 - if stripped.isdigit(): - return int(stripped) - return stripped - return video + return normalize_source(video) -def open_source(video, config): - capture = cv2.VideoCapture(normalize_video_source(video)) - capture.set(cv2.CAP_PROP_FRAME_WIDTH, config.get("camera", {}).get("resolution_raw")[0]) - capture.set(cv2.CAP_PROP_FRAME_HEIGHT, config.get("camera", {}).get("resolution_raw")[1]) - capture.set(cv2.CAP_PROP_FPS, config.get("camera", {}).get("fps", 30)) - capture.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M', 'J', 'P', 'G')) - return capture +def open_source(video, config, *, auto_reconnect: bool = False, verbose: bool = False): + """Open a capture source for finger pipelines (compat: returns ``.cap``-like handle). + + Historically returned a bare ``cv2.VideoCapture``. Callers that need the raw + OpenCV handle can use ``source.cap``; prefer ``source.read()``. + """ + capture_config = CaptureConfig.from_camera_dict( + normalize_video_source(video), + config.get("camera", {}), + reconnect=ReconnectPolicy(enabled=auto_reconnect), + verbose=verbose, + background=False, + ) + source = open_capture(capture_config, start_background=False) + cap = source.cap + cap._orisys_worker = source # type: ignore[attr-defined] + return cap class TactileProcessor: @@ -146,7 +154,7 @@ class FingerGeometry: def resolve_finger_geometry( *, export_dir, - config="./config/dd02-ov.json", + config="finger", map_cache=DEFAULT_FINGER_MAP_CACHE, source_scale=None, calib=None, @@ -208,8 +216,9 @@ class FingerRuntime: geometry: FingerGeometry, domain, tactile: TactileProcessor, - frame_processor: UVUnrollFrameProcessor, + frame_processor: FrameUnwarper, capture, + capture_source: CaptureSource | None = None, ): self.geometry = geometry self.domain = domain @@ -217,13 +226,16 @@ class FingerRuntime: self.sensor = tactile self.frame_processor = frame_processor self.capture = capture + self.capture_source = capture_source + # Deprecated alias. + self.capture_worker = capture_source @classmethod def open( cls, *, export_dir=DEFAULT_FINGER_EXPORT_DIR, - config="./config/dd02-ov.json", + config="finger", map_cache=DEFAULT_FINGER_MAP_CACHE, source_scale=None, calib=None, @@ -232,8 +244,11 @@ class FingerRuntime: flip_source_vertical=True, verbose=False, mirror=False, + auto_reconnect: bool = False, + capture_mode: str = "latest", ) -> "FingerRuntime": """Open a live finger runtime from export bundle, config, and video source.""" + del capture_mode # deprecated; latest-frame capture only if camera_id is not None: video = camera_id video = normalize_video_source(video) @@ -263,12 +278,22 @@ class FingerRuntime: ) domain.remap.ensure_fast_maps() - capture = open_source(video, geometry.config) - if not capture.isOpened(): + capture_config = CaptureConfig.from_camera_dict( + video, + geometry.config.get("camera", {}), + reconnect=ReconnectPolicy(enabled=auto_reconnect), + verbose=verbose, + background=False, + ) + capture_source = open_capture(capture_config, start_background=False) + if not capture_source.is_open(): raise RuntimeError("Cannot open camera/video source") + if is_live_camera(video): + capture_source.start_background() + capture = capture_source.cap sensor = Sensor("external", config_name=config, verbose=verbose, mirror=mirror) - frame_processor = UVUnrollFrameProcessor( + frame_processor = FrameUnwarper( domain.remap, camera_matrix=geometry.source_K if geometry.dist_coeffs is not None else None, dist_coeffs=geometry.dist_coeffs, @@ -283,6 +308,7 @@ class FingerRuntime: tactile=tactile, frame_processor=frame_processor, capture=capture, + capture_source=capture_source, ) @property @@ -290,6 +316,10 @@ class FingerRuntime: return self.frame_processor.valid_mask def read_source(self): + if self.capture_source is not None and self.capture_source.running: + return self.capture_source.read(copy=True, wait_s=1.0) + if self.capture_source is not None: + return self.capture_source.read_raw() return self.capture.read() def process_frame(self, frame, *, color=False): @@ -323,6 +353,12 @@ class FingerRuntime: return ok, unrolled def close(self): - if self.capture is not None: + if self.capture_source is not None: + self.capture_source.close() + self.capture_source = None + self.capture_worker = None + self.capture = None + elif self.capture is not None: self.capture.release() + self.capture = None self.tactile.disconnect() diff --git a/src/orisys/frame_processors.py b/src/orisys/frame_processors.py index 034169d..3b7e946 100644 --- a/src/orisys/frame_processors.py +++ b/src/orisys/frame_processors.py @@ -1,5 +1,8 @@ +from __future__ import annotations + import cv2 import numpy as np +from pathlib import Path from .unroll import UVRemap @@ -13,7 +16,7 @@ class CompoundEyeFrameProcessor: def process(self, frame): self.stitcher.get_stitch_unit(frame) self.stitcher.extract_tiles() - return self.stitcher.stitch_rt_optimized(blend=False) + return self.stitcher.stitch_rt_optimized() class SingleCameraFrameProcessor: @@ -27,7 +30,7 @@ class SingleCameraFrameProcessor: return frame -class UVUnrollFrameProcessor: +class FrameUnwarper: """Remap a raw camera frame into a flattened UV-domain BGR image.""" def __init__( @@ -227,5 +230,182 @@ def create_frame_processor(sensor_type, stitcher=None, uv_remap=None, **uv_kwarg if sensor_type in ("uv_unroll", "unroll"): if uv_remap is None: raise ValueError("UV unroll sensor requires a UVRemap instance.") - return UVUnrollFrameProcessor(uv_remap, **uv_kwargs) + return FrameUnwarper(uv_remap, **uv_kwargs) raise ValueError(f"Unsupported sensor type: {sensor_type}") + + +class FFCFrameProcessor: + """Flat-field correction with offline weight-map calibration and fast online correction.""" + + def __init__(self, blur_kernel=151, blur_sigma=0.0, epsilon=1e-6, masked_outside=True): + self.blur_kernel = self.ensure_odd(blur_kernel) + self.blur_sigma = float(blur_sigma) + self.epsilon = float(epsilon) + self.masked_outside = bool(masked_outside) + self.weight_map = None + self.weight_map_3ch = None + self.roi_mask = None + self._multiplied_buffer = None + self._output_buffer = None + self._buffer_shape = None + self._buffer_dtype = None + self._roi_bbox = None + + def calibrate(self, reference_image, roi_mask=None): + gray = self.to_gray_float32(reference_image) + roi_mask = self.normalize_mask(roi_mask, gray.shape) + active = roi_mask > 0 + if not np.any(active): + raise ValueError("ROI mask is empty; cannot calibrate FFC.") + + background = cv2.GaussianBlur( + gray, + (self.blur_kernel, self.blur_kernel), + sigmaX=self.blur_sigma, + sigmaY=self.blur_sigma, + borderType=cv2.BORDER_REPLICATE, + ) + mean_value = float(background[active].mean()) + weight = mean_value / (background + self.epsilon) + + outside_value = 0.0 if self.masked_outside else 1.0 + self.weight_map = np.where(active, weight, outside_value).astype(np.float32) + self.weight_map_3ch = None + self.roi_mask = roi_mask + self._roi_bbox = self._compute_roi_bbox(roi_mask) + self._clear_runtime_buffers() + return self.weight_map + + def correct(self, image): + if self.weight_map is None: + raise RuntimeError("FFC is not calibrated. Call calibrate() or load() first.") + if image.shape[:2] != self.weight_map.shape: + raise ValueError( + f"image shape {image.shape[:2]} does not match weight map {self.weight_map.shape}" + ) + if image.ndim not in (2, 3): + raise ValueError("image must have shape HxW or HxWxC") + + self._ensure_runtime_buffers(image) + if self.masked_outside: + self._output_buffer.fill(0) + + if self._roi_bbox is None: + src = image + weight = self.weight_map if image.ndim == 2 else self.weight_map_3ch + dst_view = self._output_buffer + else: + x1, y1, x2, y2 = self._roi_bbox + src = image[y1:y2, x1:x2] + if image.ndim == 2: + weight = self.weight_map[y1:y2, x1:x2] + else: + weight = self.weight_map_3ch[y1:y2, x1:x2] + dst_view = self._output_buffer[y1:y2, x1:x2] + + cv2.multiply(src, weight, dst=self._multiplied_buffer, dtype=cv2.CV_32F) + if np.issubdtype(image.dtype, np.uint8): + cv2.convertScaleAbs(self._multiplied_buffer, dst=dst_view, alpha=1.0, beta=0.0) + return self._output_buffer + + corrected = self.saturate_to_dtype(self._multiplied_buffer, image.dtype) + dst_view[...] = corrected + return self._output_buffer + + def save(self, path): + if self.weight_map is None: + raise RuntimeError("No weight map to save.") + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed( + path, + weight_map=self.weight_map, + roi_mask=self.roi_mask, + blur_kernel=np.array(self.blur_kernel, dtype=np.int32), + blur_sigma=np.array(self.blur_sigma, dtype=np.float32), + epsilon=np.array(self.epsilon, dtype=np.float32), + masked_outside=np.array(self.masked_outside, dtype=np.bool_), + ) + + def load(self, path): + data = np.load(path) + self.weight_map = data["weight_map"].astype(np.float32, copy=False) + self.weight_map_3ch = None + self.roi_mask = data["roi_mask"].astype(np.uint8, copy=False) + self._roi_bbox = self._compute_roi_bbox(self.roi_mask) + self.blur_kernel = int(data.get("blur_kernel", self.blur_kernel)) + self.blur_sigma = float(data.get("blur_sigma", self.blur_sigma)) + self.epsilon = float(data.get("epsilon", self.epsilon)) + self.masked_outside = bool(data.get("masked_outside", self.masked_outside)) + self._clear_runtime_buffers() + return self + + def _clear_runtime_buffers(self): + self._multiplied_buffer = None + self._output_buffer = None + self._buffer_shape = None + self._buffer_dtype = None + + @staticmethod + def _compute_roi_bbox(roi_mask): + ys, xs = np.nonzero(roi_mask > 0) + if xs.size == 0: + return None + return int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1 + + def _ensure_runtime_buffers(self, image): + shape = image.shape + dtype = image.dtype + if self._buffer_shape == shape and self._buffer_dtype == dtype: + return + + self._buffer_shape = shape + self._buffer_dtype = dtype + self._output_buffer = np.empty(shape, dtype=dtype) + + if image.ndim == 3: + channels = image.shape[2] + self.weight_map_3ch = np.repeat(self.weight_map[:, :, None], channels, axis=2) + if self._roi_bbox is None: + self._multiplied_buffer = np.empty(shape, dtype=np.float32) + else: + x1, y1, x2, y2 = self._roi_bbox + self._multiplied_buffer = np.empty((y2 - y1, x2 - x1, channels), dtype=np.float32) + else: + self.weight_map_3ch = None + if self._roi_bbox is None: + self._multiplied_buffer = np.empty(shape, dtype=np.float32) + else: + x1, y1, x2, y2 = self._roi_bbox + self._multiplied_buffer = np.empty((y2 - y1, x2 - x1), dtype=np.float32) + + @staticmethod + def ensure_odd(value, minimum=3): + value = max(int(value), int(minimum)) + return value if value % 2 == 1 else value + 1 + + @staticmethod + def to_gray_float32(image): + if image.ndim == 2: + gray = image + elif image.ndim == 3: + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + else: + raise ValueError("image must have shape HxW or HxWxC") + return gray.astype(np.float32, copy=False) + + @staticmethod + def normalize_mask(mask, shape): + if mask is None: + return np.ones(shape, dtype=np.uint8) * 255 + mask = np.asarray(mask) + if mask.shape[:2] != shape: + mask = cv2.resize(mask, (shape[1], shape[0]), interpolation=cv2.INTER_NEAREST) + return np.where(mask > 0, 255, 0).astype(np.uint8) + + @staticmethod + def saturate_to_dtype(values, dtype): + if np.issubdtype(dtype, np.integer): + info = np.iinfo(dtype) + return np.clip(values, info.min, info.max).astype(dtype) + return values.astype(dtype, copy=False) diff --git a/src/orisys/sensor.py b/src/orisys/sensor.py index 89cdddc..5632c0a 100644 --- a/src/orisys/sensor.py +++ b/src/orisys/sensor.py @@ -6,13 +6,13 @@ optical tactile sensor data with support for stitching, optical flow tracking, and Helmholtz-Hodge decomposition. """ -import os +# import os # only used by commented-out stitch_cal import numpy as np import cv2 import time -import threading from enum import Enum, auto -from typing import Tuple +from pathlib import Path +from typing import Optional, Tuple # from threading import Thread, Event # from queue import Queue, Empty @@ -23,6 +23,20 @@ from . import configs from .get_depth import Contact3D from .backend_availability import (check_cupy_available, check_opencl_available) from .frame_processors import create_frame_processor +from .signal_processing import ( + DEFAULT_SAMPLE_RATE_HZ, + LowpassFilter, + calibrate_normal_force, + calibrate_shear_components, +) +from .capture import ( + CaptureConfig, + CaptureSource, + ReconnectPolicy, + is_live_camera, + normalize_source, + open_capture, +) class Sensor: """ @@ -46,6 +60,9 @@ class Sensor: FNORMAL = auto() FSHEARX = auto() FSHEARY = auto() + FNORMAL_RAW = auto() + FSHEARX_RAW = auto() + FSHEARY_RAW = auto() CENTROID = auto() CONTOUR = auto() DEPTH = auto() @@ -57,7 +74,10 @@ class Sensor: def __init__(self, vid_src, cuda=True, isstitch=True, verbose=True, debug=False, config_name="prototype", cal_path=None, override=False, box=None, mirror=True, - backend='auto'): + backend='auto', capture_config: Optional[CaptureConfig] = None, + auto_reconnect: bool = False, capture_mode: str = "latest", + capture_queue_size: int = 1, capture_overflow: str = "drop_oldest", + **_deprecated_capture_kwargs): """ Initialize Sensor with video source and configuration. @@ -75,8 +95,11 @@ class Sensor: "cupy": Use CuPy for GPU acceleration "opencl": Use OpenCL for GPU acceleration "cpu": Use CPU for computation - hhd_downsample: Downsample factor for flow field before HHD (default: 1, no downsampling). - Factor of 2 halves each dimension, giving ~4x speedup in Poisson solve. + capture_config: Optional ``CaptureConfig`` overriding capture kwargs below. + auto_reconnect: Enable USB disconnect auto-reconnect for live cameras. + capture_mode: Deprecated; ignored (latest-frame capture only). + capture_queue_size: Deprecated; ignored. + capture_overflow: Deprecated; ignored. """ self.CUPY_AVAILABLE = check_cupy_available() @@ -88,7 +111,7 @@ class Sensor: self._external_only = isinstance(vid_src, str) and vid_src.lower() == "external" # Store settings - self.vid_src = vid_src + self.vid_src = normalize_source(vid_src) if not self._external_only else vid_src self.isstitch = isstitch self.cuda = cuda self.verbose = verbose @@ -96,6 +119,8 @@ class Sensor: self.config_name = config_name self.cal_path = cal_path self.mirror = mirror + self._capture_config_override = capture_config + self._auto_reconnect = bool(auto_reconnect) # Load configuration self.__config_cam = None @@ -126,6 +151,25 @@ class Sensor: self.vshear = self.vtemplate self.n_boxes = None self.fnormal = self.fshearx = self.fsheary = self.ftor = 0.0 + self._force_raw = (0.0, 0.0, 0.0) + self._force_cal_sensor_name = self._resolve_force_calibration_sensor_name() + self._lowpass_enabled = False + self._lowpass_cutoff_hz = 0.0 + self._lowpass_sample_rate_hz = float(DEFAULT_SAMPLE_RATE_HZ) + self._lowpass_order = 2 + self._force_filter = LowpassFilter( + cutoff_hz=self._lowpass_cutoff_hz, + sample_rate_hz=self._lowpass_sample_rate_hz, + order=self._lowpass_order, + n_channels=3, + ) + if self.verbose: + if self._force_cal_sensor_name: + print( + f"Force calibration: sensors/{self._force_cal_sensor_name}/force_calibration.json" + ) + else: + print("Force calibration: raw (no force_calibration.json)") # Contact Data Variables (public) self.centroid = (0, 0) @@ -179,21 +223,16 @@ class Sensor: self.masked_poisson_max_iters = self.__config_tracker.get("masked_poisson_max_iters", 3) self.masked_poisson_tol = self.__config_tracker.get("masked_poisson_tol", 1e-4) self.masked_poisson_reg = self.__config_tracker.get("masked_poisson_reg", 1e-8) - self._use_pyav = False - self._av_container = None - self._av_stream = None - self._av_decoder = None - self.cap = None + # Legacy PyAV capture state (unused after orisys.capture refactor). + # self._use_pyav = False + # self._av_container = None + # self._av_stream = None + # self._av_decoder = None + self.cap = None # compat alias for _capture_worker.source.cap + self._capture_source: Optional[CaptureSource] = None self._is_live_camera = ( - isinstance(self.vid_src, int) - or (isinstance(self.vid_src, str) and self.vid_src.startswith("/dev/video")) - ) and not self._external_only - self._capture_thread = None - self._capture_running = False - self._latest_frame = None - self._latest_frame_seq = 0 - self._last_consumed_seq = -1 - self._latest_frame_lock = threading.Lock() + is_live_camera(self.vid_src) and not self._external_only + ) # Setup video capture unless this sensor is fed externally. if self._external_only: @@ -397,7 +436,7 @@ class Sensor: motion_value = mag.mean() return motion_value > threshold - def compute_deformation(self, decompose=True, check_motion=True, threshold=6, tail_frames=10): + def compute_deformation(self, decompose=True, check_motion=False, threshold=6, tail_frames=10): """ Compute optical flow and optionally decompose it into force components. @@ -529,7 +568,10 @@ class Sensor: self._hhd_grid_shape = ds_shape _, forces = self.__hhd.get_force(flow_ds, valid_mask=mask_ds) - self.fnormal, self.fshearx, self.fsheary, self.ftor = forces + fn, fx, fy, ftor = forces + self._force_raw = (float(fn), float(fx), float(fy)) + self.ftor = float(ftor) + self._apply_force_filter() vnormal_ds = self.__hhd.get_normal() vshear_ds = self.__hhd.get_shear() @@ -631,6 +673,9 @@ class Sensor: self.info.FNORMAL: lambda: self.fnormal, self.info.FSHEARX: lambda: self.fshearx, self.info.FSHEARY: lambda: self.fsheary, + self.info.FNORMAL_RAW: lambda: self._force_raw[0], + self.info.FSHEARX_RAW: lambda: self._force_raw[1], + self.info.FSHEARY_RAW: lambda: self._force_raw[2], self.info.FPS: lambda: self._fps, self.info.TIMESTAMP: lambda: time.strftime('%H:%M:%S'), # Return copies of arrays to prevent external modification @@ -649,37 +694,167 @@ class Sensor: return tuple(mapping.get(arg, lambda: None)() for arg in args if isinstance(arg, self.info)) def reset(self): - """Reset optical flow tracker.""" + """Reset optical flow tracker and force-filter state.""" self.__tracker.reset() + self._force_filter.reset() if self.img is not None: self.initial_img = self.img.copy() def disconnect(self): """Release resources and stop async threads.""" - self._capture_running = False - if self._capture_thread is not None: - self._capture_thread.join(timeout=1.0) - self._capture_thread = None - - if self._use_pyav: - if self._av_container is not None: - self._av_container.close() - self._av_container = None - self._av_stream = None - self._av_decoder = None - if self.verbose: - print("Sensor disconnected.") - elif self.cap is not None and self.cap.isOpened(): - self.cap.release() + if self._capture_source is not None: + self._capture_source.close() + self._capture_source = None + self.cap = None if self.verbose: print("Sensor disconnected.") + # Legacy PyAV / direct-cap fallbacks (unused after orisys.capture refactor). + # elif self._use_pyav: + # if self._av_container is not None: + # self._av_container.close() + # self._av_container = None + # self._av_stream = None + # self._av_decoder = None + # if self.verbose: + # print("Sensor disconnected.") + # elif self.cap is not None: + # try: + # if self.cap.isOpened(): + # self.cap.release() + # except Exception: + # pass + # self.cap = None + # if self.verbose: + # print("Sensor disconnected.") cv2.destroyAllWindows() # ========================================================================= # Public API - Configuration & Calibration # ========================================================================= - + + def configure( + self, + *, + lowpass_cutoff_hz=None, + lowpass_sample_rate_hz=None, + lowpass_order=None, + lowpass_enabled=None, + hhd_downsample=None, + ): + """Update runtime-safe settings without reconnecting the camera. + + Parameters + ---------- + lowpass_cutoff_hz : float, optional + Force low-pass cutoff in Hz. ``0`` disables filtering. A positive + value enables filtering unless ``lowpass_enabled=False``. + lowpass_sample_rate_hz : float, optional + Assumed force sample rate for the Butterworth design (default 30). + lowpass_order : int, optional + Butterworth order (default 2). + lowpass_enabled : bool, optional + Explicit on/off for the causal force filter. + hhd_downsample : int, optional + Integer ``>= 1``. Rebuilds the HHD solver on the next frame. + """ + filter_changed = False + if lowpass_cutoff_hz is not None: + self._lowpass_cutoff_hz = float(lowpass_cutoff_hz) + filter_changed = True + if lowpass_enabled is None: + self._lowpass_enabled = self._lowpass_cutoff_hz > 0 + if lowpass_sample_rate_hz is not None: + rate = float(lowpass_sample_rate_hz) + if rate <= 0: + raise ValueError("lowpass_sample_rate_hz must be > 0") + self._lowpass_sample_rate_hz = rate + filter_changed = True + if lowpass_order is not None: + order = int(lowpass_order) + if order < 1: + raise ValueError("lowpass_order must be >= 1") + self._lowpass_order = order + filter_changed = True + if lowpass_enabled is not None: + self._lowpass_enabled = bool(lowpass_enabled) + filter_changed = True + + if filter_changed: + cutoff = self._lowpass_cutoff_hz if self._lowpass_enabled else 0.0 + self._force_filter.configure( + cutoff_hz=cutoff, + sample_rate_hz=self._lowpass_sample_rate_hz, + order=self._lowpass_order, + ) + if self.compute_deformation_valid: + self._apply_force_filter() + + if hhd_downsample is not None: + try: + k = max(1, int(hhd_downsample)) + except (TypeError, ValueError) as exc: + raise ValueError("hhd_downsample must be an integer >= 1") from exc + if k != self._hhd_downsample: + self._hhd_downsample = k + self.__hhd = None + self._hhd_grid_shape = None + self._update_hhd_mask_cache() + return self + + def _resolve_force_calibration_sensor_name(self) -> Optional[str]: + """Return sensors/ when force_calibration.json exists; else None. + + Lookup order: + 1. ``config_name`` stem / product alias (e.g. ``box``) + 2. ``sensors//...`` path segment in ``config_name`` + Missing JSON means ``FNORMAL/FSHEAR*`` stay raw (no hardcoded fallback). + """ + text = str(self.config_name or "").strip().replace("\\", "/") + if not text: + return None + + candidates: list[str] = [] + stem = Path(text).stem if text else "" + if stem: + candidates.append(stem) + parts = [p for p in text.split("/") if p] + if "sensors" in parts: + idx = parts.index("sensors") + if idx + 1 < len(parts): + candidates.append(parts[idx + 1]) + for name in configs.PRODUCT_BUNDLES: + if text == name or text.endswith(f"/sensors/{name}") or text.endswith(f"/sensors/{name}/sensor.json"): + candidates.append(name) + + seen: set[str] = set() + for candidate in candidates: + alias = configs.CONFIG_ALIASES.get(candidate, candidate) + for name in (alias, candidate): + if name in seen: + continue + seen.add(name) + if configs.load_force_calibration(name) is not None: + return name + return None + + def _apply_force_filter(self): + raw = np.asarray(self._force_raw, dtype=np.float64) + if self._lowpass_enabled and self._lowpass_cutoff_hz > 0: + filtered = self._force_filter.process(raw) + fn, fx, fy = (float(v) for v in filtered) + else: + fn, fx, fy = float(raw[0]), float(raw[1]), float(raw[2]) + + # Apply sensors//force_calibration.json when present; otherwise keep raw. + name = self._force_cal_sensor_name + if name is not None: + fn = float(calibrate_normal_force(fn, sensor_name=name)) + fx, fy = calibrate_shear_components(fx, fy, sensor_name=name) + fx, fy = float(fx), float(fy) + + self.fnormal, self.fshearx, self.fsheary = fn, fx, fy + # def export_config(self, path="./config/exported_config.json"): # """Return current configuration dict.""" # configs.export_config(self._config, path) @@ -727,12 +902,13 @@ class Sensor: "compound_eye" if "stitcher" in self._config else "single_camera", ) self.__config_cam = self._config["camera"] - self._config_resolution_raw = self.__config_cam["resolution_raw"] - self._config_fps = self.__config_cam["fps"] + # Capture resolution/fps now applied via CaptureConfig.from_camera_dict(__config_cam). + # self._config_resolution_raw = self.__config_cam["resolution_raw"] + # self._config_fps = self.__config_cam["fps"] self._config_units = self.__config_cam.get("units", 1) self._config_resolution_stitch = self.__config_cam["resolution_stitch"] - self._config_dis_preset = self._config["tracker"]["dis_preset"] + # self._config_dis_preset = self._config["tracker"]["dis_preset"] self.__config_tracker = self._config["tracker"] self.__config_stitcher = self._config.get("stitcher") @@ -747,71 +923,32 @@ class Sensor: def _open_video_source(self, override, box): """Initialize video capture and calibration.""" - # _is_camera_index = isinstance(self.vid_src, int) - # _is_v4l2_path = isinstance(self.vid_src, str) and self.vid_src.startswith("/dev/video") - # use_linux_camera_pyav = ( - # platform.system() == "Linux" - # and (_is_camera_index or _is_v4l2_path) - # and av is not None - # ) - - # if use_linux_camera_pyav: - # device = self.vid_src if _is_v4l2_path else f"/dev/video{self.vid_src}" - # if self.verbose: - # print(f"Using PyAV Linux camera backend: {device}") - # try: - # self._av_container = av.open( - # device, - # format="v4l2", - # options={ - # "input_format": "mjpeg", - # "video_size": f"{self._config_resolution_raw[0]}x{self._config_resolution_raw[1]}", - # "framerate": str(self._config_fps), - # }, - # ) - # self._av_stream = self._av_container.streams.video[0] - # self._av_decoder = self._av_container.decode(self._av_stream) - # self._use_pyav = True - # except Exception as exc: - # if self.verbose: - # print(f"PyAV camera open failed ({exc}), fallback to OpenCV backend.") - # self._use_pyav = False - - self.use_pyav = False - if not self._use_pyav: - self.cap = cv2.VideoCapture(self.vid_src) - if self._is_live_camera: - self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M', 'J', 'P', 'G')) - self.cap.set(cv2.CAP_PROP_FPS, self._config_fps) - self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self._config_resolution_raw[0]) - self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self._config_resolution_raw[1]) - - if self.verbose: print("Connecting to video source...") - - + + if self._capture_config_override is not None: + capture_config = self._capture_config_override + else: + capture_config = CaptureConfig.from_camera_dict( + self.vid_src, + self.__config_cam, + reconnect=ReconnectPolicy(enabled=self._auto_reconnect), + verbose=self.verbose, + background=False, + ) + + try: + self._capture_source = open_capture(capture_config, start_background=False) + except Exception as exc: + raise Exception("Failed to open video source.") from exc + + self.cap = self._capture_source.cap + if not self._is_source_open(): raise Exception("Failed to open video source.") - if self.verbose and not self._use_pyav and self.cap is not None: - fourcc_int = int(self.cap.get(cv2.CAP_PROP_FOURCC)) - fourcc_bytes = [(fourcc_int >> (8 * i)) & 0xFF for i in range(4)] - # Show printable FOURCC only; some backends return 0 or non-printable bytes. - if fourcc_int <= 0: - fourcc = "N/A" - else: - fourcc = "".join(chr(b) if 32 <= b <= 126 else "." for b in fourcc_bytes) - width = int(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH)) - height = int(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - fps = self.cap.get(cv2.CAP_PROP_FPS) - backend_id = int(self.cap.get(cv2.CAP_PROP_BACKEND)) - print( - f"OpenCV negotiated stream: FOURCC='{fourcc}' " - f"(int={fourcc_int}, hex=0x{fourcc_int:08X}), " - f"size={width}x{height}, fps={fps:.2f}, backend_id={backend_id}" - ) - + self._capture_source.warmup_frames(3) + # Wait for first valid frame frame = None while frame is None: @@ -826,17 +963,24 @@ class Sensor: if box is not None: self.stitcher.isstitch = self.isstitch = False self.stitcher.boxes = box + self.stitch_valid = bool(self.isstitch) + img = self.frame_processor.process(frame) + self.initial_img = self._make_initial_gray(img) else: print("Override=True but no box provided.") else: self._find_stitch_boxes(frame) + if not self.stitch_valid: + raise RuntimeError( + f"Failed to detect {self._config_units} vision tiles for stitching. " + "Check camera index, lighting, and that MJPG 1080p is negotiated." + ) img = self.frame_processor.process(frame) self.initial_img = self._make_initial_gray(img) - # For live cameras, start a latest-frame capture thread to avoid latency buildup. - if self._is_live_camera: - self._start_capture_thread() - + # For live cameras, start background capture to avoid latency buildup. + if self._is_live_camera and self._capture_source is not None: + self._capture_source.start_background() def _make_initial_gray(self, img_bgr): gray = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY) @@ -850,6 +994,9 @@ class Sensor: def _find_stitch_boxes(self, img_src, show=False): """Detect vision tile boxes with camera stabilization.""" n_boxes, v_mean0, timeout = 0, 0, 5 + boxes = [] + best_n_boxes = 0 + best_boxes = [] start_time = cv2.getTickCount() while n_boxes < self._config_units: @@ -860,12 +1007,18 @@ class Sensor: if abs(v_mean - v_mean0) < 5: n_boxes, boxes = self.stitcher.find_stitch_boxes(frame, show=show) + if n_boxes > best_n_boxes: + best_n_boxes = n_boxes + best_boxes = boxes if (cv2.getTickCount() - start_time) / cv2.getTickFrequency() > timeout: print("Timeout finding vision units.") - self.stitcher.isstitch = self.isstitch = False break v_mean0 = v_mean + + if n_boxes < self._config_units and best_n_boxes >= self._config_units: + n_boxes = best_n_boxes + boxes = best_boxes self.n_boxes = n_boxes self.boxes = boxes @@ -875,73 +1028,57 @@ class Sensor: print(f"Sensor: {status}") return n_boxes - def _extract_tiles(self, img_src): - """Extract tiles from source image.""" - self.stitcher.get_stitch_unit(img_src) - self.stitcher.extract_tiles() - return True + # Unused after frame_processor owns stitch extract/process. + # def _extract_tiles(self, img_src): + # """Extract tiles from source image.""" + # self.stitcher.get_stitch_unit(img_src) + # self.stitcher.extract_tiles() + # return True def _is_source_open(self): - if self._use_pyav: - return self._av_container is not None - return self.cap is not None and self.cap.isOpened() + if self._capture_source is not None: + return self._capture_source.is_open() + return False + # Legacy PyAV / direct-cap fallbacks (unused after orisys.capture refactor). + # if self._use_pyav: + # return self._av_container is not None + # return self.cap is not None and self.cap.isOpened() def _read_frame(self): - if self._capture_running: - # Startup race guard: wait briefly until capture thread publishes first frame. - if self._latest_frame is None: - deadline = time.perf_counter() + 1.0 - while time.perf_counter() < deadline: - with self._latest_frame_lock: - if self._latest_frame is not None: - break - time.sleep(0.001) - - # Return latest frame immediately (do not wait for next camera tick). - # This keeps end-to-end latency minimal; caller may receive repeated - # frames if processing runs faster than camera FPS. - with self._latest_frame_lock: - frame = self._latest_frame - seq = self._latest_frame_seq - if frame is not None: - self._last_consumed_seq = seq - return True, frame.copy() - return False, None + if self._capture_source is not None and self._capture_source.running: + return self._capture_source.read_latest(copy=True, wait_s=1.0) return self._read_frame_raw() def _read_frame_raw(self): - if self._use_pyav: - if self._av_decoder is None: - return False, None - try: - frame = next(self._av_decoder) - except StopIteration: - return False, None - except Exception: - return False, None - return True, frame.to_ndarray(format="bgr24") - if self.cap is None: - return False, None - return self.cap.read() + if self._capture_source is not None: + return self._capture_source.read_raw() + return False, None + # Legacy PyAV / direct-cap fallbacks (unused after orisys.capture refactor). + # if self._use_pyav: + # if self._av_decoder is None: + # return False, None + # try: + # frame = next(self._av_decoder) + # except StopIteration: + # return False, None + # except Exception: + # return False, None + # return True, frame.to_ndarray(format="bgr24") + # if self.cap is None: + # return False, None + # return self.cap.read() - def _start_capture_thread(self): - if self._capture_thread is not None: - return - self._capture_running = True - self._capture_thread = threading.Thread(target=self._capture_loop, daemon=True) - self._capture_thread.start() - if self.verbose: - print("Latest-frame capture thread started.") + @property + def capture_status(self): + """Return capture health snapshot, or None in external-frame mode.""" + if self._capture_source is None: + return None + return self._capture_source.status - def _capture_loop(self): - while self._capture_running: - ret, frame = self._read_frame_raw() - if not ret: - time.sleep(0.001) - continue - with self._latest_frame_lock: - self._latest_frame = frame - self._latest_frame_seq += 1 + @property + def _capture_worker(self): + """Deprecated alias for ``_capture_source``.""" + return self._capture_source # ========================================================================= diff --git a/src/orisys/signal_processing.py b/src/orisys/signal_processing.py new file mode 100644 index 0000000..7d00345 --- /dev/null +++ b/src/orisys/signal_processing.py @@ -0,0 +1,379 @@ +"""Signal helpers for force filtering and force calibration. + +Batch ``lowpass_filter`` is the zero-phase Butterworth used by example charts. +``LowpassFilter`` is a causal, stateful filter for per-frame ``Sensor`` outputs. + +Force calibration resolution order (when register support arrives): + 1. Sensor register / EEPROM calibration blob + 2. ``sensors//force_calibration.json`` + 3. Hardcoded linear defaults below +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +from scipy.signal import butter, filtfilt, sosfilt, sosfilt_zi + +# Linear calibration from raw normal-force units to Newtons (ORY-BOX / ORY-MINI). +# Prefer sensors//force_calibration.json when available. +NORMAL_FORCE_CAL_SLOPE = 3.92685554668795e-05 +NORMAL_FORCE_CAL_OFFSET = 0.0217596967278535 + +DEFAULT_SAMPLE_RATE_HZ = 30.0 +DEFAULT_LOWPASS_ORDER = 2 + + +def _parse_calibration_point(point) -> tuple[float, float]: + if isinstance(point, dict): + raw = point.get("fnormal", point.get("raw", point.get("x"))) + force_n = point.get("force_n", point.get("n", point.get("y"))) + if raw is None or force_n is None: + raise ValueError(f"invalid calibration point: {point!r}") + return float(raw), float(force_n) + if isinstance(point, (list, tuple)) and len(point) >= 2: + return float(point[0]), float(point[1]) + raise ValueError(f"invalid calibration point: {point!r}") + + +@dataclass(frozen=True) +class ForceChannelCalibration: + """One force channel: linear coeffs and/or piecewise-linear breakpoints.""" + + method: str = "linear" + slope: float | None = None + offset: float | None = None + points: tuple[tuple[float, float], ...] = () + extrapolate: bool = True + clamp_output_min: float | None = 0.0 + input_mode: str = "magnitude" + signed: bool = False + unit: str = "N" + + @classmethod + def linear(cls, slope: float, offset: float, *, unit: str = "N") -> ForceChannelCalibration: + return cls(method="linear", slope=float(slope), offset=float(offset), unit=unit) + + @classmethod + def from_channel_dict( + cls, + block: dict, + *, + unit: str = "N", + default_method: str = "piecewise_linear", + ) -> ForceChannelCalibration | None: + if not isinstance(block, dict): + return None + + method = str(block.get("method", block.get("model", default_method))).lower() + if method in ("linear", "affine"): + slope = block.get("slope") + offset = block.get("offset") + if slope is None or offset is None: + return None + return cls.linear(float(slope), float(offset), unit=str(block.get("unit", unit))) + + if method != "piecewise_linear": + return None + + raw_points = block.get("points", []) + if not isinstance(raw_points, (list, tuple)) or len(raw_points) < 2: + return None + try: + points = tuple(sorted(_parse_calibration_point(p) for p in raw_points)) + except (TypeError, ValueError): + return None + if len(points) < 2: + return None + + return cls( + method="piecewise_linear", + points=points, + extrapolate=bool(block.get("extrapolate", True)), + clamp_output_min=block.get("clamp_output_min", 0.0), + input_mode=str(block.get("input", "magnitude")).lower(), + signed=bool(block.get("signed", False)), + unit=str(block.get("unit", unit)), + ) + + def convert(self, raw: float) -> float: + raw = float(raw) + if self.method == "linear": + slope = NORMAL_FORCE_CAL_SLOPE if self.slope is None else float(self.slope) + offset = NORMAL_FORCE_CAL_OFFSET if self.offset is None else float(self.offset) + return slope * raw + offset + + points = self.points + if len(points) < 2: + return raw + + if raw <= points[0][0]: + segment = (points[0], points[1]) + if not self.extrapolate: + return self._clamp(points[0][1]) + elif raw >= points[-1][0]: + segment = (points[-2], points[-1]) + if not self.extrapolate: + return self._clamp(points[-1][1]) + else: + segment = None + for left, right in zip(points, points[1:]): + if left[0] <= raw <= right[0]: + segment = (left, right) + break + if segment is None: + return self._clamp(points[-1][1]) + + (raw0, n0), (raw1, n1) = segment + if raw1 == raw0: + value = n0 + else: + ratio = (raw - raw0) / (raw1 - raw0) + value = n0 + ratio * (n1 - n0) + return self._clamp(value) + + def convert_component(self, raw: float) -> float: + if not self.signed: + return self.convert(raw) + raw = float(raw) + sign = -1.0 if raw < 0.0 else 1.0 + return sign * self.convert(abs(raw)) + + def _clamp(self, value: float) -> float: + if self.clamp_output_min is not None: + return max(float(self.clamp_output_min), float(value)) + return float(value) + + +_DEFAULT_NORMAL = ForceChannelCalibration.linear( + NORMAL_FORCE_CAL_SLOPE, + NORMAL_FORCE_CAL_OFFSET, +) + + +def _load_force_calibration_data(sensor_name: str | None) -> dict | None: + if not sensor_name: + return None + try: + from .configs import load_force_calibration + + data = load_force_calibration(sensor_name) + except Exception: + return None + return data if isinstance(data, dict) else None + + +def _channel_from_data(data: dict | None, channel: str) -> ForceChannelCalibration | None: + if not isinstance(data, dict): + return None + + unit = str(data.get("unit", "N")) + channels = data.get("channels") + if isinstance(channels, dict): + block = channels.get(channel) + cal = ForceChannelCalibration.from_channel_dict(block, unit=unit) + if cal is not None: + return cal + + # Legacy single-channel file: {"normal_force": {"model": "linear", ...}} + if channel == "normal": + block = data.get("normal_force", data) + cal = ForceChannelCalibration.from_channel_dict( + block, unit=unit, default_method="linear" + ) + if cal is not None: + return cal + if isinstance(block, dict): + slope = block.get("slope") + offset = block.get("offset") + if slope is not None and offset is not None: + return ForceChannelCalibration.linear( + float(slope), + float(offset), + unit=str(block.get("unit", unit)), + ) + return None + + +def _normal_calibration(sensor_name: str | None = None) -> ForceChannelCalibration: + cal = _channel_from_data(_load_force_calibration_data(sensor_name), "normal") + return cal if cal is not None else _DEFAULT_NORMAL + + +def _shear_calibration(sensor_name: str | None = None) -> ForceChannelCalibration | None: + return _channel_from_data(_load_force_calibration_data(sensor_name), "shear") + + +def _load_normal_force_coeffs(sensor_name: str | None = None) -> tuple[float, float]: + """Legacy helper: linear slope/offset only (piecewise returns hardcoded defaults).""" + cal = _normal_calibration(sensor_name) + if cal.method == "linear" and cal.slope is not None and cal.offset is not None: + return float(cal.slope), float(cal.offset) + return NORMAL_FORCE_CAL_SLOPE, NORMAL_FORCE_CAL_OFFSET + + +def calibrate_normal_force(raw_value: float, *, sensor_name: str | None = None) -> float: + """Convert a raw normal-force reading to Newtons. + + Uses ``sensors//force_calibration.json`` when present (piecewise or linear). + Falls back to hardcoded linear defaults when missing. + """ + return _normal_calibration(sensor_name).convert(raw_value) + + +def calibrate_shear_force(raw_value: float, *, sensor_name: str | None = None) -> float: + """Convert a raw shear magnitude to Newtons using the shear channel curve. + + Returns ``float(raw_value)`` unchanged when no shear calibration is available. + """ + cal = _shear_calibration(sensor_name) + if cal is None: + return float(raw_value) + return cal.convert(raw_value) + + +def calibrate_shear_components( + shear_x: float, + shear_y: float, + *, + sensor_name: str | None = None, +) -> tuple[float, float]: + """Convert raw shear components to Newtons. + + When the shear channel uses ``input: components`` and ``signed: true``, each + axis is calibrated on absolute magnitude then re-signed. Otherwise the + magnitude is calibrated and components are scaled proportionally. + """ + sx = float(shear_x) + sy = float(shear_y) + cal = _shear_calibration(sensor_name) + if cal is None: + return sx, sy + + if cal.input_mode == "components": + return cal.convert_component(sx), cal.convert_component(sy) + + magnitude = (sx * sx + sy * sy) ** 0.5 + if magnitude <= 0.0: + return 0.0, 0.0 + calibrated = cal.convert(magnitude) + scale = calibrated / magnitude + return sx * scale, sy * scale + + +def _nyquist_hz(sample_rate_hz: float) -> float: + return 0.5 * float(sample_rate_hz) + + +def _validate_cutoff(cutoff_hz: float, sample_rate_hz: float) -> None: + nyquist = _nyquist_hz(sample_rate_hz) + if cutoff_hz < 0 or cutoff_hz >= nyquist: + raise ValueError( + f"cutoff_hz must be in (0, {nyquist}) for sample_rate_hz={sample_rate_hz}" + ) + + +def lowpass_filter( + data, + cutoff_hz: float = 5.0, + sample_rate_hz: float = DEFAULT_SAMPLE_RATE_HZ, + order: int = DEFAULT_LOWPASS_ORDER, +) -> np.ndarray: + """Zero-phase Butterworth low-pass for uniformly sampled 1-D data. + + Matches the example/viewer chart filter: ``cutoff_hz == 0`` is a no-op. + """ + y = np.asarray(data, dtype=np.float64) + if cutoff_hz == 0: + return y.copy() + if y.size == 0: + return y + + _validate_cutoff(cutoff_hz, sample_rate_hz) + b, a = butter(int(order), cutoff_hz / _nyquist_hz(sample_rate_hz), btype="low") + padlen = 3 * (max(len(a), len(b)) - 1) + if y.size <= padlen: + return y.copy() + try: + return filtfilt(b, a, y) + except ValueError: + return y.copy() + + +class LowpassFilter: + """Causal Butterworth low-pass with persistent SOS state (one sample at a time).""" + + def __init__( + self, + cutoff_hz: float = 0.0, + sample_rate_hz: float = DEFAULT_SAMPLE_RATE_HZ, + order: int = DEFAULT_LOWPASS_ORDER, + n_channels: int = 3, + ): + self.n_channels = int(n_channels) + self._sos = None + self._zi = None + self.configure( + cutoff_hz=cutoff_hz, + sample_rate_hz=sample_rate_hz, + order=order, + ) + + @property + def enabled(self) -> bool: + return self._sos is not None + + def configure( + self, + *, + cutoff_hz: float | None = None, + sample_rate_hz: float | None = None, + order: int | None = None, + ) -> None: + if cutoff_hz is not None: + self.cutoff_hz = float(cutoff_hz) + if sample_rate_hz is not None: + self.sample_rate_hz = float(sample_rate_hz) + if order is not None: + self.order = int(order) + + if self.cutoff_hz <= 0: + self._sos = None + self._zi = None + return + + _validate_cutoff(self.cutoff_hz, self.sample_rate_hz) + self._sos = butter( + self.order, + self.cutoff_hz / _nyquist_hz(self.sample_rate_hz), + btype="low", + output="sos", + ) + self._zi = None + + def reset(self, x0=None) -> None: + """Clear filter memory. Optional ``x0`` initializes to a steady-state DC.""" + if self._sos is None: + self._zi = None + return + if x0 is None: + self._zi = None + return + x = np.asarray(x0, dtype=np.float64).reshape(-1) + if x.size != self.n_channels: + raise ValueError(f"x0 must have {self.n_channels} channels, got {x.size}") + zi0 = sosfilt_zi(self._sos) + self._zi = zi0[:, :, None] * x[None, None, :] + + def process(self, values) -> np.ndarray: + """Filter one multi-channel sample. Returns float64 array of shape (n_channels,).""" + x = np.asarray(values, dtype=np.float64).reshape(-1) + if x.size != self.n_channels: + raise ValueError(f"expected {self.n_channels} channels, got {x.size}") + if self._sos is None: + return x.copy() + if self._zi is None: + self.reset(x) + y, self._zi = sosfilt(self._sos, x[None, :], zi=self._zi, axis=0) + return np.asarray(y[0], dtype=np.float64) diff --git a/src/orisys/stitch/__init__.py b/src/orisys/stitch/__init__.py new file mode 100644 index 0000000..bcf579b --- /dev/null +++ b/src/orisys/stitch/__init__.py @@ -0,0 +1,33 @@ +"""Compound-eye stitching package (ops, backends, feature/box/tile modules).""" + +from .api import Stitcher +from .backends import available_blend_backends, get_blend_backend +from .config import BLEND_MODES, StitchStatus, resolve_blend_mode, validate_box_grid +from .ops import ( + build_blend_coef, + compute_final_crop_rect, + copy_into_tile_buffer, + coverage_ratio, + largest_fully_covered_rect, + match_bgr_brightness, + paste_tile_clipped, + tile_weight_map, +) + +__all__ = [ + "Stitcher", + "StitchStatus", + "BLEND_MODES", + "resolve_blend_mode", + "validate_box_grid", + "get_blend_backend", + "available_blend_backends", + "tile_weight_map", + "paste_tile_clipped", + "copy_into_tile_buffer", + "build_blend_coef", + "compute_final_crop_rect", + "largest_fully_covered_rect", + "match_bgr_brightness", + "coverage_ratio", +] diff --git a/src/orisys/stitch/api.py b/src/orisys/stitch/api.py new file mode 100644 index 0000000..0038dff --- /dev/null +++ b/src/orisys/stitch/api.py @@ -0,0 +1,324 @@ +"""Public Stitcher façade composing tiles, boxes, feature stitch, and blend backends.""" + +from __future__ import annotations + +import time +from typing import Optional + +import numpy as np + +from .backends import get_blend_backend +from .config import ( + BLEND_MODES, + StitchStatus, + numba_blend_available, + resolve_blend_mode, +) +from . import boxes as boxes_mod +from . import feature as feature_mod +from .ops import build_blend_coef +from . import tiles as tiles_mod + + +class Stitcher: + """Compound-eye stitcher (feature/ORB path + feathered blend backends).""" + + def __init__(self, isstitch=False, config_stitcher=None, verbose=False): + self.isstitch = isstitch + self.config_stitcher = config_stitcher + self.verbose = verbose + + self.min_area_frac = config_stitcher["min_area_frac"] + self.max_area_frac = config_stitcher["max_area_frac"] + self.max_aspect_ratio = config_stitcher["max_aspect_ratio"] + + self.stitch_valid = False + self.boxes = self.units = None + self.index = config_stitcher["index"] + self.stitch_calibration = self.tile_list = [] + self.ncol = config_stitcher["n"][0] + self.nrow = config_stitcher["n"][1] + self.nominal_ov_pos = config_stitcher["nominal_ov_pos"] + self.margin = config_stitcher["margin"] + self.bound = self.nominal_ov_pos - self.margin + self.row_tol = config_stitcher["row_tol"] + self.box_size = config_stitcher["box_size"] + # Physical tile count for box detection (often 15); grid n is the active stitch layout. + self.n_boxes_target = int(config_stitcher.get("n_boxes_target", 15)) + + self._blend_coef_cache = {} + self._feature_stitch_cache = { + "ref_idx": None, + "moving_indices": None, + "tile_shapes": None, + "transforms": None, + "canvas": {}, + } + self._feature_frame_idx = {"idx": 0} + self._feature_stitch_print_every = 30 + self._feature_stitch_pad = 5 + self._feature_stitch_match_brightness = False + self._blend_mode, fell_back = resolve_blend_mode(config_stitcher) + self._blend_backend_name = "numpy" if self._blend_mode == "none" else self._blend_mode + self._numba_failed = False + self._blend_backend = get_blend_backend( + self._blend_backend_name, + on_numba_failure=self._mark_numba_failed, + ) + self._last_coverage = 0.0 + self._last_ms = 0.0 + + if fell_back and self.isstitch: + print( + "Warning: stitcher blend=numba requested but numba is " + "unavailable; using numpy blend path." + ) + elif self.isstitch and self.verbose: + print(f"Stitch blend: {self._blend_mode}") + + if self.verbose: + print(f"Stitcher initialized with config: {self.config_stitcher}") + + @property + def blend_mode(self): + """Configured blend mode: ``none``, ``numpy``, or ``numba``.""" + return self._blend_mode + + @property + def status(self) -> StitchStatus: + return StitchStatus( + blend_mode=self._blend_mode, + backend_name=getattr(self._blend_backend, "name", self._blend_backend_name), + numba_failed=self._numba_failed, + coverage=float(self._last_coverage), + last_ms=float(self._last_ms), + tile_count=len(self.tile_list) if self.tile_list is not None else 0, + stitch_valid=bool(self.stitch_valid), + box_count=len(self.boxes) if self.boxes is not None else 0, + expected_boxes=int(self.n_boxes_target), + ) + + def configure_feature_stitch( + self, + *, + match_brightness=None, + pad=None, + print_every=None, + blend=None, + blend_backend=None, + ): + if match_brightness is not None: + self._feature_stitch_match_brightness = bool(match_brightness) + if pad is not None: + self._feature_stitch_pad = max(0, int(pad)) + if print_every is not None: + self._feature_stitch_print_every = max(1, int(print_every)) + mode = blend if blend is not None else blend_backend + if mode is not None: + mode = str(mode).strip().lower() + if mode not in BLEND_MODES: + raise ValueError( + f"Unknown blend={mode!r}; expected one of {sorted(BLEND_MODES)}" + ) + if mode == "numba" and (not numba_blend_available() or self._numba_failed): + print("Warning: numba unavailable; using numpy blend.") + mode = "numpy" + self._blend_mode = mode + self._blend_backend_name = "numpy" if mode == "none" else mode + if mode == "numba": + self._numba_failed = False + self._blend_backend = get_blend_backend( + self._blend_backend_name, + on_numba_failure=self._mark_numba_failed, + ) + + def _resolve_call_blend(self, blend): + if blend is None: + mode = self._blend_mode + elif isinstance(blend, bool): + if blend: + mode = self._blend_mode if self._blend_mode != "none" else "numpy" + else: + return False, self._blend_backend_name + else: + mode = str(blend).strip().lower() + if mode not in BLEND_MODES: + raise ValueError( + f"Unknown blend={mode!r}; expected one of {sorted(BLEND_MODES)}" + ) + if mode == "numba" and not numba_blend_available(): + mode = "numpy" + + if mode == "numba" and self._numba_failed: + mode = "numpy" + + if mode == "none": + return False, self._blend_backend_name + return True, mode + + def _mark_numba_failed(self, exc): + if self._numba_failed: + return + self._numba_failed = True + self._blend_backend_name = "numpy" + if self._blend_mode == "numba": + self._blend_mode = "numpy" + self._blend_backend = get_blend_backend("numpy") + print( + f"Warning: numba blend failed ({type(exc).__name__}: {exc}); " + f"falling back to numpy." + ) + + def _get_blend_coef(self, ov_pos, dim, ndim): + key = (ov_pos, dim, ndim) + coef = self._blend_coef_cache.get(key) + if coef is not None: + return coef + blend_coef = build_blend_coef(ov_pos, dim, ndim) + self._blend_coef_cache[key] = blend_coef + return blend_coef + + def _ensure_blend_backend(self, backend_name: str): + name = "numpy" if backend_name == "none" else backend_name + if self._numba_failed and name == "numba": + name = "numpy" + current = getattr(self._blend_backend, "name", None) + if current != name: + self._blend_backend = get_blend_backend( + name, + on_numba_failure=self._mark_numba_failed, + ) + self._blend_backend_name = name + + def stitch_rt_optimized(self, change_sv=False, s_mean=28.0, v_mean=180.0, blend=None): + """Stitch the current tile list via the feature/ORB path. + + Returns ``None`` when a valid feature layout cannot be built. + """ + use_feather, blend_backend = self._resolve_call_blend(blend) + prev_backend = self._blend_backend_name + self._ensure_blend_backend(blend_backend) + t0 = time.perf_counter() + try: + feature_stitched = self.stitch_feature_optimized( + blend=use_feather, + match_brightness=self._feature_stitch_match_brightness, + pad=self._feature_stitch_pad, + print_every=self._feature_stitch_print_every, + ) + finally: + if self._numba_failed: + self._ensure_blend_backend("numpy") + else: + self._ensure_blend_backend(prev_backend) + self._last_ms = (time.perf_counter() - t0) * 1000.0 + canvas = self._feature_stitch_cache.get("canvas") or {} + self._last_coverage = float(canvas.get("last_coverage", self._last_coverage)) + if feature_stitched is not None: + return feature_stitched + return None + + def stitch_feature_optimized(self, blend=True, match_brightness=False, pad=5, print_every=30): + return feature_mod.stitch_feature_optimized( + isstitch=self.isstitch, + index=self.index, + nrow=self.nrow, + ncol=self.ncol, + tile_list=self.tile_list, + cache=self._feature_stitch_cache, + blend_backend=self._blend_backend, + blend=blend, + match_brightness=match_brightness, + pad=pad, + print_every=print_every, + verbose=self.verbose, + frame_idx_holder=self._feature_frame_idx, + ) + + def find_stitch_boxes(self, img_src, show=False): + n_boxes, boxes, grid_ok = boxes_mod.find_stitch_boxes( + img_src, + box_size=self.box_size, + min_area_frac=self.min_area_frac, + max_area_frac=self.max_area_frac, + max_aspect_ratio=self.max_aspect_ratio, + row_tol=self.row_tol, + nrow=self.nrow, + ncol=self.ncol, + show=show, + n_boxes_target=self.n_boxes_target, + ) + self.boxes = boxes + # Detection target is physical unit count; active stitch grid is nrow*ncol via index. + self.stitch_valid = bool(self.isstitch and n_boxes == self.n_boxes_target) + if self.isstitch and n_boxes != self.n_boxes_target: + print( + f"Warning: stitch box count {n_boxes} != expected " + f"n_boxes_target={self.n_boxes_target}; stitch_valid=False." + ) + elif self.isstitch and not grid_ok and self.verbose: + # Soft note: active index length may be nrow*ncol while more boxes exist. + print( + f"Note: detected {n_boxes} boxes; active stitch grid is " + f"{self.nrow}x{self.ncol} via index ({len(tiles_mod.normalize_index(self.index))} tiles)." + ) + return n_boxes, self.boxes + + def get_stitch_unit(self, img, flip_x=True, flip_y=True, index=np.arange(15)): + active_index = index if self.units is None else self.index + if isinstance(active_index, int): + selected_index = {active_index} + else: + selected_index = set(int(i) for i in active_index) + self.units = tiles_mod.extract_units_from_boxes( + img, + self.boxes, + selected_index=selected_index, + units=self.units, + flip_x=flip_x, + flip_y=flip_y, + ) + return True + + def extract_tiles(self): + if not self.isstitch: + tile_idx = 7 + self.index = [tile_idx] if isinstance(tile_idx, int) else tile_idx + self.ncol = 1 + self.nrow = 1 + + tile_list = tiles_mod.get_tile_by_index(self.units, self.index) + self.tile_list = tile_list + return tile_list + + def draw_boxes(self, img, thickness=2): + return boxes_mod.draw_boxes(img, self.boxes, thickness=thickness) + + # Compatibility aliases used by older call sites / tests. + def _copy_into_tile_buffer(self, src_view, dst, flip_x=True, flip_y=True): + from .ops import copy_into_tile_buffer + + copy_into_tile_buffer(src_view, dst, flip_x=flip_x, flip_y=flip_y) + + def _tile_weight_map(self, h, w, margin=16): + from .ops import tile_weight_map + + return tile_weight_map(h, w, margin) + + def _paste_tile_clipped(self, canvas, tile, y0, x0, cov=None): + from .ops import paste_tile_clipped + + paste_tile_clipped(canvas, tile, y0, x0, cov=cov) + + def _blend_tile_inplace(self, blend, wsum, cov, src, weight, y0, x0, scratch=None): + self._blend_backend.blend_tile_inplace(blend, wsum, cov, src, weight, y0, x0) + + def _compute_final_crop_rect(self, cov): + from .ops import compute_final_crop_rect + + return compute_final_crop_rect(cov) + + def _largest_fully_covered_rect(self, cov): + from .ops import largest_fully_covered_rect + + return largest_fully_covered_rect(cov) diff --git a/src/orisys/stitch/backends/__init__.py b/src/orisys/stitch/backends/__init__.py new file mode 100644 index 0000000..c251c75 --- /dev/null +++ b/src/orisys/stitch/backends/__init__.py @@ -0,0 +1,47 @@ +"""Blend backend registry (numpy / numba; future: torch / cupy).""" + +from __future__ import annotations + +from typing import Callable, Optional + +from .base import BlendBackend +from .numba_backend import NumbaBlendBackend +from .numpy_backend import NumpyBlendBackend +from ..config import numba_blend_available + +_BACKENDS = { + "numpy": NumpyBlendBackend, + "numba": NumbaBlendBackend, +} + + +def get_blend_backend( + name: str, + *, + on_numba_failure: Optional[Callable[[BaseException], None]] = None, +) -> BlendBackend: + """Return a blend backend instance by name. + + Unknown names raise ``ValueError``. ``numba`` falls back to construction + even if JIT is missing; the backend itself falls back to numpy on failure. + """ + key = str(name).strip().lower() + if key == "none": + # Feather off uses paste path; keep a numpy backend available if forced. + key = "numpy" + if key not in _BACKENDS: + raise ValueError( + f"Unknown blend backend {name!r}; expected one of " + f"{sorted(set(_BACKENDS) | {'none'})} " + f"(future backends e.g. 'torch' / 'cupy' are not registered yet)." + ) + if key == "numba": + return NumbaBlendBackend(on_failure=on_numba_failure) + return NumpyBlendBackend() + + +def available_blend_backends() -> list[str]: + names = ["none", "numpy"] + if numba_blend_available(): + names.append("numba") + return names diff --git a/src/orisys/stitch/backends/base.py b/src/orisys/stitch/backends/base.py new file mode 100644 index 0000000..4b6fa97 --- /dev/null +++ b/src/orisys/stitch/backends/base.py @@ -0,0 +1,29 @@ +"""BlendBackend protocol for CPU (and future GPU) stitch compositors.""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + + +@runtime_checkable +class BlendBackend(Protocol): + """Array-in / in-place composite kernel. + + Implementations must accept contiguous float32/uint8 NumPy arrays today. + Future Torch/CuPy backends can adapt tensors to this contract or extend it. + """ + + name: str + + def blend_tile_inplace( + self, + blend, + wsum, + cov, + src, + weight, + y0: int, + x0: int, + ) -> None: + """Composite ``src`` weighted by ``weight`` onto canvas buffers at (y0, x0).""" + ... diff --git a/src/orisys/stitch/backends/numba_backend.py b/src/orisys/stitch/backends/numba_backend.py new file mode 100644 index 0000000..17a73af --- /dev/null +++ b/src/orisys/stitch/backends/numba_backend.py @@ -0,0 +1,53 @@ +"""Numba blend backend wrapping ``orisys_blend.kernel``.""" + +from __future__ import annotations + +import numpy as np + +from .numpy_backend import NumpyBlendBackend, blend_kernel_numpy + +try: + from orisys_blend.kernel import _NUMBA_AVAILABLE, blend_kernel as _blend_kernel_numba +except Exception: + _NUMBA_AVAILABLE = False + _blend_kernel_numba = None + +if _blend_kernel_numba is None: + _NUMBA_AVAILABLE = False + + +class NumbaBlendBackend: + name = "numba" + + def __init__(self, *, on_failure=None): + self._on_failure = on_failure + self._fallback = NumpyBlendBackend() + self.failed = False + + def blend_tile_inplace(self, blend, wsum, cov, src, weight, y0, x0) -> None: + h, w = src.shape[:2] + H, W = blend.shape[:2] + y0d, x0d = max(0, y0), max(0, x0) + y1d, x1d = min(H, y0 + h), min(W, x0 + w) + if y0d >= y1d or x0d >= x1d: + return + + sy0, sx0 = y0d - y0, x0d - x0 + sh, sw = y1d - y0d, x1d - x0d + src_f = src[sy0 : sy0 + sh, sx0 : sx0 + sw].astype(np.float32, copy=False) + args = ( + blend[y0d:y1d, x0d:x1d], + wsum[y0d:y1d, x0d:x1d], + cov[y0d:y1d, x0d:x1d], + src_f, + weight[sy0 : sy0 + sh, sx0 : sx0 + sw], + ) + if not self.failed and _NUMBA_AVAILABLE and _blend_kernel_numba is not None: + try: + _blend_kernel_numba(*args) + return + except Exception as exc: + self.failed = True + if self._on_failure is not None: + self._on_failure(exc) + blend_kernel_numpy(*args) diff --git a/src/orisys/stitch/backends/numpy_backend.py b/src/orisys/stitch/backends/numpy_backend.py new file mode 100644 index 0000000..c76f66a --- /dev/null +++ b/src/orisys/stitch/backends/numpy_backend.py @@ -0,0 +1,61 @@ +"""NumPy blend backend (no JIT).""" + +from __future__ import annotations + +import numpy as np + + +def blend_kernel_numpy(blend_patch, wsum_patch, cov_patch, src_patch, w_patch): + """Incremental weighted-average composite (pure NumPy, no JIT). + + Uses contiguous arithmetic instead of boolean fancy indexing. Callers must + zero ``blend`` / ``wsum`` on uncovered pixels (as the stitcher reset does); + then the covered lerp also paints first-touch pixels correctly when ``w > 0``. + """ + w = w_patch if w_patch.dtype == np.float32 else np.asarray(w_patch, dtype=np.float32) + src = src_patch if src_patch.dtype == np.float32 else np.asarray(src_patch, dtype=np.float32) + + uncovered = cov_patch == 0 + if uncovered.all(): + np.copyto(blend_patch, src) + np.copyto(wsum_patch, w) + cov_patch.fill(1) + return + + ws = wsum_patch + w + a = w / np.maximum(ws, np.float32(1e-6)) + for c in range(src.shape[2]): + bc = blend_patch[..., c] + bc += (src[..., c] - bc) * a + np.copyto(wsum_patch, ws) + + if uncovered.any(): + need_fix = uncovered & (w < np.float32(1e-6)) + if need_fix.any(): + blend_patch[need_fix] = src[need_fix] + wsum_patch[need_fix] = w[need_fix] + + cov_patch.fill(1) + + +class NumpyBlendBackend: + name = "numpy" + + def blend_tile_inplace(self, blend, wsum, cov, src, weight, y0, x0) -> None: + h, w = src.shape[:2] + H, W = blend.shape[:2] + y0d, x0d = max(0, y0), max(0, x0) + y1d, x1d = min(H, y0 + h), min(W, x0 + w) + if y0d >= y1d or x0d >= x1d: + return + + sy0, sx0 = y0d - y0, x0d - x0 + sh, sw = y1d - y0d, x1d - x0d + src_f = src[sy0 : sy0 + sh, sx0 : sx0 + sw].astype(np.float32, copy=False) + blend_kernel_numpy( + blend[y0d:y1d, x0d:x1d], + wsum[y0d:y1d, x0d:x1d], + cov[y0d:y1d, x0d:x1d], + src_f, + weight[sy0 : sy0 + sh, sx0 : sx0 + sw], + ) diff --git a/src/orisys/stitch/boxes.py b/src/orisys/stitch/boxes.py new file mode 100644 index 0000000..c69a4a6 --- /dev/null +++ b/src/orisys/stitch/boxes.py @@ -0,0 +1,217 @@ +"""Box discovery and visualization for compound-eye calibration.""" + +from __future__ import annotations + +import os +from typing import List, Optional, Sequence, Tuple + +import cv2 +import numpy as np + + +def correct_edge_bloom(img, method: str = "divide", kernel_size=None, clip: bool = True): + """Illumination correction to reduce edge bloom (box detection only).""" + if img is None: + raise ValueError("Input image is None.") + + h, w = img.shape[:2] + if kernel_size is None: + k = max(31, min(h, w) // 2) + if k % 2 == 0: + k += 1 + kernel_size = k + img_f = img.astype(np.float32) + bg = cv2.GaussianBlur(img_f, (kernel_size, kernel_size), 0) + if method == "subtract": + mean_bg = bg.mean(axis=(0, 1), keepdims=True) + corr = img_f - (bg - mean_bg) + else: + eps = 1e-6 + mean_bg = bg.mean(axis=(0, 1), keepdims=True) + eps + gain = mean_bg * 0.5 + corr = (img_f / (bg + eps)) * gain + if clip: + corr = np.clip(corr, 0, 255).astype(np.uint8) + else: + corr = corr.astype(np.uint8) + return corr + + +def filter_boxes_by_size( + boxes: Sequence[Tuple[int, int, int, int]], + img_shape, + *, + min_area_frac: float, + max_area_frac: float, + max_aspect_ratio: float, +) -> List[Tuple[int, int, int, int]]: + h, w = img_shape + img_area = h * w + min_area = img_area * float(min_area_frac) + max_area = img_area * float(max_area_frac) + filtered = [] + for (x, y, ww, hh) in boxes: + area = ww * hh + if area < min_area or area > max_area or (abs(ww - hh) > max_aspect_ratio): + continue + filtered.append((x, y, ww, hh)) + return filtered + + +def detect_nonblack_regions( + img, + *, + box_size: Sequence[int], + min_area_frac: float, + max_area_frac: float, + max_aspect_ratio: float, + morph_k: int = 3, + n_boxes_target: int = 15, + bk_thresh_step: int = 5, + bk_thresh_max: int = 150, + show: bool = False, + row_tol: float = 0.4, + save_preview_path: str = "./config/compound_eye.png", +) -> Tuple[int, List[Tuple[int, int, int, int]]]: + """Detect tile boxes; returns ``(n_boxes, boxes as x0,y0,x1,y1)``.""" + if img is None: + raise ValueError("Input image is None.") + + h, w = img.shape[:2] + n_boxes = 0 + bk_thresh = 0 + boxes: List[Tuple[int, int, int, int]] = [] + + hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) + v = hsv[:, :, 2] + while n_boxes != n_boxes_target and bk_thresh <= bk_thresh_max: + _, v_mask = cv2.threshold(v, bk_thresh, 255, cv2.THRESH_BINARY) + mask = v_mask.copy() + edge = 20 + if w > edge * 2: + mask[:, :edge] = 0 + mask[:, -edge:] = 0 + + k = max(1, morph_k) + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (k, k)) + mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel, iterations=1) + mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel, iterations=2) + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + boxes = [] + for c in contours: + x, y, ww, hh = cv2.boundingRect(c) + boxes.append((x, y, ww, hh)) + + boxes = filter_boxes_by_size( + boxes, + (h, w), + min_area_frac=min_area_frac, + max_area_frac=max_area_frac, + max_aspect_ratio=max_aspect_ratio, + ) + n_boxes = len(boxes) + bk_thresh += bk_thresh_step + + print( + "Detected {} candidate boxes with bk_thresh={}".format( + n_boxes, bk_thresh - bk_thresh_step + ) + ) + if n_boxes == 0: + return 0, [] + + target_w = int(box_size[0]) + target_h = int(box_size[1]) + cropped_boxes = [] + for (x, y, ww, hh) in boxes: + cx = x + ww // 2 + cy = y + hh // 2 + nx0 = int(cx - target_w // 2) + ny0 = int(cy - target_h // 2) + nx0 = max(0, min(nx0, w - target_w)) + ny0 = max(0, min(ny0, h - target_h)) + cropped_boxes.append((nx0, ny0, nx0 + target_w, ny0 + target_h)) + + if cropped_boxes: + heights = [b[3] - b[1] for b in cropped_boxes] + median_height = np.median(heights) if heights else target_h + row_tolerance = max(1, int(median_height * row_tol)) + + def get_row_key(box): + y = box[1] + quantized_y = (y // row_tolerance) * row_tolerance + return (quantized_y, box[0]) + + cropped_boxes = sorted(cropped_boxes, key=get_row_key) + else: + cropped_boxes = sorted(cropped_boxes, key=lambda b: (b[1], b[0])) + + img_with_boxes = draw_boxes(img, cropped_boxes) + if show: + cv2.imshow("Detected Boxes", img_with_boxes) + cv2.waitKey(0) + + if save_preview_path: + parent = os.path.dirname(save_preview_path) + if parent: + os.makedirs(parent, exist_ok=True) + cv2.imwrite(save_preview_path, img_with_boxes) + + return len(cropped_boxes), cropped_boxes + + +def draw_boxes(img, boxes, thickness: int = 2): + if boxes is None or len(boxes) == 0: + print("No boxes to draw") + return img + + img_draw = img.copy() + color = (0, 255, 0) + for i, (x0, y0, x1, y1) in enumerate(boxes): + cv2.rectangle(img_draw, (x0, y0), (x1, y1), color, thickness) + text_size = cv2.getTextSize(str(i), cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2)[0] + text_pos = (x0 + 5, y0 + text_size[1] + 5) + cv2.rectangle( + img_draw, + (text_pos[0] - 2, text_pos[1] - text_size[1] - 2), + (text_pos[0] + text_size[0] + 2, text_pos[1] + 2), + (0, 0, 0), + -1, + ) + cv2.putText(img_draw, str(i), text_pos, cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) + return img_draw + + +def find_stitch_boxes( + img_src, + *, + box_size, + min_area_frac, + max_area_frac, + max_aspect_ratio, + row_tol, + nrow: int, + ncol: int, + show: bool = False, + n_boxes_target: Optional[int] = None, +): + """Run bloom correction + detection; return ``(n_boxes, boxes, grid_ok)``. + + ``n_boxes_target`` is the physical tile count to detect (default 15). + ``grid_ok`` is True when that count matches ``n_boxes_target``. + """ + img_corr = correct_edge_bloom(img_src) + target = int(n_boxes_target) if n_boxes_target is not None else 15 + n_boxes, boxes = detect_nonblack_regions( + img_corr, + box_size=box_size, + min_area_frac=min_area_frac, + max_area_frac=max_area_frac, + max_aspect_ratio=max_aspect_ratio, + n_boxes_target=target, + show=show, + row_tol=row_tol, + ) + grid_ok = n_boxes == target + return n_boxes, boxes, grid_ok diff --git a/src/orisys/stitch/config.py b/src/orisys/stitch/config.py new file mode 100644 index 0000000..77cbe0d --- /dev/null +++ b/src/orisys/stitch/config.py @@ -0,0 +1,76 @@ +"""Stitch configuration, blend-mode resolution, and status.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any, Mapping, Optional + +# JSON ``stitcher.blend`` / ``blend_list``: none = fast paste; numpy/numba = feathered. +BLEND_MODES = frozenset({"none", "numpy", "numba"}) +DEFAULT_BLEND_MODE = "none" + + +def numba_blend_available() -> bool: + try: + from orisys_blend.kernel import _NUMBA_AVAILABLE, blend_kernel + except Exception: + return False + return bool(_NUMBA_AVAILABLE and blend_kernel is not None) + + +def resolve_blend_mode(config_stitcher: Optional[Mapping[str, Any]] = None): + """Resolve stitcher blend mode from env, then JSON ``blend``. + + Returns ``(mode, fell_back_from_numba)`` where mode is one of + ``none`` / ``numpy`` / ``numba``. + + Priority: ``ORISYS_STITCH_BLEND``, ``ORISYS_STITCH_BLEND_BACKEND``, + then ``config_stitcher["blend"]`` / legacy ``blend_backend``, then ``none``. + """ + requested = None + for key in ("ORISYS_STITCH_BLEND", "ORISYS_STITCH_BLEND_BACKEND"): + env = os.environ.get(key, "").strip().lower() + if env in BLEND_MODES: + requested = env + break + + if requested is None and isinstance(config_stitcher, dict): + for key in ("blend", "blend_backend"): + cfg = str(config_stitcher.get(key, "")).strip().lower() + if cfg in BLEND_MODES: + requested = cfg + break + + if requested is None: + requested = DEFAULT_BLEND_MODE + + if requested == "numba" and not numba_blend_available(): + return "numpy", True + return requested, False + + +def expected_box_count(nrow: int, ncol: int) -> int: + return int(nrow) * int(ncol) + + +def validate_box_grid(boxes, nrow: int, ncol: int) -> bool: + """True when detected boxes match the configured grid size.""" + if boxes is None: + return False + return len(boxes) == expected_box_count(nrow, ncol) + + +@dataclass +class StitchStatus: + """Runtime health snapshot for compound-eye stitching.""" + + blend_mode: str = DEFAULT_BLEND_MODE + backend_name: str = "numpy" + numba_failed: bool = False + coverage: float = 0.0 + last_ms: float = 0.0 + tile_count: int = 0 + stitch_valid: bool = False + box_count: int = 0 + expected_boxes: int = 0 diff --git a/src/orisys/stitch/feature.py b/src/orisys/stitch/feature.py new file mode 100644 index 0000000..a754fe8 --- /dev/null +++ b/src/orisys/stitch/feature.py @@ -0,0 +1,599 @@ +"""Feature-based (ORB/BFMatcher) stitch plan and render orchestration.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional, Sequence + +import cv2 +import numpy as np + +from .backends.base import BlendBackend +from .ops import ( + compute_final_crop_rect, + coverage_ratio, + match_bgr_brightness, + mean_gray_bgr, + paste_tile_clipped, + tile_weight_map, + translation_from_H_to_ref, +) +from .tiles import normalize_index + + +def tile_to_bgr(img): + if img.ndim == 2: + return cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + return img + + +def tile_to_gray(bgr): + if bgr.ndim == 2: + return bgr + return cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY) + + +def ensure_u8_gray(gray): + if gray.dtype == np.uint8: + return gray + return cv2.normalize(gray, None, 0, 255, cv2.NORM_MINMAX).astype(np.uint8) + + +def preprocess_gray_for_orb(g0, g1): + g0 = cv2.GaussianBlur(g0, (5, 5), 0) + g1 = cv2.GaussianBlur(g1, (5, 5), 0) + g0 = cv2.equalizeHist(g0) + g1 = cv2.equalizeHist(g1) + return g0, g1 + + +def estimate_bfmatcher_stitch_transform( + t0, + t1, + *, + ransac_reproj_thresh=5.0, + ratio=0.9, + orb_nfeatures=5000, + verbose=True, +): + """Estimate a 3x3 transform mapping tile coordinates into the reference frame.""" + t0_bgr = tile_to_bgr(t0) + t1_bgr = tile_to_bgr(t1) + g0 = ensure_u8_gray(tile_to_gray(t0_bgr)) + g1 = ensure_u8_gray(tile_to_gray(t1_bgr)) + + if verbose: + print( + f"BF stitcher debug: t0 dtype={t0.dtype}, t1 dtype={t1.dtype}, " + f"g0 dtype={g0.dtype}, g0 range=({np.min(g0)}, {np.max(g0)}), " + f"g1 dtype={g1.dtype}, g1 range=({np.min(g1)}, {np.max(g1)})" + ) + + # g0, g1 = preprocess_gray_for_orb(g0, g1) + orb = cv2.ORB_create( + nfeatures=orb_nfeatures, + scaleFactor=1.2, + nlevels=8, + edgeThreshold=15, + fastThreshold=5, + ) + k0, d0 = orb.detectAndCompute(g0, None) + k1, d1 = orb.detectAndCompute(g1, None) + + info = { + "ok": False, + "method": None, + "inliers": 0, + "mean_err": None, + "good_matches": 0, + "k0": k0, + "k1": k1, + "t0_bgr": t0_bgr, + "t1_bgr": t1_bgr, + } + if d0 is None or d1 is None or len(k0) < 4 or len(k1) < 4: + if verbose: + print( + "OpenCV BFMatcher stitcher: not enough features.", + f"k0={0 if k0 is None else len(k0)}", + f"k1={0 if k1 is None else len(k1)}", + f"d0_is_none={d0 is None}", + f"d1_is_none={d1 is None}", + ) + return None, info + + matches = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=False).knnMatch(d0, d1, k=2) + good = [] + for m_n in matches: + if len(m_n) != 2: + continue + m, n = m_n + if m.distance < ratio * n.distance: + good.append(m) + + info["good_matches"] = len(good) + if verbose: + print(f"BFMatcher good matches: {len(good)}") + if len(good) < 4: + return None, info + + pts0 = np.float32([k0[m.queryIdx].pt for m in good]).reshape(-1, 1, 2) + pts1 = np.float32([k1[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) + method = "affine_partial2d" + M, inlier_mask = cv2.estimateAffinePartial2D( + pts1, + pts0, + method=cv2.RANSAC, + ransacReprojThreshold=ransac_reproj_thresh, + ) + if M is not None: + H = np.vstack([M, [0, 0, 1]]).astype(np.float64) + else: + method = "homography" + H, inlier_mask = cv2.findHomography(pts1, pts0, cv2.RANSAC, ransac_reproj_thresh) + + info["method"] = method + info["inliers"] = int(inlier_mask.sum()) if inlier_mask is not None else 0 + if verbose: + print(f"Estimated {method} inliers: {info['inliers']}") + if H is None: + return None, info + + if abs(H[2, 2]) > 1e-12: + H = H / H[2, 2] + + mean_err = 1e9 + if inlier_mask is not None: + inlier_bool = inlier_mask.ravel().astype(bool) + if np.any(inlier_bool): + projected = cv2.perspectiveTransform(pts1[inlier_bool], H) + err = np.linalg.norm( + projected.reshape(-1, 2) - pts0[inlier_bool].reshape(-1, 2), + axis=1, + ) + mean_err = float(err.mean()) if err.size else 1e9 + info["mean_err"] = mean_err + info["ok"] = True + return H, info + + +def get_feature_stitch_plan(isstitch, index, nrow, ncol, tile_list): + if not isstitch: + return None + + configured_indices = normalize_index(index) + total_tiles = len(configured_indices) + expected_tiles = nrow * ncol + if total_tiles <= 1 or total_tiles != expected_tiles or len(tile_list) < total_tiles: + return None + + ref_pos = (nrow // 2) * ncol + (ncol // 2) + if ref_pos >= total_tiles: + return None + + selected_positions = tuple(range(total_tiles)) + ref_idx = configured_indices[ref_pos] + moving_positions = tuple(pos for pos in selected_positions if pos != ref_pos) + moving_indices = tuple(configured_indices[pos] for pos in moving_positions) + moving_tiles = [tile_list[pos] for pos in moving_positions] + return ref_idx, tile_list[ref_pos], moving_indices, moving_tiles, selected_positions + + +def _geom_cache_key(t_ref_bgr, tiles_bgr_c, pad): + ref_h, ref_w = t_ref_bgr.shape[:2] + shapes = tuple(tb.shape[:2] for tb in tiles_bgr_c) + return ref_h, ref_w, shapes, pad + + +def ensure_feature_geom_cache( + canvas_cache, + t_ref_bgr, + tiles_bgr_c, + H_use_list, + pad, + *, + allocate_blend_bufs=True, +): + h0c, w0c = t_ref_bgr.shape[:2] + geom_key = _geom_cache_key(t_ref_bgr, tiles_bgr_c, pad) + geom_cache_hit = canvas_cache.get("geom_key") == geom_key + if geom_cache_hit: + if allocate_blend_bufs and ( + "blend_buf" not in canvas_cache + or "wsum_buf" not in canvas_cache + or "cov_buf" not in canvas_cache + ): + out_h = canvas_cache["out_h"] + out_w = canvas_cache["out_w"] + canvas_cache["blend_buf"] = np.zeros((out_h, out_w, 3), dtype=np.float32) + canvas_cache["wsum_buf"] = np.zeros((out_h, out_w), dtype=np.float32) + canvas_cache["cov_buf"] = np.zeros((out_h, out_w), dtype=np.uint8) + return True + + ref_corners = np.array([[0, 0], [w0c, 0], [w0c, h0c], [0, h0c]], dtype=np.float64) + pts = [ref_corners] + for tile_bgr, H_use in zip(tiles_bgr_c, H_use_list): + if H_use is None: + continue + hi, wi = tile_bgr.shape[:2] + tx, ty = translation_from_H_to_ref(H_use) + pts.append( + np.array( + [[tx, ty], [tx + wi, ty], [tx + wi, ty + hi], [tx, ty + hi]], + dtype=np.float64, + ) + ) + + all_xy = np.concatenate(pts, axis=0) + x_min_f = float(all_xy[:, 0].min()) - pad + y_min_f = float(all_xy[:, 1].min()) - pad + x_max_f = float(all_xy[:, 0].max()) + pad + y_max_f = float(all_xy[:, 1].max()) + pad + out_w = max(1, int(np.ceil(x_max_f - x_min_f))) + out_h = max(1, int(np.ceil(y_max_f - y_min_f))) + ref_y0 = int(np.floor(-y_min_f)) + ref_x0 = int(np.floor(-x_min_f)) + + pastes = [] + for H_use in H_use_list: + if H_use is None: + pastes.append(None) + continue + tx, ty = translation_from_H_to_ref(H_use) + pastes.append((int(np.floor(ty - y_min_f)), int(np.floor(tx - x_min_f)))) + + _dr_y0 = max(0, ref_y0) + _dr_x0 = max(0, ref_x0) + _dr_y1 = min(out_h, ref_y0 + h0c) + _dr_x1 = min(out_w, ref_x0 + w0c) + for _tb, _pa in zip(tiles_bgr_c, pastes): + if _pa is None: + continue + _py, _px = _pa + _th, _tw = _tb.shape[:2] + _dr_y0 = min(_dr_y0, max(0, _py)) + _dr_x0 = min(_dr_x0, max(0, _px)) + _dr_y1 = max(_dr_y1, min(out_h, _py + _th)) + _dr_x1 = max(_dr_x1, min(out_w, _px + _tw)) + + canvas_cache["geom_key"] = geom_key + canvas_cache["out_w"] = out_w + canvas_cache["out_h"] = out_h + canvas_cache["ref_y0"] = ref_y0 + canvas_cache["ref_x0"] = ref_x0 + canvas_cache["pastes"] = pastes + canvas_cache["x_min_f"] = x_min_f + canvas_cache["y_min_f"] = y_min_f + canvas_cache["crop_rect"] = None + canvas_cache["dirty_rect"] = (_dr_y0, _dr_y1, _dr_x0, _dr_x1) + canvas_cache["weight_maps"] = {} + canvas_cache["uint8_buf"] = np.zeros((out_h, out_w, 3), dtype=np.uint8) + if allocate_blend_bufs: + canvas_cache["blend_buf"] = np.zeros((out_h, out_w, 3), dtype=np.float32) + canvas_cache["wsum_buf"] = np.zeros((out_h, out_w), dtype=np.float32) + canvas_cache["cov_buf"] = np.zeros((out_h, out_w), dtype=np.uint8) + else: + canvas_cache["cov_buf"] = np.zeros((out_h, out_w), dtype=np.uint8) + canvas_cache.pop("blend_buf", None) + canvas_cache.pop("wsum_buf", None) + return False + + +def paste_feature_tiles_fast( + t_ref_bgr, + tiles_bgr_c, + H_use_list, + canvas_cache, + *, + geom_cache_hit, + frame_idx=0, + print_every=30, + verbose=False, + t_profile0=0.0, + t_after_brightness=0.0, + t_after_geom=0.0, +): + do_profile = verbose and frame_idx % print_every == 0 + out_w = canvas_cache["out_w"] + out_h = canvas_cache["out_h"] + ref_y0 = canvas_cache["ref_y0"] + ref_x0 = canvas_cache["ref_x0"] + pastes = canvas_cache["pastes"] + + canvas = canvas_cache.get("uint8_buf") + if canvas is None or canvas.shape[0] != out_h or canvas.shape[1] != out_w: + canvas = np.zeros((out_h, out_w, 3), dtype=np.uint8) + canvas_cache["uint8_buf"] = canvas + + dirty = canvas_cache.get("dirty_rect") + if geom_cache_hit and dirty is not None: + dy0, dy1, dx0, dx1 = dirty + canvas[dy0:dy1, dx0:dx1] = 0 + else: + canvas.fill(0) + t_after_reset = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 + + need_crop = canvas_cache.get("crop_rect") is None + cov = canvas_cache.get("cov_buf") if need_crop else None + if need_crop: + if cov is None or cov.shape != (out_h, out_w): + cov = np.zeros((out_h, out_w), dtype=np.uint8) + canvas_cache["cov_buf"] = cov + else: + if dirty is not None: + dy0, dy1, dx0, dx1 = dirty + cov[dy0:dy1, dx0:dx1] = 0 + else: + cov.fill(0) + + pasted_tiles = 0 + for tile_bgr, H_use, paste in zip(tiles_bgr_c, H_use_list, pastes): + if H_use is None or paste is None: + continue + y0, x0 = paste + paste_tile_clipped(canvas, tile_bgr, y0, x0, cov=cov) + pasted_tiles += 1 + paste_tile_clipped(canvas, t_ref_bgr, ref_y0, ref_x0, cov=cov) + t_after_paste = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 + + crop_rect = canvas_cache.get("crop_rect") + if crop_rect is None and cov is not None: + crop_rect = compute_final_crop_rect(cov) + canvas_cache["crop_rect"] = crop_rect + if crop_rect is not None: + y0, y1, x0, x1 = crop_rect + roi = canvas[y0:y1, x0:x1] + else: + roi = canvas + + rh, rw = roi.shape[:2] + stitch_buf = canvas_cache.get("stitch_buf") + if stitch_buf is None or stitch_buf.shape[0] != rh or stitch_buf.shape[1] != rw: + stitch_buf = np.empty((rh, rw, 3), dtype=np.uint8) + canvas_cache["stitch_buf"] = stitch_buf + np.copyto(stitch_buf, roi) + t_after_finalize = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 + + if do_profile: + xmf = canvas_cache.get("x_min_f", 0.0) + ymf = canvas_cache.get("y_min_f", 0.0) + print( + f"stitch_feature_optimized: canvas out_w={out_w}, out_h={out_h}, " + f"x_min={xmf:.2f}, y_min={ymf:.2f} (fast_paste)" + ) + print( + "stitch_feature_optimized steps: " + f"brightness={t_after_brightness - t_profile0:.6f}s, " + f"geom={t_after_geom - t_after_brightness:.6f}s ({'hit' if geom_cache_hit else 'miss'}), " + f"reset={t_after_reset - t_after_geom:.6f}s, " + f"paste={t_after_paste - t_after_reset:.6f}s ({pasted_tiles + 1} tiles incl ref), " + f"finalize={t_after_finalize - t_after_paste:.6f}s, " + f"total={t_after_finalize - t_profile0:.6f}s" + ) + canvas_cache["last_coverage"] = coverage_ratio(cov) if cov is not None else canvas_cache.get("last_coverage", 0.0) + return stitch_buf + + +def render_feature_tiles( + t_ref_bgr, + tiles_bgr, + H_use_list, + *, + blend_backend: BlendBackend, + pad=5, + frame_idx=0, + print_every=30, + verbose=False, + match_brightness=False, + canvas_cache=None, + blend=True, +): + if H_use_list is None: + return None + + if canvas_cache is None: + canvas_cache = {} + do_profile = verbose and frame_idx % print_every == 0 + t_profile0 = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 + + if t_ref_bgr.ndim == 2: + t_ref_bgr = cv2.cvtColor(t_ref_bgr, cv2.COLOR_GRAY2BGR) + h0c, w0c = t_ref_bgr.shape[:2] + if match_brightness: + ref_mean_gray = mean_gray_bgr(t_ref_bgr) + tiles_bgr_c = [ + match_bgr_brightness( + tb if tb.ndim == 3 else cv2.cvtColor(tb, cv2.COLOR_GRAY2BGR), + ref_mean_gray, + ) + for tb in tiles_bgr + ] + elif any(tb.ndim == 2 for tb in tiles_bgr): + tiles_bgr_c = [tb if tb.ndim == 3 else cv2.cvtColor(tb, cv2.COLOR_GRAY2BGR) for tb in tiles_bgr] + else: + tiles_bgr_c = tiles_bgr + t_after_brightness = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 + + geom_cache_hit = ensure_feature_geom_cache( + canvas_cache, + t_ref_bgr, + tiles_bgr_c, + H_use_list, + pad, + allocate_blend_bufs=bool(blend), + ) + t_after_geom = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 + + if not blend: + return paste_feature_tiles_fast( + t_ref_bgr, + tiles_bgr_c, + H_use_list, + canvas_cache, + geom_cache_hit=geom_cache_hit, + frame_idx=frame_idx, + print_every=print_every, + verbose=verbose, + t_profile0=t_profile0, + t_after_brightness=t_after_brightness, + t_after_geom=t_after_geom, + ) + + out_w = canvas_cache["out_w"] + out_h = canvas_cache["out_h"] + ref_y0 = canvas_cache["ref_y0"] + ref_x0 = canvas_cache["ref_x0"] + pastes = canvas_cache["pastes"] + + blend_buf = canvas_cache["blend_buf"] + wsum = canvas_cache["wsum_buf"] + cov = canvas_cache["cov_buf"] + weight_maps = canvas_cache["weight_maps"] + if geom_cache_hit: + _dr = canvas_cache.get("dirty_rect") + if _dr is not None: + _dy0, _dy1, _dx0, _dx1 = _dr + blend_buf[_dy0:_dy1, _dx0:_dx1] = 0 + wsum[_dy0:_dy1, _dx0:_dx1] = 0 + cov[_dy0:_dy1, _dx0:_dx1] = 0 + else: + blend_buf.fill(0) + wsum.fill(0) + cov.fill(0) + t_after_reset = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 + + blend_margin = 16 + blended_tiles = 0 + for tile_bgr, H_use, paste in zip(tiles_bgr_c, H_use_list, pastes): + if H_use is None or paste is None: + continue + y0, x0 = paste + th, tw = tile_bgr.shape[:2] + wkey = (th, tw, blend_margin) + if wkey not in weight_maps: + weight_maps[wkey] = tile_weight_map(th, tw, blend_margin) + blend_backend.blend_tile_inplace(blend_buf, wsum, cov, tile_bgr, weight_maps[wkey], y0, x0) + blended_tiles += 1 + + ref_wkey = (h0c, w0c, blend_margin) + if ref_wkey not in weight_maps: + weight_maps[ref_wkey] = tile_weight_map(h0c, w0c, blend_margin) + blend_backend.blend_tile_inplace( + blend_buf, wsum, cov, t_ref_bgr, weight_maps[ref_wkey], ref_y0, ref_x0 + ) + t_after_blend = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 + + crop_rect = canvas_cache.get("crop_rect") + if crop_rect is None: + crop_rect = compute_final_crop_rect(cov) + canvas_cache["crop_rect"] = crop_rect + if crop_rect is not None: + y0, y1, x0, x1 = crop_rect + roi = blend_buf[y0:y1, x0:x1] + else: + roi = blend_buf + rh, rw = roi.shape[:2] + stitch_buf = canvas_cache.get("stitch_buf") + if stitch_buf is None or stitch_buf.shape[0] != rh or stitch_buf.shape[1] != rw: + stitch_buf = np.empty((rh, rw, 3), dtype=np.uint8) + canvas_cache["stitch_buf"] = stitch_buf + cv2.convertScaleAbs(roi, dst=stitch_buf) + t_after_finalize = cv2.getTickCount() / cv2.getTickFrequency() if do_profile else 0.0 + + if do_profile: + xmf = canvas_cache.get("x_min_f", 0.0) + ymf = canvas_cache.get("y_min_f", 0.0) + print( + f"stitch_feature_optimized: canvas out_w={out_w}, out_h={out_h}, " + f"x_min={xmf:.2f}, y_min={ymf:.2f}" + ) + print( + "stitch_feature_optimized steps: " + f"brightness={t_after_brightness - t_profile0:.6f}s, " + f"geom={t_after_geom - t_after_brightness:.6f}s ({'hit' if geom_cache_hit else 'miss'}), " + f"reset={t_after_reset - t_after_geom:.6f}s, " + f"blend={t_after_blend - t_after_reset:.6f}s ({blended_tiles + 1} tiles incl ref), " + f"finalize={t_after_finalize - t_after_blend:.6f}s, " + f"total={t_after_finalize - t_profile0:.6f}s" + ) + + canvas_cache["last_coverage"] = coverage_ratio(cov) + canvas_cache["last_ms"] = ( + (t_after_finalize - t_profile0) * 1000.0 if do_profile else canvas_cache.get("last_ms", 0.0) + ) + return stitch_buf + + +def stitch_feature_optimized( + *, + isstitch: bool, + index, + nrow: int, + ncol: int, + tile_list: Sequence, + cache: Dict[str, Any], + blend_backend: BlendBackend, + blend: bool = True, + match_brightness: bool = False, + pad: int = 5, + print_every: int = 30, + verbose: bool = False, + frame_idx_holder: Optional[Dict[str, int]] = None, +): + """Estimate transforms once and composite the current tile grid.""" + plan = get_feature_stitch_plan(isstitch, index, nrow, ncol, tile_list) + if plan is None: + return None + + ref_idx, t_ref, moving_indices, moving_tiles, selected_positions = plan + if frame_idx_holder is None: + frame_idx_holder = {"idx": 0} + frame_idx_holder["idx"] = int(frame_idx_holder.get("idx", 0)) + 1 + frame_idx = frame_idx_holder["idx"] + tile_shapes = tuple(tile_list[pos].shape[:2] for pos in selected_positions) + cache_changed = ( + cache["ref_idx"] != ref_idx + or cache["moving_indices"] != moving_indices + or cache["tile_shapes"] != tile_shapes + ) + + if cache_changed: + cache["ref_idx"] = ref_idx + cache["moving_indices"] = moving_indices + cache["tile_shapes"] = tile_shapes + cache["transforms"] = None + cache["canvas"] = {} + if verbose: + print( + "Feature stitch tile selection: " + f"ref_idx={ref_idx}, moving_indices={list(moving_indices)}" + ) + + H_use_list = cache["transforms"] + if H_use_list is None: + do_verbose_first = verbose and frame_idx % print_every == 0 + H_use_list = [] + for j, tile in enumerate(moving_tiles): + H_use_j, _ = estimate_bfmatcher_stitch_transform( + t_ref, + tile, + verbose=(do_verbose_first and j == 0), + ) + H_use_list.append(H_use_j) + cache["transforms"] = H_use_list + + if not any(H is not None for H in H_use_list): + return None + + return render_feature_tiles( + t_ref, + moving_tiles, + H_use_list, + blend_backend=blend_backend, + pad=pad, + frame_idx=frame_idx, + print_every=print_every, + verbose=verbose, + match_brightness=match_brightness, + canvas_cache=cache["canvas"], + blend=blend, + ) diff --git a/src/orisys/stitch/ops.py b/src/orisys/stitch/ops.py new file mode 100644 index 0000000..bc23a1f --- /dev/null +++ b/src/orisys/stitch/ops.py @@ -0,0 +1,136 @@ +"""Pure array compute helpers for stitching (backend-agnostic). + +These functions take/return NumPy arrays and must not depend on ``Stitcher`` +instance state so future Torch/CuPy backends can reuse the same call shapes. +""" + +from __future__ import annotations + +from typing import Optional, Tuple + +import cv2 +import numpy as np + + +def tile_weight_map(h: int, w: int, margin: int = 16) -> np.ndarray: + """Linear ramp weight: 0 at edges, 1.0 at ``margin`` pixels inward.""" + wy = np.ones(h, dtype=np.float32) + wx = np.ones(w, dtype=np.float32) + m = min(margin, h // 2, w // 2) + if m > 0: + ramp = np.linspace(0.0, 1.0, m + 1, dtype=np.float32)[1:] + wy[:m] = ramp + wy[-m:] = ramp[::-1] + wx[:m] = ramp + wx[-m:] = ramp[::-1] + return wy[:, None] * wx[None, :] + + +def paste_tile_clipped(canvas, tile, y0: int, x0: int, cov=None) -> None: + """Paste tile into canvas with clipping; optionally mark coverage.""" + th, tw = tile.shape[:2] + H, W = canvas.shape[:2] + y0d, x0d = max(0, y0), max(0, x0) + y1d, x1d = min(H, y0 + th), min(W, x0 + tw) + if y0d >= y1d or x0d >= x1d: + return + sy0, sx0 = y0d - y0, x0d - x0 + sh, sw = y1d - y0d, x1d - x0d + np.copyto(canvas[y0d:y1d, x0d:x1d], tile[sy0 : sy0 + sh, sx0 : sx0 + sw]) + if cov is not None: + cov[y0d:y1d, x0d:x1d] = 1 + + +def copy_into_tile_buffer(src_view, dst, flip_x: bool = True, flip_y: bool = True) -> None: + """Flip/copy ``src_view`` into preallocated ``dst``.""" + if flip_x and flip_y: + cv2.flip(src_view, -1, dst) + elif flip_x: + cv2.flip(src_view, 1, dst) + elif flip_y: + cv2.flip(src_view, 0, dst) + else: + np.copyto(dst, src_view) + + +def build_blend_coef(ov_pos: int, dim: int, ndim: int) -> np.ndarray: + """1D/2D blend ramp for legacy overlap stitching.""" + blend_coef = np.linspace(1.0, 0.0, ov_pos, dtype=np.float32) + if dim == 1: + return blend_coef.reshape(1, ov_pos, 1) if ndim == 3 else blend_coef.reshape(1, ov_pos) + return blend_coef.reshape(ov_pos, 1, 1) if ndim == 3 else blend_coef.reshape(ov_pos, 1) + + +def mean_gray_bgr(bgr) -> float: + ch = cv2.mean(bgr) + return (ch[0] + ch[1] + ch[2]) / 3.0 + + +def match_bgr_brightness(tile_bgr, ref_mean_gray: float): + delta = int(round(ref_mean_gray - mean_gray_bgr(tile_bgr))) + if delta == 0: + return tile_bgr + lut = np.clip(np.arange(256, dtype=np.int16) + delta, 0, 255).astype(np.uint8) + return cv2.LUT(tile_bgr, lut) + + +def largest_fully_covered_rect(cov) -> Optional[Tuple[int, int, int, int]]: + if cov.size == 0: + return None + + mask = cov > 0 + if not np.any(mask): + return None + + h, w = mask.shape + heights = np.zeros(w, dtype=np.int32) + best_area = 0 + best_rect = None + for y in range(h): + heights = np.where(mask[y], heights + 1, 0) + stack = [] + for x in range(w + 1): + cur_h = heights[x] if x < w else 0 + start = x + while stack and stack[-1][1] > cur_h: + start0, h0 = stack.pop() + area = h0 * (x - start0) + if area > best_area: + best_area = area + best_rect = (y - h0 + 1, y + 1, start0, x) + start = start0 + if not stack or stack[-1][1] < cur_h: + stack.append((start, cur_h)) + return best_rect + + +def compute_final_crop_rect(cov) -> Optional[Tuple[int, int, int, int]]: + valid = cov > 0 + if not np.any(valid): + return None + + ys, xs = np.where(valid) + by0, by1 = int(ys.min()), int(ys.max()) + 1 + bx0, bx1 = int(xs.min()), int(xs.max()) + 1 + rect2 = largest_fully_covered_rect(cov[by0:by1, bx0:bx1]) + if rect2 is None: + return by0, by1, bx0, bx1 + + y0, y1, x0, x1 = rect2 + return by0 + y0, by0 + y1, bx0 + x0, bx0 + x1 + + +def coverage_ratio(cov) -> float: + if cov is None or cov.size == 0: + return 0.0 + return float(np.mean(cov > 0)) + + +def translation_from_H_to_ref(H) -> Tuple[float, float]: + if H is None: + return 0.0, 0.0 + H = np.asarray(H, dtype=np.float64) + if abs(H[2, 2]) > 1e-12: + H = H / H[2, 2] + pt = H @ np.array([[0.0], [0.0], [1.0]], dtype=np.float64) + return float(pt[0, 0]), float(pt[1, 0]) diff --git a/src/orisys/stitch/tiles.py b/src/orisys/stitch/tiles.py new file mode 100644 index 0000000..60c32be --- /dev/null +++ b/src/orisys/stitch/tiles.py @@ -0,0 +1,86 @@ +"""Tile extraction and indexing for compound-eye frames.""" + +from __future__ import annotations + +from typing import Iterable, List, Optional, Sequence, Tuple + +import numpy as np + +from .ops import copy_into_tile_buffer + + +def extract_units_from_boxes( + img, + boxes: Sequence[Tuple[int, int, int, int]], + *, + selected_index: Iterable[int], + units: Optional[list] = None, + flip_x: bool = True, + flip_y: bool = True, +) -> list: + """Crop/flip tiles from ``boxes`` into ``units`` buffers (reused when possible).""" + h, w = img.shape[:2] + selected = set(selected_index) + flipped_units = [] if units is None else units + + for i, (x0, y0, x1, y1) in enumerate(boxes): + if i not in selected: + continue + + x0 = max(0, int(x0)) + y0 = max(0, int(y0)) + x1 = min(w, int(x1)) + y1 = min(h, int(y1)) + if x1 <= x0 or y1 <= y0: + continue + + src_view = img[y0:y1, x0:x1] + if units is None: + tile = np.empty_like(src_view) + copy_into_tile_buffer(src_view, tile, flip_x=flip_x, flip_y=flip_y) + flipped_units.append((x0, y0, tile)) + else: + _, _, tile = flipped_units[i] + if tile.shape != src_view.shape or tile.dtype != src_view.dtype: + tile = np.empty_like(src_view) + copy_into_tile_buffer(src_view, tile, flip_x=flip_x, flip_y=flip_y) + flipped_units[i] = (x0, y0, tile) + + return flipped_units + + +def get_tile_by_index(units: Sequence, idx) -> List: + if isinstance(idx, int): + idx = [idx] + if len(idx) > len(units): + raise ValueError("Index exceeds number of tiles.") + tile_list = [] + for i in range(len(idx)): + _xi, _yi, tile = units[idx[i]] + tile_list.append(tile) + return tile_list + + +def get_tile_index_by_position(units: Sequence, x, y, tol: int = 20) -> Optional[int]: + i = 0 + for xi, yi, _tile in units: + if abs(int(x) - int(xi)) <= tol and abs(int(y) - int(yi)) <= tol: + return i + i += 1 + return None + + +def get_unit_pos(units: Sequence, idx) -> List[Tuple[int, int]]: + if isinstance(idx, int): + idx = [idx] + if len(idx) > len(units): + raise ValueError("Index exceeds number of vision units.") + return [(units[i][0], units[i][1]) for i in idx] + + +def normalize_index(index) -> Tuple[int, ...]: + if isinstance(index, int): + return (index,) + if isinstance(index, np.ndarray): + return tuple(int(i) for i in index.tolist()) + return tuple(int(i) for i in index) diff --git a/src/orisys/unroll/__init__.py b/src/orisys/unroll/__init__.py index 22e82f5..4dd2940 100644 --- a/src/orisys/unroll/__init__.py +++ b/src/orisys/unroll/__init__.py @@ -1,7 +1,7 @@ """UV-based finger mesh unroll / flatten pipeline. Offline: Blender export -> build_or_load_domain -> unroll_map_uv.npz -Runtime: live camera frame -> UVUnrollFrameProcessor -> Sensor (with valid_mask) +Runtime: live camera frame -> FrameUnwarper -> Sensor (with valid_mask) 3D display: UVDomain.surface.world_xyz """ diff --git a/src/orisys_blend/__init__.py b/src/orisys_blend/__init__.py new file mode 100644 index 0000000..3b05a79 --- /dev/null +++ b/src/orisys_blend/__init__.py @@ -0,0 +1,10 @@ +"""Pure-Python Numba stitch kernels kept outside the Nuitka-compiled orisys .pyd. + +Nuitka cannot reliably JIT ``@njit`` functions baked into a compiled extension. +This package is shipped as source next to the Nuitka wheel so Numba can see +real CPython bytecode at runtime. +""" + +from .kernel import blend_kernel + +__all__ = ["blend_kernel"] diff --git a/src/orisys_blend/kernel.py b/src/orisys_blend/kernel.py new file mode 100644 index 0000000..8c2ef0d --- /dev/null +++ b/src/orisys_blend/kernel.py @@ -0,0 +1,51 @@ +"""Fused stitch blend kernel (Numba JIT when available).""" + +from __future__ import annotations + +import numpy as np + +try: + from numba import njit + + _NUMBA_AVAILABLE = True +except ImportError: + _NUMBA_AVAILABLE = False + + def njit(*args, **kwargs): + def decorator(fn): + return fn + + return decorator + + +if _NUMBA_AVAILABLE: + + @njit(cache=True) + def blend_kernel(blend_patch, wsum_patch, cov_patch, src_patch, w_patch): + """Fused incremental weighted-average composite (numba-accelerated).""" + sh = src_patch.shape[0] + sw = src_patch.shape[1] + for y in range(sh): + for x in range(sw): + w = w_patch[y, x] + s0 = np.float32(src_patch[y, x, 0]) + s1 = np.float32(src_patch[y, x, 1]) + s2 = np.float32(src_patch[y, x, 2]) + if cov_patch[y, x] == 0: + blend_patch[y, x, 0] = s0 + blend_patch[y, x, 1] = s1 + blend_patch[y, x, 2] = s2 + wsum_patch[y, x] = w + else: + ws = wsum_patch[y, x] + w + if ws < 1e-6: + ws = 1e-6 + a = w / ws + blend_patch[y, x, 0] += (s0 - blend_patch[y, x, 0]) * a + blend_patch[y, x, 1] += (s1 - blend_patch[y, x, 1]) * a + blend_patch[y, x, 2] += (s2 - blend_patch[y, x, 2]) * a + wsum_patch[y, x] = ws + cov_patch[y, x] = 1 + +else: + blend_kernel = None diff --git a/tests/DIS_no_integrate.py b/tests/DIS_no_integrate.py new file mode 100644 index 0000000..5e21dcf --- /dev/null +++ b/tests/DIS_no_integrate.py @@ -0,0 +1,420 @@ +#!/usr/bin/env python3 +"""Compare cumulative adaptive DIS vs fixed initial-reference DIS (no integration). + +Reproduces drift on static scenes where adaptive flow integrates incremental +displacement: final_flow = queue[-1] + prev_fine_flow (see orisys.tracker). + +Usage: + python tests/DIS_no_integrate.py --source /path/to/static_clip.mp4 --frames 500 + python tests/DIS_no_integrate.py --source 0 --frames 300 --plot drift.png + python tests/DIS_no_integrate.py --config box --source 0 --no-show + +Modes compared on identical grayscale frames: + cumulative Tracker(adaptive=True) — production default + non_adaptive Tracker(adaptive=False) — single ref, warm-started DIS + fixed_ref DIS calc(ref0, frame, None) — initial frame only, no queue +""" + +from __future__ import annotations + +import argparse +import os +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +import cv2 +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[1] +SRC_ROOT = REPO_ROOT / "src" +for path in (REPO_ROOT, SRC_ROOT): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) + +from orisys.tracker import Tracker # noqa: E402 + + +DEFAULT_CONFIG = "box" + + +def parse_source(value: str): + text = str(value).strip() + for prefix in ("camera:", "cam:", "webcam:"): + if text.lower().startswith(prefix): + text = text[len(prefix) :].strip() + break + try: + return int(text) + except ValueError: + return text + + +def load_tracker_config(config_ref: str) -> dict: + """Load tracker section from an Orisys config stem or JSON path.""" + path = Path(config_ref) + if path.suffix.lower() == ".json" and path.is_file(): + import json + + data = json.loads(path.read_text(encoding="utf-8")) + else: + import orisys + + data = orisys.configs.load_config(config_ref) + tracker = dict(data.get("tracker", {})) + tracker.setdefault("algorithm", "dis") + tracker.setdefault("dis_preset", "medium") + tracker.setdefault("dis_mode", 3) + tracker.setdefault("lower_threshold", 12.0) + tracker.setdefault("upper_threshold", 18.0) + tracker.setdefault("loop_closure_interval", 1) + return tracker + + +def dis_preset_from_name(name: str) -> int: + preset = (name or "medium").lower() + if preset == "ultrafast": + return cv2.DISOPTICAL_FLOW_PRESET_ULTRAFAST + if preset == "fast": + return cv2.DISOPTICAL_FLOW_PRESET_FAST + return cv2.DISOPTICAL_FLOW_PRESET_MEDIUM + + +def flow_magnitude_stats(flow: np.ndarray) -> tuple[float, float]: + """Return (mean magnitude, max magnitude) for HxWx2 flow.""" + if flow is None: + return float("nan"), float("nan") + mag = np.linalg.norm(flow.astype(np.float32), axis=-1) + return float(mag.mean()), float(mag.max()) + + +@dataclass +class FrameMetrics: + mean_mag: float + max_mag: float + queue_len: int = 0 + + +@dataclass +class ModeSeries: + name: str + mean_mags: list[float] = field(default_factory=list) + max_mags: list[float] = field(default_factory=list) + queue_lens: list[int] = field(default_factory=list) + + def append(self, metrics: FrameMetrics) -> None: + self.mean_mags.append(metrics.mean_mag) + self.max_mags.append(metrics.max_mag) + self.queue_lens.append(metrics.queue_len) + + def slope(self) -> float: + if len(self.mean_mags) < 2: + return float("nan") + x = np.arange(len(self.mean_mags), dtype=np.float64) + y = np.asarray(self.mean_mags, dtype=np.float64) + if not np.all(np.isfinite(y)): + y = np.nan_to_num(y, nan=0.0) + return float(np.polyfit(x, y, 1)[0]) + + +class FixedRefDIS: + """DIS from a fixed initial reference frame; no integration or queue.""" + + def __init__(self, config_tracker: dict): + preset = dis_preset_from_name(config_tracker.get("dis_preset", "medium")) + self._dis = cv2.DISOpticalFlow_create(preset) + self._dis.setFinestScale(int(config_tracker.get("dis_mode", 3))) + self._ref: Optional[np.ndarray] = None + + def reset(self) -> None: + self._ref = None + + def track(self, gray: np.ndarray) -> np.ndarray: + if self._ref is None: + self._ref = gray.copy() + h, w = gray.shape[:2] + return np.zeros((h, w, 2), dtype=np.float32) + return self._dis.calc(self._ref, gray, None) + + +def make_tracker(config_tracker: dict, *, adaptive: bool) -> Tracker: + cfg = dict(config_tracker) + return Tracker( + adaptive=adaptive, + cuda=False, + verbose=False, + config_tracker=cfg, + ) + + +def read_gray_frame(cap: cv2.VideoCapture, static_image: Optional[np.ndarray]) -> tuple[bool, Optional[np.ndarray]]: + if static_image is not None: + return True, static_image.copy() + ok, frame = cap.read() + if not ok or frame is None: + return False, None + if frame.ndim == 3: + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + return True, frame + + +def run_comparison( + source, + *, + config_tracker: dict, + max_frames: int, + reset_every: Optional[int], + show: bool, +) -> dict[str, ModeSeries]: + path = Path(str(source)) if isinstance(source, str) else None + static_image: Optional[np.ndarray] = None + if isinstance(source, str) and path is not None and path.is_file(): + img = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE) + if img is not None: + static_image = img + + cap = None if static_image is not None else cv2.VideoCapture(source) + if cap is not None and not cap.isOpened(): + raise RuntimeError(f"Cannot open source: {source}") + + cumulative = make_tracker(config_tracker, adaptive=True) + non_adaptive = make_tracker(config_tracker, adaptive=False) + fixed_ref = FixedRefDIS(config_tracker) + + series = { + "cumulative": ModeSeries("cumulative"), + "non_adaptive": ModeSeries("non_adaptive"), + "fixed_ref": ModeSeries("fixed_ref"), + } + + window = "DIS compare (cumulative | non_adaptive | fixed_ref)" + if show: + cv2.namedWindow(window, cv2.WINDOW_NORMAL) + + try: + for frame_idx in range(max_frames): + ok, gray = read_gray_frame(cap, static_image) + if not ok or gray is None: + break + + if reset_every and reset_every > 0 and frame_idx > 0 and frame_idx % reset_every == 0: + cumulative.reset() + + flow_c = cumulative.track(gray.copy()) + flow_n = non_adaptive.track(gray.copy()) + flow_f = fixed_ref.track(gray) + + mean_c, max_c = flow_magnitude_stats(flow_c) + mean_n, max_n = flow_magnitude_stats(flow_n) + mean_f, max_f = flow_magnitude_stats(flow_f) + + series["cumulative"].append( + FrameMetrics(mean_c, max_c, queue_len=cumulative.getFrameQueueLength()) + ) + series["non_adaptive"].append(FrameMetrics(mean_n, max_n, queue_len=1)) + series["fixed_ref"].append(FrameMetrics(mean_f, max_f, queue_len=1)) + + if show and frame_idx % 2 == 0: + h, w = gray.shape[:2] + panels = [] + for flow, label, mean_val in ( + (flow_c, "cumulative", mean_c), + (flow_n, "non_adaptive", mean_n), + (flow_f, "fixed_ref", mean_f), + ): + vis = cv2.cvtColor(gray, cv2.COLOR_GRAY2BGR) + step = max(8, min(h, w) // 40) + ys, xs = np.mgrid[step // 2 : h : step, step // 2 : w : step] + for y, x in zip(ys.ravel(), xs.ravel()): + dx, dy = flow[y, x] + x2 = int(round(x + dx * 2)) + y2 = int(round(y + dy * 2)) + cv2.arrowedLine(vis, (x, y), (x2, y2), (0, 255, 0), 1, tipLength=0.3) + cv2.putText( + vis, + f"{label} m={mean_val:.3f}", + (8, 20), + cv2.FONT_HERSHEY_SIMPLEX, + 0.5, + (0, 255, 255), + 1, + cv2.LINE_AA, + ) + panels.append(vis) + combined = np.hstack(panels) + cv2.imshow(window, combined) + key = cv2.waitKey(1) & 0xFF + if key == ord("q"): + break + finally: + if cap is not None: + cap.release() + if show: + cv2.destroyAllWindows() + + return series + + +def print_summary(series: dict[str, ModeSeries], frames_processed: int) -> None: + print(f"\nProcessed {frames_processed} frame(s)\n") + header = f"{'mode':<14} {'initial':>10} {'final':>10} {'mean':>10} {'slope':>12} {'max_queue':>10}" + print(header) + print("-" * len(header)) + + for key in ("cumulative", "non_adaptive", "fixed_ref"): + s = series[key] + if not s.mean_mags: + print(f"{key:<14} {'(no data)':>10}") + continue + initial = s.mean_mags[0] + final = s.mean_mags[-1] + mean_all = float(np.mean(s.mean_mags)) + slope = s.slope() + max_q = max(s.queue_lens) if s.queue_lens else 0 + print( + f"{key:<14} {initial:10.4f} {final:10.4f} {mean_all:10.4f} {slope:12.6f} {max_q:10d}" + ) + + print("\nInterpretation on a static scene:") + print(" cumulative — rising mean/slope indicates integration drift") + print(" fixed_ref — should stay near a small bias if drift is removed") + + +def save_plot(series: dict[str, ModeSeries], plot_path: str) -> None: + import matplotlib.pyplot as plt + + fig, ax = plt.subplots(figsize=(10, 5)) + for key, color in ( + ("cumulative", "C0"), + ("non_adaptive", "C1"), + ("fixed_ref", "C2"), + ): + s = series[key] + if not s.mean_mags: + continue + ax.plot(s.mean_mags, label=key, color=color) + ax.set_xlabel("frame") + ax.set_ylabel("mean flow magnitude") + ax.set_title("DIS drift comparison (static scene expected: flat fixed_ref)") + ax.legend() + ax.grid(True, alpha=0.3) + fig.tight_layout() + path = Path(plot_path) + path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(path, dpi=120) + plt.close(fig) + print(f"Saved plot: {path.resolve()}") + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Compare cumulative vs fixed-reference DIS optical flow drift" + ) + parser.add_argument( + "--source", + default="0", + help="Camera index, video path, or image path (default: 0)", + ) + parser.add_argument( + "--config", + "-c", + default=DEFAULT_CONFIG, + help=f"Orisys config stem or JSON path for DIS settings (default: {DEFAULT_CONFIG})", + ) + parser.add_argument( + "--frames", + type=int, + default=300, + help="Maximum frames to process (default: 300)", + ) + parser.add_argument( + "--reset-every", + type=int, + default=0, + help="Reset cumulative tracker every N frames (0 = disabled)", + ) + parser.add_argument( + "--plot", + nargs="?", + const="dis_drift.png", + default=None, + help="Save matplotlib plot (optional path, default: dis_drift.png)", + ) + parser.add_argument( + "--no-show", + action="store_true", + help="Skip OpenCV preview window", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + source = parse_source(args.source) + config_tracker = load_tracker_config(args.config) + + if config_tracker.get("algorithm", "dis") != "dis": + print(f"Warning: config algorithm is {config_tracker.get('algorithm')}; forcing dis for test.") + + print("DIS no-integrate comparison") + print(f" source: {source}") + print(f" config: {args.config}") + print(f" dis_preset: {config_tracker.get('dis_preset')} dis_mode: {config_tracker.get('dis_mode')}") + print(f" frames: {args.frames} reset_every: {args.reset_every or 'off'}") + + series = run_comparison( + source, + config_tracker=config_tracker, + max_frames=max(args.frames, 1), + reset_every=args.reset_every or None, + show=not args.no_show, + ) + + n = len(series["cumulative"].mean_mags) + print_summary(series, n) + + if args.plot: + save_plot(series, args.plot) + + return 0 + + +def test_dis_no_integrate_smoke() -> None: + """Optional pytest entry: synthetic static frames, no camera required.""" + if os.getenv("ORISYS_RUN_DIS_DRIFT") != "1": + import pytest + + pytest.skip("Set ORISYS_RUN_DIS_DRIFT=1 to run DIS drift smoke test") + + gray = np.full((64, 64), 128, dtype=np.uint8) + # Slight per-frame noise to mimic sensor variation without true motion. + cap_frames = [] + rng = np.random.default_rng(0) + for _ in range(60): + noisy = gray.astype(np.int16) + rng.integers(-2, 3, gray.shape, dtype=np.int16) + cap_frames.append(np.clip(noisy, 0, 255).astype(np.uint8)) + + config_tracker = load_tracker_config(DEFAULT_CONFIG) + cumulative = make_tracker(config_tracker, adaptive=True) + fixed_ref = FixedRefDIS(config_tracker) + + cum_mags = [] + fix_mags = [] + for frame in cap_frames: + fc = cumulative.track(frame.copy()) + ff = fixed_ref.track(frame) + cum_mags.append(flow_magnitude_stats(fc)[0]) + fix_mags.append(flow_magnitude_stats(ff)[0]) + + assert len(cum_mags) >= 10 + # Fixed ref should not drift more than cumulative on identical static+noise input. + cum_slope = float(np.polyfit(np.arange(len(cum_mags)), cum_mags, 1)[0]) + fix_slope = float(np.polyfit(np.arange(len(fix_mags)), fix_mags, 1)[0]) + assert abs(fix_slope) <= abs(cum_slope) + 0.05, ( + f"fixed_ref slope {fix_slope:.4f} worse than cumulative {cum_slope:.4f}" + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index c6da59a..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,116 +0,0 @@ -import os -import sys -from pathlib import Path - -import pytest - -REPO_ROOT = Path(__file__).resolve().parents[1] -SRC_ROOT = REPO_ROOT / "src" -if str(SRC_ROOT) not in sys.path: - sys.path.insert(0, str(SRC_ROOT)) - -from orisys import Sensor -from orisys import configs as orisys_configs -from orisys.backend_availability import check_cupy_available, check_opencl_available - - -def _resolve_test_video() -> Path: - env_video = os.getenv("ORISYS_TEST_VIDEO") - if env_video: - raw = Path(env_video) - candidates = [ - raw, # as provided (absolute or cwd-relative) - REPO_ROOT / raw, # repo-relative - ] - # Common typo fallback: "test/..." -> "tests/..." - if env_video.startswith("test/") or env_video.startswith("test\\"): - fixed = Path("tests", *Path(env_video).parts[1:]) - candidates.append(REPO_ROOT / fixed) - - for candidate in candidates: - if candidate.exists(): - return candidate - return raw - - # Prefer tests/ fixture clip; fallback kept by caller exists check. - return REPO_ROOT / "tests" / "idle_1min.mp4" - - -@pytest.fixture(scope="session") -def test_video_path() -> Path: - path = _resolve_test_video() - if not path.exists(): - pytest.skip( - "Test video not found. Set ORISYS_TEST_VIDEO to a valid video path." - ) - return path - - -@pytest.fixture(scope="session") -def test_config_name() -> str: - requested = os.getenv("ORISYS_TEST_CONFIG") - - def _is_usable_config(name: str) -> bool: - cfg = orisys_configs.load_config(name) - return "box_size" in cfg.get("stitcher", {}) - - if requested: - if not _is_usable_config(requested): - raise RuntimeError( - f"ORISYS_TEST_CONFIG='{requested}' is missing required key " - "'stitcher.box_size'. Use a config like 'ddjx01' or provide a full path." - ) - return requested - - # Auto-pick a stable config for tests (avoid prototype variants missing keys). - for name in orisys_configs.list_config(): - try: - if _is_usable_config(name): - return name - except Exception: - continue - - raise RuntimeError( - "No usable config found for tests (missing 'stitcher.box_size'). " - "Set ORISYS_TEST_CONFIG to a valid config JSON/path." - ) - - -@pytest.fixture(scope="session") -def available_backends() -> list[str]: - backends = ["cpu"] - if check_cupy_available(): - backends.append("cupy") - if check_opencl_available(): - backends.append("opencl") - return backends - - -def make_sensor( - video_path: Path, - config_name: str = "prototype", - backend: str = "cpu", - verbose: bool = False, -) -> Sensor: - use_cuda = backend in ("cupy", "opencl") - return Sensor( - vid_src=str(video_path), - cuda=use_cuda, - isstitch=True, - verbose=verbose, - config_name=config_name, - backend=backend, - ) - - -@pytest.fixture -def sensor_factory(test_video_path, test_config_name): - def _factory(backend: str = "cpu", verbose: bool = False): - return make_sensor( - test_video_path, - config_name=test_config_name, - backend=backend, - verbose=verbose, - ) - - return _factory diff --git a/tests/dd06.py b/tests/dd06.py new file mode 100644 index 0000000..3f9462e --- /dev/null +++ b/tests/dd06.py @@ -0,0 +1,175 @@ +""" +DD06 single-camera pipeline with polygon ROI rectification. + +Flow: +1. Load sensor config + nested polygon ROI +2. Capture raw frames from the video/camera source +3. Perspective-warp the ROI into a rectangle sized to camera.resolution_stitch +4. Feed the rectified frame into Sensor.get_img(..., already_processed=True) +5. Compute deformation/contact and visualize + +SDK version: 0.5.0 +""" +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" + +import cv2 +import numpy as np +import orisys + +_REPO_ROOT = Path(__file__).resolve().parents[1] +_EXAMPLES_DIR = _REPO_ROOT / "examples" +if str(_EXAMPLES_DIR) not in sys.path: + sys.path.insert(0, str(_EXAMPLES_DIR)) + +from gui.sensors.roi_geometry import ( # noqa: E402 + build_roi_warp, + load_json_config, + load_roi_polygon, + open_video_source, + rectified_output_size, + warp_roi_to_rectangle, +) + + +def main(): + parser = argparse.ArgumentParser( + description="DD06 pipeline: polygon-ROI rectify -> Sensor.get_img(already_processed=True)" + ) + parser.add_argument( + "--video", + "-v", + type=str, + default="./data/dd06/0726.mp4", + help="视频源：摄像头编号或视频文件路径", + ) + parser.add_argument( + "--config", + "-c", + type=str, + default="./config/dd06-ov.json", + help="配置文件名称或路径", + ) + parser.add_argument( + "--verbose", + "-verbose", + action="store_true", + default=True, + help="是否输出详细日志", + ) + args = parser.parse_args() + + config = load_json_config(args.config) + roi_points, roi_image_size = load_roi_polygon(config) + out_w, out_h = rectified_output_size(config, roi_points) + + sensor = orisys.Sensor("external", config_name=args.config, verbose=args.verbose) + cap, source = open_video_source(args.video) + + warp_matrix = None + warp_src = None + last_frame_size = None + + print( + f"ROI points={len(roi_points)} source_size={roi_image_size} " + f"output={out_w}x{out_h} video={source}" + ) + print("按键说明：'q' 退出程序，'r' 重置追踪器。") + + try: + while True: + ok, frame = cap.read() + if not ok or frame is None: + print("Failed to read frame from video source.") + break + + frame_h, frame_w = frame.shape[:2] + frame_size = (frame_w, frame_h) + if warp_matrix is None or last_frame_size != frame_size: + warp_matrix, warp_src, _ = build_roi_warp( + roi_points, + roi_image_size, + frame_size, + (out_w, out_h), + ) + last_frame_size = frame_size + if args.verbose: + print(f"Built ROI warp for frame {frame_w}x{frame_h}:\n{warp_src}") + + rectified = warp_roi_to_rectangle(frame, warp_matrix, (out_w, out_h)) + sensor.get_img(frame=rectified, already_processed=True) + + is_contact_now = sensor.compute_deformation(check_motion=True, threshold=0) + _ = is_contact_now + sensor.compute_contact() + + fps, flow, vnormal, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = ( + sensor.read_info( + sensor.info.FPS, + sensor.info.VRAW, + sensor.info.VNORMAL, + sensor.info.IMG, + sensor.info.CONTOUR, + sensor.info.CENTROID, + sensor.info.DEPTH, + sensor.info.FNORMAL, + sensor.info.FSHEARX, + sensor.info.FSHEARY, + ) + ) + + preview = frame.copy() + cv2.polylines( + preview, + [np.rint(warp_src).astype(np.int32).reshape(-1, 1, 2)], + True, + (0, 255, 255), + 2, + cv2.LINE_AA, + ) + + arrows = orisys.util.draw_arrows( + img, + flow, + threshold=1, + grid_spacing=20, + arrow_scale=1.0, + ) + cv2.imshow("形变矢量场", arrows) + + div_abs = depth_map + if div_abs.max() > div_abs.min(): + div_normalized = ( + (div_abs - div_abs.min()) / (div_abs.max() - div_abs.min()) * 255 + ).astype(np.uint8) + else: + div_normalized = np.zeros_like(div_abs, dtype=np.uint8) + cv2.imshow("depth", cv2.applyColorMap(div_normalized, cv2.COLORMAP_JET)) + + print( + f"FPS={fps:.2f}, 法向力={fnormal:.4f}, 切向力X={fshearx:.4f}, " + f"切向力Y={fsheary:.4f}, 深度图尺寸={depth_map.shape}, " + f"原始分辨率={sensor.img_size_raw}, 矫正={rectified.shape[1]}x{rectified.shape[0]}" + ) + + key = cv2.waitKey(1) & 0xFF + if key == ord("q"): + print("\n正在退出示例程序...") + break + if key == ord("r"): + sensor.reset() + print("追踪器已重置。") + finally: + cap.release() + sensor.disconnect() + cv2.destroyAllWindows() + + +if __name__ == "__main__": + main() diff --git a/tests/test_capture.py b/tests/test_capture.py new file mode 100644 index 0000000..179c8ec --- /dev/null +++ b/tests/test_capture.py @@ -0,0 +1,436 @@ +"""Unit tests for orisys.capture with fake VideoCapture objects (no hardware).""" + +from __future__ import annotations + +import threading +import time + +import cv2 +import numpy as np +import pytest + +from orisys.capture import ( + MAX_SIMULTANEOUS_CAPTURES, + CameraFilter, + CameraInfo, + CaptureConfig, + CaptureHealth, + CaptureSource, + ReconnectPolicy, + list_cameras, + normalize_source, + open_capture, + pick_default_camera, +) +from orisys.capture.source import _apply_stream_props, _warmup_capture +from orisys.capture.discovery import DEFAULT_LAPTOP_WEBCAM_PATTERNS + + +MJPG = cv2.VideoWriter_fourcc(*"MJPG") +YUY2 = cv2.VideoWriter_fourcc(*"YUY2") + + +class FakeCap: + """Deterministic stand-in for cv2.VideoCapture.""" + + def __init__( + self, + frames=None, + *, + fail_after=None, + fail_reads=0, + open_ok=True, + reset_fourcc_on_resize=False, + ): + self._frames = list(frames or []) + self._idx = 0 + self._open = bool(open_ok) + self._fail_after = fail_after + self._fail_reads_remaining = int(fail_reads) + self.read_count = 0 + self.grab_count = 0 + self.release_count = 0 + self.width = 4 + self.height = 3 + self.fps = 30.0 + self.fourcc = 0 + self.reset_fourcc_on_resize = reset_fourcc_on_resize + self.set_calls: list[tuple[int, float]] = [] + + def isOpened(self): + return self._open + + def read(self): + self.read_count += 1 + if not self._open: + return False, None + if self._fail_reads_remaining > 0: + self._fail_reads_remaining -= 1 + return False, None + if self._fail_after is not None and self.read_count > self._fail_after: + self._open = False + return False, None + if self._idx >= len(self._frames): + if not self._frames: + return False, None + frame = self._frames[-1] + else: + frame = self._frames[self._idx] + self._idx += 1 + return True, frame.copy() + + def grab(self): + self.grab_count += 1 + if not self._open: + return False + ok, _ = self.read() + return ok + + def release(self): + self.release_count += 1 + self._open = False + + def get(self, prop_id): + if prop_id in (3,): # CAP_PROP_FRAME_WIDTH + return float(self.width) + if prop_id in (4,): # CAP_PROP_FRAME_HEIGHT + return float(self.height) + if prop_id in (5,): # CAP_PROP_FPS + return float(self.fps) + if prop_id in (6,): # CAP_PROP_FOURCC + return float(self.fourcc) + return 0.0 + + def set(self, prop_id, value): + self.set_calls.append((prop_id, float(value))) + if prop_id in (3,): # CAP_PROP_FRAME_WIDTH + self.width = int(value) + if self.reset_fourcc_on_resize: + self.fourcc = YUY2 + if prop_id in (4,): # CAP_PROP_FRAME_HEIGHT + self.height = int(value) + if self.reset_fourcc_on_resize: + self.fourcc = YUY2 + if prop_id in (5,): # CAP_PROP_FPS + self.fps = float(value) + if prop_id in (6,): # CAP_PROP_FOURCC + self.fourcc = int(value) + return True + + +def _frame(val: int) -> np.ndarray: + return np.full((3, 4, 3), val, dtype=np.uint8) + + +def _factory_from_caps(caps): + lock = threading.Lock() + state = {"i": 0} + + def factory(_config): + with lock: + if state["i"] >= len(caps): + raise RuntimeError("No more fake captures") + cap = caps[state["i"]] + state["i"] += 1 + return cap, "FAKE" + + return factory + + +@pytest.mark.parametrize( + "video,expected", + [ + (None, 0), + ("0", 0), + (" 1 ", 1), + (2, 2), + ("/dev/video0", "/dev/video0"), + (r".\clip.mp4" + "\\", r".\clip.mp4"), + ], +) +def test_normalize_source(video, expected): + assert normalize_source(video) == expected + + +def test_capture_config_from_camera_dict_ignores_deprecated_kwargs(): + cfg = CaptureConfig.from_camera_dict( + 0, + {"resolution_raw": [1920, 1080], "fps": 30}, + mode="queue", + queue_size=5, + overflow="drop_newest", + start_worker=False, + ) + assert cfg.width == 1920 + assert cfg.height == 1080 + assert cfg.background is False + + +def test_apply_stream_props_sets_size_before_fourcc(): + cap = FakeCap([_frame(1)], reset_fourcc_on_resize=True) + _apply_stream_props(cap, width=1920, height=1080, fps=30, fourcc="MJPG") + assert cap.fourcc == MJPG + width_idx = [p for p, _ in cap.set_calls if p == 3] + fourcc_calls = [v for p, v in cap.set_calls if p == 6] + assert width_idx + assert fourcc_calls + assert cap.set_calls.index((3, 1920.0)) < cap.set_calls.index((6, float(MJPG))) + + +def test_warmup_capture_grabs(): + cap = FakeCap([_frame(1)]) + _warmup_capture(cap, grabs=3) + assert cap.grab_count == 3 + + +def test_capture_source_read_and_status(): + frames = [_frame(7)] + cap = FakeCap(frames) + src = CaptureSource( + CaptureConfig(source=0, reconnect=ReconnectPolicy(enabled=False)), + factory=lambda cfg: (cap, "FAKE"), + ) + src.open() + assert src.is_open() + assert src.status.health == CaptureHealth.CONNECTED + ok, frame = src.read_raw() + assert ok and int(frame[0, 0, 0]) == 7 + src.release() + assert src.status.health == CaptureHealth.CLOSED + assert cap.release_count == 1 + + +def test_auto_reconnect_after_failed_reads(): + first = FakeCap([_frame(1)], fail_after=1) + second = FakeCap([_frame(9)]) + factory = _factory_from_caps([first, second]) + policy = ReconnectPolicy( + enabled=True, + fail_threshold=1, + initial_backoff_s=0.01, + max_backoff_s=0.01, + stale_timeout_s=None, + max_attempts=3, + ) + src = CaptureSource( + CaptureConfig(source=0, reconnect=policy, verbose=False), + factory=factory, + ) + src.open() + ok, _ = src.read_raw() + assert ok + ok, _ = src.read_raw() + assert not ok + assert src.should_reconnect() + assert src.reconnect() + ok, frame = src.read_raw() + assert ok and int(frame[0, 0, 0]) == 9 + assert src.status.health == CaptureHealth.CONNECTED + assert src.status.reconnect_attempts >= 1 + + +def test_reconnect_disabled_does_not_reopen(): + cap = FakeCap([_frame(1)], fail_after=1) + src = CaptureSource( + CaptureConfig( + source=0, + reconnect=ReconnectPolicy(enabled=False, fail_threshold=1), + ), + factory=lambda cfg: (cap, "FAKE"), + ) + src.open() + src.read_raw() + src.read_raw() + assert not src.should_reconnect() + + +def test_reconnect_attempts_exhausted(): + good = FakeCap([_frame(1)], fail_after=1) + state = {"n": 0} + + def factory(_cfg): + state["n"] += 1 + if state["n"] == 1: + return good, "FAKE" + raise RuntimeError("camera missing") + + policy = ReconnectPolicy( + enabled=True, + fail_threshold=1, + initial_backoff_s=0.01, + max_backoff_s=0.01, + stale_timeout_s=None, + max_attempts=2, + ) + src = CaptureSource( + CaptureConfig(source=0, reconnect=policy), + factory=factory, + ) + src.open() + src.read_raw() + src.read_raw() + assert src.should_reconnect() + assert not src.reconnect() + assert not src.reconnect() + assert src.status.health == CaptureHealth.FAILED + + +def test_background_thread_publishes_latest_frame(): + caps = [FakeCap([_frame(i) for i in range(5)])] + src = open_capture( + CaptureConfig( + source=0, + background=True, + reconnect=ReconnectPolicy(enabled=False), + ), + factory=_factory_from_caps(caps), + start_background=True, + ) + try: + deadline = time.perf_counter() + 2.0 + ok = False + frame = None + while time.perf_counter() < deadline: + ok, frame = src.read(wait_s=0.05) + if ok: + break + assert ok and frame is not None + assert src.running + finally: + src.close() + + +def test_background_reconnect(): + first = FakeCap([_frame(1)], fail_after=2) + second = FakeCap([_frame(42) for _ in range(20)]) + policy = ReconnectPolicy( + enabled=True, + fail_threshold=1, + initial_backoff_s=0.01, + max_backoff_s=0.02, + stale_timeout_s=None, + ) + src = open_capture( + CaptureConfig(source=0, reconnect=policy, background=True), + factory=_factory_from_caps([first, second]), + start_background=True, + ) + try: + saw_reconnect = False + saw_new = False + deadline = time.perf_counter() + 3.0 + while time.perf_counter() < deadline: + if src.status.reconnect_attempts > 0: + saw_reconnect = True + ok, frame = src.read(wait_s=0.05) + if ok and frame is not None and int(frame[0, 0, 0]) == 42: + saw_new = True + break + time.sleep(0.01) + assert saw_reconnect + assert saw_new + finally: + src.close() + + +def test_warmup_frames_discards_reads(): + cap = FakeCap([_frame(i) for i in range(10)]) + src = CaptureSource( + CaptureConfig(source=0), + factory=lambda cfg: (cap, "FAKE"), + ) + src.open() + src.warmup_frames(3) + ok, frame = src.read_raw() + assert ok and int(frame[0, 0, 0]) == 3 + + +def test_camera_filter_excludes_laptop_webcams(): + cams = [ + CameraInfo(0, "DSHOW", 0, 640, 480, name="Integrated Webcam"), + CameraInfo(1, "DSHOW", 0, 1920, 1080, name="Orisys Tactile Cam"), + CameraInfo(2, "DSHOW", 0, 1280, 720, name="USB Camera"), + ] + filt = CameraFilter(exclude_laptop_webcams=True) + kept = [c for c in cams if filt.matches(c)] + assert [c.index for c in kept] == [1, 2] + + +def test_camera_filter_name_include_and_vid_pid(): + cams = [ + CameraInfo(0, "V4L2", 0, 640, 480, name="foo", vendor_id=0x1234, product_id=0x5678), + CameraInfo(1, "V4L2", 0, 640, 480, name="orisys-finger", vendor_id=0xAAAA, product_id=0xBBBB), + ] + filt = CameraFilter(name_include=[r"orisys"], vid_pid_allowlist=[(0xAAAA, 0xBBBB)]) + kept = [c for c in cams if filt.matches(c)] + assert len(kept) == 1 and kept[0].index == 1 + + +def test_camera_filter_min_resolution(): + cams = [ + CameraInfo(0, "DSHOW", 0, 320, 240, name="tiny"), + CameraInfo(1, "DSHOW", 0, 1280, 720, name="hd"), + ] + filt = CameraFilter(min_width=640, min_height=480) + kept = [c for c in cams if filt.matches(c)] + assert [c.index for c in kept] == [1] + + +def test_pick_default_camera(): + cams = [ + CameraInfo(1, "DSHOW", 0, 640, 480), + CameraInfo(2, "DSHOW", 0, 640, 480), + ] + assert pick_default_camera(cams, preferred="2") == "2" + assert pick_default_camera(cams, preferred=None) == "1" + assert pick_default_camera([], preferred="0") is None + + +def test_laptop_patterns_nonempty(): + assert len(DEFAULT_LAPTOP_WEBCAM_PATTERNS) >= 5 + + +def test_camera_info_label_includes_name(): + cam = CameraInfo(3, "DSHOW", 700, 1280, 720, name="Orisys") + assert "Orisys" in cam.label() + assert cam.source == "3" + + +def test_list_cameras_is_callable(): + assert callable(list_cameras) + + +def test_max_simultaneous_captures_constant(): + assert MAX_SIMULTANEOUS_CAPTURES == 10 + + +def test_sensor_external_mode_still_works(): + from orisys import Sensor + + sensor = Sensor("external", config_name="box", verbose=False, backend="cpu") + try: + assert sensor._external_only + assert sensor.capture_status is None + assert sensor._capture_source is None + finally: + sensor.disconnect() + + +def test_capture_source_open_applies_stream_props_via_default_factory(monkeypatch): + cap = FakeCap([_frame(1)], reset_fourcc_on_resize=True) + + def fake_open(*args, **kwargs): + _apply_stream_props( + cap, + width=kwargs.get("width"), + height=kwargs.get("height"), + fps=kwargs.get("fps"), + fourcc=kwargs.get("fourcc"), + ) + _warmup_capture(cap) + return cap, "FAKE" + + monkeypatch.setattr("orisys.capture.source.open_opencv_capture", fake_open) + src = CaptureSource(CaptureConfig(source=0, width=1920, height=1080, fourcc="MJPG")) + src.open() + assert cap.fourcc == MJPG diff --git a/tests/test_ffc.py b/tests/test_ffc.py new file mode 100644 index 0000000..0d3f831 --- /dev/null +++ b/tests/test_ffc.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import cv2 + +REPO_ROOT = Path(__file__).resolve().parents[1] +SRC_ROOT = REPO_ROOT / "src" +for path in (REPO_ROOT, SRC_ROOT): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) + +from orisys import FFCFrameProcessor +from tools.ffc.roi_polygon_tool import load_roi_mask, select_polygon_roi + + +def parse_source(value): + value = str(value).strip() + for prefix in ("camera:", "cam:", "webcam:"): + if value.lower().startswith(prefix): + value = value[len(prefix):].strip() + break + try: + return int(value) + except ValueError: + return value + + +def read_image_if_possible(source): + if isinstance(source, int): + return None + path = Path(source) + if not path.exists() or not path.is_file(): + return None + image = cv2.imread(str(path), cv2.IMREAD_COLOR) + return image + + +def draw_roi_outline(image, roi_mask): + view = image.copy() + contours, _ = cv2.findContours(roi_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + cv2.drawContours(view, contours, -1, (0, 255, 255), 1) + return view + + +def parse_args(): + parser = argparse.ArgumentParser(description="Test ROI drawing and flat-field correction.") + parser.add_argument("--source", default="0", help="Camera id, image path, or video path.") + parser.add_argument("--roi-config", default=None, help="Existing ROI JSON config.") + parser.add_argument("--roi-output", default="roi_polygon.json", help="Path to save ROI JSON.") + parser.add_argument("--weights", default=None, help="Existing FFC weights .npz to load.") + parser.add_argument("--save-weights", default="ffc_weights.npz", help="Path to save calibrated weights.") + parser.add_argument("--blur-kernel", type=int, default=151) + parser.add_argument("--blur-sigma", type=float, default=0.0) + parser.add_argument("--epsilon", type=float, default=1e-6) + parser.add_argument("--keep-outside", action="store_true", help="Keep pixels outside ROI.") + return parser.parse_args() + + +def prepare_corrector(frame, args): + ffc = FFCFrameProcessor( + blur_kernel=args.blur_kernel, + blur_sigma=args.blur_sigma, + epsilon=args.epsilon, + masked_outside=not args.keep_outside, + ) + + if args.weights: + ffc.load(args.weights) + return ffc, ffc.roi_mask + + if args.roi_config: + roi_mask = load_roi_mask(args.roi_config, frame.shape) + else: + roi = select_polygon_roi(frame, output=args.roi_output) + if roi is None: + raise RuntimeError("ROI selection cancelled.") + roi_mask = roi["mask"] + + ffc.calibrate(frame, roi_mask=roi_mask) + if args.save_weights: + ffc.save(args.save_weights) + return ffc, roi_mask + + +def run_single_image(image, args): + ffc, roi_mask = prepare_corrector(image, args) + corrected = ffc.correct(image) + cv2.imshow("Input ROI", draw_roi_outline(image, roi_mask)) + cv2.imshow("Corrected ROI", draw_roi_outline(corrected, roi_mask)) + cv2.waitKey(0) + cv2.destroyAllWindows() + + +def run_video(source, args): + cap = cv2.VideoCapture(source) + if not cap.isOpened(): + raise RuntimeError(f"Cannot open source: {source}") + + try: + ok, frame = cap.read() + if not ok: + raise RuntimeError("Failed to read first frame.") + + ffc, roi_mask = prepare_corrector(frame, args) + + while True: + corrected = ffc.correct(frame) + cv2.imshow("Input ROI", draw_roi_outline(frame, roi_mask)) + cv2.imshow("Corrected ROI", draw_roi_outline(corrected, roi_mask)) + + key = cv2.waitKey(1) & 0xFF + if key in (27, ord("q")): + break + if key == ord("r"): + ffc.calibrate(frame, roi_mask=roi_mask) + if args.save_weights: + ffc.save(args.save_weights) + print("Recalibrated from current frame.") + + ok, frame = cap.read() + if not ok: + break + finally: + cap.release() + cv2.destroyAllWindows() + + +def main(): + args = parse_args() + source = parse_source(args.source) + + image = read_image_if_possible(source) + if image is not None: + run_single_image(image, args) + else: + run_video(source, args) + + +if __name__ == "__main__": + main() diff --git a/tests/test_pyproject.py b/tests/test_pyproject.py index f4533f7..c18cec4 100644 --- a/tests/test_pyproject.py +++ b/tests/test_pyproject.py @@ -12,7 +12,7 @@ def packaging_utils(): build_dir = REPO_ROOT / "build" if str(build_dir) not in sys.path: sys.path.insert(0, str(build_dir)) - from pyproject_utils import extras_require, install_requires, project_metadata + from common.pyproject_utils import extras_require, install_requires, project_metadata return install_requires, extras_require, project_metadata @@ -33,9 +33,9 @@ def test_pyproject_dynamic_version(packaging_utils): build_dir = REPO_ROOT / "build" if str(build_dir) not in sys.path: sys.path.insert(0, str(build_dir)) - from pyproject_utils import get_version + from common.pyproject_utils import get_version - assert get_version(REPO_ROOT) == "0.4.0" + assert get_version(REPO_ROOT) == "0.5.0" def test_pyproject_bundle_manifest(packaging_utils): @@ -44,11 +44,11 @@ def test_pyproject_bundle_manifest(packaging_utils): build_dir = REPO_ROOT / "build" if str(build_dir) not in sys.path: sys.path.insert(0, str(build_dir)) - from pyproject_utils import bundle_data_files, bundle_items + from common.pyproject_utils import bundle_data_files, bundle_items items = bundle_items(REPO_ROOT) names = {item["name"] for item in items} - assert names == {"configs", "examples", "docs", "assets"} + assert names == {"configs", "examples", "docs", "sensors"} data_files = dict(bundle_data_files(REPO_ROOT)) assert "orisys/configs" in data_files @@ -57,19 +57,40 @@ def test_pyproject_bundle_manifest(packaging_utils): assert all("/" in path and "*" not in path for path in config_files) assert any(path.endswith(".json") for path in config_files) - asset_dirs = [dest for dest in data_files if dest.startswith("orisys/assets/")] - assert asset_dirs + sensor_dirs = [dest for dest in data_files if dest.startswith("orisys/sensors")] + assert sensor_dirs + all_sensor_files = [path for files in (data_files[d] for d in sensor_dirs) for path in files] + assert any(path.endswith("force_calibration.json") for path in all_sensor_files) + assert any("finger" in path and path.endswith(".json") for path in all_sensor_files) + # Finger mesh/viz assets (flat or nested assets/finger/ checkout). + assert any("finger/assets" in dest or any("finger/assets" in p for p in data_files[dest]) for dest in sensor_dirs) assert all("*" not in path for files in data_files.values() for path in files) assert "examples" not in data_files +def test_stage_dev_bundle_into_writes_sensors_under_package(tmp_path): + import sys + + build_dir = REPO_ROOT / "build" + if str(build_dir) not in sys.path: + sys.path.insert(0, str(build_dir)) + from common.pyproject_utils import stage_dev_bundle_into + + pkg = tmp_path / "orisys" + packaged = stage_dev_bundle_into(REPO_ROOT, pkg) + assert any(p.replace("\\", "/").endswith("sensors/box/force_calibration.json") for p in packaged) + assert (pkg / "sensors" / "box" / "force_calibration.json").is_file() + assert (pkg / "configs").is_dir() + assert any((pkg / "configs").glob("*.json")) + + def test_bundle_explicit_files(): import sys build_dir = REPO_ROOT / "build" if str(build_dir) not in sys.path: sys.path.insert(0, str(build_dir)) - from wheel_layout import data_file_entries_for_item, resolve_bundle_source_files + from wheel.layout import data_file_entries_for_item, resolve_bundle_source_files item = { "name": "configs", diff --git a/tests/test_qt_viewer_backend.py b/tests/test_qt_viewer_backend.py new file mode 100644 index 0000000..3399bb9 --- /dev/null +++ b/tests/test_qt_viewer_backend.py @@ -0,0 +1,167 @@ +"""Tests for Qt viewer backend force history and aspect-aware display helpers.""" + +from __future__ import annotations + +import sys +import types +from pathlib import Path +from unittest.mock import MagicMock + +import numpy as np +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[1] +_EXAMPLES_DIR = _REPO_ROOT / "examples" +if str(_EXAMPLES_DIR) not in sys.path: + sys.path.insert(0, str(_EXAMPLES_DIR)) + +from gui.viewer2d.backend import ( # noqa: E402 + QtViewerBackend, + image_shape_meta, + panel_width_for_aspect, + scale_bgr_to_fit, + select_display_image, +) +from gui.sensors import ( # noqa: E402 + FrameSnapshot, + _snapshot_from_sensor, + calibrate_normal_force, +) +from orisys.signal_processing import calibrate_normal_force as sdk_calibrate_normal_force + + +class FakeForceAdapter: + """Minimal adapter that emits changing FN/FX/FY without a real sensor.""" + + def __init__(self): + self._i = 0 + + def read(self): + i = self._i + self._i += 1 + return FrameSnapshot( + image=np.zeros((100, 200, 3), dtype=np.uint8), + normal_force=1000.0 + 50.0 * i, + shear_x=-200.0 + 10.0 * i, + shear_y=300.0 - 5.0 * i, + normal_force_calibrated=calibrate_normal_force(1000.0 + 50.0 * i), + fps=30.0, + timestamp=float(i) / 30.0, + flow_overlay=np.zeros((100, 200, 3), dtype=np.uint8), + record_frame=np.zeros((100, 200, 3), dtype=np.uint8), + ) + + +def test_backend_force_history_tracks_all_channels(): + adapter = FakeForceAdapter() + backend = QtViewerBackend(npoints=32, lowpass_cutoff_hz=0.0) + + for _ in range(8): + snap = adapter.read() + backend.ingest(snap, now=snap.timestamp) + + state = backend.plot_state() + assert state is not None + assert state.sample_count == 8 + assert state.values.shape == (8, 3) + + # All three channels must change across frames (not stay flat). + fn_span = float(state.values[:, 0].max() - state.values[:, 0].min()) + fx_span = float(state.values[:, 1].max() - state.values[:, 1].min()) + fy_span = float(state.values[:, 2].max() - state.values[:, 2].min()) + assert fn_span > 100.0 + assert fx_span > 20.0 + assert fy_span > 10.0 + + # Raw FN is used for curves, not the tiny calibrated Newton scale. + assert float(state.values[-1, 0]) == pytest.approx(1000.0 + 50.0 * 7) + + +@pytest.mark.parametrize( + "shape,expected_aspect", + [ + ((100, 200, 3), 2.0), # wide + ((120, 120, 3), 1.0), # square + ((200, 100, 3), 0.5), # portrait + ], +) +def test_backend_image_aspect_metadata(shape, expected_aspect): + img = np.zeros(shape, dtype=np.uint8) + h, w, aspect = image_shape_meta(img) + assert (h, w) == (shape[0], shape[1]) + assert aspect == pytest.approx(expected_aspect) + + snap = FrameSnapshot(flow_overlay=img, normal_force=1.0, shear_x=2.0, shear_y=3.0) + backend = QtViewerBackend(lowpass_cutoff_hz=0.0) + frame = backend.ingest(snap, now=0.0) + assert frame.image_height == shape[0] + assert frame.image_width == shape[1] + assert frame.aspect_ratio == pytest.approx(expected_aspect) + + fitted = scale_bgr_to_fit(img, max_w=80, max_h=80) + fh, fw = fitted.shape[:2] + assert fw <= 80 and fh <= 80 + assert abs((fw / fh) - expected_aspect) < 0.05 + + panel_w = panel_width_for_aspect(200, expected_aspect, min_size=160) + assert panel_w >= 160 + assert panel_w == pytest.approx(max(160, int(round(200 * expected_aspect)))) + + +def test_select_display_image_prefers_flow_overlay(): + overlay = np.full((10, 20, 3), 7, dtype=np.uint8) + processed = np.full((10, 20, 3), 3, dtype=np.uint8) + snap = FrameSnapshot(flow_overlay=overlay, processed_image=processed, image=processed) + out = select_display_image(snap) + assert np.array_equal(out, overlay) + + +def test_snapshot_requests_hhd_every_frame_without_motion_gate(): + """Regression: force refresh must not use check_motion=True gating.""" + sensor = MagicMock() + sensor.img = np.zeros((32, 32), dtype=np.uint8) + sensor.frame = np.zeros((32, 32), dtype=np.uint8) + sensor.info = types.SimpleNamespace( + FPS="FPS", + VRAW="VRAW", + VNORMAL="VNORMAL", + IMG="IMG", + CONTOUR="CONTOUR", + CENTROID="CENTROID", + DEPTH="DEPTH", + FNORMAL="FNORMAL", + FSHEARX="FSHEARX", + FSHEARY="FSHEARY", + ) + sensor.read_info.return_value = ( + 30.0, + np.zeros((32, 32, 2), dtype=np.float32), + np.zeros((32, 32), dtype=np.float32), + sensor.img, + None, + None, + np.zeros((32, 32), dtype=np.float32), + 12345.0, + -100.0, + 50.0, + ) + + # Avoid depending on orisys.util.draw_arrows implementation details. + import gui.sensors.adapter as sa + + original_util = sa.orisys.util + sa.orisys.util = types.SimpleNamespace( + draw_arrows=lambda *args, **kwargs: np.zeros((32, 32, 3), dtype=np.uint8) + ) + try: + snap = _snapshot_from_sensor(sensor, raw_image=sensor.frame, processed_image=sensor.img) + finally: + sa.orisys.util = original_util + + sensor.compute_deformation.assert_called_once_with(decompose=True, check_motion=False) + assert snap is not None + assert snap.normal_force == pytest.approx(12345.0) + assert snap.shear_x == pytest.approx(-100.0) + assert snap.shear_y == pytest.approx(50.0) + assert snap.normal_force_calibrated == pytest.approx(calibrate_normal_force(12345.0)) + assert snap.normal_force_calibrated == pytest.approx(sdk_calibrate_normal_force(12345.0)) diff --git a/tests/test_signal_processing.py b/tests/test_signal_processing.py new file mode 100644 index 0000000..cdc0fb3 --- /dev/null +++ b/tests/test_signal_processing.py @@ -0,0 +1,264 @@ +"""Tests for SDK signal helpers and Sensor.configure runtime settings.""" + +from __future__ import annotations + +import numpy as np +import pytest +from scipy.signal import butter, filtfilt + +from orisys import Sensor +from orisys.signal_processing import ( + NORMAL_FORCE_CAL_OFFSET, + NORMAL_FORCE_CAL_SLOPE, + LowpassFilter, + calibrate_normal_force, + calibrate_shear_components, + calibrate_shear_force, + lowpass_filter, +) + + +def _legacy_viewer_lowpass(data, cutoff_hz, sample_rate_hz=30.0, order=2): + """Reference copy of the previous viewer2d Butterworth + filtfilt path.""" + y = np.asarray(data, dtype=np.float64) + if cutoff_hz == 0: + return y.copy() + if y.size == 0: + return y + nyquist = 0.5 * sample_rate_hz + b, a = butter(order, cutoff_hz / nyquist, btype="low") + padlen = 3 * (max(len(b), len(a)) - 1) + if y.size <= padlen: + return y.copy() + return filtfilt(b, a, y) + + +def test_batch_lowpass_matches_legacy_viewer_filter(): + rng = np.random.default_rng(0) + t = np.arange(180, dtype=np.float64) / 30.0 + signal = np.sin(2 * np.pi * 1.0 * t) + 0.4 * rng.normal(size=t.size) + cutoff = 5.0 + actual = lowpass_filter(signal, cutoff_hz=cutoff, sample_rate_hz=30.0, order=2) + expected = _legacy_viewer_lowpass(signal, cutoff_hz=cutoff, sample_rate_hz=30.0, order=2) + np.testing.assert_allclose(actual, expected, rtol=1e-10, atol=1e-10) + + +def test_batch_lowpass_cutoff_zero_is_noop(): + data = np.array([1.0, 2.0, 3.0]) + out = lowpass_filter(data, cutoff_hz=0) + np.testing.assert_array_equal(out, data) + assert out is not data + + +def test_stateful_lowpass_reduces_high_frequency_noise(): + rng = np.random.default_rng(1) + n = 240 + t = np.arange(n, dtype=np.float64) / 30.0 + clean = np.sin(2 * np.pi * 0.5 * t) + noisy = clean + 0.8 * rng.normal(size=n) + + filt = LowpassFilter(cutoff_hz=3.0, sample_rate_hz=30.0, order=2, n_channels=1) + filtered = np.array([float(filt.process([v])[0]) for v in noisy]) + + assert np.isfinite(filtered).all() + noisy_hf = float(np.std(np.diff(noisy))) + filtered_hf = float(np.std(np.diff(filtered))) + assert filtered_hf < 0.5 * noisy_hf + # Steady-state init on the first sample should not start at zero. + assert filtered[0] == pytest.approx(noisy[0]) + + +def test_calibrate_normal_force_preserves_linear_curve(): + raw = 12345.0 + expected = NORMAL_FORCE_CAL_SLOPE * raw + NORMAL_FORCE_CAL_OFFSET + assert calibrate_normal_force(raw) == pytest.approx(expected) + assert calibrate_normal_force(0.0) == pytest.approx(NORMAL_FORCE_CAL_OFFSET) + + +def test_calibrate_normal_force_mini_uses_legacy_linear_file(): + raw = 10000.0 + expected = NORMAL_FORCE_CAL_SLOPE * raw + NORMAL_FORCE_CAL_OFFSET + assert calibrate_normal_force(raw, sensor_name="mini") == pytest.approx(expected) + + +def test_calibrate_normal_force_box_uses_piecewise_curve(): + assert calibrate_normal_force(10000.0, sensor_name="box") == pytest.approx(0.377) + assert calibrate_normal_force(30000.0, sensor_name="box") == pytest.approx(1.377) + # Mid-segment interpolation between 10000→0.377 and 30000→1.377 + mid = calibrate_normal_force(20000.0, sensor_name="box") + assert mid == pytest.approx(0.877) + # Below first breakpoint with extrapolate=true + assert calibrate_normal_force(0.0, sensor_name="box") == pytest.approx(0.0) + + +def test_calibrate_shear_force_box_piecewise(): + assert calibrate_shear_force(1000.0, sensor_name="box") == pytest.approx(0.0) + assert calibrate_shear_force(36000.0, sensor_name="box") == pytest.approx(1.19) + assert calibrate_shear_force(68000.0, sensor_name="box") == pytest.approx(2.06) + + +def test_calibrate_shear_components_box_signed(): + sx, sy = calibrate_shear_components(36000.0, -36000.0, sensor_name="box") + assert sx == pytest.approx(1.19) + assert sy == pytest.approx(-1.19) + + +def test_calibrate_shear_missing_file_returns_raw(): + assert calibrate_shear_force(1234.5, sensor_name="no-such-sensor") == pytest.approx(1234.5) + sx, sy = calibrate_shear_components(1.0, -2.0, sensor_name="no-such-sensor") + assert (sx, sy) == pytest.approx((1.0, -2.0)) + + +def test_calibrate_normal_force_unknown_sensor_falls_back_to_defaults(): + raw = 5000.0 + expected = NORMAL_FORCE_CAL_SLOPE * raw + NORMAL_FORCE_CAL_OFFSET + assert calibrate_normal_force(raw, sensor_name="no-such-sensor") == pytest.approx(expected) + + +def _external_sensor(): + return Sensor("external", config_name="box", verbose=False, backend="cpu") + + +def test_configure_lowpass_filters_read_info_and_keeps_raw(): + sensor = _external_sensor() + try: + sensor._force_raw = (1000.0, -200.0, 50.0) + sensor._apply_force_filter() + fn, fx, fy = sensor.read_info( + sensor.info.FNORMAL, + sensor.info.FSHEARX, + sensor.info.FSHEARY, + ) + fn_raw, fx_raw, fy_raw = sensor.read_info( + sensor.info.FNORMAL_RAW, + sensor.info.FSHEARX_RAW, + sensor.info.FSHEARY_RAW, + ) + assert (fn_raw, fx_raw, fy_raw) == pytest.approx((1000.0, -200.0, 50.0)) + assert fn == pytest.approx(calibrate_normal_force(1000.0, sensor_name="box")) + assert (fx, fy) == pytest.approx( + calibrate_shear_components(-200.0, 50.0, sensor_name="box") + ) + + sensor.configure(lowpass_cutoff_hz=5.0, lowpass_sample_rate_hz=30.0) + noisy = [(1000.0 + (200.0 if i % 2 else -200.0), -200.0, 50.0) for i in range(40)] + filtered_fn = [] + raw_fn = [] + for sample in noisy: + sensor._force_raw = sample + sensor._apply_force_filter() + filtered_fn.append(sensor.read_info(sensor.info.FNORMAL)[0]) + raw_fn.append(sensor.read_info(sensor.info.FNORMAL_RAW)[0]) + + assert raw_fn[-1] == pytest.approx(noisy[-1][0]) + assert filtered_fn[-1] != pytest.approx(raw_fn[-1]) + assert float(np.std(np.diff(filtered_fn))) < float(np.std(np.diff(raw_fn))) + fx_raw, fy_raw = sensor.read_info(sensor.info.FSHEARX_RAW, sensor.info.FSHEARY_RAW) + assert fx_raw == pytest.approx(-200.0) + assert fy_raw == pytest.approx(50.0) + finally: + sensor.disconnect() + + +def test_configure_lowpass_disabled_or_zero_cutoff_returns_raw(): + sensor = _external_sensor() + try: + sensor.configure(lowpass_cutoff_hz=5.0) + sensor._force_raw = (10.0, 20.0, 30.0) + sensor._apply_force_filter() + + sensor.configure(lowpass_cutoff_hz=0) + sensor._force_raw = (111.0, 222.0, 333.0) + sensor._apply_force_filter() + fn, fx, fy = sensor.read_info( + sensor.info.FNORMAL, + sensor.info.FSHEARX, + sensor.info.FSHEARY, + ) + assert fn == pytest.approx(calibrate_normal_force(111.0, sensor_name="box")) + assert (fx, fy) == pytest.approx( + calibrate_shear_components(222.0, 333.0, sensor_name="box") + ) + + sensor.configure(lowpass_cutoff_hz=5.0, lowpass_enabled=False) + sensor._force_raw = (7.0, 8.0, 9.0) + sensor._apply_force_filter() + fn, fx, fy = sensor.read_info( + sensor.info.FNORMAL, + sensor.info.FSHEARX, + sensor.info.FSHEARY, + ) + fn_raw = sensor.read_info(sensor.info.FNORMAL_RAW)[0] + assert fn == pytest.approx(calibrate_normal_force(7.0, sensor_name="box")) + assert (fx, fy) == pytest.approx( + calibrate_shear_components(8.0, 9.0, sensor_name="box") + ) + assert fn_raw == pytest.approx(7.0) + finally: + sensor.disconnect() + + +def test_box_read_info_applies_force_calibration_json(): + sensor = _external_sensor() + try: + assert sensor._force_cal_sensor_name == "box" + sensor._force_raw = (10000.0, 36000.0, -36000.0) + sensor._apply_force_filter() + fn, fx, fy = sensor.read_info( + sensor.info.FNORMAL, + sensor.info.FSHEARX, + sensor.info.FSHEARY, + ) + fn_raw, fx_raw, fy_raw = sensor.read_info( + sensor.info.FNORMAL_RAW, + sensor.info.FSHEARX_RAW, + sensor.info.FSHEARY_RAW, + ) + assert (fn_raw, fx_raw, fy_raw) == pytest.approx((10000.0, 36000.0, -36000.0)) + assert fn == pytest.approx(0.377) + assert fx == pytest.approx(1.19) + assert fy == pytest.approx(-1.19) + finally: + sensor.disconnect() + + +def test_read_info_force_stays_raw_without_calibration_json(monkeypatch): + monkeypatch.setattr( + "orisys.sensor.configs.load_force_calibration", + lambda _name: None, + ) + sensor = Sensor("external", config_name="box", verbose=False, backend="cpu") + try: + assert sensor._force_cal_sensor_name is None + sensor._force_raw = (111.0, -22.0, 33.0) + sensor._apply_force_filter() + fn, fx, fy = sensor.read_info( + sensor.info.FNORMAL, + sensor.info.FSHEARX, + sensor.info.FSHEARY, + ) + assert (fn, fx, fy) == pytest.approx((111.0, -22.0, 33.0)) + finally: + sensor.disconnect() + + +def test_configure_hhd_downsample_invalidates_solver_cache(): + sensor = _external_sensor() + try: + sensor._hhd_downsample = 1 + sensor._Sensor__hhd = object() + sensor._hhd_grid_shape = (40, 40) + sensor.valid_mask = np.ones((90, 90), dtype=bool) + sensor.valid_mask_hhd = sensor.valid_mask + + sensor.configure(hhd_downsample=3) + assert sensor._hhd_downsample == 3 + assert sensor._Sensor__hhd is None + assert sensor._hhd_grid_shape is None + assert sensor.valid_mask_hhd is not None + assert sensor.valid_mask_hhd.shape == (30, 30) + + with pytest.raises(ValueError): + sensor.configure(hhd_downsample="nope") + finally: + sensor.disconnect() diff --git a/tests/test_stitch.py b/tests/test_stitch.py new file mode 100644 index 0000000..9c24762 --- /dev/null +++ b/tests/test_stitch.py @@ -0,0 +1,168 @@ +"""Unit tests for orisys.stitch (synthetic tiles, no camera).""" + +from __future__ import annotations + +import numpy as np +import pytest + +from orisys.Stitcher import Stitcher +from orisys.stitch import ( + StitchStatus, + get_blend_backend, + paste_tile_clipped, + resolve_blend_mode, + tile_weight_map, +) +from orisys.stitch.backends.numpy_backend import blend_kernel_numpy +from orisys.stitch.config import numba_blend_available, validate_box_grid +from orisys.stitch.ops import compute_final_crop_rect, coverage_ratio +from orisys.frame_processors import CompoundEyeFrameProcessor + + +def _solid_tile(val: int, h=32, w=32): + return np.full((h, w, 3), val, dtype=np.uint8) + + +def test_tile_weight_map_edges_are_low(): + wmap = tile_weight_map(32, 32, margin=4) + assert wmap.shape == (32, 32) + assert wmap.dtype == np.float32 + assert float(wmap[0, 0]) < float(wmap[16, 16]) + assert float(wmap[16, 16]) == pytest.approx(1.0) + + +def test_paste_tile_clipped_and_coverage(): + canvas = np.zeros((40, 40, 3), dtype=np.uint8) + cov = np.zeros((40, 40), dtype=np.uint8) + tile = _solid_tile(200, 10, 10) + paste_tile_clipped(canvas, tile, 5, 5, cov=cov) + assert int(canvas[5, 5, 0]) == 200 + assert int(cov[5, 5]) == 1 + assert int(cov[0, 0]) == 0 + + +def test_blend_kernel_numpy_first_paint(): + blend = np.zeros((4, 4, 3), dtype=np.float32) + wsum = np.zeros((4, 4), dtype=np.float32) + cov = np.zeros((4, 4), dtype=np.uint8) + src = np.ones((4, 4, 3), dtype=np.float32) * 50 + weight = np.ones((4, 4), dtype=np.float32) + blend_kernel_numpy(blend, wsum, cov, src, weight) + assert np.allclose(blend, 50) + assert np.all(cov == 1) + + +def test_numpy_backend_blend_tile(): + backend = get_blend_backend("numpy") + blend = np.zeros((20, 20, 3), dtype=np.float32) + wsum = np.zeros((20, 20), dtype=np.float32) + cov = np.zeros((20, 20), dtype=np.uint8) + src = np.full((8, 8, 3), 40, dtype=np.uint8) + weight = tile_weight_map(8, 8, margin=2) + backend.blend_tile_inplace(blend, wsum, cov, src, weight, 2, 3) + assert float(blend[4, 5, 0]) > 0 + assert coverage_ratio(cov) > 0 + + +@pytest.mark.skipif(not numba_blend_available(), reason="numba blend unavailable") +def test_numba_numpy_backend_parity(): + numpy_b = get_blend_backend("numpy") + numba_b = get_blend_backend("numba") + shape = (16, 16) + src = np.random.randint(0, 255, (*shape, 3), dtype=np.uint8) + weight = tile_weight_map(*shape, margin=3) + + def run(backend): + blend = np.zeros((*shape, 3), dtype=np.float32) + wsum = np.zeros(shape, dtype=np.float32) + cov = np.zeros(shape, dtype=np.uint8) + backend.blend_tile_inplace(blend, wsum, cov, src, weight, 0, 0) + return blend, cov + + b0, c0 = run(numpy_b) + b1, c1 = run(numba_b) + assert np.allclose(b0, b1, atol=1e-2) + assert np.array_equal(c0, c1) + + +def test_resolve_blend_mode_default(monkeypatch): + monkeypatch.delenv("ORISYS_STITCH_BLEND", raising=False) + monkeypatch.delenv("ORISYS_STITCH_BLEND_BACKEND", raising=False) + mode, fell = resolve_blend_mode({}) + assert mode == "none" + assert fell is False + + +def test_validate_box_grid(): + assert validate_box_grid([(0, 0, 1, 1)] * 9, 3, 3) + assert not validate_box_grid([(0, 0, 1, 1)] * 8, 3, 3) + + +def test_compute_final_crop_rect(): + cov = np.zeros((20, 20), dtype=np.uint8) + cov[4:16, 4:16] = 1 + rect = compute_final_crop_rect(cov) + assert rect is not None + y0, y1, x0, x1 = rect + assert y1 > y0 and x1 > x0 + + +def _mini_config(): + return { + "min_area_frac": 0.01, + "max_area_frac": 0.05, + "max_aspect_ratio": 50, + "index": [0, 1, 2, 3, 4, 5, 6, 7, 8], + "margin": 10, + "nominal_ov_pos": 100, + "n": [3, 3], + "row_tol": 0.6, + "box_size": [32, 32], + "blend": "numpy", + "n_boxes_target": 9, + } + + +def test_stitcher_status_and_rt_with_synthetic_tiles(): + stitcher = Stitcher(isstitch=True, config_stitcher=_mini_config(), verbose=False) + # Preload tiles as if extract_tiles already ran. + tiles = [_solid_tile(30 + i * 10) for i in range(9)] + stitcher.tile_list = tiles + stitcher.units = [(0, 0, t) for t in tiles] + stitcher.boxes = [(i * 40, 0, i * 40 + 32, 32) for i in range(9)] + + status = stitcher.status + assert isinstance(status, StitchStatus) + assert status.blend_mode == "numpy" + assert status.expected_boxes == 9 + + # Feature matching on flat solid tiles usually fails → None is OK. + out = stitcher.stitch_rt_optimized() + assert out is None or (isinstance(out, np.ndarray) and out.ndim == 3) + + +def test_compound_eye_processor_calls_stitcher(): + class FakeStitcher: + def __init__(self): + self.calls = [] + + def get_stitch_unit(self, frame): + self.calls.append("get") + + def extract_tiles(self): + self.calls.append("extract") + + def stitch_rt_optimized(self): + self.calls.append("stitch") + return np.zeros((8, 8, 3), dtype=np.uint8) + + fake = FakeStitcher() + proc = CompoundEyeFrameProcessor(fake) + out = proc.process(np.zeros((64, 64, 3), dtype=np.uint8)) + assert fake.calls == ["get", "extract", "stitch"] + assert out.shape == (8, 8, 3) + + +def test_unknown_backend_raises(): + with pytest.raises(ValueError): + get_blend_backend("torch") diff --git a/tests/test_uv_unroll_processor.py b/tests/test_uv_unroll_processor.py index ba498e7..6d34123 100644 --- a/tests/test_uv_unroll_processor.py +++ b/tests/test_uv_unroll_processor.py @@ -4,7 +4,7 @@ import cv2 import numpy as np import pytest -from orisys.frame_processors import UVUnrollFrameProcessor +from orisys.frame_processors import FrameUnwarper from orisys.unroll import UVDomain @@ -24,7 +24,7 @@ def test_uv_unroll_processor_output_shape(uv_domain): src_h = int(uv_remap.metadata.get("source_height", uv_remap.out_height)) src_w = int(uv_remap.metadata.get("source_width", uv_remap.out_width)) - processor = UVUnrollFrameProcessor( + processor = FrameUnwarper( uv_remap, source_width=src_w, source_height=src_h, @@ -45,7 +45,7 @@ def test_uv_unroll_processor_prepare_source(uv_domain): src_h = int(uv_remap.metadata.get("source_height", uv_remap.out_height)) src_w = int(uv_remap.metadata.get("source_width", uv_remap.out_width)) - processor = UVUnrollFrameProcessor( + processor = FrameUnwarper( uv_remap, source_width=src_w, source_height=src_h, @@ -61,7 +61,7 @@ def test_uv_unroll_processor_flatten_prepared_matches_process(uv_domain): src_h = int(uv_remap.metadata.get("source_height", uv_remap.out_height)) src_w = int(uv_remap.metadata.get("source_width", uv_remap.out_width)) - processor = UVUnrollFrameProcessor( + processor = FrameUnwarper( uv_remap, source_width=src_w, source_height=src_h, @@ -78,7 +78,7 @@ def test_uv_unroll_processor_gray_fast_path_matches_bgr_result(uv_domain): src_h = int(uv_remap.metadata.get("source_height", uv_remap.out_height)) src_w = int(uv_remap.metadata.get("source_width", uv_remap.out_width)) - processor = UVUnrollFrameProcessor( + processor = FrameUnwarper( uv_remap, source_width=src_w, source_height=src_h, diff --git a/tests/test_wheel_build.py b/tests/test_wheel_build.py new file mode 100644 index 0000000..b023a01 --- /dev/null +++ b/tests/test_wheel_build.py @@ -0,0 +1,54 @@ +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture +def wheel_package(): + import sys + + build_dir = REPO_ROOT / "build" + if str(build_dir) not in sys.path: + sys.path.insert(0, str(build_dir)) + from wheel import package as wheel_package + + return wheel_package + + +def test_compliant_manylinux_tags(wheel_package): + assert wheel_package.is_compliant_manylinux_wheel( + Path("orisys-0.5.0-cp310-cp310-manylinux_2_17_x86_64.whl") + ) + assert wheel_package.is_compliant_manylinux_wheel( + Path("orisys-0.5.0-cp310-cp310-manylinux2014_x86_64.whl") + ) + assert wheel_package.is_compliant_manylinux_wheel( + Path("orisys-0.5.0-cp310-cp310-manylinux_2_28_x86_64.whl") + ) + + +def test_rejects_illegal_manylinux_tags(wheel_package): + assert not wheel_package.is_compliant_manylinux_wheel( + Path("orisys-0.5.0-cp310-cp310-manylinux_x86_64.whl") + ) + assert not wheel_package.is_compliant_manylinux_wheel( + Path("orisys-0.5.0-cp310-cp310-linux_x86_64.whl") + ) + assert not wheel_package.is_compliant_manylinux_wheel( + Path("orisys-0.5.0-cp310-cp310-win_amd64.whl") + ) + + +def test_pyproject_no_private_classifier(): + import sys + + build_dir = REPO_ROOT / "build" + if str(build_dir) not in sys.path: + sys.path.insert(0, str(build_dir)) + from common.pyproject_utils import project_metadata + + classifiers = project_metadata(REPO_ROOT).get("classifiers", []) + assert "Private :: Do Not Upload" not in classifiers + assert "License :: Other/Proprietary License" in classifiers diff --git a/tools/3d_patch_generation_blender/Readme.md b/tools/3d_patch_generation_blender/Readme.md deleted file mode 100644 index 7698b9b..0000000 --- a/tools/3d_patch_generation_blender/Readme.md +++ /dev/null @@ -1,121 +0,0 @@ -# 3D Patch Generation Blender 工具 - -本文件夹包含 Blender 场景文件以及辅助脚本，用于导出网格体/相机数据，并将一张采集到的 2D 图像重新映射回 3D 表面。 - -## 主要文件 - -- `dd02c_v2.blend` - DD02C 曲面与相机配置所使用的主要 Blender 工程文件。 - -- `blender_export.py` - 在 Blender 中运行，用于导出以下数据： - - 网格几何信息 - - 与 UV 相关的 loop 数据 - - 相机内参 - - 相机位姿 - - 预计算的像素到 3D 查找表 - -- `mapto3d.py` - 读取导出的数据和一张采集图像，并生成带颜色的 PLY 格式 3D 点云。 - -## 导出 / 生成的数据 - -- `exported_data/` - 由 `blender_export.py` 生成。通常应包含以下文件： - - `mesh.npz` - - `camera_intrinsics.npz` - - `world_to_cam.npy` - - `pixel_to_xyz.npz` - -- `captured_image.png` - 要映射回导出 3D 表面的输入图像。当前该图像由 Blender 渲染得到。 - -- `mapped_points.ply` - 由 `mapto3d.py` 生成的输出点云文件。 - -- `texture/` - 保存实验过程中生成的纹理图片。 - -## 工作流程 - -### 1. 准备 Blender 场景 - -在 Blender 中打开 `dd02c_v2.blend`。 - -请确认： -- 已选中目标网格 -- 相机存在且位置正确 -- 渲染分辨率与采集图像分辨率一致 - -### 2. 导出几何和相机数据 - -在 Blender 的里面上面转到“scripts” 模式，然后点击运行 `blender_export.py`。 - -该脚本会将所需数据导出到： - -```text -exported_data/ -``` - -脚本会写出以下内容： -- 网格顶点和面信息 -- UV loop 数据 -- OpenCV 风格的相机内参 -- world-to-camera 变换矩阵 -- 可选的逐像素 3D 查找表 - -### 3. 准备输入图像 - -将需要投影到 3D 表面的图像放到当前文件夹中，可以在Blender 渲染生成，例如： - -```text -captured_image.png -``` - -`mapto3d.py` 目前在其 `__main__` 代码块中直接指定输入图像路径。 - -### 4. 将图像像素映射到 3D - -运行： - -```bash -python mapto3d.py -``` - -该脚本会： -- 加载 Blender 导出的数据 -- 加载输入图像 -- 在需要时对图像做上下翻转，以匹配导出的像素映射顺序 -- 优先使用预计算的 `pixel_to_xyz.npz` -- 将结果保存为 `mapped_points.ply` - -如果缺少预计算查找表，脚本会退回到逐像素射线求交，运行速度会慢很多。 - -### 5. 查看输出结果 - -生成的点云可以使用支持 PLY 的工具打开，例如： -- Open3D -- MeshLab -- CloudCompare - -`mapto3d.py` 中也包含可选的 Open3D 可视化步骤。 - -## 说明 - -- `mapto3d.py` 目前假设导出的相机模型与输入图像分辨率一致。 -- 脚本现在默认会对加载的图像做上下翻转，以保证映射颜色与 Blender 导出的像素顺序一致。 -- 为了获得更好的性能，建议在 `exported_data/` 中保留 `pixel_to_xyz.npz`。 - -## 依赖 - -### Blender 侧 - -- Blender Python API：`bpy` -- `numpy` - -### Python 侧 - -- `numpy` -- `opencv-python` -- `open3d`（可选，用于可视化） - diff --git a/tools/3d_patch_generation_blender/blender_export.py b/tools/3d_patch_generation_blender/blender_export.py deleted file mode 100644 index 2f4d2c2..0000000 --- a/tools/3d_patch_generation_blender/blender_export.py +++ /dev/null @@ -1,147 +0,0 @@ -import bpy -import numpy as np -import json -import os -from mathutils import Matrix, Vector - -def export_camera_intrinsics(camera_obj, scene): - """Return camera intrinsic matrix K (3x3) and distortion (none).""" - cam = camera_obj.data - render = scene.render - res_x = render.resolution_x - res_y = render.resolution_y - - # Sensor size and focal length - if cam.sensor_fit == 'HORIZONTAL': - sensor_width = cam.sensor_width - sensor_height = sensor_width / (res_x / res_y) - else: - sensor_height = cam.sensor_height - sensor_width = sensor_height * (res_x / res_y) - focal = cam.lens # in mm - - # Pixel sizes in mm - pixel_width = sensor_width / res_x - pixel_height = sensor_height / res_y - - # Intrinsic matrix (OpenCV style: fx, fy in pixels, cx, cy principal point) - fx = focal / pixel_width - fy = focal / pixel_height - cx = res_x / 2.0 - cy = res_y / 2.0 - - K = np.array([[fx, 0, cx], - [0, fy, cy], - [0, 0, 1]], dtype=np.float64) - return K, res_x, res_y - -def export_mesh_data(obj, filepath): - """Export mesh vertices, faces, and UVs to a compressed .npz file.""" - me = obj.data - # Transform vertices to world space - vertices = np.array([obj.matrix_world @ v.co for v in me.vertices], dtype=np.float32) - faces = np.array([poly.vertices[:] for poly in me.polygons], dtype=np.int32) - - # UVs (per loop, need mapping to vertices for simplicity we'll store per-face) - uv_layer = me.uv_layers.active - if uv_layer: - uv_data = np.array([uv_layer.data[i].uv for i in range(len(uv_layer.data))], dtype=np.float32) - loop_to_vertex = np.array([me.loops[i].vertex_index for i in range(len(me.loops))], dtype=np.int32) - else: - uv_data = np.zeros((0,2), dtype=np.float32) - loop_to_vertex = np.zeros(0, dtype=np.int32) - - np.savez_compressed(filepath, - vertices=vertices, - faces=faces, - uv_data=uv_data, - loop_to_vertex=loop_to_vertex) - print(f"Mesh exported to {filepath}") - -def export_camera_pose(camera_obj, filepath): - """Export camera extrinsics (world-to-camera transform matrix).""" - # Camera world matrix: transforms from camera local to world - cam_to_world = np.array(camera_obj.matrix_world, dtype=np.float32) - # We need world-to-camera transform (inverse) - world_to_cam = np.linalg.inv(cam_to_world) - np.save(filepath, world_to_cam) - print(f"Camera pose exported to {filepath}") - -def export_pixel_to_3d_lookup(obj, camera_obj, scene, filepath): - """ - Precompute for every pixel the 3D hit point (world coordinates) - and save as a 2D array (height, width, 3). This avoids raycasting later. - """ - depsgraph = bpy.context.evaluated_depsgraph_get() - render = scene.render - res_x = render.resolution_x - res_y = render.resolution_y - - # Get camera data - cam = camera_obj.data - if cam.sensor_fit == 'HORIZONTAL': - sensor_width = cam.sensor_width - sensor_height = sensor_width / (res_x / res_y) - else: - sensor_height = cam.sensor_height - sensor_width = sensor_height * (res_x / res_y) - focal = cam.lens - frame_width = sensor_width / focal - frame_height = sensor_height / focal - - # Preallocate array (height, width, 3) for XYZ world coordinates - xyz_map = np.full((res_y, res_x, 3), np.nan, dtype=np.float32) - - origin = camera_obj.matrix_world.translation - cam_rot = camera_obj.matrix_world.to_quaternion() - - for y in range(res_y): - # Normalized sensor coordinate v (0=bottom, 1=top) - v_ndc = (y + 0.5) / res_y - sy = (v_ndc - 0.5) * frame_height - for x in range(res_x): - u_ndc = (x + 0.5) / res_x - sx = (u_ndc - 0.5) * frame_width - - direction_cam = Vector((sx, sy, -1.0)) - direction_world = cam_rot @ direction_cam - direction_world.normalize() - - hit, location, normal, face_index, hit_obj, matrix = scene.ray_cast( - depsgraph, origin, direction_world) - if hit and hit_obj == obj: - xyz_map[y, x, :] = location - - np.savez_compressed(filepath, xyz_map=xyz_map) - print(f"Pixel-to-3D lookup saved to {filepath} (shape {res_y}x{res_x})") - -def export_all(): - # Configuration - output_dir = bpy.path.abspath("//exported_data/") - os.makedirs(output_dir, exist_ok=True) - - obj = bpy.context.active_object # or specify name - cam = bpy.data.objects.get("Camera") - if not cam: - cam = bpy.context.scene.camera - if not obj or not cam: - raise RuntimeError("Select mesh and ensure camera exists.") - - # 1. Mesh data - export_mesh_data(obj, os.path.join(output_dir, "mesh.npz")) - - # 2. Camera intrinsics (K matrix) and resolution - K, w, h = export_camera_intrinsics(cam, bpy.context.scene) - np.savez(os.path.join(output_dir, "camera_intrinsics.npz"), K=K, width=w, height=h) - - # 3. Camera pose (world-to-camera) - export_camera_pose(cam, os.path.join(output_dir, "world_to_cam.npy")) - - # 4. Optional: precomputed pixel-to-3D map (fastest) - export_pixel_to_3d_lookup(obj, cam, bpy.context.scene, - os.path.join(output_dir, "pixel_to_xyz.npz")) - - print("Export complete.") - -if __name__ == "__main__": - export_all() \ No newline at end of file diff --git a/tools/3d_patch_generation_blender/captured_image.png b/tools/3d_patch_generation_blender/captured_image.png deleted file mode 100644 index cb97cc9..0000000 Binary files a/tools/3d_patch_generation_blender/captured_image.png and /dev/null differ diff --git a/tools/3d_patch_generation_blender/checker_patch_dd02c.py b/tools/3d_patch_generation_blender/checker_patch_dd02c.py deleted file mode 100644 index f8d0aa8..0000000 --- a/tools/3d_patch_generation_blender/checker_patch_dd02c.py +++ /dev/null @@ -1,585 +0,0 @@ -""" -Texture Generator for Arbitrary 3D Surfaces (Blender Script) -- Mode: "patches" (black/white checker patches) or "dots" (circular markers) -- Mesh UVs are filled white so that non‑visible parts become white. -- Texture background (no‑mesh areas) stays black. -""" - -import bpy -import numpy as np -import os -import time -from mathutils import Vector - -# ---------------------------------------------------------------------- -# Common Utilities -# ---------------------------------------------------------------------- -def draw_cross(pixels, width, height, cx_uv, cy_uv, length_px, thickness_px, color): - """ - Draw a cross (plus sign) in pixel space. - cx_uv, cy_uv: centre of cross in UV coordinates (0‑1), origin at bottom‑left. - length_px: total length of each arm (pixels). - thickness_px: line thickness (odd number gives symmetric result). - """ - # Convert UV centre to pixel coordinates (y from bottom) - cx = int(round(cx_uv * (width - 1))) - cy = int(round(cy_uv * (height - 1))) - - half_len = length_px // 2 - half_thick = thickness_px // 2 - - # Horizontal arm - x0 = max(0, cx - half_len) - x1 = min(width - 1, cx + half_len) - y0 = max(0, cy - half_thick) - y1 = min(height - 1, cy + half_thick) - for y in range(y0, y1 + 1): - for x in range(x0, x1 + 1): - idx = (y * width + x) * 4 - pixels[idx:idx+4] = color - - # Vertical arm - x0 = max(0, cx - half_thick) - x1 = min(width - 1, cx + half_thick) - y0 = max(0, cy - half_len) - y1 = min(height - 1, cy + half_len) - for y in range(y0, y1 + 1): - for x in range(x0, x1 + 1): - idx = (y * width + x) * 4 - pixels[idx:idx+4] = color - -def fill_background(pixels, width, height, color): - """Fill the entire image with a solid RGBA color (0-1 floats).""" - r, g, b, a = color - for y in range(height): - for x in range(width): - idx = (y * width + x) * 4 - pixels[idx] = r - pixels[idx+1] = g - pixels[idx+2] = b - pixels[idx+3] = a - -def fill_polygon(image, pixels, width, height, polygon, color): - """Fill a polygon in the pixel array (flattened, RGBA).""" - poly = [(int(round(x)), int(round(y))) for x, y in polygon] - if len(poly) < 3: - return - min_x = max(0, min(p[0] for p in poly)) - max_x = min(width-1, max(p[0] for p in poly)) - min_y = max(0, min(p[1] for p in poly)) - max_y = min(height-1, max(p[1] for p in poly)) - for y in range(min_y, max_y + 1): - intersections = [] - for i in range(len(poly)): - x1, y1 = poly[i] - x2, y2 = poly[(i+1) % len(poly)] - if y1 == y2: - continue - if (y1 <= y < y2) or (y2 <= y < y1): - t = (y - y1) / (y2 - y1) - x = x1 + t * (x2 - x1) - intersections.append(x) - intersections.sort() - for k in range(0, len(intersections), 2): - if k+1 >= len(intersections): - break - x_start = int(round(intersections[k])) - x_end = int(round(intersections[k+1])) - x_start = max(0, min(width-1, x_start)) - x_end = max(0, min(width-1, x_end)) - for x in range(x_start, x_end + 1): - idx = (y * width + x) * 4 - pixels[idx:idx+4] = color - - -def fill_polygon_value(grid, width, height, polygon, value): - """Fill a polygon into a 2D scalar grid.""" - poly = [(int(round(x)), int(round(y))) for x, y in polygon] - if len(poly) < 3: - return - min_x = max(0, min(p[0] for p in poly)) - max_x = min(width - 1, max(p[0] for p in poly)) - min_y = max(0, min(p[1] for p in poly)) - max_y = min(height - 1, max(p[1] for p in poly)) - for y in range(min_y, max_y + 1): - intersections = [] - for i in range(len(poly)): - x1, y1 = poly[i] - x2, y2 = poly[(i + 1) % len(poly)] - if y1 == y2: - continue - if (y1 <= y < y2) or (y2 <= y < y1): - t = (y - y1) / (y2 - y1) - x = x1 + t * (x2 - x1) - intersections.append(x) - intersections.sort() - for k in range(0, len(intersections), 2): - if k + 1 >= len(intersections): - break - x_start = int(round(intersections[k])) - x_end = int(round(intersections[k + 1])) - x_start = max(0, min(width - 1, x_start)) - x_end = max(0, min(width - 1, x_end)) - grid[y, x_start:x_end + 1] = value - -def draw_ellipse(pixels, width, height, cx, cy, a, b, angle, color): - """Filled ellipse with semi‑axes a,b (pixels), rotation angle (rad).""" - cos_a = np.cos(angle) - sin_a = np.sin(angle) - rx = abs(a * cos_a) + abs(b * sin_a) - ry = abs(a * sin_a) + abs(b * cos_a) - x_min = int(max(0, cx - rx)) - x_max = int(min(width-1, cx + rx)) - y_min = int(max(0, cy - ry)) - y_max = int(min(height-1, cy + ry)) - for y in range(y_min, y_max + 1): - for x in range(x_min, x_max + 1): - dx = x - cx - dy = y - cy - u = dx * cos_a + dy * sin_a - v = -dx * sin_a + dy * cos_a - if (u/a)**2 + (v/b)**2 <= 1.0: - idx = (y * width + x) * 4 - pixels[idx:idx+4] = color - -def get_uv_from_face(obj, face_index, location): - """World‑space location → UV coordinate on the mesh face.""" - me = obj.data - local_loc = obj.matrix_world.inverted() @ location - poly = me.polygons[face_index] - vert_indices = poly.vertices - loop_indices = poly.loop_indices - if len(vert_indices) < 3: - return None - v0 = me.vertices[vert_indices[0]].co - v1 = me.vertices[vert_indices[1]].co - v2 = me.vertices[vert_indices[2]].co - uv_layer = me.uv_layers.active.data - uv0 = uv_layer[loop_indices[0]].uv - uv1 = uv_layer[loop_indices[1]].uv - uv2 = uv_layer[loop_indices[2]].uv - def tri_area(a, b, c): - return 0.5 * ((b - a).cross(c - a)).length - total_area = tri_area(v0, v1, v2) - if total_area == 0: - return None - w0 = tri_area(local_loc, v1, v2) / total_area - w1 = tri_area(local_loc, v2, v0) / total_area - w2 = 1.0 - w0 - w1 - uv = w0 * uv0 + w1 * uv1 + w2 * uv2 - return Vector((uv.x, uv.y)) - -def raycast_from_camera(scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height): - """Camera raycast through normalised sensor coordinates (0‑1).""" - sx = (u_ndc - 0.5) * frame_width - sy = (v_ndc - 0.5) * frame_height - direction_cam = Vector((sx, sy, -1.0)) - direction_world = cam.matrix_world.to_quaternion() @ direction_cam - direction_world.normalize() - origin = cam.matrix_world.translation - hit, location, normal, face_index, hit_obj, matrix = scene.ray_cast( - depsgraph, origin, direction_world) - if hit and hit_obj == obj: - return location, face_index - return None, None - -def rasterize_mesh_uvs(obj, pixels, tex_width, tex_height, color=(1,1,1,1)): - """ - Fill all triangles of the active UV map with 'color' (default white). - This marks every part of the mesh, visible or not, as white. - """ - me = obj.data - uv_layer = me.uv_layers.active - if not uv_layer: - print("Warning: No active UV layer – mesh rasterization skipped.") - return - uv_data = uv_layer.data - for poly in me.polygons: - # Get loop indices for this polygon - loops = poly.loop_indices - # Build UV polygon - uv_poly = [] - for loop_idx in loops: - uv = uv_data[loop_idx].uv - uv_poly.append((uv.x, uv.y)) - # Convert UV (0‑1) to pixel coordinates - pixel_poly = [(u * (tex_width - 1), v * (tex_height - 1)) for u, v in uv_poly] - fill_polygon(None, pixels, tex_width, tex_height, pixel_poly, color) - - -def rasterize_mesh_mask(obj, tex_width, tex_height): - """Rasterize the active UV map to a boolean mesh mask.""" - mask = np.zeros((tex_height, tex_width), dtype=bool) - me = obj.data - uv_layer = me.uv_layers.active - if not uv_layer: - return mask - uv_data = uv_layer.data - for poly in me.polygons: - uv_poly = [] - for loop_idx in poly.loop_indices: - uv = uv_data[loop_idx].uv - uv_poly.append((uv.x, uv.y)) - pixel_poly = [(u * (tex_width - 1), v * (tex_height - 1)) for u, v in uv_poly] - fill_polygon_value(mask, tex_width, tex_height, pixel_poly, True) - return mask - - -def paint_mask_color(pixels, width, height, mask, color): - """Paint all True pixels of a boolean mask with an RGBA color.""" - r, g, b, a = color - ys, xs = np.nonzero(mask) - for y, x in zip(ys, xs): - idx = (y * width + x) * 4 - pixels[idx:idx+4] = (r, g, b, a) - - -def complete_checker_gaps(pixels, width, height, mesh_mask, patch_state, max_gap_px=6): - """ - Fill small white slivers inside the mesh by expanding nearby checker colors. - Only grows locally (max_gap_px) so non-visible mesh areas remain white. - patch_state: -1 = unassigned, 0 = black patch, 1 = white patch - """ - for _ in range(max_gap_px): - candidate = mesh_mask & (patch_state < 0) - if not np.any(candidate): - break - - black_counts = np.zeros_like(patch_state, dtype=np.uint8) - white_counts = np.zeros_like(patch_state, dtype=np.uint8) - - for dy in (-1, 0, 1): - for dx in (-1, 0, 1): - if dx == 0 and dy == 0: - continue - - src_y0 = max(0, -dy) - src_y1 = height - max(0, dy) - src_x0 = max(0, -dx) - src_x1 = width - max(0, dx) - dst_y0 = max(0, dy) - dst_y1 = height - max(0, -dy) - dst_x0 = max(0, dx) - dst_x1 = width - max(0, -dx) - - neighbor_vals = patch_state[src_y0:src_y1, src_x0:src_x1] - black_counts[dst_y0:dst_y1, dst_x0:dst_x1] += (neighbor_vals == 0) - white_counts[dst_y0:dst_y1, dst_x0:dst_x1] += (neighbor_vals == 1) - - fillable = candidate & ((black_counts + white_counts) > 0) - if not np.any(fillable): - break - - patch_state[fillable] = np.where( - black_counts[fillable] > white_counts[fillable], 0, 1 - ).astype(np.int8) - - paint_mask_color(pixels, width, height, patch_state == 0, (0.0, 0.0, 0.0, 1.0)) - paint_mask_color(pixels, width, height, patch_state == 1, (1.0, 1.0, 1.0, 1.0)) - -def checker_patch_color(i, j): - """Return a black/white checker color for patch index (i, j).""" - if (i + j) % 2 == 0: - return np.array([1.0, 1.0, 1.0], dtype=np.float32) - return np.array([0.0, 0.0, 0.0], dtype=np.float32) - -# ---------------------------------------------------------------------- -# Mode 1: Colored Patches -# ---------------------------------------------------------------------- -def generate_patch_texture( - obj, cam, scene, depsgraph, - nx=40, ny=30, r=0.05, - texture_width=2048, texture_height=2048, - output_path="//patch_texture.png",uv_centre=None, cross_length=40, cross_thickness=3, cross_color=(1,0,0,1), - gap_fill_px=6 -): - cam_data = cam.data - render = scene.render - res_x = render.resolution_x - res_y = render.resolution_y - if cam_data.sensor_fit == 'HORIZONTAL': - sensor_width = cam_data.sensor_width - sensor_height = sensor_width / (res_x / res_y) - else: - sensor_height = cam_data.sensor_height - sensor_width = sensor_height * (res_x / res_y) - focal = cam_data.lens - frame_width = sensor_width / focal - frame_height = sensor_height / focal - - patch_uv_polygons = [] - patch_colors = np.zeros((ny, nx, 3), dtype=np.float32) - - print(f"Raycasting {nx*ny} patches...") - for j in range(ny): - v_start = j / ny - v_end = (j+1) / ny - for i in range(nx): - u_start = i / nx - u_end = (i+1) / nx - corners_ndc = [(u_start, v_start), (u_end, v_start), - (u_end, v_end), (u_start, v_end)] - uv_poly = [] - for (u_ndc, v_ndc) in corners_ndc: - loc, face_idx = raycast_from_camera(scene, depsgraph, cam, obj, - u_ndc, v_ndc, frame_width, frame_height) - if loc is not None: - uv = get_uv_from_face(obj, face_idx, loc) - if uv is not None: - uv_poly.append(uv) - if len(uv_poly) < 3: - patch_uv_polygons.append(None) - continue - patch_uv_polygons.append(uv_poly) - - patch_colors[j, i] = checker_patch_color(i, j) - - # Create image – start with black background - img_name = "PatchTexture" - if img_name in bpy.data.images: - bpy.data.images.remove(bpy.data.images[img_name]) - image = bpy.data.images.new(img_name, width=texture_width, height=texture_height, alpha=True) - pixels = list(image.pixels) - mesh_mask = rasterize_mesh_mask(obj, texture_width, texture_height) - patch_state = np.full((texture_height, texture_width), -1, dtype=np.int8) - - # Fill entire image with black (no mesh areas) - fill_background(pixels, texture_width, texture_height, (0,0,0,1)) - - # Rasterize all mesh UVs in white (visible + non‑visible parts) - paint_mask_color(pixels, texture_width, texture_height, mesh_mask, (1,1,1,1)) - - # Now draw the visible patches on top - idx = 0 - for j in range(ny): - for i in range(nx): - uv_poly = patch_uv_polygons[idx]; idx += 1 - if uv_poly is None: continue - pixel_poly = [(u*(texture_width-1), v*(texture_height-1)) for u,v in uv_poly] - color = patch_colors[j,i] - rgba = (color[0], color[1], color[2], 1.0) - fill_polygon(image, pixels, texture_width, texture_height, pixel_poly, rgba) - fill_polygon_value( - patch_state, - texture_width, - texture_height, - pixel_poly, - 1 if np.mean(color) >= 0.5 else 0, - ) - - # Fill small steep-edge gaps from neighboring checker cells. - complete_checker_gaps( - pixels, texture_width, texture_height, mesh_mask, patch_state, max_gap_px=gap_fill_px - ) - - # (after drawing all patches) - if uv_centre is not None: - draw_cross(pixels, texture_width, texture_height, - uv_centre[0], uv_centre[1], - cross_length, cross_thickness, cross_color) - - image.pixels[:] = pixels - image.update() - abs_path = bpy.path.abspath(output_path) - image.filepath_raw = abs_path - image.file_format = 'PNG' - image.save() - print(f"Patch texture saved to {abs_path}") - return image - -# ---------------------------------------------------------------------- -# Mode 2: Circular Dot Markers -# ---------------------------------------------------------------------- -def generate_dot_texture( - obj, cam, scene, depsgraph, - dot_diameter_pixels=12, - spacing_pixels=40, - dot_color=(0,0,0,1), # RGBA 0-1 - texture_width=2048, texture_height=2048, - output_path="//dot_texture.png",uv_centre=None, cross_length=40, cross_thickness=3, cross_color=(1,0,0,1) -): - render = scene.render - res_x = render.resolution_x - res_y = render.resolution_y - cam_data = cam.data - if cam_data.sensor_fit == 'HORIZONTAL': - sensor_width = cam_data.sensor_width - sensor_height = sensor_width / (res_x / res_y) - else: - sensor_height = cam_data.sensor_height - sensor_width = sensor_height * (res_x / res_y) - focal = cam_data.lens - frame_width = sensor_width / focal - frame_height = sensor_height / focal - - margin = spacing_pixels // 2 - x_centers = np.arange(margin, res_x, spacing_pixels) - y_centers = np.arange(margin, res_y, spacing_pixels) - offset_pix = max(2, dot_diameter_pixels // 4) - du_ndc = offset_pix / res_x - dv_ndc = offset_pix / res_y - - dots = [] # (cx_uv, cy_uv, a_uv, b_uv, angle_uv) - print(f"Raycasting {len(x_centers)*len(y_centers)} dot centers...") - for y_pix in y_centers: - v_ndc = y_pix / res_y - for x_pix in x_centers: - u_ndc = x_pix / res_x - center_loc, face_idx = raycast_from_camera(scene, depsgraph, cam, obj, - u_ndc, v_ndc, frame_width, frame_height) - if center_loc is None: continue - center_uv = get_uv_from_face(obj, face_idx, center_loc) - if center_uv is None: continue - - offsets = [ - ( u_ndc + du_ndc, v_ndc), - ( u_ndc - du_ndc, v_ndc), - ( u_ndc, v_ndc + dv_ndc), - ( u_ndc, v_ndc - dv_ndc) - ] - uv_offsets = [] - valid = True - for (u_off, v_off) in offsets: - loc_off, f_idx_off = raycast_from_camera(scene, depsgraph, cam, obj, - u_off, v_off, frame_width, frame_height) - if loc_off is None: - valid = False; break - uv_off = get_uv_from_face(obj, f_idx_off, loc_off) - if uv_off is None: - valid = False; break - uv_offsets.append(uv_off) - if not valid: continue - - scale_u = 1.0 / res_x - scale_v = 1.0 / res_y - du_dx = (uv_offsets[0] - uv_offsets[1]) / (2 * offset_pix * scale_u) - dv_dx = (uv_offsets[2] - uv_offsets[3]) / (2 * offset_pix * scale_v) - r_pix = dot_diameter_pixels / 2.0 - a_uv = r_pix * du_dx.length * scale_u - b_uv = r_pix * dv_dx.length * scale_v - angle = np.arctan2(du_dx.y, du_dx.x) if du_dx.length > 0 else 0.0 - dots.append((center_uv.x, center_uv.y, a_uv, b_uv, angle)) - - print(f"Found {len(dots)} valid dots.") - - if uv_centre is not None: - draw_cross(pixels, texture_width, texture_height, - uv_centre[0], uv_centre[1], - cross_length, cross_thickness, cross_color) - - # Create image - img_name = "DotTexture" - if img_name in bpy.data.images: - bpy.data.images.remove(bpy.data.images[img_name]) - image = bpy.data.images.new(img_name, width=texture_width, height=texture_height, alpha=True) - pixels = list(image.pixels) - - # Black background (no mesh) - fill_background(pixels, texture_width, texture_height, (0,0,0,1)) - # White for all mesh areas - rasterize_mesh_uvs(obj, pixels, texture_width, texture_height, color=(1,1,1,1)) - # Draw dots - for (cx_uv, cy_uv, a_uv, b_uv, angle_uv) in dots: - cx_px = cx_uv * (texture_width - 1) - cy_px = cy_uv * (texture_height - 1) - a_px = a_uv * texture_width - b_px = b_uv * texture_height - draw_ellipse(pixels, texture_width, texture_height, - cx_px, cy_px, a_px, b_px, angle_uv, dot_color) - - image.pixels[:] = pixels - image.update() - abs_path = bpy.path.abspath(output_path) - image.filepath_raw = abs_path - image.file_format = 'PNG' - image.save() - print(f"Dot texture saved to {abs_path}") - return image - -# ---------------------------------------------------------------------- -# Main -# ---------------------------------------------------------------------- -def main(): - # ==================== CONFIGURATION ==================== - MODE = "patches" - OBJ_NAME = "FingerMesh" - CAM_NAME = "Camera" - TEXTURE_WIDTH = 4096 - TEXTURE_HEIGHT = 4096 - OUTPUT_PATH = "//" + ("patch_texture.png" if MODE=="patches" else "dot_texture.png") - - # Patch mode - PATCH_NX = 40; PATCH_NY = 40; COLOR_R = 0.05 - # Dot mode - DOT_DIAMETER_PIX = 12; DOT_SPACING_PIX = 40; DOT_COLOR = (0,0,0,1) - # Cross - CROSS_LENGTH_PX = 40 - CROSS_THICKNESS_PX = 3 - CROSS_COLOR = (1, 0, 0, 1) # red cross - # ====================================================== - - start_time = time.time() - scene = bpy.context.scene - depsgraph = bpy.context.evaluated_depsgraph_get() - - if OBJ_NAME: - obj = bpy.data.objects.get(OBJ_NAME) - else: - obj = bpy.context.active_object - if obj is None or obj.type != 'MESH': - raise ValueError("No valid mesh object found.") - - cam = bpy.data.objects.get(CAM_NAME) if CAM_NAME else scene.camera - if cam is None or cam.type != 'CAMERA': - raise ValueError("No camera found.") - - if not obj.data.uv_layers.active: - obj.data.uv_layers.new(name="UVMap") - - # Compute UV centre - uv_layer = obj.data.uv_layers.active - uv_coords = np.array([uv_layer.data[i].uv.to_tuple() for i in range(len(uv_layer.data))]) - uv_min = uv_coords.min(axis=0) - uv_max = uv_coords.max(axis=0) - uv_centre = (uv_min + uv_max) / 2.0 - print(f"UV centre: ({uv_centre[0]:.4f}, {uv_centre[1]:.4f})") - - - # Compute UV bounding box centre - me = obj.data - uv_layer = me.uv_layers.active - uv_coords = np.array([uv_layer.data[i].uv.to_tuple() for i in range(len(uv_layer.data))]) - uv_min = uv_coords.min(axis=0) - uv_max = uv_coords.max(axis=0) - uv_centre = (uv_min + uv_max) / 2.0 - print(f"UV centre: {uv_centre[0]:.4f}, {uv_centre[1]:.4f}") - - if MODE == "patches": - generate_patch_texture( - obj, cam, scene, depsgraph, - nx=PATCH_NX, ny=PATCH_NY, r=COLOR_R, - texture_width=TEXTURE_WIDTH, texture_height=TEXTURE_HEIGHT, - output_path=OUTPUT_PATH, - uv_centre=uv_centre, - cross_length=CROSS_LENGTH_PX, - cross_thickness=CROSS_THICKNESS_PX, - cross_color=CROSS_COLOR - ) - elif MODE == "dots": - generate_dot_texture( - obj, cam, scene, depsgraph, - dot_diameter_pixels=DOT_DIAMETER_PIX, - spacing_pixels=DOT_SPACING_PIX, - dot_color=DOT_COLOR, - texture_width=TEXTURE_WIDTH, texture_height=TEXTURE_HEIGHT, - output_path=OUTPUT_PATH, - uv_centre=uv_centre, - cross_length=CROSS_LENGTH_PX, - cross_thickness=CROSS_THICKNESS_PX, - cross_color=CROSS_COLOR - ) - - print(f"Completed in {time.time()-start_time:.2f} seconds.") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/tools/3d_patch_generation_blender/dd02c_0607.blend b/tools/3d_patch_generation_blender/dd02c_0607.blend deleted file mode 100644 index 6abfed0..0000000 Binary files a/tools/3d_patch_generation_blender/dd02c_0607.blend and /dev/null differ diff --git a/tools/3d_patch_generation_blender/dd02c_0607.blend1 b/tools/3d_patch_generation_blender/dd02c_0607.blend1 deleted file mode 100644 index d140d5a..0000000 Binary files a/tools/3d_patch_generation_blender/dd02c_0607.blend1 and /dev/null differ diff --git a/tools/3d_patch_generation_blender/mapto3d_video.py b/tools/3d_patch_generation_blender/mapto3d_video.py deleted file mode 100644 index e14d108..0000000 --- a/tools/3d_patch_generation_blender/mapto3d_video.py +++ /dev/null @@ -1,411 +0,0 @@ -import argparse -import os -import time - -import cv2 -import numpy as np - - -def load_exported_data(data_dir): - """Load Blender-exported mesh/camera files.""" - mesh = np.load(os.path.join(data_dir, "mesh.npz")) - cam_data = np.load(os.path.join(data_dir, "camera_intrinsics.npz")) - - xyz_map_path = os.path.join(data_dir, "pixel_to_xyz.npz") - if not os.path.exists(xyz_map_path): - raise FileNotFoundError( - f"Real-time video viewing requires precomputed pixel_to_xyz.npz: {xyz_map_path}" - ) - - xyz_map = np.load(xyz_map_path)["xyz_map"] - width = int(cam_data["width"]) - height = int(cam_data["height"]) - if xyz_map.shape[:2] != (height, width): - raise ValueError( - f"pixel_to_xyz shape {xyz_map.shape[:2]} does not match camera size {(height, width)}" - ) - - return { - "vertices": mesh["vertices"], - "faces": mesh["faces"], - "uv_data": mesh["uv_data"], - "loop_to_vertex": mesh["loop_to_vertex"], - "K": cam_data["K"], - "width": width, - "height": height, - "world_to_cam": np.load(os.path.join(data_dir, "world_to_cam.npy")), - "xyz_map": xyz_map, - } - - -def prepare_static_points(data, display_stride=4): - """ - Prepare fixed 3D points from the precomputed pixel-to-3D map. - - The geometry does not change between video frames, so real-time viewing only - needs to update point colors. - """ - xyz_map = data["xyz_map"] - valid_mask = ~np.isnan(xyz_map[:, :, 0]) - valid_points = xyz_map[valid_mask].reshape(-1, 3) - if len(valid_points) == 0: - raise ValueError("pixel_to_xyz.npz contains no valid mesh hit points.") - - display_stride = max(1, int(display_stride)) - display_indices = np.arange(len(valid_points))[::display_stride] - return valid_mask, display_indices, valid_points[display_indices] - - -def frame_to_rgb_colors(frame_bgr, data, valid_mask, display_indices, flip_vertical=True): - """Convert a video frame to RGB point colors aligned with the exported map.""" - target_w = int(data["width"]) - target_h = int(data["height"]) - h, w = frame_bgr.shape[:2] - if (w, h) != (target_w, target_h): - frame_bgr = cv2.resize(frame_bgr, (target_w, target_h), interpolation=cv2.INTER_LINEAR) - - frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) - if flip_vertical: - frame_rgb = np.flipud(frame_rgb).copy() - - colors = frame_rgb.astype(np.float64) / 255.0 - valid_colors = colors[valid_mask].reshape(-1, 3) - return valid_colors[display_indices] - - -def init_open3d_viewer(points, colors): - import open3d as o3d - - vis = o3d.visualization.Visualizer() - vis.create_window(window_name="Real-time 3D Point Cloud", width=1280, height=720) - - pcd = o3d.geometry.PointCloud() - pcd.points = o3d.utility.Vector3dVector(points.astype(np.float64)) - pcd.colors = o3d.utility.Vector3dVector(colors.astype(np.float64)) - vis.add_geometry(pcd) - - render_option = vis.get_render_option() - render_option.point_size = 2.0 - render_option.background_color = np.array([0.0, 0.0, 0.0]) - - vis.poll_events() - vis.update_renderer() - return vis, pcd, o3d - - -def save_colored_pointcloud(points, colors, output_path): - """Save displayed points as a PLY file.""" - valid_colors = (colors * 255).clip(0, 255).astype(np.uint8) - with open(output_path, "w", encoding="utf-8") as f: - f.write("ply\n") - f.write("format ascii 1.0\n") - f.write(f"element vertex {len(points)}\n") - f.write("property float x\n") - f.write("property float y\n") - f.write("property float z\n") - f.write("property uchar red\n") - f.write("property uchar green\n") - f.write("property uchar blue\n") - f.write("end_header\n") - for pt, col in zip(points, valid_colors): - f.write(f"{pt[0]} {pt[1]} {pt[2]} {col[0]} {col[1]} {col[2]}\n") - print(f"Saved point cloud to {output_path}") - - -def view_video_pointcloud( - video_path, - data, - display_stride=4, - frame_step=1, - flip_vertical=True, - loop=False, - save_snapshot_path=None, -): - """Open a video and show it as a real-time colored 3D point cloud.""" - cap = cv2.VideoCapture(video_path) - if not cap.isOpened(): - raise FileNotFoundError(f"Could not open video: {video_path}") - - valid_mask, display_indices, display_points = prepare_static_points( - data, - display_stride=display_stride, - ) - - ok, frame = cap.read() - if not ok: - cap.release() - raise RuntimeError(f"Could not read first frame from video: {video_path}") - - display_colors = frame_to_rgb_colors( - frame, - data, - valid_mask, - display_indices, - flip_vertical=flip_vertical, - ) - vis, pcd, o3d = init_open3d_viewer(display_points, display_colors) - - print("Real-time point cloud viewer started.") - print("Close the Open3D window or press Ctrl+C in the terminal to stop.") - print(f"Displayed points: {len(display_points)} (stride={display_stride})") - - frame_idx = 0 - shown_frames = 0 - t0 = time.perf_counter() - - try: - while True: - alive = vis.poll_events() - if not alive: - break - - ok, frame = cap.read() - if not ok: - if loop: - cap.set(cv2.CAP_PROP_POS_FRAMES, 0) - continue - break - - frame_idx += 1 - if frame_idx % max(1, frame_step) != 0: - continue - - display_colors = frame_to_rgb_colors( - frame, - data, - valid_mask, - display_indices, - flip_vertical=flip_vertical, - ) - pcd.colors = o3d.utility.Vector3dVector(display_colors) - vis.update_geometry(pcd) - vis.update_renderer() - - shown_frames += 1 - if shown_frames % 30 == 0: - elapsed = max(time.perf_counter() - t0, 1e-6) - print(f"Displayed {shown_frames} frames, viewer FPS={shown_frames / elapsed:.1f}") - - finally: - if save_snapshot_path: - save_colored_pointcloud(display_points, display_colors, save_snapshot_path) - cap.release() - vis.destroy_window() - - -def parse_args(): - parser = argparse.ArgumentParser(description="View a video as a real-time 3D point cloud.") - parser.add_argument("--data-dir", default="./exported_data", help="Blender exported_data folder") - parser.add_argument("--video", default="20260602-094916.mp4", help="Input video path") - parser.add_argument("--display-stride", type=int, default=4, - help="Show every Nth valid point. Increase for faster viewing.") - parser.add_argument("--frame-step", type=int, default=1, - help="Process every Nth video frame.") - parser.add_argument("--no-flip-vertical", action="store_true", - help="Disable vertical flip before color mapping.") - parser.add_argument("--loop", action="store_true", help="Loop video playback.") - parser.add_argument("--save-snapshot", default="", - help="Optional PLY path for saving the last displayed frame.") - return parser.parse_args() - - -if __name__ == "__main__": - args = parse_args() - exported = load_exported_data(args.data_dir) - view_video_pointcloud( - args.video, - exported, - display_stride=args.display_stride, - frame_step=args.frame_step, - flip_vertical=not args.no_flip_vertical, - loop=args.loop, - save_snapshot_path=args.save_snapshot or None, - ) -import numpy as np -import cv2 -import os - -def load_exported_data(data_dir): - """Load all exported files.""" - # Mesh - mesh = np.load(os.path.join(data_dir, "mesh.npz")) - vertices = mesh['vertices'] - faces = mesh['faces'] - uv_data = mesh['uv_data'] - loop_to_vertex = mesh['loop_to_vertex'] - - # Camera intrinsics - cam_data = np.load(os.path.join(data_dir, "camera_intrinsics.npz")) - K = cam_data['K'] - width = int(cam_data['width']) - height = int(cam_data['height']) - - # Camera pose (world-to-camera) - world_to_cam = np.load(os.path.join(data_dir, "world_to_cam.npy")) - - # Precomputed pixel-to-3D map (optional) - xyz_map_path = os.path.join(data_dir, "pixel_to_xyz.npz") - if os.path.exists(xyz_map_path): - xyz_map = np.load(xyz_map_path)['xyz_map'] - else: - xyz_map = None - - return { - 'vertices': vertices, - 'faces': faces, - 'uv_data': uv_data, - 'loop_to_vertex': loop_to_vertex, - 'K': K, - 'width': width, - 'height': height, - 'world_to_cam': world_to_cam, - 'xyz_map': xyz_map - } - -def ray_intersect_triangle(ray_origin, ray_dir, v0, v1, v2): - """Möller–Trumbore intersection algorithm. Returns t (distance) or None.""" - epsilon = 1e-8 - edge1 = v1 - v0 - edge2 = v2 - v0 - h = np.cross(ray_dir, edge2) - a = np.dot(edge1, h) - if abs(a) < epsilon: - return None - f = 1.0 / a - s = ray_origin - v0 - u = f * np.dot(s, h) - if u < 0.0 or u > 1.0: - return None - q = np.cross(s, edge1) - v = f * np.dot(ray_dir, q) - if v < 0.0 or u + v > 1.0: - return None - t = f * np.dot(edge2, q) - if t > epsilon: - return t - return None - -def pixel_to_3d_ray(u, v, K, world_to_cam): - """ - Given pixel coordinates (u,v) and camera intrinsics K, - return a ray in world coordinates: origin and direction. - """ - # Invert K to get normalized image coordinates - fx = K[0,0]; fy = K[1,1]; cx = K[0,2]; cy = K[1,2] - x_norm = (u - cx) / fx - y_norm = (v - cy) / fy - - # Ray in camera space: (x_norm, y_norm, 1.0) - ray_cam = np.array([x_norm, y_norm, 1.0], dtype=np.float64) - - # Transform to world - cam_to_world = np.linalg.inv(world_to_cam) - R = cam_to_world[:3, :3] - t = cam_to_world[:3, 3] - ray_world_dir = R @ ray_cam - ray_world_dir /= np.linalg.norm(ray_world_dir) - ray_world_origin = t - - return ray_world_origin, ray_world_dir - -def map_image_to_3d(image_path, data, use_precomputed=True, flip_vertical=True): - """ - Map each pixel of the image to a 3D world point. - If use_precomputed and xyz_map exists, return that directly. - If `flip_vertical` is True, flip the loaded image top-to-bottom so its - row order matches the exported Blender camera/pixel mapping. - Otherwise, perform raycasting on the mesh. - Returns: - points: (H, W, 3) array of 3D points (NaN for misses). - colors: (H, W, 3) array of RGB colors from image. - """ - img = cv2.imread(image_path) - if img is None: - raise FileNotFoundError(f"Could not read image: {image_path}") - img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # to RGB - if flip_vertical: - img = np.flipud(img).copy() - h, w = img.shape[:2] - - if use_precomputed and data['xyz_map'] is not None: - # Use precomputed map (must match resolution) - if h != data['height'] or w != data['width']: - raise ValueError("Image resolution does not match precomputed map.") - points = data['xyz_map'].copy() - colors = img.astype(np.float32) / 255.0 - return points, colors - - # Otherwise, perform raycasting on the mesh - vertices = data['vertices'] - faces = data['faces'] - K = data['K'] - world_to_cam = data['world_to_cam'] - - points = np.full((h, w, 3), np.nan, dtype=np.float32) - colors = img.astype(np.float32) / 255.0 - - # For speed, we can use a bounding volume hierarchy (BVH). - # Here we implement a simple brute-force over all faces (slow for large meshes). - # In practice, precomputing the xyz_map is strongly recommended. - - for y in range(h): - for x in range(w): - origin, direction = pixel_to_3d_ray(x, y, K, world_to_cam) - best_t = np.inf - best_hit = None - for face in faces: - v0 = vertices[face[0]] - v1 = vertices[face[1]] - v2 = vertices[face[2]] - t = ray_intersect_triangle(origin, direction, v0, v1, v2) - if t is not None and t < best_t: - best_t = t - best_hit = origin + direction * t - if best_hit is not None: - points[y, x] = best_hit - - return points, colors - -def save_colored_pointcloud(points, colors, output_path): - """Save as PLY file (colored point cloud).""" - # Flatten valid points - mask = ~np.isnan(points[:,:,0]) - valid_points = points[mask].reshape(-1, 3) - valid_colors = (colors[mask].reshape(-1, 3) * 255).astype(np.uint8) - - with open(output_path, 'w') as f: - f.write("ply\n") - f.write("format ascii 1.0\n") - f.write(f"element vertex {len(valid_points)}\n") - f.write("property float x\n") - f.write("property float y\n") - f.write("property float z\n") - f.write("property uchar red\n") - f.write("property uchar green\n") - f.write("property uchar blue\n") - f.write("end_header\n") - for pt, col in zip(valid_points, valid_colors): - f.write(f"{pt[0]} {pt[1]} {pt[2]} {col[0]} {col[1]} {col[2]}\n") - print(f"Saved point cloud to {output_path}") - -# if __name__ == "__main__": -# data_dir = "./exported_data" # Path to exported files -# image_path = "captured_image_1.jpg" # Input camera image -# output_ply = "mapped_points.ply" - -# data = load_exported_data(data_dir) -# points, colors = map_image_to_3d( -# image_path, -# data, -# use_precomputed=True, -# flip_vertical=True, -# ) -# save_colored_pointcloud(points, colors, output_ply) - -# # Optional: Visualize using Open3D -# import open3d as o3d -# pcd = o3d.geometry.PointCloud() -# mask = ~np.isnan(points[:,:,0]) -# pcd.points = o3d.utility.Vector3dVector(points[mask].reshape(-1,3)) -# pcd.colors = o3d.utility.Vector3dVector(colors[mask].reshape(-1,3)) -# o3d.visualization.draw_geometries([pcd]) \ No newline at end of file diff --git a/tools/3d_patch_generation_blender/patch_texture.png b/tools/3d_patch_generation_blender/patch_texture.png deleted file mode 100644 index 7d3d9fa..0000000 Binary files a/tools/3d_patch_generation_blender/patch_texture.png and /dev/null differ diff --git a/tools/3d_patch_generation_blender/patch_texture_0607_X80Y45.png b/tools/3d_patch_generation_blender/patch_texture_0607_X80Y45.png deleted file mode 100644 index 830ec01..0000000 Binary files a/tools/3d_patch_generation_blender/patch_texture_0607_X80Y45.png and /dev/null differ diff --git a/tools/3d_patch_generation_blender/random_patch_dd02c.py b/tools/3d_patch_generation_blender/random_patch_dd02c.py deleted file mode 100644 index c3a282b..0000000 --- a/tools/3d_patch_generation_blender/random_patch_dd02c.py +++ /dev/null @@ -1,917 +0,0 @@ -""" -Texture Generator for Arbitrary 3D Surfaces (Blender Script) -- Mode: "patches" (colored patches), "checker" (black/white patches), - or "dots" (circular markers) -- Mesh UVs are filled white so that non‑visible parts become white. -- Texture background (no‑mesh areas) stays black. -""" - -import bpy -import numpy as np -import os -import time -from mathutils import Vector - -# ---------------------------------------------------------------------- -# Common Utilities -# ---------------------------------------------------------------------- -def draw_cross(pixels, width, height, cx_uv, cy_uv, length_px, thickness_px, color): - """ - Draw a cross (plus sign) in pixel space. - cx_uv, cy_uv: centre of cross in UV coordinates (0‑1), origin at bottom‑left. - length_px: total length of each arm (pixels). - thickness_px: line thickness (odd number gives symmetric result). - """ - # Convert UV centre to pixel coordinates (y from bottom) - cx = int(round(cx_uv * (width - 1))) - cy = int(round(cy_uv * (height - 1))) - - half_len = length_px // 2 - half_thick = thickness_px // 2 - - # Horizontal arm - x0 = max(0, cx - half_len) - x1 = min(width - 1, cx + half_len) - y0 = max(0, cy - half_thick) - y1 = min(height - 1, cy + half_thick) - for y in range(y0, y1 + 1): - for x in range(x0, x1 + 1): - idx = (y * width + x) * 4 - pixels[idx:idx+4] = color - - # Vertical arm - x0 = max(0, cx - half_thick) - x1 = min(width - 1, cx + half_thick) - y0 = max(0, cy - half_len) - y1 = min(height - 1, cy + half_len) - for y in range(y0, y1 + 1): - for x in range(x0, x1 + 1): - idx = (y * width + x) * 4 - pixels[idx:idx+4] = color - -def fill_background(pixels, width, height, color): - """Fill the entire image with a solid RGBA color (0-1 floats).""" - r, g, b, a = color - for y in range(height): - for x in range(width): - idx = (y * width + x) * 4 - pixels[idx] = r - pixels[idx+1] = g - pixels[idx+2] = b - pixels[idx+3] = a - -def fill_polygon(image, pixels, width, height, polygon, color): - """Fill a polygon in the pixel array (flattened, RGBA).""" - poly = [(int(round(x)), int(round(y))) for x, y in polygon] - if len(poly) < 3: - return - min_x = max(0, min(p[0] for p in poly)) - max_x = min(width-1, max(p[0] for p in poly)) - min_y = max(0, min(p[1] for p in poly)) - max_y = min(height-1, max(p[1] for p in poly)) - for y in range(min_y, max_y + 1): - intersections = [] - for i in range(len(poly)): - x1, y1 = poly[i] - x2, y2 = poly[(i+1) % len(poly)] - if y1 == y2: - continue - if (y1 <= y < y2) or (y2 <= y < y1): - t = (y - y1) / (y2 - y1) - x = x1 + t * (x2 - x1) - intersections.append(x) - intersections.sort() - for k in range(0, len(intersections), 2): - if k+1 >= len(intersections): - break - x_start = int(round(intersections[k])) - x_end = int(round(intersections[k+1])) - x_start = max(0, min(width-1, x_start)) - x_end = max(0, min(width-1, x_end)) - for x in range(x_start, x_end + 1): - idx = (y * width + x) * 4 - pixels[idx:idx+4] = color - -def fill_polygon_value(grid, width, height, polygon, value): - """Fill a polygon into a 2D scalar grid.""" - poly = [(int(round(x)), int(round(y))) for x, y in polygon] - if len(poly) < 3: - return - min_x = max(0, min(p[0] for p in poly)) - max_x = min(width - 1, max(p[0] for p in poly)) - min_y = max(0, min(p[1] for p in poly)) - max_y = min(height - 1, max(p[1] for p in poly)) - for y in range(min_y, max_y + 1): - intersections = [] - for i in range(len(poly)): - x1, y1 = poly[i] - x2, y2 = poly[(i + 1) % len(poly)] - if y1 == y2: - continue - if (y1 <= y < y2) or (y2 <= y < y1): - t = (y - y1) / (y2 - y1) - x = x1 + t * (x2 - x1) - intersections.append(x) - intersections.sort() - for k in range(0, len(intersections), 2): - if k + 1 >= len(intersections): - break - x_start = int(round(intersections[k])) - x_end = int(round(intersections[k + 1])) - x_start = max(0, min(width - 1, x_start)) - x_end = max(0, min(width - 1, x_end)) - grid[y, x_start:x_end + 1] = value - -def unwrap_periodic_values(values): - """ - Unwrap periodic UV coordinates so points straddling the 0/1 seam stay local. - """ - arr = np.asarray(values, dtype=np.float64).copy() - if arr.size == 0: - return arr - if arr.max() - arr.min() > 0.5: - arr[arr < 0.5] += 1.0 - return arr - -def wrap_uv_polygon_for_raster(uv_poly): - """ - Return a seam-aware version of a UV polygon. - - UV islands may cross the 0/1 seam. If we rasterize those raw coordinates as a - single polygon, Blender's repeated UVs turn into a long diagonal triangle - across the texture. We unwrap seam-crossing polygons first, then draw wrapped - copies in the texture tile(s) they overlap. - """ - if len(uv_poly) < 3: - return [] - - uv_arr = np.array([(float(u), float(v)) for u, v in uv_poly], dtype=np.float64) - uv_arr[:, 0] = unwrap_periodic_values(uv_arr[:, 0]) - uv_arr[:, 1] = unwrap_periodic_values(uv_arr[:, 1]) - return [tuple(pt) for pt in uv_arr] - -def uv_polygon_to_pixel_polygons(uv_poly, tex_width, tex_height): - """ - Convert one UV polygon into one or more pixel polygons, duplicating seam- - crossing polygons into adjacent tiles so rasterization stays local. - """ - unwrapped = wrap_uv_polygon_for_raster(uv_poly) - if len(unwrapped) < 3: - return [] - - u_vals = [u for u, _ in unwrapped] - v_vals = [v for _, v in unwrapped] - u_shift_start = int(np.floor(min(u_vals))) - u_shift_end = int(np.floor(max(u_vals))) - v_shift_start = int(np.floor(min(v_vals))) - v_shift_end = int(np.floor(max(v_vals))) - - pixel_polys = [] - for u_shift in range(u_shift_start, u_shift_end + 1): - for v_shift in range(v_shift_start, v_shift_end + 1): - shifted = [(u - u_shift, v - v_shift) for u, v in unwrapped] - shifted_u = [u for u, _ in shifted] - shifted_v = [v for _, v in shifted] - if max(shifted_u) < 0.0 or min(shifted_u) > 1.0: - continue - if max(shifted_v) < 0.0 or min(shifted_v) > 1.0: - continue - pixel_polys.append([ - (u * (tex_width - 1), v * (tex_height - 1)) - for u, v in shifted - ]) - return pixel_polys - -def draw_ellipse(pixels, width, height, cx, cy, a, b, angle, color): - """Filled ellipse with semi‑axes a,b (pixels), rotation angle (rad).""" - cos_a = np.cos(angle) - sin_a = np.sin(angle) - rx = abs(a * cos_a) + abs(b * sin_a) - ry = abs(a * sin_a) + abs(b * cos_a) - x_min = int(max(0, cx - rx)) - x_max = int(min(width-1, cx + rx)) - y_min = int(max(0, cy - ry)) - y_max = int(min(height-1, cy + ry)) - for y in range(y_min, y_max + 1): - for x in range(x_min, x_max + 1): - dx = x - cx - dy = y - cy - u = dx * cos_a + dy * sin_a - v = -dx * sin_a + dy * cos_a - if (u/a)**2 + (v/b)**2 <= 1.0: - idx = (y * width + x) * 4 - pixels[idx:idx+4] = color - -def get_uv_from_face(obj, face_index, location): - """World‑space location → UV coordinate on the mesh face.""" - me = obj.data - local_loc = obj.matrix_world.inverted() @ location - poly = me.polygons[face_index] - vert_indices = poly.vertices - loop_indices = poly.loop_indices - if len(vert_indices) < 3: - return None - v0 = me.vertices[vert_indices[0]].co - v1 = me.vertices[vert_indices[1]].co - v2 = me.vertices[vert_indices[2]].co - uv_layer = me.uv_layers.active.data - uv0 = uv_layer[loop_indices[0]].uv - uv1 = uv_layer[loop_indices[1]].uv - uv2 = uv_layer[loop_indices[2]].uv - def tri_area(a, b, c): - return 0.5 * ((b - a).cross(c - a)).length - total_area = tri_area(v0, v1, v2) - if total_area == 0: - return None - w0 = tri_area(local_loc, v1, v2) / total_area - w1 = tri_area(local_loc, v2, v0) / total_area - w2 = 1.0 - w0 - w1 - uv = w0 * uv0 + w1 * uv1 + w2 * uv2 - return Vector((uv.x, uv.y)) - -def raycast_from_camera(scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height): - """Camera raycast through normalised sensor coordinates (0‑1).""" - sx = (u_ndc - 0.5) * frame_width - sy = (v_ndc - 0.5) * frame_height - direction_cam = Vector((sx, sy, -1.0)) - direction_world = cam.matrix_world.to_quaternion() @ direction_cam - direction_world.normalize() - origin = cam.matrix_world.translation - hit, location, normal, face_index, hit_obj, matrix = scene.ray_cast( - depsgraph, origin, direction_world) - if hit and hit_obj == obj: - return location, face_index - return None, None - -def raycast_uv_from_camera(scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height): - """Camera raycast through normalized sensor coordinates and return UV.""" - loc, face_idx = raycast_from_camera( - scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height - ) - if loc is None: - return None - return get_uv_from_face(obj, face_idx, loc) - -def convex_hull_uv(points): - """Return UV points ordered around their 2D convex hull.""" - unwrapped = wrap_uv_polygon_for_raster([(p.x, p.y) for p in points]) - unique = sorted(set(unwrapped)) - if len(unique) < 3: - return None - - def cross(o, a, b): - return (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]) - - lower = [] - for p in unique: - while len(lower) >= 2 and cross(lower[-2], lower[-1], p) <= 0: - lower.pop() - lower.append(p) - - upper = [] - for p in reversed(unique): - while len(upper) >= 2 and cross(upper[-2], upper[-1], p) <= 0: - upper.pop() - upper.append(p) - - hull = lower[:-1] + upper[:-1] - if len(hull) < 3: - return None - return [Vector((u, v)) for u, v in hull] - -def sample_patch_uv_polygon( - scene, depsgraph, cam, obj, - u_start, u_end, v_start, v_end, - frame_width, frame_height, - edge_sample_grid=5 -): - """ - Build a UV polygon for one camera-space patch. - - Full interior cells use the four corners. Edge cells that only partially hit - the mesh are sampled more densely and converted to a convex hull so they are - not dropped just because some corners miss the silhouette. - """ - corners_ndc = [ - (u_start, v_start), - (u_end, v_start), - (u_end, v_end), - (u_start, v_end), - ] - - corner_uvs = [] - for u_ndc, v_ndc in corners_ndc: - uv = raycast_uv_from_camera( - scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height - ) - if uv is not None: - corner_uvs.append(uv) - - if len(corner_uvs) >= 3: - return corner_uvs - - sampled_uvs = list(corner_uvs) - for sy in range(edge_sample_grid): - v_ndc = v_start + (v_end - v_start) * sy / (edge_sample_grid - 1) - for sx in range(edge_sample_grid): - u_ndc = u_start + (u_end - u_start) * sx / (edge_sample_grid - 1) - uv = raycast_uv_from_camera( - scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height - ) - if uv is not None: - sampled_uvs.append(uv) - - return convex_hull_uv(sampled_uvs) - -def rasterize_mesh_uvs(obj, pixels, tex_width, tex_height, color=(1,1,1,1)): - """ - Fill all triangles of the active UV map with 'color' (default white). - This marks every part of the mesh, visible or not, as white. - """ - me = obj.data - uv_layer = me.uv_layers.active - if not uv_layer: - print("Warning: No active UV layer – mesh rasterization skipped.") - return - uv_data = uv_layer.data - for poly in me.polygons: - # Get loop indices for this polygon - loops = poly.loop_indices - # Build UV polygon - uv_poly = [] - for loop_idx in loops: - uv = uv_data[loop_idx].uv - uv_poly.append((uv.x, uv.y)) - # Convert UV (0‑1) to pixel coordinates - for pixel_poly in uv_polygon_to_pixel_polygons(uv_poly, tex_width, tex_height): - fill_polygon(None, pixels, tex_width, tex_height, pixel_poly, color) - -def rasterize_mesh_mask(obj, tex_width, tex_height): - """Rasterize the active UV map to a boolean mesh mask.""" - mask = np.zeros((tex_height, tex_width), dtype=bool) - me = obj.data - uv_layer = me.uv_layers.active - if not uv_layer: - return mask - uv_data = uv_layer.data - for poly in me.polygons: - uv_poly = [] - for loop_idx in poly.loop_indices: - uv = uv_data[loop_idx].uv - uv_poly.append((uv.x, uv.y)) - for pixel_poly in uv_polygon_to_pixel_polygons(uv_poly, tex_width, tex_height): - fill_polygon_value(mask, tex_width, tex_height, pixel_poly, True) - return mask - -def paint_mask_color(pixels, width, height, mask, color): - """Paint all True pixels of a boolean mask with an RGBA color.""" - r, g, b, a = color - ys, xs = np.nonzero(mask) - for y, x in zip(ys, xs): - idx = (y * width + x) * 4 - pixels[idx:idx+4] = (r, g, b, a) - -def complete_checker_gaps(pixels, width, height, mesh_mask, patch_state, max_gap_px=6): - """ - Fill small white slivers inside the mesh by expanding nearby checker colors. - - `patch_state`: -1 = unassigned, 0 = black patch, 1 = white patch. - """ - for _ in range(max_gap_px): - candidate = mesh_mask & (patch_state < 0) - if not np.any(candidate): - break - - black_counts = np.zeros_like(patch_state, dtype=np.uint8) - white_counts = np.zeros_like(patch_state, dtype=np.uint8) - - for dy in (-1, 0, 1): - for dx in (-1, 0, 1): - if dx == 0 and dy == 0: - continue - - src_y0 = max(0, -dy) - src_y1 = height - max(0, dy) - src_x0 = max(0, -dx) - src_x1 = width - max(0, dx) - dst_y0 = max(0, dy) - dst_y1 = height - max(0, -dy) - dst_x0 = max(0, dx) - dst_x1 = width - max(0, -dx) - - neighbor_vals = patch_state[src_y0:src_y1, src_x0:src_x1] - black_counts[dst_y0:dst_y1, dst_x0:dst_x1] += (neighbor_vals == 0) - white_counts[dst_y0:dst_y1, dst_x0:dst_x1] += (neighbor_vals == 1) - - fillable = candidate & ((black_counts + white_counts) > 0) - if not np.any(fillable): - break - - patch_state[fillable] = np.where( - black_counts[fillable] > white_counts[fillable], 0, 1 - ).astype(np.int8) - - paint_mask_color(pixels, width, height, patch_state == 0, (0.0, 0.0, 0.0, 1.0)) - paint_mask_color(pixels, width, height, patch_state == 1, (1.0, 1.0, 1.0, 1.0)) - -# ---------------------------------------------------------------------- -# Color generation (for patch mode) -# ---------------------------------------------------------------------- -def merge_intervals(intervals): - if not intervals: return [] - intervals.sort(key=lambda x: x[0]) - merged = [intervals[0]] - for current in intervals[1:]: - last = merged[-1] - if current[0] <= last[1]: - merged[-1] = (last[0], max(last[1], current[1])) - else: - merged.append(current) - return merged - -def get_allowed_intervals(neighbor_values, delta): - forbidden = [] - for c in neighbor_values: - low = max(0, c - delta) - high = min(1, c + delta) - forbidden.append((low, high)) - merged = merge_intervals(forbidden) - allowed = [] - if merged: - if merged[0][0] > 0: - allowed.append((0, merged[0][0])) - for i in range(1, len(merged)): - allowed.append((merged[i-1][1], merged[i][0])) - if merged[-1][1] < 1: - allowed.append((merged[-1][1], 1)) - else: - allowed.append((0, 1)) - return allowed - -def sample_from_allowed(allowed): - lengths = [end - start for start, end in allowed] - total_length = sum(lengths) - if total_length <= 0: return None - point = np.random.rand() * total_length - cumulative = 0 - for (start, end), length in zip(allowed, lengths): - cumulative += length - if point <= cumulative: - return np.random.uniform(start, end) - return allowed[-1][0] - -def sample_channel(neighbor_values, delta): - allowed = get_allowed_intervals(neighbor_values, delta) - if not allowed: return None - return sample_from_allowed(allowed) - -def generate_valid_color(neighbors, r): - delta = np.sqrt(r) - color = np.zeros(3) - for channel in range(3): - neighbor_channel = [n[channel] for n in neighbors] - sampled = sample_channel(neighbor_channel, delta) - if sampled is None: - sampled = np.random.rand() - color[channel] = sampled - return color - -def checker_patch_color(i, j): - """Return a black/white checker color for patch index (i, j).""" - if (i + j) % 2 == 0: - return np.array([1.0, 1.0, 1.0], dtype=np.float32) - return np.array([0.0, 0.0, 0.0], dtype=np.float32) - -# ---------------------------------------------------------------------- -# Mode 1: Colored Patches -# ---------------------------------------------------------------------- -def generate_patch_texture( - obj, cam, scene, depsgraph, - nx=40, ny=30, r=0.05, - texture_width=2048, texture_height=2048, - output_path="//patch_texture.png", uv_centre=None, cross_length=40, - cross_thickness=3, cross_color=(1,0,0,1), edge_sample_grid=5 -): - cam_data = cam.data - render = scene.render - res_x = render.resolution_x - res_y = render.resolution_y - if cam_data.sensor_fit == 'HORIZONTAL': - sensor_width = cam_data.sensor_width - sensor_height = sensor_width / (res_x / res_y) - else: - sensor_height = cam_data.sensor_height - sensor_width = sensor_height * (res_x / res_y) - focal = cam_data.lens - frame_width = sensor_width / focal - frame_height = sensor_height / focal - - edge_sample_grid = max(2, int(edge_sample_grid)) - patch_uv_polygons = [] - patch_colors = np.zeros((ny, nx, 3), dtype=np.float32) - skipped_patches = 0 - - print(f"Raycasting {nx*ny} patches...") - for j in range(ny): - v_start = j / ny - v_end = (j+1) / ny - for i in range(nx): - u_start = i / nx - u_end = (i+1) / nx - uv_poly = sample_patch_uv_polygon( - scene, depsgraph, cam, obj, - u_start, u_end, v_start, v_end, - frame_width, frame_height, - edge_sample_grid=edge_sample_grid - ) - if uv_poly is None: - patch_uv_polygons.append(None) - skipped_patches += 1 - continue - patch_uv_polygons.append(uv_poly) - - neighbors = [] - if j > 0: neighbors.append(patch_colors[j-1, i]) - if i > 0: neighbors.append(patch_colors[j, i-1]) - color = np.random.rand(3) if len(neighbors)==0 else generate_valid_color(neighbors, r) - patch_colors[j, i] = color - print(f"Skipped {skipped_patches} patches with fewer than 3 mesh samples.") - - # Create image – start with black background - img_name = "PatchTexture" - if img_name in bpy.data.images: - bpy.data.images.remove(bpy.data.images[img_name]) - image = bpy.data.images.new(img_name, width=texture_width, height=texture_height, alpha=True) - pixels = list(image.pixels) - - # Fill entire image with black (no mesh areas) - fill_background(pixels, texture_width, texture_height, (0,0,0,1)) - - # Rasterize all mesh UVs in white (visible + non‑visible parts) - rasterize_mesh_uvs(obj, pixels, texture_width, texture_height, color=(1,1,1,1)) - - # Now draw the visible patches on top - idx = 0 - for j in range(ny): - for i in range(nx): - uv_poly = patch_uv_polygons[idx]; idx += 1 - if uv_poly is None: continue - color = patch_colors[j,i] - rgba = (color[0], color[1], color[2], 1.0) - for pixel_poly in uv_polygon_to_pixel_polygons(uv_poly, texture_width, texture_height): - fill_polygon(image, pixels, texture_width, texture_height, pixel_poly, rgba) - - # (after drawing all patches) - if uv_centre is not None: - draw_cross(pixels, texture_width, texture_height, - uv_centre[0], uv_centre[1], - cross_length, cross_thickness, cross_color) - - image.pixels[:] = pixels - image.update() - abs_path = bpy.path.abspath(output_path) - image.filepath_raw = abs_path - image.file_format = 'PNG' - image.save() - print(f"Patch texture saved to {abs_path}") - return image - -# ---------------------------------------------------------------------- -# Mode 2: Black/White Checker Patches -# ---------------------------------------------------------------------- -def generate_checker_texture( - obj, cam, scene, depsgraph, - nx=40, ny=40, - texture_width=2048, texture_height=2048, - output_path="//checker_texture.png", uv_centre=None, cross_length=40, - cross_thickness=3, cross_color=(1,0,0,1), edge_sample_grid=5, - gap_fill_px=6 -): - cam_data = cam.data - render = scene.render - res_x = render.resolution_x - res_y = render.resolution_y - if cam_data.sensor_fit == 'HORIZONTAL': - sensor_width = cam_data.sensor_width - sensor_height = sensor_width / (res_x / res_y) - else: - sensor_height = cam_data.sensor_height - sensor_width = sensor_height * (res_x / res_y) - focal = cam_data.lens - frame_width = sensor_width / focal - frame_height = sensor_height / focal - - edge_sample_grid = max(2, int(edge_sample_grid)) - patch_uv_polygons = [] - patch_colors = np.zeros((ny, nx, 3), dtype=np.float32) - skipped_patches = 0 - - print(f"Raycasting {nx*ny} checker patches...") - for j in range(ny): - v_start = j / ny - v_end = (j+1) / ny - for i in range(nx): - u_start = i / nx - u_end = (i+1) / nx - uv_poly = sample_patch_uv_polygon( - scene, depsgraph, cam, obj, - u_start, u_end, v_start, v_end, - frame_width, frame_height, - edge_sample_grid=edge_sample_grid - ) - if uv_poly is None: - patch_uv_polygons.append(None) - skipped_patches += 1 - continue - patch_uv_polygons.append(uv_poly) - patch_colors[j, i] = checker_patch_color(i, j) - print(f"Skipped {skipped_patches} checker patches with fewer than 3 mesh samples.") - - img_name = "CheckerTexture" - if img_name in bpy.data.images: - bpy.data.images.remove(bpy.data.images[img_name]) - image = bpy.data.images.new(img_name, width=texture_width, height=texture_height, alpha=True) - pixels = list(image.pixels) - mesh_mask = rasterize_mesh_mask(obj, texture_width, texture_height) - patch_state = np.full((texture_height, texture_width), -1, dtype=np.int8) - - fill_background(pixels, texture_width, texture_height, (0,0,0,1)) - paint_mask_color(pixels, texture_width, texture_height, mesh_mask, (1,1,1,1)) - - idx = 0 - for j in range(ny): - for i in range(nx): - uv_poly = patch_uv_polygons[idx]; idx += 1 - if uv_poly is None: - continue - color = patch_colors[j,i] - rgba = (color[0], color[1], color[2], 1.0) - patch_value = 1 if np.mean(color) >= 0.5 else 0 - for pixel_poly in uv_polygon_to_pixel_polygons(uv_poly, texture_width, texture_height): - fill_polygon(image, pixels, texture_width, texture_height, pixel_poly, rgba) - fill_polygon_value( - patch_state, - texture_width, - texture_height, - pixel_poly, - patch_value, - ) - - complete_checker_gaps( - pixels, texture_width, texture_height, mesh_mask, patch_state, max_gap_px=gap_fill_px - ) - - if uv_centre is not None: - draw_cross(pixels, texture_width, texture_height, - uv_centre[0], uv_centre[1], - cross_length, cross_thickness, cross_color) - - image.pixels[:] = pixels - image.update() - abs_path = bpy.path.abspath(output_path) - image.filepath_raw = abs_path - image.file_format = 'PNG' - image.save() - print(f"Checker texture saved to {abs_path}") - return image - -# ---------------------------------------------------------------------- -# Mode 3: Circular Dot Markers -# ---------------------------------------------------------------------- -def generate_dot_texture( - obj, cam, scene, depsgraph, - dot_diameter_pixels=12, - spacing_pixels=40, - dot_color=(0,0,0,1), # RGBA 0-1 - texture_width=2048, texture_height=2048, - output_path="//dot_texture.png",uv_centre=None, cross_length=40, cross_thickness=3, cross_color=(1,0,0,1) -): - render = scene.render - res_x = render.resolution_x - res_y = render.resolution_y - cam_data = cam.data - if cam_data.sensor_fit == 'HORIZONTAL': - sensor_width = cam_data.sensor_width - sensor_height = sensor_width / (res_x / res_y) - else: - sensor_height = cam_data.sensor_height - sensor_width = sensor_height * (res_x / res_y) - focal = cam_data.lens - frame_width = sensor_width / focal - frame_height = sensor_height / focal - - margin = spacing_pixels // 2 - x_centers = np.arange(margin, res_x, spacing_pixels) - y_centers = np.arange(margin, res_y, spacing_pixels) - offset_pix = max(2, dot_diameter_pixels // 4) - du_ndc = offset_pix / res_x - dv_ndc = offset_pix / res_y - - dots = [] # (cx_uv, cy_uv, a_uv, b_uv, angle_uv) - print(f"Raycasting {len(x_centers)*len(y_centers)} dot centers...") - for y_pix in y_centers: - v_ndc = y_pix / res_y - for x_pix in x_centers: - u_ndc = x_pix / res_x - center_loc, face_idx = raycast_from_camera(scene, depsgraph, cam, obj, - u_ndc, v_ndc, frame_width, frame_height) - if center_loc is None: continue - center_uv = get_uv_from_face(obj, face_idx, center_loc) - if center_uv is None: continue - - offsets = [ - ( u_ndc + du_ndc, v_ndc), - ( u_ndc - du_ndc, v_ndc), - ( u_ndc, v_ndc + dv_ndc), - ( u_ndc, v_ndc - dv_ndc) - ] - uv_offsets = [] - for (u_off, v_off) in offsets: - uv_off = raycast_uv_from_camera( - scene, depsgraph, cam, obj, - u_off, v_off, frame_width, frame_height - ) - uv_offsets.append(uv_off) - - scale_u = 1.0 / res_x - scale_v = 1.0 / res_y - r_pix = dot_diameter_pixels / 2.0 - - right_uv, left_uv, up_uv, down_uv = uv_offsets - if right_uv is not None and left_uv is not None: - du_dx = (right_uv - left_uv) / (2 * offset_pix * scale_u) - a_uv = r_pix * du_dx.length * scale_u - elif right_uv is not None: - du_dx = (right_uv - center_uv) / (offset_pix * scale_u) - a_uv = r_pix * du_dx.length * scale_u - elif left_uv is not None: - du_dx = (center_uv - left_uv) / (offset_pix * scale_u) - a_uv = r_pix * du_dx.length * scale_u - else: - du_dx = Vector((1.0, 0.0)) - a_uv = r_pix / texture_width - - if up_uv is not None and down_uv is not None: - dv_dx = (up_uv - down_uv) / (2 * offset_pix * scale_v) - b_uv = r_pix * dv_dx.length * scale_v - elif up_uv is not None: - dv_dx = (up_uv - center_uv) / (offset_pix * scale_v) - b_uv = r_pix * dv_dx.length * scale_v - elif down_uv is not None: - dv_dx = (center_uv - down_uv) / (offset_pix * scale_v) - b_uv = r_pix * dv_dx.length * scale_v - else: - b_uv = r_pix / texture_height - - a_uv = max(a_uv, 1.0 / texture_width) - b_uv = max(b_uv, 1.0 / texture_height) - angle = np.arctan2(du_dx.y, du_dx.x) if du_dx.length > 0 else 0.0 - dots.append((center_uv.x, center_uv.y, a_uv, b_uv, angle)) - - print(f"Found {len(dots)} valid dots.") - - # Create image - img_name = "DotTexture" - if img_name in bpy.data.images: - bpy.data.images.remove(bpy.data.images[img_name]) - image = bpy.data.images.new(img_name, width=texture_width, height=texture_height, alpha=True) - pixels = list(image.pixels) - - # Black background (no mesh) - fill_background(pixels, texture_width, texture_height, (0,0,0,1)) - # White for all mesh areas - rasterize_mesh_uvs(obj, pixels, texture_width, texture_height, color=(1,1,1,1)) - # Draw dots - for (cx_uv, cy_uv, a_uv, b_uv, angle_uv) in dots: - cx_px = cx_uv * (texture_width - 1) - cy_px = cy_uv * (texture_height - 1) - a_px = a_uv * texture_width - b_px = b_uv * texture_height - draw_ellipse(pixels, texture_width, texture_height, - cx_px, cy_px, a_px, b_px, angle_uv, dot_color) - - if uv_centre is not None: - draw_cross(pixels, texture_width, texture_height, - uv_centre[0], uv_centre[1], - cross_length, cross_thickness, cross_color) - - image.pixels[:] = pixels - image.update() - abs_path = bpy.path.abspath(output_path) - image.filepath_raw = abs_path - image.file_format = 'PNG' - image.save() - print(f"Dot texture saved to {abs_path}") - return image - -# ---------------------------------------------------------------------- -# Main -# ---------------------------------------------------------------------- -def main(): - # ==================== CONFIGURATION ==================== - MODE = "patches" # "patches", "checker", or "dots" - OBJ_NAME = "FingerMesh0525.001" - CAM_NAME = "Camera0607" - TEXTURE_WIDTH = 1640*2 - TEXTURE_HEIGHT = 1600*2 - OUTPUT_NAMES = { - "patches": "patch_texture.png", - "checker": "checker_texture.png", - "dots": "dot_texture.png", - } - OUTPUT_PATH = "//" + OUTPUT_NAMES.get(MODE, "texture.png") - - # Patch mode - PATCH_NX = 80; PATCH_NY = 45; COLOR_R = 0.05 - EDGE_SAMPLE_GRID = 5 - # Checker mode - CHECKER_NX = 40; CHECKER_NY = 40; CHECKER_GAP_FILL_PX = 6 - # Dot mode - DOT_DIAMETER_PIX = 12; DOT_SPACING_PIX = 40; DOT_COLOR = (0,0,0,1) - # Cross - CROSS_LENGTH_PX = 40 - CROSS_THICKNESS_PX = 3 - CROSS_COLOR = (1, 0, 0, 1) # red cross - # ====================================================== - - start_time = time.time() - scene = bpy.context.scene - depsgraph = bpy.context.evaluated_depsgraph_get() - - if OBJ_NAME: - obj = bpy.data.objects.get(OBJ_NAME) - else: - obj = bpy.context.active_object - if obj is None or obj.type != 'MESH': - raise ValueError("No valid mesh object found.") - - cam = bpy.data.objects.get(CAM_NAME) if CAM_NAME else scene.camera - if cam is None or cam.type != 'CAMERA': - raise ValueError("No camera found.") - - if not obj.data.uv_layers.active: - obj.data.uv_layers.new(name="UVMap") - - # Compute UV centre - uv_layer = obj.data.uv_layers.active - uv_coords = np.array([uv_layer.data[i].uv.to_tuple() for i in range(len(uv_layer.data))]) - uv_min = uv_coords.min(axis=0) - uv_max = uv_coords.max(axis=0) - uv_centre = (uv_min + uv_max) / 2.0 - print(f"UV centre: ({uv_centre[0]:.4f}, {uv_centre[1]:.4f})") - - - # Compute UV bounding box centre - me = obj.data - uv_layer = me.uv_layers.active - uv_coords = np.array([uv_layer.data[i].uv.to_tuple() for i in range(len(uv_layer.data))]) - uv_min = uv_coords.min(axis=0) - uv_max = uv_coords.max(axis=0) - uv_centre = (uv_min + uv_max) / 2.0 - print(f"UV centre: {uv_centre[0]:.4f}, {uv_centre[1]:.4f}") - - if MODE == "patches": - generate_patch_texture( - obj, cam, scene, depsgraph, - nx=PATCH_NX, ny=PATCH_NY, r=COLOR_R, - texture_width=TEXTURE_WIDTH, texture_height=TEXTURE_HEIGHT, - output_path=OUTPUT_PATH, - uv_centre=uv_centre, - cross_length=CROSS_LENGTH_PX, - cross_thickness=CROSS_THICKNESS_PX, - cross_color=CROSS_COLOR, - edge_sample_grid=EDGE_SAMPLE_GRID - ) - elif MODE == "checker": - generate_checker_texture( - obj, cam, scene, depsgraph, - nx=CHECKER_NX, ny=CHECKER_NY, - texture_width=TEXTURE_WIDTH, texture_height=TEXTURE_HEIGHT, - output_path=OUTPUT_PATH, - uv_centre=uv_centre, - cross_length=CROSS_LENGTH_PX, - cross_thickness=CROSS_THICKNESS_PX, - cross_color=CROSS_COLOR, - edge_sample_grid=EDGE_SAMPLE_GRID, - gap_fill_px=CHECKER_GAP_FILL_PX - ) - elif MODE == "dots": - generate_dot_texture( - obj, cam, scene, depsgraph, - dot_diameter_pixels=DOT_DIAMETER_PIX, - spacing_pixels=DOT_SPACING_PIX, - dot_color=DOT_COLOR, - texture_width=TEXTURE_WIDTH, texture_height=TEXTURE_HEIGHT, - output_path=OUTPUT_PATH, - uv_centre=uv_centre, - cross_length=CROSS_LENGTH_PX, - cross_thickness=CROSS_THICKNESS_PX, - cross_color=CROSS_COLOR - ) - else: - raise ValueError(f"Unknown MODE: {MODE!r}. Use 'patches', 'checker', or 'dots'.") - - print(f"Completed in {time.time()-start_time:.2f} seconds.") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/tools/3d_patch_generation_blender/random_patch_dd02c_fucidals.py b/tools/3d_patch_generation_blender/random_patch_dd02c_fucidals.py deleted file mode 100644 index 06ce878..0000000 --- a/tools/3d_patch_generation_blender/random_patch_dd02c_fucidals.py +++ /dev/null @@ -1,1196 +0,0 @@ -""" -Texture Generator for Arbitrary 3D Surfaces (Blender Script) -- Mode: "patches" (colored patches), "checker" (black/white patches), - or "dots" (circular markers) -- Mesh UVs are filled white so that non‑visible parts become white. -- Texture background (no‑mesh areas) stays black. -""" - -import bpy -import numpy as np -import os -import time -from mathutils import Vector - -# ---------------------------------------------------------------------- -# Common Utilities -# ---------------------------------------------------------------------- -def draw_cross(pixels, width, height, cx_uv, cy_uv, length_px, thickness_px, color): - """ - Draw a cross (plus sign) in pixel space. - cx_uv, cy_uv: centre of cross in UV coordinates (0‑1), origin at bottom‑left. - length_px: total length of each arm (pixels). - thickness_px: line thickness (odd number gives symmetric result). - """ - # Convert UV centre to pixel coordinates (y from bottom) - cx = int(round(cx_uv * (width - 1))) - cy = int(round(cy_uv * (height - 1))) - - half_len = length_px // 2 - half_thick = thickness_px // 2 - - # Horizontal arm - x0 = max(0, cx - half_len) - x1 = min(width - 1, cx + half_len) - y0 = max(0, cy - half_thick) - y1 = min(height - 1, cy + half_thick) - for y in range(y0, y1 + 1): - for x in range(x0, x1 + 1): - idx = (y * width + x) * 4 - pixels[idx:idx+4] = color - - # Vertical arm - x0 = max(0, cx - half_thick) - x1 = min(width - 1, cx + half_thick) - y0 = max(0, cy - half_len) - y1 = min(height - 1, cy + half_len) - for y in range(y0, y1 + 1): - for x in range(x0, x1 + 1): - idx = (y * width + x) * 4 - pixels[idx:idx+4] = color - -def fill_background(pixels, width, height, color): - """Fill the entire image with a solid RGBA color (0-1 floats).""" - r, g, b, a = color - for y in range(height): - for x in range(width): - idx = (y * width + x) * 4 - pixels[idx] = r - pixels[idx+1] = g - pixels[idx+2] = b - pixels[idx+3] = a - -def fill_polygon(image, pixels, width, height, polygon, color): - """Fill a polygon in the pixel array (flattened, RGBA).""" - poly = [(int(round(x)), int(round(y))) for x, y in polygon] - if len(poly) < 3: - return - min_x = max(0, min(p[0] for p in poly)) - max_x = min(width-1, max(p[0] for p in poly)) - min_y = max(0, min(p[1] for p in poly)) - max_y = min(height-1, max(p[1] for p in poly)) - for y in range(min_y, max_y + 1): - intersections = [] - for i in range(len(poly)): - x1, y1 = poly[i] - x2, y2 = poly[(i+1) % len(poly)] - if y1 == y2: - continue - if (y1 <= y < y2) or (y2 <= y < y1): - t = (y - y1) / (y2 - y1) - x = x1 + t * (x2 - x1) - intersections.append(x) - intersections.sort() - for k in range(0, len(intersections), 2): - if k+1 >= len(intersections): - break - x_start = int(round(intersections[k])) - x_end = int(round(intersections[k+1])) - x_start = max(0, min(width-1, x_start)) - x_end = max(0, min(width-1, x_end)) - for x in range(x_start, x_end + 1): - idx = (y * width + x) * 4 - pixels[idx:idx+4] = color - -def fill_polygon_value(grid, width, height, polygon, value): - """Fill a polygon into a 2D scalar grid.""" - poly = [(int(round(x)), int(round(y))) for x, y in polygon] - if len(poly) < 3: - return - min_x = max(0, min(p[0] for p in poly)) - max_x = min(width - 1, max(p[0] for p in poly)) - min_y = max(0, min(p[1] for p in poly)) - max_y = min(height - 1, max(p[1] for p in poly)) - for y in range(min_y, max_y + 1): - intersections = [] - for i in range(len(poly)): - x1, y1 = poly[i] - x2, y2 = poly[(i + 1) % len(poly)] - if y1 == y2: - continue - if (y1 <= y < y2) or (y2 <= y < y1): - t = (y - y1) / (y2 - y1) - x = x1 + t * (x2 - x1) - intersections.append(x) - intersections.sort() - for k in range(0, len(intersections), 2): - if k + 1 >= len(intersections): - break - x_start = int(round(intersections[k])) - x_end = int(round(intersections[k + 1])) - x_start = max(0, min(width - 1, x_start)) - x_end = max(0, min(width - 1, x_end)) - grid[y, x_start:x_end + 1] = value - -def unwrap_periodic_values(values): - """ - Unwrap periodic UV coordinates so points straddling the 0/1 seam stay local. - """ - arr = np.asarray(values, dtype=np.float64).copy() - if arr.size == 0: - return arr - if arr.max() - arr.min() > 0.5: - arr[arr < 0.5] += 1.0 - return arr - -def wrap_uv_polygon_for_raster(uv_poly): - """ - Return a seam-aware version of a UV polygon. - - UV islands may cross the 0/1 seam. If we rasterize those raw coordinates as a - single polygon, Blender's repeated UVs turn into a long diagonal triangle - across the texture. We unwrap seam-crossing polygons first, then draw wrapped - copies in the texture tile(s) they overlap. - """ - if len(uv_poly) < 3: - return [] - - uv_arr = np.array([(float(u), float(v)) for u, v in uv_poly], dtype=np.float64) - uv_arr[:, 0] = unwrap_periodic_values(uv_arr[:, 0]) - uv_arr[:, 1] = unwrap_periodic_values(uv_arr[:, 1]) - return [tuple(pt) for pt in uv_arr] - -def uv_polygon_to_pixel_polygons(uv_poly, tex_width, tex_height): - """ - Convert one UV polygon into one or more pixel polygons, duplicating seam- - crossing polygons into adjacent tiles so rasterization stays local. - """ - unwrapped = wrap_uv_polygon_for_raster(uv_poly) - if len(unwrapped) < 3: - return [] - - u_vals = [u for u, _ in unwrapped] - v_vals = [v for _, v in unwrapped] - u_shift_start = int(np.floor(min(u_vals))) - u_shift_end = int(np.floor(max(u_vals))) - v_shift_start = int(np.floor(min(v_vals))) - v_shift_end = int(np.floor(max(v_vals))) - - pixel_polys = [] - for u_shift in range(u_shift_start, u_shift_end + 1): - for v_shift in range(v_shift_start, v_shift_end + 1): - shifted = [(u - u_shift, v - v_shift) for u, v in unwrapped] - shifted_u = [u for u, _ in shifted] - shifted_v = [v for _, v in shifted] - if max(shifted_u) < 0.0 or min(shifted_u) > 1.0: - continue - if max(shifted_v) < 0.0 or min(shifted_v) > 1.0: - continue - pixel_polys.append([ - (u * (tex_width - 1), v * (tex_height - 1)) - for u, v in shifted - ]) - return pixel_polys - -def draw_ellipse(pixels, width, height, cx, cy, a, b, angle, color): - """Filled ellipse with semi‑axes a,b (pixels), rotation angle (rad).""" - cos_a = np.cos(angle) - sin_a = np.sin(angle) - rx = abs(a * cos_a) + abs(b * sin_a) - ry = abs(a * sin_a) + abs(b * cos_a) - x_min = int(max(0, cx - rx)) - x_max = int(min(width-1, cx + rx)) - y_min = int(max(0, cy - ry)) - y_max = int(min(height-1, cy + ry)) - for y in range(y_min, y_max + 1): - for x in range(x_min, x_max + 1): - dx = x - cx - dy = y - cy - u = dx * cos_a + dy * sin_a - v = -dx * sin_a + dy * cos_a - if (u/a)**2 + (v/b)**2 <= 1.0: - idx = (y * width + x) * 4 - pixels[idx:idx+4] = color - -def wrap_uv_near_reference(uv, reference_uv): - """Wrap a UV so it stays closest to a reference UV across periodic seams.""" - du = uv.x - reference_uv.x - dv = uv.y - reference_uv.y - if du > 0.5: - du -= 1.0 - elif du < -0.5: - du += 1.0 - if dv > 0.5: - dv -= 1.0 - elif dv < -0.5: - dv += 1.0 - return Vector((reference_uv.x + du, reference_uv.y + dv)) - -def wrapped_uv_distance_sq(uv_a, uv_b): - """Squared UV distance on a periodic 0..1 texture domain.""" - du = abs((uv_a.x % 1.0) - (uv_b.x % 1.0)) - dv = abs((uv_a.y % 1.0) - (uv_b.y % 1.0)) - du = min(du, 1.0 - du) - dv = min(dv, 1.0 - dv) - return du * du + dv * dv - -def draw_wrapped_ellipse_uv( - pixels, width, height, - cx_uv, cy_uv, a_uv, b_uv, angle, color -): - """ - Draw an ellipse in UV space, duplicating it across the texture seam if needed. - """ - base_u = cx_uv % 1.0 - base_v = cy_uv % 1.0 - for u_shift in (-1, 0, 1): - for v_shift in (-1, 0, 1): - shifted_u = base_u + u_shift - shifted_v = base_v + v_shift - if shifted_u + a_uv < 0.0 or shifted_u - a_uv > 1.0: - continue - if shifted_v + b_uv < 0.0 or shifted_v - b_uv > 1.0: - continue - cx_px = shifted_u * (width - 1) - cy_px = shifted_v * (height - 1) - a_px = a_uv * width - b_px = b_uv * height - draw_ellipse(pixels, width, height, cx_px, cy_px, a_px, b_px, angle, color) - -def draw_wrapped_outlined_ellipse_uv( - pixels, width, height, - cx_uv, cy_uv, a_uv, b_uv, angle, - fill_color, outline_color, outline_px=2 -): - """ - Draw a seam-safe ellipse with an outline for visibility on bright textures. - """ - outline_au = a_uv + (outline_px / width) - outline_bv = b_uv + (outline_px / height) - draw_wrapped_ellipse_uv( - pixels, width, height, - cx_uv, cy_uv, outline_au, outline_bv, angle, outline_color - ) - draw_wrapped_ellipse_uv( - pixels, width, height, - cx_uv, cy_uv, a_uv, b_uv, angle, fill_color - ) - -def get_camera_frame(cam, scene): - """Return camera frame dimensions and render resolution.""" - render = scene.render - res_x = render.resolution_x - res_y = render.resolution_y - cam_data = cam.data - if cam_data.sensor_fit == 'HORIZONTAL': - sensor_width = cam_data.sensor_width - sensor_height = sensor_width / (res_x / res_y) - else: - sensor_height = cam_data.sensor_height - sensor_width = sensor_height * (res_x / res_y) - focal = cam_data.lens - frame_width = sensor_width / focal - frame_height = sensor_height / focal - return frame_width, frame_height, res_x, res_y - -def estimate_dot_uv_parameters( - scene, depsgraph, cam, obj, - u_ndc, v_ndc, center_uv, - frame_width, frame_height, res_x, res_y, - texture_width, texture_height, dot_diameter_pixels -): - """ - Estimate a UV-space dot size for a screen-space circular marker. - - We still sample local UV distortion from camera-space offsets, but collapse - the result to one isotropic UV radius so the marker stays circular on the - texture map instead of turning into an ellipse near the sides. - """ - offset_pix = max(2, dot_diameter_pixels // 4) - du_ndc = offset_pix / res_x - dv_ndc = offset_pix / res_y - offsets = [ - (u_ndc + du_ndc, v_ndc), - (u_ndc - du_ndc, v_ndc), - (u_ndc, v_ndc + dv_ndc), - (u_ndc, v_ndc - dv_ndc), - ] - - uv_offsets = [] - for u_off, v_off in offsets: - uv_off = raycast_uv_from_camera( - scene, depsgraph, cam, obj, u_off, v_off, frame_width, frame_height - ) - if uv_off is not None: - uv_off = wrap_uv_near_reference(uv_off, center_uv) - uv_offsets.append(uv_off) - - scale_u = 1.0 / res_x - scale_v = 1.0 / res_y - r_pix = dot_diameter_pixels / 2.0 - - right_uv, left_uv, up_uv, down_uv = uv_offsets - if right_uv is not None and left_uv is not None: - du_dx = (right_uv - left_uv) / (2 * offset_pix * scale_u) - a_uv = r_pix * du_dx.length * scale_u - elif right_uv is not None: - du_dx = (right_uv - center_uv) / (offset_pix * scale_u) - a_uv = r_pix * du_dx.length * scale_u - elif left_uv is not None: - du_dx = (center_uv - left_uv) / (offset_pix * scale_u) - a_uv = r_pix * du_dx.length * scale_u - else: - du_dx = Vector((1.0, 0.0)) - a_uv = r_pix / texture_width - - if up_uv is not None and down_uv is not None: - dv_dx = (up_uv - down_uv) / (2 * offset_pix * scale_v) - b_uv = r_pix * dv_dx.length * scale_v - elif up_uv is not None: - dv_dx = (up_uv - center_uv) / (offset_pix * scale_v) - b_uv = r_pix * dv_dx.length * scale_v - elif down_uv is not None: - dv_dx = (center_uv - down_uv) / (offset_pix * scale_v) - b_uv = r_pix * dv_dx.length * scale_v - else: - b_uv = r_pix / texture_height - - a_uv = max(a_uv, 1.0 / texture_width) - b_uv = max(b_uv, 1.0 / texture_height) - - # Keep the marker circular in the texture domain, but let its radius grow - # with the stronger of the local UV-space scales. - radius_uv = max(a_uv, b_uv) - return radius_uv, radius_uv, 0.0 - -def has_visible_margin( - scene, depsgraph, cam, obj, - u_ndc, v_ndc, frame_width, frame_height, - radius_ndc_u, radius_ndc_v, - extra_margin_px_x=0.0, extra_margin_px_y=0.0, - res_x=1, res_y=1, - angular_samples=12 -): - """ - Check whether a candidate center has enough visible-mesh clearance around it. - - We probe a ring around the candidate in camera space. If those neighboring - rays also hit the mesh, the dot is less likely to cross or touch the visible - silhouette. - """ - test_radius_u = radius_ndc_u + (extra_margin_px_x / max(res_x, 1)) - test_radius_v = radius_ndc_v + (extra_margin_px_y / max(res_y, 1)) - for k in range(max(angular_samples, 4)): - angle = 2.0 * np.pi * k / angular_samples - test_u = u_ndc + test_radius_u * np.cos(angle) - test_v = v_ndc + test_radius_v * np.sin(angle) - hit_loc, _ = raycast_from_camera( - scene, depsgraph, cam, obj, test_u, test_v, frame_width, frame_height - ) - if hit_loc is None: - return False - return True - -def generate_visible_fiducial_dots( - scene, depsgraph, cam, obj, - frame_width, frame_height, res_x, res_y, - texture_width, texture_height, - grid_nx, grid_ny, dot_diameter_pixels, - visibility_sample_nx=96, visibility_sample_ny=96, - edge_margin_pixels=0.0 -): - """ - Build an approximately even grid of fiducials over the visible mesh area. - - The grid is spaced in camera space over the visible mesh bounding box, then - snapped to the nearest visible raycast sample so every fiducial lands on the - actually visible surface. - """ - if grid_nx <= 0 or grid_ny <= 0: - return [] - - visibility_sample_nx = max(int(visibility_sample_nx), grid_nx * 8, 16) - visibility_sample_ny = max(int(visibility_sample_ny), grid_ny * 8, 16) - radius_ndc_u = (dot_diameter_pixels * 0.5) / res_x - radius_ndc_v = (dot_diameter_pixels * 0.5) / res_y - inset_ndc_u = radius_ndc_u + (edge_margin_pixels / max(res_x, 1)) - inset_ndc_v = radius_ndc_v + (edge_margin_pixels / max(res_y, 1)) - - all_visible_samples = [] - interior_visible_samples = [] - for j in range(visibility_sample_ny): - v_ndc = (j + 0.5) / visibility_sample_ny - for i in range(visibility_sample_nx): - u_ndc = (i + 0.5) / visibility_sample_nx - center_loc, face_idx = raycast_from_camera( - scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height - ) - if center_loc is None: - continue - center_uv = get_uv_from_face(obj, face_idx, center_loc) - if center_uv is None: - continue - all_visible_samples.append((u_ndc, v_ndc, center_uv)) - if not has_visible_margin( - scene, depsgraph, cam, obj, - u_ndc, v_ndc, frame_width, frame_height, - radius_ndc_u, radius_ndc_v, - extra_margin_px_x=edge_margin_pixels, - extra_margin_px_y=edge_margin_pixels, - res_x=res_x, - res_y=res_y - ): - continue - interior_visible_samples.append((u_ndc, v_ndc, center_uv)) - - if not all_visible_samples: - print("Warning: No visible mesh samples found for fiducials.") - return [] - if not interior_visible_samples: - print("Warning: No interior visible mesh samples found for fiducials.") - return [] - - all_u_values = np.array([sample[0] for sample in all_visible_samples], dtype=np.float64) - all_v_values = np.array([sample[1] for sample in all_visible_samples], dtype=np.float64) - u_min = float(all_u_values.min() + inset_ndc_u) - u_max = float(all_u_values.max() - inset_ndc_u) - v_min = float(all_v_values.min() + inset_ndc_v) - v_max = float(all_v_values.max() - inset_ndc_v) - - interior_u_values = np.array([sample[0] for sample in interior_visible_samples], dtype=np.float64) - interior_v_values = np.array([sample[1] for sample in interior_visible_samples], dtype=np.float64) - u_min = max(u_min, float(interior_u_values.min())) - u_max = min(u_max, float(interior_u_values.max())) - v_min = max(v_min, float(interior_v_values.min())) - v_max = min(v_max, float(interior_v_values.max())) - - if u_min > u_max: - mid_u = float(interior_u_values.mean()) - u_min = mid_u - u_max = mid_u - if v_min > v_max: - mid_v = float(interior_v_values.mean()) - v_min = mid_v - v_max = mid_v - - if grid_nx == 1: - target_us = np.array([(u_min + u_max) * 0.5], dtype=np.float64) - else: - target_us = np.linspace(u_min, u_max, grid_nx) - - if grid_ny == 1: - target_vs = np.array([(v_min + v_max) * 0.5], dtype=np.float64) - else: - target_vs = np.linspace(v_min, v_max, grid_ny) - - dots = [] - selected_uvs = [] - used_indices = set() - min_sep_u = max(2.0 * dot_diameter_pixels / texture_width, 1.0 / visibility_sample_nx) - min_sep_v = max(2.0 * dot_diameter_pixels / texture_height, 1.0 / visibility_sample_ny) - min_sep_sq = min_sep_u * min_sep_u + min_sep_v * min_sep_v - for target_v in target_vs: - for target_u in target_us: - candidate_indices = [] - for idx, (u_ndc, v_ndc, center_uv) in enumerate(interior_visible_samples): - if idx in used_indices: - continue - dist = (u_ndc - target_u) ** 2 + (v_ndc - target_v) ** 2 - candidate_indices.append((dist, idx)) - candidate_indices.sort(key=lambda item: item[0]) - - chosen_idx = None - fallback_idx = None - for _, idx in candidate_indices: - if fallback_idx is None: - fallback_idx = idx - _, _, center_uv = interior_visible_samples[idx] - if all(wrapped_uv_distance_sq(center_uv, prev_uv) >= min_sep_sq for prev_uv in selected_uvs): - chosen_idx = idx - break - - if chosen_idx is None: - chosen_idx = fallback_idx - if chosen_idx is None: - continue - - used_indices.add(chosen_idx) - u_ndc, v_ndc, center_uv = interior_visible_samples[chosen_idx] - a_uv, b_uv, angle = estimate_dot_uv_parameters( - scene, depsgraph, cam, obj, - u_ndc, v_ndc, center_uv, - frame_width, frame_height, res_x, res_y, - texture_width, texture_height, dot_diameter_pixels - ) - dots.append((center_uv.x, center_uv.y, a_uv, b_uv, angle)) - selected_uvs.append(center_uv) - - print( - f"Generated {len(dots)} fiducial dots from a {grid_nx}x{grid_ny} target grid " - f"using {len(interior_visible_samples)}/{len(all_visible_samples)} interior samples." - ) - return dots - -def get_uv_from_face(obj, face_index, location): - """World‑space location → UV coordinate on the mesh face.""" - me = obj.data - local_loc = obj.matrix_world.inverted() @ location - poly = me.polygons[face_index] - vert_indices = poly.vertices - loop_indices = poly.loop_indices - if len(vert_indices) < 3: - return None - v0 = me.vertices[vert_indices[0]].co - v1 = me.vertices[vert_indices[1]].co - v2 = me.vertices[vert_indices[2]].co - uv_layer = me.uv_layers.active.data - uv0 = uv_layer[loop_indices[0]].uv - uv1 = uv_layer[loop_indices[1]].uv - uv2 = uv_layer[loop_indices[2]].uv - def tri_area(a, b, c): - return 0.5 * ((b - a).cross(c - a)).length - total_area = tri_area(v0, v1, v2) - if total_area == 0: - return None - w0 = tri_area(local_loc, v1, v2) / total_area - w1 = tri_area(local_loc, v2, v0) / total_area - w2 = 1.0 - w0 - w1 - uv = w0 * uv0 + w1 * uv1 + w2 * uv2 - return Vector((uv.x, uv.y)) - -def raycast_from_camera(scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height): - """Camera raycast through normalised sensor coordinates (0‑1).""" - sx = (u_ndc - 0.5) * frame_width - sy = (v_ndc - 0.5) * frame_height - direction_cam = Vector((sx, sy, -1.0)) - direction_world = cam.matrix_world.to_quaternion() @ direction_cam - direction_world.normalize() - origin = cam.matrix_world.translation - hit, location, normal, face_index, hit_obj, matrix = scene.ray_cast( - depsgraph, origin, direction_world) - if hit and hit_obj == obj: - return location, face_index - return None, None - -def raycast_uv_from_camera(scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height): - """Camera raycast through normalized sensor coordinates and return UV.""" - loc, face_idx = raycast_from_camera( - scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height - ) - if loc is None: - return None - return get_uv_from_face(obj, face_idx, loc) - -def convex_hull_uv(points): - """Return UV points ordered around their 2D convex hull.""" - unwrapped = wrap_uv_polygon_for_raster([(p.x, p.y) for p in points]) - unique = sorted(set(unwrapped)) - if len(unique) < 3: - return None - - def cross(o, a, b): - return (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]) - - lower = [] - for p in unique: - while len(lower) >= 2 and cross(lower[-2], lower[-1], p) <= 0: - lower.pop() - lower.append(p) - - upper = [] - for p in reversed(unique): - while len(upper) >= 2 and cross(upper[-2], upper[-1], p) <= 0: - upper.pop() - upper.append(p) - - hull = lower[:-1] + upper[:-1] - if len(hull) < 3: - return None - return [Vector((u, v)) for u, v in hull] - -def sample_patch_uv_polygon( - scene, depsgraph, cam, obj, - u_start, u_end, v_start, v_end, - frame_width, frame_height, - edge_sample_grid=5 -): - """ - Build a UV polygon for one camera-space patch. - - Full interior cells use the four corners. Edge cells that only partially hit - the mesh are sampled more densely and converted to a convex hull so they are - not dropped just because some corners miss the silhouette. - """ - corners_ndc = [ - (u_start, v_start), - (u_end, v_start), - (u_end, v_end), - (u_start, v_end), - ] - - corner_uvs = [] - for u_ndc, v_ndc in corners_ndc: - uv = raycast_uv_from_camera( - scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height - ) - if uv is not None: - corner_uvs.append(uv) - - if len(corner_uvs) >= 3: - return corner_uvs - - sampled_uvs = list(corner_uvs) - for sy in range(edge_sample_grid): - v_ndc = v_start + (v_end - v_start) * sy / (edge_sample_grid - 1) - for sx in range(edge_sample_grid): - u_ndc = u_start + (u_end - u_start) * sx / (edge_sample_grid - 1) - uv = raycast_uv_from_camera( - scene, depsgraph, cam, obj, u_ndc, v_ndc, frame_width, frame_height - ) - if uv is not None: - sampled_uvs.append(uv) - - return convex_hull_uv(sampled_uvs) - -def rasterize_mesh_uvs(obj, pixels, tex_width, tex_height, color=(1,1,1,1)): - """ - Fill all triangles of the active UV map with 'color' (default white). - This marks every part of the mesh, visible or not, as white. - """ - me = obj.data - uv_layer = me.uv_layers.active - if not uv_layer: - print("Warning: No active UV layer – mesh rasterization skipped.") - return - uv_data = uv_layer.data - for poly in me.polygons: - # Get loop indices for this polygon - loops = poly.loop_indices - # Build UV polygon - uv_poly = [] - for loop_idx in loops: - uv = uv_data[loop_idx].uv - uv_poly.append((uv.x, uv.y)) - # Convert UV (0‑1) to pixel coordinates - for pixel_poly in uv_polygon_to_pixel_polygons(uv_poly, tex_width, tex_height): - fill_polygon(None, pixels, tex_width, tex_height, pixel_poly, color) - -def rasterize_mesh_mask(obj, tex_width, tex_height): - """Rasterize the active UV map to a boolean mesh mask.""" - mask = np.zeros((tex_height, tex_width), dtype=bool) - me = obj.data - uv_layer = me.uv_layers.active - if not uv_layer: - return mask - uv_data = uv_layer.data - for poly in me.polygons: - uv_poly = [] - for loop_idx in poly.loop_indices: - uv = uv_data[loop_idx].uv - uv_poly.append((uv.x, uv.y)) - for pixel_poly in uv_polygon_to_pixel_polygons(uv_poly, tex_width, tex_height): - fill_polygon_value(mask, tex_width, tex_height, pixel_poly, True) - return mask - -def paint_mask_color(pixels, width, height, mask, color): - """Paint all True pixels of a boolean mask with an RGBA color.""" - r, g, b, a = color - ys, xs = np.nonzero(mask) - for y, x in zip(ys, xs): - idx = (y * width + x) * 4 - pixels[idx:idx+4] = (r, g, b, a) - -def complete_checker_gaps(pixels, width, height, mesh_mask, patch_state, max_gap_px=6): - """ - Fill small white slivers inside the mesh by expanding nearby checker colors. - - `patch_state`: -1 = unassigned, 0 = black patch, 1 = white patch. - """ - for _ in range(max_gap_px): - candidate = mesh_mask & (patch_state < 0) - if not np.any(candidate): - break - - black_counts = np.zeros_like(patch_state, dtype=np.uint8) - white_counts = np.zeros_like(patch_state, dtype=np.uint8) - - for dy in (-1, 0, 1): - for dx in (-1, 0, 1): - if dx == 0 and dy == 0: - continue - - src_y0 = max(0, -dy) - src_y1 = height - max(0, dy) - src_x0 = max(0, -dx) - src_x1 = width - max(0, dx) - dst_y0 = max(0, dy) - dst_y1 = height - max(0, -dy) - dst_x0 = max(0, dx) - dst_x1 = width - max(0, -dx) - - neighbor_vals = patch_state[src_y0:src_y1, src_x0:src_x1] - black_counts[dst_y0:dst_y1, dst_x0:dst_x1] += (neighbor_vals == 0) - white_counts[dst_y0:dst_y1, dst_x0:dst_x1] += (neighbor_vals == 1) - - fillable = candidate & ((black_counts + white_counts) > 0) - if not np.any(fillable): - break - - patch_state[fillable] = np.where( - black_counts[fillable] > white_counts[fillable], 0, 1 - ).astype(np.int8) - - paint_mask_color(pixels, width, height, patch_state == 0, (0.0, 0.0, 0.0, 1.0)) - paint_mask_color(pixels, width, height, patch_state == 1, (1.0, 1.0, 1.0, 1.0)) - -# ---------------------------------------------------------------------- -# Color generation (for patch mode) -# ---------------------------------------------------------------------- -def merge_intervals(intervals): - if not intervals: return [] - intervals.sort(key=lambda x: x[0]) - merged = [intervals[0]] - for current in intervals[1:]: - last = merged[-1] - if current[0] <= last[1]: - merged[-1] = (last[0], max(last[1], current[1])) - else: - merged.append(current) - return merged - -def get_allowed_intervals(neighbor_values, delta): - forbidden = [] - for c in neighbor_values: - low = max(0, c - delta) - high = min(1, c + delta) - forbidden.append((low, high)) - merged = merge_intervals(forbidden) - allowed = [] - if merged: - if merged[0][0] > 0: - allowed.append((0, merged[0][0])) - for i in range(1, len(merged)): - allowed.append((merged[i-1][1], merged[i][0])) - if merged[-1][1] < 1: - allowed.append((merged[-1][1], 1)) - else: - allowed.append((0, 1)) - return allowed - -def sample_from_allowed(allowed): - lengths = [end - start for start, end in allowed] - total_length = sum(lengths) - if total_length <= 0: return None - point = np.random.rand() * total_length - cumulative = 0 - for (start, end), length in zip(allowed, lengths): - cumulative += length - if point <= cumulative: - return np.random.uniform(start, end) - return allowed[-1][0] - -def sample_channel(neighbor_values, delta): - allowed = get_allowed_intervals(neighbor_values, delta) - if not allowed: return None - return sample_from_allowed(allowed) - -def generate_valid_color(neighbors, r): - delta = np.sqrt(r) - color = np.zeros(3) - for channel in range(3): - neighbor_channel = [n[channel] for n in neighbors] - sampled = sample_channel(neighbor_channel, delta) - if sampled is None: - sampled = np.random.rand() - color[channel] = sampled - return color - -def checker_patch_color(i, j): - """Return a black/white checker color for patch index (i, j).""" - if (i + j) % 2 == 0: - return np.array([1.0, 1.0, 1.0], dtype=np.float32) - return np.array([0.0, 0.0, 0.0], dtype=np.float32) - -# ---------------------------------------------------------------------- -# Mode 1: Colored Patches -# ---------------------------------------------------------------------- -def generate_patch_texture( - obj, cam, scene, depsgraph, - nx=40, ny=30, r=0.05, - texture_width=2048, texture_height=2048, - output_path="//patch_texture.png", uv_centre=None, cross_length=40, - cross_thickness=3, cross_color=(1,0,0,1), edge_sample_grid=5, - fiducial_nx=0, fiducial_ny=0, fiducial_diameter_pixels=18, - fiducial_color=(1,1,1,1), fiducial_outline_color=(0,0,0,1), - fiducial_outline_px=2, fiducial_edge_margin_pixels=0.0 -): - frame_width, frame_height, res_x, res_y = get_camera_frame(cam, scene) - - edge_sample_grid = max(2, int(edge_sample_grid)) - patch_uv_polygons = [] - patch_colors = np.zeros((ny, nx, 3), dtype=np.float32) - skipped_patches = 0 - - print(f"Raycasting {nx*ny} patches...") - for j in range(ny): - v_start = j / ny - v_end = (j+1) / ny - for i in range(nx): - u_start = i / nx - u_end = (i+1) / nx - uv_poly = sample_patch_uv_polygon( - scene, depsgraph, cam, obj, - u_start, u_end, v_start, v_end, - frame_width, frame_height, - edge_sample_grid=edge_sample_grid - ) - if uv_poly is None: - patch_uv_polygons.append(None) - skipped_patches += 1 - continue - patch_uv_polygons.append(uv_poly) - - neighbors = [] - if j > 0: neighbors.append(patch_colors[j-1, i]) - if i > 0: neighbors.append(patch_colors[j, i-1]) - color = np.random.rand(3) if len(neighbors)==0 else generate_valid_color(neighbors, r) - patch_colors[j, i] = color - print(f"Skipped {skipped_patches} patches with fewer than 3 mesh samples.") - - # Create image – start with black background - img_name = "PatchTexture" - if img_name in bpy.data.images: - bpy.data.images.remove(bpy.data.images[img_name]) - image = bpy.data.images.new(img_name, width=texture_width, height=texture_height, alpha=True) - pixels = list(image.pixels) - - # Fill entire image with black (no mesh areas) - fill_background(pixels, texture_width, texture_height, (0,0,0,1)) - - # Rasterize all mesh UVs in white (visible + non‑visible parts) - rasterize_mesh_uvs(obj, pixels, texture_width, texture_height, color=(1,1,1,1)) - - # Now draw the visible patches on top - idx = 0 - for j in range(ny): - for i in range(nx): - uv_poly = patch_uv_polygons[idx]; idx += 1 - if uv_poly is None: continue - color = patch_colors[j,i] - rgba = (color[0], color[1], color[2], 1.0) - for pixel_poly in uv_polygon_to_pixel_polygons(uv_poly, texture_width, texture_height): - fill_polygon(image, pixels, texture_width, texture_height, pixel_poly, rgba) - - if fiducial_nx > 0 and fiducial_ny > 0: - fiducial_dots = generate_visible_fiducial_dots( - scene, depsgraph, cam, obj, - frame_width, frame_height, res_x, res_y, - texture_width, texture_height, - fiducial_nx, fiducial_ny, fiducial_diameter_pixels, - edge_margin_pixels=fiducial_edge_margin_pixels - ) - for cx_uv, cy_uv, a_uv, b_uv, angle_uv in fiducial_dots: - draw_wrapped_outlined_ellipse_uv( - pixels, texture_width, texture_height, - cx_uv, cy_uv, a_uv, b_uv, angle_uv, - fiducial_color, fiducial_outline_color, fiducial_outline_px - ) - - if uv_centre is not None: - draw_cross(pixels, texture_width, texture_height, - uv_centre[0], uv_centre[1], - cross_length, cross_thickness, cross_color) - - image.pixels[:] = pixels - image.update() - abs_path = bpy.path.abspath(output_path) - image.filepath_raw = abs_path - image.file_format = 'PNG' - image.save() - print(f"Patch texture saved to {abs_path}") - return image - -# ---------------------------------------------------------------------- -# Mode 2: Black/White Checker Patches -# ---------------------------------------------------------------------- -def generate_checker_texture( - obj, cam, scene, depsgraph, - nx=40, ny=40, - texture_width=2048, texture_height=2048, - output_path="//checker_texture.png", uv_centre=None, cross_length=40, - cross_thickness=3, cross_color=(1,0,0,1), edge_sample_grid=5, - gap_fill_px=6 -): - cam_data = cam.data - render = scene.render - res_x = render.resolution_x - res_y = render.resolution_y - if cam_data.sensor_fit == 'HORIZONTAL': - sensor_width = cam_data.sensor_width - sensor_height = sensor_width / (res_x / res_y) - else: - sensor_height = cam_data.sensor_height - sensor_width = sensor_height * (res_x / res_y) - focal = cam_data.lens - frame_width = sensor_width / focal - frame_height = sensor_height / focal - - edge_sample_grid = max(2, int(edge_sample_grid)) - patch_uv_polygons = [] - patch_colors = np.zeros((ny, nx, 3), dtype=np.float32) - skipped_patches = 0 - - print(f"Raycasting {nx*ny} checker patches...") - for j in range(ny): - v_start = j / ny - v_end = (j+1) / ny - for i in range(nx): - u_start = i / nx - u_end = (i+1) / nx - uv_poly = sample_patch_uv_polygon( - scene, depsgraph, cam, obj, - u_start, u_end, v_start, v_end, - frame_width, frame_height, - edge_sample_grid=edge_sample_grid - ) - if uv_poly is None: - patch_uv_polygons.append(None) - skipped_patches += 1 - continue - patch_uv_polygons.append(uv_poly) - patch_colors[j, i] = checker_patch_color(i, j) - print(f"Skipped {skipped_patches} checker patches with fewer than 3 mesh samples.") - - img_name = "CheckerTexture" - if img_name in bpy.data.images: - bpy.data.images.remove(bpy.data.images[img_name]) - image = bpy.data.images.new(img_name, width=texture_width, height=texture_height, alpha=True) - pixels = list(image.pixels) - mesh_mask = rasterize_mesh_mask(obj, texture_width, texture_height) - patch_state = np.full((texture_height, texture_width), -1, dtype=np.int8) - - fill_background(pixels, texture_width, texture_height, (0,0,0,1)) - paint_mask_color(pixels, texture_width, texture_height, mesh_mask, (1,1,1,1)) - - idx = 0 - for j in range(ny): - for i in range(nx): - uv_poly = patch_uv_polygons[idx]; idx += 1 - if uv_poly is None: - continue - color = patch_colors[j,i] - rgba = (color[0], color[1], color[2], 1.0) - patch_value = 1 if np.mean(color) >= 0.5 else 0 - for pixel_poly in uv_polygon_to_pixel_polygons(uv_poly, texture_width, texture_height): - fill_polygon(image, pixels, texture_width, texture_height, pixel_poly, rgba) - fill_polygon_value( - patch_state, - texture_width, - texture_height, - pixel_poly, - patch_value, - ) - - complete_checker_gaps( - pixels, texture_width, texture_height, mesh_mask, patch_state, max_gap_px=gap_fill_px - ) - - if uv_centre is not None: - draw_cross(pixels, texture_width, texture_height, - uv_centre[0], uv_centre[1], - cross_length, cross_thickness, cross_color) - - image.pixels[:] = pixels - image.update() - abs_path = bpy.path.abspath(output_path) - image.filepath_raw = abs_path - image.file_format = 'PNG' - image.save() - print(f"Checker texture saved to {abs_path}") - return image - -# ---------------------------------------------------------------------- -# Mode 3: Circular Dot Markers -# ---------------------------------------------------------------------- -def generate_dot_texture( - obj, cam, scene, depsgraph, - dot_diameter_pixels=12, - spacing_pixels=40, - dot_color=(0,0,0,1), # RGBA 0-1 - texture_width=2048, texture_height=2048, - output_path="//dot_texture.png",uv_centre=None, cross_length=40, cross_thickness=3, cross_color=(1,0,0,1) -): - frame_width, frame_height, res_x, res_y = get_camera_frame(cam, scene) - - margin = spacing_pixels // 2 - x_centers = np.arange(margin, res_x, spacing_pixels) - y_centers = np.arange(margin, res_y, spacing_pixels) - - dots = [] # (cx_uv, cy_uv, a_uv, b_uv, angle_uv) - print(f"Raycasting {len(x_centers)*len(y_centers)} dot centers...") - for y_pix in y_centers: - v_ndc = y_pix / res_y - for x_pix in x_centers: - u_ndc = x_pix / res_x - center_loc, face_idx = raycast_from_camera(scene, depsgraph, cam, obj, - u_ndc, v_ndc, frame_width, frame_height) - if center_loc is None: continue - center_uv = get_uv_from_face(obj, face_idx, center_loc) - if center_uv is None: continue - - a_uv, b_uv, angle = estimate_dot_uv_parameters( - scene, depsgraph, cam, obj, - u_ndc, v_ndc, center_uv, - frame_width, frame_height, res_x, res_y, - texture_width, texture_height, dot_diameter_pixels - ) - dots.append((center_uv.x, center_uv.y, a_uv, b_uv, angle)) - - print(f"Found {len(dots)} valid dots.") - - # Create image - img_name = "DotTexture" - if img_name in bpy.data.images: - bpy.data.images.remove(bpy.data.images[img_name]) - image = bpy.data.images.new(img_name, width=texture_width, height=texture_height, alpha=True) - pixels = list(image.pixels) - - # Black background (no mesh) - fill_background(pixels, texture_width, texture_height, (0,0,0,1)) - # White for all mesh areas - rasterize_mesh_uvs(obj, pixels, texture_width, texture_height, color=(1,1,1,1)) - # Draw dots - for (cx_uv, cy_uv, a_uv, b_uv, angle_uv) in dots: - draw_wrapped_ellipse_uv( - pixels, texture_width, texture_height, - cx_uv, cy_uv, a_uv, b_uv, angle_uv, dot_color - ) - - if uv_centre is not None: - draw_cross(pixels, texture_width, texture_height, - uv_centre[0], uv_centre[1], - cross_length, cross_thickness, cross_color) - - image.pixels[:] = pixels - image.update() - abs_path = bpy.path.abspath(output_path) - image.filepath_raw = abs_path - image.file_format = 'PNG' - image.save() - print(f"Dot texture saved to {abs_path}") - return image - -# ---------------------------------------------------------------------- -# Main -# ---------------------------------------------------------------------- -def main(): - # ==================== CONFIGURATION ==================== - MODE = "patches" # "patches", "checker", or "dots" - OBJ_NAME = "FingerMesh0607.001" - CAM_NAME = "ov9734-114.001" - TEXTURE_WIDTH = 1640*2 - TEXTURE_HEIGHT = 1600*2 - OUTPUT_NAMES = { - "patches": "patch_texture.png", - "checker": "checker_texture.png", - "dots": "dot_texture.png", - } - OUTPUT_PATH = "//" + OUTPUT_NAMES.get(MODE, "texture.png") - - # Patch mode - PATCH_NX = 80; PATCH_NY = 45; COLOR_R = 0.05 - EDGE_SAMPLE_GRID = 5 - FIDUCIAL_NX = 3; FIDUCIAL_NY = 3 - FIDUCIAL_DIAMETER_PIX = 24 - FIDUCIAL_COLOR = (1, 1, 1, 1) - FIDUCIAL_OUTLINE_COLOR = (0, 0, 0, 1) - FIDUCIAL_OUTLINE_PX = 2 - FIDUCIAL_EDGE_MARGIN_PIXELS = 12 - # Checker mode - CHECKER_NX = 40; CHECKER_NY = 40; CHECKER_GAP_FILL_PX = 6 - # Dot mode - DOT_DIAMETER_PIX = 12; DOT_SPACING_PIX = 40; DOT_COLOR = (0,0,0,1) - # Cross - CROSS_LENGTH_PX = 40 - CROSS_THICKNESS_PX = 3 - CROSS_COLOR = (1, 0, 0, 1) # red cross - # ====================================================== - - start_time = time.time() - scene = bpy.context.scene - depsgraph = bpy.context.evaluated_depsgraph_get() - - if OBJ_NAME: - obj = bpy.data.objects.get(OBJ_NAME) - else: - obj = bpy.context.active_object - if obj is None or obj.type != 'MESH': - raise ValueError("No valid mesh object found.") - - cam = bpy.data.objects.get(CAM_NAME) if CAM_NAME else scene.camera - if cam is None or cam.type != 'CAMERA': - raise ValueError("No camera found.") - - if not obj.data.uv_layers.active: - obj.data.uv_layers.new(name="UVMap") - - # Compute UV centre - uv_layer = obj.data.uv_layers.active - uv_coords = np.array([uv_layer.data[i].uv.to_tuple() for i in range(len(uv_layer.data))]) - uv_min = uv_coords.min(axis=0) - uv_max = uv_coords.max(axis=0) - uv_centre = (uv_min + uv_max) / 2.0 - print(f"UV centre: ({uv_centre[0]:.4f}, {uv_centre[1]:.4f})") - - - # Compute UV bounding box centre - me = obj.data - uv_layer = me.uv_layers.active - uv_coords = np.array([uv_layer.data[i].uv.to_tuple() for i in range(len(uv_layer.data))]) - uv_min = uv_coords.min(axis=0) - uv_max = uv_coords.max(axis=0) - uv_centre = (uv_min + uv_max) / 2.0 - print(f"UV centre: {uv_centre[0]:.4f}, {uv_centre[1]:.4f}") - - if MODE == "patches": - generate_patch_texture( - obj, cam, scene, depsgraph, - nx=PATCH_NX, ny=PATCH_NY, r=COLOR_R, - texture_width=TEXTURE_WIDTH, texture_height=TEXTURE_HEIGHT, - output_path=OUTPUT_PATH, - uv_centre=uv_centre, - cross_length=CROSS_LENGTH_PX, - cross_thickness=CROSS_THICKNESS_PX, - cross_color=CROSS_COLOR, - edge_sample_grid=EDGE_SAMPLE_GRID, - fiducial_nx=FIDUCIAL_NX, - fiducial_ny=FIDUCIAL_NY, - fiducial_diameter_pixels=FIDUCIAL_DIAMETER_PIX, - fiducial_color=FIDUCIAL_COLOR, - fiducial_outline_color=FIDUCIAL_OUTLINE_COLOR, - fiducial_outline_px=FIDUCIAL_OUTLINE_PX, - fiducial_edge_margin_pixels=FIDUCIAL_EDGE_MARGIN_PIXELS - ) - elif MODE == "checker": - generate_checker_texture( - obj, cam, scene, depsgraph, - nx=CHECKER_NX, ny=CHECKER_NY, - texture_width=TEXTURE_WIDTH, texture_height=TEXTURE_HEIGHT, - output_path=OUTPUT_PATH, - uv_centre=uv_centre, - cross_length=CROSS_LENGTH_PX, - cross_thickness=CROSS_THICKNESS_PX, - cross_color=CROSS_COLOR, - edge_sample_grid=EDGE_SAMPLE_GRID, - gap_fill_px=CHECKER_GAP_FILL_PX - ) - elif MODE == "dots": - generate_dot_texture( - obj, cam, scene, depsgraph, - dot_diameter_pixels=DOT_DIAMETER_PIX, - spacing_pixels=DOT_SPACING_PIX, - dot_color=DOT_COLOR, - texture_width=TEXTURE_WIDTH, texture_height=TEXTURE_HEIGHT, - output_path=OUTPUT_PATH, - uv_centre=uv_centre, - cross_length=CROSS_LENGTH_PX, - cross_thickness=CROSS_THICKNESS_PX, - cross_color=CROSS_COLOR - ) - else: - raise ValueError(f"Unknown MODE: {MODE!r}. Use 'patches', 'checker', or 'dots'.") - - print(f"Completed in {time.time()-start_time:.2f} seconds.") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/tools/3d_patch_generation_blender/real_image.jpg b/tools/3d_patch_generation_blender/real_image.jpg deleted file mode 100644 index 6a738a1..0000000 Binary files a/tools/3d_patch_generation_blender/real_image.jpg and /dev/null differ diff --git a/tools/3d_patch_generation_blender/render_image.png b/tools/3d_patch_generation_blender/render_image.png deleted file mode 100644 index 0ef5309..0000000 Binary files a/tools/3d_patch_generation_blender/render_image.png and /dev/null differ diff --git a/tools/3d_patch_generation_blender/unroll_realtime.py b/tools/3d_patch_generation_blender/unroll_realtime.py deleted file mode 100644 index 57bec0f..0000000 --- a/tools/3d_patch_generation_blender/unroll_realtime.py +++ /dev/null @@ -1,720 +0,0 @@ -#!/usr/bin/env python3 -""" -Real-time UV-based perspective correction for the exported finger mesh. - -This version replaces the previous cylindrical approximation with a remap built -from the mesh UV parameterization. The remap is precomputed once and then -applied to each live frame with cv2.remap(). -""" - -import argparse -import sys -import time - -import cv2 -import numpy as np -import orisys -from orisys.configs import load_config -from orisys.frame_processors import UVUnrollFrameProcessor -from orisys.unroll import ( - barycentric_grid, - build_or_load_domain, - compute_uv_distortion_metrics, - face_uvs_from_loop_data, - load_calib_json, - load_exported_data, - recommend_edge_expand, - resolve_uv_runtime_sizes, - scale_camera_geometry, - scaled_map_cache_path, - uv_triangle_to_image, -) - - -class StepTimer: - """Accumulate per-step wall times for rolling runtime averages.""" - - def __init__(self): - self._starts = {} - self.totals = {} - self.counts = {} - - def start(self, name): - self._starts[name] = time.perf_counter() - - def stop(self, name): - dt = time.perf_counter() - self._starts.pop(name) - self.totals[name] = self.totals.get(name, 0.0) + dt - self.counts[name] = self.counts.get(name, 0) + 1 - return dt - - def avg_ms(self, name): - if self.counts.get(name, 0) == 0: - return 0.0 - return 1000.0 * self.totals[name] / self.counts[name] - - -def benchmark_step(label, func, *args, **kwargs): - """Run func once and print its wall time.""" - t0 = time.perf_counter() - result = func(*args, **kwargs) - ms = (time.perf_counter() - t0) * 1000.0 - print(f"[benchmark] {label}: {ms:.1f} ms") - return result - - -def print_runtime_benchmark(timer): - """Format rolling average timings for the live loop.""" - parts = [ - f"capture={timer.avg_ms('capture'):.1f}ms", - f"uv_prep={timer.avg_ms('uv_prepare'):.1f}ms", - f"uv_remap={timer.avg_ms('uv_remap'):.1f}ms", - f"get_img={timer.avg_ms('get_img'):.1f}ms", - f"flow={timer.avg_ms('flow'):.1f}ms", - f"hhd={timer.avg_ms('hhd'):.1f}ms", - f"contact={timer.avg_ms('contact'):.1f}ms", - f"viz={timer.avg_ms('viz'):.1f}ms", - ] - return "bench[" + ", ".join(parts) + "]" - - -def resize_to_height(image, target_height): - """Resize an image to a target height while preserving aspect ratio.""" - if image.shape[0] == target_height: - return image - scale = target_height / float(image.shape[0]) - target_width = max(1, int(round(image.shape[1] * scale))) - return cv2.resize(image, (target_width, target_height), interpolation=cv2.INTER_AREA) - - -def resize_to_fit(image, target_width, target_height): - """Resize with aspect ratio preserved and pad to exact tile size.""" - h, w = image.shape[:2] - if h <= 0 or w <= 0: - return np.zeros((target_height, target_width, 3), dtype=np.uint8) - - scale = min(target_width / float(w), target_height / float(h)) - new_w = max(1, int(round(w * scale))) - new_h = max(1, int(round(h * scale))) - resized = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_AREA) - - if resized.ndim == 2: - resized = cv2.cvtColor(resized, cv2.COLOR_GRAY2BGR) - elif resized.ndim == 3 and resized.shape[2] == 1: - resized = cv2.cvtColor(resized[:, :, 0], cv2.COLOR_GRAY2BGR) - - canvas = np.zeros((target_height, target_width, 3), dtype=np.uint8) - y0 = (target_height - new_h) // 2 - x0 = (target_width - new_w) // 2 - canvas[y0:y0 + new_h, x0:x0 + new_w] = resized - return canvas - - -def add_panel_title(image, title): - """Overlay a simple title onto a display tile.""" - out = image.copy() - cv2.rectangle(out, (0, 0), (out.shape[1], 36), (0, 0, 0), thickness=-1) - cv2.putText(out, title, (12, 24), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2, cv2.LINE_AA) - return out - - -def apply_display_mask(image, valid_mask, border_value=0): - """Zero invalid UV pixels in visualization panels.""" - if image is None or valid_mask is None: - return image - out = image.copy() - mask = valid_mask.astype(np.uint8) - if mask.shape != out.shape[:2]: - mask = cv2.resize(mask, (out.shape[1], out.shape[0]), interpolation=cv2.INTER_NEAREST) - invalid = ~mask.astype(bool) - out[invalid] = border_value - return out - - -def contact_inside_mask(centroid, valid_mask): - if centroid is None or valid_mask is None: - return None - if len(centroid) < 2: - return None - cx, cy = int(centroid[0]), int(centroid[1]) - if cy < 0 or cy >= valid_mask.shape[0] or cx < 0 or cx >= valid_mask.shape[1]: - return False - return bool(valid_mask[cy, cx]) - - -def overlay_runtime_status(image, sensor, centroid=None): - """Annotate a panel with mask-aware runtime diagnostics.""" - out = image.copy() - stats = getattr(sensor, "mask_stats", {}) or {} - coverage = 100.0 * float(stats.get("coverage", 0.0)) - flow_valid = float(stats.get("flow_mean_valid", 0.0)) - flow_invalid = float(stats.get("flow_mean_invalid", 0.0)) - centroid_ok = contact_inside_mask(centroid, getattr(sensor, "valid_mask", None)) - centroid_label = "n/a" if centroid_ok is None else ("inside" if centroid_ok else "outside") - lines = [ - f"mask {coverage:.1f}%", - f"flow valid {flow_valid:.3f}", - f"flow invalid {flow_invalid:.3f}", - f"contact {centroid_label}", - ] - y = 54 - for line in lines: - cv2.putText(out, line, (12, y), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (255, 255, 255), 2, cv2.LINE_AA) - y += 24 - return out - - -def make_tiled_dashboard(panels, tile_width=640, tile_height=360): - """Arrange four panels into a 2x2 dashboard.""" - tiles = [ - add_panel_title(resize_to_fit(image, tile_width, tile_height), title) - for title, image in panels - ] - top_row = np.hstack(tiles[:2]) - bottom_row = np.hstack(tiles[2:4]) - return np.vstack([top_row, bottom_row]) - - -def make_uv_debug_image(data, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0): - """Create a UV-domain reference image to verify orientation of the remap.""" - face_uvs = face_uvs_from_loop_data(data["faces"], data["uv_data"], data["loop_to_vertex"]) - uv_img = np.zeros((out_height, out_width, 3), dtype=np.uint8) - label_color = (255, 255, 255) - - for uv_tri in face_uvs: - tri = uv_triangle_to_image( - uv_tri, - out_width, - out_height, - flip_v=flip_v, - edge_band=edge_band, - edge_expand=edge_expand, - ) - pts = np.round(tri).astype(np.int32) - - mask = np.zeros((out_height, out_width), dtype=np.uint8) - cv2.fillConvexPoly(mask, pts, 255) - - x0 = max(0, int(np.floor(np.min(tri[:, 0])))) - x1 = min(out_width - 1, int(np.ceil(np.max(tri[:, 0])))) - y0 = max(0, int(np.floor(np.min(tri[:, 1])))) - y1 = min(out_height - 1, int(np.ceil(np.max(tri[:, 1])))) - if x1 < x0 or y1 < y0: - continue - - yy, xx = np.mgrid[y0:y1 + 1, x0:x1 + 1] - inside = mask[y0:y1 + 1, x0:x1 + 1] > 0 - if not np.any(inside): - continue - - bary = barycentric_grid(xx + 0.5, yy + 0.5, tri) - if bary is None: - continue - w0, w1, w2 = bary - valid = inside & (w0 >= -1e-6) & (w1 >= -1e-6) & (w2 >= -1e-6) - if not np.any(valid): - continue - - uv = ( - w0[valid, None] * uv_tri[0] - + w1[valid, None] * uv_tri[1] - + w2[valid, None] * uv_tri[2] - ) - colors = np.zeros((uv.shape[0], 3), dtype=np.uint8) - colors[:, 0] = np.clip(uv[:, 0] * 255.0, 0, 255).astype(np.uint8) - colors[:, 1] = np.clip(uv[:, 1] * 255.0, 0, 255).astype(np.uint8) - colors[:, 2] = np.clip((1.0 - 0.5 * (uv[:, 0] + uv[:, 1])) * 255.0, 0, 255).astype(np.uint8) - uv_img[yy[valid], xx[valid]] = colors - - cv2.polylines(uv_img, [pts.reshape(-1, 1, 2)], isClosed=True, color=(255, 255, 255), thickness=1) - - cv2.arrowedLine(uv_img, (20, out_height - 20), (min(out_width - 20, 140), out_height - 20), (0, 0, 255), 2, tipLength=0.15) - cv2.arrowedLine(uv_img, (20, out_height - 20), (20, max(20, out_height - 140)), (0, 255, 0), 2, tipLength=0.15) - cv2.putText(uv_img, "U", (min(out_width - 30, 145), out_height - 28), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2, cv2.LINE_AA) - cv2.putText(uv_img, "V", (28, max(30, out_height - 145)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2, cv2.LINE_AA) - cv2.putText(uv_img, "UV Map", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.9, label_color, 2, cv2.LINE_AA) - if edge_expand > 1.0: - cv2.putText(uv_img, f"Edge expand {edge_expand:.2f}x", (20, 64), cv2.FONT_HERSHEY_SIMPLEX, 0.55, label_color, 2, cv2.LINE_AA) - return uv_img - - -def make_uv_distortion_image(data, distortion, out_width, out_height, flip_v=True, edge_band=0.12, edge_expand=1.0): - """Render UV distortion heatmap; red means more 3D surface per UV area.""" - face_uvs = face_uvs_from_loop_data(data["faces"], data["uv_data"], data["loop_to_vertex"]) - uv_img = np.zeros((out_height, out_width, 3), dtype=np.uint8) - rel = distortion["relative_scale"] - edge_mask = distortion["edge_mask"] - valid_rel = np.isfinite(rel) & (rel > 0) - if not np.any(valid_rel): - return uv_img - - log_rel = np.log2(rel[valid_rel]) - clip = max(0.25, float(np.percentile(np.abs(log_rel), 95))) - - for idx, uv_tri in enumerate(face_uvs): - tri = uv_triangle_to_image( - uv_tri, - out_width, - out_height, - flip_v=flip_v, - edge_band=edge_band, - edge_expand=edge_expand, - ) - pts = np.round(tri).astype(np.int32) - - if not np.isfinite(rel[idx]) or rel[idx] <= 0: - color = (64, 64, 64) - else: - value = np.clip((np.log2(rel[idx]) + clip) / (2.0 * clip), 0.0, 1.0) - color_idx = np.uint8(round(value * 255.0)) - color = tuple(int(c) for c in cv2.applyColorMap(np.array([[color_idx]], dtype=np.uint8), cv2.COLORMAP_JET)[0, 0]) - - cv2.fillConvexPoly(uv_img, pts, color) - edge_color = (255, 255, 255) if edge_mask[idx] else (180, 180, 180) - edge_thickness = 2 if edge_mask[idx] else 1 - cv2.polylines(uv_img, [pts.reshape(-1, 1, 2)], isClosed=True, color=edge_color, thickness=edge_thickness) - - cv2.putText(uv_img, "UV Distortion", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 255), 2, cv2.LINE_AA) - cv2.putText(uv_img, "Blue: denser UV Red: sparser UV", (20, 64), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (220, 220, 220), 2, cv2.LINE_AA) - cv2.putText(uv_img, "White outlines: edge-band faces", (20, 92), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (220, 220, 220), 2, cv2.LINE_AA) - middle_stats = distortion["middle_stats"] - edge_stats = distortion["edge_stats"] - cv2.putText( - uv_img, - f"middle p05/p95: {middle_stats['relative_scale_p05']:.2f}x / {middle_stats['relative_scale_p95']:.2f}x", - (20, 120), - cv2.FONT_HERSHEY_SIMPLEX, - 0.55, - (220, 220, 220), - 2, - cv2.LINE_AA, - ) - cv2.putText( - uv_img, - f"edge p05/p95: {edge_stats['relative_scale_p05']:.2f}x / {edge_stats['relative_scale_p95']:.2f}x", - (20, 148), - cv2.FONT_HERSHEY_SIMPLEX, - 0.55, - (220, 220, 220), - 2, - cv2.LINE_AA, - ) - return uv_img - - -def open_source(video_path, camera_id): - return cv2.VideoCapture(video_path) if video_path else cv2.VideoCapture(camera_id) - - -PERF_TUNING_EPILOG = """ -Performance tuning (config JSON + CLI): - camera.resolution_stitch [280, 300] UV output + SDK flow/HHD grid - camera.source_scale 0.4 shrink source before uv_prep/remap - tracker.dis_preset medium -> fast faster DIS optical flow - tracker.hhd_downsample 5 -> 4 smaller HHD Poisson grid - contact.denoise_sigma 12 -> 6 or 0 faster FoE contact step - --source_scale 0.4 override config source scale - --masked_poisson_solver sparse vs capfft benchmark HHD solve time - --no-viz skip dashboard ~20 ms/frame savings - -Recommended dd02-ov fast profile: - resolution_stitch [280, 300], source_scale 0.4, hhd_downsample 5 - Delete or rename stale unroll_map_uv.npz after changing output scale. -""" - - -def run_deformation_timed(sensor, timer, check_motion=True, threshold=0, tail_frames=10): - """Run flow + gated HHD with separate benchmark buckets.""" - timer.start("flow") - sensor.compute_flow() - timer.stop("flow") - - timer.start("hhd") - run_hhd = True - if check_motion: - is_motion = sensor.detect_motion(threshold=threshold) - try: - tail_frames = max(0, int(tail_frames)) - except (TypeError, ValueError): - tail_frames = 0 - if is_motion: - sensor.tail_frames_count = 0 - elif sensor.tail_frames_count < tail_frames: - sensor.tail_frames_count += 1 - else: - run_hhd = False - sensor.tail_frames_count = tail_frames - if run_hhd: - sensor.compute_hhd() - timer.stop("hhd") - - -def configure_masked_poisson(sensor, args): - """Apply masked-HHD solver settings to the externally fed SDK sensor.""" - sensor.masked_poisson_solver = args.masked_poisson_solver - sensor.masked_poisson_max_boundary = args.masked_poisson_max_boundary - sensor.masked_poisson_boundary_stride = args.masked_poisson_boundary_stride - sensor.masked_poisson_max_iters = args.masked_poisson_max_iters - sensor.masked_poisson_tol = args.masked_poisson_tol - sensor.masked_poisson_reg = args.masked_poisson_reg - - -def resolve_runtime_geometry(args): - """Apply config defaults and optional source-scale to UV/camera geometry.""" - sdk_config = load_config(args.sdk_config) - camera_config = sdk_config.get("camera", {}) - out_width, out_height, source_scale = resolve_uv_runtime_sizes( - camera_config, - out_width=args.out_width, - out_height=args.out_height, - source_scale=args.source_scale, - ) - stitch = camera_config.get("resolution_stitch") - if stitch and len(stitch) >= 2 and (int(stitch[0]) != out_width or int(stitch[1]) != out_height): - print( - "Warning: UV output " - f"{out_width}x{out_height} differs from resolution_stitch " - f"{int(stitch[0])}x{int(stitch[1])} in {args.sdk_config}; " - "Sensor.get_img will resize again." - ) - - args.out_width = out_width - args.out_height = out_height - args.source_scale = source_scale - args.map_cache = scaled_map_cache_path(args.map_cache, source_scale) - return sdk_config, camera_config - - -def main(): - parser = argparse.ArgumentParser( - description="Real-time UV-based finger unrolling", - epilog=PERF_TUNING_EPILOG, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("--mesh", required=True, help="Exported mesh.npz path") - parser.add_argument("--K", dest="intrinsics_path", required=True, help="Exported camera_intrinsics.npz path") - parser.add_argument("--pose", required=True, help="Exported world_to_cam.npy path") - parser.add_argument("--calib", default=None, help="OpenCV calibration JSON for the live camera") - # With a 12% UV edge band on each side, only about 76% of each axis is the - # lower-distortion middle region. Use a slightly larger full remap so the - # useful center keeps roughly the same sampling density as the prior 506x550. - parser.add_argument("--out_width", type=int, default=None, help="UV flatten width (default: camera.resolution_stitch in sdk config)") - parser.add_argument("--out_height", type=int, default=None, help="UV flatten height (default: camera.resolution_stitch in sdk config)") - parser.add_argument( - "--source_scale", - type=float, - default=None, - help="Resize live source to this fraction before uv_prep/uv_remap; scales K accordingly (default: camera.source_scale or work4[3] in sdk config, else 1.0)", - ) - parser.add_argument("--camera_id", type=int, default=0) - parser.add_argument("--video", help="Video file path (overrides camera)") - parser.add_argument("--map_cache", default="unroll_map_uv.npz") - parser.add_argument("--no_flip_v", action="store_true", help="Keep UV V increasing downward") - parser.add_argument("--no_flip_source_vertical", action="store_true", help="Do not vertically flip the live source before UV remap") - parser.add_argument("--edge_band", type=float, default=0.12, help="Normalized UV band near the boundary treated as high-distortion edge") - parser.add_argument("--edge_expand", type=float, default=0.0, help="Extra UV allocation for edge band; 0 chooses automatically from distortion stats") - parser.add_argument("--sdk_config", default="./config/dd02-ov.json", help="Orisys SDK config path/name") - parser.add_argument("--sdk_source", default=None, help="Dummy source used to initialize Sensor; defaults to --video or --camera_id") - parser.add_argument("--masked_poisson_solver", choices=["sparse", "capfft"], default="sparse", help="Masked HHD Poisson solver used inside the SDK") - parser.add_argument("--masked_poisson_max_boundary", type=int, default=96, help="Maximum number of UV-boundary samples used by the capfft solver") - parser.add_argument("--masked_poisson_boundary_stride", type=int, default=1, help="Subsample stride applied before capfft boundary sampling") - parser.add_argument("--masked_poisson_max_iters", type=int, default=3, help="Maximum capfft correction iterations per Poisson solve") - parser.add_argument("--masked_poisson_tol", type=float, default=1e-4, help="Residual tolerance used by the capfft correction loop") - parser.add_argument("--masked_poisson_reg", type=float, default=1e-8, help="Diagonal regularization applied to the capacitance matrix") - parser.add_argument("--verbose", action="store_true", help="Enable verbose SDK logs") - parser.add_argument( - "--no-viz", - action="store_true", - help="Skip OpenCV dashboard; print forces and benchmark timings only", - ) - args = parser.parse_args() - _, camera_config = resolve_runtime_geometry(args) - - print("[benchmark] Startup timing:") - data = benchmark_step("load_exported_data", load_exported_data, args.mesh, args.intrinsics_path, args.pose) - flip_v = not args.no_flip_v - flip_source_vertical = not args.no_flip_source_vertical - distortion = benchmark_step( - "compute_uv_distortion_metrics", - compute_uv_distortion_metrics, - data, - edge_band=args.edge_band, - ) - distortion_stats = distortion["stats"] - middle_stats = distortion["middle_stats"] - edge_stats = distortion["edge_stats"] - edge_expand = args.edge_expand if args.edge_expand > 1.0 else recommend_edge_expand(distortion) - print( - "UV distortion stats: " - f"median mm/uv={distortion_stats['median_mm_per_uv']:.3f}, " - f"relative scale p05/p95={distortion_stats['relative_scale_p05']:.2f}x/" - f"{distortion_stats['relative_scale_p95']:.2f}x, " - f"anisotropy p50/p95={distortion_stats['anisotropy_p50']:.2f}x/" - f"{distortion_stats['anisotropy_p95']:.2f}x" - ) - print(f"Using UV edge warp: band={args.edge_band:.2f}, expand={edge_expand:.2f}x") - print( - "UV middle-region stats: " - f"faces={middle_stats['face_count']}, " - f"relative scale p05/p95={middle_stats['relative_scale_p05']:.2f}x/" - f"{middle_stats['relative_scale_p95']:.2f}x, " - f"anisotropy p50/p95={middle_stats['anisotropy_p50']:.2f}x/" - f"{middle_stats['anisotropy_p95']:.2f}x" - ) - print( - "UV edge-band stats: " - f"faces={edge_stats['face_count']}, " - f"relative scale p05/p95={edge_stats['relative_scale_p05']:.2f}x/" - f"{edge_stats['relative_scale_p95']:.2f}x, " - f"anisotropy p50/p95={edge_stats['anisotropy_p50']:.2f}x/" - f"{edge_stats['anisotropy_p95']:.2f}x" - ) - - if args.calib: - source_K, dist_coeffs, img_size = benchmark_step("load_calib_json", load_calib_json, args.calib) - native_width, native_height = img_size - print(f"Calibration loaded. Live image size: {(native_width, native_height)}") - else: - source_K = data["K"] - dist_coeffs = None - native_width = data["width"] - native_height = data["height"] - print("No calibration file provided; using exported intrinsics for projection.") - - source_K, source_width, source_height = scale_camera_geometry( - source_K, - native_width, - native_height, - args.source_scale, - ) - print( - "UV runtime geometry: " - f"native={native_width}x{native_height}, " - f"source={source_width}x{source_height} (scale={args.source_scale:g}), " - f"output={args.out_width}x{args.out_height}, " - f"map_cache={args.map_cache}" - ) - - domain = benchmark_step( - "build_or_load_domain", - build_or_load_domain, - args.mesh, - args.intrinsics_path, - args.pose, - args.map_cache, - source_K, - source_width, - source_height, - args.out_width, - args.out_height, - flip_v=flip_v, - edge_band=args.edge_band, - edge_expand=edge_expand, - include_surface=True, - source_scale=args.source_scale, - ) - remap = domain.remap - remap.ensure_fast_maps() - edge_expand = float(remap.metadata.get("edge_expand", edge_expand)) - - uv_debug = benchmark_step( - "make_uv_debug_image", - make_uv_debug_image, - data, - out_width=args.out_width, - out_height=args.out_height, - flip_v=flip_v, - edge_band=args.edge_band, - edge_expand=edge_expand, - ) - uv_distortion = benchmark_step( - "make_uv_distortion_image", - make_uv_distortion_image, - data, - distortion, - out_width=args.out_width, - out_height=args.out_height, - flip_v=flip_v, - edge_band=args.edge_band, - edge_expand=edge_expand, - ) - uv_reference = benchmark_step( - "uv_reference_compose", - lambda: np.hstack( - [ - resize_to_height(uv_debug, args.out_height), - resize_to_height(uv_distortion, args.out_height), - ] - ), - ) - - cap = open_source(args.video, args.camera_id) - if not cap.isOpened(): - print("Cannot open camera/video source") - sys.exit(1) - - cap.set(cv2.CAP_PROP_FRAME_WIDTH, native_width) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, native_height) - - sensor = benchmark_step( - "sensor_init", - lambda: orisys.Sensor( - "external", - config_name=args.sdk_config, - verbose=args.verbose, - mirror=False, - ), - ) - configure_masked_poisson(sensor, args) - print( - "Masked HHD solver: " - f"{sensor.masked_poisson_solver} " - f"(max_boundary={sensor.masked_poisson_max_boundary}, " - f"stride={sensor.masked_poisson_boundary_stride}, " - f"iters={sensor.masked_poisson_max_iters}, " - f"tol={sensor.masked_poisson_tol:.1e})" - ) - - frame_processor = UVUnrollFrameProcessor( - domain.remap, - camera_matrix=source_K if dist_coeffs is not None else None, - dist_coeffs=dist_coeffs, - source_width=source_width, - source_height=source_height, - flip_source_vertical=flip_source_vertical, - border_value=(0, 0, 0), - ) - - ok, warmup_frame = cap.read() - if ok: - warmup_prepared = frame_processor.prepare_source(warmup_frame) - warmup_unrolled = frame_processor.flatten_prepared(warmup_prepared) - sensor.get_img( - frame=warmup_unrolled, - already_processed=True, - valid_mask=frame_processor.valid_mask, - ) - benchmark_step("hhd_warmup", sensor.warmup_deformation) - else: - print("[benchmark] hhd_warmup: skipped (no frame from source)") - - if args.no_viz: - print("Running with --no-viz (dashboard disabled).") - else: - print("Press 'q' to quit.") - runtime_timer = StepTimer() - try: - while True: - runtime_timer.start("capture") - ok, frame = cap.read() - runtime_timer.stop("capture") - if not ok: - break - - runtime_timer.start("uv_prepare") - frame_source = frame_processor.prepare_source(frame) - runtime_timer.stop("uv_prepare") - - runtime_timer.start("uv_remap") - unrolled = frame_processor.flatten_prepared(frame_source) - runtime_timer.stop("uv_remap") - - runtime_timer.start("get_img") - sensor.get_img( - frame=unrolled, - already_processed=True, - valid_mask=frame_processor.valid_mask, - ) - runtime_timer.stop("get_img") - - run_deformation_timed(sensor, runtime_timer, check_motion=True, threshold=0) - - runtime_timer.start("contact") - sensor.compute_contact() - runtime_timer.stop("contact") - - fps = sensor._fps - fnormal = sensor.fnormal - fshearx = sensor.fshearx - fsheary = sensor.fsheary - contact_ok = contact_inside_mask(sensor.centroid, sensor.valid_mask) - - if not args.no_viz: - runtime_timer.start("viz") - fps, flow, vnormal, vshear, img, contour, centroid, depth_map, fnormal, fshearx, fsheary = sensor.read_info( - sensor.info.FPS, - sensor.info.VRAW, - sensor.info.VNORMAL, - sensor.info.VSHEAR, - sensor.info.IMG, - sensor.info.CONTOUR, - sensor.info.CENTROID, - sensor.info.DEPTH, - sensor.info.FNORMAL, - sensor.info.FSHEARX, - sensor.info.FSHEARY, - ) - - arrows = orisys.util.draw_arrows( - img, - flow, - threshold=0.4, - grid_spacing=10, - arrow_scale=1.0, - ) - contact_img = orisys.util.draw_contact(img, contour, centroid) - - depth_norm = depth_map - if depth_norm.size > 0 and depth_norm.max() > depth_norm.min(): - depth_u8 = ((depth_norm - depth_norm.min()) / (depth_norm.max() - depth_norm.min()) * 255).astype(np.uint8) - else: - depth_u8 = np.zeros_like(depth_norm, dtype=np.uint8) - depth_colored = cv2.applyColorMap(depth_u8, cv2.COLORMAP_JET) - - display_mask = sensor.valid_mask - arrows = apply_display_mask(arrows, display_mask, border_value=(0, 0, 0)) - contact_img = apply_display_mask(contact_img, display_mask, border_value=(0, 0, 0)) - depth_colored = apply_display_mask(depth_colored, display_mask, border_value=(0, 0, 0)) - arrows = overlay_runtime_status(arrows, sensor, centroid=centroid) - contact_ok = contact_inside_mask(centroid, display_mask) - - dashboard = make_tiled_dashboard( - [ - ("Original", frame_source), - ("UV Map | Distortion", uv_reference), - ("UV Unrolled", unrolled), - ("Flow", arrows), - ], - tile_width=640, - tile_height=360, - ) - cv2.imshow("UV Unroll Dashboard", dashboard) - runtime_timer.stop("viz") - - print( - f"FPS={fps:.2f}, 法向力={fnormal:.4f}, 切向力X={fshearx:.4f}, " - f"切向力Y={fsheary:.4f}, 展开图尺寸={unrolled.shape[:2]}, " - f"solver={sensor.masked_poisson_solver}, " - f"mask={sensor.mask_stats['coverage']:.3f}, " - f"flow(valid/invalid)=({sensor.mask_stats['flow_mean_valid']:.3f}/" - f"{sensor.mask_stats['flow_mean_invalid']:.3f}), " - f"contact_in_mask={contact_ok}, " - f"{print_runtime_benchmark(runtime_timer)}", - end="\r", - ) - - key = cv2.waitKey(1) & 0xFF - if key == ord("q"): - print("\n正在退出程序...") - break - if key == ord("r"): - sensor.reset() - print("\n追踪器已重置。") - finally: - cap.release() - sensor.disconnect() - cv2.destroyAllWindows() - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/tools/3d_patch_generation_blender/unrolled.png b/tools/3d_patch_generation_blender/unrolled.png deleted file mode 100644 index 1de9fcf..0000000 Binary files a/tools/3d_patch_generation_blender/unrolled.png and /dev/null differ diff --git a/tools/dd02_3dviewer/exported_data/camera_intrinsics.npz b/tools/dd02_3dviewer/exported_data/camera_intrinsics.npz deleted file mode 100644 index 56b7885..0000000 Binary files a/tools/dd02_3dviewer/exported_data/camera_intrinsics.npz and /dev/null differ diff --git a/tools/dd02_3dviewer/exported_data/flatten_remap.npz b/tools/dd02_3dviewer/exported_data/flatten_remap.npz deleted file mode 100644 index 02ca287..0000000 Binary files a/tools/dd02_3dviewer/exported_data/flatten_remap.npz and /dev/null differ diff --git a/tools/dd02_3dviewer/exported_data/mesh.npz b/tools/dd02_3dviewer/exported_data/mesh.npz deleted file mode 100644 index 0b95784..0000000 Binary files a/tools/dd02_3dviewer/exported_data/mesh.npz and /dev/null differ diff --git a/tools/dd02_3dviewer/exported_data/pixel_to_xyz.npz b/tools/dd02_3dviewer/exported_data/pixel_to_xyz.npz deleted file mode 100644 index 7c83a7b..0000000 Binary files a/tools/dd02_3dviewer/exported_data/pixel_to_xyz.npz and /dev/null differ diff --git a/tools/dd02_3dviewer/exported_data/world_to_cam.npy b/tools/dd02_3dviewer/exported_data/world_to_cam.npy deleted file mode 100644 index e59cfe7..0000000 Binary files a/tools/dd02_3dviewer/exported_data/world_to_cam.npy and /dev/null differ diff --git a/tools/dd02_3dviewer/mapto3d.py b/tools/dd02_3dviewer/mapto3d.py deleted file mode 100644 index 49ffb05..0000000 --- a/tools/dd02_3dviewer/mapto3d.py +++ /dev/null @@ -1,188 +0,0 @@ -import numpy as np -import cv2 -import os - -def load_exported_data(data_dir): - """Load all exported files.""" - # Mesh - mesh = np.load(os.path.join(data_dir, "mesh.npz")) - vertices = mesh['vertices'] - faces = mesh['faces'] - uv_data = mesh['uv_data'] - loop_to_vertex = mesh['loop_to_vertex'] - - # Camera intrinsics - cam_data = np.load(os.path.join(data_dir, "camera_intrinsics.npz")) - K = cam_data['K'] - width = int(cam_data['width']) - height = int(cam_data['height']) - - # Camera pose (world-to-camera) - world_to_cam = np.load(os.path.join(data_dir, "world_to_cam.npy")) - - # Precomputed pixel-to-3D map (optional) - xyz_map_path = os.path.join(data_dir, "pixel_to_xyz.npz") - if os.path.exists(xyz_map_path): - xyz_map = np.load(xyz_map_path)['xyz_map'] - else: - xyz_map = None - - return { - 'vertices': vertices, - 'faces': faces, - 'uv_data': uv_data, - 'loop_to_vertex': loop_to_vertex, - 'K': K, - 'width': width, - 'height': height, - 'world_to_cam': world_to_cam, - 'xyz_map': xyz_map - } - -def ray_intersect_triangle(ray_origin, ray_dir, v0, v1, v2): - """Möller–Trumbore intersection algorithm. Returns t (distance) or None.""" - epsilon = 1e-8 - edge1 = v1 - v0 - edge2 = v2 - v0 - h = np.cross(ray_dir, edge2) - a = np.dot(edge1, h) - if abs(a) < epsilon: - return None - f = 1.0 / a - s = ray_origin - v0 - u = f * np.dot(s, h) - if u < 0.0 or u > 1.0: - return None - q = np.cross(s, edge1) - v = f * np.dot(ray_dir, q) - if v < 0.0 or u + v > 1.0: - return None - t = f * np.dot(edge2, q) - if t > epsilon: - return t - return None - -def pixel_to_3d_ray(u, v, K, world_to_cam): - """ - Given pixel coordinates (u,v) and camera intrinsics K, - return a ray in world coordinates: origin and direction. - """ - # Invert K to get normalized image coordinates - fx = K[0,0]; fy = K[1,1]; cx = K[0,2]; cy = K[1,2] - x_norm = (u - cx) / fx - y_norm = (v - cy) / fy - - # Ray in camera space: (x_norm, y_norm, 1.0) - ray_cam = np.array([x_norm, y_norm, 1.0], dtype=np.float64) - - # Transform to world - cam_to_world = np.linalg.inv(world_to_cam) - R = cam_to_world[:3, :3] - t = cam_to_world[:3, 3] - ray_world_dir = R @ ray_cam - ray_world_dir /= np.linalg.norm(ray_world_dir) - ray_world_origin = t - - return ray_world_origin, ray_world_dir - -def map_image_to_3d(image_path, data, use_precomputed=True, flip_vertical=True): - """ - Map each pixel of the image to a 3D world point. - If use_precomputed and xyz_map exists, return that directly. - If `flip_vertical` is True, flip the loaded image top-to-bottom so its - row order matches the exported Blender camera/pixel mapping. - Otherwise, perform raycasting on the mesh. - Returns: - points: (H, W, 3) array of 3D points (NaN for misses). - colors: (H, W, 3) array of RGB colors from image. - """ - img = cv2.imread(image_path) - if img is None: - raise FileNotFoundError(f"Could not read image: {image_path}") - img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # to RGB - if flip_vertical: - img = np.flipud(img).copy() - h, w = img.shape[:2] - - if use_precomputed and data['xyz_map'] is not None: - # Use precomputed map (must match resolution) - if h != data['height'] or w != data['width']: - raise ValueError("Image resolution does not match precomputed map.") - points = data['xyz_map'].copy() - colors = img.astype(np.float32) / 255.0 - return points, colors - - # Otherwise, perform raycasting on the mesh - vertices = data['vertices'] - faces = data['faces'] - K = data['K'] - world_to_cam = data['world_to_cam'] - - points = np.full((h, w, 3), np.nan, dtype=np.float32) - colors = img.astype(np.float32) / 255.0 - - # For speed, we can use a bounding volume hierarchy (BVH). - # Here we implement a simple brute-force over all faces (slow for large meshes). - # In practice, precomputing the xyz_map is strongly recommended. - - for y in range(h): - for x in range(w): - origin, direction = pixel_to_3d_ray(x, y, K, world_to_cam) - best_t = np.inf - best_hit = None - for face in faces: - v0 = vertices[face[0]] - v1 = vertices[face[1]] - v2 = vertices[face[2]] - t = ray_intersect_triangle(origin, direction, v0, v1, v2) - if t is not None and t < best_t: - best_t = t - best_hit = origin + direction * t - if best_hit is not None: - points[y, x] = best_hit - - return points, colors - -def save_colored_pointcloud(points, colors, output_path): - """Save as PLY file (colored point cloud).""" - # Flatten valid points - mask = ~np.isnan(points[:,:,0]) - valid_points = points[mask].reshape(-1, 3) - valid_colors = (colors[mask].reshape(-1, 3) * 255).astype(np.uint8) - - with open(output_path, 'w') as f: - f.write("ply\n") - f.write("format ascii 1.0\n") - f.write(f"element vertex {len(valid_points)}\n") - f.write("property float x\n") - f.write("property float y\n") - f.write("property float z\n") - f.write("property uchar red\n") - f.write("property uchar green\n") - f.write("property uchar blue\n") - f.write("end_header\n") - for pt, col in zip(valid_points, valid_colors): - f.write(f"{pt[0]} {pt[1]} {pt[2]} {col[0]} {col[1]} {col[2]}\n") - print(f"Saved point cloud to {output_path}") - -if __name__ == "__main__": - data_dir = "./exported_data" # Path to exported files - image_path = "captured_image_1.jpg" # Input camera image - output_ply = "mapped_points.ply" - - data = load_exported_data(data_dir) - points, colors = map_image_to_3d( - image_path, - data, - use_precomputed=True, - flip_vertical=True, - ) - save_colored_pointcloud(points, colors, output_ply) - - # Optional: Visualize using Open3D - import open3d as o3d - pcd = o3d.geometry.PointCloud() - mask = ~np.isnan(points[:,:,0]) - pcd.points = o3d.utility.Vector3dVector(points[mask].reshape(-1,3)) - pcd.colors = o3d.utility.Vector3dVector(colors[mask].reshape(-1,3)) - o3d.visualization.draw_geometries([pcd]) \ No newline at end of file diff --git a/tools/dd02_3dviewer/sdk_mapto3d.py b/tools/dd02_3dviewer/sdk_mapto3d.py deleted file mode 100644 index 31cd6d7..0000000 --- a/tools/dd02_3dviewer/sdk_mapto3d.py +++ /dev/null @@ -1,629 +0,0 @@ -import argparse -import os -import sys -import time - -import cv2 -import numpy as np - -# UV-unroll mesh viewer (grey-white surface, UV-space field mapping): -# from examples.viz.mesh_viewer import UVMeshViewer, build_uv_surface_mesh -# # or: from orisys.finger import UVMeshViewer, build_uv_surface_mesh -# See examples/finger_pipeline_3dview.py - - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -REPO_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..")) -SRC_DIR = os.path.join(REPO_ROOT, "src") -if SRC_DIR not in sys.path: - sys.path.insert(0, SRC_DIR) - -import orisys - - -def load_exported_data(data_dir): - mesh = np.load(os.path.join(data_dir, "mesh.npz")) - cam_data = np.load(os.path.join(data_dir, "camera_intrinsics.npz")) - xyz_map_path = os.path.join(data_dir, "pixel_to_xyz.npz") - if not os.path.exists(xyz_map_path): - raise FileNotFoundError(f"Missing precomputed 3D map: {xyz_map_path}") - - xyz_map = np.load(xyz_map_path)["xyz_map"] - width = int(cam_data["width"]) - height = int(cam_data["height"]) - if xyz_map.shape[:2] != (height, width): - raise ValueError( - f"pixel_to_xyz shape {xyz_map.shape[:2]} does not match camera size {(height, width)}" - ) - - return { - "vertices": mesh["vertices"], - "faces": mesh["faces"], - "uv_data": mesh["uv_data"], - "loop_to_vertex": mesh["loop_to_vertex"], - "K": cam_data["K"], - "width": width, - "height": height, - "world_to_cam": np.load(os.path.join(data_dir, "world_to_cam.npy")), - "xyz_map": xyz_map, - } - - -def parse_source(value): - value = str(value).strip() - for prefix in ("camera:", "cam:", "webcam:"): - if value.lower().startswith(prefix): - value = value[len(prefix):].strip() - break - - try: - return int(value) - except ValueError: - return value - - -def prepare_static_points(data, display_stride): - xyz_map = data["xyz_map"] - height, width = xyz_map.shape[:2] - valid_mask = ~np.isnan(xyz_map[:, :, 0]) - valid_points = xyz_map[valid_mask].reshape(-1, 3) - if len(valid_points) == 0: - raise ValueError("pixel_to_xyz.npz contains no valid mesh hit points.") - - display_indices = np.arange(len(valid_points))[::max(1, int(display_stride))] - flat_indices = np.flatnonzero(valid_mask.ravel())[display_indices] - display_ys = flat_indices // width - display_xs = flat_indices % width - return ( - valid_mask, - display_indices, - valid_points[display_indices], - display_ys, - display_xs, - ) - - -def mesh_pixel_to_world_scale(xyz_map, valid_mask): - valid_points = xyz_map[valid_mask] - if len(valid_points) == 0: - return 1e-3 - extent = np.nanmax(valid_points, axis=0) - np.nanmin(valid_points, axis=0) - mesh_span = float(np.max(extent)) - height, width = xyz_map.shape[:2] - return mesh_span / max(height, width, 1) - - -def sample_field_at_display_points(field, display_ys, display_xs): - if field.ndim == 2: - return field[display_ys, display_xs] - return field[display_ys, display_xs, :] - - -def deform_point_cloud(rest_points, displacement, pixel_to_world, deform_scale): - return rest_points + displacement * pixel_to_world * float(deform_scale) - - -def lift_flow_to_3d(flow, tangent_x, tangent_y, normals): - """Lift raw 2D optical flow to tangent-plane 3D displacement.""" - tx_norm = np.linalg.norm(tangent_x, axis=2, keepdims=True) - ty_norm = np.linalg.norm(tangent_y, axis=2, keepdims=True) - t1 = np.divide(tangent_x, tx_norm, out=np.zeros_like(tangent_x), where=tx_norm > 1e-8) - t2 = np.divide(tangent_y, ty_norm, out=np.zeros_like(tangent_y), where=ty_norm > 1e-8) - flow_3d = flow[:, :, 0:1] * t1 + flow[:, :, 1:2] * t2 - normal_component = np.sum(flow_3d * normals, axis=2, keepdims=True) - return flow_3d - normal_component * normals - - -def align_image_to_map(img_bgr, data, valid_mask, display_indices, flip_vertical=True): - target_w = int(data["width"]) - target_h = int(data["height"]) - h, w = img_bgr.shape[:2] - if (w, h) != (target_w, target_h): - img_bgr = cv2.resize(img_bgr, (target_w, target_h), interpolation=cv2.INTER_LINEAR) - - img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB) - if flip_vertical: - img_rgb = np.flipud(img_rgb).copy() - - colors = img_rgb.astype(np.float64) / 255.0 - return colors[valid_mask].reshape(-1, 3)[display_indices] - - -def arrow_map_reference_colors(count, color=(0.04, 0.04, 0.04)): - return np.tile(np.asarray(color, dtype=np.float64), (count, 1)) - - -def magnitude_map_colors(flow, valid_mask, display_indices, max_magnitude=None): - mag = np.linalg.norm(flow, axis=2) - valid_mag = mag[valid_mask].reshape(-1) - display_mag = valid_mag[display_indices] - if max_magnitude is None or max_magnitude <= 0: - max_magnitude = max(float(np.percentile(display_mag, 95)), 1e-6) - - value = np.clip(display_mag / max_magnitude, 0.0, 1.0) - return np.column_stack( - ( - value, - np.clip(1.0 - np.abs(value - 0.5) * 2.0, 0.0, 1.0), - 1.0 - value, - ) - ).astype(np.float64) - - -def align_scalar_field_to_map(field, data, flip_vertical=True): - if field.ndim == 3: - field = np.linalg.norm(field, axis=2) - - target_w = int(data["width"]) - target_h = int(data["height"]) - h, w = field.shape[:2] - if (w, h) != (target_w, target_h): - field = cv2.resize(field, (target_w, target_h), interpolation=cv2.INTER_LINEAR) - else: - field = field.copy() - - if flip_vertical: - field = np.flipud(field).copy() - return field - - -def scalar_map_colors(field, valid_mask, display_indices, value_min=None, value_max=None): - valid_values = field[valid_mask].reshape(-1) - display_values = valid_values[display_indices] - finite_values = display_values[np.isfinite(display_values)] - if len(finite_values) == 0: - return arrow_map_reference_colors(len(display_indices)) - - if value_min is None: - value_min = float(np.percentile(finite_values, 5)) - if value_max is None: - value_max = float(np.percentile(finite_values, 95)) - if value_max <= value_min: - value_max = value_min + 1e-6 - - value = np.clip((display_values - value_min) / (value_max - value_min), 0.0, 1.0) - return np.column_stack( - ( - value, - np.clip(1.0 - np.abs(value - 0.5) * 2.0, 0.0, 1.0), - 1.0 - value, - ) - ).astype(np.float64) - - -def align_vector_field_to_map(field, data, flip_vertical=True, scale_as_flow=True): - target_w = int(data["width"]) - target_h = int(data["height"]) - h, w = field.shape[:2] - if (w, h) != (target_w, target_h): - sx = target_w / max(w, 1) - sy = target_h / max(h, 1) - field = cv2.resize(field, (target_w, target_h), interpolation=cv2.INTER_LINEAR) - if scale_as_flow: - field[:, :, 0] *= sx - field[:, :, 1] *= sy - else: - field = field.copy() - - if flip_vertical: - field = np.flipud(field).copy() - field[:, :, 1] *= -1.0 - return field - - -def estimate_surface_normals(xyz_map): - valid = ~np.isnan(xyz_map[:, :, 0]) - filled = xyz_map.copy() - filled[~valid] = 0 - d_y = np.gradient(filled, axis=0) - d_x = np.gradient(filled, axis=1) - normals = np.cross(d_x, d_y) - norm = np.linalg.norm(normals, axis=2, keepdims=True) - normals = np.divide(normals, norm, out=np.zeros_like(normals), where=norm > 1e-8) - normals[~valid] = 0 - return normals - - -def compute_surface_frames(xyz_map): - valid = ~np.isnan(xyz_map[:, :, 0]) - filled = xyz_map.copy() - filled[~valid] = 0 - d_y = np.gradient(filled, axis=0) - d_x = np.gradient(filled, axis=1) - normals = np.cross(d_x, d_y) - norm = np.linalg.norm(normals, axis=2, keepdims=True) - normals = np.divide(normals, norm, out=np.zeros_like(normals), where=norm > 1e-8) - d_x[~valid] = 0 - d_y[~valid] = 0 - normals[~valid] = 0 - return d_x, d_y, normals, valid - - -def lift_deformation_to_3d(vshear, vnormal, tangent_x, tangent_y, normals, normal_gain=1.0): - # Unit tangents: map 2D pixel shear to 3D direction without world-scale Jacobian blow-up. - tx_norm = np.linalg.norm(tangent_x, axis=2, keepdims=True) - ty_norm = np.linalg.norm(tangent_y, axis=2, keepdims=True) - t1 = np.divide(tangent_x, tx_norm, out=np.zeros_like(tangent_x), where=tx_norm > 1e-8) - t2 = np.divide(tangent_y, ty_norm, out=np.zeros_like(tangent_y), where=ty_norm > 1e-8) - - shear_3d = vshear[:, :, 0:1] * t1 + vshear[:, :, 1:2] * t2 - normal_mag = np.linalg.norm(vnormal, axis=2, keepdims=True) * float(normal_gain) - normal_3d = normal_mag * normals - - normal_component = np.sum(shear_3d * normals, axis=2, keepdims=True) - shear_tangent = shear_3d - normal_component * normals - return shear_tangent + normal_3d - - -def line_colors(mag, arrow_map_style): - if len(mag) == 0: - return np.empty((0, 3), dtype=np.float64) - if arrow_map_style: - return np.tile(np.array([0.0, 0.9, 1.0], dtype=np.float64), (len(mag), 1)) - norm_mag = mag / max(float(np.percentile(mag, 95)), 1e-6) - norm_mag = np.clip(norm_mag, 0.0, 1.0) - return np.column_stack((norm_mag, 1.0 - norm_mag, np.zeros_like(norm_mag))).astype(np.float64) - - -def lines_from_vectors(starts, vectors, mag, flow_scale, arrow_map_style, unit_direction=False): - vec_norm = np.linalg.norm(vectors, axis=1) - good = np.isfinite(vec_norm) & (vec_norm > 1e-8) - starts, vectors, mag = starts[good], vectors[good], mag[good] - if len(starts) == 0: - return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) - - if unit_direction: - vec_norm = vec_norm[:, None] - vectors = np.divide(vectors, vec_norm, out=np.zeros_like(vectors), where=vec_norm > 1e-8) - - points = np.empty((len(starts) * 2, 3), dtype=np.float64) - points[0::2] = starts - points[1::2] = starts + vectors * flow_scale - lines = np.column_stack((np.arange(0, len(points), 2), np.arange(1, len(points), 2))).astype(np.int32) - return points, lines, line_colors(mag, arrow_map_style) - - -def build_flow_lines( - xyz_map, - valid_mask, - flow, - arrow_stride=32, - flow_scale=1.0, - min_magnitude=0.5, - arrow_map_style=False, -): - h, w = flow.shape[:2] - ys, xs = np.mgrid[0:h:arrow_stride, 0:w:arrow_stride] - ys = ys.reshape(-1) - xs = xs.reshape(-1) - sample_flow = flow[ys, xs] - mag = np.linalg.norm(sample_flow, axis=1) - keep = valid_mask[ys, xs] & (mag >= min_magnitude) - xs, ys, sample_flow, mag = xs[keep], ys[keep], sample_flow[keep], mag[keep] - if len(xs) == 0: - return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) - - tx = np.rint(xs + sample_flow[:, 0]).astype(np.int32) - ty = np.rint(ys + sample_flow[:, 1]).astype(np.int32) - inside = (0 <= tx) & (tx < w) & (0 <= ty) & (ty < h) - xs, ys, tx, ty, mag = xs[inside], ys[inside], tx[inside], ty[inside], mag[inside] - if len(xs) == 0: - return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) - - valid_target = valid_mask[ty, tx] - xs, ys, tx, ty, mag = xs[valid_target], ys[valid_target], tx[valid_target], ty[valid_target], mag[valid_target] - starts = xyz_map[ys, xs] - vectors = xyz_map[ty, tx] - starts - return lines_from_vectors(starts, vectors, mag, flow_scale, arrow_map_style) - - -def build_resultant_force_lines( - xyz_map, - valid_mask, - resultant_force, - arrow_stride=32, - flow_scale=1.0, - min_magnitude=0.01, - arrow_map_style=False, -): - h, w = resultant_force.shape[:2] - ys, xs = np.mgrid[0:h:arrow_stride, 0:w:arrow_stride] - ys = ys.reshape(-1) - xs = xs.reshape(-1) - - vectors = resultant_force[ys, xs] - mag = np.linalg.norm(vectors, axis=1) - keep = valid_mask[ys, xs] & (mag >= min_magnitude) - xs, ys, vectors, mag = ( - xs[keep], - ys[keep], - vectors[keep], - mag[keep], - ) - if len(xs) == 0: - return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) - starts = xyz_map[ys, xs] - return lines_from_vectors( - starts, vectors, mag, flow_scale, arrow_map_style, unit_direction=True - ) - - -def init_viewer( - cloud_points, - cloud_colors, - flow_points, - flow_lines, - flow_colors, - arrow_map_style=False, - show_arrows=True, -): - import open3d as o3d - - vis = o3d.visualization.Visualizer() - vis.create_window(window_name="SDK 3D Flow", width=1280, height=720) - - pcd = o3d.geometry.PointCloud() - pcd.points = o3d.utility.Vector3dVector(cloud_points.astype(np.float64)) - pcd.colors = o3d.utility.Vector3dVector(cloud_colors.astype(np.float64)) - vis.add_geometry(pcd) - - line_set = None - if show_arrows: - line_set = o3d.geometry.LineSet() - line_set.points = o3d.utility.Vector3dVector(flow_points.astype(np.float64)) - line_set.lines = o3d.utility.Vector2iVector(flow_lines) - line_set.colors = o3d.utility.Vector3dVector(flow_colors.astype(np.float64)) - vis.add_geometry(line_set) - - render_option = vis.get_render_option() - render_option.point_size = 1.0 if arrow_map_style else 2.0 - render_option.line_width = 2.0 - render_option.background_color = np.array([0.0, 0.0, 0.0]) - vis.poll_events() - vis.update_renderer() - return vis, pcd, line_set, o3d - - -def run_sdk_3d_flow(args): - data = load_exported_data(args.data_dir) - valid_mask, display_indices, rest_cloud_points, display_ys, display_xs = prepare_static_points( - data, args.display_stride - ) - tangent_x, tangent_y, surface_normals, valid_mask = compute_surface_frames(data["xyz_map"]) - pixel_to_world = mesh_pixel_to_world_scale(data["xyz_map"], valid_mask) - show_arrows = args.display_mode == "arrows" - deform_cloud = args.display_mode == "deform" - - sensor = orisys.Sensor( - parse_source(args.source), - config_name=args.config, - isstitch=False, - cuda=args.cuda, - verbose=args.verbose, - mirror=not args.no_mirror, - ) - - vis = pcd = line_set = o3d = None - frame_count = 0 - t0 = time.perf_counter() - - try: - while True: - if vis is not None and not vis.poll_events(): - break - - if sensor.get_img() is None: - break - needs_decomposition = args.vector_mode in ("shear-normal", "depth-map") - sensor.compute_deformation(decompose=needs_decomposition, check_motion=False) - if args.vector_mode == "depth-map": - sensor.compute_contact() - flow, img_bgr, vshear, vnormal, depth_map = sensor.read_info( - sensor.info.VRAW, - sensor.info.IMG, - sensor.info.VSHEAR, - sensor.info.VNORMAL, - sensor.info.DEPTH, - ) - - flow_map = align_vector_field_to_map(flow, data, flip_vertical=not args.no_flip_vertical) - if args.vector_mode == "vraw-magnitude": - cloud_colors = magnitude_map_colors(flow_map, valid_mask, display_indices, args.magnitude_max) - elif args.vector_mode == "depth-map": - depth_map = align_scalar_field_to_map(depth_map, data, flip_vertical=not args.no_flip_vertical) - cloud_colors = scalar_map_colors( - depth_map, - valid_mask, - display_indices, - value_min=args.depth_min, - value_max=args.depth_max, - ) - elif args.arrow_map_style: - cloud_colors = arrow_map_reference_colors(len(rest_cloud_points)) - else: - cloud_colors = align_image_to_map( - img_bgr, - data, - valid_mask, - display_indices, - flip_vertical=not args.no_flip_vertical, - ) - - flip_vertical = not args.no_flip_vertical - displacement_map = None - if args.vector_mode == "shear-normal": - vshear_map = align_vector_field_to_map(vshear, data, flip_vertical=flip_vertical) - vnormal_map = align_vector_field_to_map(vnormal, data, flip_vertical=flip_vertical) - displacement_map = lift_deformation_to_3d( - vshear_map, - vnormal_map, - tangent_x, - tangent_y, - surface_normals, - normal_gain=args.normal_gain, - ) - elif args.vector_mode == "depth-map" and deform_cloud: - vnormal_map = align_vector_field_to_map(vnormal, data, flip_vertical=flip_vertical) - zero_shear = np.zeros_like(vnormal_map) - displacement_map = lift_deformation_to_3d( - zero_shear, - vnormal_map, - tangent_x, - tangent_y, - surface_normals, - normal_gain=args.normal_gain, - ) - elif args.vector_mode in ("vraw", "vraw-magnitude"): - displacement_map = lift_flow_to_3d(flow_map, tangent_x, tangent_y, surface_normals) - - if deform_cloud and args.deform_color == "magnitude" and displacement_map is not None: - cloud_colors = magnitude_map_colors( - displacement_map, valid_mask, display_indices, args.magnitude_max - ) - - if deform_cloud: - if displacement_map is None: - raise ValueError( - f"display-mode deform is not supported with vector-mode {args.vector_mode!r}" - ) - point_displacement = sample_field_at_display_points( - displacement_map, display_ys, display_xs - ) - cloud_points = deform_point_cloud( - rest_cloud_points, - point_displacement, - pixel_to_world, - args.deform_scale, - ) - flow_points = np.empty((0, 3), dtype=np.float64) - flow_lines = np.empty((0, 2), dtype=np.int32) - flow_colors = np.empty((0, 3), dtype=np.float64) - else: - cloud_points = rest_cloud_points - if args.vector_mode == "shear-normal": - flow_points, flow_lines, flow_colors = build_resultant_force_lines( - data["xyz_map"], - valid_mask, - displacement_map, - arrow_stride=args.arrow_stride, - flow_scale=args.flow_scale, - min_magnitude=args.min_flow, - arrow_map_style=args.arrow_map_style, - ) - elif args.vector_mode in ("vraw-magnitude", "depth-map"): - flow_points = np.empty((0, 3), dtype=np.float64) - flow_lines = np.empty((0, 2), dtype=np.int32) - flow_colors = np.empty((0, 3), dtype=np.float64) - else: - flow_points, flow_lines, flow_colors = build_flow_lines( - data["xyz_map"], - valid_mask, - flow_map, - arrow_stride=args.arrow_stride, - flow_scale=args.flow_scale, - min_magnitude=args.min_flow, - arrow_map_style=args.arrow_map_style, - ) - - if vis is None: - vis, pcd, line_set, o3d = init_viewer( - cloud_points, - cloud_colors, - flow_points, - flow_lines, - flow_colors, - arrow_map_style=args.arrow_map_style, - show_arrows=show_arrows, - ) - mode_label = "deformed cloud" if deform_cloud else "arrows" - print(f"SDK 3D flow viewer started ({mode_label}). Close the Open3D window to stop.") - else: - pcd.points = o3d.utility.Vector3dVector(cloud_points.astype(np.float64)) - pcd.colors = o3d.utility.Vector3dVector(cloud_colors) - if line_set is not None: - line_set.points = o3d.utility.Vector3dVector(flow_points.astype(np.float64)) - line_set.lines = o3d.utility.Vector2iVector(flow_lines) - line_set.colors = o3d.utility.Vector3dVector(flow_colors.astype(np.float64)) - vis.update_geometry(line_set) - vis.update_geometry(pcd) - vis.update_renderer() - - frame_count += 1 - if frame_count % 30 == 0: - elapsed = max(time.perf_counter() - t0, 1e-6) - arrow_count = len(flow_lines) if show_arrows else 0 - print(f"frames={frame_count}, viewer FPS={frame_count / elapsed:.1f}, arrows={arrow_count}") - finally: - sensor.disconnect() - if vis is not None: - vis.destroy_window() - - -def parse_args(): - parser = argparse.ArgumentParser(description="Map SDK flow fields to a 3D point cloud.") - parser.add_argument("--data-dir", default=os.path.join(SCRIPT_DIR, "exported_data")) - parser.add_argument( - "--source", - default=os.path.join(SCRIPT_DIR, "20260602-094916.mp4"), - help="Video path or live camera ID, for example 0, 1, camera:0, or cam:1.", - ) - parser.add_argument("--config", default=os.path.join(REPO_ROOT, "config", "dd02-ov.json")) - parser.add_argument("--display-stride", type=int, default=4) - parser.add_argument("--arrow-stride", type=int, default=8) - parser.add_argument( - "--flow-scale", - type=float, - default=3.0, - help="Arrow length in 3D viewer units (shear-normal uses unit direction; magnitude shown by color).", - ) - parser.add_argument( - "--min-flow", - type=float, - default=0.5, - help="Minimum 2D-equivalent resultant magnitude before drawing an arrow.", - ) - parser.add_argument("--normal-gain", type=float, default=1.0) - parser.add_argument( - "--display-mode", - choices=["arrows", "deform"], - default="arrows", - help="arrows: line vectors; deform: displace the point cloud by the 3D deformation field.", - ) - parser.add_argument( - "--deform-scale", - type=float, - default=1.0, - help="Gain on cloud displacement (multiplied by mesh pixel-to-world scale).", - ) - parser.add_argument( - "--deform-color", - choices=["texture", "magnitude"], - default="texture", - help="Point colors in deform mode: camera texture or deformation magnitude heat-map.", - ) - parser.add_argument("--magnitude-max", type=float, default=None, - help="Upper VRAW magnitude for heat-map color scaling; default uses frame percentile.") - parser.add_argument("--depth-min", type=float, default=None, - help="Minimum depth value for heat-map color scaling; default uses frame percentile.") - parser.add_argument("--depth-max", type=float, default=None, - help="Maximum depth value for heat-map color scaling; default uses frame percentile.") - parser.add_argument( - "--vector-mode", - choices=["vraw", "vraw-magnitude", "shear-normal", "depth-map"], - default="vraw", - help="Vector display mode. shear-normal shows 3D resultant vectors (surface-tangent shear + surface-normal component).", - ) - parser.add_argument( - "--arrow-map-style", - action="store_true", - help="Hide real texture and use a dark Qt-viewer-like arrow map style.", - ) - parser.add_argument("--no-flip-vertical", action="store_true") - parser.add_argument("--no-mirror", action="store_true") - parser.add_argument("--cuda", action="store_true") - parser.add_argument("--verbose", action="store_true") - return parser.parse_args() - - -if __name__ == "__main__": - run_sdk_3d_flow(parse_args()) diff --git a/tools/dd02_3dviewer/sdk_mapto3d_flatten.py b/tools/dd02_3dviewer/sdk_mapto3d_flatten.py deleted file mode 100644 index 919c712..0000000 --- a/tools/dd02_3dviewer/sdk_mapto3d_flatten.py +++ /dev/null @@ -1,564 +0,0 @@ -import argparse -import os -import sys -import time - -import cv2 -import numpy as np - - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -REPO_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..")) -SRC_DIR = os.path.join(REPO_ROOT, "src") -if SRC_DIR not in sys.path: - sys.path.insert(0, SRC_DIR) - -import orisys -from orisys.unroll import UVDomain - - -def load_flatten_remap(path): - """Load flatten cache; returns dict compatible with legacy viewer code.""" - domain = UVDomain.load(path) - result = { - "map_x": domain.remap.map_x, - "map_y": domain.remap.map_y, - "valid_mask": domain.valid_mask, - } - if domain.surface is not None: - result["flat_xyz"] = domain.surface.flat_xyz - return result - - -def flip_sensor_frames_vertical(sensor): - """Flip captured BGR/gray frames to match Blender-exported map orientation.""" - if sensor._img_bgr_raw is not None: - sensor._img_bgr_raw = cv2.flip(sensor._img_bgr_raw, 0) - if sensor.img is not None: - cv2.flip(sensor.img, 0, dst=sensor.img) - - -def flatten_frame(frame, remap, border_value=0): - flat = cv2.remap( - frame, - remap["map_x"], - remap["map_y"], - interpolation=cv2.INTER_LINEAR, - borderMode=cv2.BORDER_CONSTANT, - borderValue=border_value, - ) - flat = flat.copy() - flat[~remap["valid_mask"]] = border_value - return flat - - -def load_exported_data(data_dir): - mesh = np.load(os.path.join(data_dir, "mesh.npz")) - cam_data = np.load(os.path.join(data_dir, "camera_intrinsics.npz")) - xyz_map_path = os.path.join(data_dir, "pixel_to_xyz.npz") - if not os.path.exists(xyz_map_path): - raise FileNotFoundError(f"Missing precomputed 3D map: {xyz_map_path}") - - xyz_map = np.load(xyz_map_path)["xyz_map"] - width = int(cam_data["width"]) - height = int(cam_data["height"]) - if xyz_map.shape[:2] != (height, width): - raise ValueError( - f"pixel_to_xyz shape {xyz_map.shape[:2]} does not match camera size {(height, width)}" - ) - - return { - "vertices": mesh["vertices"], - "faces": mesh["faces"], - "uv_data": mesh["uv_data"], - "loop_to_vertex": mesh["loop_to_vertex"], - "K": cam_data["K"], - "width": width, - "height": height, - "world_to_cam": np.load(os.path.join(data_dir, "world_to_cam.npy")), - "xyz_map": xyz_map, - } - - -def parse_source(value): - value = str(value).strip() - for prefix in ("camera:", "cam:", "webcam:"): - if value.lower().startswith(prefix): - value = value[len(prefix):].strip() - break - - try: - return int(value) - except ValueError: - return value - - -def prepare_static_points(data, display_stride): - xyz_map = data["xyz_map"] - valid_mask = ~np.isnan(xyz_map[:, :, 0]) - valid_points = xyz_map[valid_mask].reshape(-1, 3) - if len(valid_points) == 0: - raise ValueError("pixel_to_xyz.npz contains no valid mesh hit points.") - - display_indices = np.arange(len(valid_points))[::max(1, int(display_stride))] - return valid_mask, display_indices, valid_points[display_indices] - - -def align_image_to_map(img_bgr, data, valid_mask, display_indices, flip_vertical=True, flip_horizontal=True): - target_w = int(data["width"]) - target_h = int(data["height"]) - h, w = img_bgr.shape[:2] - if (w, h) != (target_w, target_h): - img_bgr = cv2.resize(img_bgr, (target_w, target_h), interpolation=cv2.INTER_LINEAR) - - img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB) - if flip_vertical: - img_rgb = np.flipud(img_rgb).copy() - if flip_horizontal: - img_rgb = np.fliplr(img_rgb).copy() - - colors = img_rgb.astype(np.float64) / 255.0 - return colors[valid_mask].reshape(-1, 3)[display_indices] - - -def arrow_map_reference_colors(count, color=(0.04, 0.04, 0.04)): - return np.tile(np.asarray(color, dtype=np.float64), (count, 1)) - - -def magnitude_map_colors(flow, valid_mask, display_indices, max_magnitude=None): - mag = np.linalg.norm(flow, axis=2) - valid_mag = mag[valid_mask].reshape(-1) - display_mag = valid_mag[display_indices] - if max_magnitude is None or max_magnitude <= 0: - max_magnitude = max(float(np.percentile(display_mag, 95)), 1e-6) - - value = np.clip(display_mag / max_magnitude, 0.0, 1.0) - return np.column_stack( - ( - value, - np.clip(1.0 - np.abs(value - 0.5) * 2.0, 0.0, 1.0), - 1.0 - value, - ) - ).astype(np.float64) - - -def align_scalar_field_to_map(field, data, flip_vertical=True): - if field.ndim == 3: - field = np.linalg.norm(field, axis=2) - - target_w = int(data["width"]) - target_h = int(data["height"]) - h, w = field.shape[:2] - if (w, h) != (target_w, target_h): - field = cv2.resize(field, (target_w, target_h), interpolation=cv2.INTER_LINEAR) - else: - field = field.copy() - - if flip_vertical: - field = np.flipud(field).copy() - return field - - -def scalar_map_colors(field, valid_mask, display_indices, value_min=None, value_max=None): - valid_values = field[valid_mask].reshape(-1) - display_values = valid_values[display_indices] - finite_values = display_values[np.isfinite(display_values)] - if len(finite_values) == 0: - return arrow_map_reference_colors(len(display_indices)) - - if value_min is None: - value_min = float(np.percentile(finite_values, 5)) - if value_max is None: - value_max = float(np.percentile(finite_values, 95)) - if value_max <= value_min: - value_max = value_min + 1e-6 - - value = np.clip((display_values - value_min) / (value_max - value_min), 0.0, 1.0) - return np.column_stack( - ( - value, - np.clip(1.0 - np.abs(value - 0.5) * 2.0, 0.0, 1.0), - 1.0 - value, - ) - ).astype(np.float64) - - -def align_vector_field_to_map(field, data, flip_vertical=True, scale_as_flow=True): - target_w = int(data["width"]) - target_h = int(data["height"]) - h, w = field.shape[:2] - if (w, h) != (target_w, target_h): - sx = target_w / max(w, 1) - sy = target_h / max(h, 1) - field = cv2.resize(field, (target_w, target_h), interpolation=cv2.INTER_LINEAR) - if scale_as_flow: - field[:, :, 0] *= sx - field[:, :, 1] *= sy - else: - field = field.copy() - - if flip_vertical: - field = np.flipud(field).copy() - field[:, :, 1] *= -1.0 - return field - - -def estimate_surface_normals(xyz_map): - valid = ~np.isnan(xyz_map[:, :, 0]) - filled = xyz_map.copy() - filled[~valid] = 0 - d_y = np.gradient(filled, axis=0) - d_x = np.gradient(filled, axis=1) - normals = np.cross(d_x, d_y) - norm = np.linalg.norm(normals, axis=2, keepdims=True) - normals = np.divide(normals, norm, out=np.zeros_like(normals), where=norm > 1e-8) - normals[~valid] = 0 - return normals - - -def line_colors(mag, arrow_map_style): - if len(mag) == 0: - return np.empty((0, 3), dtype=np.float64) - if arrow_map_style: - return np.tile(np.array([0.0, 0.9, 1.0], dtype=np.float64), (len(mag), 1)) - norm_mag = mag / max(float(np.percentile(mag, 95)), 1e-6) - norm_mag = np.clip(norm_mag, 0.0, 1.0) - return np.column_stack((norm_mag, 1.0 - norm_mag, np.zeros_like(norm_mag))).astype(np.float64) - - -def lines_from_vectors(starts, vectors, mag, flow_scale, arrow_map_style): - vec_norm = np.linalg.norm(vectors, axis=1) - good = np.isfinite(vec_norm) & (vec_norm > 1e-8) - starts, vectors, mag = starts[good], vectors[good], mag[good] - if len(starts) == 0: - return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) - - points = np.empty((len(starts) * 2, 3), dtype=np.float64) - points[0::2] = starts - points[1::2] = starts + vectors * flow_scale - lines = np.column_stack((np.arange(0, len(points), 2), np.arange(1, len(points), 2))).astype(np.int32) - return points, lines, line_colors(mag, arrow_map_style) - - -def build_flow_lines( - xyz_map, - valid_mask, - flow, - arrow_stride=32, - flow_scale=1.0, - min_magnitude=0.5, - arrow_map_style=False, -): - h, w = flow.shape[:2] - ys, xs = np.mgrid[0:h:arrow_stride, 0:w:arrow_stride] - ys = ys.reshape(-1) - xs = xs.reshape(-1) - sample_flow = flow[ys, xs] - mag = np.linalg.norm(sample_flow, axis=1) - keep = valid_mask[ys, xs] & (mag >= min_magnitude) - xs, ys, sample_flow, mag = xs[keep], ys[keep], sample_flow[keep], mag[keep] - if len(xs) == 0: - return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) - - tx = np.rint(xs + sample_flow[:, 0]).astype(np.int32) - ty = np.rint(ys + sample_flow[:, 1]).astype(np.int32) - inside = (0 <= tx) & (tx < w) & (0 <= ty) & (ty < h) - xs, ys, tx, ty, mag = xs[inside], ys[inside], tx[inside], ty[inside], mag[inside] - if len(xs) == 0: - return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) - - valid_target = valid_mask[ty, tx] - xs, ys, tx, ty, mag = xs[valid_target], ys[valid_target], tx[valid_target], ty[valid_target], mag[valid_target] - starts = xyz_map[ys, xs] - vectors = xyz_map[ty, tx] - starts - return lines_from_vectors(starts, vectors, mag, flow_scale, arrow_map_style) - - -def build_shear_normal_lines( - xyz_map, - valid_mask, - vshear, - vnormal, - normals, - arrow_stride=32, - flow_scale=1.0, - min_magnitude=0.01, - normal_gain=1.0, - arrow_map_style=False, -): - h, w = vshear.shape[:2] - ys, xs = np.mgrid[0:h:arrow_stride, 0:w:arrow_stride] - ys = ys.reshape(-1) - xs = xs.reshape(-1) - - shear = vshear[ys, xs] - normal_field = vnormal[ys, xs] - shear_mag = np.linalg.norm(shear, axis=1) - normal_mag = np.linalg.norm(normal_field, axis=1) * normal_gain - combined_mag = shear_mag + normal_mag - keep = valid_mask[ys, xs] & (combined_mag >= min_magnitude) - xs, ys, shear, shear_mag, normal_mag, combined_mag = ( - xs[keep], - ys[keep], - shear[keep], - shear_mag[keep], - normal_mag[keep], - combined_mag[keep], - ) - if len(xs) == 0: - return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) - - tx = np.rint(xs + shear[:, 0]).astype(np.int32) - ty = np.rint(ys + shear[:, 1]).astype(np.int32) - inside = (0 <= tx) & (tx < w) & (0 <= ty) & (ty < h) - xs, ys, tx, ty, shear_mag, normal_mag, combined_mag = ( - xs[inside], - ys[inside], - tx[inside], - ty[inside], - shear_mag[inside], - normal_mag[inside], - combined_mag[inside], - ) - if len(xs) == 0: - return np.empty((0, 3)), np.empty((0, 2), dtype=np.int32), np.empty((0, 3)) - - valid_target = valid_mask[ty, tx] - xs, ys, tx, ty, shear_mag, normal_mag, combined_mag = ( - xs[valid_target], - ys[valid_target], - tx[valid_target], - ty[valid_target], - shear_mag[valid_target], - normal_mag[valid_target], - combined_mag[valid_target], - ) - - starts = xyz_map[ys, xs] - tangent = xyz_map[ty, tx] - starts - tangent_norm = np.linalg.norm(tangent, axis=1, keepdims=True) - tangent_unit = np.divide(tangent, tangent_norm, out=np.zeros_like(tangent), where=tangent_norm > 1e-8) - normal_unit = normals[ys, xs] - - shear_ratio = shear_mag / np.maximum(combined_mag, 1e-8) - normal_ratio = normal_mag / np.maximum(combined_mag, 1e-8) - vectors = tangent_unit * shear_ratio[:, None] + normal_unit * normal_ratio[:, None] - vector_norm = np.linalg.norm(vectors, axis=1, keepdims=True) - vectors = np.divide(vectors, vector_norm, out=np.zeros_like(vectors), where=vector_norm > 1e-8) - return lines_from_vectors(starts, vectors, combined_mag, flow_scale, arrow_map_style) - - -def init_viewer(cloud_points, cloud_colors, flow_points, flow_lines, flow_colors, arrow_map_style=False): - import open3d as o3d - - vis = o3d.visualization.Visualizer() - vis.create_window(window_name="SDK 3D Flow", width=1280, height=720) - - pcd = o3d.geometry.PointCloud() - pcd.points = o3d.utility.Vector3dVector(cloud_points.astype(np.float64)) - pcd.colors = o3d.utility.Vector3dVector(cloud_colors.astype(np.float64)) - vis.add_geometry(pcd) - - line_set = o3d.geometry.LineSet() - line_set.points = o3d.utility.Vector3dVector(flow_points.astype(np.float64)) - line_set.lines = o3d.utility.Vector2iVector(flow_lines) - line_set.colors = o3d.utility.Vector3dVector(flow_colors.astype(np.float64)) - vis.add_geometry(line_set) - - render_option = vis.get_render_option() - render_option.point_size = 1.0 if arrow_map_style else 2.0 - render_option.line_width = 2.0 - render_option.background_color = np.array([0.0, 0.0, 0.0]) - vis.poll_events() - vis.update_renderer() - return vis, pcd, line_set, o3d - - -def run_sdk_3d_flow(args): - data = load_exported_data(args.data_dir) - flatten_remap = load_flatten_remap(args.flatten_remap) if args.flatten_remap else None - view_data = data - if flatten_remap is not None: - if "flat_xyz" not in flatten_remap: - raise ValueError("Flatten remap must contain flat_xyz for 3D visualization.") - view_data = dict(data) - view_data["xyz_map"] = flatten_remap["flat_xyz"] - view_data["height"], view_data["width"] = view_data["xyz_map"].shape[:2] - - valid_mask, display_indices, cloud_points = prepare_static_points(view_data, args.display_stride) - surface_normals = estimate_surface_normals(view_data["xyz_map"]) - flip_vertical = not args.no_flip_vertical - - sensor = orisys.Sensor( - parse_source(args.source), - config_name=args.config, - isstitch=False, - cuda=args.cuda, - verbose=args.verbose, - mirror=not args.no_mirror, - ) - - vis = pcd = line_set = o3d = None - frame_count = 0 - t0 = time.perf_counter() - - try: - while True: - if vis is not None and not vis.poll_events(): - break - - if sensor.get_img() is None: - break - if flip_vertical: - flip_sensor_frames_vertical(sensor) - flat_bgr = None - if flatten_remap is not None: - flat_bgr = flatten_frame(sensor.read_info(sensor.info.IMG)[0], flatten_remap) - flat_gray = cv2.cvtColor(flat_bgr, cv2.COLOR_BGR2GRAY) - sensor.img = flat_gray - sensor._img_bgr_raw = flat_bgr - sensor.img_size_raw = flat_bgr.shape - needs_decomposition = args.vector_mode in ("shear-normal", "depth-map") - sensor.compute_deformation(decompose=needs_decomposition, check_motion=False) - if args.vector_mode == "depth-map": - sensor.compute_contact() - flow, img_bgr, vshear, vnormal, depth_map = sensor.read_info( - sensor.info.VRAW, - sensor.info.IMG, - sensor.info.VSHEAR, - sensor.info.VNORMAL, - sensor.info.DEPTH, - ) - if flat_bgr is not None: - img_bgr = flat_bgr - - active_data = view_data if flatten_remap is not None else data - # Input is already flipped above; flattened domain uses flat_xyz directly. - flip_for_map = False if (flatten_remap is not None or flip_vertical) else True - flow_map = align_vector_field_to_map(flow, active_data, flip_vertical=flip_for_map) - if args.vector_mode in ("vraw-magnitude", "vraw-magnitude-arrows"): - cloud_colors = magnitude_map_colors(flow_map, valid_mask, display_indices, args.magnitude_max) - elif args.vector_mode == "depth-map": - depth_map = align_scalar_field_to_map(depth_map, active_data, flip_vertical=flip_for_map) - cloud_colors = scalar_map_colors( - depth_map, - valid_mask, - display_indices, - value_min=args.depth_min, - value_max=args.depth_max, - ) - elif args.arrow_map_style: - cloud_colors = arrow_map_reference_colors(len(cloud_points)) - else: - cloud_colors = align_image_to_map( - img_bgr, - active_data, - valid_mask, - display_indices, - flip_vertical=flip_for_map, - ) - - if args.vector_mode == "shear-normal": - vshear_map = align_vector_field_to_map(vshear, active_data, flip_vertical=flip_for_map) - vnormal_map = align_vector_field_to_map(vnormal, active_data, flip_vertical=flip_for_map) - flow_points, flow_lines, flow_colors = build_shear_normal_lines( - active_data["xyz_map"], - valid_mask, - vshear_map, - vnormal_map, - surface_normals, - arrow_stride=args.arrow_stride, - flow_scale=args.flow_scale, - min_magnitude=args.min_flow, - normal_gain=args.normal_gain, - arrow_map_style=args.arrow_map_style, - ) - elif args.vector_mode in ("vraw-magnitude", "depth-map"): - flow_points = np.empty((0, 3), dtype=np.float64) - flow_lines = np.empty((0, 2), dtype=np.int32) - flow_colors = np.empty((0, 3), dtype=np.float64) - else: - flow_points, flow_lines, flow_colors = build_flow_lines( - active_data["xyz_map"], - valid_mask, - flow_map, - arrow_stride=args.arrow_stride, - flow_scale=args.flow_scale, - min_magnitude=args.min_flow, - arrow_map_style=args.arrow_map_style, - ) - - if vis is None: - vis, pcd, line_set, o3d = init_viewer( - cloud_points, - cloud_colors, - flow_points, - flow_lines, - flow_colors, - arrow_map_style=args.arrow_map_style, - ) - print("SDK 3D flow viewer started. Close the Open3D window to stop.") - else: - pcd.colors = o3d.utility.Vector3dVector(cloud_colors) - line_set.points = o3d.utility.Vector3dVector(flow_points.astype(np.float64)) - line_set.lines = o3d.utility.Vector2iVector(flow_lines) - line_set.colors = o3d.utility.Vector3dVector(flow_colors.astype(np.float64)) - vis.update_geometry(pcd) - vis.update_geometry(line_set) - vis.update_renderer() - - frame_count += 1 - if frame_count % 30 == 0: - elapsed = max(time.perf_counter() - t0, 1e-6) - print(f"frames={frame_count}, viewer FPS={frame_count / elapsed:.1f}, arrows={len(flow_lines)}") - finally: - sensor.disconnect() - if vis is not None: - vis.destroy_window() - - -def parse_args(): - parser = argparse.ArgumentParser(description="Map SDK flow fields to a 3D point cloud.") - parser.add_argument("--data-dir", default=os.path.join(SCRIPT_DIR, "exported_data")) - parser.add_argument( - "--source", - default=os.path.join(SCRIPT_DIR, "20260602-094916.mp4"), - help="Video path or live camera ID, for example 0, 1, camera:0, or cam:1.", - ) - parser.add_argument("--config", default=os.path.join(REPO_ROOT, "config", "dd02-ov.json")) - parser.add_argument("--display-stride", type=int, default=4) - parser.add_argument("--arrow-stride", type=int, default=32) - parser.add_argument("--flow-scale", type=float, default=3.0) - parser.add_argument("--min-flow", type=float, default=0.5) - parser.add_argument("--normal-gain", type=float, default=1.0) - parser.add_argument("--magnitude-max", type=float, default=None, - help="Upper VRAW magnitude for heat-map color scaling; default uses frame percentile.") - parser.add_argument("--depth-min", type=float, default=None, - help="Minimum depth value for heat-map color scaling; default uses frame percentile.") - parser.add_argument("--depth-max", type=float, default=None, - help="Maximum depth value for heat-map color scaling; default uses frame percentile.") - parser.add_argument("--flatten-remap", default=None, - help="Optional flatten_remap.npz to run optical flow in UV-flattened image space.") - parser.add_argument( - "--vector-mode", - choices=["vraw", "vraw-magnitude", "vraw-magnitude-arrows", "shear-normal", "depth-map"], - default="vraw", - ) - parser.add_argument( - "--arrow-map-style", - action="store_true", - help="Hide real texture and use a dark Qt-viewer-like arrow map style.", - ) - parser.add_argument( - "--no-flip-vertical", - action="store_true", - help="Do not flip captured video vertically before processing.", - ) - parser.add_argument("--no-mirror", action="store_true") - parser.add_argument("--cuda", action="store_true") - parser.add_argument("--verbose", action="store_true") - return parser.parse_args() - - -if __name__ == "__main__": - run_sdk_3d_flow(parse_args()) diff --git a/tools/dd02_3dviewer/texture/patch_texture_0525_N120.png b/tools/dd02_3dviewer/texture/patch_texture_0525_N120.png deleted file mode 100644 index 769f728..0000000 Binary files a/tools/dd02_3dviewer/texture/patch_texture_0525_N120.png and /dev/null differ diff --git a/tools/dd02_3dviewer/texture/patch_texture_0525_N60.png b/tools/dd02_3dviewer/texture/patch_texture_0525_N60.png deleted file mode 100644 index bf4adc6..0000000 Binary files a/tools/dd02_3dviewer/texture/patch_texture_0525_N60.png and /dev/null differ diff --git a/tools/dd02_3dviewer/texture/patch_texture_0525_N75.png b/tools/dd02_3dviewer/texture/patch_texture_0525_N75.png deleted file mode 100644 index 5356d64..0000000 Binary files a/tools/dd02_3dviewer/texture/patch_texture_0525_N75.png and /dev/null differ diff --git a/tools/dd02_3dviewer/texture/patch_texture_0525_N90.png b/tools/dd02_3dviewer/texture/patch_texture_0525_N90.png deleted file mode 100644 index 5775e41..0000000 Binary files a/tools/dd02_3dviewer/texture/patch_texture_0525_N90.png and /dev/null differ diff --git a/tools/dd02_3dviewer/uv_mesh_viewer.py b/tools/dd02_3dviewer/uv_mesh_viewer.py deleted file mode 100644 index 8ca871a..0000000 --- a/tools/dd02_3dviewer/uv_mesh_viewer.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Backward-compatible re-exports; prefer ``examples/viz/mesh_viewer.py`` or ``orisys.finger``.""" - -from orisys.finger.mesh import * # noqa: F403 -from orisys.finger.viewer import * # noqa: F403 diff --git a/tools/downstream_sync_manifest.json b/tools/downstream_sync_manifest.json new file mode 100644 index 0000000..3b50f09 --- /dev/null +++ b/tools/downstream_sync_manifest.json @@ -0,0 +1,48 @@ +{ + "version": 1, + "downstreams": [ + { + "name": "orisys-sdk-release", + "path": "../orisys-sdk-release", + "sync": [ + {"from": "docs", "to": "docs"}, + {"from": "examples", "to": "examples", "include": [ + "README.md", + "basic_pipeline.py", + "mini_pipeline.py", + "finger_pipeline.py", + "finger_pipeline_3dview.py", + "plot_data.py", + "two_sensor_pipeline.py" + ]}, + {"from": "config", "to": "config", "include": ["box.json", "finger.json", "mini.json"]} + ], + "checks": [ + "python -m py_compile examples/basic_pipeline.py examples/finger_pipeline.py examples/finger_pipeline_3dview.py examples/plot_data.py", + "python -c \"import pathlib; p=pathlib.Path('.'); hits=[f for f in p.rglob('*') if f.is_file() and 'examples/products' in f.read_text(encoding='utf-8', errors='ignore').replace('\\\\','/')]; print('stale products paths:' if hits else 'no stale products paths'); [print(x) for x in hits]\"" + ] + }, + { + "name": "sensor-bench", + "path": "../sensor-bench", + "sync": [ + {"from": "config", "to": "config", "include": ["box.json", "mini.json", "dd06-ov-0810.json"]} + ], + "checks": [ + "python tests/test_smoke.py" + ] + }, + { + "name": "finger-bench", + "path": "../finger-bench", + "sync": [ + {"from": "config", "to": "config", "include": ["finger.json", "dd02-ov.json"]}, + {"from": "assets/finger", "to": "assets/finger"} + ], + "checks": [ + "python tests/test_smoke.py", + "python -c \"import pathlib; t=pathlib.Path('README.md').read_text(encoding='utf-8'); print('README still mentions examples/products' if 'examples/products' in t else 'README products path ok')\"" + ] + } + ] +} diff --git a/tools/ffc/roi_polygon_tool.py b/tools/ffc/roi_polygon_tool.py new file mode 100644 index 0000000..d6bccfd --- /dev/null +++ b/tools/ffc/roi_polygon_tool.py @@ -0,0 +1,484 @@ +from __future__ import annotations + +import argparse +import json +import os +from datetime import datetime +from pathlib import Path + +import cv2 +import numpy as np + + +WINDOW_NAME = "ROI Editor" +POLYGON_WINDOW_NAME = "ROI Polygon Editor" + + +def to_uint8_image(image): + array = np.asarray(image) + if array.dtype == np.uint8: + return array + array = np.nan_to_num(array.astype(np.float32), copy=False) + if array.size > 0 and float(np.max(array)) <= 1.0: + array = array * 255.0 + return np.clip(array, 0, 255).astype(np.uint8) + + +def normalize_editor_frame(frame): + if not isinstance(frame, np.ndarray): + raise TypeError("frame must be a numpy ndarray with shape HxW or HxWxC") + + frame = to_uint8_image(frame) + if frame.ndim == 2: + return cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR) + if frame.ndim != 3: + raise ValueError("frame must have shape HxW or HxWxC") + + channels = frame.shape[2] + if channels == 1: + return cv2.cvtColor(frame[:, :, 0], cv2.COLOR_GRAY2BGR) + if channels == 3: + return np.ascontiguousarray(frame) + if channels == 4: + return cv2.cvtColor(frame, cv2.COLOR_BGRA2BGR) + raise ValueError("frame channel count must be 1, 3, or 4") + + +def polygon_to_mask(points, image_size): + width, height = int(image_size[0]), int(image_size[1]) + mask = np.zeros((height, width), dtype=np.uint8) + polygon = np.asarray(points, dtype=np.int32).reshape(-1, 1, 2) + cv2.fillPoly(mask, [polygon], 255) + return mask + + +def scale_points(points, source_size, target_size): + source_w, source_h = int(source_size[0]), int(source_size[1]) + target_w, target_h = int(target_size[0]), int(target_size[1]) + pts = np.asarray(points, dtype=np.float32).reshape(-1, 2) + if source_w > 0 and source_h > 0 and (source_w, source_h) != (target_w, target_h): + pts[:, 0] *= target_w / float(source_w) + pts[:, 1] *= target_h / float(source_h) + pts[:, 0] = np.clip(pts[:, 0], 0, max(target_w - 1, 0)) + pts[:, 1] = np.clip(pts[:, 1], 0, max(target_h - 1, 0)) + return np.rint(pts).astype(np.int32) + + +def normalize_mask(mask, shape): + if mask is None: + return np.ones(shape, dtype=np.uint8) * 255 + mask = np.asarray(mask) + if mask.shape[:2] != shape: + mask = cv2.resize(mask, (shape[1], shape[0]), interpolation=cv2.INTER_NEAREST) + return np.where(mask > 0, 255, 0).astype(np.uint8) + + +def build_roi_payload(image_size, points, mask_path=None, metadata=None): + """Build the shared ``roi`` object written into sensor / FFC JSON configs.""" + points = np.asarray(points, dtype=np.int32).reshape(-1, 2) + if len(points) < 3: + raise ValueError("polygon ROI requires at least 3 points") + payload = { + "type": "polygon", + "image_size": [int(image_size[0]), int(image_size[1])], + "points": [[int(x), int(y)] for x, y in points], + "created_at": datetime.now().isoformat(timespec="seconds"), + } + + if mask_path is not None: + payload["mask_path"] = str(mask_path) + if metadata: + payload["metadata"] = metadata + return payload + + +def extract_roi_section(data): + """Return the ROI dict from either a nested ``roi`` field or a legacy top-level ROI file.""" + if not isinstance(data, dict): + raise TypeError("ROI config must be a JSON object") + if isinstance(data.get("roi"), dict): + return data["roi"] + return data + + +def load_roi_config(path): + path = Path(path) + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + data["_config_path"] = str(path) + return data + + +def save_polygon_roi_config(path, points, image_size, mask_path=None, metadata=None): + """Backward-compatible standalone polygon ROI JSON writer.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + data = build_roi_payload( + image_size, + points, + mask_path=mask_path, + metadata=metadata, + ) + data["version"] = "1.0" + with path.open("w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + return data + + +def upsert_roi_in_config(path, roi_payload, *, create_if_missing=True): + """Read-modify-write ``roi`` into an existing sensor config, preserving other fields.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, dict): + raise ValueError(f"Config must be a JSON object: {path}") + elif create_if_missing: + data = {"version": "1.0"} + else: + raise FileNotFoundError(f"Config not found: {path}") + + data["roi"] = roi_payload + with path.open("w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + f.write("\n") + return data + + +def load_roi_mask(config_path, frame_shape): + config_path = Path(config_path) + data = load_roi_config(config_path) + roi = extract_roi_section(data) + + height, width = frame_shape[:2] + target_size = (width, height) + source_size = roi.get("image_size") or data.get("image_size") or [width, height] + points = roi.get("points") + points = roi.get("points") or roi.get("polygon") or roi.get("roi_polygon") + if points: + polygon = scale_points(points, source_size, target_size).reshape(-1, 1, 2) + output = np.zeros((height, width), dtype=np.uint8) + cv2.fillPoly(output, [polygon], 255) + return output + + mask_path = roi.get("mask_path") or data.get("mask_path") or data.get("roi_mask") + if mask_path: + mask_path = Path(mask_path) + if not mask_path.is_absolute(): + mask_path = config_path.parent / mask_path + loaded = cv2.imread(str(mask_path), cv2.IMREAD_GRAYSCALE) + if loaded is None: + raise FileNotFoundError(f"Failed to read ROI mask: {mask_path}") + return normalize_mask(loaded, (height, width)) + + raise ValueError(f"Unsupported ROI config: {config_path}") + + +def resolve_video_source(video): + """Accept camera index (int/str digit) or filesystem path.""" + if isinstance(video, int): + return video + text = str(video).strip() + if text.isdigit(): + return int(text) + return text + + +def open_video_capture(video): + source = resolve_video_source(video) + # Avoid MSMF HW transform stalls on Windows camera capture. + os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0") + if isinstance(source, int): + cap = cv2.VideoCapture(source, cv2.CAP_DSHOW) + if not cap.isOpened(): + cap.release() + cap = cv2.VideoCapture(source) + else: + path = Path(source) + if not path.exists(): + raise FileNotFoundError(f"Video file not found: {path}") + cap = cv2.VideoCapture(str(path)) + if not cap.isOpened(): + raise RuntimeError(f"Failed to open video source: {video}") + return cap + + +def capture_frame(cap, *, max_attempts=60): + """Grab the next readable frame from an opened capture.""" + last_error = None + for _ in range(max_attempts): + ok, frame = cap.read() + if ok and frame is not None and getattr(frame, "size", 0) > 0: + return normalize_editor_frame(frame) + last_error = "empty frame" if ok else "read failed" + raise RuntimeError(f"Failed to capture frame from video source ({last_error})") + + +def capture_frame_from_source(video): + cap = open_video_capture(video) + try: + return capture_frame(cap) + finally: + cap.release() + + +class RoiPolygonEditor: + def __init__(self, frame): + self.frame = normalize_editor_frame(frame) + self.points = [] + + def set_frame(self, frame): + self.frame = normalize_editor_frame(frame) + self.points = [] + + def on_mouse(self, event, x, y, flags, param): + if event == cv2.EVENT_LBUTTONDOWN: + self.points.append((int(x), int(y))) + elif event == cv2.EVENT_RBUTTONDOWN and self.points: + self.points.pop() + + def clear(self): + self.points = [] + + def undo(self): + if self.points: + self.points.pop() + + def render(self): + view = self.frame.copy() + if len(self.points) >= 3: + overlay = view.copy() + polygon = np.asarray(self.points, dtype=np.int32).reshape(-1, 1, 2) + cv2.fillPoly(overlay, [polygon], (0, 128, 255)) + view = cv2.addWeighted(overlay, 0.25, view, 0.75, 0.0) + cv2.polylines(view, [polygon], True, (0, 255, 255), 2, cv2.LINE_AA) + elif len(self.points) >= 2: + cv2.polylines( + view, + [np.asarray(self.points, dtype=np.int32).reshape(-1, 1, 2)], + False, + (0, 255, 255), + 2, + cv2.LINE_AA, + ) + + for idx, (x, y) in enumerate(self.points): + cv2.circle(view, (x, y), 4, (0, 255, 0), -1, cv2.LINE_AA) + cv2.putText( + view, + str(idx + 1), + (x + 6, y - 6), + cv2.FONT_HERSHEY_SIMPLEX, + 0.45, + (0, 255, 0), + 1, + cv2.LINE_AA, + ) + + lines = [ + "LClick add RClick/U undo C clear R recapture S/Enter save Q/Esc quit", + f"points={len(self.points)} resolution={view.shape[1]}x{view.shape[0]}", + ] + for i, text in enumerate(lines): + y = 24 + i * 24 + cv2.putText(view, text, (10, y), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 0), 3, cv2.LINE_AA) + cv2.putText(view, text, (10, y), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 1, cv2.LINE_AA) + return view + + +def build_roi_result( + points, + image_size, + mask, + config=None, + config_path=None, + mask_path=None, +): + result = { + "type": "polygon", + "points": [[int(x), int(y)] for x, y in points] if points is not None else None, + "image_size": [int(image_size[0]), int(image_size[1])], + "mask": mask, + "config": config, + "config_path": str(config_path) if config_path is not None else None, + "mask_path": str(mask_path) if mask_path is not None else None, + } + return result + + +def _save_roi_outputs( + *, + image_size, + output, + points, + metadata=None, + save_mask=True, + nest_in_config=True, +): + mask = polygon_to_mask(points, image_size) + config = None + config_path = None + mask_path = None + + if output is not None: + config_path = Path(output) + config_path.parent.mkdir(parents=True, exist_ok=True) + saved_mask = None + if save_mask: + mask_path = config_path.with_name(config_path.stem + "_mask.png") + cv2.imwrite(str(mask_path), mask) + saved_mask = mask_path.name + + roi_payload = build_roi_payload( + image_size, + points, + mask_path=saved_mask, + metadata=metadata, + ) + if nest_in_config: + config = upsert_roi_in_config(config_path, roi_payload) + else: + # Standalone ROI file (legacy FFC polygon style). + standalone = dict(roi_payload) + standalone["version"] = "1.0" + with config_path.open("w", encoding="utf-8") as f: + json.dump(standalone, f, indent=2, ensure_ascii=False) + f.write("\n") + config = standalone + + return build_roi_result( + points, + image_size, + mask, + config=config, + config_path=config_path, + mask_path=mask_path, + ) + + +def select_polygon_roi( + frame, + output=None, + window_name=POLYGON_WINDOW_NAME, + metadata=None, + save_mask=True, + destroy_window=True, + recapture_callback=None, + nest_in_config=False, +): + """Open an OpenCV window, draw a polygon ROI, and return points/mask/config info.""" + editor = RoiPolygonEditor(frame) + cv2.namedWindow(window_name, cv2.WINDOW_AUTOSIZE) + cv2.setMouseCallback(window_name, editor.on_mouse) + + try: + while True: + cv2.imshow(window_name, editor.render()) + key = cv2.waitKey(16) & 0xFF + if key in (27, ord("q")): + return None + if key in (8, ord("u")): + editor.undo() + elif key == ord("c"): + editor.clear() + elif key == ord("r") and recapture_callback is not None: + editor.set_frame(recapture_callback()) + elif key in (13, ord("s")): + if len(editor.points) < 3: + print("Need at least 3 points to save ROI.") + continue + + image_size = (editor.frame.shape[1], editor.frame.shape[0]) + return _save_roi_outputs( + image_size=image_size, + output=output, + points=editor.points, + metadata=metadata, + save_mask=save_mask, + nest_in_config=nest_in_config, + ) + finally: + if destroy_window: + cv2.destroyWindow(window_name) + + +select_roi = select_polygon_roi + + +def parse_args(argv=None): + parser = argparse.ArgumentParser( + description="Capture a frame from a camera/video and save a polygon ROI to JSON." + ) + parser.add_argument( + "--video", + "-v", + required=True, + help="Camera index (e.g. 0) or path to a video file", + ) + parser.add_argument( + "--config", + "-c", + required=True, + help="Output JSON config path (existing sensor configs are updated in place)", + ) + parser.add_argument( + "--frame-output", + default=None, + help="Optional path to save the captured reference frame (PNG/JPG)", + ) + parser.add_argument( + "--no-mask", + action="store_true", + help="Do not write a companion *_mask.png next to the config", + ) + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + video = resolve_video_source(args.video) + cap = open_video_capture(video) + + def recapture(): + # For file sources, rewind so R always gets a usable frame. + if not isinstance(video, int): + cap.set(cv2.CAP_PROP_POS_FRAMES, 0) + return capture_frame(cap) + + try: + frame = capture_frame(cap) + if args.frame_output: + frame_path = Path(args.frame_output) + frame_path.parent.mkdir(parents=True, exist_ok=True) + cv2.imwrite(str(frame_path), frame) + print(f"Saved reference frame: {frame_path}") + + result = select_polygon_roi( + frame, + output=args.config, + metadata={"source": str(args.video)}, + save_mask=not args.no_mask, + recapture_callback=recapture, + nest_in_config=True, + ) + finally: + cap.release() + cv2.destroyAllWindows() + + if result is None: + print("ROI capture cancelled.") + return 1 + + print(f"Saved ROI type={result['type']} image_size={result['image_size']}") + if result.get("points") is not None: + print(f"points={result['points']}") + if result.get("config_path"): + print(f"config={result['config_path']}") + if result.get("mask_path"): + print(f"mask={result['mask_path']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/patch_generation/3.6mm.svg b/tools/patch_generation/3.6mm.svg deleted file mode 100644 index 7748a5e..0000000 --- a/tools/patch_generation/3.6mm.svg +++ /dev/null @@ -1,45 +0,0 @@ - - - - diff --git a/tools/patch_generation/checker_pattern.png b/tools/patch_generation/checker_pattern.png deleted file mode 100644 index 8051c8b..0000000 Binary files a/tools/patch_generation/checker_pattern.png and /dev/null differ diff --git a/tools/patch_generation/cmyk_generated_rgb.png b/tools/patch_generation/cmyk_generated_rgb.png deleted file mode 100644 index 802435a..0000000 Binary files a/tools/patch_generation/cmyk_generated_rgb.png and /dev/null differ diff --git a/tools/patch_generation/cmyk_pattern.tiff b/tools/patch_generation/cmyk_pattern.tiff deleted file mode 100644 index 539699f..0000000 Binary files a/tools/patch_generation/cmyk_pattern.tiff and /dev/null differ diff --git a/tools/patch_generation/color_pattern.png b/tools/patch_generation/color_pattern.png deleted file mode 100644 index 9733268..0000000 Binary files a/tools/patch_generation/color_pattern.png and /dev/null differ diff --git a/tools/patch_generation/dd01.svg b/tools/patch_generation/dd01.svg deleted file mode 100644 index 165ea5d..0000000 --- a/tools/patch_generation/dd01.svg +++ /dev/null @@ -1,49 +0,0 @@ - - - - diff --git a/tools/patch_generation/dd01_checker.svg b/tools/patch_generation/dd01_checker.svg deleted file mode 100644 index 5367551..0000000 --- a/tools/patch_generation/dd01_checker.svg +++ /dev/null @@ -1,49 +0,0 @@ - - - - diff --git a/tools/patch_generation/patch_generation.py b/tools/patch_generation/patch_generation.py deleted file mode 100644 index 931db44..0000000 --- a/tools/patch_generation/patch_generation.py +++ /dev/null @@ -1,53 +0,0 @@ -#%% -import numpy as np -import cv2 - -def generate_color_pattern(h, w, d, r): - # Initialize the pattern with zeros - pattern = np.zeros((h, w, 3)) - - # Calculate the number of patches in each dimension - num_patches_x = h // d - num_patches_y = w // d - - # Maximum number of attempts to find a suitable color - max_attempts = 1000 - - # Iterate over the patches - for i in range(num_patches_x): - for j in range(num_patches_y): - # Get the neighbors - neighbors = [] - if i > 0: - neighbors.append(pattern[(i-1)*d:i*d, j*d:(j+1)*d]) - if j > 0: - neighbors.append(pattern[i*d:(i+1)*d, (j-1)*d:j*d]) - - # Generate a new color - for attempt in range(max_attempts): - new_color = np.random.rand(3) - if all(np.min((new_color - neighbor.mean(axis=(0, 1)))**2) >= r for neighbor in neighbors): - break - if attempt == max_attempts - 1: - print(f"Warning: Could not find a suitable color for patch ({i}, {j}) after {max_attempts} attempts") - - # Apply the new color to the patch - pattern[i*d:(i+1)*d, j*d:(j+1)*d] = new_color - - print('Patch ({}, {})'.format(i, j)) - - return pattern - -# Test the function -pattern = generate_color_pattern(162, 162, 3, 0.75) - -# Convert the pattern to a format suitable for display with OpenCV -pattern_display = (pattern * 255).astype(np.uint8) - -# # Display the pattern -# cv2.imshow('Pattern', pattern_display) -# cv2.waitKey(0) -# cv2.destroyAllWindows() - -# #%% -cv2.imwrite('color_pattern.png', pattern_display) diff --git a/tools/patch_generation/patch_generation_3.py b/tools/patch_generation/patch_generation_3.py deleted file mode 100644 index ab68ca2..0000000 --- a/tools/patch_generation/patch_generation_3.py +++ /dev/null @@ -1,773 +0,0 @@ -#%% -import numpy as np -import cv2 -import matplotlib.pyplot as plt -from PIL import Image - -def merge_intervals(intervals): - # Merge overlapping intervals. - if not intervals: - return [] - intervals.sort(key=lambda x: x[0]) - merged = [intervals[0]] - for current in intervals[1:]: - last = merged[-1] - if current[0] <= last[1]: - merged[-1] = (last[0], max(last[1], current[1])) - else: - merged.append(current) - return merged - -def get_allowed_intervals(neighbor_values, delta): - # Compute forbidden intervals for this channel from neighbor values. - forbidden = [] - for c in neighbor_values: - low = max(0, c - delta) - high = min(1, c + delta) - forbidden.append((low, high)) - # Merge overlapping forbidden intervals. - merged = merge_intervals(forbidden) - allowed = [] - # Allowed region before the first forbidden interval. - if merged: - if merged[0][0] > 0: - allowed.append((0, merged[0][0])) - # Allowed regions between forbidden intervals. - for i in range(1, len(merged)): - allowed.append((merged[i-1][1], merged[i][0])) - # Allowed region after the last forbidden interval. - if merged[-1][1] < 1: - allowed.append((merged[-1][1], 1)) - else: - allowed.append((0, 1)) - return allowed - -def sample_from_allowed(allowed): - # Choose a random value from the allowed intervals with probability proportional to interval length. - lengths = [end - start for start, end in allowed] - total_length = sum(lengths) - if total_length <= 0: - return None - point = np.random.rand() * total_length - cumulative = 0 - for (start, end), length in zip(allowed, lengths): - cumulative += length - if point <= cumulative: - return np.random.uniform(start, end) - return allowed[-1][0] - -def sample_channel(neighbor_values, delta): - # Compute allowed intervals and sample a value. - allowed = get_allowed_intervals(neighbor_values, delta) - if not allowed: - return None - return sample_from_allowed(allowed) - -def generate_valid_color(neighbors, r): - """ - Generate a random color that satisfies, for each neighbor, - |R - R_neighbor| >= sqrt(r), and similarly for G and B. - neighbors: list of neighbor colors (each is an array of shape (3,)) - r: threshold for squared difference. - """ - delta = np.sqrt(r) - color = np.zeros(3) - for channel in range(3): - neighbor_channel = [n[channel] for n in neighbors] - sampled = sample_channel(neighbor_channel, delta) - # If the allowed intervals are empty, use a fallback. - if sampled is None: - sampled = np.random.rand() - color[channel] = sampled - return color - -def generate_color_pattern(h, w, d, r): - """ - Generate a dense color pattern. - h, w: height and width of the output image in pixels. - d: patch size in pixels (each patch is d x d). - r: randomness regulation factor. For each neighbor the condition is: - (|channel_new - channel_neighbor|)^2 >= r. - """ - pattern = np.zeros((h, w, 3), dtype=np.float32) - num_patches_x = h // d - num_patches_y = w // d - - for i in range(num_patches_x): - for j in range(num_patches_y): - # Collect neighbor patch colors (using mean because patches have uniform color). - neighbors = [] - if i > 0: - patch_above = pattern[(i-1)*d:i*d, j*d:(j+1)*d] - neighbors.append(np.mean(patch_above, axis=(0, 1))) - if j > 0: - patch_left = pattern[i*d:(i+1)*d, (j-1)*d:j*d] - neighbors.append(np.mean(patch_left, axis=(0, 1))) - - if len(neighbors) == 0: - # For the first patch use a random color. - new_color = np.random.rand(3) - else: - new_color = generate_valid_color(neighbors, r) - - pattern[i*d:(i+1)*d, j*d:(j+1)*d] = new_color - # print('Patch ({}, {})'.format(i, j)) - return pattern - -def convert_to_grayscale(color_pattern): - """ - Convert color pattern to grayscale using standard luminance weights. - - Parameters: - color_pattern: Input color image (H x W x 3) with values in range [0, 1] - - Returns: - Grayscale image (H x W) with values in range [0, 1] - """ - # Use standard RGB to grayscale conversion: 0.299*R + 0.587*G + 0.114*B - grayscale = 0.299 * color_pattern[:, :, 2] + 0.587 * color_pattern[:, :, 1] + 0.114 * color_pattern[:, :, 0] - return grayscale - -def optimize_grayscale_patches(grayscale, d, min_diff=0.1, max_iterations=100): - """ - Optimize grayscale image to improve value difference between neighboring patches. - - Parameters: - grayscale: Input grayscale image (H x W) with values in range [0, 1] - d: patch size in pixels (each patch is d x d) - min_diff: minimum absolute difference required between neighboring patches - max_iterations: maximum number of optimization iterations - - Returns: - Optimized grayscale image with improved patch differences - """ - h, w = grayscale.shape - num_patches_x = h // d - num_patches_y = w // d - - # Create optimized version - optimized = grayscale.copy() - - for iteration in range(max_iterations): - improved = False - - for i in range(num_patches_x): - for j in range(num_patches_y): - # Get current patch value (mean of patch) - current_patch = optimized[i*d:(i+1)*d, j*d:(j+1)*d] - current_value = np.mean(current_patch) - - # Collect neighbor patch values - neighbor_values = [] - if i > 0: - patch_above = optimized[(i-1)*d:i*d, j*d:(j+1)*d] - neighbor_values.append(np.mean(patch_above)) - if j > 0: - patch_left = optimized[i*d:(i+1)*d, (j-1)*d:j*d] - neighbor_values.append(np.mean(patch_left)) - if i < num_patches_x - 1: - patch_below = optimized[(i+1)*d:(i+2)*d, j*d:(j+1)*d] - neighbor_values.append(np.mean(patch_below)) - if j < num_patches_y - 1: - patch_right = optimized[i*d:(i+1)*d, (j+1)*d:(j+2)*d] - neighbor_values.append(np.mean(patch_right)) - - if len(neighbor_values) == 0: - continue - - # Check if current value is too close to any neighbor - min_neighbor_diff = min([abs(current_value - nv) for nv in neighbor_values]) - - if min_neighbor_diff < min_diff: - # Need to adjust this patch - # Find valid range that maintains min_diff from all neighbors - forbidden = [] - for nv in neighbor_values: - forbidden.append((max(0, nv - min_diff), min(1, nv + min_diff))) - - # Merge forbidden intervals - merged_forbidden = merge_intervals(forbidden) - - # Find allowed intervals - allowed = [] - if merged_forbidden: - if merged_forbidden[0][0] > 0: - allowed.append((0, merged_forbidden[0][0])) - for k in range(1, len(merged_forbidden)): - gap = merged_forbidden[k][0] - merged_forbidden[k-1][1] - if gap > 0: - allowed.append((merged_forbidden[k-1][1], merged_forbidden[k][0])) - if merged_forbidden[-1][1] < 1: - allowed.append((merged_forbidden[-1][1], 1)) - else: - allowed.append((0, 1)) - - # Sample new value from allowed intervals - if allowed: - new_value = sample_from_allowed(allowed) - if new_value is not None: - optimized[i*d:(i+1)*d, j*d:(j+1)*d] = new_value - improved = True - - if not improved: - print(f"Converged after {iteration + 1} iterations") - break - - return optimized - -def analyze_patch_differences(image, d): - """ - Analyze and report the minimum differences between neighboring patches. - - Parameters: - image: Grayscale image (H x W) with values in range [0, 1] - d: patch size in pixels - - Returns: - Dictionary with statistics about patch differences - """ - h, w = image.shape - num_patches_x = h // d - num_patches_y = w // d - - horizontal_diffs = [] - vertical_diffs = [] - - for i in range(num_patches_x): - for j in range(num_patches_y): - current_patch = image[i*d:(i+1)*d, j*d:(j+1)*d] - current_value = np.mean(current_patch) - - # Check horizontal neighbor (right) - if j < num_patches_y - 1: - patch_right = image[i*d:(i+1)*d, (j+1)*d:(j+2)*d] - right_value = np.mean(patch_right) - horizontal_diffs.append(abs(current_value - right_value)) - - # Check vertical neighbor (below) - if i < num_patches_x - 1: - patch_below = image[(i+1)*d:(i+2)*d, j*d:(j+1)*d] - below_value = np.mean(patch_below) - vertical_diffs.append(abs(current_value - below_value)) - - stats = { - 'min_diff': min(horizontal_diffs + vertical_diffs) if (horizontal_diffs + vertical_diffs) else 0, - 'max_diff': max(horizontal_diffs + vertical_diffs) if (horizontal_diffs + vertical_diffs) else 0, - 'mean_diff': np.mean(horizontal_diffs + vertical_diffs) if (horizontal_diffs + vertical_diffs) else 0, - 'median_diff': np.median(horizontal_diffs + vertical_diffs) if (horizontal_diffs + vertical_diffs) else 0, - } - - return stats - -def rgb_to_cmyk(rgb_image): - """ - Convert RGB image to CMYK color space for printing. - - Parameters: - rgb_image: RGB image (H x W x 3) with values in range [0, 1], BGR order (OpenCV format) - - Returns: - CMYK image (H x W x 4) with values in range [0, 1] - """ - # OpenCV uses BGR, so we need to handle it as B, G, R in channels 0, 1, 2 - b = rgb_image[:, :, 0] - g = rgb_image[:, :, 1] - r = rgb_image[:, :, 2] - - # Calculate K (black) channel - k = 1 - np.maximum(np.maximum(r, g), b) - - # Avoid division by zero - k_inv = 1 - k - k_inv = np.where(k_inv == 0, 1e-10, k_inv) - - # Calculate CMY channels - c = (1 - r - k) / k_inv - m = (1 - g - k) / k_inv - y = (1 - b - k) / k_inv - - # Stack into CMYK image - cmyk = np.stack([c, m, y, k], axis=2) - cmyk = np.clip(cmyk, 0, 1) - - return cmyk - -def cmyk_to_rgb(cmyk_image): - """ - Convert CMYK image back to RGB color space. - - Parameters: - cmyk_image: CMYK image (H x W x 4) with values in range [0, 1] - - Returns: - RGB image (H x W x 3) with values in range [0, 1], BGR order (OpenCV format) - """ - c = cmyk_image[:, :, 0] - m = cmyk_image[:, :, 1] - y = cmyk_image[:, :, 2] - k = cmyk_image[:, :, 3] - - # Convert back to RGB - r = (1 - c) * (1 - k) - g = (1 - m) * (1 - k) - b = (1 - y) * (1 - k) - - # Stack in BGR order for OpenCV - rgb = np.stack([b, g, r], axis=2) - rgb = np.clip(rgb, 0, 1) - - return rgb - -def optimize_cmyk_patches(cmyk_image, d, min_diff=0.1, max_iterations=100): - """ - Optimize CMYK image to improve color difference between neighboring patches. - Uses Euclidean distance in CMYK space. - - Parameters: - cmyk_image: Input CMYK image (H x W x 4) with values in range [0, 1] - d: patch size in pixels (each patch is d x d) - min_diff: minimum Euclidean distance required between neighboring patches in CMYK space - max_iterations: maximum number of optimization iterations - - Returns: - Optimized CMYK image with improved patch differences - """ - h, w = cmyk_image.shape[:2] - num_patches_x = h // d - num_patches_y = w // d - - optimized = cmyk_image.copy() - - for iteration in range(max_iterations): - improved = False - - for i in range(num_patches_x): - for j in range(num_patches_y): - # Get current patch CMYK values (mean of patch) - current_patch = optimized[i*d:(i+1)*d, j*d:(j+1)*d] - current_cmyk = np.mean(current_patch, axis=(0, 1)) - - # Collect neighbor patch CMYK values - neighbor_cmyk = [] - if i > 0: - patch_above = optimized[(i-1)*d:i*d, j*d:(j+1)*d] - neighbor_cmyk.append(np.mean(patch_above, axis=(0, 1))) - if j > 0: - patch_left = optimized[i*d:(i+1)*d, (j-1)*d:j*d] - neighbor_cmyk.append(np.mean(patch_left, axis=(0, 1))) - if i < num_patches_x - 1: - patch_below = optimized[(i+1)*d:(i+2)*d, j*d:(j+1)*d] - neighbor_cmyk.append(np.mean(patch_below, axis=(0, 1))) - if j < num_patches_y - 1: - patch_right = optimized[i*d:(i+1)*d, (j+1)*d:(j+2)*d] - neighbor_cmyk.append(np.mean(patch_right, axis=(0, 1))) - - if len(neighbor_cmyk) == 0: - continue - - # Calculate minimum Euclidean distance to neighbors in CMYK space - distances = [np.linalg.norm(current_cmyk - n) for n in neighbor_cmyk] - min_distance = min(distances) - - if min_distance < min_diff: - # Need to adjust this patch - try to move it away from neighbors - # Find the direction away from the closest neighbor - closest_idx = np.argmin(distances) - closest_neighbor = neighbor_cmyk[closest_idx] - - # Move away from closest neighbor - direction = current_cmyk - closest_neighbor - direction_norm = np.linalg.norm(direction) - - if direction_norm > 0: - direction = direction / direction_norm - # Move by the deficit amount - move_amount = (min_diff - min_distance) * 1.2 # 1.2 factor for faster convergence - new_cmyk = current_cmyk + direction * move_amount - new_cmyk = np.clip(new_cmyk, 0, 1) - - # Check if new value is better - new_distances = [np.linalg.norm(new_cmyk - n) for n in neighbor_cmyk] - new_min_distance = min(new_distances) - - if new_min_distance > min_distance: - optimized[i*d:(i+1)*d, j*d:(j+1)*d] = new_cmyk - improved = True - - if not improved: - print(f"CMYK optimization converged after {iteration + 1} iterations") - break - - return optimized - -def analyze_cmyk_patch_differences(cmyk_image, d): - """ - Analyze CMYK patch differences using Euclidean distance in CMYK space. - - Parameters: - cmyk_image: CMYK image (H x W x 4) with values in range [0, 1] - d: patch size in pixels - - Returns: - Dictionary with statistics about CMYK patch differences - """ - h, w = cmyk_image.shape[:2] - num_patches_x = h // d - num_patches_y = w // d - - horizontal_diffs = [] - vertical_diffs = [] - - for i in range(num_patches_x): - for j in range(num_patches_y): - current_patch = cmyk_image[i*d:(i+1)*d, j*d:(j+1)*d] - current_cmyk = np.mean(current_patch, axis=(0, 1)) - - # Check horizontal neighbor (right) - if j < num_patches_y - 1: - patch_right = cmyk_image[i*d:(i+1)*d, (j+1)*d:(j+2)*d] - right_cmyk = np.mean(patch_right, axis=(0, 1)) - horizontal_diffs.append(np.linalg.norm(current_cmyk - right_cmyk)) - - # Check vertical neighbor (below) - if i < num_patches_x - 1: - patch_below = cmyk_image[(i+1)*d:(i+2)*d, j*d:(j+1)*d] - below_cmyk = np.mean(patch_below, axis=(0, 1)) - vertical_diffs.append(np.linalg.norm(current_cmyk - below_cmyk)) - - all_diffs = horizontal_diffs + vertical_diffs - stats = { - 'min_diff': min(all_diffs) if all_diffs else 0, - 'max_diff': max(all_diffs) if all_diffs else 0, - 'mean_diff': np.mean(all_diffs) if all_diffs else 0, - 'median_diff': np.median(all_diffs) if all_diffs else 0, - } - - return stats - -def grayscale_to_cmyk_optimized(grayscale_target, d, min_cmyk_diff=0.2, max_iterations=100, color_diversity=0.3): - """ - Generate a CMYK image that when converted to grayscale matches the target grayscale, - while maintaining good color separation between patches. - - Parameters: - grayscale_target: Target grayscale image (H x W) with values in range [0, 1] - d: patch size in pixels - min_cmyk_diff: minimum Euclidean distance required between neighboring patches in CMYK space - max_iterations: maximum number of optimization iterations - color_diversity: amount of color diversity to inject (0.0 to 1.0) - - Returns: - CMYK image (H x W x 4) optimized for printing - """ - h, w = grayscale_target.shape - num_patches_x = h // d - num_patches_y = w // d - - # Initialize CMYK image - cmyk_image = np.zeros((h, w, 4), dtype=np.float32) - - # Generate initial CMYK values for each patch - for i in range(num_patches_x): - for j in range(num_patches_y): - # Get target grayscale value for this patch - target_gray = np.mean(grayscale_target[i*d:(i+1)*d, j*d:(j+1)*d]) - - # Generate random CMY values with diversity - c = np.random.rand() * color_diversity - m = np.random.rand() * color_diversity - y = np.random.rand() * color_diversity - - # Calculate K based on target grayscale - # Grayscale from CMYK: gray ≈ 1 - K when CMY are small - # For more accurate: gray = (1-C)*(1-K)*0.114 + (1-M)*(1-K)*0.587 + (1-Y)*(1-K)*0.299 - k = 1 - target_gray # Initial approximation - - # Adjust to match target grayscale - # Create temporary CMYK and test - temp_cmyk = np.array([c, m, y, k]) - temp_rgb = np.array([(1-c)*(1-k), (1-m)*(1-k), (1-y)*(1-k)]) # R, G, B - temp_gray = 0.299 * temp_rgb[0] + 0.587 * temp_rgb[1] + 0.114 * temp_rgb[2] - - # Adjust K to match target - k_adjustment = target_gray - temp_gray - k = np.clip(k - k_adjustment, 0, 1) - - cmyk_image[i*d:(i+1)*d, j*d:(j+1)*d] = [c, m, y, k] - - # Optimize CMYK to improve patch differences while maintaining grayscale target - print("Optimizing CMYK for color separation while matching grayscale...") - for iteration in range(max_iterations): - improved = False - - for i in range(num_patches_x): - for j in range(num_patches_y): - # Get current patch - current_patch = cmyk_image[i*d:(i+1)*d, j*d:(j+1)*d] - current_cmyk = np.mean(current_patch, axis=(0, 1)) - - # Get target grayscale for this patch - target_gray = np.mean(grayscale_target[i*d:(i+1)*d, j*d:(j+1)*d]) - - # Collect neighbor CMYK values - neighbor_cmyk = [] - if i > 0: - neighbor_cmyk.append(np.mean(cmyk_image[(i-1)*d:i*d, j*d:(j+1)*d], axis=(0, 1))) - if j > 0: - neighbor_cmyk.append(np.mean(cmyk_image[i*d:(i+1)*d, (j-1)*d:j*d], axis=(0, 1))) - if i < num_patches_x - 1: - neighbor_cmyk.append(np.mean(cmyk_image[(i+1)*d:(i+2)*d, j*d:(j+1)*d], axis=(0, 1))) - if j < num_patches_y - 1: - neighbor_cmyk.append(np.mean(cmyk_image[i*d:(i+1)*d, (j+1)*d:(j+2)*d], axis=(0, 1))) - - if len(neighbor_cmyk) == 0: - continue - - # Check CMYK distance to neighbors - distances = [np.linalg.norm(current_cmyk - n) for n in neighbor_cmyk] - min_distance = min(distances) - - if min_distance < min_cmyk_diff: - # Try to improve color separation - # Vary CMY channels while keeping grayscale constant - for attempt in range(5): - # Generate new CMY values - new_c = np.clip(current_cmyk[0] + (np.random.rand() - 0.5) * 0.2, 0, 1) - new_m = np.clip(current_cmyk[1] + (np.random.rand() - 0.5) * 0.2, 0, 1) - new_y = np.clip(current_cmyk[2] + (np.random.rand() - 0.5) * 0.2, 0, 1) - - # Calculate K to maintain grayscale - # gray = (1-C)*(1-K)*0.114 + (1-M)*(1-K)*0.587 + (1-Y)*(1-K)*0.299 - # gray = (1-K) * [0.114*(1-C) + 0.587*(1-M) + 0.299*(1-Y)] - rgb_factor = 0.114*(1-new_c) + 0.587*(1-new_m) + 0.299*(1-new_y) - if rgb_factor > 0: - new_k = 1 - (target_gray / rgb_factor) - new_k = np.clip(new_k, 0, 1) - else: - new_k = current_cmyk[3] - - new_cmyk = np.array([new_c, new_m, new_y, new_k]) - - # Verify grayscale match - test_rgb = np.array([(1-new_c)*(1-new_k), (1-new_m)*(1-new_k), (1-new_y)*(1-new_k)]) - test_gray = 0.299 * test_rgb[0] + 0.587 * test_rgb[1] + 0.114 * test_rgb[2] - - # Check if grayscale is close enough and CMYK distance improved - if abs(test_gray - target_gray) < 0.05: # Grayscale tolerance - new_distances = [np.linalg.norm(new_cmyk - n) for n in neighbor_cmyk] - new_min_distance = min(new_distances) - - if new_min_distance > min_distance: - cmyk_image[i*d:(i+1)*d, j*d:(j+1)*d] = new_cmyk - improved = True - break - - if not improved: - print(f"Grayscale-to-CMYK optimization converged after {iteration + 1} iterations") - break - - return cmyk_image - -def cmyk_to_grayscale(cmyk_image): - """ - Convert CMYK image to grayscale. - - Parameters: - cmyk_image: CMYK image (H x W x 4) with values in range [0, 1] - - Returns: - Grayscale image (H x W) with values in range [0, 1] - """ - # Convert to RGB first, then to grayscale - rgb = cmyk_to_rgb(cmyk_image) - grayscale = 0.299 * rgb[:, :, 2] + 0.587 * rgb[:, :, 1] + 0.114 * rgb[:, :, 0] - return grayscale - -def main(): - # Parameters for the pattern. - # Image dimensions in millimeters - width_mm = 50 # Width in millimeters - height_mm = 50 # Height in millimeters - dpi = 600 # Dots per inch for printing - patch_size = 1 # 3 for 300dpi, 6 for 600dpi - - # Convert mm to pixels: pixels = (mm / 25.4) * dpi - width = int((width_mm / 25.4) * dpi) - height = int((height_mm / 25.4) * dpi) - - print(f"Image dimensions: {width_mm}mm x {height_mm}mm at {dpi} DPI") - print(f"Pixel dimensions: {width} x {height} pixels") - - - # Set r to a value that gives sufficient difference. - # Note: r is the threshold for squared difference per channel, and the condition is - # |c - c_neighbor| >= sqrt(r). Very high r may make it impossible to satisfy. - r = 1 - - # Step 1: Generate initial color pattern - print("Generating color pattern...") - pattern = generate_color_pattern(height, width, patch_size, r) - pattern_display = (pattern * 255).astype(np.uint8) - - # Step 2: Convert to grayscale - print("\nConverting to grayscale...") - grayscale = convert_to_grayscale(pattern) - grayscale_display = (grayscale * 255).astype(np.uint8) - - # Step 3: Analyze original grayscale differences - print("\nAnalyzing original grayscale patch differences...") - original_stats = analyze_patch_differences(grayscale, patch_size) - print(f"Original - Min: {original_stats['min_diff']:.4f}, Max: {original_stats['max_diff']:.4f}, " - f"Mean: {original_stats['mean_diff']:.4f}, Median: {original_stats['median_diff']:.4f}") - - # Step 4: Optimize grayscale to improve patch differences - print("\nOptimizing grayscale patch differences...") - min_diff_threshold = 0.15 # Adjust this value to control minimum difference - optimized_grayscale = optimize_grayscale_patches(grayscale, patch_size, min_diff=min_diff_threshold, max_iterations=50) - optimized_display = (optimized_grayscale * 255).astype(np.uint8) - - # Step 5: Analyze optimized grayscale differences - print("\nAnalyzing optimized grayscale patch differences...") - optimized_stats = analyze_patch_differences(optimized_grayscale, patch_size) - print(f"Optimized - Min: {optimized_stats['min_diff']:.4f}, Max: {optimized_stats['max_diff']:.4f}, " - f"Mean: {optimized_stats['mean_diff']:.4f}, Median: {optimized_stats['median_diff']:.4f}") - - # Step 6: Generate CMYK image from optimized grayscale - print("\n" + "="*60) - print("GENERATING CMYK FROM OPTIMIZED GRAYSCALE") - print("="*60) - print("\nGenerating CMYK image to match optimized grayscale...") - cmyk_from_grayscale = grayscale_to_cmyk_optimized( - optimized_grayscale, - patch_size, - min_cmyk_diff=0.2, - max_iterations=50, - color_diversity=0.5 - ) - - # Analyze CMYK differences - print("\nAnalyzing CMYK patch differences...") - cmyk_stats = analyze_cmyk_patch_differences(cmyk_from_grayscale, patch_size) - print(f"CMYK - Min: {cmyk_stats['min_diff']:.4f}, Max: {cmyk_stats['max_diff']:.4f}, " - f"Mean: {cmyk_stats['mean_diff']:.4f}, Median: {cmyk_stats['median_diff']:.4f}") - - # Convert CMYK back to RGB for display - rgb_from_cmyk = cmyk_to_rgb(cmyk_from_grayscale) - rgb_from_cmyk_display = (rgb_from_cmyk * 255).astype(np.uint8) - - # Convert CMYK back to grayscale to verify - grayscale_from_cmyk = cmyk_to_grayscale(cmyk_from_grayscale) - grayscale_from_cmyk_display = (grayscale_from_cmyk * 255).astype(np.uint8) - - # Analyze how well grayscale was preserved - grayscale_from_cmyk_stats = analyze_patch_differences(grayscale_from_cmyk, patch_size) - print(f"Grayscale from CMYK - Min: {grayscale_from_cmyk_stats['min_diff']:.4f}") - - # Calculate grayscale reconstruction error - gray_error = np.mean(np.abs(optimized_grayscale - grayscale_from_cmyk)) - print(f"Grayscale reconstruction error: {gray_error:.4f}") - - # Display all images with matplotlib - fig, axes = plt.subplots(2, 3, figsize=(18, 12), dpi=300) - - # Row 1: Original grayscale optimization - # Color pattern (convert BGR to RGB for matplotlib) - pattern_rgb = cv2.cvtColor(pattern_display, cv2.COLOR_BGR2RGB) - axes[0, 0].imshow(pattern_rgb) - axes[0, 0].set_title(f'1. Original Color Pattern\nSize: {height}x{width} pixels', fontsize=10) - axes[0, 0].axis('on') - axes[0, 0].grid(True, alpha=0.3) - - # Grayscale from color - axes[0, 1].imshow(grayscale_display, cmap='gray', vmin=0, vmax=255) - axes[0, 1].set_title(f'2. Grayscale (from color)\nSize: {height}x{width} pixels\nMin diff: {original_stats["min_diff"]:.4f}', fontsize=10) - axes[0, 1].axis('on') - axes[0, 1].grid(True, alpha=0.3) - - # Optimized grayscale - axes[0, 2].imshow(optimized_display, cmap='gray', vmin=0, vmax=255) - axes[0, 2].set_title(f'3. Optimized Grayscale (Target)\nSize: {height}x{width} pixels\nMin diff: {optimized_stats["min_diff"]:.4f}', fontsize=10, fontweight='bold') - axes[0, 2].axis('on') - axes[0, 2].grid(True, alpha=0.3) - - # Row 2: CMYK generation from optimized grayscale - # RGB from CMYK - rgb_from_cmyk_viz = cv2.cvtColor(rgb_from_cmyk_display, cv2.COLOR_BGR2RGB) - axes[1, 0].imshow(rgb_from_cmyk_viz) - axes[1, 0].set_title(f'4. RGB from Generated CMYK\nCMYK Min diff: {cmyk_stats["min_diff"]:.4f}\n(Best for printing)', fontsize=10, fontweight='bold') - axes[1, 0].axis('on') - axes[1, 0].grid(True, alpha=0.3) - - # Grayscale from CMYK (verification) - axes[1, 1].imshow(grayscale_from_cmyk_display, cmap='gray', vmin=0, vmax=255) - axes[1, 1].set_title(f'5. Grayscale from CMYK\nMin diff: {grayscale_from_cmyk_stats["min_diff"]:.4f}\nError: {gray_error:.4f}', fontsize=10) - axes[1, 1].axis('on') - axes[1, 1].grid(True, alpha=0.3) - - # Show CMYK channels visualization - cmyk_viz = np.zeros((height, width, 3), dtype=np.uint8) - cmyk_viz[:, :, 0] = (cmyk_from_grayscale[:, :, 0] * 255).astype(np.uint8) # Cyan in B channel - cmyk_viz[:, :, 1] = (cmyk_from_grayscale[:, :, 1] * 255).astype(np.uint8) # Magenta in G channel - cmyk_viz[:, :, 2] = (cmyk_from_grayscale[:, :, 2] * 255).astype(np.uint8) # Yellow in R channel - axes[1, 2].imshow(cv2.cvtColor(cmyk_viz, cv2.COLOR_BGR2RGB)) - axes[1, 2].set_title(f'6. CMYK Channels (CMY only)\nK channel separate', fontsize=10) - axes[1, 2].axis('on') - axes[1, 2].grid(True, alpha=0.3) - - plt.tight_layout() - print(f"\nDisplay info: Figure DPI={fig.dpi}, Size={fig.get_size_inches()} inches") - plt.show() - - # Save all images - cv2.imwrite('color_pattern.png', pattern_display) - cv2.imwrite('grayscale_from_color.png', grayscale_display) - cv2.imwrite('optimized_grayscale.png', optimized_display) - cv2.imwrite('cmyk_generated_rgb.png', rgb_from_cmyk_display) - cv2.imwrite('grayscale_from_cmyk.png', grayscale_from_cmyk_display) - - # Save CMYK data - np.save('cmyk_from_grayscale.npy', cmyk_from_grayscale) - - # Save CMYK as TIFF (proper format for printing) - print("\nSaving CMYK as TIFF...") - try: - # Convert CMYK to uint8 (0-255 range) - cmyk_uint8 = (cmyk_from_grayscale * 255).astype(np.uint8) - - # For TIFF CMYK, we need to ensure proper channel order - # PIL CMYK expects channels in C, M, Y, K order - # Split channels - c_channel = cmyk_uint8[:, :, 0] - m_channel = cmyk_uint8[:, :, 1] - y_channel = cmyk_uint8[:, :, 2] - k_channel = cmyk_uint8[:, :, 3] - - # Create PIL images for each channel and merge - c_img = Image.fromarray(c_channel, mode='L') - m_img = Image.fromarray(m_channel, mode='L') - y_img = Image.fromarray(y_channel, mode='L') - k_img = Image.fromarray(k_channel, mode='L') - - # Merge into CMYK image - cmyk_pil = Image.merge('CMYK', (c_img, m_img, y_img, k_img)) - - # Save as TIFF with CMYK color space at 300 DPI for professional printing - cmyk_pil.save('cmyk_pattern.tiff', format='TIFF', dpi=(300, 300)) - - # Verify the TIFF was saved by checking file size - import os - if os.path.exists('cmyk_pattern.tiff'): - file_size = os.path.getsize('cmyk_pattern.tiff') - print(f"CMYK TIFF saved successfully: {file_size} bytes at 300 DPI") - - except Exception as e: - print(f"Error saving CMYK TIFF: {e}") - - print("\nImages saved successfully!") - print("CMYK-generated RGB: cmyk_generated_rgb.png") - print("Grayscale verification: grayscale_from_cmyk.png") - print("CMYK data: cmyk_from_grayscale.npy") - print("CMYK TIFF (300 DPI): cmyk_pattern.tiff (professional printing format)") - -if __name__ == '__main__': - main() diff --git a/tools/patch_generation/patch_generation_3_org.py b/tools/patch_generation/patch_generation_3_org.py deleted file mode 100644 index 0a09359..0000000 --- a/tools/patch_generation/patch_generation_3_org.py +++ /dev/null @@ -1,140 +0,0 @@ -#%% -import numpy as np -import cv2 - -def merge_intervals(intervals): - # Merge overlapping intervals. - if not intervals: - return [] - intervals.sort(key=lambda x: x[0]) - merged = [intervals[0]] - for current in intervals[1:]: - last = merged[-1] - if current[0] <= last[1]: - merged[-1] = (last[0], max(last[1], current[1])) - else: - merged.append(current) - return merged - -def get_allowed_intervals(neighbor_values, delta): - # Compute forbidden intervals for this channel from neighbor values. - forbidden = [] - for c in neighbor_values: - low = max(0, c - delta) - high = min(1, c + delta) - forbidden.append((low, high)) - # Merge overlapping forbidden intervals. - merged = merge_intervals(forbidden) - allowed = [] - # Allowed region before the first forbidden interval. - if merged: - if merged[0][0] > 0: - allowed.append((0, merged[0][0])) - # Allowed regions between forbidden intervals. - for i in range(1, len(merged)): - allowed.append((merged[i-1][1], merged[i][0])) - # Allowed region after the last forbidden interval. - if merged[-1][1] < 1: - allowed.append((merged[-1][1], 1)) - else: - allowed.append((0, 1)) - return allowed - -def sample_from_allowed(allowed): - # Choose a random value from the allowed intervals with probability proportional to interval length. - lengths = [end - start for start, end in allowed] - total_length = sum(lengths) - if total_length <= 0: - return None - point = np.random.rand() * total_length - cumulative = 0 - for (start, end), length in zip(allowed, lengths): - cumulative += length - if point <= cumulative: - return np.random.uniform(start, end) - return allowed[-1][0] - -def sample_channel(neighbor_values, delta): - # Compute allowed intervals and sample a value. - allowed = get_allowed_intervals(neighbor_values, delta) - if not allowed: - return None - return sample_from_allowed(allowed) - -def generate_valid_color(neighbors, r): - """ - Generate a random color that satisfies, for each neighbor, - |R - R_neighbor| >= sqrt(r), and similarly for G and B. - neighbors: list of neighbor colors (each is an array of shape (3,)) - r: threshold for squared difference. - """ - delta = np.sqrt(r) - color = np.zeros(3) - for channel in range(3): - neighbor_channel = [n[channel] for n in neighbors] - sampled = sample_channel(neighbor_channel, delta) - # If the allowed intervals are empty, use a fallback. - if sampled is None: - sampled = np.random.rand() - color[channel] = sampled - return color - -def generate_color_pattern(h, w, d, r): - """ - Generate a dense color pattern. - h, w: height and width of the output image in pixels. - d: patch size in pixels (each patch is d x d). - r: randomness regulation factor. For each neighbor the condition is: - (|channel_new - channel_neighbor|)^2 >= r. - """ - pattern = np.zeros((h, w, 3), dtype=np.float32) - num_patches_x = h // d - num_patches_y = w // d - - for i in range(num_patches_x): - for j in range(num_patches_y): - # Collect neighbor patch colors (using mean because patches have uniform color). - neighbors = [] - if i > 0: - patch_above = pattern[(i-1)*d:i*d, j*d:(j+1)*d] - neighbors.append(np.mean(patch_above, axis=(0, 1))) - if j > 0: - patch_left = pattern[i*d:(i+1)*d, (j-1)*d:j*d] - neighbors.append(np.mean(patch_left, axis=(0, 1))) - - if len(neighbors) == 0: - # For the first patch use a random color. - new_color = np.random.rand(3) - else: - new_color = generate_valid_color(neighbors, r) - - pattern[i*d:(i+1)*d, j*d:(j+1)*d] = new_color - print('Patch ({}, {})'.format(i, j)) - return pattern - -def main(): - # Parameters for the pattern. - # For example, using 100x100 pixels image with patches of 3x3 pixels. - # 11111111 - height = 1200 - width = 1000 - patch_size = 12 - # Set r to a value that gives sufficient difference. - # Note: r is the threshold for squared difference per channel, and the condition is - # |c - c_neighbor| >= sqrt(r). Very high r may make it impossible to satisfy. - r = 1000 - - pattern = generate_color_pattern(height, width, patch_size, r) - # Scale to 0-255 for display and saving. - pattern_display = (pattern * 255).astype(np.uint8) - - # Display the pattern with OpenCV. - cv2.imshow('Pattern', pattern_display) - cv2.waitKey(0) - cv2.destroyAllWindows() - - # Save the pattern as an image file. - cv2.imwrite('color_pattern.png', pattern_display) - -if __name__ == '__main__': - main() diff --git a/tools/patch_generation/patchgen_DD01 checker.py b/tools/patch_generation/patchgen_DD01 checker.py deleted file mode 100644 index 3105639..0000000 --- a/tools/patch_generation/patchgen_DD01 checker.py +++ /dev/null @@ -1,46 +0,0 @@ -#%% -import numpy as np -import cv2 - - -def generate_checker_pattern(height, width, patch_size): - """ - Generate a black-and-white checkerboard pattern. - - Each `patch_size x patch_size` block alternates between black and white. - Partial blocks at the image borders are also filled correctly. - """ - pattern = np.zeros((height, width, 3), dtype=np.uint8) - - num_patches_x = (height + patch_size - 1) // patch_size - num_patches_y = (width + patch_size - 1) // patch_size - - for i in range(num_patches_x): - for j in range(num_patches_y): - color = 255 if (i + j) % 2 == 0 else 0 - y0 = i * patch_size - y1 = min((i + 1) * patch_size, height) - x0 = j * patch_size - x1 = min((j + 1) * patch_size, width) - pattern[y0:y1, x0:x1] = color - print("Patch ({}, {}) -> {}".format(i, j, "white" if color == 255 else "black")) - - return pattern - - -def main(): - height = 1200 - width = 1000 - patch_size = 12 - - pattern = generate_checker_pattern(height, width, patch_size) - - cv2.imshow("Checker Pattern", pattern) - cv2.waitKey(0) - cv2.destroyAllWindows() - - cv2.imwrite("checker_pattern.png", pattern) - - -if __name__ == '__main__': - main() diff --git a/tools/patch_generation/patchgen_DD01.py b/tools/patch_generation/patchgen_DD01.py deleted file mode 100644 index ca60363..0000000 --- a/tools/patch_generation/patchgen_DD01.py +++ /dev/null @@ -1,139 +0,0 @@ -#%% -import numpy as np -import cv2 - -def merge_intervals(intervals): - # Merge overlapping intervals. - if not intervals: - return [] - intervals.sort(key=lambda x: x[0]) - merged = [intervals[0]] - for current in intervals[1:]: - last = merged[-1] - if current[0] <= last[1]: - merged[-1] = (last[0], max(last[1], current[1])) - else: - merged.append(current) - return merged - -def get_allowed_intervals(neighbor_values, delta): - # Compute forbidden intervals for this channel from neighbor values. - forbidden = [] - for c in neighbor_values: - low = max(0, c - delta) - high = min(1, c + delta) - forbidden.append((low, high)) - # Merge overlapping forbidden intervals. - merged = merge_intervals(forbidden) - allowed = [] - # Allowed region before the first forbidden interval. - if merged: - if merged[0][0] > 0: - allowed.append((0, merged[0][0])) - # Allowed regions between forbidden intervals. - for i in range(1, len(merged)): - allowed.append((merged[i-1][1], merged[i][0])) - # Allowed region after the last forbidden interval. - if merged[-1][1] < 1: - allowed.append((merged[-1][1], 1)) - else: - allowed.append((0, 1)) - return allowed - -def sample_from_allowed(allowed): - # Choose a random value from the allowed intervals with probability proportional to interval length. - lengths = [end - start for start, end in allowed] - total_length = sum(lengths) - if total_length <= 0: - return None - point = np.random.rand() * total_length - cumulative = 0 - for (start, end), length in zip(allowed, lengths): - cumulative += length - if point <= cumulative: - return np.random.uniform(start, end) - return allowed[-1][0] - -def sample_channel(neighbor_values, delta): - # Compute allowed intervals and sample a value. - allowed = get_allowed_intervals(neighbor_values, delta) - if not allowed: - return None - return sample_from_allowed(allowed) - -def generate_valid_color(neighbors, r): - """ - Generate a random color that satisfies, for each neighbor, - |R - R_neighbor| >= sqrt(r), and similarly for G and B. - neighbors: list of neighbor colors (each is an array of shape (3,)) - r: threshold for squared difference. - """ - delta = np.sqrt(r) - color = np.zeros(3) - for channel in range(3): - neighbor_channel = [n[channel] for n in neighbors] - sampled = sample_channel(neighbor_channel, delta) - # If the allowed intervals are empty, use a fallback. - if sampled is None: - sampled = np.random.rand() - color[channel] = sampled - return color - -def generate_color_pattern(h, w, d, r): - """ - Generate a dense color pattern. - h, w: height and width of the output image in pixels. - d: patch size in pixels (each patch is d x d). - r: randomness regulation factor. For each neighbor the condition is: - (|channel_new - channel_neighbor|)^2 >= r. - """ - pattern = np.zeros((h, w, 3), dtype=np.float32) - num_patches_x = h // d - num_patches_y = w // d - - for i in range(num_patches_x): - for j in range(num_patches_y): - # Collect neighbor patch colors (using mean because patches have uniform color). - neighbors = [] - if i > 0: - patch_above = pattern[(i-1)*d:i*d, j*d:(j+1)*d] - neighbors.append(np.mean(patch_above, axis=(0, 1))) - if j > 0: - patch_left = pattern[i*d:(i+1)*d, (j-1)*d:j*d] - neighbors.append(np.mean(patch_left, axis=(0, 1))) - - if len(neighbors) == 0: - # For the first patch use a random color. - new_color = np.random.rand(3) - else: - new_color = generate_valid_color(neighbors, r) - - pattern[i*d:(i+1)*d, j*d:(j+1)*d] = new_color - print('Patch ({}, {})'.format(i, j)) - return pattern - -def main(): - # Parameters for the pattern. - # For example, using 100x100 pixels image with patches of 3x3 pixels. - height = 1200 - width = 1000 - patch_size = 12 - # Set r to a value that gives sufficient difference. - # Note: r is the threshold for squared difference per channel, and the condition is - # |c - c_neighbor| >= sqrt(r). Very high r may make it impossible to satisfy. - r = 1000 - - pattern = generate_color_pattern(height, width, patch_size, r) - # Scale to 0-255 for display and saving. - pattern_display = (pattern * 255).astype(np.uint8) - - # Display the pattern with OpenCV. - cv2.imshow('Pattern', pattern_display) - cv2.waitKey(0) - cv2.destroyAllWindows() - - # Save the pattern as an image file. - cv2.imwrite('color_pattern.png', pattern_display) - -if __name__ == '__main__': - main() diff --git a/tools/patch_generation/patchgen_DD02C_dot.py b/tools/patch_generation/patchgen_DD02C_dot.py deleted file mode 100644 index 948d6b3..0000000 --- a/tools/patch_generation/patchgen_DD02C_dot.py +++ /dev/null @@ -1,44 +0,0 @@ -import numpy as np -from PIL import Image, ImageDraw - -# Parameters -R = 9.0 # mm -H = 20.0 # mm -fov_half = np.radians(75) # 75 degrees half-angle -pixels_per_mm = 100 - -# Texture dimensions -width = int(np.pi * 2 * R * pixels_per_mm) # circumference -height = int(H * pixels_per_mm) - -# Desired dot pattern in image space (normalized) -delta_theta = np.radians(10) # 10° azimuth spacing -delta_r = 0.1 # radial spacing in normalized units -rho = 0.08 # normalized dot radius (from reference) - -# Generate dot centers in image space -r_vals = np.arange(R/H, np.tan(fov_half), delta_r) -theta_vals = np.arange(0, 2*np.pi, delta_theta) - -# Create empty texture (white background) -img = Image.new('L', (width, height), color=255) -draw = ImageDraw.Draw(img) - -for r_img in r_vals: - z = R / r_img - v = z / H - # Ellipse axes in texture coordinates - a_u = (rho * z) / (2 * np.pi * R) - a_v = (rho * z**2) / (R * H) - # Convert to pixels - a_u_px = a_u * width - a_v_px = a_v * height - for theta in theta_vals: - u = theta / (2 * np.pi) - cx = u * width - cy = v * height - # Draw ellipse (bounding box) - bbox = [cx - a_u_px, cy - a_v_px, cx + a_u_px, cy + a_v_px] - draw.ellipse(bbox, fill=0) # black dot - -img.save('cylinder_marker_texture.png') diff --git a/tools/patch_generation/patchgen_DD02M.py b/tools/patch_generation/patchgen_DD02M.py deleted file mode 100644 index 9b869ec..0000000 --- a/tools/patch_generation/patchgen_DD02M.py +++ /dev/null @@ -1,141 +0,0 @@ -#%% -import numpy as np -import cv2 - -def merge_intervals(intervals): - # Merge overlapping intervals. - if not intervals: - return [] - intervals.sort(key=lambda x: x[0]) - merged = [intervals[0]] - for current in intervals[1:]: - last = merged[-1] - if current[0] <= last[1]: - merged[-1] = (last[0], max(last[1], current[1])) - else: - merged.append(current) - return merged - -def get_allowed_intervals(neighbor_values, delta): - # Compute forbidden intervals for this channel from neighbor values. - forbidden = [] - for c in neighbor_values: - low = max(0, c - delta) - high = min(1, c + delta) - forbidden.append((low, high)) - # Merge overlapping forbidden intervals. - merged = merge_intervals(forbidden) - allowed = [] - # Allowed region before the first forbidden interval. - if merged: - if merged[0][0] > 0: - allowed.append((0, merged[0][0])) - # Allowed regions between forbidden intervals. - for i in range(1, len(merged)): - allowed.append((merged[i-1][1], merged[i][0])) - # Allowed region after the last forbidden interval. - if merged[-1][1] < 1: - allowed.append((merged[-1][1], 1)) - else: - allowed.append((0, 1)) - return allowed - -def sample_from_allowed(allowed): - # Choose a random value from the allowed intervals with probability proportional to interval length. - lengths = [end - start for start, end in allowed] - total_length = sum(lengths) - if total_length <= 0: - return None - point = np.random.rand() * total_length - cumulative = 0 - for (start, end), length in zip(allowed, lengths): - cumulative += length - if point <= cumulative: - return np.random.uniform(start, end) - return allowed[-1][0] - -def sample_channel(neighbor_values, delta): - # Compute allowed intervals and sample a value. - allowed = get_allowed_intervals(neighbor_values, delta) - if not allowed: - return None - return sample_from_allowed(allowed) - -def generate_valid_color(neighbors, r): - """ - Generate a random color that satisfies, for each neighbor, - |R - R_neighbor| >= sqrt(r), and similarly for G and B. - neighbors: list of neighbor colors (each is an array of shape (3,)) - r: threshold for squared difference. - """ - delta = np.sqrt(r) - color = np.zeros(3) - for channel in range(3): - neighbor_channel = [n[channel] for n in neighbors] - sampled = sample_channel(neighbor_channel, delta) - # If the allowed intervals are empty, use a fallback. - if sampled is None: - sampled = np.random.rand() - color[channel] = sampled - return color - -def generate_color_pattern(h, w, d, r): - """ - Generate a dense color pattern. - h, w: height and width of the output image in pixels. - d: patch size in pixels (each patch is d x d). - r: randomness regulation factor. For each neighbor the condition is: - (|channel_new - channel_neighbor|)^2 >= r. - """ - pattern = np.zeros((h, w, 3), dtype=np.float32) - num_patches_x = h // d - num_patches_y = w // d - - for i in range(num_patches_x): - for j in range(num_patches_y): - # Collect neighbor patch colors (using mean because patches have uniform color). - neighbors = [] - if i > 0: - patch_above = pattern[(i-1)*d:i*d, j*d:(j+1)*d] - neighbors.append(np.mean(patch_above, axis=(0, 1))) - if j > 0: - patch_left = pattern[i*d:(i+1)*d, (j-1)*d:j*d] - neighbors.append(np.mean(patch_left, axis=(0, 1))) - - if len(neighbors) == 0: - # For the first patch use a random color. - new_color = np.random.rand(3) - else: - new_color = generate_valid_color(neighbors, r) - - pattern[i*d:(i+1)*d, j*d:(j+1)*d] = new_color - print('Patch ({}, {})'.format(i, j)) - return pattern - -def main(): - # Parameters for the pattern. - # For example, using 100x100 pixels image with patches of 3x3 pixels. - # height = 2750 - # width = 1950 - height = 1200 - width = 1000 - patch_size = 12 - # Set r to a value that gives sufficient difference. - # Note: r is the threshold for squared difference per channel, and the condition is - # |c - c_neighbor| >= sqrt(r). Very high r may make it impossible to satisfy. - r = 1000 - - pattern = generate_color_pattern(height, width, patch_size, r) - # Scale to 0-255 for display and saving. - pattern_display = (pattern * 255).astype(np.uint8) - - # Display the pattern with OpenCV. - cv2.imshow('Pattern', pattern_display) - cv2.waitKey(0) - cv2.destroyAllWindows() - - # Save the pattern as an image file. - cv2.imwrite('color_pattern.png', pattern_display) - -if __name__ == '__main__': - main() diff --git a/tools/patch_generation/patchgen_DD03.py b/tools/patch_generation/patchgen_DD03.py deleted file mode 100644 index 27341d9..0000000 --- a/tools/patch_generation/patchgen_DD03.py +++ /dev/null @@ -1,141 +0,0 @@ -#%% -import numpy as np -import cv2 - -def merge_intervals(intervals): - # Merge overlapping intervals. - if not intervals: - return [] - intervals.sort(key=lambda x: x[0]) - merged = [intervals[0]] - for current in intervals[1:]: - last = merged[-1] - if current[0] <= last[1]: - merged[-1] = (last[0], max(last[1], current[1])) - else: - merged.append(current) - return merged - -def get_allowed_intervals(neighbor_values, delta): - # Compute forbidden intervals for this channel from neighbor values. - forbidden = [] - for c in neighbor_values: - low = max(0, c - delta) - high = min(1, c + delta) - forbidden.append((low, high)) - # Merge overlapping forbidden intervals. - merged = merge_intervals(forbidden) - allowed = [] - # Allowed region before the first forbidden interval. - if merged: - if merged[0][0] > 0: - allowed.append((0, merged[0][0])) - # Allowed regions between forbidden intervals. - for i in range(1, len(merged)): - allowed.append((merged[i-1][1], merged[i][0])) - # Allowed region after the last forbidden interval. - if merged[-1][1] < 1: - allowed.append((merged[-1][1], 1)) - else: - allowed.append((0, 1)) - return allowed - -def sample_from_allowed(allowed): - # Choose a random value from the allowed intervals with probability proportional to interval length. - lengths = [end - start for start, end in allowed] - total_length = sum(lengths) - if total_length <= 0: - return None - point = np.random.rand() * total_length - cumulative = 0 - for (start, end), length in zip(allowed, lengths): - cumulative += length - if point <= cumulative: - return np.random.uniform(start, end) - return allowed[-1][0] - -def sample_channel(neighbor_values, delta): - # Compute allowed intervals and sample a value. - allowed = get_allowed_intervals(neighbor_values, delta) - if not allowed: - return None - return sample_from_allowed(allowed) - -def generate_valid_color(neighbors, r): - """ - Generate a random color that satisfies, for each neighbor, - |R - R_neighbor| >= sqrt(r), and similarly for G and B. - neighbors: list of neighbor colors (each is an array of shape (3,)) - r: threshold for squared difference. - """ - delta = np.sqrt(r) - color = np.zeros(3) - for channel in range(3): - neighbor_channel = [n[channel] for n in neighbors] - sampled = sample_channel(neighbor_channel, delta) - # If the allowed intervals are empty, use a fallback. - if sampled is None: - sampled = np.random.rand() - color[channel] = sampled - return color - -def generate_color_pattern(h, w, d, r): - """ - Generate a dense color pattern. - h, w: height and width of the output image in pixels. - d: patch size in pixels (each patch is d x d). - r: randomness regulation factor. For each neighbor the condition is: - (|channel_new - channel_neighbor|)^2 >= r. - """ - pattern = np.zeros((h, w, 3), dtype=np.float32) - num_patches_x = h // d - num_patches_y = w // d - - for i in range(num_patches_x): - for j in range(num_patches_y): - # Collect neighbor patch colors (using mean because patches have uniform color). - neighbors = [] - if i > 0: - patch_above = pattern[(i-1)*d:i*d, j*d:(j+1)*d] - neighbors.append(np.mean(patch_above, axis=(0, 1))) - if j > 0: - patch_left = pattern[i*d:(i+1)*d, (j-1)*d:j*d] - neighbors.append(np.mean(patch_left, axis=(0, 1))) - - if len(neighbors) == 0: - # For the first patch use a random color. - new_color = np.random.rand(3) - else: - new_color = generate_valid_color(neighbors, r) - - pattern[i*d:(i+1)*d, j*d:(j+1)*d] = new_color - print('Patch ({}, {})'.format(i, j)) - return pattern - -def main(): - # Parameters for the pattern. - # For example, using 100x100 pixels image with patches of 3x3 pixels. - # height = 2750 - # width = 1950 - height = 1340 - width = 1360 - patch_size = 12 - # Set r to a value that gives sufficient difference. - # Note: r is the threshold for squared difference per channel, and the condition is - # |c - c_neighbor| >= sqrt(r). Very high r may make it impossible to satisfy. - r = 1000 - - pattern = generate_color_pattern(height, width, patch_size, r) - # Scale to 0-255 for display and saving. - pattern_display = (pattern * 255).astype(np.uint8) - - # Display the pattern with OpenCV. - cv2.imshow('Pattern', pattern_display) - cv2.waitKey(0) - cv2.destroyAllWindows() - - # Save the pattern as an image file. - cv2.imwrite('color_pattern.png', pattern_display) - -if __name__ == '__main__': - main() diff --git a/tools/patch_generation/w19.5_l27.5_p12.pdf b/tools/patch_generation/w19.5_l27.5_p12.pdf deleted file mode 100644 index 8a1bf4d..0000000 Binary files a/tools/patch_generation/w19.5_l27.5_p12.pdf and /dev/null differ diff --git a/tools/patch_generation/w19.5_l27.5_p12_checker.pdf b/tools/patch_generation/w19.5_l27.5_p12_checker.pdf deleted file mode 100644 index bb22801..0000000 Binary files a/tools/patch_generation/w19.5_l27.5_p12_checker.pdf and /dev/null differ diff --git a/tools/patch_generation/彩色+黑白-svg-pdf.zip b/tools/patch_generation/彩色+黑白-svg-pdf.zip deleted file mode 100644 index 8d68d23..0000000 Binary files a/tools/patch_generation/彩色+黑白-svg-pdf.zip and /dev/null differ diff --git a/tools/sync_downstreams.py b/tools/sync_downstreams.py new file mode 100644 index 0000000..c4f78aa --- /dev/null +++ b/tools/sync_downstreams.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Plan and apply downstream file syncs from tools/downstream_sync_manifest.json. + +The ``apply`` command copies files/directories listed in the manifest. It does +not delete extra downstream files and does not run downstream tests. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +from pathlib import Path + + +MANIFEST_NAME = "downstream_sync_manifest.json" + + +def sdk_root() -> Path: + return Path(__file__).resolve().parent.parent + + +def load_manifest(root: Path) -> dict: + path = Path(__file__).resolve().parent / MANIFEST_NAME + with path.open(encoding="utf-8") as handle: + data = json.load(handle) + if data.get("version") != 1: + raise SystemExit(f"unsupported manifest version: {data.get('version')}") + if not isinstance(data.get("downstreams"), list): + raise SystemExit("manifest is missing a downstreams list") + return data + + +def selected_downstreams(manifest: dict, target: str | None) -> list[dict]: + items = manifest["downstreams"] + if target is None: + return items + matches = [item for item in items if item.get("name") == target] + if not matches: + names = ", ".join(item.get("name", "") for item in items) + raise SystemExit(f"unknown target {target!r}; known: {names}") + return matches + + +def expand_jobs(root: Path, item: dict) -> list[tuple[Path, Path]]: + dest_root = (root / item["path"]).resolve() + jobs: list[tuple[Path, Path]] = [] + for entry in item.get("sync", []): + src_base = root / entry["from"] + dest_base = dest_root / entry["to"] + includes = entry.get("include") + if includes: + for name in includes: + jobs.append((src_base / name, dest_base / name)) + else: + jobs.append((src_base, dest_base)) + return jobs + + +def print_plan(root: Path, items: list[dict]) -> None: + print(f"SDK root: {root}") + print() + for item in items: + dest_root = (root / item["path"]).resolve() + print(f"== {item['name']} ==") + print(f"path: {dest_root}") + jobs = expand_jobs(root, item) + if jobs: + print("sync:") + for src, dest in jobs: + print(f" {src} -> {dest}") + else: + print("sync: (none)") + checks = item.get("checks") or [] + if checks: + print("checks:") + for command in checks: + print(f" {command}") + else: + print("checks: (none)") + print() + + +def check_paths(root: Path, items: list[dict]) -> int: + missing: list[str] = [] + print(f"SDK root: {root}") + print() + for item in items: + dest_root = (root / item["path"]).resolve() + print(f"== {item['name']} ==") + if dest_root.is_dir(): + print(f"ok downstream root: {dest_root}") + else: + print(f"MISS downstream root: {dest_root}") + missing.append(str(dest_root)) + for src, dest in expand_jobs(root, item): + src_ok = src.exists() + dest_note = "exists" if dest.exists() else "missing (ok until apply)" + status = "ok " if src_ok else "MISS" + print(f"{status} source: {src}") + print(f" dest: {dest} [{dest_note}]") + if not src_ok: + missing.append(str(src)) + print() + if missing: + print(f"{len(missing)} missing path(s).") + return 1 + print("all required source and downstream roots exist.") + return 0 + + +def copy_job(src: Path, dest: Path) -> None: + if not src.exists(): + raise FileNotFoundError(src) + if src.is_dir(): + shutil.copytree(src, dest, dirs_exist_ok=True) + return + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dest) + + +def apply_sync(root: Path, items: list[dict]) -> int: + if check_paths(root, items) != 0: + print("apply aborted because required paths are missing.") + return 1 + + copied = 0 + for item in items: + print(f"== {item['name']} ==") + for src, dest in expand_jobs(root, item): + copy_job(src, dest) + copied += 1 + print(f"copied {src} -> {dest}") + print() + print(f"copied {copied} sync job(s).") + print("checks were not run; execute the listed check commands manually.") + return 0 + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + plan = sub.add_parser("plan", help="print intended sync jobs and checks") + plan.add_argument("--target", help="limit to one downstream name") + + check = sub.add_parser("check-paths", help="verify source and dest roots exist") + check.add_argument("--target", help="limit to one downstream name") + + apply_cmd = sub.add_parser("apply", help="copy manifest sources to downstreams") + apply_cmd.add_argument("--target", help="limit to one downstream name") + + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv if argv is not None else sys.argv[1:]) + root = sdk_root() + items = selected_downstreams(load_manifest(root), getattr(args, "target", None)) + if args.command == "plan": + print_plan(root, items) + return 0 + if args.command == "check-paths": + return check_paths(root, items) + if args.command == "apply": + return apply_sync(root, items) + raise SystemExit(f"unknown command: {args.command}") + + +if __name__ == "__main__": + raise SystemExit(main())