faster linux build

build/README.md

@@ -39,9 +39,14 @@ Direct module paths (same behavior):
 ```bash
 python build/wheel/build.py
 build\wheel\win.bat
-.\build\wheel\docker\run.ps1
+.\build\wheel\docker\run.ps1 -SkipImageBuild   # reuse the image when only sources changed
 ```
 
+The image bakes in per-ABI build tooling and a per-ABI wheelhouse of the runtime
+deps, so the entrypoint no longer re-resolves them over the network. See
+`docs/publishing.md` §6 for build-time details, why `-SkipImageBuild` matters,
+and what to change before attempting to parallelize the per-ABI builds.
+
 ### Release gate
 
 Local only: never publishes, and makes no network call unless `--check-pypi` is

build/wheel/compile.py

@@ -32,12 +32,20 @@ NUITKA_NOFOLLOW_IMPORTS = (
     "cv2",
     "numba",
     "orisys_blend",
+    # Optional accelerators / GUIs: imported behind try/except, resolved from the
+    # user's environment at runtime. Listing them here is not optional - without
+    # it, Nuitka follows whatever happens to be installed in the *build*
+    # environment, which blows up compile time and makes wheel contents depend on
+    # ambient packages. tests/test_nuitka_nofollow.py enforces full coverage.
     "cupy",
     "cupyx",
     "pyopencl",
+    "pyvkfft",
     "open3d",
     "PyQt5",
     "pyqtgraph",
+    "torch",
+    "torchvision",
 )
 
 

build/wheel/docker/Dockerfile.manylinux

@@ -4,23 +4,51 @@
 FROM quay.io/pypa/manylinux_2_28_x86_64:latest
 
 ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_ROOT_USER_ACTION=ignore
+    PIP_ROOT_USER_ACTION=ignore \
+    ORISYS_WHEELHOUSE_ROOT=/opt/orisys-wheelhouse
 
-# Only install compile/repair tooling here. Runtime deps (numpy/scipy/opencv/numba)
-# are declared in the wheel metadata and installed by pip when users install orisys.
-# Nuitka is configured with --nofollow-import-to for those packages.
-RUN for PYBIN in \
-        /opt/python/cp310-cp310/bin \
-        /opt/python/cp311-cp311/bin \
-        /opt/python/cp312-cp312/bin; do \
-      echo "=== Installing build tooling into ${PYBIN} ===" && \
-      "${PYBIN}/python" -m pip install --upgrade pip setuptools wheel && \
+# Per-ABI build tooling plus a per-ABI wheelhouse of every package the release
+# flow installs at build time:
+#
+#   * build tooling  - nuitka/auditwheel etc. baked in, so the entrypoint does
+#     not have to re-resolve them from PyPI on every container start
+#     (ORISYS_UPDATE_BUILD_TOOLS=1 opts back in to that).
+#   * wheelhouse     - `build/wheel/build.py --target pypi-linux` force-enables
+#     verification, which creates a throwaway venv and pip-installs the built
+#     wheel plus its declared runtime deps. Runtime deps are deliberately NOT
+#     installed into the image interpreter (users get them from the wheel
+#     metadata), so without a wheelhouse every build re-downloaded
+#     numpy/scipy/opencv/numba — roughly 200 MB, three times over, once per ABI.
+#     The entrypoint points pip at the wheelhouse with
+#     PIP_NO_INDEX/PIP_FIND_LINKS, which keeps the venv isolated and still
+#     exercises real dependency resolution, just without the network.
+#
+# --only-binary=:all: makes a missing wheel a loud image-build failure instead of
+# a slow source build inside the container.
+RUN set -eux; \
+    for TAG in cp310-cp310 cp311-cp311 cp312-cp312; do \
+        PYBIN="/opt/python/${TAG}/bin"; \
+        echo "=== build tooling for ${TAG} ==="; \
+        "${PYBIN}/python" -m pip install --upgrade pip setuptools wheel; \
         "${PYBIN}/python" -m pip install \
             "nuitka>=2.0" \
             ordered-set \
             zstandard \
             auditwheel; \
-    done
+        echo "=== wheelhouse for ${TAG} ==="; \
+        "${PYBIN}/python" -m pip download \
+            --only-binary=:all: \
+            --dest "${ORISYS_WHEELHOUSE_ROOT}/${TAG}" \
+            pip \
+            setuptools \
+            wheel \
+            auditwheel \
+            numpy \
+            scipy \
+            opencv-python \
+            numba; \
+    done; \
+    du -sh "${ORISYS_WHEELHOUSE_ROOT}"/*
 
 WORKDIR /src
 ENTRYPOINT ["/bin/bash", "/src/build/wheel/docker/entrypoint.sh"]

build/wheel/docker/entrypoint.sh

@@ -6,12 +6,18 @@ set -euo pipefail
 TARGET="${ORISYS_BUILD_TARGET:-pypi-linux}"
 # Comma-separated manylinux tags, e.g. "cp310-cp310,cp311-cp311,cp312-cp312"
 PYTHON_TAGS="${ORISYS_PYTHON_TAGS:-cp310-cp310,cp311-cp311,cp312-cp312}"
+WHEELHOUSE_ROOT="${ORISYS_WHEELHOUSE_ROOT:-/opt/orisys-wheelhouse}"
+# The image already bakes in pip/setuptools/wheel/nuitka/auditwheel. Re-resolving
+# them from PyPI on every container start costs a network round-trip per ABI and
+# lets the running container drift from the image it was built from.
+UPDATE_BUILD_TOOLS="${ORISYS_UPDATE_BUILD_TOOLS:-0}"
 
 cd /src
 
 echo "=== manylinux entrypoint ==="
 echo "Target: ${TARGET}"
 echo "Python tags: ${PYTHON_TAGS}"
+echo "Wheelhouse root: ${WHEELHOUSE_ROOT}"
 uname -a
 
 IFS=',' read -r -a TAG_ARRAY <<< "${PYTHON_TAGS}"
@@ -40,7 +46,27 @@ for tag in "${TAG_ARRAY[@]}"; do
   export PATH="${PYBIN}:${PATH}"
   export PYTHONPATH="/src/src${PYTHONPATH:+:${PYTHONPATH}}"
 
+  # Point pip at the per-ABI wheelhouse baked into the image. The pypi-linux
+  # target force-enables verification, which creates a throwaway venv and
+  # pip-installs the built wheel plus its declared runtime deps; serving those
+  # from the wheelhouse keeps the venv isolated (no --system-site-packages) while
+  # removing a ~200 MB download per ABI. PIP_* is inherited by every pip
+  # subprocess, including that venv.
+  wheelhouse="${WHEELHOUSE_ROOT}/${tag}"
+  if [[ -d "${wheelhouse}" ]]; then
+    export PIP_NO_INDEX=1
+    export PIP_FIND_LINKS="${wheelhouse}"
+    echo "[info] pip is offline, installing from ${wheelhouse}"
+  else
+    echo "[warn] no wheelhouse at ${wheelhouse}; pip will use the network." >&2
+    echo "[warn] rebuild the image without -SkipImageBuild to bake it in." >&2
+    unset PIP_NO_INDEX PIP_FIND_LINKS || true
+  fi
+
+  if [[ "${UPDATE_BUILD_TOOLS}" == "1" ]]; then
+    echo "[info] refreshing build tooling (ORISYS_UPDATE_BUILD_TOOLS=1)"
     "${PYBIN}/python" -m pip install -q --upgrade pip setuptools wheel auditwheel
+  fi
 
   if "${PYBIN}/python" build/wheel/build.py --target "${TARGET}"; then
     BUILT+=("${tag}")
@@ -48,6 +74,8 @@ for tag in "${TAG_ARRAY[@]}"; do
     echo "[ERROR] Build failed for ${tag}" >&2
     FAILED=1
   fi
+
+  unset PIP_NO_INDEX PIP_FIND_LINKS || true
 done
 
 echo ""

docs/publishing.md

@@ -234,3 +234,58 @@ root = distribution("orisys").locate_file("orisys")   # site-packages/orisys
 
 如果以后需要离线 API 参考，加一个只包含 `docs/api.md` 的窄条目即可，不要整个 `docs/` 都塞进去。
 
+---
+
+## 6. 缩短 manylinux 构建时间
+
+`python build/run.py manylinux` 走 `build/wheel/docker/run.ps1` → 容器内 `entrypoint.sh`，对 `cp310/cp311/cp312` **串行**各做一次 Nuitka 编译 + 打 wheel + auditwheel repair + 隔离 venv 校验。
+
+### 6.1 镜像里已经烘好的东西（所以每次构建不必再拉）
+
+`Dockerfile.manylinux` 在每个 ABI 下预置了两类内容：
+
+1. **构建工具**：`pip`/`setuptools`/`wheel`/`nuitka`/`ordered-set`/`zstandard`/`auditwheel`。
+   因此 `entrypoint.sh` 默认**不再**在每个 ABI 上重跑 `pip install --upgrade pip setuptools wheel auditwheel` —— 那是每次构建 3 次多余的 PyPI 往返，还会让容器里的工具版本与镜像不一致。需要用最新工具时显式开启：
+
+   ```powershell
+   docker run ... -e ORISYS_UPDATE_BUILD_TOOLS=1 ...
+   ```
+
+2. **每个 ABI 一个 wheelhouse**（`/opt/orisys-wheelhouse/<tag>`），内含 `numpy`/`scipy`/`opencv-python`/`numba` 及其依赖闭包，外加 `pip`/`setuptools`/`wheel`/`auditwheel` 的 wheel。
+
+   `--target pypi-linux` 会**强制**校验：为每个 ABI 建一个一次性 venv，再 `pip install` 刚构建的 wheel 和它声明的运行期依赖。镜像故意不把运行期依赖装进解释器（用户是凭 wheel 元数据装的），所以以前每次构建都要现下约 200 MB × 3。`entrypoint.sh` 现在用 `PIP_NO_INDEX=1` + `PIP_FIND_LINKS=<wheelhouse>` 把 pip 指向本地 wheelhouse：**venv 仍然是隔离的**（没有 `--system-site-packages`），依赖解析与安装依然真实发生，只是不走网络。
+
+   找不到 wheelhouse 时会打印警告并回退到联网，不会静默失败。镜像因此增大约 400–500 MB（一次性代价，换每次构建省三次下载）。
+
+### 6.2 不要为了省事直接重建镜像
+
+只改了源码（`src/`、`config/`、`sensors/`、`examples/`、`pyproject.toml`）时加 `-SkipImageBuild`：
+
+```powershell
+.\build\wheel\docker\run.ps1 -SkipImageBuild
+```
+
+Dockerfile 的 `RUN` 层要装 3 份工具链并下载 3 个 wheelhouse，缓存未命中就是数分钟。**改了 `Dockerfile.manylinux` 或需要更新 wheelhouse 时才重建镜像**。
+
+### 6.3 Nuitka 的排除列表必须完整
+
+`build/wheel/compile.py::NUITKA_NOFOLLOW_IMPORTS` 列出所有**不打进 wheel、运行期从用户环境解析**的第三方包。漏一个意味着 Nuitka 会尝试把它跟进来：编译时间暴涨，而且 wheel 内容开始取决于构建机上恰好装了什么 —— `torch`、`torchvision`（`src/orisys/tracker.py` 的 RAFT 追踪器）和 `pyvkfft`（`backend_availability.py`）就是这样漏掉的，之前只是靠「构建机上没装」才没出事。
+
+`tests/test_nuitka_nofollow.py` 会扫描 `src/orisys` 的全部第三方导入，任何不在排除列表里的都会让测试失败。**新增可选依赖时同步更新列表。**
+
+### 6.4 想要并行？先读这一段
+
+`--jobs` 的默认值是**全部 CPU 数**，单次编译的 C 阶段已经吃满所有核心。真正能靠并行省下的是 Nuitka 的单线程前端（分析/优化/代码生成）、scons 启动、`auditwheel` 与 venv 校验 —— 做法是 3 路并行 + 每个 build 限 `--jobs=<核数/3>`。
+
+但**现在不能直接并行**：`build/common/paths.py` 里的工作目录是写死的共享路径
+
+```
+build/nuitka/compile
+build/nuitka/wheel
+build/nuitka/wheel/repaired
+```
+
+而 `build/wheel/build.py` 每次开头都 `_reset_dir()`（`rmtree`）这两个目录，`auditwheel repair` 又会再 `rmtree` 一次 `repaired`。仓库是挂载进容器的（`-v ${RepoRoot}:/src`），所以无论「一个容器 3 路并行」还是「3 个容器并行」，只要共享同一个挂载点，三个 build 就会互相清空暂存目录 —— 表现为随机失败或 wheel 内容错乱。
+
+要并行必须先让工作目录带上 ABI 标签（如 `build/nuitka/<tag>/compile`）。输出路径不用动：`SDK/<version>/`、`dist/repaired/` 的文件名本来就按 ABI 区分，下游 `release preflight` 不受影响。另外 3 个 Nuitka 并行很吃内存，先把 Docker Desktop 内存调到 16 GB 以上再试。
+

tests/conftest.py

@@ -22,8 +22,25 @@ import pytest
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
 SRC_ROOT = REPO_ROOT / "src"
-if str(SRC_ROOT) not in sys.path:
-    sys.path.insert(0, str(SRC_ROOT))
+BUILD_ROOT = REPO_ROOT / "build"
+EXAMPLES_ROOT = REPO_ROOT / "examples"
+
+# Establish import precedence once, here, before any test module is imported. The
+# test modules insert the same directories, but all of those inserts are guarded
+# by `if path not in sys.path`, so once the paths are present the guards turn
+# them into no-ops and this order sticks.
+#
+# Order matters and is not incidental:
+#   * examples/ must win over build/ because `build/gui` is also a package named
+#     `gui`; with build/ first, `import gui.viewer2d` silently resolves to the
+#     build tooling and fails.
+#   * build/ must precede site-packages so `wheel`, `common` and `release` mean
+#     the repository's build tooling, not the unrelated PyPI `wheel` package.
+#   * src/ provides the checkout's `orisys`.
+# Insert in reverse priority (each insert goes to the front).
+for _path in (SRC_ROOT, BUILD_ROOT, EXAMPLES_ROOT):
+    if str(_path) not in sys.path:
+        sys.path.insert(0, str(_path))
 
 from orisys import Sensor
 from orisys import configs as orisys_configs

tests/test_harness_guards.py

@@ -93,6 +93,32 @@ def test_fixtures_are_registered_with_pytest(request):
     )
 
 
+def test_import_precedence_is_correct():
+    """examples/ must beat build/ for `gui`; build/ must beat site-packages.
+
+    Regression guard: build/gui is a package named `gui`, so if build/ precedes
+    examples/ on sys.path then `import gui.viewer2d` resolves to the build
+    tooling and tests/test_qt_viewer_backend.py fails at collection.
+    """
+    import common
+    import gui
+    import wheel
+
+    examples_gui = (REPO_ROOT / "examples" / "gui").resolve()
+    build_root = (REPO_ROOT / "build").resolve()
+
+    assert Path(gui.__file__).resolve().parent == examples_gui, (
+        f"`gui` resolved to {gui.__file__} instead of {examples_gui}; "
+        "examples/ must precede build/ on sys.path"
+    )
+    for module in (common, wheel):
+        resolved = Path(module.__file__).resolve()
+        assert build_root in resolved.parents, (
+            f"`{module.__name__}` resolved to {resolved}, outside {build_root}; "
+            "build/ must precede site-packages on sys.path"
+        )
+
+
 def test_tests_run_against_checkout_not_stale_install():
     """Fail loudly when an old orisys in site-packages shadows src/orisys."""
     src_pkg = (REPO_ROOT / "src" / "orisys").resolve()

tests/test_nuitka_nofollow.py

@@ -0,0 +1,72 @@
+"""Every third-party import in src/orisys must be excluded from the Nuitka compile.
+
+Omitting one means Nuitka tries to follow it into the compiled extension: compile
+time explodes, and the wheel's contents start depending on whatever happens to be
+installed in the *build* environment rather than on what is declared. ``torch``,
+``torchvision`` and ``pyvkfft`` were imported by the package but missing from the
+exclusion list; this test keeps that list complete.
+"""
+
+from __future__ import annotations
+
+import ast
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+BUILD_ROOT = REPO_ROOT / "build"
+if str(BUILD_ROOT) not in sys.path:
+    sys.path.insert(0, str(BUILD_ROOT))
+
+from wheel.compile import NUITKA_INCLUDE_PACKAGES, NUITKA_NOFOLLOW_IMPORTS  # noqa: E402
+
+PACKAGE_ROOT = REPO_ROOT / "src" / "orisys"
+FIRST_PARTY = {"orisys", "orisys_blend"}
+
+
+def _third_party_imports() -> dict[str, set[str]]:
+    """Top-level third-party import names -> the files that import them."""
+    stdlib = set(sys.stdlib_module_names)
+    found: dict[str, set[str]] = {}
+
+    for path in sorted(PACKAGE_ROOT.rglob("*.py")):
+        try:
+            tree = ast.parse(path.read_text(encoding="utf-8", errors="ignore"))
+        except SyntaxError:
+            continue
+        for node in ast.walk(tree):
+            names: list[str] = []
+            if isinstance(node, ast.Import):
+                names = [alias.name for alias in node.names]
+            elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
+                names = [node.module]
+            for name in names:
+                top = name.split(".")[0]
+                if top in stdlib or top in FIRST_PARTY:
+                    continue
+                found.setdefault(top, set()).add(path.relative_to(REPO_ROOT).as_posix())
+    return found
+
+
+def test_every_third_party_import_is_nofollowed():
+    found = _third_party_imports()
+    assert found, "expected to find third-party imports in src/orisys"
+
+    missing = sorted(set(found) - set(NUITKA_NOFOLLOW_IMPORTS))
+    assert not missing, (
+        "these third-party modules are imported by src/orisys but missing from "
+        "build/wheel/compile.py::NUITKA_NOFOLLOW_IMPORTS:\n  "
+        + "\n  ".join(f"{name} (imported by {sorted(found[name])})" for name in missing)
+        + "\nNuitka will try to follow them into the compiled module; add them to "
+        "the exclusion list so they resolve from the user's environment at runtime."
+    )
+
+
+def test_included_packages_are_first_party_or_subpackages():
+    assert "orisys" in NUITKA_INCLUDE_PACKAGES
+    for name in NUITKA_INCLUDE_PACKAGES:
+        assert name == "orisys" or name.startswith("orisys."), name
+
+
+def test_nofollow_list_has_no_duplicates():
+    assert len(NUITKA_NOFOLLOW_IMPORTS) == len(set(NUITKA_NOFOLLOW_IMPORTS))