diff --git a/build/README.md b/build/README.md index 8ae20d4..d9f2c3b 100644 --- a/build/README.md +++ b/build/README.md @@ -39,9 +39,14 @@ Direct module paths (same behavior): ```bash python build/wheel/build.py build\wheel\win.bat -.\build\wheel\docker\run.ps1 +.\build\wheel\docker\run.ps1 -SkipImageBuild # reuse the image when only sources changed ``` +The image bakes in per-ABI build tooling and a per-ABI wheelhouse of the runtime +deps, so the entrypoint no longer re-resolves them over the network. See +`docs/publishing.md` §6 for build-time details, why `-SkipImageBuild` matters, +and what to change before attempting to parallelize the per-ABI builds. + ### Release gate Local only: never publishes, and makes no network call unless `--check-pypi` is diff --git a/build/wheel/compile.py b/build/wheel/compile.py index 6b4d525..06b9a9a 100644 --- a/build/wheel/compile.py +++ b/build/wheel/compile.py @@ -32,12 +32,20 @@ NUITKA_NOFOLLOW_IMPORTS = ( "cv2", "numba", "orisys_blend", + # Optional accelerators / GUIs: imported behind try/except, resolved from the + # user's environment at runtime. Listing them here is not optional - without + # it, Nuitka follows whatever happens to be installed in the *build* + # environment, which blows up compile time and makes wheel contents depend on + # ambient packages. tests/test_nuitka_nofollow.py enforces full coverage. "cupy", "cupyx", "pyopencl", + "pyvkfft", "open3d", "PyQt5", "pyqtgraph", + "torch", + "torchvision", ) diff --git a/build/wheel/docker/Dockerfile.manylinux b/build/wheel/docker/Dockerfile.manylinux index fdc6ca0..5107e1f 100644 --- a/build/wheel/docker/Dockerfile.manylinux +++ b/build/wheel/docker/Dockerfile.manylinux @@ -4,23 +4,51 @@ FROM quay.io/pypa/manylinux_2_28_x86_64:latest ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ - PIP_ROOT_USER_ACTION=ignore + PIP_ROOT_USER_ACTION=ignore \ + ORISYS_WHEELHOUSE_ROOT=/opt/orisys-wheelhouse -# Only install compile/repair tooling here. Runtime deps (numpy/scipy/opencv/numba) -# are declared in the wheel metadata and installed by pip when users install orisys. -# Nuitka is configured with --nofollow-import-to for those packages. -RUN for PYBIN in \ - /opt/python/cp310-cp310/bin \ - /opt/python/cp311-cp311/bin \ - /opt/python/cp312-cp312/bin; do \ - echo "=== Installing build tooling into ${PYBIN} ===" && \ - "${PYBIN}/python" -m pip install --upgrade pip setuptools wheel && \ - "${PYBIN}/python" -m pip install \ - "nuitka>=2.0" \ - ordered-set \ - zstandard \ - auditwheel; \ - done +# Per-ABI build tooling plus a per-ABI wheelhouse of every package the release +# flow installs at build time: +# +# * build tooling - nuitka/auditwheel etc. baked in, so the entrypoint does +# not have to re-resolve them from PyPI on every container start +# (ORISYS_UPDATE_BUILD_TOOLS=1 opts back in to that). +# * wheelhouse - `build/wheel/build.py --target pypi-linux` force-enables +# verification, which creates a throwaway venv and pip-installs the built +# wheel plus its declared runtime deps. Runtime deps are deliberately NOT +# installed into the image interpreter (users get them from the wheel +# metadata), so without a wheelhouse every build re-downloaded +# numpy/scipy/opencv/numba — roughly 200 MB, three times over, once per ABI. +# The entrypoint points pip at the wheelhouse with +# PIP_NO_INDEX/PIP_FIND_LINKS, which keeps the venv isolated and still +# exercises real dependency resolution, just without the network. +# +# --only-binary=:all: makes a missing wheel a loud image-build failure instead of +# a slow source build inside the container. +RUN set -eux; \ + for TAG in cp310-cp310 cp311-cp311 cp312-cp312; do \ + PYBIN="/opt/python/${TAG}/bin"; \ + echo "=== build tooling for ${TAG} ==="; \ + "${PYBIN}/python" -m pip install --upgrade pip setuptools wheel; \ + "${PYBIN}/python" -m pip install \ + "nuitka>=2.0" \ + ordered-set \ + zstandard \ + auditwheel; \ + echo "=== wheelhouse for ${TAG} ==="; \ + "${PYBIN}/python" -m pip download \ + --only-binary=:all: \ + --dest "${ORISYS_WHEELHOUSE_ROOT}/${TAG}" \ + pip \ + setuptools \ + wheel \ + auditwheel \ + numpy \ + scipy \ + opencv-python \ + numba; \ + done; \ + du -sh "${ORISYS_WHEELHOUSE_ROOT}"/* WORKDIR /src ENTRYPOINT ["/bin/bash", "/src/build/wheel/docker/entrypoint.sh"] diff --git a/build/wheel/docker/entrypoint.sh b/build/wheel/docker/entrypoint.sh index 470b754..f4b92ba 100644 --- a/build/wheel/docker/entrypoint.sh +++ b/build/wheel/docker/entrypoint.sh @@ -6,12 +6,18 @@ set -euo pipefail TARGET="${ORISYS_BUILD_TARGET:-pypi-linux}" # Comma-separated manylinux tags, e.g. "cp310-cp310,cp311-cp311,cp312-cp312" PYTHON_TAGS="${ORISYS_PYTHON_TAGS:-cp310-cp310,cp311-cp311,cp312-cp312}" +WHEELHOUSE_ROOT="${ORISYS_WHEELHOUSE_ROOT:-/opt/orisys-wheelhouse}" +# The image already bakes in pip/setuptools/wheel/nuitka/auditwheel. Re-resolving +# them from PyPI on every container start costs a network round-trip per ABI and +# lets the running container drift from the image it was built from. +UPDATE_BUILD_TOOLS="${ORISYS_UPDATE_BUILD_TOOLS:-0}" cd /src echo "=== manylinux entrypoint ===" echo "Target: ${TARGET}" echo "Python tags: ${PYTHON_TAGS}" +echo "Wheelhouse root: ${WHEELHOUSE_ROOT}" uname -a IFS=',' read -r -a TAG_ARRAY <<< "${PYTHON_TAGS}" @@ -40,7 +46,27 @@ for tag in "${TAG_ARRAY[@]}"; do export PATH="${PYBIN}:${PATH}" export PYTHONPATH="/src/src${PYTHONPATH:+:${PYTHONPATH}}" - "${PYBIN}/python" -m pip install -q --upgrade pip setuptools wheel auditwheel + # Point pip at the per-ABI wheelhouse baked into the image. The pypi-linux + # target force-enables verification, which creates a throwaway venv and + # pip-installs the built wheel plus its declared runtime deps; serving those + # from the wheelhouse keeps the venv isolated (no --system-site-packages) while + # removing a ~200 MB download per ABI. PIP_* is inherited by every pip + # subprocess, including that venv. + wheelhouse="${WHEELHOUSE_ROOT}/${tag}" + if [[ -d "${wheelhouse}" ]]; then + export PIP_NO_INDEX=1 + export PIP_FIND_LINKS="${wheelhouse}" + echo "[info] pip is offline, installing from ${wheelhouse}" + else + echo "[warn] no wheelhouse at ${wheelhouse}; pip will use the network." >&2 + echo "[warn] rebuild the image without -SkipImageBuild to bake it in." >&2 + unset PIP_NO_INDEX PIP_FIND_LINKS || true + fi + + if [[ "${UPDATE_BUILD_TOOLS}" == "1" ]]; then + echo "[info] refreshing build tooling (ORISYS_UPDATE_BUILD_TOOLS=1)" + "${PYBIN}/python" -m pip install -q --upgrade pip setuptools wheel auditwheel + fi if "${PYBIN}/python" build/wheel/build.py --target "${TARGET}"; then BUILT+=("${tag}") @@ -48,6 +74,8 @@ for tag in "${TAG_ARRAY[@]}"; do echo "[ERROR] Build failed for ${tag}" >&2 FAILED=1 fi + + unset PIP_NO_INDEX PIP_FIND_LINKS || true done echo "" diff --git a/docs/publishing.md b/docs/publishing.md index 714c069..db64d3e 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -234,3 +234,58 @@ root = distribution("orisys").locate_file("orisys") # site-packages/orisys 如果以后需要离线 API 参考,加一个只包含 `docs/api.md` 的窄条目即可,不要整个 `docs/` 都塞进去。 +--- + +## 6. 缩短 manylinux 构建时间 + +`python build/run.py manylinux` 走 `build/wheel/docker/run.ps1` → 容器内 `entrypoint.sh`,对 `cp310/cp311/cp312` **串行**各做一次 Nuitka 编译 + 打 wheel + auditwheel repair + 隔离 venv 校验。 + +### 6.1 镜像里已经烘好的东西(所以每次构建不必再拉) + +`Dockerfile.manylinux` 在每个 ABI 下预置了两类内容: + +1. **构建工具**:`pip`/`setuptools`/`wheel`/`nuitka`/`ordered-set`/`zstandard`/`auditwheel`。 + 因此 `entrypoint.sh` 默认**不再**在每个 ABI 上重跑 `pip install --upgrade pip setuptools wheel auditwheel` —— 那是每次构建 3 次多余的 PyPI 往返,还会让容器里的工具版本与镜像不一致。需要用最新工具时显式开启: + + ```powershell + docker run ... -e ORISYS_UPDATE_BUILD_TOOLS=1 ... + ``` + +2. **每个 ABI 一个 wheelhouse**(`/opt/orisys-wheelhouse/`),内含 `numpy`/`scipy`/`opencv-python`/`numba` 及其依赖闭包,外加 `pip`/`setuptools`/`wheel`/`auditwheel` 的 wheel。 + + `--target pypi-linux` 会**强制**校验:为每个 ABI 建一个一次性 venv,再 `pip install` 刚构建的 wheel 和它声明的运行期依赖。镜像故意不把运行期依赖装进解释器(用户是凭 wheel 元数据装的),所以以前每次构建都要现下约 200 MB × 3。`entrypoint.sh` 现在用 `PIP_NO_INDEX=1` + `PIP_FIND_LINKS=` 把 pip 指向本地 wheelhouse:**venv 仍然是隔离的**(没有 `--system-site-packages`),依赖解析与安装依然真实发生,只是不走网络。 + + 找不到 wheelhouse 时会打印警告并回退到联网,不会静默失败。镜像因此增大约 400–500 MB(一次性代价,换每次构建省三次下载)。 + +### 6.2 不要为了省事直接重建镜像 + +只改了源码(`src/`、`config/`、`sensors/`、`examples/`、`pyproject.toml`)时加 `-SkipImageBuild`: + +```powershell +.\build\wheel\docker\run.ps1 -SkipImageBuild +``` + +Dockerfile 的 `RUN` 层要装 3 份工具链并下载 3 个 wheelhouse,缓存未命中就是数分钟。**改了 `Dockerfile.manylinux` 或需要更新 wheelhouse 时才重建镜像**。 + +### 6.3 Nuitka 的排除列表必须完整 + +`build/wheel/compile.py::NUITKA_NOFOLLOW_IMPORTS` 列出所有**不打进 wheel、运行期从用户环境解析**的第三方包。漏一个意味着 Nuitka 会尝试把它跟进来:编译时间暴涨,而且 wheel 内容开始取决于构建机上恰好装了什么 —— `torch`、`torchvision`(`src/orisys/tracker.py` 的 RAFT 追踪器)和 `pyvkfft`(`backend_availability.py`)就是这样漏掉的,之前只是靠「构建机上没装」才没出事。 + +`tests/test_nuitka_nofollow.py` 会扫描 `src/orisys` 的全部第三方导入,任何不在排除列表里的都会让测试失败。**新增可选依赖时同步更新列表。** + +### 6.4 想要并行?先读这一段 + +`--jobs` 的默认值是**全部 CPU 数**,单次编译的 C 阶段已经吃满所有核心。真正能靠并行省下的是 Nuitka 的单线程前端(分析/优化/代码生成)、scons 启动、`auditwheel` 与 venv 校验 —— 做法是 3 路并行 + 每个 build 限 `--jobs=<核数/3>`。 + +但**现在不能直接并行**:`build/common/paths.py` 里的工作目录是写死的共享路径 + +``` +build/nuitka/compile +build/nuitka/wheel +build/nuitka/wheel/repaired +``` + +而 `build/wheel/build.py` 每次开头都 `_reset_dir()`(`rmtree`)这两个目录,`auditwheel repair` 又会再 `rmtree` 一次 `repaired`。仓库是挂载进容器的(`-v ${RepoRoot}:/src`),所以无论「一个容器 3 路并行」还是「3 个容器并行」,只要共享同一个挂载点,三个 build 就会互相清空暂存目录 —— 表现为随机失败或 wheel 内容错乱。 + +要并行必须先让工作目录带上 ABI 标签(如 `build/nuitka//compile`)。输出路径不用动:`SDK//`、`dist/repaired/` 的文件名本来就按 ABI 区分,下游 `release preflight` 不受影响。另外 3 个 Nuitka 并行很吃内存,先把 Docker Desktop 内存调到 16 GB 以上再试。 + diff --git a/tests/conftest.py b/tests/conftest.py index e75c9c3..078f9fb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,8 +22,25 @@ import pytest REPO_ROOT = Path(__file__).resolve().parents[1] SRC_ROOT = REPO_ROOT / "src" -if str(SRC_ROOT) not in sys.path: - sys.path.insert(0, str(SRC_ROOT)) +BUILD_ROOT = REPO_ROOT / "build" +EXAMPLES_ROOT = REPO_ROOT / "examples" + +# Establish import precedence once, here, before any test module is imported. The +# test modules insert the same directories, but all of those inserts are guarded +# by `if path not in sys.path`, so once the paths are present the guards turn +# them into no-ops and this order sticks. +# +# Order matters and is not incidental: +# * examples/ must win over build/ because `build/gui` is also a package named +# `gui`; with build/ first, `import gui.viewer2d` silently resolves to the +# build tooling and fails. +# * build/ must precede site-packages so `wheel`, `common` and `release` mean +# the repository's build tooling, not the unrelated PyPI `wheel` package. +# * src/ provides the checkout's `orisys`. +# Insert in reverse priority (each insert goes to the front). +for _path in (SRC_ROOT, BUILD_ROOT, EXAMPLES_ROOT): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) from orisys import Sensor from orisys import configs as orisys_configs diff --git a/tests/test_harness_guards.py b/tests/test_harness_guards.py index d079398..b2cf3cd 100644 --- a/tests/test_harness_guards.py +++ b/tests/test_harness_guards.py @@ -93,6 +93,32 @@ def test_fixtures_are_registered_with_pytest(request): ) +def test_import_precedence_is_correct(): + """examples/ must beat build/ for `gui`; build/ must beat site-packages. + + Regression guard: build/gui is a package named `gui`, so if build/ precedes + examples/ on sys.path then `import gui.viewer2d` resolves to the build + tooling and tests/test_qt_viewer_backend.py fails at collection. + """ + import common + import gui + import wheel + + examples_gui = (REPO_ROOT / "examples" / "gui").resolve() + build_root = (REPO_ROOT / "build").resolve() + + assert Path(gui.__file__).resolve().parent == examples_gui, ( + f"`gui` resolved to {gui.__file__} instead of {examples_gui}; " + "examples/ must precede build/ on sys.path" + ) + for module in (common, wheel): + resolved = Path(module.__file__).resolve() + assert build_root in resolved.parents, ( + f"`{module.__name__}` resolved to {resolved}, outside {build_root}; " + "build/ must precede site-packages on sys.path" + ) + + def test_tests_run_against_checkout_not_stale_install(): """Fail loudly when an old orisys in site-packages shadows src/orisys.""" src_pkg = (REPO_ROOT / "src" / "orisys").resolve() diff --git a/tests/test_nuitka_nofollow.py b/tests/test_nuitka_nofollow.py new file mode 100644 index 0000000..f6c4360 --- /dev/null +++ b/tests/test_nuitka_nofollow.py @@ -0,0 +1,72 @@ +"""Every third-party import in src/orisys must be excluded from the Nuitka compile. + +Omitting one means Nuitka tries to follow it into the compiled extension: compile +time explodes, and the wheel's contents start depending on whatever happens to be +installed in the *build* environment rather than on what is declared. ``torch``, +``torchvision`` and ``pyvkfft`` were imported by the package but missing from the +exclusion list; this test keeps that list complete. +""" + +from __future__ import annotations + +import ast +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +BUILD_ROOT = REPO_ROOT / "build" +if str(BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(BUILD_ROOT)) + +from wheel.compile import NUITKA_INCLUDE_PACKAGES, NUITKA_NOFOLLOW_IMPORTS # noqa: E402 + +PACKAGE_ROOT = REPO_ROOT / "src" / "orisys" +FIRST_PARTY = {"orisys", "orisys_blend"} + + +def _third_party_imports() -> dict[str, set[str]]: + """Top-level third-party import names -> the files that import them.""" + stdlib = set(sys.stdlib_module_names) + found: dict[str, set[str]] = {} + + for path in sorted(PACKAGE_ROOT.rglob("*.py")): + try: + tree = ast.parse(path.read_text(encoding="utf-8", errors="ignore")) + except SyntaxError: + continue + for node in ast.walk(tree): + names: list[str] = [] + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0: + names = [node.module] + for name in names: + top = name.split(".")[0] + if top in stdlib or top in FIRST_PARTY: + continue + found.setdefault(top, set()).add(path.relative_to(REPO_ROOT).as_posix()) + return found + + +def test_every_third_party_import_is_nofollowed(): + found = _third_party_imports() + assert found, "expected to find third-party imports in src/orisys" + + missing = sorted(set(found) - set(NUITKA_NOFOLLOW_IMPORTS)) + assert not missing, ( + "these third-party modules are imported by src/orisys but missing from " + "build/wheel/compile.py::NUITKA_NOFOLLOW_IMPORTS:\n " + + "\n ".join(f"{name} (imported by {sorted(found[name])})" for name in missing) + + "\nNuitka will try to follow them into the compiled module; add them to " + "the exclusion list so they resolve from the user's environment at runtime." + ) + + +def test_included_packages_are_first_party_or_subpackages(): + assert "orisys" in NUITKA_INCLUDE_PACKAGES + for name in NUITKA_INCLUDE_PACKAGES: + assert name == "orisys" or name.startswith("orisys."), name + + +def test_nofollow_list_has_no_duplicates(): + assert len(NUITKA_NOFOLLOW_IMPORTS) == len(set(NUITKA_NOFOLLOW_IMPORTS))