orisys-tech/orisys-sdk
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

finished docker manylinux compliation for pypi

Browse Source
This commit is contained in:
ykcheungab
2026-08-03 17:07:54 +08:00
parent 3117ddd5b9
commit d7c5845646
31 changed files with 709 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/Dockerfile.manylinux
+26
View File
@@ -0,0 +1,26 @@
# manylinux_2_28: modern glibc baseline with binary wheels for numpy/scipy/opencv.
# Prefer this over manylinux2014 — recent NumPy requires GCC >= 10.3 and will try
# (and fail) to build from source on the older image.
FROM quay.io/pypa/manylinux_2_28_x86_64:latest
ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_ROOT_USER_ACTION=ignore
# Only install compile/repair tooling here. Runtime deps (numpy/scipy/opencv/numba)
# are declared in the wheel metadata and installed by pip when users install orisys.
# Nuitka is configured with --nofollow-import-to for those packages.
RUN for PYBIN in \
/opt/python/cp310-cp310/bin \
/opt/python/cp311-cp311/bin \
/opt/python/cp312-cp312/bin; do \
echo "=== Installing build tooling into ${PYBIN} ===" && \
"${PYBIN}/python" -m pip install --upgrade pip setuptools wheel && \
"${PYBIN}/python" -m pip install \
"nuitka>=2.0" \
ordered-set \
zstandard \
auditwheel; \
done
WORKDIR /src
ENTRYPOINT ["/bin/bash", "/src/build/manylinux_entrypoint.sh"]
build/build_docker.txt
+2
View File
@@ -0,0 +1,2 @@
powershell -ExecutionPolicy Bypass -File .\build\build_manylinux_docker.ps1
powershell -ExecutionPolicy Bypass -File .\build\build_manylinux_docker.ps1 -PythonVersions 3.10,3.12
build/build_manylinux_docker.ps1
+104
View File
@@ -0,0 +1,104 @@
# Build PyPI-compliant manylinux wheels for CPython 3.10/3.11/3.12 using Docker Desktop.
#
# Usage (from repo root, PowerShell):
# .\build\build_manylinux_docker.ps1
# .\build\build_manylinux_docker.ps1 -PythonVersions 3.10,3.11,3.12
# .\build\build_manylinux_docker.ps1 -PythonVersions 3.11
# .\build\build_manylinux_docker.ps1 -SkipImageBuild
#
# From cmd.exe:
# powershell -ExecutionPolicy Bypass -File .\build\build_manylinux_docker.ps1
#
# Prerequisites:
# - Docker Desktop running in Linux container mode
# - Enough disk for the manylinux image (~2GB+)
param(
[ValidateSet("pypi-linux", "internal")]
[string]$Target = "pypi-linux",
[string]$PythonVersions = "3.10,3.11,3.12",
[switch]$SkipImageBuild,
[string]$ImageName = "orisys-manylinux"
)
$ErrorActionPreference = "Stop"
function Convert-ToManylinuxTag {
param([string]$Version)
$v = $Version.Trim()
switch ($v) {
"3.10" { return "cp310-cp310" }
"3.11" { return "cp311-cp311" }
"3.12" { return "cp312-cp312" }
"cp310-cp310" { return "cp310-cp310" }
"cp311-cp311" { return "cp311-cp311" }
"cp312-cp312" { return "cp312-cp312" }
default {
throw "Unsupported Python version '$Version'. Use 3.10, 3.11, and/or 3.12."
}
}
}
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
Set-Location $RepoRoot
$tags = @()
foreach ($part in ($PythonVersions -split ",")) {
if ([string]::IsNullOrWhiteSpace($part)) { continue }
$tags += (Convert-ToManylinuxTag $part)
}
if ($tags.Count -eq 0) {
throw "No Python versions specified."
}
$pythonTags = [string]::Join(",", $tags)
Write-Host "=== Orisys manylinux Docker build ==="
Write-Host "Repo: $RepoRoot"
Write-Host "Target: $Target"
Write-Host "Image: $ImageName"
Write-Host "Python tags: $pythonTags"
# Docker Desktop must be in Linux container mode for manylinux.
$dockerInfo = docker version --format "{{.Server.Os}}" 2>$null
if ($LASTEXITCODE -ne 0) {
throw "Docker is not available. Start Docker Desktop and retry."
}
if ($dockerInfo -ne "linux") {
throw "Docker server OS is '$dockerInfo'. Switch Docker Desktop to Linux containers."
}
$entrypoint = Join-Path $RepoRoot "build\manylinux_entrypoint.sh"
# Normalize CRLF -> LF so bash in the container can execute the script.
$raw = [System.IO.File]::ReadAllText($entrypoint)
$normalized = $raw -replace "`r`n", "`n" -replace "`r", "`n"
[System.IO.File]::WriteAllText($entrypoint, $normalized)
if (-not $SkipImageBuild) {
Write-Host "`n=== Building Docker image ==="
docker build -f build/Dockerfile.manylinux -t $ImageName .
if ($LASTEXITCODE -ne 0) {
throw "docker build failed"
}
}
Write-Host "`n=== Running manylinux container ==="
# Mount the repo so artifacts land under dist/repaired and SDK/<version>/ on the host.
docker run --rm `
-e "ORISYS_BUILD_TARGET=$Target" `
-e "ORISYS_PYTHON_TAGS=$pythonTags" `
-v "${RepoRoot}:/src" `
$ImageName
if ($LASTEXITCODE -ne 0) {
throw "manylinux container build failed"
}
Write-Host "`n=== Host artifacts ==="
$repaired = Join-Path $RepoRoot "dist\repaired"
if (Test-Path $repaired) {
Get-ChildItem $repaired -Filter *.whl | ForEach-Object { Write-Host $_.FullName }
} else {
Write-Warning "No dist/repaired directory found."
}
Write-Host "`nNext steps: see docs/publishing.md (TestPyPI then PyPI)."
build/build_nuitka.py
+123 -22
View File
@@ -1,36 +1,80 @@
"""Build a Nuitka-compiled orisys wheel and publish it to SDK/{version}/.
Pipeline:
1. Compile src/orisys → .pyd
1. Compile src/orisys → extension module (.pyd / .so)
2. Stage wheel tree (compiled module + bundled files from pyproject.toml)
3. Run bdist_wheel and move output to SDK/{version}/
4. Smoke-test imports in a temporary venv
3. Run bdist_wheel; on Linux, auditwheel-repair to a manylinux tag
4. Smoke-test imports in a temporary venv (required for --target pypi-linux)
Run from repo root: python build/build_nuitka.py
Targets:
internal – local/SDK delivery (default). Linux still repairs with auditwheel.
pypi-linux – PyPI binary wheel; requires Linux + CPython 3.10/3.11/3.12 + compliant tag.
Run from repo root:
python build/build_nuitka.py
python build/build_nuitka.py --target pypi-linux
"""
from __future__ import annotations
import argparse
import os
import shutil
import subprocess
import sys
from pathlib import Path
from nuitka_compile import compile_orisys, copy_compiled_orisys, install_nuitka
from paths import NUITKA_COMPILE_DIR, NUITKA_WHEEL_DIR, REPO_ROOT, repo_path
from paths import NUITKA_COMPILE_DIR, NUITKA_WHEEL_DIR, REPO_ROOT, repaired_wheel_dir, repo_path
from pyproject_utils import bundle_items, extras_require, get_version, install_requires
from verify import verify_built_wheel
from wheel_build import publish_wheel_to_sdk, run_bdist_wheel, write_wheel_setup
from wheel_build import (
is_compliant_manylinux_wheel,
publish_wheel_to_sdk,
repair_linux_wheel,
run_bdist_wheel,
write_wheel_setup,
)
from wheel_layout import copy_bundle_items, copy_orisys_blend, package_data_from_bundle
VALID_TARGETS = ("internal", "pypi-linux")
# ABI tags that manylinux images ship under /opt/python/cpXYY-cpXYY.
PYPI_LINUX_PYTHON_VERSIONS = ((3, 10), (3, 11), (3, 12))
def _reset_dir(path) -> None:
def _reset_dir(path: Path) -> None:
"""Remove and recreate a build output directory."""
if path.exists():
shutil.rmtree(path)
path.mkdir(parents=True)
def build() -> None:
def _validate_pypi_linux_environment() -> None:
if sys.platform != "linux":
raise SystemExit(
"[ERROR] --target pypi-linux must run inside a Linux manylinux container "
"(for example quay.io/pypa/manylinux_2_28_x86_64 via Docker Desktop)."
)
if sys.version_info[:2] not in PYPI_LINUX_PYTHON_VERSIONS:
supported = ", ".join(f"{major}.{minor}" for major, minor in PYPI_LINUX_PYTHON_VERSIONS)
raise SystemExit(
f"[ERROR] --target pypi-linux requires CPython {supported}, got "
f"{sys.version_info.major}.{sys.version_info.minor}. "
"Use /opt/python/cp310-cp310 (or cp311/cp312) in the manylinux_2_28 image."
)
def build(*, target: str = "internal", verify: bool | None = None) -> Path:
if target not in VALID_TARGETS:
raise SystemExit(f"[ERROR] Unknown target {target!r}; choose from {VALID_TARGETS}")
if target == "pypi-linux":
_validate_pypi_linux_environment()
# pypi-linux always verifies; internal opts in with --verify.
if verify is None:
verify = target == "pypi-linux"
# --- 1. Compile ---
install_nuitka()
@@ -38,7 +82,7 @@ def build() -> None:
_reset_dir(NUITKA_COMPILE_DIR)
_reset_dir(NUITKA_WHEEL_DIR)
print("\n=== Compiling with Nuitka ===")
print(f"\n=== Compiling with Nuitka (target={target}, version={version}) ===")
env = os.environ.copy()
src_dir = repo_path("src")
env["PYTHONPATH"] = src_dir.as_posix() + os.pathsep + env.get("PYTHONPATH", "")
@@ -71,23 +115,80 @@ def build() -> None:
extras_require=extras_require(REPO_ROOT),
)
# --- 3. Build wheel and publish to SDK/ ---
# --- 3. Build wheel, repair on Linux, publish to SDK/ ---
wheel_path = run_bdist_wheel(NUITKA_WHEEL_DIR)
if wheel_path is None:
print("\n[ERROR] Wheel not found.")
return
raise SystemExit("\n[ERROR] Wheel not found after bdist_wheel.")
final_wheel = publish_wheel_to_sdk(wheel_path, version)
print(f"\n[SUCCESS] Wheel created and moved to: {final_wheel}")
repair_dir = repaired_wheel_dir()
final_candidate = repair_linux_wheel(wheel_path, repair_dir)
# # --- 4. Verify (isolated venv; does not touch the active environment) ---
# print("\n=== Verifying finger mesh/viewer imports ===")
# try:
# verify_built_wheel(final_wheel)
# except subprocess.CalledProcessError as exc:
# print(f"\n[WARNING] Wheel verify failed: {exc}")
# print("The wheel was built successfully; verification runs in an isolated temp venv.")
if target == "pypi-linux" and not is_compliant_manylinux_wheel(final_candidate):
raise SystemExit(
f"[ERROR] Wheel is not PyPI-compliant: {final_candidate.name}. "
"Expected a manylinux_2_17_x86_64 (or newer) platform tag."
)
# Keep a durable copy under dist/repaired for upload tooling.
dist_repaired = REPO_ROOT / "dist" / "repaired"
dist_repaired.mkdir(parents=True, exist_ok=True)
dist_copy = dist_repaired / final_candidate.name
if dist_copy.resolve() != final_candidate.resolve():
shutil.copy2(final_candidate, dist_copy)
final_wheel = publish_wheel_to_sdk(final_candidate, version)
print(f"\n[SUCCESS] Wheel created and copied to: {final_wheel}")
print(f"[SUCCESS] Upload candidate also at: {dist_copy}")
# --- 4. Verify (isolated venv; does not touch the active environment) ---
if verify:
print("\n=== Verifying wheel imports in isolated venv ===")
try:
verify_built_wheel(final_wheel)
print("[SUCCESS] Wheel smoke test passed.")
except subprocess.CalledProcessError as exc:
if target == "pypi-linux":
raise SystemExit(
f"[ERROR] Wheel verify failed (required for pypi-linux): {exc}"
) from exc
print(f"\n[WARNING] Wheel verify failed: {exc}")
print("The wheel was built successfully; verification runs in an isolated temp venv.")
else:
print("\n[INFO] Skipping wheel verify (pass --verify to enable).")
return final_wheel
def main(argv: list[str] | None = None) -> None:
parser = argparse.ArgumentParser(description="Build a Nuitka-compiled orisys wheel.")
parser.add_argument(
"--target",
choices=VALID_TARGETS,
default="internal",
help="internal: SDK delivery; pypi-linux: manylinux PyPI binary wheel",
)
parser.add_argument(
"--verify",
action="store_true",
help="Run isolated venv smoke test (always on for --target pypi-linux)",
)
parser.add_argument(
"--skip-verify",
action="store_true",
help="Skip smoke test when using --target internal",
)
args = parser.parse_args(argv)
verify: bool | None
if args.target == "pypi-linux":
verify = True
elif args.verify:
verify = True
elif args.skip_verify:
verify = False
else:
verify = None
build(target=args.target, verify=verify)
if __name__ == "__main__":
build()
main()
build/manylinux_entrypoint.sh
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env bash
# Runs inside quay.io/pypa/manylinux_2_28_x86_64 with the repo mounted at /src.
# Builds one Nuitka wheel per CPython ABI listed in ORISYS_PYTHON_TAGS.
set -euo pipefail
TARGET="${ORISYS_BUILD_TARGET:-pypi-linux}"
# Comma-separated manylinux tags, e.g. "cp310-cp310,cp311-cp311,cp312-cp312"
PYTHON_TAGS="${ORISYS_PYTHON_TAGS:-cp310-cp310,cp311-cp311,cp312-cp312}"
cd /src
echo "=== manylinux entrypoint ==="
echo "Target: ${TARGET}"
echo "Python tags: ${PYTHON_TAGS}"
uname -a
IFS=',' read -r -a TAG_ARRAY <<< "${PYTHON_TAGS}"
FAILED=0
BUILT=()
for tag in "${TAG_ARRAY[@]}"; do
tag="$(echo "${tag}" | tr -d '[:space:]')"
if [[ -z "${tag}" ]]; then
continue
fi
PYBIN="/opt/python/${tag}/bin"
if [[ ! -x "${PYBIN}/python" ]]; then
echo "[ERROR] Python not found for tag ${tag}: ${PYBIN}/python" >&2
FAILED=1
continue
fi
echo ""
echo "========================================"
echo "Building orisys wheel: ${tag}"
echo "Python: $("${PYBIN}/python" -V)"
echo "========================================"
export PATH="${PYBIN}:${PATH}"
export PYTHONPATH="/src/src${PYTHONPATH:+:${PYTHONPATH}}"
"${PYBIN}/python" -m pip install -q --upgrade pip setuptools wheel auditwheel
if "${PYBIN}/python" build/build_nuitka.py --target "${TARGET}"; then
BUILT+=("${tag}")
else
echo "[ERROR] Build failed for ${tag}" >&2
FAILED=1
fi
done
echo ""
echo "=== Build artifacts ==="
ls -la /src/dist/repaired/ || true
ls -la /src/SDK/*/ || true
if [[ ${#BUILT[@]} -gt 0 ]]; then
echo "Built tags: ${BUILT[*]}"
fi
if [[ "${FAILED}" -ne 0 ]]; then
echo "[ERROR] One or more Python ABI builds failed." >&2
exit 1
fi
echo "[SUCCESS] All requested manylinux wheels built."
build/nuitka_compile.py
+7
View File
@@ -19,11 +19,18 @@ NUITKA_INCLUDE_PACKAGES = (
)
NUITKA_NOFOLLOW_IMPORTS = (
# Keep as runtime deps (shipped via wheel install_requires / extras).
"numpy",
"scipy",
"cv2",
"numba",
"orisys_blend",
"cupy",
"cupyx",
"pyopencl",
"open3d",
"PyQt5",
"pyqtgraph",
)
build/paths.py
+5
View File
@@ -18,3 +18,8 @@ def repo_path(*parts: str) -> Path:
def sdk_dir(version: str) -> Path:
return REPO_ROOT / "SDK" / version
def repaired_wheel_dir() -> Path:
"""Directory for auditwheel-repaired wheels under the Nuitka wheel build tree."""
return NUITKA_WHEEL_DIR / "repaired"
build/pyproject_utils.py
+21 -9
View File
@@ -2,7 +2,7 @@
from __future__ import annotations
import importlib
import ast
import subprocess
import sys
from pathlib import Path
@@ -41,17 +41,29 @@ def extras_require(repo_root: Path) -> dict[str, list[str]]:
def _resolve_version_attr(repo_root: Path, attr: str) -> str:
"""Resolve a setuptools dynamic version attr (e.g. orisys.__version__)."""
src_dir = str(repo_root / "src")
if src_dir not in sys.path:
sys.path.insert(0, src_dir)
module_name, _, attr_name = attr.partition(".")
if not attr_name:
raise RuntimeError(f"Invalid dynamic version attr: {attr!r}")
module = importlib.import_module(module_name)
value = module
for name in attr_name.split("."):
value = getattr(value, name)
return str(value)
# Avoid importing package modules here: __init__ may import heavy runtime deps
# (e.g., cv2/numpy) that are not required just to read __version__.
rel_module = Path(*module_name.split("."))
module_file = repo_root / "src" / rel_module.with_suffix(".py")
if not module_file.is_file():
module_file = repo_root / "src" / rel_module / "__init__.py"
if not module_file.is_file():
raise RuntimeError(f"Cannot resolve module file for dynamic version attr: {attr!r}")
tree = ast.parse(module_file.read_text(encoding="utf-8"), filename=str(module_file))
target_name = attr_name.split(".")[0]
for node in tree.body:
if isinstance(node, ast.Assign):
for target in node.targets:
if isinstance(target, ast.Name) and target.id == target_name:
value = ast.literal_eval(node.value)
return str(value)
raise RuntimeError(f"Attribute {target_name!r} not found in {module_file}")
def get_version(repo_root: Path) -> str:
build/verify.py
+30 -12
View File
@@ -11,6 +11,32 @@ from pathlib import Path
from paths import GUI_DIST_DIR
# Public SDK surface that must import from a released wheel.
_SMOKE_IMPORT_SCRIPT = """
import orisys
from orisys import Sensor, list_config, export_config, util, finger
from orisys.finger import UVMeshViewer, build_uv_surface_mesh, FingerRuntime
from orisys.finger.mesh import UVSurfaceMesh
from orisys.finger.viewer import init_uv_mesh_viewer
from orisys.hhd import nHHD
from orisys.tracker import Tracker
assert orisys.__version__
assert Sensor is not None
assert callable(list_config)
assert callable(export_config)
assert util is not None
assert finger is not None
assert UVMeshViewer is not None
assert build_uv_surface_mesh is not None
assert FingerRuntime is not None
assert UVSurfaceMesh is not None
assert init_uv_mesh_viewer is not None
assert nHHD is not None
assert Tracker is not None
print(f"orisys {orisys.__version__} public API imports OK")
"""
def _venv_python(venv_dir: Path) -> Path:
if sys.platform == "win32":
@@ -19,7 +45,7 @@ def _venv_python(venv_dir: Path) -> Path:
def verify_built_wheel(wheel_path: Path) -> None:
"""Install the wheel in a temporary venv and smoke-test finger imports."""
"""Install the wheel in a temporary venv and smoke-test public imports."""
wheel_path = wheel_path.resolve()
with tempfile.TemporaryDirectory(prefix="orisys-wheel-verify-") as tmp:
venv_dir = Path(tmp) / "venv"
@@ -27,20 +53,12 @@ def verify_built_wheel(wheel_path: Path) -> None:
python = _venv_python(venv_dir)
subprocess.check_call(
[python, "-m", "pip", "install", "-q", str(wheel_path)],
[python, "-m", "pip", "install", "-q", "--upgrade", "pip"],
)
subprocess.check_call(
[
python,
"-c",
(
"from orisys.finger import UVMeshViewer, build_uv_surface_mesh, FingerRuntime; "
"from orisys.finger.mesh import UVSurfaceMesh; "
"from orisys.finger.viewer import init_uv_mesh_viewer; "
"print('finger mesh/viewer imports OK')"
),
],
[python, "-m", "pip", "install", "-q", str(wheel_path)],
)
subprocess.check_call([python, "-c", _SMOKE_IMPORT_SCRIPT])
def verify_gui_dist(dist_dir: Path | None = None) -> None:
build/wheel_build.py
+74 -1
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import re
import shutil
import subprocess
import sys
@@ -9,6 +10,11 @@ from pathlib import Path
from paths import sdk_dir
# PyPI rejects bare tags like "manylinux_x86_64"; require a glibc ABI version.
_MANYLINUX_TAG_RE = re.compile(
r"(?:manylinux2014|manylinux_2_(?:17|24|27|28|31|34|35))_x86_64"
)
def write_wheel_setup(
wheel_dir: Path,
@@ -52,11 +58,78 @@ def run_bdist_wheel(wheel_dir: Path) -> Path | None:
return wheels[0] if wheels else None
def _ensure_auditwheel() -> None:
try:
subprocess.check_call(
[sys.executable, "-m", "auditwheel", "--version"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
except (subprocess.CalledProcessError, FileNotFoundError):
print("Installing auditwheel...")
subprocess.check_call([sys.executable, "-m", "pip", "install", "auditwheel", "-q"])
def is_compliant_manylinux_wheel(wheel_path: Path) -> bool:
"""True if the wheel filename uses a PyPI-accepted manylinux x86_64 tag."""
return bool(_MANYLINUX_TAG_RE.search(wheel_path.name))
def repair_linux_wheel(wheel_path: Path, repair_dir: Path) -> Path:
"""Run auditwheel show/repair and return a PyPI-compliant manylinux wheel.
Raises RuntimeError if repair fails or the result still lacks a valid tag.
On non-Linux platforms the input wheel is returned unchanged.
"""
if sys.platform != "linux":
print("Skipping auditwheel repair (not running on Linux).")
return wheel_path
_ensure_auditwheel()
repair_dir = repair_dir.resolve()
if repair_dir.exists():
shutil.rmtree(repair_dir)
repair_dir.mkdir(parents=True)
print("\n=== auditwheel show ===")
subprocess.check_call([sys.executable, "-m", "auditwheel", "show", str(wheel_path)])
print("\n=== auditwheel repair ===")
subprocess.check_call(
[
sys.executable,
"-m",
"auditwheel",
"repair",
str(wheel_path),
"--wheel-dir",
str(repair_dir),
]
)
repaired = sorted(repair_dir.glob("*.whl"))
if not repaired:
raise RuntimeError(f"auditwheel repair produced no wheels in {repair_dir}")
compliant = [path for path in repaired if is_compliant_manylinux_wheel(path)]
if not compliant:
names = ", ".join(path.name for path in repaired)
raise RuntimeError(
"auditwheel repair did not produce a PyPI-compliant manylinux tag. "
f"Got: {names}. Rebuild inside quay.io/pypa/manylinux_2_28_x86_64 "
"(or manylinux2014_x86_64) instead of a generic Linux host."
)
final = compliant[0]
print(f"Repaired wheel: {final.name}")
return final
def publish_wheel_to_sdk(wheel_path: Path, version: str) -> Path:
sdk_version_dir = sdk_dir(version)
sdk_version_dir.mkdir(parents=True, exist_ok=True)
dest_wheel = sdk_version_dir / wheel_path.name
if dest_wheel.exists():
dest_wheel.unlink()
shutil.move(str(wheel_path), str(dest_wheel))
shutil.copy2(str(wheel_path), str(dest_wheel))
return dest_wheel