diff --git a/build/README.md b/build/README.md index 4521fab..3ff1140 100644 --- a/build/README.md +++ b/build/README.md @@ -27,6 +27,7 @@ build/ ```bash python build/run.py wheel-win python build/run.py wheel-manylinux +python build/run.py wheel-manylinux -Architecture aarch64 -PythonVersions 3.10 # Single Version python build/run.py wheel @@ -42,6 +43,29 @@ build\wheel\win.bat .\build\wheel\docker\run.ps1 -SkipImageBuild # reuse the image when only sources changed ``` +`wheel-manylinux` defaults to `x86_64`; `-Architecture aarch64` builds ARM64 Linux +wheels in `quay.io/pypa/manylinux_2_28_aarch64`. Docker Desktop must use Linux +containers and support ARM64 emulation on an Intel/AMD host. This runs the ARM64 +compiler under QEMU, so compilation can be much slower than on native ARM64. +The default Python versions remain 3.10, 3.11 and 3.12; `-PythonVersions` selects +which ABIs to compile. The image still prepares tooling for all three ABIs. + +The default image names are `orisys-manylinux` and `orisys-manylinux-aarch64`. +`-ImageName` overrides the name; a reused image must match the requested +architecture. For example, reuse the ARM64 image after source-only changes: + +```powershell +.\build\wheel\docker\run.ps1 -Architecture aarch64 -PythonVersions 3.10 -SkipImageBuild +``` + +ARM64 wheels target glibc Linux, including Q6A and Jetson systems with compatible +Python ABIs and glibc versions (2.28 or newer for this build image). They do not +target Android. The build requires binary ARM64 wheels for the runtime +dependencies and performs the existing isolated import check. Validate on each +board before delivery; CUDA, JetPack and Qualcomm GPU/NPU runtimes need separate +compatibility checks. Builds share staging directories, so run architectures +and Python ABIs sequentially. + The image bakes in per-ABI build tooling and a per-ABI wheelhouse of the runtime deps, so the entrypoint no longer re-resolves them over the network. See `docs/publishing.md` §6 for build-time details, why `-SkipImageBuild` matters, diff --git a/build/common/verify/wheel_contents.py b/build/common/verify/wheel_contents.py index 1857ea4..af5a6dc 100644 --- a/build/common/verify/wheel_contents.py +++ b/build/common/verify/wheel_contents.py @@ -427,7 +427,7 @@ def _check_platform_tag(result: WheelCheckResult) -> None: result.add( "platform-tag", ok, - f"{tag!r}" if ok else f"{tag!r} is not a PyPI-accepted manylinux x86_64 tag", + f"{tag!r}" if ok else f"{tag!r} is not a supported manylinux x86_64 or aarch64 tag", ) diff --git a/build/run.py b/build/run.py index 5296e12..68234a7 100644 --- a/build/run.py +++ b/build/run.py @@ -86,11 +86,11 @@ def main(argv: list[str] | None = None) -> int: argv = list(sys.argv[1:] if argv is None else argv) if not argv or argv[0] in ("-h", "--help"): print(__doc__) - print("Commands: wheel | wheel-win | manylinux | gui | verify | release") + print("Commands: wheel | wheel-win | wheel-manylinux | gui | verify | release") print("Examples:") print(" python build/run.py wheel --help") print(" python build/run.py gui finger") - print(" python build/run.py manylinux -PythonVersions 3.10") + print(" python build/run.py wheel-manylinux -Architecture aarch64 -PythonVersions 3.10") print(" python build/run.py release preflight --version ") print(" python build/run.py release audit SDK\\\\orisys--cp312-cp312-win_amd64.whl") return 0 @@ -107,7 +107,7 @@ def main(argv: list[str] | None = None) -> int: if command not in dispatch: print(f"Unknown command: {command!r}", file=sys.stderr) print( - "Choose from: wheel, wheel-win, manylinux, gui, verify, release", + "Choose from: wheel, wheel-win, wheel-manylinux, gui, verify, release", file=sys.stderr, ) return 2 diff --git a/build/wheel/build.py b/build/wheel/build.py index b5dfabc..0341122 100644 --- a/build/wheel/build.py +++ b/build/wheel/build.py @@ -76,7 +76,7 @@ def _validate_pypi_linux_environment() -> None: if sys.platform != "linux": raise SystemExit( "[ERROR] --target pypi-linux must run inside a Linux manylinux container " - "(for example quay.io/pypa/manylinux_2_28_x86_64 via Docker Desktop)." + "(manylinux_2_28_x86_64 or manylinux_2_28_aarch64 via Docker Desktop)." ) if sys.version_info[:2] not in PYPI_LINUX_PYTHON_VERSIONS: supported = ", ".join(f"{major}.{minor}" for major, minor in PYPI_LINUX_PYTHON_VERSIONS) @@ -153,7 +153,7 @@ def build(*, target: str = "internal", verify: bool | None = None) -> Path: if target == "pypi-linux" and not is_compliant_manylinux_wheel(final_candidate): raise SystemExit( f"[ERROR] Wheel is not PyPI-compliant: {final_candidate.name}. " - "Expected a manylinux_2_17_x86_64 (or newer) platform tag." + "Expected a supported manylinux x86_64 or aarch64 platform tag." ) # Keep a durable copy under dist/repaired for upload tooling. diff --git a/build/wheel/docker/Dockerfile.manylinux b/build/wheel/docker/Dockerfile.manylinux index 5107e1f..92606a0 100644 --- a/build/wheel/docker/Dockerfile.manylinux +++ b/build/wheel/docker/Dockerfile.manylinux @@ -1,7 +1,8 @@ # manylinux_2_28: modern glibc baseline with binary wheels for numpy/scipy/opencv. # Prefer this over manylinux2014 — recent NumPy requires GCC >= 10.3 and will try # (and fail) to build from source on the older image. -FROM quay.io/pypa/manylinux_2_28_x86_64:latest +ARG MANYLINUX_ARCH=x86_64 +FROM quay.io/pypa/manylinux_2_28_${MANYLINUX_ARCH}:latest ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ PIP_ROOT_USER_ACTION=ignore \ diff --git a/build/wheel/docker/entrypoint.sh b/build/wheel/docker/entrypoint.sh index f4b92ba..ad3ad93 100644 --- a/build/wheel/docker/entrypoint.sh +++ b/build/wheel/docker/entrypoint.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Runs inside quay.io/pypa/manylinux_2_28_x86_64 with the repo mounted at /src. +# Runs inside the selected manylinux_2_28 image with the repo mounted at /src. # Builds one Nuitka wheel per CPython ABI listed in ORISYS_PYTHON_TAGS. set -euo pipefail diff --git a/build/wheel/docker/run.ps1 b/build/wheel/docker/run.ps1 index 219674e..880f87f 100644 --- a/build/wheel/docker/run.ps1 +++ b/build/wheel/docker/run.ps1 @@ -1,21 +1,24 @@ -# Build PyPI-compliant manylinux wheels for CPython 3.10/3.11/3.12 using Docker Desktop. +# Build x86_64 or aarch64 manylinux wheels using Docker Desktop. # # Usage (from repo root, PowerShell): # .\build\wheel\docker\run.ps1 -# .\build\build_manylinux_docker.ps1 # compatibility wrapper +# .\build\wheel\docker\run.ps1 -Architecture aarch64 -PythonVersions 3.10 # # From cmd.exe: -# powershell -ExecutionPolicy Bypass -File .\build\build_manylinux_docker.ps1 +# powershell -ExecutionPolicy Bypass -File .\build\wheel\docker\run.ps1 param( [ValidateSet("pypi-linux", "internal")] [string]$Target = "pypi-linux", [string]$PythonVersions = "3.10,3.11,3.12", + [ValidateSet("x86_64", "aarch64")] + [string]$Architecture = "x86_64", [switch]$SkipImageBuild, - [string]$ImageName = "orisys-manylinux" + [string]$ImageName = "" ) $ErrorActionPreference = "Stop" +$Architecture = $Architecture.ToLowerInvariant() function Convert-ToManylinuxTag { param([string]$Version) @@ -45,11 +48,17 @@ if ($tags.Count -eq 0) { throw "No Python versions specified." } $pythonTags = [string]::Join(",", $tags) +$dockerArchitecture = if ($Architecture -eq "aarch64") { "arm64" } else { "amd64" } +$dockerPlatform = "linux/$dockerArchitecture" +if ([string]::IsNullOrWhiteSpace($ImageName)) { + $ImageName = if ($Architecture -eq "aarch64") { "orisys-manylinux-aarch64" } else { "orisys-manylinux" } +} Write-Host "=== Orisys manylinux Docker build ===" Write-Host "Repo: $RepoRoot" Write-Host "Target: $Target" Write-Host "Image: $ImageName" +Write-Host "Architecture: $Architecture ($dockerPlatform)" Write-Host "Python tags: $pythonTags" # Docker Desktop must be in Linux container mode for manylinux. @@ -71,15 +80,25 @@ $dockerfile = "build/wheel/docker/Dockerfile.manylinux" if (-not $SkipImageBuild) { Write-Host "`n=== Building Docker image ===" - docker build -f $dockerfile -t $ImageName . + docker build --platform $dockerPlatform --build-arg "MANYLINUX_ARCH=$Architecture" -f $dockerfile -t $ImageName . if ($LASTEXITCODE -ne 0) { throw "docker build failed" } } +# A custom or reused image must contain the requested architecture. +$imageArchitecture = docker image inspect --format "{{.Architecture}}" $ImageName +if ($LASTEXITCODE -ne 0) { + throw "Cannot inspect image '$ImageName'. Rebuild without -SkipImageBuild." +} +if ($imageArchitecture -ne $dockerArchitecture) { + throw "Image '$ImageName' architecture is '$imageArchitecture'; expected '$dockerArchitecture'. Rebuild without -SkipImageBuild." +} + Write-Host "`n=== Running manylinux container ===" # Mount the repo so artifacts land under dist/repaired and SDK// on the host. docker run --rm ` + --platform $dockerPlatform ` -e "ORISYS_BUILD_TARGET=$Target" ` -e "ORISYS_PYTHON_TAGS=$pythonTags" ` -v "${RepoRoot}:/src" ` diff --git a/build/wheel/package.py b/build/wheel/package.py index 416d46c..1e0062b 100644 --- a/build/wheel/package.py +++ b/build/wheel/package.py @@ -16,7 +16,7 @@ from common.paths import sdk_dir # PyPI rejects bare tags like "manylinux_x86_64"; require a glibc ABI version. _MANYLINUX_TAG_RE = re.compile( - r"(?:manylinux2014|manylinux_2_(?:17|24|27|28|31|34|35))_x86_64" + r"(?:manylinux2014|manylinux_2_(?:17|24|27|28|31|34|35))_(?:x86_64|aarch64)(?=\.|$)" ) # Source of the PyPI project description (see read_long_description). @@ -155,7 +155,7 @@ def _ensure_auditwheel() -> None: def is_compliant_manylinux_wheel(wheel_path: Path) -> bool: - """True if the wheel filename uses a PyPI-accepted manylinux x86_64 tag.""" + """True if the wheel filename uses a supported manylinux x86_64 or aarch64 tag.""" return bool(_MANYLINUX_TAG_RE.search(wheel_path.name)) @@ -200,8 +200,8 @@ def repair_linux_wheel(wheel_path: Path, repair_dir: Path) -> Path: names = ", ".join(path.name for path in repaired) raise RuntimeError( "auditwheel repair did not produce a PyPI-compliant manylinux tag. " - f"Got: {names}. Rebuild inside quay.io/pypa/manylinux_2_28_x86_64 " - "(or manylinux2014_x86_64) instead of a generic Linux host." + f"Got: {names}. Rebuild inside the manylinux_2_28 image for the " + "target architecture instead of a generic Linux host." ) final = compliant[0] diff --git a/tests/test_manylinux_docker.py b/tests/test_manylinux_docker.py new file mode 100644 index 0000000..17599f7 --- /dev/null +++ b/tests/test_manylinux_docker.py @@ -0,0 +1,121 @@ +"""Exercise the Windows launcher without downloading or compiling Docker images.""" + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +POWERSHELL = shutil.which("powershell") +REPO_ROOT = Path(__file__).resolve().parents[1] +pytestmark = pytest.mark.skipif(not POWERSHELL, reason="Windows PowerShell is required") + + +def run_launcher(tmp_path, arguments, image_architecture): + docker_dir = tmp_path / "build" / "wheel" / "docker" + docker_dir.mkdir(parents=True) + shutil.copy2(REPO_ROOT / "build/wheel/docker/run.ps1", docker_dir) + (docker_dir / "entrypoint.sh").write_text("#!/bin/bash\n", encoding="utf-8") + parameters = {} + iterator = iter(arguments) + for argument in iterator: + parameters[argument.lstrip("-")] = ( + True if argument == "-SkipImageBuild" else next(iterator) + ) + (tmp_path / "arguments.json").write_text(json.dumps(parameters), encoding="utf-8") + harness = tmp_path / "harness.ps1" + harness.write_text( + """ +$ErrorActionPreference = 'Stop' +$global:Calls = [System.Collections.Generic.List[object]]::new() +function docker { + $global:Calls.Add(@($args)) + $global:LASTEXITCODE = 0 + if ($args[0] -eq 'version') { return 'linux' } + if ($args[0] -eq 'image') { return '__ARCH__' } +} +$decoded = Get-Content (Join-Path $PSScriptRoot 'arguments.json') -Raw | ConvertFrom-Json +$parameters = @{} +foreach ($property in $decoded.PSObject.Properties) { $parameters[$property.Name] = $property.Value } +try { + & (Join-Path $PSScriptRoot 'build/wheel/docker/run.ps1') @parameters +} finally { + ConvertTo-Json -InputObject @($global:Calls.ToArray()) -Depth 5 | + Set-Content (Join-Path $PSScriptRoot 'calls.json') -Encoding UTF8 +} +""".replace("__ARCH__", image_architecture), + encoding="utf-8", + ) + result = subprocess.run( + [POWERSHELL, "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", str(harness)], + capture_output=True, + text=True, + check=False, + ) + calls = json.loads((tmp_path / "calls.json").read_text(encoding="utf-8-sig")) + return result, calls + + +@pytest.mark.parametrize( + "arguments,image_architecture,platform,architecture,image_name", + [ + ([], "amd64", "linux/amd64", "x86_64", "orisys-manylinux"), + ( + ["-Architecture", "aarch64"], + "arm64", + "linux/arm64", + "aarch64", + "orisys-manylinux-aarch64", + ), + ( + ["-Architecture", "AARCH64"], + "arm64", + "linux/arm64", + "aarch64", + "orisys-manylinux-aarch64", + ), + ], +) +def test_architecture_controls_build_and_run( + tmp_path, arguments, image_architecture, platform, architecture, image_name +): + result, calls = run_launcher(tmp_path, arguments, image_architecture) + assert result.returncode == 0, result.stderr + build = next(call for call in calls if call[0] == "build") + run = next(call for call in calls if call[0] == "run") + assert build[build.index("--platform") + 1] == platform + assert build[build.index("--build-arg") + 1] == f"MANYLINUX_ARCH={architecture}" + assert build[build.index("-t") + 1] == image_name + assert run[run.index("--platform") + 1] == platform + assert run[-1] == image_name + + +@pytest.mark.parametrize( + "image_architecture,success", [("arm64", True), ("amd64", False)] +) +def test_reused_image_must_match_requested_architecture( + tmp_path, image_architecture, success +): + result, calls = run_launcher( + tmp_path, + [ + "-Architecture", + "aarch64", + "-SkipImageBuild", + "-ImageName", + "custom-arm", + "-PythonVersions", + "3.10", + ], + image_architecture, + ) + assert not any(call[0] == "build" for call in calls) + assert (result.returncode == 0) == success + runs = [call for call in calls if call[0] == "run"] + if success: + assert runs[0][-1] == "custom-arm" + assert "ORISYS_PYTHON_TAGS=cp310-cp310" in runs[0] + else: + assert not runs + assert "architecture" in result.stderr.lower() diff --git a/tests/test_wheel_build.py b/tests/test_wheel_build.py index 60cc150..c627b94 100644 --- a/tests/test_wheel_build.py +++ b/tests/test_wheel_build.py @@ -29,6 +29,13 @@ def test_compliant_manylinux_tags(wheel_package): ) +@pytest.mark.parametrize("tag", ["manylinux2014_aarch64", "manylinux_2_28_aarch64"]) +def test_compliant_arm_manylinux_tags(wheel_package, tag): + assert wheel_package.is_compliant_manylinux_wheel( + Path(f"orisys-1.2.3-cp310-cp310-{tag}.whl") + ) + + def test_rejects_illegal_manylinux_tags(wheel_package): assert not wheel_package.is_compliant_manylinux_wheel( Path("orisys-1.2.3-cp310-cp310-manylinux_x86_64.whl") diff --git a/tests/test_wheel_contents.py b/tests/test_wheel_contents.py index 5181c79..ef861ce 100644 --- a/tests/test_wheel_contents.py +++ b/tests/test_wheel_contents.py @@ -350,10 +350,11 @@ def test_bad_manylinux_tag_fails(tmp_path): assert "platform-tag" in _failed_names(inspect_wheel(wheel, repo_root=repo)) -def test_good_manylinux_tag_passes(tmp_path): +@pytest.mark.parametrize("architecture", ["x86_64", "aarch64"]) +def test_good_manylinux_tag_passes(tmp_path, architecture): wheel, repo = _make_wheel( tmp_path, - filename=f"orisys-{VERSION}-cp312-cp312-manylinux_2_28_x86_64.whl", + filename=f"orisys-{VERSION}-cp312-cp312-manylinux_2_28_{architecture}.whl", ) assert inspect_wheel(wheel, repo_root=repo).ok