From 9c951e9d3c922cd5461438abea820b9cbc5a7e1f Mon Sep 17 00:00:00 2001 From: ykcheungab Date: Thu, 15 Oct 2026 21:39:31 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=B0=86=20ARM64=20wheel=20=E7=BA=B3?= =?UTF-8?q?=E5=85=A5=E5=8F=91=E5=B8=83=E6=A0=A1=E9=AA=8C=E5=92=8C=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补齐全部架构的运行证据、GitHub 发布说明和 PyPI 逐文件下载核验。已通过 161 项测试、代码和文档门禁、九个 wheel 的本地索引下载核验，以及 ARM 三个 ABI 的 40 帧 CPU 检查。 --- build/README.md | 11 ++- build/release/cli.py | 2 +- build/release/github.py | 11 ++- build/release/preflight.py | 44 +++++----- build/release/publish.py | 11 +++ build/release/pypi.py | 52 +++++++----- build/release/runtime.py | 117 +++++++++++++++++++++++++++ docs/publishing.en.md | 35 +++++++- tests/test_release_publish.py | 112 ++++++++++++++++++++++++- tests/test_release_runtime.py | 148 ++++++++++++++++++++++++++++++++++ 10 files changed, 496 insertions(+), 47 deletions(-) create mode 100644 build/release/runtime.py create mode 100644 tests/test_release_runtime.py diff --git a/build/README.md b/build/README.md index 3ff1140..fe30d21 100644 --- a/build/README.md +++ b/build/README.md @@ -73,8 +73,9 @@ and what to change before attempting to parallelize the per-ABI builds. ### Release gate -Local only: never publishes, and makes no network call unless `--check-pypi` is -passed. See `docs/publishing.md` for the full flow. +Local only: never publishes. `--check-pypi` probes the index; native Windows +venvs may download dependencies. Linux verification uses local Docker images +and their offline wheelhouses. See `docs/publishing.en.md` for the full flow. ```bash python build/run.py release preflight --version @@ -91,6 +92,8 @@ is the expected input, so build first: ```bash python build/run.py wheel-win +python build/run.py wheel-manylinux +python build/run.py wheel-manylinux -Architecture aarch64 python build/run.py release preflight --all-wheel-pythons --check-pypi ``` @@ -112,6 +115,10 @@ python build/run.py release pypi --version --pypi-only --apply Both refuse to act unless `release preflight` passed on a clean commit with no waived options, and every wheel still hashes to what that report recorded. +Runtime evidence must cover every wheel hash. `--all-wheel-pythons` verifies all +staged platforms and ABIs, using conda for Windows and matching Docker images for +Linux. GitHub attaches all certified wheels; PyPI verifies every uploaded +Python/platform wheel by downloading it and comparing its hash. Tokens are only ever read from a hidden prompt or stdin, never from argv. See `docs/publishing.md` §3. diff --git a/build/release/cli.py b/build/release/cli.py index 867f9be..239ac7e 100644 --- a/build/release/cli.py +++ b/build/release/cli.py @@ -52,7 +52,7 @@ def _cmd_preflight(argv: list[str]) -> int: parser.add_argument( "--all-wheel-pythons", action="store_true", - help="verify cp310/cp311/cp312 wheels, each in its own conda env", + help="verify every staged wheel (Windows conda environments and Linux Docker images)", ) parser.add_argument("--offline", action="store_true", help="reuse parent-env deps instead of PyPI") parser.add_argument("--no-runtime", action="store_true", help="skip the isolated runtime check") diff --git a/build/release/github.py b/build/release/github.py index 518ee45..f91bea2 100644 --- a/build/release/github.py +++ b/build/release/github.py @@ -31,6 +31,7 @@ if str(_BUILD_ROOT) not in sys.path: from common.paths import REPO_ROOT # noqa: E402 from release.publish import GateResult, PublishGateError, load_gate # noqa: E402 +from release.runtime import PLATFORM_LABELS, wheel_target # noqa: E402 MANIFEST_PATH = REPO_ROOT / "tools" / "downstream_sync_manifest.json" SYNC_TOOL = REPO_ROOT / "tools" / "sync_downstreams.py" @@ -93,10 +94,11 @@ def _git(repo: Path, *args: str, check: bool = True) -> str: def build_release_notes(version: str, wheels: list[Path], git_commit: str | None) -> str: + platforms = sorted({PLATFORM_LABELS[wheel_target(wheel)[1]] for wheel in wheels}) lines = [ f"orisys {version}", "", - "Wheels (Python 3.10 / 3.11 / 3.12, Windows x64 and Linux x86_64):", + f"Wheels ({', '.join(platforms)}; Python ABI is shown in each filename):", ] lines.extend(f"- `{wheel.name}`" for wheel in wheels) lines.append("") @@ -335,7 +337,8 @@ def publish(options: Options) -> int: asset_dir = repo / "release" / version assets = [asset_dir / wheel.name for wheel in wheels] - notes_file = options.notes_file + notes_file = options.notes_file or gate.report_path.parent / "github-release-notes.md" + generated_notes = build_release_notes(version, wheels, gate.git_commit) if options.notes_file is None else None steps = build_github_steps( version=version, branch=branch, @@ -361,11 +364,15 @@ def publish(options: Options) -> int: print() if not options.apply: + if generated_notes: + print(generated_notes) print("dry run: nothing was changed. Re-run with --apply to execute.") return 0 # --- 4. execute ---------------------------------------------------- try: + if generated_notes: + notes_file.write_text(generated_notes, encoding="utf-8") commit_before = _git(repo, "rev-parse", "HEAD") _sync(run_checks=not options.skip_checks) _stage_wheels(wheels, asset_dir) diff --git a/build/release/preflight.py b/build/release/preflight.py index f0bc9b1..6182c68 100644 --- a/build/release/preflight.py +++ b/build/release/preflight.py @@ -1,8 +1,8 @@ """Local release gate: everything that must hold before publishing a wheel. ``release preflight`` answers one question with an exit code: is this checkout, -this version and these wheels safe to publish? It never publishes anything, and -by default it makes no network calls. +this version and these wheels safe to publish? It never publishes anything; +native verification environments may download dependencies from PyPI. Steps, in order: @@ -52,6 +52,7 @@ from wheel.layout import ( # noqa: E402 from release import report as report_mod # noqa: E402 from release import version as version_mod # noqa: E402 +from release.runtime import verify_manylinux_wheel, wheel_target # noqa: E402 @dataclass @@ -411,31 +412,29 @@ class Preflight: return Step("wheel-inspection", not problems, detail, data={"count": len(self.wheels)}) def _platform_token(self) -> str: - return "win_amd64" if sys.platform == "win32" else "manylinux" + return "win_amd64" if sys.platform == "win32" else platform.machine().lower() def _runtime_targets(self) -> list[tuple[str, Path, Path | None]]: token = self._platform_token() if self.options.all_wheel_pythons: targets: list[tuple[str, Path, Path | None]] = [] - for tag in ("cp310", "cp311", "cp312"): - match = next( - (w for w in self.wheels if f"-{tag}-" in w.name and token in w.name), None - ) - if match is None: - raise RuntimeError( - f"no {tag} {token} wheel among {[w.name for w in self.wheels]}" - ) - executable = version_mod.python_for_tag(tag) - if executable is None: + for wheel in self.wheels: + tag, target = wheel_target(wheel) + executable = None + if target == "win_amd64": + if sys.platform != "win32": + raise RuntimeError("Verify Windows wheels from a Windows host") + executable = version_mod.python_for_tag(tag) + if target == "win_amd64" and executable is None: raise RuntimeError( f"cannot resolve an interpreter for {tag}; create the conda env " f"py{tag[2:]} or set CONDA_ENVS_ROOT" ) - targets.append((tag, match, executable)) + targets.append((tag, wheel, executable)) return targets tag = f"cp{sys.version_info.major}{sys.version_info.minor}" - match = next((w for w in self.wheels if f"-{tag}-" in w.name and token in w.name), None) + match = next((w for w in self.wheels if wheel_target(w) == (tag, token)), None) if match is None: raise RuntimeError( f"no {tag} {token} wheel among {[w.name for w in self.wheels]}; pass " @@ -475,17 +474,24 @@ class Preflight: for tag, wheel, executable in targets: label = f"{tag} {wheel.name}" try: + digest = _sha256_file(wheel) # One venv per wheel: it runs the import smoke test, the bundled # asset check and the frame pipeline together. - record = verify_wheel( + verifier = verify_wheel + extra = {"python_executable": executable, "offline": self.options.offline} + if wheel_target(wheel)[1] != "win_amd64": + verifier = verify_manylinux_wheel + extra = {"repo_root": self.repo_root} + record = verifier( wheel, video=video, config_name=self.options.config_name, frames=self.options.frames, - python_executable=executable, - offline=self.options.offline, check_assets=not self.options.no_assets, + **extra, ) + if _sha256_file(wheel) != digest: + raise RuntimeError(f"{wheel.name} changed during runtime verification") except subprocess.CalledProcessError as exc: problems.append(f"{label}: command failed ({exc})") continue @@ -497,7 +503,7 @@ class Preflight: f"{label}: {record['assets']} | frames={record['frames']} " f"flow={record['flow_shape']}" ) - self.runtime_records.append({"tag": tag, **record}) + self.runtime_records.append({"tag": tag, **record, "sha256": digest}) detail = "\n".join(lines) if problems: diff --git a/build/release/publish.py b/build/release/publish.py index 73c20c8..d175c85 100644 --- a/build/release/publish.py +++ b/build/release/publish.py @@ -86,6 +86,17 @@ def evaluate_gate( if not recorded: problems.append("report lists no wheels") + runtime_records = {} + for step in payload.get("steps") or []: + if step.get("name") == "wheel-runtime" and step.get("ok"): + runtime_records.update({item.get("wheel"): item.get("sha256") + for item in (step.get("data") or {}).get("records", [])}) + uncovered = sorted(name for name, digest in recorded.items() + if runtime_records.get(name) != digest) + if uncovered: + problems.append(f"wheels lack matching runtime verification: {uncovered}; " + "re-run preflight --all-wheel-pythons") + missing = sorted(set(recorded) - set(wheels)) if missing: problems.append(f"wheels in the report are missing from disk: {missing}") diff --git a/build/release/pypi.py b/build/release/pypi.py index 5292bf4..f258dac 100644 --- a/build/release/pypi.py +++ b/build/release/pypi.py @@ -18,6 +18,7 @@ probe, and ``--apply`` being required even to reach TestPyPI. from __future__ import annotations import json +import os import subprocess import sys import tempfile @@ -34,7 +35,7 @@ if str(_BUILD_ROOT) not in sys.path: from common.paths import REPO_ROOT # noqa: E402 from release import auth # noqa: E402 -from release.publish import GateResult, PublishGateError, load_gate # noqa: E402 +from release.publish import GateResult, PublishGateError, load_gate, sha256_file # noqa: E402 INDEX_JSON = {name: str(meta["json_url"]) for name, meta in auth.INDEXES.items()} INDEX_SIMPLE = {name: str(meta["simple_url"]) for name, meta in auth.INDEXES.items()} @@ -283,9 +284,10 @@ def _upload(wheels: list[Path], repository: str | None) -> int: def _fetch_back(index: str, version: str, wheel: Path) -> int: - """Download the freshly uploaded wheel back out of the index.""" + """Fetch the exact Python/platform wheel and verify its certified bytes.""" with tempfile.TemporaryDirectory(prefix="orisys-fetch-") as tmp: target = Path(tmp) / "out" + python_tag, abi_tag, platform_tags = wheel.stem.rsplit("-", 3)[1:] argv = [ sys.executable, "-m", @@ -293,18 +295,23 @@ def _fetch_back(index: str, version: str, wheel: Path) -> int: "download", "--no-deps", "--no-cache-dir", + "--only-binary=:all:", + "--implementation", "cp", + "--python-version", python_tag[2:], + "--abi", abi_tag, "--index-url", INDEX_SIMPLE[index], "--dest", str(target), f"orisys=={version}", ] - if index == "testpypi": - # TestPyPI mirrors only what was uploaded there; deps live on PyPI. - # --no-deps keeps this a pure "is the artifact fetchable" probe. - argv += ["--extra-index-url", INDEX_SIMPLE["pypi"]] + for platform_tag in platform_tags.split("."): + argv += ["--platform", platform_tag] + # Index verification must not inherit a fallback index or wheelhouse. + env = {key: value for key, value in os.environ.items() if not key.startswith("PIP_")} + env["PIP_CONFIG_FILE"] = os.devnull print(f"[run ] {' '.join(argv)}") - completed = subprocess.run(argv, check=False) + completed = subprocess.run(argv, check=False, env=env) if completed.returncode != 0: return completed.returncode @@ -313,11 +320,10 @@ def _fetch_back(index: str, version: str, wheel: Path) -> int: print(f"[FAIL] pip download produced nothing for orisys=={version}") return 1 print(f"[ok ] fetched {[p.name for p in fetched]} from {index}") - if wheel.name not in {path.name for path in fetched}: - print( - f"[warn] {wheel.name} was not the file fetched " - "(the index may serve a different platform first)" - ) + if {path.name for path in fetched} != {wheel.name}: + raise PublishGateError(f"{index} returned a different wheel than {wheel.name}") + if sha256_file(fetched[0]) != sha256_file(wheel): + raise PublishGateError(f"{index} returned different bytes for {wheel.name}") return 0 @@ -436,13 +442,21 @@ def publish(options: Options) -> int: print(f"[ok ] {index} serves all {len(published)} file(s) for {version}") if options.verify_install: - if _fetch_back(index, version, wheels[0]) != 0: - print( - f"[warn] {index} artifact is not fetchable yet; the upload is " - "published, so this is a propagation delay rather than a failure" - ) - else: - print(f"[ok ] fetched the published artifact back from {index}") + fetch_pending = False + for wheel in wheels: + try: + if _fetch_back(index, version, wheel) != 0: + fetch_pending = True + print(f"[warn] {wheel.name} is uploaded but download verification is inconclusive") + else: + print(f"[ok ] fetched and hash-verified {wheel.name} from {index}") + except PublishGateError as exc: + print(f"[FAIL] {exc}") + return 1 + if fetch_pending: + print(f"[warn] {index} upload succeeded but download verification is inconclusive; " + "later indexes were not uploaded. Re-run after checking propagation.") + return 0 if phases == ["testpypi"]: print() diff --git a/build/release/runtime.py b/build/release/runtime.py new file mode 100644 index 0000000..a291cb2 --- /dev/null +++ b/build/release/runtime.py @@ -0,0 +1,117 @@ +"""Run Linux wheel release checks in the matching manylinux Docker image.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +from packaging.utils import parse_wheel_filename + +PLATFORM_LABELS = { + "win_amd64": "Windows x64", + "x86_64": "Linux x86_64", + "aarch64": "Linux ARM64", +} + + +def wheel_target(wheel: Path) -> tuple[str, str]: + """Return the CPython tag and supported OS/architecture; reject ambiguity.""" + _, _, _, tags = parse_wheel_filename(wheel.name) + identities = set() + for tag in tags: + if ( + tag.interpreter not in {"cp310", "cp311", "cp312"} + or tag.abi != tag.interpreter + ): + raise RuntimeError(f"Unsupported Python ABI in {wheel.name}") + if tag.platform == "win_amd64": + target = "win_amd64" + elif tag.platform.startswith("manylinux") and tag.platform.endswith("_aarch64"): + target = "aarch64" + elif tag.platform.startswith("manylinux") and tag.platform.endswith("_x86_64"): + target = "x86_64" + else: + raise RuntimeError(f"Unsupported release platform in {wheel.name}") + identities.add((tag.interpreter, target)) + if len(identities) != 1: + raise RuntimeError(f"Ambiguous release platform in {wheel.name}") + return identities.pop() + + +_CONTAINER_CHECK = """ +import json, sys +from pathlib import Path +sys.path.insert(0, '/src/build') +from common.verify.wheel import verify_wheel +record = verify_wheel(Path(sys.argv[1]), video=Path(sys.argv[2]), + config_name=sys.argv[3], frames=int(sys.argv[4]), check_assets=sys.argv[5] == '1') +print('ORISYS_RUNTIME_RESULT=' + json.dumps(record)) +""" + + +def verify_manylinux_wheel( + wheel: Path, + *, + repo_root: Path, + video: Path, + config_name: str, + frames: int, + check_assets: bool = True, +) -> dict: + tag, architecture = wheel_target(wheel) + if architecture == "win_amd64": + raise RuntimeError("Windows wheels must be verified with a Windows interpreter") + platform = "linux/arm64" if architecture == "aarch64" else "linux/amd64" + image = ( + "orisys-manylinux-aarch64" if architecture == "aarch64" else "orisys-manylinux" + ) + abi = f"{tag}-{tag}" + argv = [ + "docker", + "run", + "--rm", + "--pull=never", + "--platform", + platform, + "-v", + f"{repo_root.resolve()}:/src:ro", + "-v", + f"{wheel.resolve().parent}:/wheels:ro", + "-v", + f"{video.resolve().parent}:/video:ro", + "-e", + "PIP_NO_INDEX=1", + "-e", + f"PIP_FIND_LINKS=/opt/orisys-wheelhouse/{abi}", + "--entrypoint", + f"/opt/python/{abi}/bin/python", + image, + "-c", + _CONTAINER_CHECK, + f"/wheels/{wheel.name}", + f"/video/{video.name}", + config_name, + str(frames), + "1" if check_assets else "0", + ] + print(f" verifying {wheel.name} in {image} ({platform}) ...", flush=True) + result = subprocess.run(argv, capture_output=True, text=True, check=False) + if result.returncode: + raise RuntimeError( + f"{image} verification failed:\n{result.stdout}\n{result.stderr}" + ) + marker = "ORISYS_RUNTIME_RESULT=" + records = [ + line[len(marker) :] + for line in result.stdout.splitlines() + if line.startswith(marker) + ] + if len(records) != 1: + raise RuntimeError( + f"{image} returned no unique verification record:\n{result.stdout}" + ) + record = json.loads(records[0]) + if record.get("wheel") != wheel.name or record.get("frames") != frames: + raise RuntimeError(f"Unexpected verification record for {wheel.name}: {record}") + return record diff --git a/docs/publishing.en.md b/docs/publishing.en.md index 6a5ea4b..5a4b061 100644 --- a/docs/publishing.en.md +++ b/docs/publishing.en.md @@ -29,12 +29,13 @@ assets are present; `pyproject.toml` declares which non-code files are packaged. ## Build the candidate Use the launcher appropriate to the target. `wheel-win` invokes the Windows build; -`manylinux` invokes the Docker-backed Linux build. A single-interpreter `wheel` +`wheel-manylinux` invokes the Docker-backed Linux build. A single-interpreter `wheel` build is useful for a local candidate, but does not cover every supported ABI. ```bash python build/run.py wheel-win -python build/run.py manylinux +python build/run.py wheel-manylinux +python build/run.py wheel-manylinux -Architecture aarch64 ``` Windows builds use the configured `py310`, `py311`, and `py312` environments. @@ -44,6 +45,11 @@ See [build/README.md](../build/README.md) for build prerequisites and narrower commands. A contributor without these environments can still run tests and hand off a source change; do not describe it as a verified release candidate. +Build Windows x64, Linux x86_64 and Linux ARM64 before preparing the release +report. With Python 3.10, 3.11 and 3.12 this produces nine wheels. Both publishers +use every wheel in `SDK//`; files added or rebuilt after preflight +invalidate its certification. ARM64 CPU wheels do not require CUDA. + ## Run the release gate After building, run `release preflight` on a clean, reviewed commit. It checks @@ -56,6 +62,16 @@ version check against PyPI. The report is written under python build/run.py release preflight --all-wheel-pythons --check-pypi ``` +`--all-wheel-pythons` checks every staged ABI and architecture. Run the complete +mixed-platform gate from Windows: Windows wheels use the `py310`, `py311` and +`py312` conda environments, while Linux wheels use `orisys-manylinux` or +`orisys-manylinux-aarch64` with the matching container Python. Docker must already +have these locally built images; verification never pulls or rebuilds them. +Linux dependencies come from each image's ABI wheelhouse, in an isolated venv. +ARM64 verification on Intel/AMD uses QEMU and can take longer. Every wheel must +pass imports, bundled-asset loading and the CPU video pipeline; emulation does +not certify Q6A or Jetson hardware, GPU or NPU behavior. + For work in progress, `--allow-dirty --keep-going` can collect diagnostics, but that report does **not** certify a release. Do not use `--skip-tests`, `--no-runtime`, or `--no-assets` to turn missing verification into approval. If the runtime clip is @@ -72,6 +88,8 @@ not replace the full preflight gate. Hand the maintainer the version, source commit, wheel filenames, preflight report, and any hardware or platform checks that were not run. The publish commands read the passing preflight report and verify the wheel hashes again. +The report must also contain successful runtime evidence matching every wheel's +hash. Older reports that checked only Windows wheels must be regenerated. ```bash python build/run.py release github --version @@ -87,3 +105,16 @@ GitHub publication can sync the release mirror, commit, tag, push, and create a Release. See the launcher help and [build/README.md](../build/README.md) for the exact flags. A published version cannot be overwritten, so do not treat a rerun as a way to replace bad wheels. + +The release order is build all candidates, commit source changes, run the clean +preflight, preview both publishers, then execute the authorized publications. +GitHub attaches all certified wheels and generates notes naming their actual +platforms, including ARM64. PyPI defaults to TestPyPI first; after confirming it, +use `--confirm-pypi --apply` for production. Each upload checks the complete +filename set and downloads every Python/platform wheel from that index to +compare its hash. TestPyPI download verification does not fall back to PyPI. +If downloads are not yet available, the command reports the successful upload +as unconfirmed and stops before uploading to later indexes; check propagation +before rerunning. +If a published version has a different file set, the commands stop rather than +silently extending or replacing that release. diff --git a/tests/test_release_publish.py b/tests/test_release_publish.py index ccdf3cc..b4760f3 100644 --- a/tests/test_release_publish.py +++ b/tests/test_release_publish.py @@ -41,6 +41,9 @@ def _report_payload(version: str, wheels: list[Path], **overrides) -> dict: {"name": wheel.name, "sha256": publish.sha256_file(wheel), "size": wheel.stat().st_size} for wheel in wheels ], + "steps": [{"name": "wheel-runtime", "ok": True, "data": { + "records": [{"wheel": wheel.name, "sha256": publish.sha256_file(wheel)} for wheel in wheels] + }}], } payload.update(overrides) return payload @@ -89,6 +92,21 @@ def test_gate_rejects_changed_wheel(tmp_path): assert any("changed after preflight" in problem for problem in problems) +def test_gate_rejects_arm_wheel_without_runtime_evidence(tmp_path): + wheel = _wheel(tmp_path, "orisys-1.2.3-cp311-cp311-manylinux_2_28_aarch64.whl") + payload = _report_payload("1.2.3", [wheel], steps=[]) + problems, _ = publish.evaluate_gate(payload, "1.2.3", {wheel.name: wheel}) + assert any("runtime" in problem and wheel.name in problem for problem in problems) + + +def test_gate_rejects_runtime_evidence_for_different_bytes(tmp_path): + wheel = _wheel(tmp_path, "orisys-1.2.3-cp311-cp311-manylinux_2_28_aarch64.whl") + payload = _report_payload("1.2.3", [wheel]) + payload["steps"][0]["data"]["records"][0]["sha256"] = "stale" + problems, _ = publish.evaluate_gate(payload, "1.2.3", {wheel.name: wheel}) + assert any("runtime" in problem for problem in problems) + + def test_gate_rejects_uncertified_and_missing_wheels(tmp_path): wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")] payload = _report_payload("1.2.3", wheels) @@ -186,9 +204,12 @@ def test_github_steps_include_notes_file(tmp_path): def test_release_notes_list_every_wheel(): notes = github.build_release_notes( - "1.2.3", [Path("a.whl"), Path("b.whl")], "c" * 40 + "1.2.3", [Path("orisys-1.2.3-cp311-cp311-win_amd64.whl"), + Path("orisys-1.2.3-cp311-cp311-manylinux_2_28_aarch64.whl")], "c" * 40 ) - assert "a.whl" in notes and "b.whl" in notes + assert "win_amd64.whl" in notes and "aarch64.whl" in notes + assert "Linux ARM64" in notes + assert "Linux x86_64" not in notes assert "cccccccccccc" in notes @@ -622,3 +643,90 @@ def test_publish_aborts_on_a_conflicting_existing_release(tmp_path, monkeypatch, def test_index_json_urls_cover_both_indexes(): assert pypi.INDEX_JSON["testpypi"].startswith("https://test.pypi.org/") assert pypi.INDEX_JSON["pypi"].startswith("https://pypi.org/") + + +@pytest.mark.parametrize("abi", ["cp310", "cp311", "cp312"]) +@pytest.mark.parametrize("platform", ["win_amd64", "manylinux_2_28_x86_64", "manylinux_2_28_aarch64"]) +def test_fetch_back_targets_every_abi_and_platform(tmp_path, monkeypatch, abi, platform): + wheel = _wheel(tmp_path, f"orisys-1.2.3-{abi}-{abi}-{platform}.whl") + monkeypatch.setenv("PIP_EXTRA_INDEX_URL", "https://fallback.invalid/simple") + + def run(argv, **kwargs): + assert "PIP_EXTRA_INDEX_URL" not in kwargs["env"] + assert kwargs["env"]["PIP_CONFIG_FILE"] == pypi.os.devnull + assert argv[argv.index("--python-version") + 1] == abi[2:] + assert argv[argv.index("--abi") + 1] == abi + assert argv[argv.index("--platform") + 1] == platform + assert argv[argv.index("--index-url") + 1] == pypi.INDEX_SIMPLE["testpypi"] + assert "--extra-index-url" not in argv + target = Path(argv[argv.index("--dest") + 1]) + target.mkdir() + (target / wheel.name).write_bytes(wheel.read_bytes()) + return subprocess.CompletedProcess(argv, 0) + + monkeypatch.setattr(pypi.subprocess, "run", run) + assert pypi._fetch_back("testpypi", "1.2.3", wheel) == 0 + + +def test_fetch_back_rejects_modified_download(tmp_path, monkeypatch): + wheel = _wheel(tmp_path, "orisys-1.2.3-cp311-cp311-manylinux_2_28_aarch64.whl") + + def run(argv, **kwargs): + target = Path(argv[argv.index("--dest") + 1]) + target.mkdir() + (target / wheel.name).write_bytes(b"tampered") + return subprocess.CompletedProcess(argv, 0) + + monkeypatch.setattr(pypi.subprocess, "run", run) + with pytest.raises(publish.PublishGateError, match="different bytes"): + pypi._fetch_back("pypi", "1.2.3", wheel) + + +def test_pypi_upload_and_fetch_include_arm_wheels(tmp_path, monkeypatch): + sdk, original = _synthetic_report_layout(tmp_path, "1.2.3") + wheels = [original, _wheel(original.parent, "orisys-1.2.3-cp311-cp311-manylinux_2_28_aarch64.whl")] + report = sdk / "build/release/reports/1.2.3/preflight.json" + report.write_text(json.dumps(_report_payload("1.2.3", wheels)), encoding="utf-8") + monkeypatch.setattr(pypi, "REPO_ROOT", sdk) + monkeypatch.setattr(pypi, "_ensure_credentials", lambda *a, **k: True) + monkeypatch.setattr(pypi, "index_files", lambda *a, **k: (None, None)) + monkeypatch.setattr(pypi, "index_state", lambda *a, **k: pypi.IndexState(versions=[])) + monkeypatch.setattr(pypi, "wait_for_release", lambda *a, **k: ([w.name for w in wheels], None)) + uploaded, fetched = [], [] + monkeypatch.setattr(pypi, "_upload", lambda files, index: uploaded.extend(files) or 0) + monkeypatch.setattr(pypi, "_fetch_back", lambda index, version, wheel: fetched.append(wheel) or 0) + assert pypi.publish(pypi.Options(version="1.2.3", apply=True, pypi_only=True, skip_check=True)) == 0 + assert set(uploaded) == set(fetched) == set(wheels) + + +def test_github_dry_run_includes_arm_asset_and_generated_notes(tmp_path, monkeypatch, capsys): + sdk, release, original = _synthetic_publish_layout(tmp_path, "1.2.3", tag_exists=False) + wheels = [original, _wheel(original.parent, "orisys-1.2.3-cp311-cp311-manylinux_2_28_aarch64.whl")] + report = sdk / "build/release/reports/1.2.3/preflight.json" + report.write_text(json.dumps(_report_payload("1.2.3", wheels)), encoding="utf-8") + monkeypatch.setattr(github, "REPO_ROOT", sdk) + monkeypatch.setattr(github, "MANIFEST_PATH", sdk / "tools/downstream_sync_manifest.json") + monkeypatch.setattr(github, "_check_gh", lambda repo: []) + monkeypatch.setattr(github, "_release_exists", lambda repo, tag: False) + assert github.publish(github.Options(version="1.2.3", skip_fetch=True)) == 0 + output = capsys.readouterr().out + assert wheels[1].name in output + assert "Linux ARM64" in output + assert "--notes-file" in output + assert not (release / "release").exists() + assert not (report.parent / "github-release-notes.md").exists() + + +def test_inconclusive_testpypi_download_stops_before_production(tmp_path, monkeypatch, capsys): + sdk, wheel = _synthetic_report_layout(tmp_path, "1.2.3") + monkeypatch.setattr(pypi, "REPO_ROOT", sdk) + monkeypatch.setattr(pypi, "_ensure_credentials", lambda *a, **k: True) + monkeypatch.setattr(pypi, "index_files", lambda *a, **k: (None, None)) + monkeypatch.setattr(pypi, "index_state", lambda *a, **k: pypi.IndexState(versions=[])) + monkeypatch.setattr(pypi, "wait_for_release", lambda *a, **k: ([wheel.name], None)) + uploaded = [] + monkeypatch.setattr(pypi, "_upload", lambda files, index: uploaded.append(index) or 0) + monkeypatch.setattr(pypi, "_fetch_back", lambda *a: 1) + assert pypi.publish(pypi.Options(version="1.2.3", apply=True, confirm_pypi=True, skip_check=True)) == 0 + assert uploaded == ["testpypi"] + assert "later indexes were not uploaded" in capsys.readouterr().out diff --git a/tests/test_release_runtime.py b/tests/test_release_runtime.py new file mode 100644 index 0000000..b01ce8a --- /dev/null +++ b/tests/test_release_runtime.py @@ -0,0 +1,148 @@ +"""Release verification must cover each staged architecture and Python ABI.""" + +import json +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from release import preflight, runtime + + +def test_all_wheel_pythons_selects_all_nine_wheels(tmp_path, monkeypatch): + check = preflight.Preflight( + preflight.PreflightOptions(version="1.2.3", all_wheel_pythons=True), + repo_root=tmp_path, + ) + check.wheels = [ + tmp_path / f"orisys-1.2.3-{abi}-{abi}-{platform}.whl" + for abi in ("cp310", "cp311", "cp312") + for platform in ("win_amd64", "manylinux_2_28_x86_64", "manylinux_2_28_aarch64") + ] + monkeypatch.setattr(preflight.sys, "platform", "win32") + monkeypatch.setattr( + preflight.version_mod, "python_for_tag", lambda tag: Path(f"{tag}/python.exe") + ) + assert {wheel for _, wheel, _ in check._runtime_targets()} == set(check.wheels) + + +@pytest.mark.parametrize( + "architecture,platform,image", + [ + ("x86_64", "linux/amd64", "orisys-manylinux"), + ("aarch64", "linux/arm64", "orisys-manylinux-aarch64"), + ], +) +def test_manylinux_verifier_uses_matching_container( + tmp_path, monkeypatch, architecture, platform, image +): + wheel = tmp_path / f"orisys-1.2.3-cp311-cp311-manylinux_2_28_{architecture}.whl" + wheel.touch() + video = tmp_path / "sample.mp4" + video.touch() + record = { + "wheel": wheel.name, + "assets": "OK", + "imports": "OK", + "frames": 40, + "flow_shape": [1, 2], + } + calls = [] + + def run(argv, **kwargs): + calls.append(argv) + return subprocess.CompletedProcess( + argv, 0, "progress\nORISYS_RUNTIME_RESULT=" + json.dumps(record), "" + ) + + monkeypatch.setattr(runtime.subprocess, "run", run) + assert ( + runtime.verify_manylinux_wheel( + wheel, repo_root=tmp_path, video=video, config_name="box", frames=40 + ) + == record + ) + argv = calls[0] + assert argv[argv.index("--platform") + 1] == platform + assert image in argv + assert "/opt/python/cp311-cp311/bin/python" in argv + assert any( + str(tmp_path.resolve()) in arg and arg.endswith(":/src:ro") for arg in argv + ) + + +def test_manylinux_verifier_propagates_container_failure(tmp_path, monkeypatch): + monkeypatch.setattr( + runtime.subprocess, + "run", + lambda argv, **kwargs: subprocess.CompletedProcess( + argv, 1, "", "ARM import failed" + ), + ) + with pytest.raises(RuntimeError, match="ARM import failed"): + runtime.verify_manylinux_wheel( + tmp_path / "orisys-1.2.3-cp312-cp312-manylinux_2_28_aarch64.whl", + repo_root=tmp_path, + video=tmp_path / "sample.mp4", + config_name="box", + frames=40, + ) + + +def test_runtime_step_records_hashes_for_windows_and_arm(tmp_path, monkeypatch): + video = tmp_path / "sample.mp4" + video.touch() + check = preflight.Preflight( + preflight.PreflightOptions( + version="1.2.3", all_wheel_pythons=True, video=video + ), + repo_root=tmp_path, + ) + check.wheels = [ + tmp_path / f"orisys-1.2.3-cp311-cp311-{target}.whl" + for target in ("win_amd64", "manylinux_2_28_aarch64") + ] + for wheel in check.wheels: + wheel.write_bytes(b"wheel") + monkeypatch.setattr(preflight.sys, "platform", "win32") + monkeypatch.setattr( + preflight.version_mod, "python_for_tag", lambda tag: Path("python.exe") + ) + seen = [] + + def verify(wheel, **kwargs): + seen.append(wheel) + return {"wheel": wheel.name, "assets": "OK", "frames": 40, "flow_shape": [1, 2]} + + monkeypatch.setattr(preflight, "verify_wheel", verify) + monkeypatch.setattr(preflight, "verify_manylinux_wheel", verify) + step = check._wheel_runtime() + assert step.ok, step.detail + assert set(seen) == set(check.wheels) + assert all( + record["sha256"] == preflight._sha256_file(check.wheels[0]) + for record in step.data["records"] + ) + + +@pytest.mark.parametrize( + "name", + [ + "orisys-1.2.3-cp311-cp311-manylinux_2_28_aarch64.whl", + "orisys-1.2.3-cp311-cp311-manylinux_2_28_x86_64.whl", + ], +) +def test_native_linux_selection_matches_machine(tmp_path, monkeypatch, name): + check = preflight.Preflight( + preflight.PreflightOptions(version="1.2.3"), repo_root=tmp_path + ) + check.wheels = [tmp_path / name] + monkeypatch.setattr(preflight.sys, "platform", "linux") + monkeypatch.setattr(preflight.sys, "version_info", SimpleNamespace(major=3, minor=11)) + monkeypatch.setattr(preflight.platform, "machine", lambda: "aarch64") + if "aarch64" in name: + assert check._runtime_targets()[0][1].name == name + else: + with pytest.raises(RuntimeError, match="no cp311 aarch64 wheel"): + check._runtime_targets()