orisys-tech/orisys-sdk
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 将 ARM64 wheel 纳入发布校验和上传流程

Browse Source 
补齐全部架构的运行证据、GitHub 发布说明和 PyPI 逐文件下载核验。已通过 161 项测试、代码和文档门禁、九个 wheel 的本地索引下载核验，以及 ARM 三个 ABI 的 40 帧 CPU 检查。
This commit is contained in:
ykcheungab
2026-10-15 21:39:31 +08:00
parent bb15b750fb
commit 9c951e9d3c
10 changed files with 496 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/release/cli.py
+1 -1
View File
@@ -52,7 +52,7 @@ def _cmd_preflight(argv: list[str]) -> int:
parser.add_argument(
"--all-wheel-pythons",
action="store_true",
help="verify cp310/cp311/cp312 wheels, each in its own conda env",
help="verify every staged wheel (Windows conda environments and Linux Docker images)",
)
parser.add_argument("--offline", action="store_true", help="reuse parent-env deps instead of PyPI")
parser.add_argument("--no-runtime", action="store_true", help="skip the isolated runtime check")
build/release/github.py
+9 -2
View File
@@ -31,6 +31,7 @@ if str(_BUILD_ROOT) not in sys.path:
from common.paths import REPO_ROOT # noqa: E402
from release.publish import GateResult, PublishGateError, load_gate # noqa: E402
from release.runtime import PLATFORM_LABELS, wheel_target # noqa: E402
MANIFEST_PATH = REPO_ROOT / "tools" / "downstream_sync_manifest.json"
SYNC_TOOL = REPO_ROOT / "tools" / "sync_downstreams.py"
@@ -93,10 +94,11 @@ def _git(repo: Path, *args: str, check: bool = True) -> str:
def build_release_notes(version: str, wheels: list[Path], git_commit: str | None) -> str:
platforms = sorted({PLATFORM_LABELS[wheel_target(wheel)[1]] for wheel in wheels})
lines = [
f"orisys {version}",
"",
"Wheels (Python 3.10 / 3.11 / 3.12, Windows x64 and Linux x86_64):",
f"Wheels ({', '.join(platforms)}; Python ABI is shown in each filename):",
]
lines.extend(f"- `{wheel.name}`" for wheel in wheels)
lines.append("")
@@ -335,7 +337,8 @@ def publish(options: Options) -> int:
asset_dir = repo / "release" / version
assets = [asset_dir / wheel.name for wheel in wheels]
notes_file = options.notes_file
notes_file = options.notes_file or gate.report_path.parent / "github-release-notes.md"
generated_notes = build_release_notes(version, wheels, gate.git_commit) if options.notes_file is None else None
steps = build_github_steps(
version=version,
branch=branch,
@@ -361,11 +364,15 @@ def publish(options: Options) -> int:
print()
if not options.apply:
if generated_notes:
print(generated_notes)
print("dry run: nothing was changed. Re-run with --apply to execute.")
return 0
# --- 4. execute ----------------------------------------------------
try:
if generated_notes:
notes_file.write_text(generated_notes, encoding="utf-8")
commit_before = _git(repo, "rev-parse", "HEAD")
_sync(run_checks=not options.skip_checks)
_stage_wheels(wheels, asset_dir)
build/release/preflight.py
+25 -19
View File
@@ -1,8 +1,8 @@
"""Local release gate: everything that must hold before publishing a wheel.
``release preflight`` answers one question with an exit code: is this checkout,
this version and these wheels safe to publish? It never publishes anything, and
by default it makes no network calls.
this version and these wheels safe to publish? It never publishes anything;
native verification environments may download dependencies from PyPI.
Steps, in order:
@@ -52,6 +52,7 @@ from wheel.layout import ( # noqa: E402
from release import report as report_mod # noqa: E402
from release import version as version_mod # noqa: E402
from release.runtime import verify_manylinux_wheel, wheel_target # noqa: E402
@dataclass
@@ -411,31 +412,29 @@ class Preflight:
return Step("wheel-inspection", not problems, detail, data={"count": len(self.wheels)})
def _platform_token(self) -> str:
return "win_amd64" if sys.platform == "win32" else "manylinux"
return "win_amd64" if sys.platform == "win32" else platform.machine().lower()
def _runtime_targets(self) -> list[tuple[str, Path, Path | None]]:
token = self._platform_token()
if self.options.all_wheel_pythons:
targets: list[tuple[str, Path, Path | None]] = []
for tag in ("cp310", "cp311", "cp312"):
match = next(
(w for w in self.wheels if f"-{tag}-" in w.name and token in w.name), None
)
if match is None:
raise RuntimeError(
f"no {tag} {token} wheel among {[w.name for w in self.wheels]}"
)
executable = version_mod.python_for_tag(tag)
if executable is None:
for wheel in self.wheels:
tag, target = wheel_target(wheel)
executable = None
if target == "win_amd64":
if sys.platform != "win32":
raise RuntimeError("Verify Windows wheels from a Windows host")
executable = version_mod.python_for_tag(tag)
if target == "win_amd64" and executable is None:
raise RuntimeError(
f"cannot resolve an interpreter for {tag}; create the conda env "
f"py{tag[2:]} or set CONDA_ENVS_ROOT"
)
targets.append((tag, match, executable))
targets.append((tag, wheel, executable))
return targets
tag = f"cp{sys.version_info.major}{sys.version_info.minor}"
match = next((w for w in self.wheels if f"-{tag}-" in w.name and token in w.name), None)
match = next((w for w in self.wheels if wheel_target(w) == (tag, token)), None)
if match is None:
raise RuntimeError(
f"no {tag} {token} wheel among {[w.name for w in self.wheels]}; pass "
@@ -475,17 +474,24 @@ class Preflight:
for tag, wheel, executable in targets:
label = f"{tag} {wheel.name}"
try:
digest = _sha256_file(wheel)
# One venv per wheel: it runs the import smoke test, the bundled
# asset check and the frame pipeline together.
record = verify_wheel(
verifier = verify_wheel
extra = {"python_executable": executable, "offline": self.options.offline}
if wheel_target(wheel)[1] != "win_amd64":
verifier = verify_manylinux_wheel
extra = {"repo_root": self.repo_root}
record = verifier(
wheel,
video=video,
config_name=self.options.config_name,
frames=self.options.frames,
python_executable=executable,
offline=self.options.offline,
check_assets=not self.options.no_assets,
**extra,
)
if _sha256_file(wheel) != digest:
raise RuntimeError(f"{wheel.name} changed during runtime verification")
except subprocess.CalledProcessError as exc:
problems.append(f"{label}: command failed ({exc})")
continue
@@ -497,7 +503,7 @@ class Preflight:
f"{label}: {record['assets']} | frames={record['frames']} "
f"flow={record['flow_shape']}"
)
self.runtime_records.append({"tag": tag, **record})
self.runtime_records.append({"tag": tag, **record, "sha256": digest})
detail = "\n".join(lines)
if problems:
build/release/publish.py
+11
View File
@@ -86,6 +86,17 @@ def evaluate_gate(
if not recorded:
problems.append("report lists no wheels")
runtime_records = {}
for step in payload.get("steps") or []:
if step.get("name") == "wheel-runtime" and step.get("ok"):
runtime_records.update({item.get("wheel"): item.get("sha256")
for item in (step.get("data") or {}).get("records", [])})
uncovered = sorted(name for name, digest in recorded.items()
if runtime_records.get(name) != digest)
if uncovered:
problems.append(f"wheels lack matching runtime verification: {uncovered}; "
"re-run preflight --all-wheel-pythons")
missing = sorted(set(recorded) - set(wheels))
if missing:
problems.append(f"wheels in the report are missing from disk: {missing}")
build/release/pypi.py
+33 -19
View File
@@ -18,6 +18,7 @@ probe, and ``--apply`` being required even to reach TestPyPI.
from __future__ import annotations
import json
import os
import subprocess
import sys
import tempfile
@@ -34,7 +35,7 @@ if str(_BUILD_ROOT) not in sys.path:
from common.paths import REPO_ROOT # noqa: E402
from release import auth # noqa: E402
from release.publish import GateResult, PublishGateError, load_gate # noqa: E402
from release.publish import GateResult, PublishGateError, load_gate, sha256_file # noqa: E402
INDEX_JSON = {name: str(meta["json_url"]) for name, meta in auth.INDEXES.items()}
INDEX_SIMPLE = {name: str(meta["simple_url"]) for name, meta in auth.INDEXES.items()}
@@ -283,9 +284,10 @@ def _upload(wheels: list[Path], repository: str | None) -> int:
def _fetch_back(index: str, version: str, wheel: Path) -> int:
"""Download the freshly uploaded wheel back out of the index."""
"""Fetch the exact Python/platform wheel and verify its certified bytes."""
with tempfile.TemporaryDirectory(prefix="orisys-fetch-") as tmp:
target = Path(tmp) / "out"
python_tag, abi_tag, platform_tags = wheel.stem.rsplit("-", 3)[1:]
argv = [
sys.executable,
"-m",
@@ -293,18 +295,23 @@ def _fetch_back(index: str, version: str, wheel: Path) -> int:
"download",
"--no-deps",
"--no-cache-dir",
"--only-binary=:all:",
"--implementation", "cp",
"--python-version", python_tag[2:],
"--abi", abi_tag,
"--index-url",
INDEX_SIMPLE[index],
"--dest",
str(target),
f"orisys=={version}",
]
if index == "testpypi":
# TestPyPI mirrors only what was uploaded there; deps live on PyPI.
# --no-deps keeps this a pure "is the artifact fetchable" probe.
argv += ["--extra-index-url", INDEX_SIMPLE["pypi"]]
for platform_tag in platform_tags.split("."):
argv += ["--platform", platform_tag]
# Index verification must not inherit a fallback index or wheelhouse.
env = {key: value for key, value in os.environ.items() if not key.startswith("PIP_")}
env["PIP_CONFIG_FILE"] = os.devnull
print(f"[run ] {' '.join(argv)}")
completed = subprocess.run(argv, check=False)
completed = subprocess.run(argv, check=False, env=env)
if completed.returncode != 0:
return completed.returncode
@@ -313,11 +320,10 @@ def _fetch_back(index: str, version: str, wheel: Path) -> int:
print(f"[FAIL] pip download produced nothing for orisys=={version}")
return 1
print(f"[ok ] fetched {[p.name for p in fetched]} from {index}")
if wheel.name not in {path.name for path in fetched}:
print(
f"[warn] {wheel.name} was not the file fetched "
"(the index may serve a different platform first)"
)
if {path.name for path in fetched} != {wheel.name}:
raise PublishGateError(f"{index} returned a different wheel than {wheel.name}")
if sha256_file(fetched[0]) != sha256_file(wheel):
raise PublishGateError(f"{index} returned different bytes for {wheel.name}")
return 0
@@ -436,13 +442,21 @@ def publish(options: Options) -> int:
print(f"[ok ] {index} serves all {len(published)} file(s) for {version}")
if options.verify_install:
if _fetch_back(index, version, wheels[0]) != 0:
print(
f"[warn] {index} artifact is not fetchable yet; the upload is "
"published, so this is a propagation delay rather than a failure"
)
else:
print(f"[ok ] fetched the published artifact back from {index}")
fetch_pending = False
for wheel in wheels:
try:
if _fetch_back(index, version, wheel) != 0:
fetch_pending = True
print(f"[warn] {wheel.name} is uploaded but download verification is inconclusive")
else:
print(f"[ok ] fetched and hash-verified {wheel.name} from {index}")
except PublishGateError as exc:
print(f"[FAIL] {exc}")
return 1
if fetch_pending:
print(f"[warn] {index} upload succeeded but download verification is inconclusive; "
"later indexes were not uploaded. Re-run after checking propagation.")
return 0
if phases == ["testpypi"]:
print()
build/release/runtime.py
+117
View File
@@ -0,0 +1,117 @@
"""Run Linux wheel release checks in the matching manylinux Docker image."""
from __future__ import annotations
import json
import subprocess
from pathlib import Path
from packaging.utils import parse_wheel_filename
PLATFORM_LABELS = {
"win_amd64": "Windows x64",
"x86_64": "Linux x86_64",
"aarch64": "Linux ARM64",
}
def wheel_target(wheel: Path) -> tuple[str, str]:
"""Return the CPython tag and supported OS/architecture; reject ambiguity."""
_, _, _, tags = parse_wheel_filename(wheel.name)
identities = set()
for tag in tags:
if (
tag.interpreter not in {"cp310", "cp311", "cp312"}
or tag.abi != tag.interpreter
):
raise RuntimeError(f"Unsupported Python ABI in {wheel.name}")
if tag.platform == "win_amd64":
target = "win_amd64"
elif tag.platform.startswith("manylinux") and tag.platform.endswith("_aarch64"):
target = "aarch64"
elif tag.platform.startswith("manylinux") and tag.platform.endswith("_x86_64"):
target = "x86_64"
else:
raise RuntimeError(f"Unsupported release platform in {wheel.name}")
identities.add((tag.interpreter, target))
if len(identities) != 1:
raise RuntimeError(f"Ambiguous release platform in {wheel.name}")
return identities.pop()
_CONTAINER_CHECK = """
import json, sys
from pathlib import Path
sys.path.insert(0, '/src/build')
from common.verify.wheel import verify_wheel
record = verify_wheel(Path(sys.argv[1]), video=Path(sys.argv[2]),
config_name=sys.argv[3], frames=int(sys.argv[4]), check_assets=sys.argv[5] == '1')
print('ORISYS_RUNTIME_RESULT=' + json.dumps(record))
"""
def verify_manylinux_wheel(
wheel: Path,
*,
repo_root: Path,
video: Path,
config_name: str,
frames: int,
check_assets: bool = True,
) -> dict:
tag, architecture = wheel_target(wheel)
if architecture == "win_amd64":
raise RuntimeError("Windows wheels must be verified with a Windows interpreter")
platform = "linux/arm64" if architecture == "aarch64" else "linux/amd64"
image = (
"orisys-manylinux-aarch64" if architecture == "aarch64" else "orisys-manylinux"
)
abi = f"{tag}-{tag}"
argv = [
"docker",
"run",
"--rm",
"--pull=never",
"--platform",
platform,
"-v",
f"{repo_root.resolve()}:/src:ro",
"-v",
f"{wheel.resolve().parent}:/wheels:ro",
"-v",
f"{video.resolve().parent}:/video:ro",
"-e",
"PIP_NO_INDEX=1",
"-e",
f"PIP_FIND_LINKS=/opt/orisys-wheelhouse/{abi}",
"--entrypoint",
f"/opt/python/{abi}/bin/python",
image,
"-c",
_CONTAINER_CHECK,
f"/wheels/{wheel.name}",
f"/video/{video.name}",
config_name,
str(frames),
"1" if check_assets else "0",
]
print(f" verifying {wheel.name} in {image} ({platform}) ...", flush=True)
result = subprocess.run(argv, capture_output=True, text=True, check=False)
if result.returncode:
raise RuntimeError(
f"{image} verification failed:\n{result.stdout}\n{result.stderr}"
)
marker = "ORISYS_RUNTIME_RESULT="
records = [
line[len(marker) :]
for line in result.stdout.splitlines()
if line.startswith(marker)
]
if len(records) != 1:
raise RuntimeError(
f"{image} returned no unique verification record:\n{result.stdout}"
)
record = json.loads(records[0])
if record.get("wheel") != wheel.name or record.get("frames") != frames:
raise RuntimeError(f"Unexpected verification record for {wheel.name}: {record}")
return record