ykcheungab
9c951e9d3c
补齐全部架构的运行证据、GitHub 发布说明和 PyPI 逐文件下载核验。已通过 161 项测试、代码和文档门禁、九个 wheel 的本地索引下载核验，以及 ARM 三个 ABI 的 40 帧 CPU 检查。
473 lines
18 KiB
Python
473 lines
18 KiB
Python
"""Upload the certified wheels to TestPyPI, then (on confirmation) to PyPI.
|
|
|
|
``python build/run.py release pypi`` runs:
|
|
|
|
1. certified-wheels gate (same contract as ``release github``)
|
|
2. ``twine check`` on every wheel - the wheel metadata is generated, so its
|
|
long description and classifiers are worth validating before an upload that
|
|
cannot be undone
|
|
3. refuse if the version already exists on the target index (read-only probe)
|
|
4. upload to TestPyPI, then verify by fetching it back from the index
|
|
5. stop there unless ``--confirm-pypi`` is given; then repeat 3-4 against PyPI
|
|
|
|
PyPI uploads are permanent: a version number can never be reused, and a bad
|
|
artifact cannot be replaced. Hence the TestPyPI stage, the read-only existence
|
|
probe, and ``--apply`` being required even to reach TestPyPI.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
_BUILD_ROOT = Path(__file__).resolve().parents[1]
|
|
if str(_BUILD_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(_BUILD_ROOT))
|
|
|
|
from common.paths import REPO_ROOT # noqa: E402
|
|
|
|
from release import auth # noqa: E402
|
|
from release.publish import GateResult, PublishGateError, load_gate, sha256_file # noqa: E402
|
|
|
|
INDEX_JSON = {name: str(meta["json_url"]) for name, meta in auth.INDEXES.items()}
|
|
INDEX_SIMPLE = {name: str(meta["simple_url"]) for name, meta in auth.INDEXES.items()}
|
|
|
|
|
|
@dataclass
|
|
class Options:
|
|
version: str
|
|
apply: bool = False
|
|
confirm_pypi: bool = False
|
|
verify_install: bool = True
|
|
skip_check: bool = False
|
|
# Upload straight to PyPI, skipping the TestPyPI rehearsal. Needed when you
|
|
# only hold a production token, since the two indexes reject each other's.
|
|
pypi_only: bool = False
|
|
# Offer to enter a missing token now instead of just failing.
|
|
prompt_credentials: bool = False
|
|
# Token supplied non-interactively (e.g. --token-stdin), used once.
|
|
token: str | None = None
|
|
|
|
|
|
def upload_phases(options: Options) -> list[str]:
|
|
"""The indexes to upload to, in order."""
|
|
if options.pypi_only:
|
|
return ["pypi"]
|
|
return ["testpypi", "pypi"] if options.confirm_pypi else ["testpypi"]
|
|
|
|
|
|
@dataclass
|
|
class IndexState:
|
|
versions: list[str] = field(default_factory=list)
|
|
error: str | None = None
|
|
|
|
@property
|
|
def known(self) -> bool:
|
|
return self.error is None
|
|
|
|
|
|
def twine_check_argv(wheels: list[Path]) -> list[str]:
|
|
return [sys.executable, "-m", "twine", "check", *[str(w) for w in wheels]]
|
|
|
|
|
|
def twine_upload_argv(wheels: list[Path], repository: str | None) -> list[str]:
|
|
argv = [sys.executable, "-m", "twine", "upload"]
|
|
if repository:
|
|
argv += ["--repository", repository]
|
|
argv += [str(wheel) for wheel in wheels]
|
|
return argv
|
|
|
|
|
|
def credentials_hint(index: str) -> str | None:
|
|
"""Guidance when ``index`` has no usable credentials, else ``None``.
|
|
|
|
More than "does the file exist": a placeholder or the wrong ``username`` is
|
|
treated as missing, because twine would only find out at upload time.
|
|
"""
|
|
state = auth.credential_state(index)
|
|
if state.configured:
|
|
return None
|
|
lines = [f"no usable credentials for [{index}]: {state.problem}"]
|
|
lines.append(f" set it up with: python build/run.py release auth --set-token {index}")
|
|
if index == "testpypi":
|
|
lines.append(
|
|
" (TestPyPI needs its own account and token; if you only hold a "
|
|
"production token, use --pypi-only to skip the rehearsal)"
|
|
)
|
|
lines.append(
|
|
" or set TWINE_USERNAME=__token__ and TWINE_PASSWORD for this session only"
|
|
)
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _ensure_credentials(
|
|
indexes: list[str], options: Options, *, interactive: bool | None = None
|
|
) -> bool:
|
|
"""Make sure every index that will be uploaded to has credentials.
|
|
|
|
Prompts for a missing token when there is a terminal to prompt on; a piped or
|
|
CI run fails fast instead of hanging on a hidden-input prompt.
|
|
"""
|
|
missing = [index for index in indexes if not auth.credential_state(index).configured]
|
|
if not missing:
|
|
for index in indexes:
|
|
state = auth.credential_state(index)
|
|
print(f"[ok ] credentials for [{index}] ({state.source})")
|
|
return True
|
|
|
|
if interactive is None:
|
|
interactive = sys.stdin.isatty()
|
|
allow_prompt = options.prompt_credentials or (interactive and options.apply)
|
|
|
|
if not allow_prompt:
|
|
for index in missing:
|
|
print(f"[{'FAIL' if options.apply else 'warn'}] {credentials_hint(index)}")
|
|
if not options.apply:
|
|
print(
|
|
"[info] dry run continues so you can review the plan; "
|
|
"--apply would stop here until the credentials exist"
|
|
)
|
|
return True
|
|
if not interactive:
|
|
print(
|
|
"[info] stdin is not a terminal, so nothing was prompted; run "
|
|
"`release auth --set-token <index>` or pass --prompt-credentials"
|
|
)
|
|
return False
|
|
|
|
for index in missing:
|
|
print(f"[warn] {credentials_hint(index)}")
|
|
token = options.token or auth.prompt_for_token(index)
|
|
if not token:
|
|
print(f"[FAIL] no token entered for [{index}]")
|
|
return False
|
|
for problem in auth.validate_token(token):
|
|
print(f"[FAIL] {problem.removeprefix('ERROR ')}")
|
|
return False
|
|
for warning in auth.token_warnings(token):
|
|
print(f"[warn] {warning.removeprefix('WARN ')}")
|
|
path = auth.write_token(index, token)
|
|
print(f"[ok ] wrote [{index}] credentials to {path}")
|
|
options.token = None # only the first index may consume a piped token
|
|
return True
|
|
|
|
|
|
def index_state(index: str, *, timeout: float = 20.0) -> IndexState:
|
|
"""Read the published versions of orisys from an index (read-only)."""
|
|
url = INDEX_JSON[index]
|
|
request = urllib.request.Request(
|
|
url, headers={"User-Agent": "orisys-release-publish", "Accept": "application/json"}
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
payload = json.load(response)
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code == 404:
|
|
return IndexState(versions=[])
|
|
return IndexState(error=f"{index} returned HTTP {exc.code}")
|
|
except Exception as exc: # network, DNS, TLS, JSON
|
|
return IndexState(error=f"{index} lookup failed: {type(exc).__name__}: {exc}")
|
|
|
|
releases = payload.get("releases")
|
|
if not isinstance(releases, dict):
|
|
return IndexState(error=f"unexpected {index} response (no 'releases' mapping)")
|
|
return IndexState(
|
|
versions=sorted(name for name, files in releases.items() if files)
|
|
)
|
|
|
|
|
|
def index_files(
|
|
index: str, version: str, *, timeout: float = 20.0
|
|
) -> tuple[list[str] | None, str | None]:
|
|
"""Filenames published for ``version``, as ``(filenames, error)``.
|
|
|
|
``(None, None)`` - the index does not serve this version (yet)
|
|
``(None, "why")`` - the lookup itself failed, which is NOT evidence of absence
|
|
``([...], None)`` - the version is published with those files
|
|
|
|
Keeping those apart matters: reporting a network hiccup as "not published
|
|
yet" is how a successful upload gets mistaken for a failure.
|
|
"""
|
|
url = INDEX_JSON[index]
|
|
request = urllib.request.Request(
|
|
url, headers={"User-Agent": "orisys-release-publish", "Accept": "application/json"}
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
payload = json.load(response)
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code == 404:
|
|
return None, None
|
|
return None, f"{index} returned HTTP {exc.code}"
|
|
except Exception as exc:
|
|
return None, f"{index} lookup failed: {type(exc).__name__}: {exc}"
|
|
|
|
files = (payload.get("releases") or {}).get(version)
|
|
if not isinstance(files, list) or not files:
|
|
return None, None
|
|
return sorted(str(item.get("filename", "")) for item in files), None
|
|
|
|
|
|
def wait_for_release(
|
|
index: str,
|
|
version: str,
|
|
*,
|
|
attempts: int = 6,
|
|
delay: float = 5.0,
|
|
sleep=None,
|
|
timeout: float = 20.0,
|
|
) -> tuple[list[str] | None, str | None]:
|
|
"""Poll until the index serves ``version`` (uploads are not instantly visible).
|
|
|
|
TestPyPI in particular lags the upload response by seconds. Returns the last
|
|
``(filenames, error)`` observed.
|
|
"""
|
|
sleep = sleep or time.sleep
|
|
last_error: str | None = None
|
|
for attempt in range(1, max(1, attempts) + 1):
|
|
filenames, error = index_files(index, version, timeout=timeout)
|
|
if filenames is not None:
|
|
return filenames, None
|
|
last_error = error
|
|
if attempt < attempts:
|
|
print(
|
|
f"[info] {index} does not serve {version} yet "
|
|
f"({'lookup failed' if error else 'indexing'}), retry {attempt}/{attempts - 1}",
|
|
flush=True,
|
|
)
|
|
sleep(delay)
|
|
return None, last_error
|
|
|
|
|
|
def existing_release_state(index: str, version: str, expected: set[str]) -> str:
|
|
"""``"satisfied"`` | ``"conflict"`` | ``"absent"`` for this index and version.
|
|
|
|
An index version can never be replaced, so "already published with exactly
|
|
these files" is the only successful terminal state; treating it as an error
|
|
would make a re-run after a partial failure look like a new failure. A
|
|
different file set is a genuine conflict and must stop the run.
|
|
"""
|
|
published, error = index_files(index, version)
|
|
if published is None:
|
|
if error:
|
|
print(f"[warn] could not check {index} for an existing release: {error}")
|
|
return "absent"
|
|
if set(published) == expected:
|
|
print(
|
|
f"[ok ] {index} already serves orisys {version} with all "
|
|
f"{len(published)} file(s)"
|
|
)
|
|
return "satisfied"
|
|
extra = sorted(set(published) - expected)
|
|
missing = sorted(expected - set(published))
|
|
print(
|
|
f"[FAIL] {index} already serves orisys {version} with a different file set"
|
|
+ (f"; unexpected: {extra}" if extra else "")
|
|
+ (f"; missing: {missing}" if missing else "")
|
|
)
|
|
return "conflict"
|
|
|
|
|
|
def _upload(wheels: list[Path], repository: str | None) -> int:
|
|
argv = twine_upload_argv(wheels, repository)
|
|
print(f"[run ] {' '.join(str(part) for part in argv)}")
|
|
completed = subprocess.run(argv, check=False)
|
|
return completed.returncode
|
|
|
|
|
|
def _fetch_back(index: str, version: str, wheel: Path) -> int:
|
|
"""Fetch the exact Python/platform wheel and verify its certified bytes."""
|
|
with tempfile.TemporaryDirectory(prefix="orisys-fetch-") as tmp:
|
|
target = Path(tmp) / "out"
|
|
python_tag, abi_tag, platform_tags = wheel.stem.rsplit("-", 3)[1:]
|
|
argv = [
|
|
sys.executable,
|
|
"-m",
|
|
"pip",
|
|
"download",
|
|
"--no-deps",
|
|
"--no-cache-dir",
|
|
"--only-binary=:all:",
|
|
"--implementation", "cp",
|
|
"--python-version", python_tag[2:],
|
|
"--abi", abi_tag,
|
|
"--index-url",
|
|
INDEX_SIMPLE[index],
|
|
"--dest",
|
|
str(target),
|
|
f"orisys=={version}",
|
|
]
|
|
for platform_tag in platform_tags.split("."):
|
|
argv += ["--platform", platform_tag]
|
|
# Index verification must not inherit a fallback index or wheelhouse.
|
|
env = {key: value for key, value in os.environ.items() if not key.startswith("PIP_")}
|
|
env["PIP_CONFIG_FILE"] = os.devnull
|
|
print(f"[run ] {' '.join(argv)}")
|
|
completed = subprocess.run(argv, check=False, env=env)
|
|
if completed.returncode != 0:
|
|
return completed.returncode
|
|
|
|
fetched = sorted(target.glob(f"orisys-{version}*"))
|
|
if not fetched:
|
|
print(f"[FAIL] pip download produced nothing for orisys=={version}")
|
|
return 1
|
|
print(f"[ok ] fetched {[p.name for p in fetched]} from {index}")
|
|
if {path.name for path in fetched} != {wheel.name}:
|
|
raise PublishGateError(f"{index} returned a different wheel than {wheel.name}")
|
|
if sha256_file(fetched[0]) != sha256_file(wheel):
|
|
raise PublishGateError(f"{index} returned different bytes for {wheel.name}")
|
|
return 0
|
|
|
|
|
|
def publish(options: Options) -> int:
|
|
version = options.version
|
|
mode = "APPLY" if options.apply else "DRY RUN"
|
|
print(f"=== release pypi: orisys {version} ({mode}) ===")
|
|
|
|
# --- 1. gate -------------------------------------------------------
|
|
try:
|
|
gate: GateResult = load_gate(REPO_ROOT, version)
|
|
except PublishGateError as exc:
|
|
print(f"[FAIL] {exc}")
|
|
return 1
|
|
print(f"[ok ] preflight report: {gate.report_path}")
|
|
for wheel in gate.wheels:
|
|
print(f" [ok] {wheel.name} sha256={wheel.sha256[:12]}")
|
|
for warning in gate.warnings:
|
|
print(f"[warn] {warning}")
|
|
|
|
wheels = [REPO_ROOT / "SDK" / version / record.name for record in gate.wheels]
|
|
missing = [str(wheel) for wheel in wheels if not wheel.is_file()]
|
|
if missing:
|
|
print(f"[FAIL] wheels missing from disk: {missing}")
|
|
return 1
|
|
|
|
# --- 2. credentials + twine check (before anything irreversible) ----
|
|
phases = upload_phases(options)
|
|
if options.pypi_only:
|
|
print(
|
|
"[warn] --pypi-only: uploading straight to PyPI with no TestPyPI "
|
|
"rehearsal. An index version can never be reused or replaced."
|
|
)
|
|
if not _ensure_credentials(phases, options):
|
|
return 1
|
|
|
|
if not options.skip_check:
|
|
argv = twine_check_argv(wheels)
|
|
print(f"[run ] {' '.join(str(part) for part in argv)}")
|
|
completed = subprocess.run(argv, check=False)
|
|
if completed.returncode != 0:
|
|
print("[FAIL] twine check reported problems; fix the wheel metadata first")
|
|
return 1
|
|
print("[ok ] twine check passed")
|
|
|
|
# --- 3. phase plan -------------------------------------------------
|
|
expected = {wheel.name for wheel in wheels}
|
|
remaining: list[str] = []
|
|
for index in phases:
|
|
state = existing_release_state(index, version, expected)
|
|
if state == "conflict":
|
|
return 1
|
|
if state == "absent":
|
|
remaining.append(index)
|
|
phases = remaining
|
|
|
|
print()
|
|
for index in phases:
|
|
state = index_state(index)
|
|
if not state.known:
|
|
print(f"[FAIL] {state.error}; cannot safely check for duplicate uploads")
|
|
return 1
|
|
if version in state.versions:
|
|
print(
|
|
f"[FAIL] orisys {version} already exists on {index} "
|
|
f"(published: {', '.join(state.versions) or 'none'}) - "
|
|
"an index version cannot be reused"
|
|
)
|
|
return 1
|
|
print(f"[ok ] {index}: {version} not present (published: {', '.join(state.versions) or 'none'})")
|
|
|
|
if not phases:
|
|
print()
|
|
print("=== release pypi: every requested index already has this release ===")
|
|
return 0
|
|
|
|
print()
|
|
for index in phases:
|
|
print(f"[plan] upload {len(wheels)} wheel(s) to {index}")
|
|
if phases == ["testpypi"]:
|
|
print(
|
|
"[plan] stop after TestPyPI. Re-run with "
|
|
"`--confirm-pypi --apply` to publish to the real PyPI."
|
|
)
|
|
print()
|
|
|
|
if not options.apply:
|
|
print("dry run: nothing was uploaded. Re-run with --apply to execute.")
|
|
return 0
|
|
|
|
# --- 4. upload ----------------------------------------------------
|
|
for index in phases:
|
|
if _upload(wheels, index) != 0:
|
|
print(f"[FAIL] twine upload to {index} failed")
|
|
return 1
|
|
print(f"[ok ] uploaded to {index}")
|
|
|
|
published, error = wait_for_release(index, version)
|
|
if published is None:
|
|
# The upload itself succeeded, so this is a confirmation problem, not
|
|
# a publishing failure. Say so plainly and do not imply data loss.
|
|
reason = f"lookup kept failing ({error})" if error else "the index has not indexed it yet"
|
|
print(
|
|
f"[warn] uploaded, but could not confirm it on {index} yet: {reason}.\n"
|
|
f" check https://{'test.' if index == 'testpypi' else ''}pypi.org/project/orisys/{version}/"
|
|
"\n re-running this command is safe: an already-published version is skipped"
|
|
)
|
|
print(f"\n=== release pypi: {index} upload SUCCEEDED (verification inconclusive) ===")
|
|
return 0
|
|
if set(published) != expected:
|
|
print(
|
|
f"[FAIL] {index} file set mismatch:\n expected {sorted(expected)}\n"
|
|
f" got {sorted(published)}"
|
|
)
|
|
return 1
|
|
print(f"[ok ] {index} serves all {len(published)} file(s) for {version}")
|
|
|
|
if options.verify_install:
|
|
fetch_pending = False
|
|
for wheel in wheels:
|
|
try:
|
|
if _fetch_back(index, version, wheel) != 0:
|
|
fetch_pending = True
|
|
print(f"[warn] {wheel.name} is uploaded but download verification is inconclusive")
|
|
else:
|
|
print(f"[ok ] fetched and hash-verified {wheel.name} from {index}")
|
|
except PublishGateError as exc:
|
|
print(f"[FAIL] {exc}")
|
|
return 1
|
|
if fetch_pending:
|
|
print(f"[warn] {index} upload succeeded but download verification is inconclusive; "
|
|
"later indexes were not uploaded. Re-run after checking propagation.")
|
|
return 0
|
|
|
|
if phases == ["testpypi"]:
|
|
print()
|
|
print("=== release pypi: TestPyPI stage PASSED; PyPI untouched ===")
|
|
print("next: python build/run.py release pypi --confirm-pypi --apply")
|
|
return 0
|
|
|
|
print()
|
|
if options.pypi_only:
|
|
print("=== release pypi PASSED (PyPI only, no TestPyPI rehearsal) ===")
|
|
else:
|
|
print("=== release pypi PASSED (TestPyPI + PyPI) ===")
|
|
return 0
|