"""Upload the certified wheels to TestPyPI, then (on confirmation) to PyPI. ``python build/run.py release pypi`` runs: 1. certified-wheels gate (same contract as ``release github``) 2. ``twine check`` on every wheel - the wheel metadata is generated, so its long description and classifiers are worth validating before an upload that cannot be undone 3. refuse if the version already exists on the target index (read-only probe) 4. upload to TestPyPI, then verify by fetching it back from the index 5. stop there unless ``--confirm-pypi`` is given; then repeat 3-4 against PyPI PyPI uploads are permanent: a version number can never be reused, and a bad artifact cannot be replaced. Hence the TestPyPI stage, the read-only existence probe, and ``--apply`` being required even to reach TestPyPI. """ from __future__ import annotations import json import os import subprocess import sys import tempfile import time import urllib.error import urllib.request from dataclasses import dataclass, field from pathlib import Path _BUILD_ROOT = Path(__file__).resolve().parents[1] if str(_BUILD_ROOT) not in sys.path: sys.path.insert(0, str(_BUILD_ROOT)) from common.paths import REPO_ROOT # noqa: E402 from release import auth # noqa: E402 from release.publish import GateResult, PublishGateError, load_gate, sha256_file # noqa: E402 INDEX_JSON = {name: str(meta["json_url"]) for name, meta in auth.INDEXES.items()} INDEX_SIMPLE = {name: str(meta["simple_url"]) for name, meta in auth.INDEXES.items()} @dataclass class Options: version: str apply: bool = False confirm_pypi: bool = False verify_install: bool = True skip_check: bool = False # Upload straight to PyPI, skipping the TestPyPI rehearsal. Needed when you # only hold a production token, since the two indexes reject each other's. pypi_only: bool = False # Offer to enter a missing token now instead of just failing. prompt_credentials: bool = False # Token supplied non-interactively (e.g. --token-stdin), used once. token: str | None = None def upload_phases(options: Options) -> list[str]: """The indexes to upload to, in order.""" if options.pypi_only: return ["pypi"] return ["testpypi", "pypi"] if options.confirm_pypi else ["testpypi"] @dataclass class IndexState: versions: list[str] = field(default_factory=list) error: str | None = None @property def known(self) -> bool: return self.error is None def twine_check_argv(wheels: list[Path]) -> list[str]: return [sys.executable, "-m", "twine", "check", *[str(w) for w in wheels]] def twine_upload_argv(wheels: list[Path], repository: str | None) -> list[str]: argv = [sys.executable, "-m", "twine", "upload"] if repository: argv += ["--repository", repository] argv += [str(wheel) for wheel in wheels] return argv def credentials_hint(index: str) -> str | None: """Guidance when ``index`` has no usable credentials, else ``None``. More than "does the file exist": a placeholder or the wrong ``username`` is treated as missing, because twine would only find out at upload time. """ state = auth.credential_state(index) if state.configured: return None lines = [f"no usable credentials for [{index}]: {state.problem}"] lines.append(f" set it up with: python build/run.py release auth --set-token {index}") if index == "testpypi": lines.append( " (TestPyPI needs its own account and token; if you only hold a " "production token, use --pypi-only to skip the rehearsal)" ) lines.append( " or set TWINE_USERNAME=__token__ and TWINE_PASSWORD for this session only" ) return "\n".join(lines) def _ensure_credentials( indexes: list[str], options: Options, *, interactive: bool | None = None ) -> bool: """Make sure every index that will be uploaded to has credentials. Prompts for a missing token when there is a terminal to prompt on; a piped or CI run fails fast instead of hanging on a hidden-input prompt. """ missing = [index for index in indexes if not auth.credential_state(index).configured] if not missing: for index in indexes: state = auth.credential_state(index) print(f"[ok ] credentials for [{index}] ({state.source})") return True if interactive is None: interactive = sys.stdin.isatty() allow_prompt = options.prompt_credentials or (interactive and options.apply) if not allow_prompt: for index in missing: print(f"[{'FAIL' if options.apply else 'warn'}] {credentials_hint(index)}") if not options.apply: print( "[info] dry run continues so you can review the plan; " "--apply would stop here until the credentials exist" ) return True if not interactive: print( "[info] stdin is not a terminal, so nothing was prompted; run " "`release auth --set-token ` or pass --prompt-credentials" ) return False for index in missing: print(f"[warn] {credentials_hint(index)}") token = options.token or auth.prompt_for_token(index) if not token: print(f"[FAIL] no token entered for [{index}]") return False for problem in auth.validate_token(token): print(f"[FAIL] {problem.removeprefix('ERROR ')}") return False for warning in auth.token_warnings(token): print(f"[warn] {warning.removeprefix('WARN ')}") path = auth.write_token(index, token) print(f"[ok ] wrote [{index}] credentials to {path}") options.token = None # only the first index may consume a piped token return True def index_state(index: str, *, timeout: float = 20.0) -> IndexState: """Read the published versions of orisys from an index (read-only).""" url = INDEX_JSON[index] request = urllib.request.Request( url, headers={"User-Agent": "orisys-release-publish", "Accept": "application/json"} ) try: with urllib.request.urlopen(request, timeout=timeout) as response: payload = json.load(response) except urllib.error.HTTPError as exc: if exc.code == 404: return IndexState(versions=[]) return IndexState(error=f"{index} returned HTTP {exc.code}") except Exception as exc: # network, DNS, TLS, JSON return IndexState(error=f"{index} lookup failed: {type(exc).__name__}: {exc}") releases = payload.get("releases") if not isinstance(releases, dict): return IndexState(error=f"unexpected {index} response (no 'releases' mapping)") return IndexState( versions=sorted(name for name, files in releases.items() if files) ) def index_files( index: str, version: str, *, timeout: float = 20.0 ) -> tuple[list[str] | None, str | None]: """Filenames published for ``version``, as ``(filenames, error)``. ``(None, None)`` - the index does not serve this version (yet) ``(None, "why")`` - the lookup itself failed, which is NOT evidence of absence ``([...], None)`` - the version is published with those files Keeping those apart matters: reporting a network hiccup as "not published yet" is how a successful upload gets mistaken for a failure. """ url = INDEX_JSON[index] request = urllib.request.Request( url, headers={"User-Agent": "orisys-release-publish", "Accept": "application/json"} ) try: with urllib.request.urlopen(request, timeout=timeout) as response: payload = json.load(response) except urllib.error.HTTPError as exc: if exc.code == 404: return None, None return None, f"{index} returned HTTP {exc.code}" except Exception as exc: return None, f"{index} lookup failed: {type(exc).__name__}: {exc}" files = (payload.get("releases") or {}).get(version) if not isinstance(files, list) or not files: return None, None return sorted(str(item.get("filename", "")) for item in files), None def wait_for_release( index: str, version: str, *, attempts: int = 6, delay: float = 5.0, sleep=None, timeout: float = 20.0, ) -> tuple[list[str] | None, str | None]: """Poll until the index serves ``version`` (uploads are not instantly visible). TestPyPI in particular lags the upload response by seconds. Returns the last ``(filenames, error)`` observed. """ sleep = sleep or time.sleep last_error: str | None = None for attempt in range(1, max(1, attempts) + 1): filenames, error = index_files(index, version, timeout=timeout) if filenames is not None: return filenames, None last_error = error if attempt < attempts: print( f"[info] {index} does not serve {version} yet " f"({'lookup failed' if error else 'indexing'}), retry {attempt}/{attempts - 1}", flush=True, ) sleep(delay) return None, last_error def existing_release_state(index: str, version: str, expected: set[str]) -> str: """``"satisfied"`` | ``"conflict"`` | ``"absent"`` for this index and version. An index version can never be replaced, so "already published with exactly these files" is the only successful terminal state; treating it as an error would make a re-run after a partial failure look like a new failure. A different file set is a genuine conflict and must stop the run. """ published, error = index_files(index, version) if published is None: if error: print(f"[warn] could not check {index} for an existing release: {error}") return "absent" if set(published) == expected: print( f"[ok ] {index} already serves orisys {version} with all " f"{len(published)} file(s)" ) return "satisfied" extra = sorted(set(published) - expected) missing = sorted(expected - set(published)) print( f"[FAIL] {index} already serves orisys {version} with a different file set" + (f"; unexpected: {extra}" if extra else "") + (f"; missing: {missing}" if missing else "") ) return "conflict" def _upload(wheels: list[Path], repository: str | None) -> int: argv = twine_upload_argv(wheels, repository) print(f"[run ] {' '.join(str(part) for part in argv)}") completed = subprocess.run(argv, check=False) return completed.returncode def _fetch_back(index: str, version: str, wheel: Path) -> int: """Fetch the exact Python/platform wheel and verify its certified bytes.""" with tempfile.TemporaryDirectory(prefix="orisys-fetch-") as tmp: target = Path(tmp) / "out" python_tag, abi_tag, platform_tags = wheel.stem.rsplit("-", 3)[1:] argv = [ sys.executable, "-m", "pip", "download", "--no-deps", "--no-cache-dir", "--only-binary=:all:", "--implementation", "cp", "--python-version", python_tag[2:], "--abi", abi_tag, "--index-url", INDEX_SIMPLE[index], "--dest", str(target), f"orisys=={version}", ] for platform_tag in platform_tags.split("."): argv += ["--platform", platform_tag] # Index verification must not inherit a fallback index or wheelhouse. env = {key: value for key, value in os.environ.items() if not key.startswith("PIP_")} env["PIP_CONFIG_FILE"] = os.devnull print(f"[run ] {' '.join(argv)}") completed = subprocess.run(argv, check=False, env=env) if completed.returncode != 0: return completed.returncode fetched = sorted(target.glob(f"orisys-{version}*")) if not fetched: print(f"[FAIL] pip download produced nothing for orisys=={version}") return 1 print(f"[ok ] fetched {[p.name for p in fetched]} from {index}") if {path.name for path in fetched} != {wheel.name}: raise PublishGateError(f"{index} returned a different wheel than {wheel.name}") if sha256_file(fetched[0]) != sha256_file(wheel): raise PublishGateError(f"{index} returned different bytes for {wheel.name}") return 0 def publish(options: Options) -> int: version = options.version mode = "APPLY" if options.apply else "DRY RUN" print(f"=== release pypi: orisys {version} ({mode}) ===") # --- 1. gate ------------------------------------------------------- try: gate: GateResult = load_gate(REPO_ROOT, version) except PublishGateError as exc: print(f"[FAIL] {exc}") return 1 print(f"[ok ] preflight report: {gate.report_path}") for wheel in gate.wheels: print(f" [ok] {wheel.name} sha256={wheel.sha256[:12]}") for warning in gate.warnings: print(f"[warn] {warning}") wheels = [REPO_ROOT / "SDK" / version / record.name for record in gate.wheels] missing = [str(wheel) for wheel in wheels if not wheel.is_file()] if missing: print(f"[FAIL] wheels missing from disk: {missing}") return 1 # --- 2. credentials + twine check (before anything irreversible) ---- phases = upload_phases(options) if options.pypi_only: print( "[warn] --pypi-only: uploading straight to PyPI with no TestPyPI " "rehearsal. An index version can never be reused or replaced." ) if not _ensure_credentials(phases, options): return 1 if not options.skip_check: argv = twine_check_argv(wheels) print(f"[run ] {' '.join(str(part) for part in argv)}") completed = subprocess.run(argv, check=False) if completed.returncode != 0: print("[FAIL] twine check reported problems; fix the wheel metadata first") return 1 print("[ok ] twine check passed") # --- 3. phase plan ------------------------------------------------- expected = {wheel.name for wheel in wheels} remaining: list[str] = [] for index in phases: state = existing_release_state(index, version, expected) if state == "conflict": return 1 if state == "absent": remaining.append(index) phases = remaining print() for index in phases: state = index_state(index) if not state.known: print(f"[FAIL] {state.error}; cannot safely check for duplicate uploads") return 1 if version in state.versions: print( f"[FAIL] orisys {version} already exists on {index} " f"(published: {', '.join(state.versions) or 'none'}) - " "an index version cannot be reused" ) return 1 print(f"[ok ] {index}: {version} not present (published: {', '.join(state.versions) or 'none'})") if not phases: print() print("=== release pypi: every requested index already has this release ===") return 0 print() for index in phases: print(f"[plan] upload {len(wheels)} wheel(s) to {index}") if phases == ["testpypi"]: print( "[plan] stop after TestPyPI. Re-run with " "`--confirm-pypi --apply` to publish to the real PyPI." ) print() if not options.apply: print("dry run: nothing was uploaded. Re-run with --apply to execute.") return 0 # --- 4. upload ---------------------------------------------------- for index in phases: if _upload(wheels, index) != 0: print(f"[FAIL] twine upload to {index} failed") return 1 print(f"[ok ] uploaded to {index}") published, error = wait_for_release(index, version) if published is None: # The upload itself succeeded, so this is a confirmation problem, not # a publishing failure. Say so plainly and do not imply data loss. reason = f"lookup kept failing ({error})" if error else "the index has not indexed it yet" print( f"[warn] uploaded, but could not confirm it on {index} yet: {reason}.\n" f" check https://{'test.' if index == 'testpypi' else ''}pypi.org/project/orisys/{version}/" "\n re-running this command is safe: an already-published version is skipped" ) print(f"\n=== release pypi: {index} upload SUCCEEDED (verification inconclusive) ===") return 0 if set(published) != expected: print( f"[FAIL] {index} file set mismatch:\n expected {sorted(expected)}\n" f" got {sorted(published)}" ) return 1 print(f"[ok ] {index} serves all {len(published)} file(s) for {version}") if options.verify_install: fetch_pending = False for wheel in wheels: try: if _fetch_back(index, version, wheel) != 0: fetch_pending = True print(f"[warn] {wheel.name} is uploaded but download verification is inconclusive") else: print(f"[ok ] fetched and hash-verified {wheel.name} from {index}") except PublishGateError as exc: print(f"[FAIL] {exc}") return 1 if fetch_pending: print(f"[warn] {index} upload succeeded but download verification is inconclusive; " "later indexes were not uploaded. Re-run after checking propagation.") return 0 if phases == ["testpypi"]: print() print("=== release pypi: TestPyPI stage PASSED; PyPI untouched ===") print("next: python build/run.py release pypi --confirm-pypi --apply") return 0 print() if options.pypi_only: print("=== release pypi PASSED (PyPI only, no TestPyPI rehearsal) ===") else: print("=== release pypi PASSED (TestPyPI + PyPI) ===") return 0