From 174bceabc4a36b10829270fc74678047e02b9da3 Mon Sep 17 00:00:00 2001 From: ykcheungab Date: Sat, 12 Sep 2026 16:50:55 +0800 Subject: [PATCH] automate publishing: release github + release pypi --- build/README.md | 15 ++ build/release/cli.py | 102 +++++++- build/release/github.py | 369 ++++++++++++++++++++++++++++ build/release/preflight.py | 4 +- build/release/publish.py | 158 ++++++++++++ build/release/pypi.py | 299 ++++++++++++++++++++++ docs/publishing.md | 62 ++++- tests/test_release_publish.py | 312 +++++++++++++++++++++++ tools/downstream_sync_manifest.json | 32 +-- tools/sync_downstreams.py | 97 ++++++-- 10 files changed, 1395 insertions(+), 55 deletions(-) create mode 100644 build/release/github.py create mode 100644 build/release/publish.py create mode 100644 build/release/pypi.py create mode 100644 tests/test_release_publish.py diff --git a/build/README.md b/build/README.md index d9f2c3b..87ab493 100644 --- a/build/README.md +++ b/build/README.md @@ -72,6 +72,21 @@ python build/run.py release preflight --all-wheel-pythons --check-pypi Reports land in `build/release/reports//`. +### Publish + +Both default to a dry run; `--apply` is required for anything irreversible. + +```bash +python build/run.py release github --version # preview +python build/run.py release github --version --apply # sync + tag + push + Release +python build/run.py release pypi --version --apply # TestPyPI only +python build/run.py release pypi --version --confirm-pypi --apply # + PyPI +``` + +Both refuse to act unless `release preflight` passed on a clean commit with no +waived options, and every wheel still hashes to what that report recorded. +See `docs/publishing.md` §3. + ### GUI ```bash diff --git a/build/release/cli.py b/build/release/cli.py index ca8652d..55923ec 100644 --- a/build/release/cli.py +++ b/build/release/cli.py @@ -1,4 +1,9 @@ -"""Command line for ``python build/run.py release``.""" +"""Command line for ``python build/run.py release``. + +Commands: ``preflight`` (local gate), ``audit`` (single wheel), ``github`` (sync +the public repo + publish a GitHub Release), ``pypi`` (TestPyPI then PyPI). +The publish commands are dry-run unless ``--apply`` is passed. +""" from __future__ import annotations @@ -19,6 +24,7 @@ from common.verify.wheel import ( # noqa: E402 verify_wheel_runtime, ) from common.verify.wheel_contents import inspect_wheel # noqa: E402 +from release import github, pypi # noqa: E402 from release.preflight import PreflightOptions, run_preflight # noqa: E402 @@ -170,11 +176,97 @@ def _cmd_audit(argv: list[str]) -> int: return exit_code +def _cmd_github(argv: list[str]) -> int: + parser = argparse.ArgumentParser( + prog="build/run.py release github", + description="Sync orisys-sdk-release, tag, and publish a GitHub Release.", + ) + parser.add_argument("--version", required=True, help="release version, e.g. 0.5.2") + parser.add_argument( + "--apply", + action="store_true", + help="actually mutate the release repo, push, and create the release", + ) + parser.add_argument("--notes-file", type=Path, default=None, help="release notes markdown") + parser.add_argument( + "--attach-preflight", + action="store_true", + help="also upload the preflight report as a release asset (contains local paths)", + ) + parser.add_argument( + "--allow-dirty-release", + action="store_true", + help="proceed even if the release repo has uncommitted changes", + ) + parser.add_argument("--skip-fetch", action="store_true", help="skip `git fetch --tags`") + parser.add_argument( + "--skip-checks", action="store_true", help="skip the manifest's downstream checks" + ) + parser.add_argument( + "--skip-release-upload", + action="store_true", + help="stop after pushing the tag and print the `gh release create` command", + ) + args = parser.parse_args(argv) + + return github.publish( + github.Options( + version=args.version, + apply=args.apply, + notes_file=args.notes_file, + attach_preflight=args.attach_preflight, + allow_dirty_release=args.allow_dirty_release, + skip_fetch=args.skip_fetch, + skip_checks=args.skip_checks, + skip_release_upload=args.skip_release_upload, + ) + ) + + +def _cmd_pypi(argv: list[str]) -> int: + parser = argparse.ArgumentParser( + prog="build/run.py release pypi", + description="Upload the certified wheels to TestPyPI and then PyPI.", + ) + parser.add_argument("--version", required=True, help="release version, e.g. 0.5.2") + parser.add_argument( + "--apply", + action="store_true", + help="actually upload (uploads are permanent and cannot be overwritten)", + ) + parser.add_argument( + "--confirm-pypi", + action="store_true", + help="after TestPyPI, also publish to the real PyPI", + ) + parser.add_argument( + "--repository", default="testpypi", help="twine repository for the first phase" + ) + parser.add_argument( + "--skip-install-verify", + action="store_true", + help="skip fetching the artifact back out of the index", + ) + parser.add_argument("--skip-check", action="store_true", help="skip `twine check`") + args = parser.parse_args(argv) + + return pypi.publish( + pypi.Options( + version=args.version, + apply=args.apply, + confirm_pypi=args.confirm_pypi, + repository=args.repository, + verify_install=not args.skip_install_verify, + skip_check=args.skip_check, + ) + ) + + def main(argv: list[str] | None = None) -> int: argv = list(sys.argv[1:] if argv is None else argv) if not argv or argv[0] in ("-h", "--help"): print(__doc__) - print("Commands: preflight | audit") + print("Commands: preflight | audit | github | pypi") return 0 command, rest = argv[0], argv[1:] @@ -182,9 +274,13 @@ def main(argv: list[str] | None = None) -> int: return _cmd_preflight(rest) if command == "audit": return _cmd_audit(rest) + if command == "github": + return _cmd_github(rest) + if command == "pypi": + return _cmd_pypi(rest) print(f"Unknown release command: {command!r}", file=sys.stderr) - print("Choose from: preflight, audit", file=sys.stderr) + print("Choose from: preflight, audit, github, pypi", file=sys.stderr) return 2 diff --git a/build/release/github.py b/build/release/github.py new file mode 100644 index 0000000..b4544a4 --- /dev/null +++ b/build/release/github.py @@ -0,0 +1,369 @@ +"""Sync the public release repo and publish a GitHub Release. + +``python build/run.py release github`` runs, in order: + +1. certified-wheels gate (a passing ``release preflight`` report must exist and + every wheel must still hash to what it recorded) +2. refuse if the version is already tagged or already has a GitHub Release +3. sync the open-source subset of this checkout into ``orisys-sdk-release`` + (``tools/sync_downstreams.py`` + the manifest's ``checks``) +4. stage ``SDK//*.whl`` into ``orisys-sdk-release/release//`` +5. commit, tag ``v``, push branch + tag +6. create the GitHub Release with the wheels attached, then verify the assets + +Nothing mutates anything unless ``--apply`` is passed: the default is a dry run +that prints every command it would run. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[1] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import REPO_ROOT # noqa: E402 +from release.publish import GateResult, PublishGateError, load_gate # noqa: E402 + +MANIFEST_PATH = REPO_ROOT / "tools" / "downstream_sync_manifest.json" +SYNC_TOOL = REPO_ROOT / "tools" / "sync_downstreams.py" +RELEASE_TARGET = "orisys-sdk-release" + + +@dataclass +class Options: + version: str + apply: bool = False + notes_file: Path | None = None + attach_preflight: bool = False + allow_dirty_release: bool = False + skip_fetch: bool = False + skip_checks: bool = False + skip_release_upload: bool = False + + +@dataclass +class CommandStep: + description: str + argv: list[str] + cwd: Path | None = None + + +@dataclass +class Plan: + release_repo: Path + branch: str + tag: str + assets: list[Path] = field(default_factory=list) + steps: list[CommandStep] = field(default_factory=list) + + +def manifest_entry() -> dict: + """The manifest entry for the public release repo.""" + payload = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + for item in payload.get("downstreams", []): + if item.get("name") == RELEASE_TARGET: + return item + raise PublishGateError(f"{MANIFEST_PATH} has no downstream named {RELEASE_TARGET!r}") + + +def release_repo_path() -> Path: + return (REPO_ROOT / manifest_entry()["path"]).resolve() + + +def _git(repo: Path, *args: str, check: bool = True) -> str: + completed = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, + text=True, + check=False, + ) + if check and completed.returncode != 0: + raise PublishGateError( + f"git {' '.join(args)} failed in {repo}:\n{completed.stderr.strip()}" + ) + return (completed.stdout or "").strip() + + +def build_release_notes(version: str, wheels: list[Path], git_commit: str | None) -> str: + lines = [ + f"orisys {version}", + "", + "Wheels (Python 3.10 / 3.11 / 3.12, Windows x64 and Linux x86_64):", + ] + lines.extend(f"- `{wheel.name}`" for wheel in wheels) + lines.append("") + if git_commit: + lines.append( + f"Built from `{git_commit[:12]}` and certified by `release preflight`." + ) + return "\n".join(lines) + "\n" + + +def build_github_steps( + *, + version: str, + branch: str, + tag: str, + assets: list[Path], + notes_file: Path | None, + title: str | None = None, +) -> list[CommandStep]: + """The mutating commands, as pure data so they can be asserted in tests.""" + release_argv = [ + "gh", + "release", + "create", + tag, + "--verify-tag", + "--title", + title or f"v{version}", + "--latest", + ] + if notes_file is not None: + release_argv += ["--notes-file", str(notes_file)] + release_argv += [str(asset) for asset in assets] + + return [ + CommandStep("create tag", ["git", "tag", tag]), + CommandStep("push branch", ["git", "push", "origin", branch]), + CommandStep("push tag", ["git", "push", "origin", tag]), + CommandStep("create GitHub Release with assets", release_argv), + ] + + +def _check_gh() -> list[str]: + """Problems that would stop the release step, reported before any mutation.""" + problems: list[str] = [] + if shutil.which("gh") is None: + problems.append( + "the `gh` CLI is not on PATH. Install it (`winget install --id GitHub.cli`) " + "and authenticate (`gh auth login`), or use --skip-release-upload" + ) + return problems + completed = subprocess.run( + ["gh", "auth", "status"], capture_output=True, text=True, check=False + ) + if completed.returncode != 0: + problems.append("`gh auth status` failed; run `gh auth login` first") + return problems + + +def _existing_release_tags(repo: Path, version: str) -> list[str]: + wanted = {version, f"v{version}"} + tags = set(_git(repo, "tag", "--list").splitlines()) + return sorted(wanted & tags) + + +def _release_exists(tag: str) -> bool: + completed = subprocess.run( + ["gh", "release", "view", tag, "--json", "tagName"], + capture_output=True, + text=True, + check=False, + ) + return completed.returncode == 0 + + +def _sync(*, run_checks: bool) -> None: + """Copy the public subset into the release repo and run its checks.""" + python = sys.executable + for command in ( + [python, str(SYNC_TOOL), "apply", "--target", RELEASE_TARGET], + *( + [[python, str(SYNC_TOOL), "run-checks", "--target", RELEASE_TARGET]] + if run_checks + else [] + ), + ): + completed = subprocess.run(command, cwd=REPO_ROOT, check=False) + if completed.returncode != 0: + raise PublishGateError( + f"`{' '.join(command[1:])}` failed with exit {completed.returncode}" + ) + + +def _stage_wheels(wheels: list[Path], asset_dir: Path) -> None: + asset_dir.mkdir(parents=True, exist_ok=True) + for wheel in wheels: + shutil.copy2(wheel, asset_dir / wheel.name) + + +def publish(options: Options) -> int: + version = options.version + mode = "APPLY" if options.apply else "DRY RUN" + print(f"=== release github: orisys {version} ({mode}) ===") + + # --- 1. gate ------------------------------------------------------- + try: + gate: GateResult = load_gate(REPO_ROOT, version) + except PublishGateError as exc: + print(f"[FAIL] {exc}") + return 1 + print(f"[ok ] preflight report: {gate.report_path}") + print(f" certified commit: {gate.git_commit}") + for wheel in gate.wheels: + print(f" [ok] {wheel.name} sha256={wheel.sha256[:12]} {wheel.size} bytes") + for warning in gate.warnings: + print(f"[warn] {warning}") + + # --- 2. target repo ------------------------------------------------ + try: + repo = release_repo_path() + except PublishGateError as exc: + print(f"[FAIL] {exc}") + return 1 + if not (repo / ".git").exists(): + print(f"[FAIL] {repo} is not a git repository") + return 1 + + try: + if not options.skip_fetch: + _git(repo, "fetch", "--tags", "--quiet") + print(f"[ok ] git fetch --tags in {repo}") + branch = _git(repo, "rev-parse", "--abbrev-ref", "HEAD") + dirty = _git(repo, "status", "--porcelain") + except PublishGateError as exc: + print(f"[FAIL] {exc}") + return 1 + + if dirty and not options.allow_dirty_release: + preview = "\n".join(dirty.splitlines()[:10]) + print( + f"[FAIL] {repo} has uncommitted changes; commit or stash them first " + f"(or pass --allow-dirty-release):\n{preview}" + ) + return 1 + print(f"[ok ] release repo: {repo} ({branch}) clean={not dirty}") + + tag = f"v{version}" + existing = _existing_release_tags(repo, version) + if existing: + print( + f"[FAIL] tag(s) {existing} already exist in {repo}; " + f"{version} looks already released" + ) + return 1 + if shutil.which("gh") and _release_exists(tag): + print(f"[FAIL] GitHub Release {tag} already exists") + return 1 + print(f"[ok ] no existing tag or release for {tag}") + + # A dry run is a preview of what would happen, so missing tooling is a warning + # there and a hard stop only when we are about to mutate something. + if options.skip_release_upload: + print("[warn] --skip-release-upload: the GitHub Release will be left to you") + else: + gh_problems = _check_gh() + if gh_problems: + for problem in gh_problems: + print(f"[{'FAIL' if options.apply else 'warn'}] {problem}") + if options.apply: + return 1 + else: + print("[ok ] gh CLI is installed and authenticated") + + # --- 3. plan the mutating steps ------------------------------------ + wheels = [REPO_ROOT / "SDK" / version / record.name for record in gate.wheels] + missing = [str(wheel) for wheel in wheels if not wheel.is_file()] + if missing: + print(f"[FAIL] wheels missing from disk: {missing}") + return 1 + + asset_dir = repo / "release" / version + assets = [asset_dir / wheel.name for wheel in wheels] + notes_file = options.notes_file + steps = build_github_steps( + version=version, branch=branch, tag=tag, assets=assets, notes_file=notes_file + ) + + print() + print(f"[plan] sync {len(manifest_entry().get('sync', []))} entry(ies) into {repo}") + print(f"[plan] stage {len(wheels)} wheel(s) into {asset_dir}") + print(f"[plan] commit {repo} and tag {tag}") + for step in steps: + manual = options.skip_release_upload and step.description.startswith( + "create GitHub Release" + ) + prefix = "[plan] (left manual)" if manual else "[plan]" + print(f"{prefix} {step.description}: {' '.join(step.argv)}") + print() + + if not options.apply: + print("dry run: nothing was changed. Re-run with --apply to execute.") + return 0 + + # --- 4. execute ---------------------------------------------------- + try: + commit_before = _git(repo, "rev-parse", "HEAD") + _sync(run_checks=not options.skip_checks) + _stage_wheels(wheels, asset_dir) + + if options.attach_preflight: + shutil.copy2(gate.report_path, asset_dir / f"{version}-preflight.json") + + _git(repo, "add", "-A") + pending = _git(repo, "status", "--porcelain") + if pending: + _git(repo, "commit", "-m", f"release {version}") + print(f"[ok ] committed {len(pending.splitlines())} change(s) in {repo}") + else: + print("[ok ] nothing new to commit in the release repo") + + _git(repo, "tag", tag) + _git(repo, "push", "origin", branch) + _git(repo, "push", "origin", tag) + print(f"[ok ] pushed {branch} and {tag}") + except PublishGateError as exc: + print(f"[FAIL] {exc}") + return 1 + + release_argv = next( + step.argv for step in steps if step.description.startswith("create GitHub Release") + ) + + if options.skip_release_upload: + print() + print("[todo] publish the release yourself:") + print(" " + " ".join(release_argv)) + print() + print("=== release github: repo synced and tagged; release upload left manual ===") + return 0 + + completed = subprocess.run(release_argv, check=False) + if completed.returncode != 0: + print(f"[FAIL] gh release create exited {completed.returncode}") + return 1 + print(f"[ok ] GitHub Release {tag} created") + + # --- 5. verify the published assets -------------------------------- + completed = subprocess.run( + ["gh", "release", "view", tag, "--json", "assets,tagName"], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + print("[FAIL] could not read back the release to verify assets") + return 1 + payload = json.loads(completed.stdout or "{}") + published = {asset["name"] for asset in payload.get("assets", [])} + expected = {asset.name for asset in assets} + if options.attach_preflight: + expected.add(f"{version}-preflight.json") + if published != expected: + print( + f"[FAIL] release asset mismatch:\n expected {sorted(expected)}\n" + f" published {sorted(published)}" + ) + return 1 + print(f"[ok ] verified {len(published)} asset(s) on the release") + print() + print(f"=== release github PASSED (commit {commit_before[:12]} -> {_git(repo, 'rev-parse', 'HEAD')[:12]}) ===") + return 0 diff --git a/build/release/preflight.py b/build/release/preflight.py index 81b00fc..11bc0ae 100644 --- a/build/release/preflight.py +++ b/build/release/preflight.py @@ -16,7 +16,9 @@ Steps, in order: 8. ``wheel-runtime`` - one isolated venv per wheel: bundled assets load and real frames flow through the public pipeline -Output lands in ``build/release/reports//preflight.{json,txt}``. +Output lands in ``build/release/reports//preflight.{json,txt}``, and is +the contract the publish commands (``release github`` / ``release pypi``) consume: +they refuse to ship anything this run did not certify. """ from __future__ import annotations diff --git a/build/release/publish.py b/build/release/publish.py new file mode 100644 index 0000000..73c20c8 --- /dev/null +++ b/build/release/publish.py @@ -0,0 +1,158 @@ +"""Shared publish gate: certify a release from its preflight report. + +Both publish commands (``release github`` and ``release pypi``) refuse to act +unless a passing ``release preflight`` report exists for the exact version and +every wheel still hashes to what that report recorded. The report is the contract +between "we verified this" and "we shipped this"; re-deriving the checks here +would just be a second, weaker opinion. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from pathlib import Path + +# Options that weaken the gate. A release must not be certified by a run that +# waived any of them: re-run preflight on a clean commit instead. +WAIVED_GUARD_KEYS = ("skip_tests", "allow_dirty", "force", "no_runtime", "no_assets") + + +class PublishGateError(RuntimeError): + """The preflight report does not certify these wheels.""" + + +@dataclass +class WheelRecord: + name: str + sha256: str + size: int = 0 + + +@dataclass +class GateResult: + version: str + git_commit: str | None + report_path: Path + wheels: list[WheelRecord] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def evaluate_gate( + payload: dict, + version: str, + wheels: dict[str, Path], + *, + head_commit: str | None = None, +) -> tuple[list[str], list[str]]: + """Return ``(problems, warnings)`` for a report payload and the wheels on disk. + + Pure function: no I/O beyond hashing the wheels, so it is cheap to test. + """ + problems: list[str] = [] + warnings: list[str] = [] + + if str(payload.get("version")) != version: + problems.append( + f"report is for version {payload.get('version')!r}, not {version!r}" + ) + if not payload.get("ok"): + problems.append( + "report result is FAIL - run `release preflight --keep-going` and fix " + "everything before publishing" + ) + if payload.get("aborted"): + problems.append("report stopped early (aborted); it certifies nothing") + + options = payload.get("options") or {} + waived = sorted(key for key in WAIVED_GUARD_KEYS if options.get(key)) + if waived: + problems.append( + "report waived gate options " + + ", ".join(f"--{key.replace('_', '-')}" for key in waived) + + "; commit the tree and re-run `release preflight` without them" + ) + + recorded = {item.get("name"): item.get("sha256") for item in payload.get("wheels") or []} + if not recorded: + problems.append("report lists no wheels") + + missing = sorted(set(recorded) - set(wheels)) + if missing: + problems.append(f"wheels in the report are missing from disk: {missing}") + extra = sorted(set(wheels) - set(recorded)) + if extra: + problems.append(f"wheels on disk are not covered by the report: {extra}") + + for name in sorted(set(recorded) & set(wheels)): + actual = sha256_file(wheels[name]) + if actual != recorded[name]: + problems.append( + f"{name} changed after preflight: report {recorded[name][:12]}, " + f"disk {actual[:12]} - rebuild and re-run preflight" + ) + + commit = payload.get("git", {}).get("commit") if isinstance(payload.get("git"), dict) else None + if head_commit and commit and commit != head_commit: + warnings.append( + f"report certifies commit {commit[:12]} but HEAD is {head_commit[:12]}; " + "the wheels are unchanged, so this is usually just a later commit" + ) + return problems, warnings + + +def load_gate(repo_root: Path, version: str) -> GateResult: + """Load and enforce the preflight report for ``version``. + + Raises :class:`PublishGateError` with every problem found, so one run reports + all of them instead of stopping at the first. + """ + repo_root = Path(repo_root) + report_path = repo_root / "build" / "release" / "reports" / version / "preflight.json" + if not report_path.is_file(): + raise PublishGateError( + f"no preflight report at {report_path}. Run:\n" + f" python build/run.py release preflight\n" + f"(the publish commands only ship what preflight certified)" + ) + + try: + payload = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise PublishGateError(f"cannot read {report_path}: {exc}") from exc + + wheel_dir = repo_root / "SDK" / version + wheels = {path.name: path for path in sorted(wheel_dir.glob("*.whl"))} + if not wheels: + raise PublishGateError( + f"no wheels in {wheel_dir}; build first (`python build/run.py wheel-win` / " + "`manylinux`) and re-run preflight" + ) + + problems, warnings = evaluate_gate(payload, version, wheels) + if problems: + raise PublishGateError( + f"the preflight report at {report_path} does not certify orisys {version}:\n " + + "\n ".join(problems) + ) + + git_payload = payload.get("git") if isinstance(payload.get("git"), dict) else {} + return GateResult( + version=version, + git_commit=git_payload.get("commit"), + report_path=report_path, + wheels=[ + WheelRecord(name=name, sha256=sha256_file(path), size=path.stat().st_size) + for name, path in wheels.items() + ], + warnings=warnings, + ) diff --git a/build/release/pypi.py b/build/release/pypi.py new file mode 100644 index 0000000..649aff5 --- /dev/null +++ b/build/release/pypi.py @@ -0,0 +1,299 @@ +"""Upload the certified wheels to TestPyPI, then (on confirmation) to PyPI. + +``python build/run.py release pypi`` runs: + +1. certified-wheels gate (same contract as ``release github``) +2. ``twine check`` on every wheel - the wheel metadata is generated, so its + long description and classifiers are worth validating before an upload that + cannot be undone +3. refuse if the version already exists on the target index (read-only probe) +4. upload to TestPyPI, then verify by fetching it back from the index +5. stop there unless ``--confirm-pypi`` is given; then repeat 3-4 against PyPI + +PyPI uploads are permanent: a version number can never be reused, and a bad +artifact cannot be replaced. Hence the TestPyPI stage, the read-only existence +probe, and ``--apply`` being required even to reach TestPyPI. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +import tempfile +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[1] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import REPO_ROOT # noqa: E402 +from release.publish import GateResult, PublishGateError, load_gate # noqa: E402 + +INDEX_JSON = { + "testpypi": "https://test.pypi.org/pypi/orisys/json", + "pypi": "https://pypi.org/pypi/orisys/json", +} +INDEX_SIMPLE = { + "testpypi": "https://test.pypi.org/simple/", + "pypi": "https://pypi.org/simple/", +} + + +@dataclass +class Options: + version: str + apply: bool = False + confirm_pypi: bool = False + repository: str = "testpypi" + verify_install: bool = True + skip_check: bool = False + + +@dataclass +class IndexState: + versions: list[str] = field(default_factory=list) + error: str | None = None + + @property + def known(self) -> bool: + return self.error is None + + +def twine_check_argv(wheels: list[Path]) -> list[str]: + return [sys.executable, "-m", "twine", "check", *[str(w) for w in wheels]] + + +def twine_upload_argv(wheels: list[Path], repository: str | None) -> list[str]: + argv = [sys.executable, "-m", "twine", "upload"] + if repository: + argv += ["--repository", repository] + argv += [str(wheel) for wheel in wheels] + return argv + + +def credentials_hint() -> str | None: + """Return guidance when twine has no obvious credentials, else ``None``. + + twine itself reads ``~/.pypirc`` or ``TWINE_USERNAME``/``TWINE_PASSWORD``; + failing here is much better than failing halfway through an upload. + """ + import os + + if os.environ.get("TWINE_PASSWORD") or os.environ.get("TWINE_USERNAME"): + return None + pypirc = Path.home() / ".pypirc" + if pypirc.is_file(): + return None + return ( + "no PyPI credentials found: create ~/.pypirc with [testpypi] and [pypi] " + "sections, or set TWINE_USERNAME=__token__ and TWINE_PASSWORD=. " + "Never commit the token." + ) + + +def index_state(index: str, *, timeout: float = 20.0) -> IndexState: + """Read the published versions of orisys from an index (read-only).""" + url = INDEX_JSON[index] + request = urllib.request.Request( + url, headers={"User-Agent": "orisys-release-publish", "Accept": "application/json"} + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.load(response) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return IndexState(versions=[]) + return IndexState(error=f"{index} returned HTTP {exc.code}") + except Exception as exc: # network, DNS, TLS, JSON + return IndexState(error=f"{index} lookup failed: {type(exc).__name__}: {exc}") + + releases = payload.get("releases") + if not isinstance(releases, dict): + return IndexState(error=f"unexpected {index} response (no 'releases' mapping)") + return IndexState( + versions=sorted(name for name, files in releases.items() if files) + ) + + +def index_files(index: str, version: str, *, timeout: float = 20.0) -> list[str] | None: + """Filenames published for ``version`` on ``index``, or ``None`` if absent.""" + url = INDEX_JSON[index] + request = urllib.request.Request( + url, headers={"User-Agent": "orisys-release-publish", "Accept": "application/json"} + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.load(response) + except Exception: + return None + files = (payload.get("releases") or {}).get(version) + if not isinstance(files, list): + return None + return sorted(item.get("filename", "") for item in files) + + +def _upload(wheels: list[Path], repository: str | None) -> int: + argv = twine_upload_argv(wheels, repository) + print(f"[run ] {' '.join(str(part) for part in argv)}") + completed = subprocess.run(argv, check=False) + return completed.returncode + + +def _fetch_back(index: str, version: str, wheel: Path) -> int: + """Download the freshly uploaded wheel back out of the index.""" + with tempfile.TemporaryDirectory(prefix="orisys-fetch-") as tmp: + target = Path(tmp) / "out" + argv = [ + sys.executable, + "-m", + "pip", + "download", + "--no-deps", + "--no-cache-dir", + "--index-url", + INDEX_SIMPLE[index], + "--dest", + str(target), + f"orisys=={version}", + ] + if index == "testpypi": + # TestPyPI mirrors only what was uploaded there; deps live on PyPI. + # --no-deps keeps this a pure "is the artifact fetchable" probe. + argv += ["--extra-index-url", INDEX_SIMPLE["pypi"]] + print(f"[run ] {' '.join(argv)}") + completed = subprocess.run(argv, check=False) + if completed.returncode != 0: + return completed.returncode + + fetched = sorted(target.glob(f"orisys-{version}*")) + if not fetched: + print(f"[FAIL] pip download produced nothing for orisys=={version}") + return 1 + print(f"[ok ] fetched {[p.name for p in fetched]} from {index}") + if wheel.name not in {path.name for path in fetched}: + print( + f"[warn] {wheel.name} was not the file fetched " + "(the index may serve a different platform first)" + ) + return 0 + + +def publish(options: Options) -> int: + version = options.version + mode = "APPLY" if options.apply else "DRY RUN" + print(f"=== release pypi: orisys {version} ({mode}) ===") + + # --- 1. gate ------------------------------------------------------- + try: + gate: GateResult = load_gate(REPO_ROOT, version) + except PublishGateError as exc: + print(f"[FAIL] {exc}") + return 1 + print(f"[ok ] preflight report: {gate.report_path}") + for wheel in gate.wheels: + print(f" [ok] {wheel.name} sha256={wheel.sha256[:12]}") + for warning in gate.warnings: + print(f"[warn] {warning}") + + wheels = [REPO_ROOT / "SDK" / version / record.name for record in gate.wheels] + missing = [str(wheel) for wheel in wheels if not wheel.is_file()] + if missing: + print(f"[FAIL] wheels missing from disk: {missing}") + return 1 + + # --- 2. credentials + twine check (before anything irreversible) ---- + # Dry runs are previews, so missing credentials warn there and stop only when + # an upload is actually about to happen. + hint = credentials_hint() + if hint: + print(f"[{'FAIL' if options.apply else 'warn'}] {hint}") + if options.apply: + return 1 + else: + print("[ok ] PyPI credentials are configured (twine will read them)") + + if not options.skip_check: + argv = twine_check_argv(wheels) + print(f"[run ] {' '.join(str(part) for part in argv)}") + completed = subprocess.run(argv, check=False) + if completed.returncode != 0: + print("[FAIL] twine check reported problems; fix the wheel metadata first") + return 1 + print("[ok ] twine check passed") + + # --- 3. phase plan ------------------------------------------------- + phases = ["testpypi"] + if options.confirm_pypi: + phases.append("pypi") + + print() + for index in phases: + state = index_state(index) + if not state.known: + print(f"[FAIL] {state.error}; cannot safely check for duplicate uploads") + return 1 + if version in state.versions: + print( + f"[FAIL] orisys {version} already exists on {index} " + f"(published: {', '.join(state.versions) or 'none'}) - " + "an index version cannot be reused" + ) + return 1 + print(f"[ok ] {index}: {version} not present (published: {', '.join(state.versions) or 'none'})") + + print() + for index in phases: + print(f"[plan] upload {len(wheels)} wheel(s) to {index}") + if "pypi" not in phases: + print( + "[plan] stop after TestPyPI. Re-run with " + "`--confirm-pypi --apply` to publish to the real PyPI." + ) + print() + + if not options.apply: + print("dry run: nothing was uploaded. Re-run with --apply to execute.") + return 0 + + # --- 4. upload ---------------------------------------------------- + for index in phases: + if _upload(wheels, index) != 0: + print(f"[FAIL] twine upload to {index} failed") + return 1 + print(f"[ok ] uploaded to {index}") + + published = index_files(index, version) + if published is None: + print( + f"[FAIL] {index} does not serve orisys {version} yet; " + "the upload may still be indexing, verify manually" + ) + return 1 + expected = {wheel.name for wheel in wheels} + if set(published) != expected: + print( + f"[FAIL] {index} file set mismatch:\n expected {sorted(expected)}\n" + f" got {sorted(published)}" + ) + return 1 + print(f"[ok ] {index} serves all {len(published)} file(s) for {version}") + + if options.verify_install: + if _fetch_back(index, version, wheels[0]) != 0: + print(f"[FAIL] {index} artifact is not fetchable") + return 1 + + if "pypi" not in phases: + print() + print("=== release pypi: TestPyPI stage PASSED; PyPI untouched ===") + print("next: python build/run.py release pypi --confirm-pypi --apply") + return 0 + + print() + print("=== release pypi PASSED (TestPyPI + PyPI) ===") + return 0 diff --git a/docs/publishing.md b/docs/publishing.md index db64d3e..77731fa 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -120,24 +120,62 @@ python build/run.py verify --gui-dist --product ory_finger --- -## 3. 发布（目前仍需人工） +## 3. 发布（两条命令） -阶段一不包含自动发布。当前人工步骤： - -1. 用 `release preflight` 拿到 PASS（并保留 `preflight.json`）。 -2. 把 `SDK//` 同步到 `orisys-sdk-release` 仓库的 `release//`。 -3. GitHub：打 tag **`v`** 并创建 Release，上传 6 个 wheel 作为附件。 - - 注意历史 tag 里有一个 typo：`0.50`（与 `v0.5.1` 指向同一 commit）。 -4. PyPI：先 TestPyPI 再正式仓库。 +先决条件：`release preflight` PASS，**而且是在干净提交上跑的**。两个发布命令都只发布「preflight 认证过的东西」——它们会读取 `build/release/reports//preflight.json`，逐个核对 wheel 的 sha256，并拒绝使用带豁免项（`--skip-tests` / `--allow-dirty` / `--no-runtime` / `--force`）的报告。所以正确顺序是： ```powershell -python -m twine upload --repository testpypi SDK//*.whl -python -m twine upload SDK//*.whl +git add -A; git commit -m "..." +python build/run.py wheel-win && python build/run.py manylinux +python build/run.py release preflight --all-wheel-pythons --check-pypi # 干净树 + 无豁免 ``` -**不要把 API token 写进仓库**。`release preflight --check-pypi` 只做只读查询，不需要 token。 +两个命令**默认都是 dry run**，只打印将要执行的动作；必须显式加 `--apply` 才会真正动手。GitHub 的 `gh release create`、`git push`，以及 PyPI 上传（版本号永久占用、不可覆盖）都属于不可撤销操作。 -### 已知的历史发布空洞 +### 3.1 GitHub：同步公开仓库 + 上传 Release + +```powershell +python build/run.py release github --version 0.5.2 # 预览 +python build/run.py release github --version 0.5.2 --apply # 执行 +``` + +执行顺序： + +1. 认证书校验（上面那份报告 + 逐 wheel sha256） +2. `git fetch --tags`，若 `v0.5.2` 或 `0.5.2` 已存在则拒绝；`gh release view` 已存在也拒绝 +3. 按 `tools/downstream_sync_manifest.json` 同步公开内容到 `../orisys-sdk-release`，并运行 manifest 里的 `checks` +4. 把 `SDK/0.5.2/*.whl` 放到 `orisys-sdk-release/release/0.5.2/`（该目录在 release 仓库里是 gitignore 的，只用于上传附件） +5. 提交 → 打 tag `v0.5.2` → push 分支与 tag +6. `gh release create v0.5.2 --verify-tag --latest <6 个 wheel>`，然后回读 `gh release view --json assets` 核对附件齐全 + +需要：`gh` 已安装并登录（`winget install --id GitHub.cli` 然后 `gh auth login`）。没装 gh 时： + +- dry run 仍会完整打印计划（只告警） +- `--apply` 会失败，或改用 `--skip-release-upload`：同步、提交、打 tag、push 都自动完成，然后打印出你自己要跑的 `gh release create` + +其他开关：`--notes-file`（自定义 release notes）、`--attach-preflight`（把 `preflight.json` 也作为附件上传，便于审计；注意报告里有本机绝对路径）、`--allow-dirty-release`、`--skip-fetch`、`--skip-checks`。 + +> tag 拼写沿用历史风格 `v`（注意历史里那个 typo tag `0.50` 指向 `v0.5.1`）。命令会在动手前拒绝已存在的 tag，所以不会重复发布。 + +### 3.2 PyPI：TestPyPI 先行，确认后再上正式仓库 + +```powershell +python build/run.py release pypi --version 0.5.2 # 预览 +python build/run.py release pypi --version 0.5.2 --apply # 只发 TestPyPI +python build/run.py release pypi --version 0.5.2 --confirm-pypi --apply # TestPyPI + 正式 PyPI +``` + +执行顺序： + +1. 同样的证书校验 +2. `twine check` 全部 wheel（校验 METADATA/long description，上传前就能发现问题） +3. 只读查询目标索引，确认该版本不存在（TestPyPI 与 PyPI 各查各的） +4. 上传 TestPyPI → 回读索引确认 6 个文件都在 → `pip download` 把刚上传的 wheel 拉回来，证明产物可获取 +5. 只有加了 `--confirm-pypi` 才会对正式 PyPI 重复第 3-4 步；否则停在这里并打印后续命令 + +凭证由 twine 自己解析：`~/.pypirc` 的 `[testpypi]` / `[pypi]` 段，或 `TWINE_USERNAME=__token__` + `TWINE_PASSWORD=`。命令会在**任何写操作之前**检查凭证是否配置好并给出提示。**不要把 token 写进仓库。** + +### 3.3 已知的历史发布空洞 | 版本 | 本地 `SDK/` | PyPI | GitHub Release | | --- | --- | --- | --- | diff --git a/tests/test_release_publish.py b/tests/test_release_publish.py new file mode 100644 index 0000000..c30799f --- /dev/null +++ b/tests/test_release_publish.py @@ -0,0 +1,312 @@ +"""Tests for the publish gate, the GitHub plan and the PyPI command shape. + +Nothing here touches the network, ``gh`` or an index: the mutating paths are +verified through their pure planning functions, which is exactly what the CLI +executes after the gate passes. +""" + +from __future__ import annotations + +import json +import sys +import urllib.error +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +BUILD_ROOT = REPO_ROOT / "build" +if str(BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(BUILD_ROOT)) + +from release import github, publish, pypi # noqa: E402 + + +def _wheel(tmp_path: Path, name: str, payload: bytes = b"wheel") -> Path: + path = tmp_path / name + path.write_bytes(payload) + return path + + +def _report_payload(version: str, wheels: list[Path], **overrides) -> dict: + payload = { + "schema": 1, + "version": version, + "ok": True, + "aborted": False, + "git": {"commit": "a" * 40}, + "options": {"skip_tests": False, "allow_dirty": False}, + "wheels": [ + {"name": wheel.name, "sha256": publish.sha256_file(wheel), "size": wheel.stat().st_size} + for wheel in wheels + ], + } + payload.update(overrides) + return payload + + +# --- gate ------------------------------------------------------------------- + + +def test_gate_passes_for_matching_wheels(tmp_path): + wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")] + problems, warnings = publish.evaluate_gate( + _report_payload("1.2.3", wheels), "1.2.3", {w.name: w for w in wheels} + ) + assert problems == [] + assert warnings == [] + + +def test_gate_rejects_version_mismatch(tmp_path): + wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")] + problems, _ = publish.evaluate_gate( + _report_payload("9.9.9", wheels), "1.2.3", {w.name: w for w in wheels} + ) + assert any("report is for version" in problem for problem in problems) + + +def test_gate_rejects_failed_report(tmp_path): + wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")] + problems, _ = publish.evaluate_gate( + _report_payload("1.2.3", wheels, ok=False), "1.2.3", {w.name: w for w in wheels} + ) + assert any("result is FAIL" in problem for problem in problems) + + +def test_gate_rejects_waived_guard_options(tmp_path): + wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")] + payload = _report_payload("1.2.3", wheels, options={"skip_tests": True, "no_runtime": True}) + problems, _ = publish.evaluate_gate(payload, "1.2.3", {w.name: w for w in wheels}) + assert any("--skip-tests" in problem and "--no-runtime" in problem for problem in problems) + + +def test_gate_rejects_changed_wheel(tmp_path): + wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")] + payload = _report_payload("1.2.3", wheels) + wheels[0].write_bytes(b"tampered") + problems, _ = publish.evaluate_gate(payload, "1.2.3", {w.name: w for w in wheels}) + assert any("changed after preflight" in problem for problem in problems) + + +def test_gate_rejects_uncertified_and_missing_wheels(tmp_path): + wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")] + payload = _report_payload("1.2.3", wheels) + extra = _wheel(tmp_path, "orisys-1.2.3-cp311-cp311-win_amd64.whl") + problems, _ = publish.evaluate_gate( + payload, "1.2.3", {w.name: w for w in [*wheels, extra]} + ) + assert any("not covered by the report" in problem for problem in problems) + + +def test_gate_warns_on_commit_drift(tmp_path): + wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")] + payload = _report_payload("1.2.3", wheels) + problems, warnings = publish.evaluate_gate( + payload, "1.2.3", {w.name: w for w in wheels}, head_commit="b" * 40 + ) + assert problems == [] + assert any("HEAD is" in warning for warning in warnings) + + +def test_load_gate_requires_a_report(tmp_path, monkeypatch): + monkeypatch.setattr(publish, "REPO_ROOT", tmp_path, raising=False) + with pytest.raises(publish.PublishGateError, match="no preflight report"): + publish.load_gate(tmp_path, "1.2.3") + + +def test_load_gate_explains_waived_report(tmp_path): + report_dir = tmp_path / "build" / "release" / "reports" / "1.2.3" + report_dir.mkdir(parents=True) + wheel_dir = tmp_path / "SDK" / "1.2.3" + wheel_dir.mkdir(parents=True) + wheel = wheel_dir / "orisys-1.2.3-cp310-cp310-win_amd64.whl" + wheel.write_bytes(b"wheel") + (report_dir / "preflight.json").write_text( + json.dumps(_report_payload("1.2.3", [wheel], options={"skip_tests": True})), + encoding="utf-8", + ) + with pytest.raises(publish.PublishGateError, match="--skip-tests"): + publish.load_gate(tmp_path, "1.2.3") + + +def test_load_gate_returns_certified_wheels(tmp_path): + report_dir = tmp_path / "build" / "release" / "reports" / "1.2.3" + report_dir.mkdir(parents=True) + wheel_dir = tmp_path / "SDK" / "1.2.3" + wheel_dir.mkdir(parents=True) + wheel = wheel_dir / "orisys-1.2.3-cp310-cp310-win_amd64.whl" + wheel.write_bytes(b"wheel") + (report_dir / "preflight.json").write_text( + json.dumps(_report_payload("1.2.3", [wheel])), encoding="utf-8" + ) + gate = publish.load_gate(tmp_path, "1.2.3") + assert [record.name for record in gate.wheels] == [wheel.name] + assert gate.wheels[0].sha256 == publish.sha256_file(wheel) + assert gate.git_commit == "a" * 40 + + +# --- github ----------------------------------------------------------------- + + +def test_github_steps_shape(tmp_path): + assets = [_wheel(tmp_path, f"orisys-1.2.3-cp3n{n}-cp3n{n}-win_amd64.whl") for n in (0, 1)] + steps = github.build_github_steps( + version="1.2.3", + branch="main", + tag="v1.2.3", + assets=assets, + notes_file=None, + ) + by_description = {step.description: step.argv for step in steps} + assert by_description["create tag"] == ["git", "tag", "v1.2.3"] + assert by_description["push branch"] == ["git", "push", "origin", "main"] + assert by_description["push tag"] == ["git", "push", "origin", "v1.2.3"] + + release_argv = by_description["create GitHub Release with assets"] + assert release_argv[:2] == ["gh", "release"] + assert "v1.2.3" in release_argv + assert "--verify-tag" in release_argv + assert "--latest" in release_argv + assert release_argv[-2:] == [str(asset) for asset in assets] + + +def test_github_steps_include_notes_file(tmp_path): + notes = tmp_path / "notes.md" + notes.write_text("hello", encoding="utf-8") + steps = github.build_github_steps( + version="1.2.3", branch="main", tag="v1.2.3", assets=[], notes_file=notes + ) + release_argv = next( + step.argv for step in steps if step.description.startswith("create GitHub Release") + ) + assert "--notes-file" in release_argv + assert str(notes) in release_argv + + +def test_release_notes_list_every_wheel(): + notes = github.build_release_notes( + "1.2.3", [Path("a.whl"), Path("b.whl")], "c" * 40 + ) + assert "a.whl" in notes and "b.whl" in notes + assert "cccccccccccc" in notes + + +def test_manifest_scope_excludes_internal_docs(): + """The public repo must not receive our internal process documents.""" + entry = github.manifest_entry() + docs_entry = next(item for item in entry["sync"] if item["from"] == "docs") + assert set(docs_entry["include"]) == {"get_started.md", "api.md"} + assert "publishing.md" not in docs_entry["include"] + assert "pypi_readme.md" not in docs_entry["include"] + + +def test_manifest_scope_covers_examples_and_sensor_json_only(): + entry = github.manifest_entry() + examples_entry = next(item for item in entry["sync"] if item["from"] == "examples") + # Whole tree (no include list), with caches excluded. + assert "include" not in examples_entry + assert set(examples_entry["exclude"]) >= {"__pycache__", "*.pyc"} + + sensors_entry = next(item for item in entry["sync"] if item["from"] == "sensors") + assert sensors_entry["include"] + assert all( + name.endswith(".json") for name in sensors_entry["include"] + ), "binary sensor assets belong in the wheel, not the public repo" + + +def test_manifest_checks_use_the_python_placeholder(): + entry = github.manifest_entry() + for command in entry["checks"]: + assert command.startswith("{python} "), command + + +# --- pypi ------------------------------------------------------------------- + + +def test_twine_upload_argv_targets_the_repository(): + argv = pypi.twine_upload_argv([Path("a.whl")], "testpypi") + assert argv[:3] == [sys.executable, "-m", "twine"] + assert "--repository" in argv and "testpypi" in argv + assert argv[-1] == "a.whl" + + +def test_twine_upload_argv_defaults_to_pypi(): + argv = pypi.twine_upload_argv([Path("a.whl")], None) + assert "--repository" not in argv + + +def test_credentials_hint_reports_missing(monkeypatch, tmp_path): + monkeypatch.delenv("TWINE_PASSWORD", raising=False) + monkeypatch.delenv("TWINE_USERNAME", raising=False) + monkeypatch.setattr(pypi.Path, "home", lambda: tmp_path) + hint = pypi.credentials_hint() + assert hint is not None + assert "TWINE_PASSWORD" in hint + + +def test_credentials_hint_accepts_env(monkeypatch): + monkeypatch.setenv("TWINE_USERNAME", "__token__") + monkeypatch.setenv("TWINE_PASSWORD", "pypi-secret") + assert pypi.credentials_hint() is None + + +def test_credentials_hint_accepts_pypirc(monkeypatch, tmp_path): + monkeypatch.delenv("TWINE_PASSWORD", raising=False) + monkeypatch.delenv("TWINE_USERNAME", raising=False) + (tmp_path / ".pypirc").write_text("[pypi]\n", encoding="utf-8") + monkeypatch.setattr(pypi.Path, "home", lambda: tmp_path) + assert pypi.credentials_hint() is None + + +def test_index_state_parses_versions(monkeypatch): + class Response: + def __init__(self, payload): + self._payload = json.dumps(payload).encode() + + def read(self, *args): + return self._payload + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + monkeypatch.setattr( + pypi.urllib.request, + "urlopen", + lambda *a, **k: Response({"releases": {"1.2.3": [{"filename": "a"}], "0.0.0": []}}), + ) + state = pypi.index_state("pypi") + assert state.known + assert state.versions == ["1.2.3"] + + +def test_index_state_reports_lookup_failure(monkeypatch): + def _raise(*args, **kwargs): + raise urllib.error.URLError("offline") + + monkeypatch.setattr(pypi.urllib.request, "urlopen", _raise) + state = pypi.index_state("pypi") + assert not state.known + assert "offline" in state.error + + +def test_index_files_returns_none_when_version_absent(monkeypatch): + class Response: + def read(self, *args): + return json.dumps({"releases": {}}).encode() + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + monkeypatch.setattr(pypi.urllib.request, "urlopen", lambda *a, **k: Response()) + assert pypi.index_files("pypi", "1.2.3") is None + + +def test_index_json_urls_cover_both_indexes(): + assert pypi.INDEX_JSON["testpypi"].startswith("https://test.pypi.org/") + assert pypi.INDEX_JSON["pypi"].startswith("https://pypi.org/") diff --git a/tools/downstream_sync_manifest.json b/tools/downstream_sync_manifest.json index 3b50f09..7ff2659 100644 --- a/tools/downstream_sync_manifest.json +++ b/tools/downstream_sync_manifest.json @@ -5,21 +5,21 @@ "name": "orisys-sdk-release", "path": "../orisys-sdk-release", "sync": [ - {"from": "docs", "to": "docs"}, - {"from": "examples", "to": "examples", "include": [ - "README.md", - "basic_pipeline.py", - "mini_pipeline.py", - "finger_pipeline.py", - "finger_pipeline_3dview.py", - "plot_data.py", - "two_sensor_pipeline.py" - ]}, - {"from": "config", "to": "config", "include": ["box.json", "finger.json", "mini.json"]} + {"from": "docs", "to": "docs", "include": ["get_started.md", "api.md"]}, + {"from": "examples", "to": "examples", "exclude": ["__pycache__", "*.pyc", "*.pyo"]}, + {"from": "config", "to": "config", "include": ["box.json", "finger.json", "mini.json"]}, + {"from": "sensors", "to": "sensors", "include": [ + "box/sensor.json", + "box/force_calibration.json", + "finger/sensor.json", + "finger/manifest.json", + "mini/sensor.json" + ]} ], "checks": [ - "python -m py_compile examples/basic_pipeline.py examples/finger_pipeline.py examples/finger_pipeline_3dview.py examples/plot_data.py", - "python -c \"import pathlib; p=pathlib.Path('.'); hits=[f for f in p.rglob('*') if f.is_file() and 'examples/products' in f.read_text(encoding='utf-8', errors='ignore').replace('\\\\','/')]; print('stale products paths:' if hits else 'no stale products paths'); [print(x) for x in hits]\"" + "{python} -c \"import pathlib, py_compile; files=sorted(pathlib.Path('examples').rglob('*.py')); [py_compile.compile(str(f), doraise=True) for f in files]; print('compiled %d example module(s)' % len(files))\"", + "{python} -c \"import pathlib, sys; bad=[str(p) for p in pathlib.Path('.').rglob('*') if p.is_file() and p.suffix in ('.md','.py') and 'examples/products' in p.read_text(encoding='utf-8', errors='ignore').replace(chr(92),'/')]; print('stale products paths:' if bad else 'no stale products paths'); [print(' ', x) for x in bad]; sys.exit(1 if bad else 0)\"", + "{python} -c \"import pathlib, sys; leaked=[str(p) for p in pathlib.Path('.').rglob('*') if p.name in ('publishing.md','pypi_readme.md')]; print('internal docs leaked into the public repo:', leaked); sys.exit(1 if leaked else 0)\"" ] }, { @@ -29,7 +29,7 @@ {"from": "config", "to": "config", "include": ["box.json", "mini.json", "dd06-ov-0810.json"]} ], "checks": [ - "python tests/test_smoke.py" + "{python} tests/test_smoke.py" ] }, { @@ -40,8 +40,8 @@ {"from": "assets/finger", "to": "assets/finger"} ], "checks": [ - "python tests/test_smoke.py", - "python -c \"import pathlib; t=pathlib.Path('README.md').read_text(encoding='utf-8'); print('README still mentions examples/products' if 'examples/products' in t else 'README products path ok')\"" + "{python} tests/test_smoke.py", + "{python} -c \"import pathlib; t=pathlib.Path('README.md').read_text(encoding='utf-8'); print('README still mentions examples/products' if 'examples/products' in t else 'README products path ok')\"" ] } ] diff --git a/tools/sync_downstreams.py b/tools/sync_downstreams.py index c4f78aa..c412480 100644 --- a/tools/sync_downstreams.py +++ b/tools/sync_downstreams.py @@ -2,7 +2,9 @@ """Plan and apply downstream file syncs from tools/downstream_sync_manifest.json. The ``apply`` command copies files/directories listed in the manifest. It does -not delete extra downstream files and does not run downstream tests. +not delete extra downstream files and does not run downstream tests by default; +``run-checks`` (or ``release github``) runs the manifest's ``checks`` commands +with ``{python}`` replaced by the running interpreter. """ from __future__ import annotations @@ -10,13 +12,24 @@ from __future__ import annotations import argparse import json import shutil +import subprocess import sys +from dataclasses import dataclass from pathlib import Path MANIFEST_NAME = "downstream_sync_manifest.json" +@dataclass(frozen=True) +class SyncJob: + """One source -> destination copy, with optional name patterns to skip.""" + + source: Path + dest: Path + exclude: tuple[str, ...] = () + + def sdk_root() -> Path: return Path(__file__).resolve().parent.parent @@ -43,18 +56,19 @@ def selected_downstreams(manifest: dict, target: str | None) -> list[dict]: return matches -def expand_jobs(root: Path, item: dict) -> list[tuple[Path, Path]]: +def expand_jobs(root: Path, item: dict) -> list[SyncJob]: dest_root = (root / item["path"]).resolve() - jobs: list[tuple[Path, Path]] = [] + jobs: list[SyncJob] = [] for entry in item.get("sync", []): src_base = root / entry["from"] dest_base = dest_root / entry["to"] + exclude = tuple(entry.get("exclude") or ()) includes = entry.get("include") if includes: for name in includes: - jobs.append((src_base / name, dest_base / name)) + jobs.append(SyncJob(src_base / name, dest_base / name, exclude)) else: - jobs.append((src_base, dest_base)) + jobs.append(SyncJob(src_base, dest_base, exclude)) return jobs @@ -68,8 +82,9 @@ def print_plan(root: Path, items: list[dict]) -> None: jobs = expand_jobs(root, item) if jobs: print("sync:") - for src, dest in jobs: - print(f" {src} -> {dest}") + for job in jobs: + note = f" (excluding {', '.join(job.exclude)})" if job.exclude else "" + print(f" {job.source} -> {job.dest}{note}") else: print("sync: (none)") checks = item.get("checks") or [] @@ -94,14 +109,14 @@ def check_paths(root: Path, items: list[dict]) -> int: else: print(f"MISS downstream root: {dest_root}") missing.append(str(dest_root)) - for src, dest in expand_jobs(root, item): - src_ok = src.exists() - dest_note = "exists" if dest.exists() else "missing (ok until apply)" + for job in expand_jobs(root, item): + src_ok = job.source.exists() + dest_note = "exists" if job.dest.exists() else "missing (ok until apply)" status = "ok " if src_ok else "MISS" - print(f"{status} source: {src}") - print(f" dest: {dest} [{dest_note}]") + print(f"{status} source: {job.source}") + print(f" dest: {job.dest} [{dest_note}]") if not src_ok: - missing.append(str(src)) + missing.append(str(job.source)) print() if missing: print(f"{len(missing)} missing path(s).") @@ -110,14 +125,16 @@ def check_paths(root: Path, items: list[dict]) -> int: return 0 -def copy_job(src: Path, dest: Path) -> None: - if not src.exists(): - raise FileNotFoundError(src) - if src.is_dir(): - shutil.copytree(src, dest, dirs_exist_ok=True) +def copy_job(job: SyncJob) -> None: + if not job.source.exists(): + raise FileNotFoundError(job.source) + if job.source.is_dir(): + # exclude keeps caches out of the public repo (e.g. __pycache__, *.pyc) + ignore = shutil.ignore_patterns(*job.exclude) if job.exclude else None + shutil.copytree(job.source, job.dest, dirs_exist_ok=True, ignore=ignore) return - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, dest) + job.dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(job.source, job.dest) def apply_sync(root: Path, items: list[dict]) -> int: @@ -128,16 +145,45 @@ def apply_sync(root: Path, items: list[dict]) -> int: copied = 0 for item in items: print(f"== {item['name']} ==") - for src, dest in expand_jobs(root, item): - copy_job(src, dest) + for job in expand_jobs(root, item): + copy_job(job) copied += 1 - print(f"copied {src} -> {dest}") + print(f"copied {job.source} -> {job.dest}") print() print(f"copied {copied} sync job(s).") print("checks were not run; execute the listed check commands manually.") return 0 +def run_checks(root: Path, items: list[dict], *, python: str | None = None) -> int: + """Run each manifest ``checks`` command inside the downstream repo. + + ``{python}`` in a command is replaced with ``python`` (or the interpreter + given). Returns the number of failed checks. + """ + interpreter = python or sys.executable + failures = 0 + for item in items: + dest_root = (root / item["path"]).resolve() + checks = item.get("checks") or [] + if not checks: + continue + print(f"== {item['name']} ==") + for command in checks: + resolved = command.replace("{python}", interpreter) + print(f"running: {resolved}") + completed = subprocess.run( + resolved, shell=True, cwd=dest_root, check=False + ) + if completed.returncode != 0: + print(f"[FAIL] exit {completed.returncode}: {resolved}") + failures += 1 + else: + print("[ok ] check passed") + print() + return failures + + def parse_args(argv: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="command", required=True) @@ -151,6 +197,9 @@ def parse_args(argv: list[str]) -> argparse.Namespace: apply_cmd = sub.add_parser("apply", help="copy manifest sources to downstreams") apply_cmd.add_argument("--target", help="limit to one downstream name") + run = sub.add_parser("run-checks", help="run the manifest checks in each downstream") + run.add_argument("--target", help="limit to one downstream name") + return parser.parse_args(argv) @@ -165,6 +214,8 @@ def main(argv: list[str] | None = None) -> int: return check_paths(root, items) if args.command == "apply": return apply_sync(root, items) + if args.command == "run-checks": + return 1 if run_checks(root, items) else 0 raise SystemExit(f"unknown command: {args.command}")