automate publishing: release github + release pypi

tools/sync_downstreams.py

@@ -2,7 +2,9 @@
 """Plan and apply downstream file syncs from tools/downstream_sync_manifest.json.
 
 The ``apply`` command copies files/directories listed in the manifest. It does
-not delete extra downstream files and does not run downstream tests.
+not delete extra downstream files and does not run downstream tests by default;
+``run-checks`` (or ``release github``) runs the manifest's ``checks`` commands
+with ``{python}`` replaced by the running interpreter.
 """
 
 from __future__ import annotations
@@ -10,13 +12,24 @@ from __future__ import annotations
 import argparse
 import json
 import shutil
+import subprocess
 import sys
+from dataclasses import dataclass
 from pathlib import Path
 
 
 MANIFEST_NAME = "downstream_sync_manifest.json"
 
 
+@dataclass(frozen=True)
+class SyncJob:
+    """One source -> destination copy, with optional name patterns to skip."""
+
+    source: Path
+    dest: Path
+    exclude: tuple[str, ...] = ()
+
+
 def sdk_root() -> Path:
     return Path(__file__).resolve().parent.parent
 
@@ -43,18 +56,19 @@ def selected_downstreams(manifest: dict, target: str | None) -> list[dict]:
     return matches
 
 
-def expand_jobs(root: Path, item: dict) -> list[tuple[Path, Path]]:
+def expand_jobs(root: Path, item: dict) -> list[SyncJob]:
     dest_root = (root / item["path"]).resolve()
-    jobs: list[tuple[Path, Path]] = []
+    jobs: list[SyncJob] = []
     for entry in item.get("sync", []):
         src_base = root / entry["from"]
         dest_base = dest_root / entry["to"]
+        exclude = tuple(entry.get("exclude") or ())
         includes = entry.get("include")
         if includes:
             for name in includes:
-                jobs.append((src_base / name, dest_base / name))
+                jobs.append(SyncJob(src_base / name, dest_base / name, exclude))
         else:
-            jobs.append((src_base, dest_base))
+            jobs.append(SyncJob(src_base, dest_base, exclude))
     return jobs
 
 
@@ -68,8 +82,9 @@ def print_plan(root: Path, items: list[dict]) -> None:
         jobs = expand_jobs(root, item)
         if jobs:
             print("sync:")
-            for src, dest in jobs:
-                print(f"  {src} -> {dest}")
+            for job in jobs:
+                note = f"  (excluding {', '.join(job.exclude)})" if job.exclude else ""
+                print(f"  {job.source} -> {job.dest}{note}")
         else:
             print("sync: (none)")
         checks = item.get("checks") or []
@@ -94,14 +109,14 @@ def check_paths(root: Path, items: list[dict]) -> int:
         else:
             print(f"MISS downstream root: {dest_root}")
             missing.append(str(dest_root))
-        for src, dest in expand_jobs(root, item):
-            src_ok = src.exists()
-            dest_note = "exists" if dest.exists() else "missing (ok until apply)"
+        for job in expand_jobs(root, item):
+            src_ok = job.source.exists()
+            dest_note = "exists" if job.dest.exists() else "missing (ok until apply)"
             status = "ok  " if src_ok else "MISS"
-            print(f"{status} source: {src}")
-            print(f"     dest:   {dest} [{dest_note}]")
+            print(f"{status} source: {job.source}")
+            print(f"     dest:   {job.dest} [{dest_note}]")
             if not src_ok:
-                missing.append(str(src))
+                missing.append(str(job.source))
         print()
     if missing:
         print(f"{len(missing)} missing path(s).")
@@ -110,14 +125,16 @@ def check_paths(root: Path, items: list[dict]) -> int:
     return 0
 
 
-def copy_job(src: Path, dest: Path) -> None:
-    if not src.exists():
-        raise FileNotFoundError(src)
-    if src.is_dir():
-        shutil.copytree(src, dest, dirs_exist_ok=True)
+def copy_job(job: SyncJob) -> None:
+    if not job.source.exists():
+        raise FileNotFoundError(job.source)
+    if job.source.is_dir():
+        # exclude keeps caches out of the public repo (e.g. __pycache__, *.pyc)
+        ignore = shutil.ignore_patterns(*job.exclude) if job.exclude else None
+        shutil.copytree(job.source, job.dest, dirs_exist_ok=True, ignore=ignore)
         return
-    dest.parent.mkdir(parents=True, exist_ok=True)
-    shutil.copy2(src, dest)
+    job.dest.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(job.source, job.dest)
 
 
 def apply_sync(root: Path, items: list[dict]) -> int:
@@ -128,16 +145,45 @@ def apply_sync(root: Path, items: list[dict]) -> int:
     copied = 0
     for item in items:
         print(f"== {item['name']} ==")
-        for src, dest in expand_jobs(root, item):
-            copy_job(src, dest)
+        for job in expand_jobs(root, item):
+            copy_job(job)
             copied += 1
-            print(f"copied {src} -> {dest}")
+            print(f"copied {job.source} -> {job.dest}")
         print()
     print(f"copied {copied} sync job(s).")
     print("checks were not run; execute the listed check commands manually.")
     return 0
 
 
+def run_checks(root: Path, items: list[dict], *, python: str | None = None) -> int:
+    """Run each manifest ``checks`` command inside the downstream repo.
+
+    ``{python}`` in a command is replaced with ``python`` (or the interpreter
+    given). Returns the number of failed checks.
+    """
+    interpreter = python or sys.executable
+    failures = 0
+    for item in items:
+        dest_root = (root / item["path"]).resolve()
+        checks = item.get("checks") or []
+        if not checks:
+            continue
+        print(f"== {item['name']} ==")
+        for command in checks:
+            resolved = command.replace("{python}", interpreter)
+            print(f"running: {resolved}")
+            completed = subprocess.run(
+                resolved, shell=True, cwd=dest_root, check=False
+            )
+            if completed.returncode != 0:
+                print(f"[FAIL] exit {completed.returncode}: {resolved}")
+                failures += 1
+            else:
+                print("[ok  ] check passed")
+        print()
+    return failures
+
+
 def parse_args(argv: list[str]) -> argparse.Namespace:
     parser = argparse.ArgumentParser(description=__doc__)
     sub = parser.add_subparsers(dest="command", required=True)
@@ -151,6 +197,9 @@ def parse_args(argv: list[str]) -> argparse.Namespace:
     apply_cmd = sub.add_parser("apply", help="copy manifest sources to downstreams")
     apply_cmd.add_argument("--target", help="limit to one downstream name")
 
+    run = sub.add_parser("run-checks", help="run the manifest checks in each downstream")
+    run.add_argument("--target", help="limit to one downstream name")
+
     return parser.parse_args(argv)
 
 
@@ -165,6 +214,8 @@ def main(argv: list[str] | None = None) -> int:
         return check_paths(root, items)
     if args.command == "apply":
         return apply_sync(root, items)
+    if args.command == "run-checks":
+        return 1 if run_checks(root, items) else 0
     raise SystemExit(f"unknown command: {args.command}")