From 5f78115429e9590d4dc759279f1816743018002d Mon Sep 17 00:00:00 2001 From: ykcheungab Date: Sat, 12 Sep 2026 17:43:40 +0800 Subject: [PATCH] fix pypi cannot check after upload --- build/release/pypi.py | 127 ++++++++++++++++++++++++++++++---- docs/publishing.md | 9 ++- tests/test_release_publish.py | 117 ++++++++++++++++++++++++++++++- 3 files changed, 237 insertions(+), 16 deletions(-) diff --git a/build/release/pypi.py b/build/release/pypi.py index 6f205f4..98e7355 100644 --- a/build/release/pypi.py +++ b/build/release/pypi.py @@ -22,6 +22,7 @@ import shutil import subprocess import sys import tempfile +import time import urllib.error import urllib.request from dataclasses import dataclass, field @@ -182,8 +183,18 @@ def index_state(index: str, *, timeout: float = 20.0) -> IndexState: ) -def index_files(index: str, version: str, *, timeout: float = 20.0) -> list[str] | None: - """Filenames published for ``version`` on ``index``, or ``None`` if absent.""" +def index_files( + index: str, version: str, *, timeout: float = 20.0 +) -> tuple[list[str] | None, str | None]: + """Filenames published for ``version``, as ``(filenames, error)``. + + ``(None, None)`` - the index does not serve this version (yet) + ``(None, "why")`` - the lookup itself failed, which is NOT evidence of absence + ``([...], None)`` - the version is published with those files + + Keeping those apart matters: reporting a network hiccup as "not published + yet" is how a successful upload gets mistaken for a failure. + """ url = INDEX_JSON[index] request = urllib.request.Request( url, headers={"User-Agent": "orisys-release-publish", "Accept": "application/json"} @@ -191,12 +202,77 @@ def index_files(index: str, version: str, *, timeout: float = 20.0) -> list[str] try: with urllib.request.urlopen(request, timeout=timeout) as response: payload = json.load(response) - except Exception: - return None + except urllib.error.HTTPError as exc: + if exc.code == 404: + return None, None + return None, f"{index} returned HTTP {exc.code}" + except Exception as exc: + return None, f"{index} lookup failed: {type(exc).__name__}: {exc}" + files = (payload.get("releases") or {}).get(version) - if not isinstance(files, list): - return None - return sorted(item.get("filename", "") for item in files) + if not isinstance(files, list) or not files: + return None, None + return sorted(str(item.get("filename", "")) for item in files), None + + +def wait_for_release( + index: str, + version: str, + *, + attempts: int = 6, + delay: float = 5.0, + sleep=None, + timeout: float = 20.0, +) -> tuple[list[str] | None, str | None]: + """Poll until the index serves ``version`` (uploads are not instantly visible). + + TestPyPI in particular lags the upload response by seconds. Returns the last + ``(filenames, error)`` observed. + """ + sleep = sleep or time.sleep + last_error: str | None = None + for attempt in range(1, max(1, attempts) + 1): + filenames, error = index_files(index, version, timeout=timeout) + if filenames is not None: + return filenames, None + last_error = error + if attempt < attempts: + print( + f"[info] {index} does not serve {version} yet " + f"({'lookup failed' if error else 'indexing'}), retry {attempt}/{attempts - 1}", + flush=True, + ) + sleep(delay) + return None, last_error + + +def existing_release_state(index: str, version: str, expected: set[str]) -> str: + """``"satisfied"`` | ``"conflict"`` | ``"absent"`` for this index and version. + + An index version can never be replaced, so "already published with exactly + these files" is the only successful terminal state; treating it as an error + would make a re-run after a partial failure look like a new failure. A + different file set is a genuine conflict and must stop the run. + """ + published, error = index_files(index, version) + if published is None: + if error: + print(f"[warn] could not check {index} for an existing release: {error}") + return "absent" + if set(published) == expected: + print( + f"[ok ] {index} already serves orisys {version} with all " + f"{len(published)} file(s)" + ) + return "satisfied" + extra = sorted(set(published) - expected) + missing = sorted(expected - set(published)) + print( + f"[FAIL] {index} already serves orisys {version} with a different file set" + + (f"; unexpected: {extra}" if extra else "") + + (f"; missing: {missing}" if missing else "") + ) + return "conflict" def _upload(wheels: list[Path], repository: str | None) -> int: @@ -288,6 +364,16 @@ def publish(options: Options) -> int: print("[ok ] twine check passed") # --- 3. phase plan ------------------------------------------------- + expected = {wheel.name for wheel in wheels} + remaining: list[str] = [] + for index in phases: + state = existing_release_state(index, version, expected) + if state == "conflict": + return 1 + if state == "absent": + remaining.append(index) + phases = remaining + print() for index in phases: state = index_state(index) @@ -303,6 +389,11 @@ def publish(options: Options) -> int: return 1 print(f"[ok ] {index}: {version} not present (published: {', '.join(state.versions) or 'none'})") + if not phases: + print() + print("=== release pypi: every requested index already has this release ===") + return 0 + print() for index in phases: print(f"[plan] upload {len(wheels)} wheel(s) to {index}") @@ -324,14 +415,18 @@ def publish(options: Options) -> int: return 1 print(f"[ok ] uploaded to {index}") - published = index_files(index, version) + published, error = wait_for_release(index, version) if published is None: + # The upload itself succeeded, so this is a confirmation problem, not + # a publishing failure. Say so plainly and do not imply data loss. + reason = f"lookup kept failing ({error})" if error else "the index has not indexed it yet" print( - f"[FAIL] {index} does not serve orisys {version} yet; " - "the upload may still be indexing, verify manually" + f"[warn] uploaded, but could not confirm it on {index} yet: {reason}.\n" + f" check https://{'test.' if index == 'testpypi' else ''}pypi.org/project/orisys/{version}/" + "\n re-running this command is safe: an already-published version is skipped" ) - return 1 - expected = {wheel.name for wheel in wheels} + print(f"\n=== release pypi: {index} upload SUCCEEDED (verification inconclusive) ===") + return 0 if set(published) != expected: print( f"[FAIL] {index} file set mismatch:\n expected {sorted(expected)}\n" @@ -342,8 +437,12 @@ def publish(options: Options) -> int: if options.verify_install: if _fetch_back(index, version, wheels[0]) != 0: - print(f"[FAIL] {index} artifact is not fetchable") - return 1 + print( + f"[warn] {index} artifact is not fetchable yet; the upload is " + "published, so this is a propagation delay rather than a failure" + ) + else: + print(f"[ok ] fetched the published artifact back from {index}") if phases == ["testpypi"]: print() diff --git a/docs/publishing.md b/docs/publishing.md index cda6179..496e9f0 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -175,9 +175,16 @@ python build/run.py release pypi --version 0.5.2 --pypi-only --apply 1. 认证书校验（与 `release github` 同一份报告） 2. `twine check` 全部 wheel（校验 METADATA/long description，上传前就能发现问题） 3. 只读查询目标索引，确认该版本不存在（TestPyPI 与 PyPI 各查各的） -4. 上传 → 回读索引确认 6 个文件都在 → `pip download` 把刚上传的 wheel 拉回来，证明产物可获取 +4. 上传 → **轮询**索引确认 6 个文件都在 → `pip download` 把刚上传的 wheel 拉回来，证明产物可获取 5. 未加 `--confirm-pypi` 时只发 TestPyPI，然后停在这里并打印后续命令 +两个针对「索引不即时一致」的设计（TestPyPI 尤其明显，上传成功后 JSON API 可能滞后若干秒）： + +- **上传后的确认会重试**（默认 6 次 × 5 秒）。确认不了也**不算失败**：twine 上传成功就说明产物已发布，命令会打印警告、给出项目页链接并以 0 退出，不会把一个成功的上传报成失败。真正的失败只有两种：twine 上传本身失败，或索引上的文件集与本地不一致。 +- **重复运行是安全的**：若索引上已经存在该版本且文件集完全一致，该阶段会被视为「已完成」而跳过（`[ok] testpypi already serves orisys 0.5.2 with all 6 file(s)`），不会重传、也不报错。这让你在确认不明确时可以直接重跑。如果文件集**不一致**（索引上是另一次构建），才会失败并停下。 + +所以「上传后确认失败」不等于要重新上传——索引版本号不可覆盖，重跑只会做一次只读核对。 + ### 3.3 凭证：`release auth` 与 `~/.pypirc` ```powershell diff --git a/tests/test_release_publish.py b/tests/test_release_publish.py index 2fde341..ccdf3cc 100644 --- a/tests/test_release_publish.py +++ b/tests/test_release_publish.py @@ -501,7 +501,122 @@ def test_index_files_returns_none_when_version_absent(monkeypatch): return False monkeypatch.setattr(pypi.urllib.request, "urlopen", lambda *a, **k: Response()) - assert pypi.index_files("pypi", "1.2.3") is None + assert pypi.index_files("pypi", "1.2.3") == (None, None) + + +def test_index_files_distinguishes_lookup_failure_from_absence(monkeypatch): + """A network hiccup must not be reported as "not published yet".""" + + def _raise(*args, **kwargs): + raise urllib.error.URLError("connection reset") + + monkeypatch.setattr(pypi.urllib.request, "urlopen", _raise) + filenames, error = pypi.index_files("testpypi", "1.2.3") + assert filenames is None + assert error is not None and "connection reset" in error + + +def test_index_files_returns_filenames(monkeypatch): + class Response: + def read(self, *args): + return json.dumps( + {"releases": {"1.2.3": [{"filename": "b.whl"}, {"filename": "a.whl"}]}} + ).encode() + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + monkeypatch.setattr(pypi.urllib.request, "urlopen", lambda *a, **k: Response()) + assert pypi.index_files("pypi", "1.2.3") == (["a.whl", "b.whl"], None) + + +def test_wait_for_release_retries_until_indexing_finishes(monkeypatch): + """TestPyPI lags the upload response; the check must poll, not give up.""" + calls = {"n": 0} + slept: list[float] = [] + + def _fake(index, version, *, timeout=20.0): + calls["n"] += 1 + if calls["n"] < 3: + return None, None + return ["a.whl"], None + + monkeypatch.setattr(pypi, "index_files", _fake) + filenames, error = pypi.wait_for_release( + "testpypi", "1.2.3", attempts=5, delay=1.0, sleep=slept.append + ) + assert filenames == ["a.whl"] + assert error is None + assert calls["n"] == 3 + assert slept == [1.0, 1.0] + + +def test_wait_for_release_reports_the_last_error(monkeypatch): + def _fake(index, version, *, timeout=20.0): + return None, "HTTP 503" + + monkeypatch.setattr(pypi, "index_files", _fake) + filenames, error = pypi.wait_for_release( + "pypi", "1.2.3", attempts=2, delay=0.0, sleep=lambda _: None + ) + assert filenames is None + assert error == "HTTP 503" + + +def test_existing_release_state_satisfied_conflict_absent(monkeypatch, capsys): + expected = {"orisys-1.2.3-cp310-cp310-win_amd64.whl"} + monkeypatch.setattr(pypi, "index_files", lambda i, v, **k: (None, None)) + assert pypi.existing_release_state("pypi", "1.2.3", expected) == "absent" + + monkeypatch.setattr(pypi, "index_files", lambda i, v, **k: (sorted(expected), None)) + assert pypi.existing_release_state("pypi", "1.2.3", expected) == "satisfied" + + monkeypatch.setattr( + pypi, "index_files", lambda i, v, **k: (["orisys-1.2.3-cp311-cp311-win_amd64.whl"], None) + ) + assert pypi.existing_release_state("pypi", "1.2.3", expected) == "conflict" + output = capsys.readouterr().out + assert "different file set" in output and "missing" in output + + +def test_publish_rerun_after_success_is_not_a_failure(tmp_path, monkeypatch, capsys): + """Re-running a finished phase must succeed, not look like a new error.""" + sdk, wheel = _synthetic_report_layout(tmp_path, "1.2.3") + monkeypatch.setattr(pypi, "REPO_ROOT", sdk) + from release import auth + + auth.write_token("pypi", "pypi-" + "A" * 30, path=tmp_path / ".pypirc") + monkeypatch.setattr(auth, "pypirc_path", lambda: tmp_path / ".pypirc") + monkeypatch.setattr( + pypi, "index_files", lambda i, v, **k: ([wheel.name], None) + ) + + exit_code = pypi.publish( + pypi.Options(version="1.2.3", apply=True, pypi_only=True, skip_check=True) + ) + output = capsys.readouterr().out + assert exit_code == 0 + assert "already serves orisys 1.2.3" in output + assert "every requested index already has this release" in output + + +def test_publish_aborts_on_a_conflicting_existing_release(tmp_path, monkeypatch, capsys): + sdk, wheel = _synthetic_report_layout(tmp_path, "1.2.3") + monkeypatch.setattr(pypi, "REPO_ROOT", sdk) + from release import auth + + auth.write_token("pypi", "pypi-" + "A" * 30, path=tmp_path / ".pypirc") + monkeypatch.setattr(auth, "pypirc_path", lambda: tmp_path / ".pypirc") + monkeypatch.setattr(pypi, "index_files", lambda i, v, **k: (["other.whl"], None)) + + exit_code = pypi.publish( + pypi.Options(version="1.2.3", apply=True, pypi_only=True, skip_check=True) + ) + assert exit_code == 1 + assert "different file set" in capsys.readouterr().out def test_index_json_urls_cover_both_indexes():