orisys-tech/orisys-sdk
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: orisys-tech/orisys-sdk:5f78115429e9590d4dc759279f1816743018002d
Branches Tags
orisys-tech/orisys-sdk:dev orisys-tech/orisys-sdk:main
..
compare: orisys-tech/orisys-sdk:main
Branches Tags
orisys-tech/orisys-sdk:dev orisys-tech/orisys-sdk:main

1 Commits
5f78115429 .. main

Author SHA1 Message Date
zhangyijian 75fa98eff0 Merge pull request 'Dev' (#19) from dev into main 
Reviewed-on: #19
 2026-09-02 21:39:03 +08:00
56 changed files with 288 additions and 8403 deletions
Expand all files Collapse all files
.gitignore
-1
View File
@@ -10,7 +10,6 @@ __pycache__/
/build/dist/
/build/lib/
/build/pyinstaller_work/
/build/release/reports/
/build/*.spec
/dist/
/dist_obfuscated/
MANIFEST.in
-2
View File
@@ -2,8 +2,6 @@
include pyproject.toml
include setup.py
include README.md
# Build-time input for the PyPI long description (read from the checkout).
include docs/pypi_readme.md
recursive-include config *.json
recursive-include sensors *
recursive-include build/common *.py
build/README.md
+2 -55
View File
@@ -12,7 +12,6 @@ build/
README.md
common/ shared helpers (paths, pyproject, verify)
wheel/ SDK .whl (Nuitka + manylinux Docker)
release/ local release gate (preflight / audit)
gui/
common/ shared PyInstaller driver
ory_finger/ ORY-FINGER dashboard
@@ -39,58 +38,9 @@ Direct module paths (same behavior):
```bash
python build/wheel/build.py
build\wheel\win.bat
.\build\wheel\docker\run.ps1 -SkipImageBuild # reuse the image when only sources changed
.\build\wheel\docker\run.ps1
```
The image bakes in per-ABI build tooling and a per-ABI wheelhouse of the runtime
deps, so the entrypoint no longer re-resolves them over the network. See
`docs/publishing.md` §6 for build-time details, why `-SkipImageBuild` matters,
and what to change before attempting to parallelize the per-ABI builds.
### Release gate
Local only: never publishes, and makes no network call unless `--check-pypi` is
passed. See `docs/publishing.md` for the full flow.
```bash
python build/run.py release preflight --version <version>
python build/run.py release preflight --all-wheel-pythons --check-pypi
python build/run.py release audit SDK\<version>\orisys-<version>-cp312-cp312-win_amd64.whl
```
`preflight` runs: repo state → version consistency → version-string scan →
duplicate guard (fails only when the version is already released: a local git
tag, or PyPI with `--check-pypi`) → pytest → bundle precheck → per-wheel static
content/metadata assertions → isolated runtime verification in one venv per wheel
(imports + bundled assets + real frames, no camera). A populated `SDK/<version>/`
is the expected input, so build first:
```bash
python build/run.py wheel-win
python build/run.py release preflight --all-wheel-pythons --check-pypi
```
Reports land in `build/release/reports/<version>/`.
### Publish
Both default to a dry run; `--apply` is required for anything irreversible.
```bash
python build/run.py release auth # credential status
python build/run.py release auth --set-token pypi # hidden prompt, writes ~/.pypirc
python build/run.py release github --version <version> # preview
python build/run.py release github --version <version> --apply # sync + tag + push + Release
python build/run.py release pypi --version <version> --apply # TestPyPI only
python build/run.py release pypi --version <version> --confirm-pypi --apply # + PyPI
python build/run.py release pypi --version <version> --pypi-only --apply # skip TestPyPI
```
Both refuse to act unless `release preflight` passed on a clean commit with no
waived options, and every wheel still hashes to what that report recorded.
Tokens are only ever read from a hidden prompt or stdin, never from argv.
See `docs/publishing.md` §3.
### GUI
```bash
@@ -103,16 +53,13 @@ python build/run.py gui box # scaffold / NotImplemented
```bash
python build/run.py verify path\to\orisys.whl
python build/run.py verify path\to\orisys.whl --contents --assets --runtime
python build/run.py verify --gui-dist --product ory_finger
```
## Outputs
- Wheels: `SDK/<version>/`, `dist/repaired/`
- Preflight reports: `build/release/reports/<version>/`
- Finger GUI: `dist/OrisysDashboard/`
- Future mini/box: `dist/OryMini/`, `dist/OryBox/`
Generated work files under `build/nuitka/`, `build/release/reports/` and
`build/pyinstaller_work/` are gitignored.
Generated work files under `build/nuitka/` and `build/pyinstaller_work/` are gitignored.
build/common/verify/__init__.py
+2 -17
View File
@@ -1,21 +1,6 @@
"""Post-build smoke tests for wheels and product GUI folders."""
from common.verify.gui import GUI_PRODUCTS, verify_gui_dist
from common.verify.wheel import (
verify_built_wheel,
verify_wheel,
verify_wheel_assets,
verify_wheel_runtime,
)
from common.verify.wheel_contents import WheelCheckResult, inspect_wheel
from common.verify.wheel import verify_built_wheel
__all__ = [
"GUI_PRODUCTS",
"WheelCheckResult",
"inspect_wheel",
"verify_built_wheel",
"verify_gui_dist",
"verify_wheel",
"verify_wheel_assets",
"verify_wheel_runtime",
]
__all__ = ["GUI_PRODUCTS", "verify_built_wheel", "verify_gui_dist"]
build/common/verify/cli.py
+1 -105
View File
@@ -3,24 +3,9 @@
from __future__ import annotations
import argparse
import sys
from pathlib import Path
_BUILD_ROOT = Path(__file__).resolve().parents[2]
if str(_BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(_BUILD_ROOT))
from common.paths import REPO_ROOT # noqa: E402
from common.verify import ( # noqa: E402
GUI_PRODUCTS,
inspect_wheel,
verify_built_wheel,
verify_gui_dist,
verify_wheel,
verify_wheel_assets,
verify_wheel_runtime,
)
from common.verify.wheel import DEFAULT_RUNTIME_CONFIG, default_verification_video # noqa: E402
from common.verify import GUI_PRODUCTS, verify_built_wheel, verify_gui_dist
def main(argv: list[str] | None = None) -> int:
@@ -33,31 +18,6 @@ def main(argv: list[str] | None = None) -> int:
help="GUI product profile (default: ory_finger)",
)
parser.add_argument("--dist-dir", type=Path, default=None, help="Override GUI dist folder")
parser.add_argument(
"--contents",
action="store_true",
help="Run static metadata/content assertions on the wheel",
)
parser.add_argument(
"--assets",
action="store_true",
help="Install the wheel in a temp venv and assert bundled assets load",
)
parser.add_argument(
"--runtime",
action="store_true",
help="Install the wheel and process real frames (no camera required)",
)
parser.add_argument("--expected-version", default=None, help="Assert the wheel version")
parser.add_argument(
"--strict-bundle",
action="store_true",
help="Treat bundled-file drift from the source tree as a failure",
)
parser.add_argument("--video", type=Path, default=None, help="Clip for --runtime")
parser.add_argument("--config", default=DEFAULT_RUNTIME_CONFIG, help="Config for --runtime")
parser.add_argument("--frames", type=int, default=40, help="Frames for --runtime")
parser.add_argument("--offline", action="store_true", help="Reuse parent-env dependencies")
parser.add_argument("wheel", nargs="?", type=Path, help="Wheel path for import smoke test")
args = parser.parse_args(argv)
@@ -66,70 +26,6 @@ def main(argv: list[str] | None = None) -> int:
return 0
if args.wheel is None:
parser.error("Provide a wheel path or pass --gui-dist")
exit_code = 0
if args.contents:
result = inspect_wheel(
args.wheel,
expected_version=args.expected_version,
repo_root=REPO_ROOT,
strict_bundle=args.strict_bundle,
)
print(result.render())
if not result.ok:
exit_code = 1
if args.assets and args.runtime:
video = args.video or default_verification_video(REPO_ROOT)
if video is None:
print("[FAIL] runtime: no clip found; pass --video or set ORISYS_VERIFY_VIDEO")
return 1
try:
record = verify_wheel(
args.wheel,
video=video,
config_name=args.config,
frames=args.frames,
offline=args.offline,
)
print(f"[ok ] {record['imports']}")
print(f"[ok ] {record['assets']}")
print(f"[ok ] runtime: frames={record['frames']} flow={record['flow_shape']}")
except Exception as exc:
print(f"[FAIL] isolated verification: {type(exc).__name__}: {exc}")
exit_code = 1
return exit_code
if args.assets:
try:
print(verify_wheel_assets(args.wheel))
except Exception as exc:
print(f"[FAIL] bundled assets: {type(exc).__name__}: {exc}")
exit_code = 1
if args.runtime:
video = args.video or default_verification_video(REPO_ROOT)
if video is None:
print("[FAIL] runtime: no clip found; pass --video or set ORISYS_VERIFY_VIDEO")
return 1
try:
info = verify_wheel_runtime(
args.wheel,
video=video,
config_name=args.config,
frames=args.frames,
offline=args.offline,
)
print(f"[ok ] runtime: {info}")
except Exception as exc:
print(f"[FAIL] runtime: {type(exc).__name__}: {exc}")
exit_code = 1
if args.contents or args.assets or args.runtime:
return exit_code
# Default behaviour is unchanged: import smoke test in an isolated venv.
verify_built_wheel(args.wheel)
return 0
build/common/verify/wheel.py
+14 -398
View File
@@ -1,26 +1,11 @@
"""Isolated venv smoke tests for packaged orisys wheels.
Three levels of confidence, all in a throwaway virtualenv so the active
environment is never touched:
* :func:`verify_built_wheel` - the public API imports (fast, original check);
* :func:`verify_wheel_assets` - bundled configs/sensors resolve and load via
``importlib.resources``;
* :func:`verify_wheel_runtime` - the compiled wheel actually processes real
frames through the public pipeline with no camera attached.
Every level can be bootstrapped from another interpreter so each supported
CPython ABI can be verified against its own wheel.
"""
"""Isolated venv smoke tests for packaged orisys wheels."""
from __future__ import annotations
import os
import subprocess
import sys
import tempfile
import venv
from contextlib import contextmanager
from pathlib import Path
# Public SDK surface that must import from a released wheel.
@@ -53,167 +38,6 @@ assert Stitcher is not None
print(f"orisys {orisys.__version__} public API imports OK")
"""
# Bundled data that must be resolvable and loadable from inside site-packages.
# The wheel ships the compiled package, so these paths only exist if the bundle
# step actually ran for this build.
_ASSET_SCRIPT = r"""
import sys
from importlib.metadata import distribution
from pathlib import Path
import numpy as np
import orisys
# Locate bundled data through the installed distribution, NOT through
# orisys.__file__ / importlib.resources. Nuitka compiles the package into one
# extension module whose __init__ installs nuitka_module_loader, which reports a
# synthetic __file__ (e.g. ...\site-packages\orisys\orisys\__init__.py) and a
# synthetic __path__ ending in '\not_existing'. Neither points at the real data
# directory, so importlib.resources silently finds nothing in a Nuitka wheel.
dist = distribution("orisys")
dist_root = Path(dist.locate_file("")).resolve()
package_root = Path(dist.locate_file("orisys")).resolve()
_diagnosis = (
f"orisys distribution resolved to {dist_root} (sys.prefix={sys.prefix!r}, "
f"sys.executable={sys.executable!r}), outside the verification venv. A stale "
"orisys in the parent environment is visible through --system-site-packages; "
"run `pip uninstall -y orisys` there, or re-run without --offline."
)
assert str(Path(sys.prefix).resolve()) in str(dist_root), _diagnosis
assert package_root.is_dir(), f"package directory not found: {package_root}"
required = [
"configs/box.json",
"configs/finger.json",
"sensors/box/force_calibration.json",
"sensors/box/sensor.json",
"sensors/finger/manifest.json",
"sensors/finger/sensor.json",
"sensors/mini/sensor.json",
"sensors/finger/assets/30fps-0703/mesh.npz",
"sensors/finger/assets/30fps-0703/camera_intrinsics.npz",
"sensors/finger/assets/30fps-0703/world_to_cam.npy",
"sensors/finger/assets/visualization/mesh.npz",
"sensors/finger/assets/visualization/world_to_cam.npy",
]
missing = [name for name in required if not (package_root / name).is_file()]
assert not missing, (
f"bundled assets missing from the wheel: {missing}\n"
f"package root: {package_root}\ninstalled tree: "
f"{sorted(p.name for p in package_root.iterdir()) if package_root.is_dir() else '<missing>'}"
)
for name in required:
path = package_root / name
if name.endswith(".npz"):
data = np.load(str(path))
assert len(data.files) > 0, f"{name} is an empty npz"
elif name.endswith(".npy"):
arr = np.load(str(path))
assert arr.size > 0, f"{name} is an empty npy"
configs = orisys.list_config()
assert configs, "orisys.list_config() returned nothing"
print(f"bundled assets OK ({len(required)} files, {len(configs)} configs) "
f"from {package_root}")
"""
# End-to-end pipeline run against a video file: no camera, no GUI, no imshow.
_RUNTIME_SCRIPT = r"""
import json
import os
import sys
from importlib.metadata import distribution
from pathlib import Path
import numpy as np
os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0")
import orisys
# Provenance via the installed distribution: a Nuitka-compiled package reports a
# synthetic orisys.__file__ (see the asset check) and a stale orisys visible
# through --system-site-packages would otherwise satisfy this check.
dist = distribution("orisys")
dist_root = Path(dist.locate_file("")).resolve()
assert str(Path(sys.prefix).resolve()) in str(dist_root), (
f"the installed orisys came from {dist_root}, outside the verification venv "
f"{sys.prefix!r}; run `pip uninstall -y orisys` in the parent environment or "
"re-run without --offline."
)
video, config_name, frames = sys.argv[1], sys.argv[2], int(sys.argv[3])
sensor = orisys.Sensor(
vid_src=video,
config_name=config_name,
backend="cpu",
verbose=False,
)
processed = 0
try:
for _ in range(frames):
img = sensor.get_img()
assert img is not None, "get_img returned None"
assert isinstance(img, np.ndarray) and img.ndim in (2, 3)
sensor.compute_deformation(check_motion=False)
flow, vnormal, vshear = sensor.read_info(
sensor.info.VRAW, sensor.info.VNORMAL, sensor.info.VSHEAR
)
for name, array in (("flow", flow), ("vnormal", vnormal), ("vshear", vshear)):
assert np.isfinite(array).all(), f"{name} contains NaN/Inf"
contact = sensor.compute_contact()
assert isinstance(contact, dict) and "centroid_found" in contact
fnormal, fsx, fsy = sensor.read_info(
sensor.info.FNORMAL, sensor.info.FSHEARX, sensor.info.FSHEARY
)
assert np.isfinite([fnormal, fsx, fsy]).all(), "force output contains NaN/Inf"
processed += 1
finally:
sensor.disconnect()
assert processed == frames, f"only processed {processed}/{frames} frames"
print(json.dumps({
"version": orisys.__version__,
"frames": processed,
"flow_shape": list(flow.shape),
"video": video,
"config": config_name,
"package_root": str(Path(dist.locate_file("orisys")).resolve()),
}))
"""
# Default runtime-verification inputs. Overridable by CLI flags or environment.
DEFAULT_RUNTIME_CONFIG = os.getenv("ORISYS_VERIFY_CONFIG", "box")
# Clips tracked in the repository, smallest first.
DEFAULT_VIDEO_CANDIDATES = (
Path("data") / "sample" / "s1_480p.mp4",
Path("data") / "sample" / "s1_720p.mp4",
Path("tests") / "idle_1min.mp4",
)
def default_verification_video(repo_root: Path) -> Path | None:
"""Resolve the clip used for the end-to-end runtime check, if any."""
env_video = os.getenv("ORISYS_VERIFY_VIDEO")
repo_root = Path(repo_root)
if env_video:
candidate = Path(env_video)
return candidate if candidate.is_absolute() else repo_root / candidate
for relative in DEFAULT_VIDEO_CANDIDATES:
candidate = repo_root / relative
if candidate.is_file():
return candidate
return None
def _venv_python(venv_dir: Path) -> Path:
if sys.platform == "win32":
@@ -221,226 +45,18 @@ def _venv_python(venv_dir: Path) -> Path:
return venv_dir / "bin" / "python"
def _hermetic_env() -> dict:
"""Environment for the verification venv with PYTHONPATH removed.
An inherited PYTHONPATH=src would let the checkout satisfy the checks
instead of the installed wheel, which is precisely the failure mode the
provenance assertions in the scripts exist to catch.
"""
return {key: value for key, value in os.environ.items() if key != "PYTHONPATH"}
def _create_venv(
venv_dir: Path,
*,
python_executable: Path | None = None,
system_site_packages: bool = False,
) -> Path:
"""Create a throwaway venv, optionally from a different interpreter."""
if python_executable is None:
venv.create(venv_dir, with_pip=True, system_site_packages=system_site_packages)
else:
command = [str(python_executable), "-m", "venv"]
if system_site_packages:
command.append("--system-site-packages")
command.append(str(venv_dir))
subprocess.check_call(command, env=_hermetic_env())
return _venv_python(venv_dir)
def _install_wheel(
python: Path, wheel_path: Path, *, offline: bool = False, env: dict | None = None
) -> None:
"""Install the wheel under test, guaranteeing it lands in the venv.
``--force-reinstall --no-deps`` is not optional: pip reports a same-version
requirement as "already satisfied" and skips the file, which is fatal here
because ``--offline`` also exposes the parent environment's site-packages.
Without it the venv check can silently validate a *different* orisys than
the one passed in (the scripts additionally assert provenance).
"""
if offline:
# Dependencies come from the parent environment via --system-site-packages.
commands = [
[
str(python),
"-m",
"pip",
"install",
"-q",
"--no-index",
"--no-deps",
"--force-reinstall",
str(wheel_path),
]
]
else:
commands = [
[str(python), "-m", "pip", "install", "-q", "--upgrade", "pip"],
[str(python), "-m", "pip", "install", "-q", str(wheel_path)],
[
str(python),
"-m",
"pip",
"install",
"-q",
"--force-reinstall",
"--no-deps",
str(wheel_path),
],
]
for command in commands:
subprocess.check_call(command, stdout=subprocess.DEVNULL, env=env)
class WheelVerificationError(RuntimeError):
"""A verification step failed inside the throwaway venv.
Carries the child's stdout/stderr so the real traceback is visible instead of
a bare non-zero exit code.
"""
def _run_script(
python: Path, script: str, args: list[str] | None = None, *, env: dict
) -> str:
"""Run a verification script in an already-prepared venv."""
completed = subprocess.run(
[str(python), "-c", script, *(args or [])],
capture_output=True,
text=True,
env=env,
check=False,
)
if completed.returncode != 0:
command = " ".join(str(part) for part in (args or []))
raise WheelVerificationError(
f"verification script failed in the venv (exit {completed.returncode}"
+ (f", args: {command}" if command else "")
+ ").\n--- stdout ---\n"
+ (completed.stdout.strip() or "<empty>")
+ "\n--- stderr ---\n"
+ (completed.stderr.strip() or "<empty>")
)
return completed.stdout.strip()
@contextmanager
def _installed_venv(
wheel_path: Path,
*,
python_executable: Path | None = None,
offline: bool = False,
prefix: str = "orisys-wheel-verify-",
):
"""Create a venv with the wheel installed, for one or more checks.
Creating a venv and installing the runtime dependencies is by far the most
expensive part of verification (tens of seconds plus a large download when
not using ``--offline``), so callers that run several checks must share one
venv instead of building one per check.
"""
wheel_path = wheel_path.resolve()
env = _hermetic_env()
with tempfile.TemporaryDirectory(prefix=prefix) as tmp:
venv_dir = Path(tmp) / "venv"
interpreter = python_executable or Path(sys.executable)
print(
f" [{wheel_path.name}] creating venv from {interpreter.name}"
+ ("" if offline else " and installing dependencies (first run downloads packages)")
+ " ...",
flush=True,
)
python = _create_venv(
venv_dir,
python_executable=python_executable,
system_site_packages=offline,
)
_install_wheel(python, wheel_path, offline=offline, env=env)
yield python, env
def verify_built_wheel(wheel_path: Path, *, python_executable: Path | None = None) -> None:
def verify_built_wheel(wheel_path: Path) -> None:
"""Install the wheel in a temporary venv and smoke-test public imports."""
with _installed_venv(wheel_path, python_executable=python_executable) as (python, env):
_run_script(python, _SMOKE_IMPORT_SCRIPT, env=env)
wheel_path = wheel_path.resolve()
with tempfile.TemporaryDirectory(prefix="orisys-wheel-verify-") as tmp:
venv_dir = Path(tmp) / "venv"
venv.create(venv_dir, with_pip=True)
python = _venv_python(venv_dir)
def verify_wheel_assets(wheel_path: Path, *, python_executable: Path | None = None) -> str:
"""Assert the wheel carries loadable bundled configs and sensor assets."""
with _installed_venv(wheel_path, python_executable=python_executable) as (python, env):
return _run_script(python, _ASSET_SCRIPT, env=env)
def verify_wheel_runtime(
wheel_path: Path,
*,
video: Path,
config_name: str,
frames: int = 40,
python_executable: Path | None = None,
offline: bool = False,
) -> dict:
"""Run real frames through the installed wheel with no camera attached.
This is the only check that proves the compiled extension plus the bundled
sensor assets actually produce finite flow/force output end to end.
"""
with _installed_venv(
wheel_path, python_executable=python_executable, offline=offline
) as (python, env):
return _runtime_check(python, env, video=video, config_name=config_name, frames=frames)
def _runtime_check(
python: Path, env: dict, *, video: Path, config_name: str, frames: int
) -> dict:
video = Path(video).resolve()
if not video.is_file():
raise FileNotFoundError(f"verification video not found: {video}")
if frames < 1:
raise ValueError("frames must be >= 1")
print(f" processing {frames} frame(s) with config {config_name!r} ...", flush=True)
output = _run_script(
python, _RUNTIME_SCRIPT, [str(video), config_name, str(frames)], env=env
)
import json
return json.loads(output.splitlines()[-1])
def verify_wheel(
wheel_path: Path,
*,
video: Path | None = None,
config_name: str = DEFAULT_RUNTIME_CONFIG,
frames: int = 40,
python_executable: Path | None = None,
offline: bool = False,
check_assets: bool = True,
) -> dict:
"""Run every isolated-check level for one wheel inside a single venv.
Use this instead of calling the individual ``verify_wheel_*`` helpers: it
pays the venv/dependency cost once and returns the runtime record plus the
asset and import summaries.
"""
with _installed_venv(
wheel_path, python_executable=python_executable, offline=offline
) as (python, env):
imports = _run_script(python, _SMOKE_IMPORT_SCRIPT, env=env)
record: dict = {"wheel": Path(wheel_path).name, "imports": imports}
if check_assets:
record["assets"] = _run_script(python, _ASSET_SCRIPT, env=env)
else:
record["assets"] = "asset check skipped (--no-assets)"
if video is not None:
record.update(
_runtime_check(
python, env, video=video, config_name=config_name, frames=frames
)
)
return record
subprocess.check_call(
[python, "-m", "pip", "install", "-q", "--upgrade", "pip"],
)
subprocess.check_call(
[python, "-m", "pip", "install", "-q", str(wheel_path)],
)
subprocess.check_call([python, "-c", _SMOKE_IMPORT_SCRIPT])
build/common/verify/wheel_contents.py
-501
View File
@@ -1,501 +0,0 @@
"""Static content and metadata validation for built orisys wheels.
Everything here is pure standard library: the wheel is opened as a zip and
inspected in place. No installation, no network, no hardware.
The checks encode the guarantees a Nuitka-compiled release must satisfy:
* the filename, ``METADATA`` and ``RECORD`` agree on name/version/tags;
* PyPI-visible metadata (``Requires-Python``, classifiers, project URLs,
license) is present instead of silently dropped by the hand-written
``setup.py`` template in :mod:`wheel.package`;
* no Python sources ship inside ``orisys/`` (IP containment) while the compiled
extension is present;
* every file declared by ``[[tool.orisys.bundle.items]]`` is present and
byte-identical to the source tree;
* no caches, build leftovers or scratch directories leak in.
"""
from __future__ import annotations
import base64
import hashlib
import re
import sys
import zipfile
from dataclasses import dataclass, field
from email.parser import Parser
from pathlib import Path
_BUILD_ROOT = Path(__file__).resolve().parents[1]
if str(_BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(_BUILD_ROOT))
from common.paths import REPO_ROOT # noqa: E402
from common.pyproject_utils import bundle_items, extras_require, install_requires, project_metadata # noqa: E402
from wheel.layout import resolve_bundle_source_files # noqa: E402
from wheel.package import is_compliant_manylinux_wheel # noqa: E402
# PyPI's default per-project file limit is 100 MB; keep headroom.
MAX_WHEEL_BYTES = 90 * 1024 * 1024
_FILENAME_RE = re.compile(
r"^(?P<name>[^-]+)-(?P<version>[^-]+)-(?P<python>[^-]+)-(?P<abi>[^-]+)-(?P<platform>.+)\.whl$"
)
_COMPILED_EXT_RE = re.compile(r"^orisys/__init__\..+\.(?:pyd|so)$")
_FORBIDDEN_PREFIXES = (
"test/",
"tests/",
"data/",
"build/",
"dist/",
"sdk/",
".git",
)
_FORBIDDEN_SUFFIXES = (".pyc", ".pyo", ".c", ".pdb", ".map", ".spec")
@dataclass(frozen=True)
class Check:
name: str
ok: bool
detail: str = ""
def render(self) -> str:
return f"[{'ok' if self.ok else 'FAIL'}] {self.name}: {self.detail}"
@dataclass
class WheelCheckResult:
wheel: Path
version: str | None = None
name: str | None = None
platform_tag: str | None = None
entry_count: int = 0
uncompressed_bytes: int = 0
file_bytes: int = 0
checks: list[Check] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
@property
def ok(self) -> bool:
return all(check.ok for check in self.checks)
@property
def failures(self) -> list[Check]:
return [check for check in self.checks if not check.ok]
def add(self, name: str, ok: bool, detail: str = "") -> bool:
self.checks.append(Check(name=name, ok=ok, detail=detail))
return ok
def warn(self, message: str) -> None:
self.warnings.append(message)
def render(self) -> str:
lines = [
f"wheel: {self.wheel.name}",
f" version={self.version} platform={self.platform_tag} "
f"entries={self.entry_count} file_size={self.file_bytes} "
f"uncompressed={self.uncompressed_bytes}",
]
lines.extend(f" {check.render()}" for check in self.checks)
lines.extend(f" [warn] {message}" for message in self.warnings)
lines.append(f" RESULT: {'PASS' if self.ok else 'FAIL'}")
return "\n".join(lines)
def _sha256(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def _requirement_name(spec: str) -> str:
return re.split(r"[<>=!~\[; ]", spec, maxsplit=1)[0].strip().lower().replace("_", "-")
def _metadata_message(zf: zipfile.ZipFile, result: WheelCheckResult):
candidates = [name for name in zf.namelist() if name.endswith(".dist-info/METADATA")]
if not candidates:
result.add("metadata-present", False, "no *.dist-info/METADATA entry in the wheel")
return None
raw = zf.read(candidates[0]).decode("utf-8", errors="replace")
message = Parser().parsestr(raw)
result.add("metadata-present", True, candidates[0])
return message
def _check_filename(result: WheelCheckResult, expected_version: str | None) -> None:
match = _FILENAME_RE.match(result.wheel.name)
if match is None:
result.add("filename", False, f"unparsable wheel filename: {result.wheel.name}")
return
result.name = match.group("name")
result.version = match.group("version")
result.platform_tag = match.group("platform")
problems = []
if result.name != "orisys":
problems.append(f"distribution name is {result.name!r}, expected 'orisys'")
if expected_version is not None and result.version != expected_version:
problems.append(f"filename version {result.version!r} != expected {expected_version!r}")
result.add("filename", not problems, "; ".join(problems) or result.wheel.name)
def _check_metadata(
result: WheelCheckResult,
message,
expected_version: str | None,
repo_root: Path | None,
) -> None:
if message is None:
return
result.add(
"metadata-name",
message.get("Name") == "orisys",
f"Name={message.get('Name')!r}",
)
declared = message.get("Version")
expected = expected_version or result.version
if expected is not None and declared != expected:
result.add(
"metadata-version",
False,
f"METADATA Version={declared!r} != expected {expected!r}",
)
else:
result.add("metadata-version", True, f"Version={declared!r}")
requires_python = (message.get("Requires-Python") or "").strip()
result.add(
"metadata-requires-python",
bool(requires_python),
f"Requires-Python={requires_python!r}" if requires_python else "Requires-Python is missing",
)
classifiers = message.get_all("Classifier") or []
result.add(
"metadata-classifiers",
bool(classifiers),
f"{len(classifiers)} classifier(s)" if classifiers else "no Classifier fields",
)
project_urls = message.get_all("Project-URL") or []
result.add(
"metadata-project-urls",
bool(project_urls),
f"{len(project_urls)} Project-URL(s)" if project_urls else "no Project-URL fields",
)
license_values = [
(message.get("License") or "").strip(),
(message.get("License-Expression") or "").strip(),
]
has_license = any(license_values) or any(
value.startswith("License ::") for value in classifiers
)
result.add(
"metadata-license",
has_license,
f"License={license_values[0]!r} License-Expression={license_values[1]!r}"
if has_license
else "no License / License-Expression / 'License ::' classifier",
)
dynamic = message.get_all("Dynamic") or []
result.add(
"metadata-no-dynamic-version",
"version" not in dynamic,
f"Dynamic={dynamic!r}" if dynamic else "no Dynamic fields",
)
if dynamic:
result.warn(f"METADATA still declares Dynamic fields: {dynamic!r}")
if repo_root is not None and (repo_root / "pyproject.toml").is_file():
requires_dist = message.get_all("Requires-Dist") or []
provided = {_requirement_name(entry) for entry in requires_dist}
missing_core = [
spec for spec in install_requires(repo_root) if _requirement_name(spec) not in provided
]
result.add(
"metadata-requires-dist",
not missing_core,
f"missing core Requires-Dist entries: {missing_core!r}"
if missing_core
else f"{len(requires_dist)} Requires-Dist entries",
)
extras = message.get_all("Provides-Extra") or []
missing_extras = sorted(set(extras_require(repo_root)) - set(extras))
result.add(
"metadata-extras",
not missing_extras,
f"missing Provides-Extra: {missing_extras!r}"
if missing_extras
else f"Provides-Extra={sorted(extras)!r}",
)
def _check_compiled_only(result: WheelCheckResult, names: list[str]) -> None:
compiled = [name for name in names if _COMPILED_EXT_RE.match(name)]
result.add(
"compiled-extension",
bool(compiled),
f"{compiled[0]}" if compiled else "no orisys/__init__.<abi>.pyd|.so entry",
)
# orisys/examples/** is shipped as package data on purpose; every other
# orisys/**/*.py means source code leaked past the Nuitka compile step.
sources = [
name
for name in names
if name.startswith("orisys/")
and name.endswith(".py")
and not name.startswith("orisys/examples/")
]
result.add(
"no-python-sources",
not sources,
f"python sources leaked into the wheel: {sources[:5]!r}"
if sources
else "compiled package ships no python sources (examples excluded by design)",
)
blend = [name for name in names if name.startswith("orisys_blend/")]
result.add(
"orisys-blend-present",
"orisys_blend/__init__.py" in blend and "orisys_blend/kernel.py" in blend,
f"{sorted(blend)!r}",
)
def _check_bundle_fidelity(
result: WheelCheckResult,
zf: zipfile.ZipFile,
names: set[str],
repo_root: Path | None,
strict: bool,
) -> None:
if repo_root is None or not (repo_root / "pyproject.toml").is_file():
result.warn("pyproject.toml not found; bundle fidelity checks skipped")
return
missing: list[str] = []
mismatched: list[str] = []
checked = 0
for item in bundle_items(repo_root):
target = str(item["target"]).replace("\\", "/")
source_dir = repo_root / item["source"]
for source_file in resolve_bundle_source_files(repo_root, item):
if item.get("preserve_tree", False):
relative = source_file.relative_to(source_dir).as_posix()
entry = f"orisys/{target}/{relative}"
else:
entry = f"orisys/{target}/{source_file.name}"
if entry not in names:
missing.append(entry)
continue
checked += 1
if _sha256(zf.read(entry)) != _sha256(source_file.read_bytes()):
mismatched.append(entry)
# Both entries here mean "the wheel does not match this revision of the
# source tree". Against a fresh build that is fatal; against an already
# released wheel it is expected, so ``strict`` (set by preflight, unset by
# audit) decides whether it fails or warns.
if missing:
detail = (
f"{len(missing)} declared file(s) absent from the wheel (added to the "
f"source tree after this build?): {missing[:5]!r}"
)
result.add("bundle-completeness", not strict, detail)
if not strict:
result.warn(detail)
else:
result.add(
"bundle-completeness", True, f"{checked} declared bundled file(s) present"
)
if mismatched:
detail = (
f"{len(mismatched)} of {checked} bundled file(s) differ from the source "
f"tree (wheel predates or postdates it): {mismatched[:5]!r}"
)
result.add("bundle-fidelity", not strict, detail)
if not strict:
result.warn(detail)
else:
result.add(
"bundle-fidelity",
True,
f"{checked} bundled file(s) byte-identical to source",
)
def _check_forbidden_entries(result: WheelCheckResult, names: list[str]) -> None:
offenders = []
for name in names:
lowered = name.lower()
if lowered.startswith(_FORBIDDEN_PREFIXES):
offenders.append(name)
elif lowered.endswith(_FORBIDDEN_SUFFIXES):
offenders.append(name)
elif "__pycache__" in lowered or ".egg-info" in lowered:
offenders.append(name)
result.add(
"no-forbidden-entries",
not offenders,
f"unexpected entries: {offenders[:5]!r}" if offenders else "no caches/leftovers/scratch paths",
)
def _check_archive_paths(result: WheelCheckResult, names: list[str]) -> None:
unsafe = [
name
for name in names
if name.startswith(("/", "\\")) or ".." in Path(name.replace("\\", "/")).parts
]
result.add(
"archive-paths",
not unsafe,
f"absolute or traversing entries: {unsafe[:5]!r}" if unsafe else "all entries are relative",
)
def _check_record(result: WheelCheckResult, zf: zipfile.ZipFile, names: list[str]) -> None:
records = [name for name in names if name.endswith(".dist-info/RECORD")]
if not records:
result.add("record", False, "no *.dist-info/RECORD entry")
return
bad_paths: list[str] = []
bad_hashes: list[str] = []
bad_sizes: list[str] = []
present = set(names)
for line in zf.read(records[0]).decode("utf-8").splitlines():
if not line.strip():
continue
parts = line.split(",")
while len(parts) < 3:
parts.append("")
path, hash_field, size_field = parts[0], parts[1], parts[2]
if path not in present:
bad_paths.append(path)
continue
if not hash_field.startswith("sha256="):
continue
expected = hash_field[len("sha256=") :]
actual = base64.urlsafe_b64encode(
hashlib.sha256(zf.read(path)).digest()
).rstrip(b"=").decode("ascii")
if actual != expected:
bad_hashes.append(path)
if size_field and size_field.isdigit():
if int(size_field) != len(zf.read(path)):
bad_sizes.append(path)
problems = []
if bad_paths:
problems.append(f"paths not in archive: {bad_paths[:5]!r}")
if bad_hashes:
problems.append(f"sha256 mismatch: {bad_hashes[:5]!r}")
if bad_sizes:
problems.append(f"size mismatch: {bad_sizes[:5]!r}")
result.add(
"record",
not problems,
"; ".join(problems) if problems else f"{records[0]} verifies",
)
def _check_platform_tag(result: WheelCheckResult) -> None:
tag = result.platform_tag
if tag is None:
return
if "win" in tag:
ok = tag == "win_amd64"
result.add("platform-tag", ok, f"{tag!r} (expected 'win_amd64')")
return
ok = is_compliant_manylinux_wheel(result.wheel)
result.add(
"platform-tag",
ok,
f"{tag!r}" if ok else f"{tag!r} is not a PyPI-accepted manylinux x86_64 tag",
)
def _check_size(result: WheelCheckResult, max_bytes: int) -> None:
result.add(
"size-limit",
result.file_bytes <= max_bytes,
f"{result.file_bytes} bytes (limit {max_bytes})",
)
def inspect_wheel(
wheel_path: Path,
*,
expected_version: str | None = None,
repo_root: Path | None = None,
max_bytes: int = MAX_WHEEL_BYTES,
strict_bundle: bool = True,
) -> WheelCheckResult:
"""Validate one built wheel without installing it.
``strict_bundle`` controls how the wheel's bundled files are compared with the
current source tree: ``True`` (gate a fresh build, used by
``release preflight``) fails on any missing or changed declared file;
``False`` (audit an already-released wheel) reports the same findings as
warnings.
"""
wheel_path = Path(wheel_path).resolve()
if repo_root is None:
repo_root = REPO_ROOT if (REPO_ROOT / "pyproject.toml").is_file() else None
else:
repo_root = Path(repo_root).resolve()
result = WheelCheckResult(wheel=wheel_path)
if not wheel_path.is_file():
result.add("wheel-exists", False, f"not a file: {wheel_path}")
return result
result.file_bytes = wheel_path.stat().st_size
result.add("wheel-exists", True, str(wheel_path))
_check_filename(result, expected_version)
_check_platform_tag(result)
_check_size(result, max_bytes)
try:
zf = zipfile.ZipFile(wheel_path)
except zipfile.BadZipFile as exc:
result.add("zip-integrity", False, f"not a valid zip: {exc}")
return result
with zf:
corrupt = zf.testzip()
result.add(
"zip-integrity",
corrupt is None,
"CRC check passed" if corrupt is None else f"corrupt entry: {corrupt}",
)
names = zf.namelist()
name_set = set(names)
result.entry_count = len(names)
result.uncompressed_bytes = sum(info.file_size for info in zf.infolist())
_check_archive_paths(result, names)
_check_forbidden_entries(result, names)
message = _metadata_message(zf, result)
_check_metadata(result, message, expected_version, repo_root)
_check_compiled_only(result, names)
_check_bundle_fidelity(result, zf, name_set, repo_root, strict_bundle)
_check_record(result, zf, names)
return result
build/release/__init__.py
-7
View File
@@ -1,7 +0,0 @@
"""Local release tooling: preflight gate and per-wheel audit.
Entry point is ``python build/run.py release <command>``; see
:mod:`release.preflight` for the gate steps and :mod:`release.cli` for the CLI.
Imports are intentionally not re-exported here so importing a submodule never
triggers the whole subsystem.
"""
build/release/auth.py
-398
View File
@@ -1,398 +0,0 @@
"""PyPI credential setup: locate, inspect and write ``~/.pypirc``.
Editing the config from a command rather than by hand avoids the two mistakes
that actually happen: putting something other than ``__token__`` in ``username``,
and pasting a TestPyPI token into the ``[pypi]`` section (the two indexes reject
each other's tokens with 403, and both token kinds start with ``pypi-``).
The writer does targeted text edits rather than rewriting through configparser,
so the explanatory comments in the file survive.
"""
from __future__ import annotations
import configparser
import getpass
import os
import sys
from dataclasses import dataclass, field
from pathlib import Path
INDEXES: dict[str, dict[str, str | None]] = {
"pypi": {
"upload_url": "https://upload.pypi.org/legacy/",
"json_url": "https://pypi.org/pypi/orisys/json",
"simple_url": "https://pypi.org/simple/",
"token_url": "https://pypi.org/manage/account/token/",
"register_url": None,
},
"testpypi": {
"upload_url": "https://test.pypi.org/legacy/",
"json_url": "https://test.pypi.org/pypi/orisys/json",
"simple_url": "https://test.pypi.org/simple/",
"token_url": "https://test.pypi.org/manage/account/token/",
"register_url": "https://test.pypi.org/account/register/",
},
}
# Substrings that mark a token slot that was never filled in.
PLACEHOLDER_MARKERS = ("PASTE", "REPLACE", "TOKEN-HERE", "YOUR-TOKEN", "XXXX")
DEFAULT_TEMPLATE = """\
# PyPI / TestPyPI credentials for twine.
#
# Keep this file out of any git repository: a token committed once stays in the
# history even after you delete it.
#
# * username is literally "__token__" - never an account name or email
# * PyPI and TestPyPI tokens both start with "pypi-" but are NOT
# interchangeable: each index rejects the other's with 403
# * TestPyPI is a separate service with a separate account
[distutils]
index-servers =
"""
@dataclass
class CredentialState:
index: str
configured: bool
source: str | None = None
problem: str | None = None
@dataclass
class TwineLookup:
"""What twine's own resolver says about one index."""
available: bool
known: bool
config: dict = field(default_factory=dict)
error: str | None = None
def pypirc_path() -> Path:
"""Where twine looks by default: ``~/.pypirc`` (same expansion twine uses)."""
return Path(os.path.expanduser("~")) / ".pypirc"
def looks_like_placeholder(token: str) -> bool:
upper = token.upper()
return any(marker in upper for marker in PLACEHOLDER_MARKERS)
def parse_pypirc(text: str) -> dict[str, dict[str, str]]:
"""Section -> key -> value, tolerating comments and continuation lines."""
sections: dict[str, dict[str, str]] = {}
current: str | None = None
for raw in text.splitlines():
line = raw.strip()
if not line or line.startswith(("#", ";", "%")):
continue
if line.startswith("[") and line.endswith("]"):
current = line[1:-1].strip()
sections.setdefault(current, {})
continue
if current is not None and "=" in line:
key, _, value = line.partition("=")
sections[current][key.strip()] = value.strip()
return sections
def credential_state(
index: str,
*,
path: Path | None = None,
env: dict | None = None,
) -> CredentialState:
"""Is ``index`` usable by twine, and why not when it is not.
Delegates to ``twine.utils.get_config`` when twine is importable, because the
only definition that matters is twine's. The important subtlety that a naive
check misses: **twine ignores any section not listed in
``[distutils] index-servers``** (with a fallback of ``pypi testpypi`` when the
key is absent). A ``[testpypi]`` section without that listing looks configured
but twine reports "Missing 'testpypi' section".
"""
if index not in INDEXES:
raise ValueError(f"unknown index {index!r}; expected one of {sorted(INDEXES)}")
env = os.environ if env is None else env
if env.get("TWINE_PASSWORD"):
return CredentialState(index, True, "environment variables")
path = path or pypirc_path()
if not path.is_file():
return CredentialState(index, False, None, f"{path} does not exist yet")
try:
text = path.read_text(encoding="utf-8", errors="ignore")
except OSError as exc:
return CredentialState(index, False, None, f"cannot read {path}: {exc}")
sections = parse_pypirc(text)
if index not in sections and index not in index_servers(text):
# Do not blame index-servers for a section that simply is not there.
return CredentialState(index, False, None, f"{path} has no [{index}] section")
lookup = _twine_lookup(index, path)
if lookup.error:
return CredentialState(index, False, None, f"twine cannot read {path}: {lookup.error}")
if lookup.available:
if not lookup.known:
return CredentialState(index, False, None, _ignored_section(index, path, text, sections))
username = str(lookup.config.get("username") or "").strip()
password = str(lookup.config.get("password") or "").strip()
if not password:
# twine knows the index but resolved no password for it: almost always
# a section that index-servers does not list.
detail = _ignored_section(index, path, text, sections)
return CredentialState(index, False, None, detail)
else:
# twine is not installed: replicate its rule locally.
if index not in index_servers(text):
return CredentialState(index, False, None, _ignored_section(index, path, text, sections))
section = sections[index]
username = section.get("username", "").strip()
password = section.get("password", "").strip()
if looks_like_placeholder(password):
return CredentialState(
index, False, str(path), f"[{index}] still contains a placeholder token"
)
if username != "__token__":
return CredentialState(
index,
False,
str(path),
f"[{index}] username is {username!r}; API tokens need the literal '__token__'",
)
return CredentialState(index, True, str(path))
def _ignored_section(
index: str, path: Path, text: str, sections: dict[str, dict[str, str]]
) -> str:
"""Name the real cause: absent section, or present but not listed by twine."""
if index not in sections:
return f"{path} has no [{index}] section"
if index not in index_servers(text):
return (
f"[{index}] exists in {path} but is not listed in "
"[distutils] index-servers, so twine ignores it entirely; "
"fix with `release auth --repair`"
)
return f"twine resolves no password for [{index}] from {path}"
def _twine_lookup(index: str, path: Path) -> TwineLookup:
"""Ask twine itself; ``available=False`` when twine is not installed.
``get_config`` is not enough on its own: it always contains a ``pypi`` entry
(and a ``testpypi`` one when listed) even when it never copied any
credentials, so the resolved username/password are what matter.
"""
try:
from twine import utils
except ImportError:
return TwineLookup(available=False, known=False)
try:
config = utils.get_config(str(path))
except Exception as exc: # twine reports malformed files as exceptions
return TwineLookup(available=True, known=False, error=f"{type(exc).__name__}: {exc}")
section = config.get(index)
if section is None:
return TwineLookup(available=True, known=False)
return TwineLookup(available=True, known=True, config=dict(section))
def index_servers(text: str) -> set[str]:
"""Names twine honours from ``[distutils] index-servers``.
Replicates ``twine.utils.get_config``, including its surprising asymmetry:
* key **absent** -> falls back to ``pypi testpypi``
* key **present but empty** -> an empty list, so *no* section is honoured
(the fallback does not apply), and every section looks configured while
twine silently ignores it
* continuation lines are part of the value, so ``index-servers =`` followed
by indented names is the normal form
"""
parser = configparser.RawConfigParser()
try:
parser.read_string(text)
except configparser.Error:
return set()
raw = parser.get("distutils", "index-servers", fallback="pypi testpypi")
return set(raw.split())
def validate_token(token: str) -> list[str]:
"""Problems with a pasted token. Empty and whitespace are fatal; the rest warn."""
return [problem for problem in _token_problems(token) if problem.startswith("ERROR")]
def token_warnings(token: str) -> list[str]:
return [problem for problem in _token_problems(token) if problem.startswith("WARN")]
def _token_problems(token: str) -> list[str]:
problems: list[str] = []
if not token:
problems.append("ERROR the token is empty")
return problems
if any(char.isspace() for char in token):
problems.append("ERROR the token contains whitespace (copy/paste artefact?)")
if not token.startswith("pypi-"):
problems.append(
"WARN PyPI API tokens start with 'pypi-'; a legacy account password also "
"works but needs username != '__token__'"
)
if len(token) < 20:
problems.append("WARN the token looks too short; check the paste was complete")
return problems
def unlisted_sections(path: Path | None = None) -> list[str]:
"""Known indexes whose section exists but is missing from ``index-servers``."""
path = path or pypirc_path()
if not path.is_file():
return []
text = path.read_text(encoding="utf-8", errors="ignore")
present = set(parse_pypirc(text))
listed = index_servers(text)
return [index for index in sorted(INDEXES) if index in present and index not in listed]
def repair_index_servers(path: Path | None = None) -> list[str]:
"""List sections that exist but are not in ``index-servers``.
This is the state a hand-edited file lands in: the section is present, so it
looks configured, but twine ignores it completely. Returns the indexes added.
"""
path = path or pypirc_path()
if not path.is_file():
return []
text = path.read_text(encoding="utf-8")
present = set(parse_pypirc(text))
added: list[str] = []
for index in sorted(INDEXES):
if index in present and index not in index_servers(text):
text = ensure_index_server(text, index)
added.append(index)
if added:
path.write_text(text, encoding="utf-8", newline="")
_restrict_permissions(path)
return added
def upsert_token(text: str, index: str, token: str) -> str:
"""Set ``password`` inside ``[index]``, creating the section if needed."""
lines = text.splitlines()
start = next((i for i, line in enumerate(lines) if line.strip() == f"[{index}]"), None)
if start is None:
if lines and lines[-1].strip():
lines.append("")
lines.extend(
[
f"[{index}]",
f"repository = {INDEXES[index]['upload_url']}",
"username = __token__",
f"password = {token}",
]
)
return "\n".join(lines) + "\n"
end = len(lines)
for offset in range(start + 1, len(lines)):
stripped = lines[offset].strip()
if stripped.startswith("[") and stripped.endswith("]"):
end = offset
break
insert_at = end
for offset in range(start + 1, end):
stripped = lines[offset].strip()
if stripped.startswith("password"):
lines[offset] = f"password = {token}"
return "\n".join(lines) + "\n"
if stripped.startswith(("username", "repository")):
insert_at = offset + 1
lines.insert(insert_at, f"password = {token}")
return "\n".join(lines) + "\n"
def ensure_index_server(text: str, index: str) -> str:
"""Make sure ``[distutils] index-servers`` lists ``index``."""
if index in index_servers(text):
return text
lines = text.splitlines()
if "[distutils]" not in (line.strip() for line in lines):
block = ["[distutils]", "index-servers =", f" {index}", ""]
return "\n".join(block + lines) + "\n"
start = next(i for i, line in enumerate(lines) if line.strip() == "[distutils]")
for offset in range(start + 1, len(lines)):
if lines[offset].strip().startswith("index-servers"):
lines.insert(offset + 1, f" {index}")
return "\n".join(lines) + "\n"
lines.insert(start + 1, f" {index}")
return "\n".join(lines) + "\n"
def _restrict_permissions(path: Path) -> None:
"""Best effort: owner-only. Windows ACLs need icacls, not chmod."""
try:
os.chmod(path, 0o600)
except OSError:
pass
def write_token(index: str, token: str, *, path: Path | None = None) -> Path:
"""Write ``token`` into ``[index]`` of the pypirc file, preserving comments."""
if index not in INDEXES:
raise ValueError(f"unknown index {index!r}; expected one of {sorted(INDEXES)}")
path = path or pypirc_path()
if path.is_file():
text = path.read_text(encoding="utf-8")
else:
text = DEFAULT_TEMPLATE
text = upsert_token(text, index, token)
text = ensure_index_server(text, index)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8", newline="")
_restrict_permissions(path)
return path
def prompt_for_token(index: str, *, prompt_fn=None, out=None) -> str | None:
"""Ask for a token with hidden input. Returns ``None`` if input is unavailable."""
out = out or sys.stdout
meta = INDEXES[index]
print(f"PyPI API token for [{index}]", file=out)
print(f" create one at: {meta['token_url']}", file=out)
if meta.get("register_url"):
print(f" needs an account first: {meta['register_url']}", file=out)
print(" input is hidden and is written only to your pypirc file", file=out)
ask = prompt_fn or getpass.getpass
try:
return ask("token: ").strip()
except (EOFError, KeyboardInterrupt, OSError):
# OSError covers environments where stdin cannot be read at all (pytest
# capture, a detached CI stdin): treat that as "cannot prompt", never as a
# crash.
print("", file=out)
print("aborted: no token entered", file=out)
return None
def read_token_from_stdin() -> str:
"""For CI: read the token from stdin instead of a prompt or the command line."""
return (sys.stdin.readline() or "").strip()
build/release/cli.py
-416
View File
@@ -1,416 +0,0 @@
"""Command line for ``python build/run.py release``.
Commands: ``preflight`` (local gate), ``audit`` (single wheel), ``auth`` (PyPI
credentials in ~/.pypirc), ``github`` (sync the public repo + publish a GitHub
Release), ``pypi`` (TestPyPI then PyPI). The publish commands are dry-run unless
``--apply`` is passed, and tokens are never accepted as command-line arguments.
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
_BUILD_ROOT = Path(__file__).resolve().parents[1]
if str(_BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(_BUILD_ROOT))
from common.paths import REPO_ROOT # noqa: E402
from common.verify.wheel import ( # noqa: E402
DEFAULT_RUNTIME_CONFIG,
default_verification_video,
verify_wheel,
verify_wheel_assets,
verify_wheel_runtime,
)
from common.verify.wheel_contents import inspect_wheel # noqa: E402
from release import auth, github, pypi # noqa: E402
from release.preflight import PreflightOptions, run_preflight # noqa: E402
def _cmd_preflight(argv: list[str]) -> int:
parser = argparse.ArgumentParser(
prog="build/run.py release preflight",
description="Gate a release locally: version, tests, wheel content, runtime.",
)
parser.add_argument("--version", help="release version (default: pyproject's dynamic version)")
parser.add_argument("--wheel-dir", type=Path, help="directory to inspect (default: SDK/<version>)")
parser.add_argument("--video", type=Path, help="clip for the end-to-end runtime check")
parser.add_argument("--config", default=DEFAULT_RUNTIME_CONFIG, help="config name for that check")
parser.add_argument("--frames", type=int, default=40, help="frames to process (default: 40)")
parser.add_argument("--skip-tests", action="store_true", help="skip pytest (recorded in the report)")
parser.add_argument("--longrun", action="store_true", help="include the slow long-run stability test")
parser.add_argument("--allow-dirty", action="store_true", help="allow a dirty working tree")
parser.add_argument(
"--force",
action="store_true",
help="accept a version that a git tag already marks as released",
)
parser.add_argument("--check-pypi", action="store_true", help="read PyPI and fail if the version exists")
parser.add_argument(
"--all-wheel-pythons",
action="store_true",
help="verify cp310/cp311/cp312 wheels, each in its own conda env",
)
parser.add_argument("--offline", action="store_true", help="reuse parent-env deps instead of PyPI")
parser.add_argument("--no-runtime", action="store_true", help="skip the isolated runtime check")
parser.add_argument("--no-assets", action="store_true", help="skip the bundled-asset check")
parser.add_argument("--keep-going", action="store_true", help="collect every failure instead of stopping")
parser.add_argument("--report-dir", type=Path, help="override build/release/reports")
args = parser.parse_args(argv)
options = PreflightOptions(
version=args.version,
wheel_dir=args.wheel_dir,
video=args.video,
config_name=args.config,
frames=args.frames,
skip_tests=args.skip_tests,
longrun=args.longrun,
allow_dirty=args.allow_dirty,
force=args.force,
check_pypi=args.check_pypi,
all_wheel_pythons=args.all_wheel_pythons,
offline=args.offline,
keep_going=args.keep_going,
report_dir=args.report_dir,
no_runtime=args.no_runtime,
no_assets=args.no_assets,
)
return run_preflight(options, repo_root=REPO_ROOT)
def _cmd_audit(argv: list[str]) -> int:
parser = argparse.ArgumentParser(
prog="build/run.py release audit",
description="Inspect one built wheel without a release context.",
)
parser.add_argument("wheel", type=Path, help="path to an .whl")
parser.add_argument("--expected-version", help="assert the wheel reports this version")
parser.add_argument(
"--strict-bundle",
action="store_true",
help="treat bundled-file drift from the current source tree as a failure",
)
parser.add_argument("--runtime", action="store_true", help="also install and run the wheel")
parser.add_argument("--assets", action="store_true", help="also assert bundled assets load")
parser.add_argument("--video", type=Path, help="clip for --runtime")
parser.add_argument("--config", default=DEFAULT_RUNTIME_CONFIG)
parser.add_argument("--frames", type=int, default=40)
parser.add_argument("--offline", action="store_true")
parser.add_argument(
"--python",
type=Path,
default=None,
help="interpreter to build the verification venv from (default: the running one)",
)
args = parser.parse_args(argv)
wheel = args.wheel
if not wheel.is_file():
print(f"[ERROR] wheel not found: {wheel}", file=sys.stderr)
return 2
exit_code = 0
result = inspect_wheel(
wheel,
expected_version=args.expected_version,
repo_root=REPO_ROOT,
strict_bundle=args.strict_bundle,
)
print(result.render())
if not result.ok:
exit_code = 1
if args.assets and args.runtime:
# One venv for both checks: building it (and downloading dependencies in
# online mode) is the expensive part.
video = args.video or default_verification_video(REPO_ROOT)
if video is None:
print("[FAIL] runtime: no verification clip; pass --video or set ORISYS_VERIFY_VIDEO")
return 1
try:
record = verify_wheel(
wheel,
video=video,
config_name=args.config,
frames=args.frames,
python_executable=args.python,
offline=args.offline,
)
print(f"[ok ] {record['imports']}")
print(f"[ok ] {record['assets']}")
print(f"[ok ] runtime: frames={record['frames']} flow={record['flow_shape']}")
except Exception as exc:
print(f"[FAIL] isolated verification: {type(exc).__name__}: {exc}")
exit_code = 1
return exit_code
if args.assets:
try:
print(verify_wheel_assets(wheel, python_executable=args.python))
except Exception as exc:
print(f"[FAIL] bundled assets: {type(exc).__name__}: {exc}")
exit_code = 1
if args.runtime:
video = args.video or default_verification_video(REPO_ROOT)
if video is None:
print("[FAIL] runtime: no verification clip; pass --video or set ORISYS_VERIFY_VIDEO")
return 1
try:
info = verify_wheel_runtime(
wheel,
video=video,
config_name=args.config,
frames=args.frames,
python_executable=args.python,
offline=args.offline,
)
print(f"[ok ] runtime: {info}")
except Exception as exc:
print(f"[FAIL] runtime: {type(exc).__name__}: {exc}")
exit_code = 1
return exit_code
def _cmd_github(argv: list[str]) -> int:
parser = argparse.ArgumentParser(
prog="build/run.py release github",
description="Sync orisys-sdk-release, tag, and publish a GitHub Release.",
)
parser.add_argument("--version", required=True, help="release version, e.g. 0.5.2")
parser.add_argument(
"--apply",
action="store_true",
help="actually mutate the release repo, push, and create the release",
)
parser.add_argument("--notes-file", type=Path, default=None, help="release notes markdown")
parser.add_argument(
"--attach-preflight",
action="store_true",
help="also upload the preflight report as a release asset (contains local paths)",
)
parser.add_argument(
"--allow-dirty-release",
action="store_true",
help="proceed even if the release repo has uncommitted changes",
)
parser.add_argument("--skip-fetch", action="store_true", help="skip `git fetch --tags`")
parser.add_argument(
"--skip-checks", action="store_true", help="skip the manifest's downstream checks"
)
parser.add_argument(
"--skip-release-upload",
action="store_true",
help="stop after pushing the tag and print the `gh release create` command",
)
args = parser.parse_args(argv)
return github.publish(
github.Options(
version=args.version,
apply=args.apply,
notes_file=args.notes_file,
attach_preflight=args.attach_preflight,
allow_dirty_release=args.allow_dirty_release,
skip_fetch=args.skip_fetch,
skip_checks=args.skip_checks,
skip_release_upload=args.skip_release_upload,
)
)
def _cmd_auth(argv: list[str]) -> int:
parser = argparse.ArgumentParser(
prog="build/run.py release auth",
description=(
"Inspect or set up PyPI credentials in ~/.pypirc. "
"Tokens are read from a hidden prompt or stdin, never from argv."
),
)
parser.add_argument(
"--set-token",
choices=sorted(auth.INDEXES),
default=None,
help="store a token for this index (prompts unless --token-stdin)",
)
parser.add_argument(
"--token-stdin",
action="store_true",
help="read the token from stdin instead of prompting (for scripts/CI)",
)
parser.add_argument(
"--repair",
action="store_true",
help=(
"add existing sections to [distutils] index-servers; twine ignores "
"sections that are not listed there"
),
)
parser.add_argument(
"--index",
choices=sorted(auth.INDEXES),
default="pypi",
help="index to report on when not setting a token (default: pypi)",
)
args = parser.parse_args(argv)
if args.repair:
path = auth.pypirc_path()
added = auth.repair_index_servers(path)
if added:
print(f"[ok ] added {added} to [distutils] index-servers in {path}")
else:
print(f"[ok ] nothing to repair in {path}")
for index in sorted(auth.INDEXES):
state = auth.credential_state(index)
flag = "[ok ]" if state.configured else "[-- ]"
detail = state.source if state.configured else state.problem
print(f"{flag} [{index}] {detail}")
return 0 if auth.credential_state(args.index).configured else 1
if args.set_token:
if args.token_stdin:
token = auth.read_token_from_stdin()
else:
token = auth.prompt_for_token(args.set_token)
if not token:
print("[FAIL] no token entered")
return 1
for problem in auth.validate_token(token):
print(f"[FAIL] {problem.removeprefix('ERROR ')}")
return 1
for warning in auth.token_warnings(token):
print(f"[warn] {warning.removeprefix('WARN ')}")
path = auth.write_token(args.set_token, token)
print(f"[ok ] wrote [{args.set_token}] credentials to {path}")
state = auth.credential_state(args.set_token)
if not state.configured:
print(f"[FAIL] the file still does not validate: {state.problem}")
return 1
print(f"[ok ] [{args.set_token}] credentials verified ({state.source})")
return 0
path = auth.pypirc_path()
print(f"config file: {path} ({'exists' if path.is_file() else 'missing'})")
for index in sorted(auth.INDEXES):
state = auth.credential_state(index)
if state.configured:
print(f"[ok ] [{index}] configured ({state.source})")
else:
print(f"[-- ] [{index}] not usable: {state.problem}")
unlisted = auth.unlisted_sections(path)
if unlisted:
print()
print(
f"[warn] {unlisted} have sections that [distutils] index-servers does not "
"list; twine ignores those entirely"
)
print(" fix with: python build/run.py release auth --repair")
print()
print("set a token with:")
for index in sorted(auth.INDEXES):
if not auth.credential_state(index).configured:
print(f" python build/run.py release auth --set-token {index}")
print(" (TestPyPI needs its own account; if you only hold a production token,")
print(" skip it and publish with `release pypi --pypi-only`)")
return 0 if auth.credential_state(args.index).configured else 1
def _cmd_pypi(argv: list[str]) -> int:
parser = argparse.ArgumentParser(
prog="build/run.py release pypi",
description="Upload the certified wheels to TestPyPI and then PyPI.",
)
parser.add_argument("--version", required=True, help="release version, e.g. 0.5.2")
parser.add_argument(
"--apply",
action="store_true",
help="actually upload (uploads are permanent and cannot be overwritten)",
)
parser.add_argument(
"--confirm-pypi",
action="store_true",
help="after TestPyPI, also publish to the real PyPI",
)
parser.add_argument(
"--pypi-only",
action="store_true",
help=(
"upload straight to PyPI, skipping the TestPyPI rehearsal "
"(for when you only hold a production token)"
),
)
parser.add_argument(
"--prompt-credentials",
action="store_true",
help="if a token is missing, prompt for it and store it in ~/.pypirc",
)
parser.add_argument(
"--token-stdin",
action="store_true",
help="read a missing token from stdin instead of prompting (scripts/CI)",
)
parser.add_argument(
"--skip-install-verify",
action="store_true",
help="skip fetching the artifact back out of the index",
)
parser.add_argument("--skip-check", action="store_true", help="skip `twine check`")
args = parser.parse_args(argv)
if args.pypi_only and args.confirm_pypi:
parser.error("--pypi-only and --confirm-pypi are mutually exclusive")
if args.token_stdin and not args.prompt_credentials:
parser.error("--token-stdin requires --prompt-credentials")
token = auth.read_token_from_stdin() if args.token_stdin else None
return pypi.publish(
pypi.Options(
version=args.version,
apply=args.apply,
confirm_pypi=args.confirm_pypi,
verify_install=not args.skip_install_verify,
skip_check=args.skip_check,
pypi_only=args.pypi_only,
prompt_credentials=args.prompt_credentials,
token=token,
)
)
def main(argv: list[str] | None = None) -> int:
argv = list(sys.argv[1:] if argv is None else argv)
if not argv or argv[0] in ("-h", "--help"):
print(__doc__)
print("Commands: preflight | audit | auth | github | pypi")
return 0
command, rest = argv[0], argv[1:]
if command == "preflight":
return _cmd_preflight(rest)
if command == "audit":
return _cmd_audit(rest)
if command == "auth":
return _cmd_auth(rest)
if command == "github":
return _cmd_github(rest)
if command == "pypi":
return _cmd_pypi(rest)
print(f"Unknown release command: {command!r}", file=sys.stderr)
print("Choose from: preflight, audit, auth, github, pypi", file=sys.stderr)
return 2
if __name__ == "__main__":
raise SystemExit(main())
build/release/github.py
-438
View File
@@ -1,438 +0,0 @@
"""Sync the public release repo and publish a GitHub Release.
``python build/run.py release github`` runs, in order:
1. certified-wheels gate (a passing ``release preflight`` report must exist and
every wheel must still hash to what it recorded)
2. refuse if the version is already tagged or already has a GitHub Release
3. sync the open-source subset of this checkout into ``orisys-sdk-release``
(``tools/sync_downstreams.py`` + the manifest's ``checks``)
4. stage ``SDK/<version>/*.whl`` into ``orisys-sdk-release/release/<version>/``
5. commit, tag ``v<version>``, push branch + tag
6. create the GitHub Release with the wheels attached, then verify the assets
Nothing mutates anything unless ``--apply`` is passed: the default is a dry run
that prints every command it would run.
"""
from __future__ import annotations
import json
import shutil
import subprocess
import sys
from dataclasses import dataclass, field
from pathlib import Path
_BUILD_ROOT = Path(__file__).resolve().parents[1]
if str(_BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(_BUILD_ROOT))
from common.paths import REPO_ROOT # noqa: E402
from release.publish import GateResult, PublishGateError, load_gate # noqa: E402
MANIFEST_PATH = REPO_ROOT / "tools" / "downstream_sync_manifest.json"
SYNC_TOOL = REPO_ROOT / "tools" / "sync_downstreams.py"
RELEASE_TARGET = "orisys-sdk-release"
@dataclass
class Options:
version: str
apply: bool = False
notes_file: Path | None = None
attach_preflight: bool = False
allow_dirty_release: bool = False
skip_fetch: bool = False
skip_checks: bool = False
skip_release_upload: bool = False
@dataclass
class CommandStep:
description: str
argv: list[str]
cwd: Path | None = None
@dataclass
class Plan:
release_repo: Path
branch: str
tag: str
assets: list[Path] = field(default_factory=list)
steps: list[CommandStep] = field(default_factory=list)
def manifest_entry() -> dict:
"""The manifest entry for the public release repo."""
payload = json.loads(MANIFEST_PATH.read_text(encoding="utf-8"))
for item in payload.get("downstreams", []):
if item.get("name") == RELEASE_TARGET:
return item
raise PublishGateError(f"{MANIFEST_PATH} has no downstream named {RELEASE_TARGET!r}")
def release_repo_path() -> Path:
return (REPO_ROOT / manifest_entry()["path"]).resolve()
def _git(repo: Path, *args: str, check: bool = True) -> str:
completed = subprocess.run(
["git", "-C", str(repo), *args],
capture_output=True,
text=True,
check=False,
)
if check and completed.returncode != 0:
raise PublishGateError(
f"git {' '.join(args)} failed in {repo}:\n{completed.stderr.strip()}"
)
return (completed.stdout or "").strip()
def build_release_notes(version: str, wheels: list[Path], git_commit: str | None) -> str:
lines = [
f"orisys {version}",
"",
"Wheels (Python 3.10 / 3.11 / 3.12, Windows x64 and Linux x86_64):",
]
lines.extend(f"- `{wheel.name}`" for wheel in wheels)
lines.append("")
if git_commit:
lines.append(
f"Built from `{git_commit[:12]}` and certified by `release preflight`."
)
return "\n".join(lines) + "\n"
def build_github_steps(
*,
version: str,
branch: str,
tag: str,
assets: list[Path],
notes_file: Path | None,
create_tag: bool = True,
title: str | None = None,
) -> list[CommandStep]:
"""The mutating commands, as pure data so they can be asserted in tests.
``create_tag=False`` is the resume path: the tag already exists (a previous
run pushed it before failing), so only the remaining steps are emitted.
"""
release_argv = [
"gh",
"release",
"create",
tag,
"--verify-tag",
"--title",
title or f"v{version}",
"--latest",
]
if notes_file is not None:
release_argv += ["--notes-file", str(notes_file)]
release_argv += [str(asset) for asset in assets]
steps: list[CommandStep] = []
if create_tag:
steps.append(CommandStep("create tag", ["git", "tag", tag]))
steps.append(CommandStep("push branch", ["git", "push", "origin", branch]))
steps.append(CommandStep("push tag", ["git", "push", "origin", tag]))
steps.append(CommandStep("create GitHub Release with assets", release_argv))
return steps
def non_github_remote_problem(repo: Path) -> str | None:
"""Why ``gh`` cannot target this repo, or ``None`` if the remote is fine.
``gh`` resolves its repository from the working directory. Running it from
this checkout instead of the release repo is a real failure mode: the SDK
repo's origin is an internal host, and ``gh`` reports the confusing "none of
the git remotes ... point to a known GitHub host".
"""
remote = _git(repo, "remote", "get-url", "origin", check=False)
if "github.com" in remote:
return None
return (
f"the release repo's origin is {remote!r}, which is not a GitHub host. "
"`gh` resolves the repository from the working directory, so point origin "
"at github.com before publishing a release"
)
def _check_gh(repo: Path) -> list[str]:
"""Problems that would stop the release step, reported before any mutation."""
problems: list[str] = []
if shutil.which("gh") is None:
problems.append(
"the `gh` CLI is not on PATH. Install it (`winget install --id GitHub.cli`) "
"and authenticate (`gh auth login`), or use --skip-release-upload"
)
return problems
remote_problem = non_github_remote_problem(repo)
if remote_problem:
problems.append(remote_problem)
completed = subprocess.run(
["gh", "auth", "status"], capture_output=True, text=True, check=False
)
if completed.returncode != 0:
problems.append("`gh auth status` failed; run `gh auth login` first")
return problems
def _existing_release_tags(repo: Path, version: str) -> list[str]:
wanted = {version, f"v{version}"}
tags = set(_git(repo, "tag", "--list").splitlines())
return sorted(wanted & tags)
def _release_exists(repo: Path, tag: str) -> bool | None:
"""Is there already a GitHub Release for ``tag``?
Returns ``None`` when that cannot be determined (``gh`` unavailable), so the
caller can distinguish "not published yet" from "unknown" instead of treating
a missing tool as proof that nothing was published.
Must run with ``cwd`` set to the release repo: `gh` resolves the target
repository from the working directory, and this checkout's own remote is an
internal host.
"""
if shutil.which("gh") is None:
return None
completed = subprocess.run(
["gh", "release", "view", tag, "--json", "tagName"],
capture_output=True,
text=True,
check=False,
cwd=str(repo),
)
return completed.returncode == 0
def _sync(*, run_checks: bool) -> None:
"""Copy the public subset into the release repo and run its checks."""
python = sys.executable
for command in (
[python, str(SYNC_TOOL), "apply", "--target", RELEASE_TARGET],
*(
[[python, str(SYNC_TOOL), "run-checks", "--target", RELEASE_TARGET]]
if run_checks
else []
),
):
completed = subprocess.run(command, cwd=REPO_ROOT, check=False)
if completed.returncode != 0:
raise PublishGateError(
f"`{' '.join(command[1:])}` failed with exit {completed.returncode}"
)
def _stage_wheels(wheels: list[Path], asset_dir: Path) -> None:
asset_dir.mkdir(parents=True, exist_ok=True)
for wheel in wheels:
shutil.copy2(wheel, asset_dir / wheel.name)
def publish(options: Options) -> int:
version = options.version
mode = "APPLY" if options.apply else "DRY RUN"
print(f"=== release github: orisys {version} ({mode}) ===")
# --- 1. gate -------------------------------------------------------
try:
gate: GateResult = load_gate(REPO_ROOT, version)
except PublishGateError as exc:
print(f"[FAIL] {exc}")
return 1
print(f"[ok ] preflight report: {gate.report_path}")
print(f" certified commit: {gate.git_commit}")
for wheel in gate.wheels:
print(f" [ok] {wheel.name} sha256={wheel.sha256[:12]} {wheel.size} bytes")
for warning in gate.warnings:
print(f"[warn] {warning}")
# --- 2. target repo ------------------------------------------------
try:
repo = release_repo_path()
except PublishGateError as exc:
print(f"[FAIL] {exc}")
return 1
if not (repo / ".git").exists():
print(f"[FAIL] {repo} is not a git repository")
return 1
try:
if not options.skip_fetch:
_git(repo, "fetch", "--tags", "--quiet")
print(f"[ok ] git fetch --tags in {repo}")
branch = _git(repo, "rev-parse", "--abbrev-ref", "HEAD")
dirty = _git(repo, "status", "--porcelain")
except PublishGateError as exc:
print(f"[FAIL] {exc}")
return 1
if dirty and not options.allow_dirty_release:
preview = "\n".join(dirty.splitlines()[:10])
print(
f"[FAIL] {repo} has uncommitted changes; commit or stash them first "
f"(or pass --allow-dirty-release):\n{preview}"
)
return 1
print(f"[ok ] release repo: {repo} ({branch}) clean={not dirty}")
tag = f"v{version}"
existing = _existing_release_tags(repo, version)
# A tag that exists without a GitHub Release means a previous run got as far
# as pushing the tag and then failed (or the upload was left manual). That is
# exactly the resumable state, so continue instead of refusing.
create_tag = not existing
if existing:
print(
f"[warn] tag(s) {existing} already exist in {repo}; skipping tag creation "
"and continuing to the release upload"
)
release_state = _release_exists(repo, tag)
if release_state is True:
print(f"[FAIL] GitHub Release {tag} already exists; nothing to publish")
return 1
if release_state is None:
print(
"[warn] cannot tell whether a GitHub Release for "
f"{tag} already exists (`gh` not installed); --apply will stop at the gh check"
)
if create_tag:
print(f"[ok ] tag {tag} does not exist in {repo}")
elif create_tag:
print(f"[ok ] no existing tag or GitHub Release for {tag}")
else:
print(f"[ok ] no GitHub Release for {tag} yet")
# A dry run is a preview of what would happen, so missing tooling is a warning
# there and a hard stop only when we are about to mutate something.
if options.skip_release_upload:
print("[warn] --skip-release-upload: the GitHub Release will be left to you")
else:
gh_problems = _check_gh(repo)
if gh_problems:
for problem in gh_problems:
print(f"[{'FAIL' if options.apply else 'warn'}] {problem}")
if options.apply:
return 1
else:
print("[ok ] gh CLI is installed and authenticated")
# --- 3. plan the mutating steps ------------------------------------
wheels = [REPO_ROOT / "SDK" / version / record.name for record in gate.wheels]
missing = [str(wheel) for wheel in wheels if not wheel.is_file()]
if missing:
print(f"[FAIL] wheels missing from disk: {missing}")
return 1
asset_dir = repo / "release" / version
assets = [asset_dir / wheel.name for wheel in wheels]
notes_file = options.notes_file
steps = build_github_steps(
version=version,
branch=branch,
tag=tag,
assets=assets,
notes_file=notes_file,
create_tag=create_tag,
)
print()
print(f"[plan] sync {len(manifest_entry().get('sync', []))} entry(ies) into {repo}")
print(f"[plan] stage {len(wheels)} wheel(s) into {asset_dir}")
if create_tag:
print(f"[plan] commit {repo} and create tag {tag}")
else:
print(f"[plan] commit {repo} (tag {tag} already exists)")
for step in steps:
manual = options.skip_release_upload and step.description.startswith(
"create GitHub Release"
)
prefix = "[plan] (left manual)" if manual else "[plan]"
print(f"{prefix} {step.description}: {' '.join(step.argv)}")
print()
if not options.apply:
print("dry run: nothing was changed. Re-run with --apply to execute.")
return 0
# --- 4. execute ----------------------------------------------------
try:
commit_before = _git(repo, "rev-parse", "HEAD")
_sync(run_checks=not options.skip_checks)
_stage_wheels(wheels, asset_dir)
if options.attach_preflight:
shutil.copy2(gate.report_path, asset_dir / f"{version}-preflight.json")
_git(repo, "add", "-A")
pending = _git(repo, "status", "--porcelain")
if pending:
_git(repo, "commit", "-m", f"release {version}")
print(f"[ok ] committed {len(pending.splitlines())} change(s) in {repo}")
else:
print("[ok ] nothing new to commit in the release repo")
if create_tag:
_git(repo, "tag", tag)
# Pushing an already-up-to-date ref is a no-op, so resuming is safe.
_git(repo, "push", "origin", branch)
_git(repo, "push", "origin", tag)
print(f"[ok ] pushed {branch} and {tag}")
except PublishGateError as exc:
print(f"[FAIL] {exc}")
return 1
release_argv = next(
step.argv for step in steps if step.description.startswith("create GitHub Release")
)
if options.skip_release_upload:
print()
print("[todo] publish the release yourself:")
print(" " + " ".join(release_argv))
print()
print("=== release github: repo synced and tagged; release upload left manual ===")
return 0
# `gh` resolves its target repository from the working directory, so it must
# run inside the release repo - this checkout's own origin is an internal host.
completed = subprocess.run(release_argv, check=False, cwd=str(repo))
if completed.returncode != 0:
print(f"[FAIL] gh release create exited {completed.returncode}")
return 1
print(f"[ok ] GitHub Release {tag} created")
# --- 5. verify the published assets --------------------------------
completed = subprocess.run(
["gh", "release", "view", tag, "--json", "assets,tagName"],
capture_output=True,
text=True,
check=False,
cwd=str(repo),
)
if completed.returncode != 0:
print("[FAIL] could not read back the release to verify assets")
return 1
payload = json.loads(completed.stdout or "{}")
published = {asset["name"] for asset in payload.get("assets", [])}
expected = {asset.name for asset in assets}
if options.attach_preflight:
expected.add(f"{version}-preflight.json")
if published != expected:
print(
f"[FAIL] release asset mismatch:\n expected {sorted(expected)}\n"
f" published {sorted(published)}"
)
return 1
print(f"[ok ] verified {len(published)} asset(s) on the release")
print()
print(f"=== release github PASSED (commit {commit_before[:12]} -> {_git(repo, 'rev-parse', 'HEAD')[:12]}) ===")
return 0
build/release/preflight.py
-570
View File
@@ -1,570 +0,0 @@
"""Local release gate: everything that must hold before publishing a wheel.
``release preflight`` answers one question with an exit code: is this checkout,
this version and these wheels safe to publish? It never publishes anything, and
by default it makes no network calls.
Steps, in order:
1. ``repo-state`` - git repository, clean working tree, recorded commit
2. ``version-consistency`` - __version__ == pyproject dynamic attr == wheel names
3. ``version-strings`` - no stale version literal left in tracked prose
4. ``duplicate-guard`` - already-released version (git tag, optionally PyPI)
5. ``tests`` - pytest (slow/hardware markers excluded)
6. ``bundle-precheck`` - every bundle item resolves; required assets present
7. ``wheel-inspection`` - static metadata/content assertions per wheel
8. ``wheel-runtime`` - one isolated venv per wheel: bundled assets load and
real frames flow through the public pipeline
Output lands in ``build/release/reports/<version>/preflight.{json,txt}``, and is
the contract the publish commands (``release github`` / ``release pypi``) consume:
they refuse to ship anything this run did not certify.
"""
from __future__ import annotations
import hashlib
import json
import platform
import subprocess
import sys
import time
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
_BUILD_ROOT = Path(__file__).resolve().parents[1]
if str(_BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(_BUILD_ROOT))
from common.paths import REPO_ROOT # noqa: E402
from common.pyproject_utils import bundle_items, get_version # noqa: E402
from common.verify.wheel import ( # noqa: E402
DEFAULT_RUNTIME_CONFIG,
default_verification_video,
verify_wheel,
)
from common.verify.wheel_contents import inspect_wheel # noqa: E402
from release import report as report_mod # noqa: E402
from release import version as version_mod # noqa: E402
from wheel.layout import resolve_bundle_source_files, verify_required_assets # noqa: E402
@dataclass
class Step:
name: str
ok: bool
detail: str = ""
duration_s: float = 0.0
data: dict = field(default_factory=dict)
@dataclass
class PreflightOptions:
version: str | None = None
wheel_dir: Path | None = None
video: Path | None = None
config_name: str = DEFAULT_RUNTIME_CONFIG
frames: int = 40
skip_tests: bool = False
longrun: bool = False
allow_dirty: bool = False
force: bool = False
check_pypi: bool = False
all_wheel_pythons: bool = False
offline: bool = False
keep_going: bool = False
report_dir: Path | None = None
no_runtime: bool = False
no_assets: bool = False
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with Path(path).open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _now() -> str:
return datetime.now(timezone.utc).isoformat(timespec="seconds")
class Preflight:
def __init__(self, options: PreflightOptions, repo_root: Path | None = None) -> None:
self.options = options
self.repo_root = Path(repo_root or REPO_ROOT)
self.steps: list[Step] = []
self.git: dict = {"commit": None, "branch": None, "dirty": None}
self.wheels: list[Path] = []
self.wheel_dir: Path | None = None
self.runtime_records: list[dict] = []
self.aborted = False
self.ok = False
if options.version:
self.version = options.version
self.version_source = "explicit --version"
elif (self.repo_root / "pyproject.toml").is_file():
self.version = get_version(self.repo_root)
self.version_source = "pyproject.toml dynamic attr"
else:
# Not version-shaped on purpose: this is a sentinel for "unresolved",
# and it must not look like a release version to the version-string scan.
self.version = "unknown"
self.version_source = "unresolved"
self.report_dir = report_mod.reports_root(self.repo_root, options.report_dir) / self.version
# -- helpers ---------------------------------------------------------
def _git(self, *args: str) -> str:
try:
completed = subprocess.run(
["git", *args],
cwd=self.repo_root,
capture_output=True,
text=True,
check=False,
)
except OSError:
return ""
return (completed.stdout or "").strip()
def _execute(self, step_fn) -> None:
started = time.perf_counter()
try:
step = step_fn()
except Exception as exc: # a crashed step is a failed step, never a silent pass
step = Step(name=getattr(step_fn, "__name__", "step").lstrip("_").replace("_", "-"),
ok=False, detail=f"crashed: {type(exc).__name__}: {exc}")
step.duration_s = round(time.perf_counter() - started, 3)
self.steps.append(step)
flag = "ok " if step.ok else "FAIL"
print(f"[{flag}] {step.name} ({step.duration_s:.2f}s)")
for line in str(step.detail).splitlines():
print(f" {line}")
if not step.ok and not self.options.keep_going:
self.aborted = True
print(" stopping: pass --keep-going to collect every failure")
# -- steps -----------------------------------------------------------
def _repo_state(self) -> Step:
if not (self.repo_root / ".git").exists():
return Step("repo-state", False, f"{self.repo_root} is not a git repository")
commit = self._git("rev-parse", "HEAD")
branch = self._git("rev-parse", "--abbrev-ref", "HEAD")
porcelain = self._git("status", "--porcelain")
dirty = bool(porcelain)
self.git = {"commit": commit, "branch": branch, "dirty": dirty}
if dirty and not self.options.allow_dirty:
preview = "\n".join(porcelain.splitlines()[:20])
return Step(
"repo-state",
False,
f"working tree is dirty; commit, stash or pass --allow-dirty:\n{preview}",
data=self.git,
)
detail = f"{branch} @ {commit[:12]} dirty={dirty} version={self.version} ({self.version_source})"
if dirty:
detail += " (accepted via --allow-dirty)"
return Step("repo-state", True, detail, data=self.git)
def _version_consistency(self) -> Step:
problems: list[str] = []
data: dict = {"version": self.version, "source": self.version_source}
attr = version_mod.dynamic_version_attr(self.repo_root)
data["dynamic_attr"] = attr
if attr is None:
problems.append(
"pyproject.toml no longer declares tool.setuptools.dynamic.version.attr"
)
if (self.repo_root / "pyproject.toml").is_file():
resolved = get_version(self.repo_root)
data["resolved_version"] = resolved
if resolved != self.version:
problems.append(
f"src/orisys/__init__.py __version__ is {resolved!r} but the release "
f"version is {self.version!r}"
)
sdk_dirs = version_mod.sdk_versions(self.repo_root)
data["sdk_versions"] = sdk_dirs
for wheel in version_mod.sdk_wheels(self.repo_root, self.version):
found = version_mod.wheel_version(wheel)
if found != self.version:
problems.append(f"{wheel.name} carries version {found!r}, expected {self.version!r}")
detail = (
f"version={self.version} ({self.version_source}) dynamic_attr={attr}\n"
f"SDK/ versions present: {sdk_dirs or 'none'}"
)
if problems:
detail += "\n" + "\n".join(problems)
return Step("version-consistency", not problems, detail, data=data)
def _duplicate_guard(self) -> Step:
"""Fail only when this version has already been *released*.
A populated ``SDK/<version>/`` is the normal input to preflight (it is
what gets verified), so it is reported, not blocked. What must not happen
silently is re-releasing a version that is already out: that is detected
offline through git tags and, opt-in, against PyPI.
"""
problems: list[str] = []
lines: list[str] = []
existing = version_mod.sdk_wheels(self.repo_root, self.version)
if existing:
lines.append(
f"SDK/{self.version}/ holds {len(existing)} staged wheel(s) to verify: "
f"{[w.name for w in existing]}"
)
else:
lines.append(
f"SDK/{self.version}/ is empty; nothing is staged yet "
"(build with `python build/run.py wheel-win` / `manylinux`)"
)
tags = version_mod.git_tags(self.repo_root)
tag_hits = version_mod.released_tags_for(self.version, tags)
if tag_hits:
message = f"git tag(s) {tag_hits} already mark {self.version} as released"
if self.options.force:
lines.append(message + " (accepted via --force)")
else:
problems.append(
message + "; pass --force to re-verify or intentionally re-release"
)
else:
lines.append(
f"no local git tag marks {self.version} as released "
f"({len(tags)} tag(s) checked in this repo and {version_mod.RELEASE_REPO_DIRNAME})"
)
if self.options.check_pypi:
versions, error = version_mod.pypi_released_versions()
if error:
problems.append(f"PyPI lookup failed: {error}")
elif versions is None:
problems.append("PyPI lookup returned no version list")
elif self.version in versions:
problems.append(
f"orisys {self.version} is already published on PyPI; a PyPI upload "
"cannot be undone. Published: " + ", ".join(versions)
)
else:
lines.append(
f"PyPI has no {self.version}; published: " + (", ".join(versions) or "none")
)
else:
lines.append("PyPI existence check skipped (pass --check-pypi to enable)")
detail = "\n".join(lines)
if problems:
detail += "\n" + "\n".join(problems)
return Step("duplicate-guard", not problems, detail)
def _version_strings(self) -> Step:
"""Keep the version out of prose: one source, everything else derived.
Scans tracked text for version-looking literals that disagree with the
release version. Dependency constraints are ignored by design; deliberate
exceptions live in ``version.VERSION_STRING_ALLOWLIST``.
"""
findings = version_mod.find_version_literals(self.repo_root, self.version)
if not findings:
scanned = len(version_mod.tracked_text_files(self.repo_root))
return Step(
"version-strings",
True,
f"no stale version literal in {scanned} scanned file(s); "
f"version {self.version} lives only where it is derived from",
)
by_file: dict[str, list[str]] = {}
for path, lineno, literal in findings:
by_file.setdefault(path, []).append(f"{path}:{lineno} -> {literal}")
detail = (
f"{len(findings)} version literal(s) disagree with {self.version}. "
"Update them, or make the file version-agnostic, or add a documented "
"entry to build/release/version.py::VERSION_STRING_ALLOWLIST:\n"
+ "\n".join(line for lines in by_file.values() for line in lines)
)
return Step("version-strings", False, detail, data={"count": len(findings)})
def _tests(self) -> Step:
if self.options.skip_tests:
return Step(
"tests",
True,
"WARNING: test gate SKIPPED via --skip-tests; this run cannot certify a release",
data={"status": "skipped"},
)
marker = "not hardware" if self.options.longrun else "not slow and not hardware"
self.report_dir.mkdir(parents=True, exist_ok=True)
junit = self.report_dir / "junit.xml"
command = [
sys.executable,
"-m",
"pytest",
"-q",
"-m",
marker,
f"--junitxml={junit}",
]
try:
completed = subprocess.run(
command, cwd=self.repo_root, capture_output=True, text=True, check=False
)
except FileNotFoundError as exc:
return Step("tests", False, f"cannot run pytest: {exc}")
ok = completed.returncode == 0
tail = ((completed.stdout or "") + (completed.stderr or "")).strip().splitlines()[-12:]
detail = f"pytest -m {marker!r} in {sys.executable}\nexit code {completed.returncode}"
if tail:
detail += "\n" + "\n".join(tail)
return Step("tests", ok, detail, data={"marker": marker, "junit": str(junit)})
def _bundle_precheck(self) -> Step:
if not (self.repo_root / "pyproject.toml").is_file():
return Step("bundle-precheck", False, "pyproject.toml not found")
items = bundle_items(self.repo_root)
problems: list[str] = []
counts: dict[str, int] = {}
for item in items:
files = resolve_bundle_source_files(self.repo_root, item)
counts[str(item.get("name"))] = len(files)
if not files:
problems.append(
f"bundle item {item.get('name')!r} resolved 0 files; its patterns match nothing"
)
try:
verify_required_assets(self.repo_root, items)
except FileNotFoundError as exc:
problems.append(str(exc))
detail = "resolved files per bundle item: " + json.dumps(counts)
if problems:
detail += "\n" + "\n".join(problems)
return Step("bundle-precheck", not problems, detail, data={"counts": counts})
def _discover_wheels(self) -> str | None:
"""Populate self.wheels; return an error message when nothing was found."""
directory = (
Path(self.options.wheel_dir)
if self.options.wheel_dir
else self.repo_root / "SDK" / self.version
)
self.wheel_dir = directory
if not directory.is_dir():
return (
f"{directory} does not exist. Build first "
"(`python build/run.py wheel`) or pass --wheel-dir."
)
self.wheels = sorted(directory.glob("*.whl"))
if not self.wheels:
return f"no *.whl found in {directory}. Build first or pass --wheel-dir."
return None
def _wheel_inspection(self) -> Step:
problem = self._discover_wheels()
if problem:
return Step("wheel-inspection", False, problem)
problems: list[str] = []
blocks: list[str] = []
for wheel in self.wheels:
result = inspect_wheel(
wheel,
expected_version=self.version,
repo_root=self.repo_root,
strict_bundle=True,
)
blocks.append(result.render())
if not result.ok:
problems.append(
f"{wheel.name}: " + ", ".join(check.name for check in result.failures)
)
detail = f"{len(self.wheels)} wheel(s) in {self.wheel_dir}\n" + "\n".join(blocks)
if problems:
detail += "\nfailed checks:\n" + "\n".join(problems)
return Step("wheel-inspection", not problems, detail, data={"count": len(self.wheels)})
def _platform_token(self) -> str:
return "win_amd64" if sys.platform == "win32" else "manylinux"
def _runtime_targets(self) -> list[tuple[str, Path, Path | None]]:
token = self._platform_token()
if self.options.all_wheel_pythons:
targets: list[tuple[str, Path, Path | None]] = []
for tag in ("cp310", "cp311", "cp312"):
match = next(
(w for w in self.wheels if f"-{tag}-" in w.name and token in w.name), None
)
if match is None:
raise RuntimeError(
f"no {tag} {token} wheel among {[w.name for w in self.wheels]}"
)
executable = version_mod.python_for_tag(tag)
if executable is None:
raise RuntimeError(
f"cannot resolve an interpreter for {tag}; create the conda env "
f"py{tag[2:]} or set CONDA_ENVS_ROOT"
)
targets.append((tag, match, executable))
return targets
tag = f"cp{sys.version_info.major}{sys.version_info.minor}"
match = next((w for w in self.wheels if f"-{tag}-" in w.name and token in w.name), None)
if match is None:
raise RuntimeError(
f"no {tag} {token} wheel among {[w.name for w in self.wheels]}; pass "
"--all-wheel-pythons to verify every ABI in its own environment, or run from "
"a matching interpreter"
)
return [(tag, match, None)]
def _wheel_runtime(self) -> Step:
if self.options.no_runtime:
return Step(
"wheel-runtime",
True,
"WARNING: runtime verification SKIPPED via --no-runtime",
data={"status": "skipped"},
)
video = self.options.video or default_verification_video(self.repo_root)
if video is None:
return Step(
"wheel-runtime",
False,
"no verification clip found. Pass --video, set ORISYS_VERIFY_VIDEO, or commit a "
"clip under data/sample/. The runtime check is never silently skipped.",
)
video = Path(video)
if not video.is_file():
return Step("wheel-runtime", False, f"verification clip not found: {video}")
try:
targets = self._runtime_targets()
except RuntimeError as exc:
return Step("wheel-runtime", False, str(exc))
problems: list[str] = []
lines: list[str] = [f"clip: {video}", f"config: {self.options.config_name}"]
for tag, wheel, executable in targets:
label = f"{tag} {wheel.name}"
try:
# One venv per wheel: it runs the import smoke test, the bundled
# asset check and the frame pipeline together.
record = verify_wheel(
wheel,
video=video,
config_name=self.options.config_name,
frames=self.options.frames,
python_executable=executable,
offline=self.options.offline,
check_assets=not self.options.no_assets,
)
except subprocess.CalledProcessError as exc:
problems.append(f"{label}: command failed ({exc})")
continue
except Exception as exc:
problems.append(f"{label}: {type(exc).__name__}: {exc}")
continue
lines.append(
f"{label}: {record['assets']} | frames={record['frames']} "
f"flow={record['flow_shape']}"
)
self.runtime_records.append({"tag": tag, **record})
detail = "\n".join(lines)
if problems:
detail += "\nfailures:\n" + "\n".join(problems)
return Step("wheel-runtime", not problems, detail, data={"records": self.runtime_records})
# -- driver ----------------------------------------------------------
def run(self) -> int:
started_at = _now()
started = time.perf_counter()
print(f"=== orisys release preflight: version {self.version} ({self.version_source}) ===")
print(f"repo: {self.repo_root}")
print(f"report: {self.report_dir}")
print()
try:
for step_fn in (
self._repo_state,
self._version_consistency,
self._version_strings,
self._duplicate_guard,
self._tests,
self._bundle_precheck,
self._wheel_inspection,
self._wheel_runtime,
):
self._execute(step_fn)
if self.aborted:
break
finally:
self.ok = bool(self.steps) and all(step.ok for step in self.steps) and not self.aborted
duration = round(time.perf_counter() - started, 3)
payload = {
"version": self.version,
"version_source": self.version_source,
"ok": self.ok,
"aborted": self.aborted,
"started_at": started_at,
"finished_at": _now(),
"duration_s": duration,
"git": self.git,
"interpreter": {
"executable": sys.executable,
"version": platform.python_version(),
"platform": sys.platform,
},
"options": {key: value for key, value in asdict(self.options).items()
if not isinstance(value, Path)},
"steps": [asdict(step) for step in self.steps],
"wheels": [
{
"name": wheel.name,
"size": wheel.stat().st_size,
"sha256": _sha256_file(wheel),
}
for wheel in self.wheels
],
}
payload["options"]["wheel_dir"] = str(self.options.wheel_dir) if self.options.wheel_dir else None
payload["options"]["video"] = str(self.options.video) if self.options.video else None
json_path, text_path = report_mod.write_report(payload, self.report_dir)
print()
print(f"report: {json_path}")
print(f"report: {text_path}")
print()
print(f"=== preflight {'PASSED' if self.ok else 'FAILED'} ({duration}s) ===")
return 0 if self.ok else 1
def run_preflight(options: PreflightOptions, repo_root: Path | None = None) -> int:
return Preflight(options, repo_root=repo_root).run()
build/release/publish.py
-158
View File
@@ -1,158 +0,0 @@
"""Shared publish gate: certify a release from its preflight report.
Both publish commands (``release github`` and ``release pypi``) refuse to act
unless a passing ``release preflight`` report exists for the exact version and
every wheel still hashes to what that report recorded. The report is the contract
between "we verified this" and "we shipped this"; re-deriving the checks here
would just be a second, weaker opinion.
"""
from __future__ import annotations
import hashlib
import json
from dataclasses import dataclass, field
from pathlib import Path
# Options that weaken the gate. A release must not be certified by a run that
# waived any of them: re-run preflight on a clean commit instead.
WAIVED_GUARD_KEYS = ("skip_tests", "allow_dirty", "force", "no_runtime", "no_assets")
class PublishGateError(RuntimeError):
"""The preflight report does not certify these wheels."""
@dataclass
class WheelRecord:
name: str
sha256: str
size: int = 0
@dataclass
class GateResult:
version: str
git_commit: str | None
report_path: Path
wheels: list[WheelRecord] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with Path(path).open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def evaluate_gate(
payload: dict,
version: str,
wheels: dict[str, Path],
*,
head_commit: str | None = None,
) -> tuple[list[str], list[str]]:
"""Return ``(problems, warnings)`` for a report payload and the wheels on disk.
Pure function: no I/O beyond hashing the wheels, so it is cheap to test.
"""
problems: list[str] = []
warnings: list[str] = []
if str(payload.get("version")) != version:
problems.append(
f"report is for version {payload.get('version')!r}, not {version!r}"
)
if not payload.get("ok"):
problems.append(
"report result is FAIL - run `release preflight --keep-going` and fix "
"everything before publishing"
)
if payload.get("aborted"):
problems.append("report stopped early (aborted); it certifies nothing")
options = payload.get("options") or {}
waived = sorted(key for key in WAIVED_GUARD_KEYS if options.get(key))
if waived:
problems.append(
"report waived gate options "
+ ", ".join(f"--{key.replace('_', '-')}" for key in waived)
+ "; commit the tree and re-run `release preflight` without them"
)
recorded = {item.get("name"): item.get("sha256") for item in payload.get("wheels") or []}
if not recorded:
problems.append("report lists no wheels")
missing = sorted(set(recorded) - set(wheels))
if missing:
problems.append(f"wheels in the report are missing from disk: {missing}")
extra = sorted(set(wheels) - set(recorded))
if extra:
problems.append(f"wheels on disk are not covered by the report: {extra}")
for name in sorted(set(recorded) & set(wheels)):
actual = sha256_file(wheels[name])
if actual != recorded[name]:
problems.append(
f"{name} changed after preflight: report {recorded[name][:12]}, "
f"disk {actual[:12]} - rebuild and re-run preflight"
)
commit = payload.get("git", {}).get("commit") if isinstance(payload.get("git"), dict) else None
if head_commit and commit and commit != head_commit:
warnings.append(
f"report certifies commit {commit[:12]} but HEAD is {head_commit[:12]}; "
"the wheels are unchanged, so this is usually just a later commit"
)
return problems, warnings
def load_gate(repo_root: Path, version: str) -> GateResult:
"""Load and enforce the preflight report for ``version``.
Raises :class:`PublishGateError` with every problem found, so one run reports
all of them instead of stopping at the first.
"""
repo_root = Path(repo_root)
report_path = repo_root / "build" / "release" / "reports" / version / "preflight.json"
if not report_path.is_file():
raise PublishGateError(
f"no preflight report at {report_path}. Run:\n"
f" python build/run.py release preflight\n"
f"(the publish commands only ship what preflight certified)"
)
try:
payload = json.loads(report_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise PublishGateError(f"cannot read {report_path}: {exc}") from exc
wheel_dir = repo_root / "SDK" / version
wheels = {path.name: path for path in sorted(wheel_dir.glob("*.whl"))}
if not wheels:
raise PublishGateError(
f"no wheels in {wheel_dir}; build first (`python build/run.py wheel-win` / "
"`manylinux`) and re-run preflight"
)
problems, warnings = evaluate_gate(payload, version, wheels)
if problems:
raise PublishGateError(
f"the preflight report at {report_path} does not certify orisys {version}:\n "
+ "\n ".join(problems)
)
git_payload = payload.get("git") if isinstance(payload.get("git"), dict) else {}
return GateResult(
version=version,
git_commit=git_payload.get("commit"),
report_path=report_path,
wheels=[
WheelRecord(name=name, sha256=sha256_file(path), size=path.stat().st_size)
for name, path in wheels.items()
],
warnings=warnings,
)
build/release/pypi.py
-458
View File
@@ -1,458 +0,0 @@
"""Upload the certified wheels to TestPyPI, then (on confirmation) to PyPI.
``python build/run.py release pypi`` runs:
1. certified-wheels gate (same contract as ``release github``)
2. ``twine check`` on every wheel - the wheel metadata is generated, so its
long description and classifiers are worth validating before an upload that
cannot be undone
3. refuse if the version already exists on the target index (read-only probe)
4. upload to TestPyPI, then verify by fetching it back from the index
5. stop there unless ``--confirm-pypi`` is given; then repeat 3-4 against PyPI
PyPI uploads are permanent: a version number can never be reused, and a bad
artifact cannot be replaced. Hence the TestPyPI stage, the read-only existence
probe, and ``--apply`` being required even to reach TestPyPI.
"""
from __future__ import annotations
import json
import shutil
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
from dataclasses import dataclass, field
from pathlib import Path
_BUILD_ROOT = Path(__file__).resolve().parents[1]
if str(_BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(_BUILD_ROOT))
from common.paths import REPO_ROOT # noqa: E402
from release import auth # noqa: E402
from release.publish import GateResult, PublishGateError, load_gate # noqa: E402
INDEX_JSON = {name: str(meta["json_url"]) for name, meta in auth.INDEXES.items()}
INDEX_SIMPLE = {name: str(meta["simple_url"]) for name, meta in auth.INDEXES.items()}
@dataclass
class Options:
version: str
apply: bool = False
confirm_pypi: bool = False
verify_install: bool = True
skip_check: bool = False
# Upload straight to PyPI, skipping the TestPyPI rehearsal. Needed when you
# only hold a production token, since the two indexes reject each other's.
pypi_only: bool = False
# Offer to enter a missing token now instead of just failing.
prompt_credentials: bool = False
# Token supplied non-interactively (e.g. --token-stdin), used once.
token: str | None = None
def upload_phases(options: Options) -> list[str]:
"""The indexes to upload to, in order."""
if options.pypi_only:
return ["pypi"]
return ["testpypi", "pypi"] if options.confirm_pypi else ["testpypi"]
@dataclass
class IndexState:
versions: list[str] = field(default_factory=list)
error: str | None = None
@property
def known(self) -> bool:
return self.error is None
def twine_check_argv(wheels: list[Path]) -> list[str]:
return [sys.executable, "-m", "twine", "check", *[str(w) for w in wheels]]
def twine_upload_argv(wheels: list[Path], repository: str | None) -> list[str]:
argv = [sys.executable, "-m", "twine", "upload"]
if repository:
argv += ["--repository", repository]
argv += [str(wheel) for wheel in wheels]
return argv
def credentials_hint(index: str) -> str | None:
"""Guidance when ``index`` has no usable credentials, else ``None``.
More than "does the file exist": a placeholder or the wrong ``username`` is
treated as missing, because twine would only find out at upload time.
"""
state = auth.credential_state(index)
if state.configured:
return None
lines = [f"no usable credentials for [{index}]: {state.problem}"]
lines.append(f" set it up with: python build/run.py release auth --set-token {index}")
if index == "testpypi":
lines.append(
" (TestPyPI needs its own account and token; if you only hold a "
"production token, use --pypi-only to skip the rehearsal)"
)
lines.append(
" or set TWINE_USERNAME=__token__ and TWINE_PASSWORD for this session only"
)
return "\n".join(lines)
def _ensure_credentials(
indexes: list[str], options: Options, *, interactive: bool | None = None
) -> bool:
"""Make sure every index that will be uploaded to has credentials.
Prompts for a missing token when there is a terminal to prompt on; a piped or
CI run fails fast instead of hanging on a hidden-input prompt.
"""
missing = [index for index in indexes if not auth.credential_state(index).configured]
if not missing:
for index in indexes:
state = auth.credential_state(index)
print(f"[ok ] credentials for [{index}] ({state.source})")
return True
if interactive is None:
interactive = sys.stdin.isatty()
allow_prompt = options.prompt_credentials or (interactive and options.apply)
if not allow_prompt:
for index in missing:
print(f"[{'FAIL' if options.apply else 'warn'}] {credentials_hint(index)}")
if not options.apply:
print(
"[info] dry run continues so you can review the plan; "
"--apply would stop here until the credentials exist"
)
return True
if not interactive:
print(
"[info] stdin is not a terminal, so nothing was prompted; run "
"`release auth --set-token <index>` or pass --prompt-credentials"
)
return False
for index in missing:
print(f"[warn] {credentials_hint(index)}")
token = options.token or auth.prompt_for_token(index)
if not token:
print(f"[FAIL] no token entered for [{index}]")
return False
for problem in auth.validate_token(token):
print(f"[FAIL] {problem.removeprefix('ERROR ')}")
return False
for warning in auth.token_warnings(token):
print(f"[warn] {warning.removeprefix('WARN ')}")
path = auth.write_token(index, token)
print(f"[ok ] wrote [{index}] credentials to {path}")
options.token = None # only the first index may consume a piped token
return True
def index_state(index: str, *, timeout: float = 20.0) -> IndexState:
"""Read the published versions of orisys from an index (read-only)."""
url = INDEX_JSON[index]
request = urllib.request.Request(
url, headers={"User-Agent": "orisys-release-publish", "Accept": "application/json"}
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
payload = json.load(response)
except urllib.error.HTTPError as exc:
if exc.code == 404:
return IndexState(versions=[])
return IndexState(error=f"{index} returned HTTP {exc.code}")
except Exception as exc: # network, DNS, TLS, JSON
return IndexState(error=f"{index} lookup failed: {type(exc).__name__}: {exc}")
releases = payload.get("releases")
if not isinstance(releases, dict):
return IndexState(error=f"unexpected {index} response (no 'releases' mapping)")
return IndexState(
versions=sorted(name for name, files in releases.items() if files)
)
def index_files(
index: str, version: str, *, timeout: float = 20.0
) -> tuple[list[str] | None, str | None]:
"""Filenames published for ``version``, as ``(filenames, error)``.
``(None, None)`` - the index does not serve this version (yet)
``(None, "why")`` - the lookup itself failed, which is NOT evidence of absence
``([...], None)`` - the version is published with those files
Keeping those apart matters: reporting a network hiccup as "not published
yet" is how a successful upload gets mistaken for a failure.
"""
url = INDEX_JSON[index]
request = urllib.request.Request(
url, headers={"User-Agent": "orisys-release-publish", "Accept": "application/json"}
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
payload = json.load(response)
except urllib.error.HTTPError as exc:
if exc.code == 404:
return None, None
return None, f"{index} returned HTTP {exc.code}"
except Exception as exc:
return None, f"{index} lookup failed: {type(exc).__name__}: {exc}"
files = (payload.get("releases") or {}).get(version)
if not isinstance(files, list) or not files:
return None, None
return sorted(str(item.get("filename", "")) for item in files), None
def wait_for_release(
index: str,
version: str,
*,
attempts: int = 6,
delay: float = 5.0,
sleep=None,
timeout: float = 20.0,
) -> tuple[list[str] | None, str | None]:
"""Poll until the index serves ``version`` (uploads are not instantly visible).
TestPyPI in particular lags the upload response by seconds. Returns the last
``(filenames, error)`` observed.
"""
sleep = sleep or time.sleep
last_error: str | None = None
for attempt in range(1, max(1, attempts) + 1):
filenames, error = index_files(index, version, timeout=timeout)
if filenames is not None:
return filenames, None
last_error = error
if attempt < attempts:
print(
f"[info] {index} does not serve {version} yet "
f"({'lookup failed' if error else 'indexing'}), retry {attempt}/{attempts - 1}",
flush=True,
)
sleep(delay)
return None, last_error
def existing_release_state(index: str, version: str, expected: set[str]) -> str:
"""``"satisfied"`` | ``"conflict"`` | ``"absent"`` for this index and version.
An index version can never be replaced, so "already published with exactly
these files" is the only successful terminal state; treating it as an error
would make a re-run after a partial failure look like a new failure. A
different file set is a genuine conflict and must stop the run.
"""
published, error = index_files(index, version)
if published is None:
if error:
print(f"[warn] could not check {index} for an existing release: {error}")
return "absent"
if set(published) == expected:
print(
f"[ok ] {index} already serves orisys {version} with all "
f"{len(published)} file(s)"
)
return "satisfied"
extra = sorted(set(published) - expected)
missing = sorted(expected - set(published))
print(
f"[FAIL] {index} already serves orisys {version} with a different file set"
+ (f"; unexpected: {extra}" if extra else "")
+ (f"; missing: {missing}" if missing else "")
)
return "conflict"
def _upload(wheels: list[Path], repository: str | None) -> int:
argv = twine_upload_argv(wheels, repository)
print(f"[run ] {' '.join(str(part) for part in argv)}")
completed = subprocess.run(argv, check=False)
return completed.returncode
def _fetch_back(index: str, version: str, wheel: Path) -> int:
"""Download the freshly uploaded wheel back out of the index."""
with tempfile.TemporaryDirectory(prefix="orisys-fetch-") as tmp:
target = Path(tmp) / "out"
argv = [
sys.executable,
"-m",
"pip",
"download",
"--no-deps",
"--no-cache-dir",
"--index-url",
INDEX_SIMPLE[index],
"--dest",
str(target),
f"orisys=={version}",
]
if index == "testpypi":
# TestPyPI mirrors only what was uploaded there; deps live on PyPI.
# --no-deps keeps this a pure "is the artifact fetchable" probe.
argv += ["--extra-index-url", INDEX_SIMPLE["pypi"]]
print(f"[run ] {' '.join(argv)}")
completed = subprocess.run(argv, check=False)
if completed.returncode != 0:
return completed.returncode
fetched = sorted(target.glob(f"orisys-{version}*"))
if not fetched:
print(f"[FAIL] pip download produced nothing for orisys=={version}")
return 1
print(f"[ok ] fetched {[p.name for p in fetched]} from {index}")
if wheel.name not in {path.name for path in fetched}:
print(
f"[warn] {wheel.name} was not the file fetched "
"(the index may serve a different platform first)"
)
return 0
def publish(options: Options) -> int:
version = options.version
mode = "APPLY" if options.apply else "DRY RUN"
print(f"=== release pypi: orisys {version} ({mode}) ===")
# --- 1. gate -------------------------------------------------------
try:
gate: GateResult = load_gate(REPO_ROOT, version)
except PublishGateError as exc:
print(f"[FAIL] {exc}")
return 1
print(f"[ok ] preflight report: {gate.report_path}")
for wheel in gate.wheels:
print(f" [ok] {wheel.name} sha256={wheel.sha256[:12]}")
for warning in gate.warnings:
print(f"[warn] {warning}")
wheels = [REPO_ROOT / "SDK" / version / record.name for record in gate.wheels]
missing = [str(wheel) for wheel in wheels if not wheel.is_file()]
if missing:
print(f"[FAIL] wheels missing from disk: {missing}")
return 1
# --- 2. credentials + twine check (before anything irreversible) ----
phases = upload_phases(options)
if options.pypi_only:
print(
"[warn] --pypi-only: uploading straight to PyPI with no TestPyPI "
"rehearsal. An index version can never be reused or replaced."
)
if not _ensure_credentials(phases, options):
return 1
if not options.skip_check:
argv = twine_check_argv(wheels)
print(f"[run ] {' '.join(str(part) for part in argv)}")
completed = subprocess.run(argv, check=False)
if completed.returncode != 0:
print("[FAIL] twine check reported problems; fix the wheel metadata first")
return 1
print("[ok ] twine check passed")
# --- 3. phase plan -------------------------------------------------
expected = {wheel.name for wheel in wheels}
remaining: list[str] = []
for index in phases:
state = existing_release_state(index, version, expected)
if state == "conflict":
return 1
if state == "absent":
remaining.append(index)
phases = remaining
print()
for index in phases:
state = index_state(index)
if not state.known:
print(f"[FAIL] {state.error}; cannot safely check for duplicate uploads")
return 1
if version in state.versions:
print(
f"[FAIL] orisys {version} already exists on {index} "
f"(published: {', '.join(state.versions) or 'none'}) - "
"an index version cannot be reused"
)
return 1
print(f"[ok ] {index}: {version} not present (published: {', '.join(state.versions) or 'none'})")
if not phases:
print()
print("=== release pypi: every requested index already has this release ===")
return 0
print()
for index in phases:
print(f"[plan] upload {len(wheels)} wheel(s) to {index}")
if phases == ["testpypi"]:
print(
"[plan] stop after TestPyPI. Re-run with "
"`--confirm-pypi --apply` to publish to the real PyPI."
)
print()
if not options.apply:
print("dry run: nothing was uploaded. Re-run with --apply to execute.")
return 0
# --- 4. upload ----------------------------------------------------
for index in phases:
if _upload(wheels, index) != 0:
print(f"[FAIL] twine upload to {index} failed")
return 1
print(f"[ok ] uploaded to {index}")
published, error = wait_for_release(index, version)
if published is None:
# The upload itself succeeded, so this is a confirmation problem, not
# a publishing failure. Say so plainly and do not imply data loss.
reason = f"lookup kept failing ({error})" if error else "the index has not indexed it yet"
print(
f"[warn] uploaded, but could not confirm it on {index} yet: {reason}.\n"
f" check https://{'test.' if index == 'testpypi' else ''}pypi.org/project/orisys/{version}/"
"\n re-running this command is safe: an already-published version is skipped"
)
print(f"\n=== release pypi: {index} upload SUCCEEDED (verification inconclusive) ===")
return 0
if set(published) != expected:
print(
f"[FAIL] {index} file set mismatch:\n expected {sorted(expected)}\n"
f" got {sorted(published)}"
)
return 1
print(f"[ok ] {index} serves all {len(published)} file(s) for {version}")
if options.verify_install:
if _fetch_back(index, version, wheels[0]) != 0:
print(
f"[warn] {index} artifact is not fetchable yet; the upload is "
"published, so this is a propagation delay rather than a failure"
)
else:
print(f"[ok ] fetched the published artifact back from {index}")
if phases == ["testpypi"]:
print()
print("=== release pypi: TestPyPI stage PASSED; PyPI untouched ===")
print("next: python build/run.py release pypi --confirm-pypi --apply")
return 0
print()
if options.pypi_only:
print(f"=== release pypi PASSED (PyPI only, no TestPyPI rehearsal) ===")
else:
print("=== release pypi PASSED (TestPyPI + PyPI) ===")
return 0
build/release/report.py
-71
View File
@@ -1,71 +0,0 @@
"""Structured preflight reports.
A preflight run writes both a machine-readable ``preflight.json`` (the contract a
future publish step will consume and re-verify) and a human ``preflight.txt``.
"""
from __future__ import annotations
import json
from pathlib import Path
REPORTS_RELPATH = Path("build") / "release" / "reports"
SCHEMA_VERSION = 1
def reports_root(repo_root: Path, override: Path | None = None) -> Path:
return Path(override) if override else Path(repo_root) / REPORTS_RELPATH
def render_text(payload: dict) -> str:
git = payload.get("git") or {}
interpreter = payload.get("interpreter") or {}
lines = [
f"orisys release preflight :: version {payload.get('version')}",
f" result: {'PASS' if payload.get('ok') else 'FAIL'}",
f" git: {git.get('branch')} @ {git.get('commit')} (dirty={git.get('dirty')})",
f" interpreter: {interpreter.get('version')} at {interpreter.get('executable')}",
f" duration: {payload.get('duration_s')}s",
"",
]
for step in payload.get("steps", []):
flag = "ok " if step.get("ok") else "FAIL"
lines.append(f"[{flag}] {step.get('name')} ({step.get('duration_s')}s)")
for detail_line in str(step.get("detail") or "").splitlines():
lines.append(f" {detail_line}")
lines.append("")
wheels = payload.get("wheels") or []
if wheels:
lines.append("wheels:")
for wheel in wheels:
lines.append(
f" {wheel.get('name')} {wheel.get('size')} bytes sha256={wheel.get('sha256')}"
)
lines.append("")
options = payload.get("options") or {}
waived = [
key
for key, value in options.items()
if value and key in {"skip_tests", "allow_dirty", "force", "no_runtime", "no_assets"}
]
if waived:
lines.append(f"waived guards: {', '.join(sorted(waived))}")
lines.append("")
return "\n".join(lines).rstrip() + "\n"
def write_report(payload: dict, out_dir: Path) -> tuple[Path, Path]:
out_dir = Path(out_dir)
out_dir.mkdir(parents=True, exist_ok=True)
payload = {**payload, "schema": SCHEMA_VERSION}
json_path = out_dir / "preflight.json"
text_path = out_dir / "preflight.txt"
json_path.write_text(json.dumps(payload, indent=2, sort_keys=False), encoding="utf-8")
text_path.write_text(render_text(payload), encoding="utf-8")
return json_path, text_path
build/release/version.py
-272
View File
@@ -1,272 +0,0 @@
"""Version consistency and duplicate-release guards for the release gate."""
from __future__ import annotations
import json
import os
import re
import subprocess
import urllib.error
import urllib.request
from pathlib import Path
PYPI_JSON_URL = "https://pypi.org/pypi/orisys/json"
# Sibling repository that carries the public GitHub releases and their tags.
RELEASE_REPO_DIRNAME = "orisys-sdk-release"
# orisys-<version>-<pytag>-<abitag>-<platformtag>.whl
_WHEEL_NAME_RE = re.compile(r"^orisys-(?P<version>[^-]+)-[^-]+-[^-]+-.+\.whl$")
def wheel_version(wheel_path: Path) -> str | None:
match = _WHEEL_NAME_RE.match(Path(wheel_path).name)
return match.group("version") if match else None
def sdk_versions(repo_root: Path) -> list[str]:
"""Version directory names present under ``SDK/``, in ascending order.
Only digit-leading names count as versions, so stray directories (``tmp``,
``latest``) never appear as releases.
"""
sdk_root = Path(repo_root) / "SDK"
if not sdk_root.is_dir():
return []
return sorted(
(
entry.name
for entry in sdk_root.iterdir()
if entry.is_dir() and entry.name[:1].isdigit()
),
key=_version_sort_key,
)
def sdk_wheels(repo_root: Path, version: str) -> list[Path]:
"""Wheels already published into ``SDK/<version>/``."""
version_dir = Path(repo_root) / "SDK" / version
if not version_dir.is_dir():
return []
return sorted(version_dir.glob("*.whl"))
def dynamic_version_attr(repo_root: Path) -> str | None:
"""Return pyproject's configured dynamic version attr (e.g. orisys.__version__)."""
from common.pyproject_utils import load_pyproject
config = (
load_pyproject(Path(repo_root))
.get("tool", {})
.get("setuptools", {})
.get("dynamic", {})
.get("version")
)
if isinstance(config, dict):
attr = config.get("attr")
if isinstance(attr, str):
return attr
return None
def pypi_released_versions(timeout: float = 20.0) -> tuple[list[str] | None, str | None]:
"""Read the published version list from PyPI.
Returns ``(versions, error)``; ``versions`` is ``None`` when the lookup
failed. This is the only network call in the release gate and it is
read-only, so it is safe to run before an irreversible upload.
"""
request = urllib.request.Request(
PYPI_JSON_URL,
headers={"User-Agent": "orisys-release-preflight", "Accept": "application/json"},
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
payload = json.load(response)
except urllib.error.HTTPError as exc:
if exc.code == 404:
return [], None
return None, f"PyPI returned HTTP {exc.code}"
except Exception as exc: # network, DNS, TLS, JSON
return None, f"{type(exc).__name__}: {exc}"
releases = payload.get("releases")
if not isinstance(releases, dict):
return None, "unexpected PyPI response shape (no 'releases' mapping)"
listed = [
str(version)
for version, files in releases.items()
if isinstance(files, list) and files
]
return sorted(listed, key=_version_sort_key), None
def _version_sort_key(name: str):
parts = re.findall(r"\d+", name)
return ([int(part) for part in parts] or [0], name)
def git_tags(repo_root: Path) -> list[str]:
"""Local git tags from this repo and the sibling release repo.
Tags are the only offline evidence that a version was already released
(``v0.5.1`` in ``orisys-sdk-release`` marks a published GitHub release), so
the duplicate guard works without network access.
"""
roots = [Path(repo_root)]
sibling = Path(repo_root).parent / RELEASE_REPO_DIRNAME
if sibling.is_dir():
roots.append(sibling)
tags: set[str] = set()
for root in roots:
try:
completed = subprocess.run(
["git", "-C", str(root), "tag", "--list"],
capture_output=True,
text=True,
check=False,
)
except OSError:
continue
if completed.returncode != 0:
continue
tags.update(
line.strip() for line in (completed.stdout or "").splitlines() if line.strip()
)
return sorted(tags)
def released_tags_for(version: str, tags: list[str]) -> list[str]:
"""Tag spellings that mark ``version`` as already released."""
return sorted({version, f"v{version}"} & set(tags))
# Version literals that are deliberately not the package version. Keyed by
# repo-relative path; each entry documents *why* the literal may stay.
VERSION_STRING_ALLOWLIST: dict[str, frozenset[str]] = {
# Historical release records: rewriting these would destroy the audit trail.
"docs/publishing.md": frozenset({"0.4.2", "0.4.3", "0.4.4", "0.5.0", "0.5.1", "0.5.2"}),
# Unrelated third-party/CUDA versions quoted in prose.
"docs/get_started.md": frozenset({"12.9.0"}),
}
# Paths never scanned: tests deliberately use synthetic versions, reports are
# generated output, and this module's own allowlist/sentinel literals would
# otherwise flag themselves.
VERSION_SCAN_EXCLUDED_PREFIXES = (
"tests/",
"build/release/reports/",
"build/nuitka/",
"build/lib/",
"dist/",
"SDK/",
"data/",
)
VERSION_SCAN_EXCLUDED_FILES = ("build/release/version.py",)
_VERSION_LITERAL_RE = re.compile(
r"(?<![<>=!~\w.])v?(\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.]+)?)(?![0-9A-Za-z.])"
)
_SCANNED_SUFFIXES = (".md", ".py", ".txt", ".toml", ".cfg", ".ini", ".rst")
def tracked_text_files(repo_root: Path) -> list[Path]:
"""Tracked files worth scanning for version literals."""
repo_root = Path(repo_root)
try:
completed = subprocess.run(
["git", "-C", str(repo_root), "ls-files", "-z"],
capture_output=True,
text=True,
check=False,
)
names = [n for n in (completed.stdout or "").split("\0") if n]
except OSError:
names = []
if not names:
names = [
str(path.relative_to(repo_root)).replace("\\", "/")
for path in repo_root.rglob("*")
if path.is_file()
]
selected = []
for name in names:
if name.startswith(VERSION_SCAN_EXCLUDED_PREFIXES):
continue
if name in VERSION_SCAN_EXCLUDED_FILES:
continue
if not name.endswith(_SCANNED_SUFFIXES):
continue
selected.append(Path(name))
return selected
def find_version_literals(
repo_root: Path, version: str
) -> list[tuple[str, int, str]]:
"""``(path, lineno, literal)`` for every version-looking string != ``version``.
Dependency constraints are skipped by the regex lookbehind (``numpy>=1.24.4``
is not a version reference), and historical/third-party literals require an
explicit entry in :data:`VERSION_STRING_ALLOWLIST`.
"""
repo_root = Path(repo_root)
findings: list[tuple[str, int, str]] = []
for relative in tracked_text_files(repo_root):
absolute = repo_root / relative
try:
text = absolute.read_text(encoding="utf-8", errors="ignore")
except OSError:
continue
if "\0" in text:
continue
allowed = VERSION_STRING_ALLOWLIST.get(relative.as_posix(), frozenset())
for lineno, line in enumerate(text.splitlines(), start=1):
for match in _VERSION_LITERAL_RE.finditer(line):
literal = match.group(1)
if literal == version or literal in allowed:
continue
findings.append((relative.as_posix(), lineno, literal))
return findings
def conda_envs_root() -> Path | None:
"""Locate the conda envs directory the same way build/wheel/win.bat does."""
override = os.getenv("CONDA_ENVS_ROOT")
if override:
path = Path(override)
return path if path.is_dir() else None
home = Path.home()
candidates = (
home / "anaconda3" / "envs",
home / "miniconda3" / "envs",
Path(os.getenv("LOCALAPPDATA", "")) / "anaconda3" / "envs",
Path(os.getenv("LOCALAPPDATA", "")) / "miniconda3" / "envs",
)
for candidate in candidates:
if candidate.is_dir():
return candidate
return None
def python_for_tag(tag: str, envs_root: Path | None = None) -> Path | None:
"""Resolve an interpreter for a wheel tag like ``cp310`` (env ``py310``)."""
digits = tag[2:] if tag.startswith("cp") else tag
env_name = f"py{digits}"
root = envs_root or conda_envs_root()
if root is None:
return None
for relative in (
Path(env_name) / "python.exe",
Path(env_name) / "bin" / "python",
):
candidate = root / relative
if candidate.is_file():
return candidate
return None
build/run.py
+2 -16
View File
@@ -8,8 +8,6 @@ Usage (from repo root):
python build/run.py gui finger|mini|box
python build/run.py verify [wheel.whl]
python build/run.py verify --gui-dist [--product ory_finger]
python build/run.py release preflight [--version V]
python build/run.py release audit <wheel.whl>
See build/README.md for details.
"""
@@ -76,23 +74,15 @@ def _cmd_verify(argv: list[str]) -> int:
return verify_main(argv)
def _cmd_release(argv: list[str]) -> int:
from release.cli import main as release_main
return release_main(argv)
def main(argv: list[str] | None = None) -> int:
argv = list(sys.argv[1:] if argv is None else argv)
if not argv or argv[0] in ("-h", "--help"):
print(__doc__)
print("Commands: wheel | wheel-win | manylinux | gui | verify | release")
print("Commands: wheel | wheel-win | manylinux | gui | verify")
print("Examples:")
print(" python build/run.py wheel --help")
print(" python build/run.py gui finger")
print(" python build/run.py manylinux -PythonVersions 3.10")
print(" python build/run.py release preflight --version <version>")
print(" python build/run.py release audit SDK\\<version>\\orisys-<version>-cp312-cp312-win_amd64.whl")
return 0
command, rest = argv[0], argv[1:]
@@ -102,14 +92,10 @@ def main(argv: list[str] | None = None) -> int:
"wheel-manylinux": _cmd_manylinux,
"gui": _cmd_gui,
"verify": _cmd_verify,
"release": _cmd_release,
}
if command not in dispatch:
print(f"Unknown command: {command!r}", file=sys.stderr)
print(
"Choose from: wheel, wheel-win, manylinux, gui, verify, release",
file=sys.stderr,
)
print("Choose from: wheel, wheel-win, manylinux, gui, verify", file=sys.stderr)
return 2
return dispatch[command](rest)
build/wheel/build.py
+5 -16
View File
@@ -30,13 +30,7 @@ if str(_BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(_BUILD_ROOT))
from common.paths import NUITKA_COMPILE_DIR, NUITKA_WHEEL_DIR, REPO_ROOT, repaired_wheel_dir, repo_path
from common.pyproject_utils import (
bundle_items,
extras_require,
get_version,
install_requires,
project_metadata,
)
from common.pyproject_utils import bundle_items, extras_require, get_version, install_requires
from common.verify.wheel import verify_built_wheel
from wheel.compile import compile_orisys, copy_compiled_orisys, install_nuitka
from wheel.layout import (
@@ -47,7 +41,6 @@ from wheel.layout import (
from wheel.package import (
is_compliant_manylinux_wheel,
publish_wheel_to_sdk,
read_long_description,
repair_linux_wheel,
run_bdist_wheel,
write_wheel_setup,
@@ -118,12 +111,10 @@ def build(*, target: str = "internal", verify: bool | None = None) -> Path:
# Bundle rules live in pyproject.toml → [[tool.orisys.bundle.items]]
bundle = bundle_items(REPO_ROOT)
copied_counts = copy_bundle_items(REPO_ROOT, pkg_dir, bundle)
for item in bundle:
destination = f"orisys/{item['target']}/"
print(
f"Copied {copied_counts.get(item['name'], 0)} file(s) "
f"for bundle item {item['name']!r} to {destination}"
)
print(f"Copied {copied_counts['configs']} JSON config files")
print(f"Copied {copied_counts['examples']} example files to orisys/examples/")
print(f"Copied {copied_counts['docs']} doc files to orisys/docs/")
print(f"Copied {copied_counts['sensors']} sensor bundle files to orisys/sensors/")
write_wheel_setup(
NUITKA_WHEEL_DIR,
@@ -131,8 +122,6 @@ def build(*, target: str = "internal", verify: bool | None = None) -> Path:
package_data=package_data_from_bundle(ext_suffix, bundle),
install_requires=install_requires(REPO_ROOT),
extras_require=extras_require(REPO_ROOT),
project=project_metadata(REPO_ROOT),
long_description=read_long_description(REPO_ROOT),
)
# --- 3. Build wheel, repair on Linux, publish to SDK/ ---
build/wheel/compile.py
-8
View File
@@ -32,20 +32,12 @@ NUITKA_NOFOLLOW_IMPORTS = (
"cv2",
"numba",
"orisys_blend",
# Optional accelerators / GUIs: imported behind try/except, resolved from the
# user's environment at runtime. Listing them here is not optional - without
# it, Nuitka follows whatever happens to be installed in the *build*
# environment, which blows up compile time and makes wheel contents depend on
# ambient packages. tests/test_nuitka_nofollow.py enforces full coverage.
"cupy",
"cupyx",
"pyopencl",
"pyvkfft",
"open3d",
"PyQt5",
"pyqtgraph",
"torch",
"torchvision",
)
build/wheel/docker/Dockerfile.manylinux
+16 -44
View File
@@ -4,51 +4,23 @@
FROM quay.io/pypa/manylinux_2_28_x86_64:latest
ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_ROOT_USER_ACTION=ignore \
ORISYS_WHEELHOUSE_ROOT=/opt/orisys-wheelhouse
PIP_ROOT_USER_ACTION=ignore
# Per-ABI build tooling plus a per-ABI wheelhouse of every package the release
# flow installs at build time:
#
# * build tooling - nuitka/auditwheel etc. baked in, so the entrypoint does
# not have to re-resolve them from PyPI on every container start
# (ORISYS_UPDATE_BUILD_TOOLS=1 opts back in to that).
# * wheelhouse - `build/wheel/build.py --target pypi-linux` force-enables
# verification, which creates a throwaway venv and pip-installs the built
# wheel plus its declared runtime deps. Runtime deps are deliberately NOT
# installed into the image interpreter (users get them from the wheel
# metadata), so without a wheelhouse every build re-downloaded
# numpy/scipy/opencv/numba — roughly 200 MB, three times over, once per ABI.
# The entrypoint points pip at the wheelhouse with
# PIP_NO_INDEX/PIP_FIND_LINKS, which keeps the venv isolated and still
# exercises real dependency resolution, just without the network.
#
# --only-binary=:all: makes a missing wheel a loud image-build failure instead of
# a slow source build inside the container.
RUN set -eux; \
for TAG in cp310-cp310 cp311-cp311 cp312-cp312; do \
PYBIN="/opt/python/${TAG}/bin"; \
echo "=== build tooling for ${TAG} ==="; \
"${PYBIN}/python" -m pip install --upgrade pip setuptools wheel; \
"${PYBIN}/python" -m pip install \
"nuitka>=2.0" \
ordered-set \
zstandard \
auditwheel; \
echo "=== wheelhouse for ${TAG} ==="; \
"${PYBIN}/python" -m pip download \
--only-binary=:all: \
--dest "${ORISYS_WHEELHOUSE_ROOT}/${TAG}" \
pip \
setuptools \
wheel \
auditwheel \
numpy \
scipy \
opencv-python \
numba; \
done; \
du -sh "${ORISYS_WHEELHOUSE_ROOT}"/*
# Only install compile/repair tooling here. Runtime deps (numpy/scipy/opencv/numba)
# are declared in the wheel metadata and installed by pip when users install orisys.
# Nuitka is configured with --nofollow-import-to for those packages.
RUN for PYBIN in \
/opt/python/cp310-cp310/bin \
/opt/python/cp311-cp311/bin \
/opt/python/cp312-cp312/bin; do \
echo "=== Installing build tooling into ${PYBIN} ===" && \
"${PYBIN}/python" -m pip install --upgrade pip setuptools wheel && \
"${PYBIN}/python" -m pip install \
"nuitka>=2.0" \
ordered-set \
zstandard \
auditwheel; \
done
WORKDIR /src
ENTRYPOINT ["/bin/bash", "/src/build/wheel/docker/entrypoint.sh"]
build/wheel/docker/entrypoint.sh
+1 -29
View File
@@ -6,18 +6,12 @@ set -euo pipefail
TARGET="${ORISYS_BUILD_TARGET:-pypi-linux}"
# Comma-separated manylinux tags, e.g. "cp310-cp310,cp311-cp311,cp312-cp312"
PYTHON_TAGS="${ORISYS_PYTHON_TAGS:-cp310-cp310,cp311-cp311,cp312-cp312}"
WHEELHOUSE_ROOT="${ORISYS_WHEELHOUSE_ROOT:-/opt/orisys-wheelhouse}"
# The image already bakes in pip/setuptools/wheel/nuitka/auditwheel. Re-resolving
# them from PyPI on every container start costs a network round-trip per ABI and
# lets the running container drift from the image it was built from.
UPDATE_BUILD_TOOLS="${ORISYS_UPDATE_BUILD_TOOLS:-0}"
cd /src
echo "=== manylinux entrypoint ==="
echo "Target: ${TARGET}"
echo "Python tags: ${PYTHON_TAGS}"
echo "Wheelhouse root: ${WHEELHOUSE_ROOT}"
uname -a
IFS=',' read -r -a TAG_ARRAY <<< "${PYTHON_TAGS}"
@@ -46,27 +40,7 @@ for tag in "${TAG_ARRAY[@]}"; do
export PATH="${PYBIN}:${PATH}"
export PYTHONPATH="/src/src${PYTHONPATH:+:${PYTHONPATH}}"
# Point pip at the per-ABI wheelhouse baked into the image. The pypi-linux
# target force-enables verification, which creates a throwaway venv and
# pip-installs the built wheel plus its declared runtime deps; serving those
# from the wheelhouse keeps the venv isolated (no --system-site-packages) while
# removing a ~200 MB download per ABI. PIP_* is inherited by every pip
# subprocess, including that venv.
wheelhouse="${WHEELHOUSE_ROOT}/${tag}"
if [[ -d "${wheelhouse}" ]]; then
export PIP_NO_INDEX=1
export PIP_FIND_LINKS="${wheelhouse}"
echo "[info] pip is offline, installing from ${wheelhouse}"
else
echo "[warn] no wheelhouse at ${wheelhouse}; pip will use the network." >&2
echo "[warn] rebuild the image without -SkipImageBuild to bake it in." >&2
unset PIP_NO_INDEX PIP_FIND_LINKS || true
fi
if [[ "${UPDATE_BUILD_TOOLS}" == "1" ]]; then
echo "[info] refreshing build tooling (ORISYS_UPDATE_BUILD_TOOLS=1)"
"${PYBIN}/python" -m pip install -q --upgrade pip setuptools wheel auditwheel
fi
"${PYBIN}/python" -m pip install -q --upgrade pip setuptools wheel auditwheel
if "${PYBIN}/python" build/wheel/build.py --target "${TARGET}"; then
BUILT+=("${tag}")
@@ -74,8 +48,6 @@ for tag in "${TAG_ARRAY[@]}"; do
echo "[ERROR] Build failed for ${tag}" >&2
FAILED=1
fi
unset PIP_NO_INDEX PIP_FIND_LINKS || true
done
echo ""
build/wheel/package.py
+3 -83
View File
@@ -19,76 +19,6 @@ _MANYLINUX_TAG_RE = re.compile(
r"(?:manylinux2014|manylinux_2_(?:17|24|27|28|31|34|35))_x86_64"
)
# Source of the PyPI project description (see read_long_description).
PYPI_README_RELPATH = Path("docs") / "pypi_readme.md"
def _license_text(project: dict) -> str | None:
"""Extract free-text license from either pyproject license form."""
declared = project.get("license")
if isinstance(declared, str):
return declared
if isinstance(declared, dict):
text = declared.get("text")
if isinstance(text, str):
return text
return None
def _person_strings(entries) -> list[str]:
"""Render PEP 621 author/maintainer tables as ``Name <email>`` strings."""
rendered: list[str] = []
for entry in entries or []:
if not isinstance(entry, dict):
continue
name = (entry.get("name") or "").strip()
email = (entry.get("email") or "").strip()
if name and email:
rendered.append(f"{name} <{email}>")
elif name or email:
rendered.append(name or email)
return rendered
def metadata_kwargs(project: dict) -> dict:
"""Translate a pyproject ``[project]`` table into ``setup()`` keywords.
The Nuitka wheel is built by a generated ``setup.py`` outside the normal
setuptools pipeline, so every PyPI-visible field has to be passed through
explicitly. Omitting them is what left the early PyPI releases with no
``Requires-Python``, classifiers, project URLs or license on the project page.
"""
authors = _person_strings(project.get("authors"))
maintainers = _person_strings(project.get("maintainers"))
candidates = {
"description": project.get("description"),
"python_requires": project.get("requires-python"),
"classifiers": list(project.get("classifiers") or []),
"project_urls": dict(project.get("urls") or {}),
"license": _license_text(project),
"author": authors[0] if authors else None,
"author_email": None,
"maintainer": maintainers[0] if maintainers else None,
}
return {key: value for key, value in candidates.items() if value}
def read_long_description(repo_root: Path) -> str | None:
"""Return the PyPI long description, or ``None`` when it is missing.
Deliberately not the repository ``README.md``: that file documents the
closed-source build/publishing layout, which should not end up on the PyPI
project page.
"""
path = Path(repo_root) / PYPI_README_RELPATH
if not path.is_file():
print(
f"[WARN] {PYPI_README_RELPATH.as_posix()} not found; the wheel will be "
"published without a long description."
)
return None
return path.read_text(encoding="utf-8")
def write_wheel_setup(
wheel_dir: Path,
@@ -97,17 +27,7 @@ def write_wheel_setup(
package_data: dict[str, list[str]],
install_requires: list[str],
extras_require: dict[str, list[str]],
project: dict | None = None,
long_description: str | None = None,
) -> Path:
metadata_lines = ""
if project:
for key, value in sorted(metadata_kwargs(project).items()):
metadata_lines += f" {key}={value!r},\n"
if long_description:
metadata_lines += " long_description=_LONG_DESCRIPTION,\n"
metadata_lines += ' long_description_content_type="text/markdown",\n'
setup_py_path = wheel_dir / "setup.py"
setup_content = f"""
from setuptools import setup, Distribution
@@ -116,12 +36,12 @@ class BinaryDistribution(Distribution):
def has_ext_modules(self):
return True
{'_LONG_DESCRIPTION = ' + repr(long_description) if long_description else ''}
setup(
name="orisys",
version="{version}",
{metadata_lines} packages=['orisys', 'orisys.examples', 'orisys_blend'],
description="Orisys - Optical Tactile Sensor SDK (Nuitka Compiled)",
author="Orisys",
packages=['orisys', 'orisys.examples', 'orisys_blend'],
package_dir={{'': 'wheel_root'}},
package_data={package_data},
include_package_data=True,
docs/api.md
+1 -21
View File
@@ -17,7 +17,7 @@ description: Orisys SDK Python API 中文参考
* **vid_src** (`int | str`, 必填): 传感器 ID、相机编号或视频路径。默认为 `0`。
* **config_name** (`str`，可选): 传感器配置文件名称或 `json` 文件路径，用于加载对应的参数配置；例如 `box` 或 `./config/box.json`。
* **cal_path** (`str`，可选): 显式的 `force_calibration.json` 路径；传入后优先于 `sensors/<type>/force_calibration.json`。不传时按 `config_name` 自动加载产品包中的标定文件。
* **cal_path** (`str`，可选，已弃用): 该参数仅为兼容旧代码保留，当前版本中不再使用，传入后也会被忽略。
* **backend** (`str`，可选): 光流场解耦计算后端，默认为 `auto`，可选`cupy`,`opencl`,`cpu`。
* **verbose** (`bool`，可选): 是否输出初始化及运行过程中的详细日志信息，默认为 `False`。
@@ -577,26 +577,6 @@ finally:
* 导出内置的`config_name`配置参数字典至`path`
### `export_sensor_bundle` / `export_sensor_bundles` 方法
#### 描述
将已安装的 `sensors/<name>/` 传感器包复制到目标目录，供下游应用（如桌面 App）做本地可编辑配置，无需硬编码 wheel 路径。解析顺序与 `resolve_sensor_bundle` 相同（含 `importlib.resources` 打包布局）。
#### 输入参数
* **name** / **names**: 单个名字，或名字列表；`export_sensor_bundles` 省略 `names` 时导出所有可解析的产品包（`box` / `mini` / `finger`）
* **dest_dir**: 目标根目录；实际写入 `dest_dir/<name>/`
* **include_assets**: 默认 `False`，仅复制 `sensor.json` 以及存在的 `force_calibration.json` / `manifest.json`；为 `True` 时复制完整目录树（含 `assets/`）
* **overwrite**: 默认 `True`；为 `False` 且目标已存在时抛出 `FileExistsError`
#### 返回
* `Path` 或 `list[Path]`：导出后的包目录路径
#### CLI
```bash
python -m orisys.export_sensors --dest ./out/sensors --names box,mini
python -m orisys.export_sensors --dest ./out/sensors --include-assets
```
## 常见问题解答 (FAQ)
### Q1:为什么必须先调用`get_img()`，再调用`compute_deformation()`？
A：`compute_deformation()`依赖最近一次获取并拼接的图像数据进行形变与力计算。
docs/get_started.md
+6 -4
View File
@@ -5,6 +5,8 @@ description: 安装并运行 Orisys SDK 的基础步骤
# 快速开始
版本：0.5.0
## 简介
本文件介绍如何上手和运行叠动视触觉传感器配套的开发套件。本套件是配合叠动视触觉传感器进行图像处理，计算形变、力以及接触信息。它可以从摄像头或视频文件中读取图像，处理后打印和展示处理后信息。
支持操作系统：Windows、Ubuntu
@@ -82,9 +84,9 @@ pip install --upgrade orisys
### 使用 .whl 文件安装（离线或指定版本）
从官方渠道获取 .whl 文件，文件名格式通常为：
orisys-{版本}-{Python版本}-{Python版本}-{操作系统}_{硬件架构}.whl
例如 `orisys-<版本>-cp310-cp310-win_amd64.whl` 表示：
例如：orisys-0.5.0-cp310-cp310-win_amd64.whl 表示：
版本：`<版本>`（以实际下载的文件名为准，可用 `python -c "import orisys; print(orisys.__version__)"` 确认）
版本：0.5.0
Python 版本：3.10（cp310）
操作系统：Windows（win）
硬件架构：64位（amd64）
@@ -95,7 +97,7 @@ Python 版本：3.10（cp310）
```bash
# 请将文件名替换为您实际下载的文件名
pip install orisys-<版本>-cp310-cp310-win_amd64.whl
pip install orisys-0.5.0-cp310-cp310-win_amd64.whl
```
### 安装 CuPy（可选）
@@ -137,7 +139,7 @@ python
import orisys
print(orisys.__version__)
```
如果打印出版本号，则说明安装成功。
如果输出版本号（例如 0.5.0），则说明安装成功。
## 使用方法
### 运行示例程序
docs/publishing.md
-399
View File
@@ -1,399 +0,0 @@
# 发布流程（构建 → 校验 → 发布）
本文档说明如何构建 orisys 的 Nuitka 二进制 wheel、如何在发布前用一条命令做完整门禁，以及当前仍需人工执行的发布步骤。
> 历史说明：`README.md` 与 `build/wheel/docker/run.ps1` 早就引用了本文件，但它此前并不存在。本文件补齐该引用。
---
## 0. 环境准备
推荐用 `py310` conda 环境执行构建与校验（它同时具备 pytest / twine / numba / Nuitka / open3d / PyQt5）：
```powershell
conda activate py310
python -m pytest -m "not slow and not hardware" # 开发期快速回归
```
`py311`、`py312` 仅作为 Nuitka 各 ABI 的编译环境按需调用（`build/wheel/win.bat` 会依次使用 `py310`/`py311`/`py312`）。
**不要**在同一个环境里既装旧版 `orisys` 又从源码目录跑测试：site-packages 里的旧包会静默遮蔽 `src/orisys`。
`tests/test_harness_guards.py` 会显式报错提示执行 `pip uninstall -y orisys`。
---
## 1. 构建 wheel
```powershell
# Windows：py310 / py311 / py312 三个 ABI
python build/run.py wheel-win
# 单个解释器（当前环境）
python build/run.py wheel
# Linux manylinux（Docker Desktop，Linux 容器模式）
python build/run.py manylinux
python build/run.py wheel-manylinux -PythonVersions 3.10
```
产物落点：
- `SDK/<version>/` — 交付用 wheel（git 忽略）
- `dist/repaired/` — 上传候选（git 忽略）
---
## 2. 发布前门禁：`release preflight`
```powershell
# --version 可省略：默认取 pyproject 的 dynamic version（即 src/orisys/__init__.py 里的 __version__）
python build/run.py release preflight
```
按顺序执行 8 个步骤，任一步失败即整体失败（`--keep-going` 可改为收集全部失败）：
| 步骤 | 检查内容 |
| --- | --- |
| `repo-state` | 是 git 仓库、工作树干净、记录 HEAD（`--allow-dirty` 可豁免，用于提交前预检） |
| `version-consistency` | `src/orisys/__init__.py::__version__` == pyproject 的 dynamic attr == `SDK/<version>/` 中 wheel 文件名里的版本 |
| `version-strings` | 追踪的文本文件里没有与当前版本不符的版本字面量（见 §5.3） |
| `duplicate-guard` | 该版本是否**已经发布过**：本地 git tag（`v<version>` / `<version>`，含 sibling `orisys-sdk-release` 仓库）命中即拒绝；`--check-pypi` 时再查 PyPI。`SDK/<version>/` 里已有 wheel 只作信息展示——那是 preflight 的正常输入，不是错误 |
| `tests` | `pytest -m "not slow and not hardware"`（`--longrun` 时包含 slow） |
| `bundle-precheck` | 每个 `[[tool.orisys.bundle.items]]` 至少解析出 1 个文件；finger 必需资产存在 |
| `wheel-inspection` | 逐个 wheel 做静态断言：文件名/版本/METADATA 字段/RECORD 哈希/压缩包路径/体积上限/无 pyc 与缓存/无 `orisys/**/*.py` 源码泄漏/与源码树逐文件字节一致 |
| `wheel-runtime` | 每个 wheel **一个** venv，在里面依次做：导入冒烟 → 内置 configs/sensors 资产可加载 → 对真实视频跑 40 帧公开流水线（无需相机、无 GUI） |
正常顺序是**先构建、再 preflight**，不需要额外开关：
```powershell
python build/run.py wheel-win
python build/run.py manylinux
python build/run.py release preflight --all-wheel-pythons --check-pypi
```
提交前想先看结果（此时工作树通常还是脏的）：加 `--allow-dirty --keep-going`。
产物报告：
- `build/release/reports/<version>/preflight.json` — 机器可读，含每个 wheel 的 sha256，供后续发布步骤复用
- `build/release/reports/<version>/preflight.txt` — 人工阅读
- `build/release/reports/<version>/junit.xml` — pytest 结果
（`build/release/reports/` 已加入 `.gitignore`。）
### 常用参数
| 参数 | 作用 |
| --- | --- |
| `--wheel-dir DIR` | 校验指定目录下的 wheel（默认 `SDK/<version>`） |
| `--all-wheel-pythons` | cp310/cp311/cp312 各自在对应 conda 环境里独立校验 |
| `--video PATH` / `--frames N` / `--config NAME` | 覆盖运行时校验输入（默认 `data/sample/s1_480p.mp4` / 40 帧 / `box`） |
| `--offline` | 复用父环境的 numpy/opencv/numba，不去 PyPI 装依赖 |
| `--check-pypi` | 唯一一处网络调用：只读 PyPI，防止重复发布 |
| `--skip-tests` / `--no-runtime` / `--no-assets` | 显式跳过并在报告中留痕；**跳过的运行不能作为发布依据** |
| `--report-dir DIR` | 覆盖报告根目录 |
`wheel-runtime` **不会静默跳过**：找不到视频时直接失败，提示 `--video` / `ORISYS_VERIFY_VIDEO`。
### 单体体检：`release audit`
对单个 wheel 做静态断言（+ 可选运行时校验），不需要版本上下文。适合体检历史产物：
```powershell
python build/run.py release audit SDK\<version>\orisys-<version>-cp312-cp312-win_amd64.whl
python build/run.py release audit dist\repaired\orisys-<version>-cp310-cp310-win_amd64.whl --strict-bundle
# 追加运行时校验（--assets 与 --runtime 同时给时共用一个 venv，只装一次依赖）
python build/run.py release audit <wheel> --assets --runtime --video data/sample/s1_480p.mp4
```
默认对「内置文件与当前源码树不一致」只告警（历史 wheel 与当前分支不一致是正常的）；加 `--strict-bundle` 改成失败（刚构建完的 wheel 应当一致）。
旧的导入冒烟测试仍然可用，行为不变：
```powershell
python build/run.py verify <wheel>
python build/run.py verify <wheel> --contents # 静态断言
python build/run.py verify <wheel> --assets --runtime # 资产 + 真实帧
python build/run.py verify --gui-dist --product ory_finger
```
---
## 3. 发布（两条命令）
先决条件：`release preflight` PASS，**而且是在干净提交上跑的**。两个发布命令都只发布「preflight 认证过的东西」——它们会读取 `build/release/reports/<version>/preflight.json`，逐个核对 wheel 的 sha256，并拒绝使用带豁免项（`--skip-tests` / `--allow-dirty` / `--no-runtime` / `--force`）的报告。所以正确顺序是：
```powershell
git add -A; git commit -m "..."
python build/run.py wheel-win && python build/run.py manylinux
python build/run.py release preflight --all-wheel-pythons --check-pypi # 干净树 + 无豁免
```
两个命令**默认都是 dry run**，只打印将要执行的动作；必须显式加 `--apply` 才会真正动手。GitHub 的 `gh release create`、`git push`，以及 PyPI 上传（版本号永久占用、不可覆盖）都属于不可撤销操作。
### 3.1 GitHub：同步公开仓库 + 上传 Release
```powershell
python build/run.py release github --version 0.5.2 # 预览
python build/run.py release github --version 0.5.2 --apply # 执行
```
执行顺序：
1. 认证书校验（上面那份报告 + 逐 wheel sha256）
2. `git fetch --tags`，若 `v0.5.2` 或 `0.5.2` 已存在则拒绝；`gh release view` 已存在也拒绝
3. 按 `tools/downstream_sync_manifest.json` 同步公开内容到 `../orisys-sdk-release`，并运行 manifest 里的 `checks`
4. 把 `SDK/0.5.2/*.whl` 放到 `orisys-sdk-release/release/0.5.2/`（该目录在 release 仓库里是 gitignore 的，只用于上传附件）
5. 提交 → 打 tag `v0.5.2` → push 分支与 tag
6. `gh release create v0.5.2 --verify-tag --latest <6 个 wheel>`，然后回读 `gh release view --json assets` 核对附件齐全
需要：`gh` 已安装并登录（`winget install --id GitHub.cli` 然后 `gh auth login`）。没装 gh 时：
- dry run 仍会完整打印计划（只告警）
- `--apply` 会失败，或改用 `--skip-release-upload`：同步、提交、打 tag、push 都自动完成，然后打印出你自己要跑的 `gh release create`
其他开关：`--notes-file`（自定义 release notes）、`--attach-preflight`（把 `preflight.json` 也作为附件上传，便于审计；注意报告里有本机绝对路径）、`--allow-dirty-release`、`--skip-fetch`、`--skip-checks`。
> tag 拼写沿用历史风格 `v<version>`（注意历史里那个 typo tag `0.50` 指向 `v0.5.1`）。命令会在动手前拒绝已存在的 tag，所以不会重复发布。**如果 tag 已经 push 但 Release 上传失败（例如 `gh` 没配好），直接重跑同一条 `--apply` 命令即可续跑**：它会跳过创建 tag，只补做剩下的步骤；只有「GitHub Release 已存在」才拒绝。
### 3.2 PyPI：TestPyPI 先行，确认后再上正式仓库
```powershell
python build/run.py release pypi --version 0.5.2 # 预览
python build/run.py release pypi --version 0.5.2 --apply # 只发 TestPyPI
python build/run.py release pypi --version 0.5.2 --confirm-pypi --apply # TestPyPI + 正式 PyPI
# 只有正式 token、没有 TestPyPI 账号时：跳过彩排直接发正式库
python build/run.py release pypi --version 0.5.2 --pypi-only --apply
```
`--pypi-only` 与 `--confirm-pypi` 互斥。**注意它的代价**：没有 TestPyPI 彩排，而正式库的版本号一旦占用就无法复用、无法替换。它仍然会做完整门禁、`twine check`、上传前查重、上传后回读索引核对文件、以及把产物重新拉回来验证。
执行顺序：
1. 认证书校验（与 `release github` 同一份报告）
2. `twine check` 全部 wheel（校验 METADATA/long description，上传前就能发现问题）
3. 只读查询目标索引，确认该版本不存在（TestPyPI 与 PyPI 各查各的）
4. 上传 → **轮询**索引确认 6 个文件都在 → `pip download` 把刚上传的 wheel 拉回来，证明产物可获取
5. 未加 `--confirm-pypi` 时只发 TestPyPI，然后停在这里并打印后续命令
两个针对「索引不即时一致」的设计（TestPyPI 尤其明显，上传成功后 JSON API 可能滞后若干秒）：
- **上传后的确认会重试**（默认 6 次 × 5 秒）。确认不了也**不算失败**：twine 上传成功就说明产物已发布，命令会打印警告、给出项目页链接并以 0 退出，不会把一个成功的上传报成失败。真正的失败只有两种：twine 上传本身失败，或索引上的文件集与本地不一致。
- **重复运行是安全的**：若索引上已经存在该版本且文件集完全一致，该阶段会被视为「已完成」而跳过（`[ok] testpypi already serves orisys 0.5.2 with all 6 file(s)`），不会重传、也不报错。这让你在确认不明确时可以直接重跑。如果文件集**不一致**（索引上是另一次构建），才会失败并停下。
所以「上传后确认失败」不等于要重新上传——索引版本号不可覆盖，重跑只会做一次只读核对。
### 3.3 凭证：`release auth` 与 `~/.pypirc`
```powershell
python build/run.py release auth # 查看两个索引的凭证状态
python build/run.py release auth --set-token pypi # 隐藏输入，写入 ~/.pypirc
python build/run.py release auth --set-token testpypi
python build/run.py release auth --repair # 修「有段但没列进 index-servers」
```
`--set-token` 的 token 只从**隐藏输入**或 `--token-stdin`（供脚本/CI）读取，**不接受命令行参数**——避免落进 shell 历史和进程列表。写入采用定点文本编辑，文件里的说明注释会保留；同时会把该索引补进 `[distutils] index-servers`（这一步很关键，见下）。
`release auth` 只有在你真的运行 `--set-token` 时才写文件；不带参数时只读并报告，状态判断比「文件是否存在」严格：
```
config file: C:\Users\cheun\.pypirc (exists)
[-- ] [pypi] not usable: [pypi] still contains a placeholder token
[-- ] [testpypi] not usable: C:\Users\cheun\.pypirc has no [testpypi] section
```
**占位符、空密码、`username` 不是 `__token__`，都会被判定为「不可用」**——否则 twine 只会在真正上传时才发现，而那时已经动过索引了。
`release pypi` 在 `--apply` 时若凭证缺失：有终端就地提示输入（隐藏），无终端则直接失败并打印 `release auth --set-token <index>`；也可显式加 `--prompt-credentials`。dry run 只告警并继续展示计划，不会卡在输入上。
#### twine 直接读取的格式
默认路径 `~/.pypirc`（Windows 上即 `C:\Users\<你>\.pypirc`，由 `USERPROFILE` 展开）。**没有**环境变量可以改这个路径；要用别的文件就传 `--config-file`。
```ini
[distutils]
index-servers =
pypi
testpypi
[pypi]
repository = https://upload.pypi.org/legacy/
username = __token__
password = pypi-你的正式token
[testpypi]
repository = https://test.pypi.org/legacy/
username = __token__
password = pypi-你的TestPyPI token
```
容易搞错的点：
- **`[distutils] index-servers` 必须列出该索引，否则 twine 完全忽略那一节** —— 这是最常见的坑：文件里明明有 `[testpypi]` 段，twine 却报 `InvalidConfiguration: Missing 'testpypi' section`。用 `release auth --repair` 修（把已存在但未列出的段补进 `index-servers`）。
- 更细的语义差别：`index-servers =` **值为空**表示「一个都不认」；只有**整行缺失**时 twine 才会回退到默认的 `pypi testpypi`。所以手工编辑时别把名字删空。
- `username` 就是字面量 `__token__`，不是账号名或邮箱
- 两类 token **都以 `pypi-` 开头**，但**不能互换**：拿正式 token 打 `test.pypi.org` 会被 403 拒绝
- TestPyPI 是**独立服务、独立账号**，注册在 <https://test.pypi.org/account/register/>，你的 PyPI 账号在那里不存在
- 正式 token 建议限定到 `orisys` 项目（项目已存在，可以建 project-scoped token）
- 该文件必须在**任何 git 仓库之外**（放在用户目录里）；token 一旦提交进仓库，即使后来删文件也还在历史里——那属于安全事故
- 临时替代方案：会话内设 `TWINE_USERNAME=__token__` + `TWINE_PASSWORD=<token>`，命令会识别这种情形
- PyPI token **没有只读的校验方式**，有效性只能在真正上传时才会被索引确认；所以所有本地检查、查重都排在写操作之前
`release auth` 的「可用」判定**直接问 twine 自己**（`twine.utils.get_config` 解析出的 username/password），而不是自己解析 ini，这样报告结果与 twine 的行为不会分叉。twine 未安装时才回退到本地复刻同一套规则。
只需要 TestPyPI 时才把 `testpypi` 写进 `index-servers` 并填对应 token；只发正式库时 `--pypi-only` 用不到 `[testpypi]` 段。
### 3.4 已知的历史发布空洞
| 版本 | 本地 `SDK/` | PyPI | GitHub Release |
| --- | --- | --- | --- |
| 0.4.2 | — | 有 | 有 |
| 0.4.3 | — | 有 | 有 |
| 0.4.4 | 有 | **无** | **无** |
| 0.5.0 | 有 | **无** | **无** |
| 0.5.1 | 有 | 有 | 有（tag `v0.5.1` + typo tag `0.50`） |
| 0.5.2 | 有 | **待发布** | **待发布**（`release preflight --check-pypi` 确认 PyPI 尚无 0.5.2） |
> 本表按 2026 年 0.5.2 构建时的实际状态填写。`0.5.1` 这一行曾被一次全局替换误改成 `0.5.2` 并丢失，请勿再用「全局把 0.5.1 替换成新版本号」的方式改本文件——历史事实不应该被改。
---
## 4. 环境变量
| 变量 | 用途 |
| --- | --- |
| `ORISYS_TEST_VIDEO` | pytest 用的视频；默认自动取 `data/sample/s1_480p.mp4` |
| `ORISYS_TEST_CONFIG` | pytest 用的配置名；默认自动挑一个含 `stitcher.box_size` 的配置 |
| `ORISYS_VERIFY_VIDEO` | 运行时校验用的视频 |
| `ORISYS_VERIFY_CONFIG` | 运行时校验用的配置名（默认 `box`） |
| `ORISYS_LONGRUN_BACKEND` / `ORISYS_LONGRUN_FRAMES` | `slow` 长稳测试的参数 |
| `CONDA_ENVS_ROOT` | conda envs 根目录（`--all-wheel-pythons` 用它解析 `py310`/`py311`/`py312`） |
---
## 5. 三个必须记住的坑
### 5.1 Nuitka 编译包的 `__file__` 是假的
Nuitka 把整个 `orisys` 编译成单个扩展模块，其 `__init__` 会安装 `nuitka_module_loader`。于是：
```python
>>> import orisys
>>> orisys.__file__ # 合成的、磁盘上并不存在的路径
'...\\site-packages\\orisys\\orisys\\__init__.py'
>>> orisys.__path__ # 第二项是 Nuitka 的哨兵值
['...\\site-packages\\orisys\\orisys', '\\not_existing']
>>> orisys.__loader__.__class__.__name__
'nuitka_module_loader'
```
后果：**在 Nuitka wheel 里不能靠 `orisys.__file__` 或 `importlib.resources.files("orisys")` 定位内置数据**——它们会指向不存在的目录，静默返回「文件不存在」。请改用发行包定位：
```python
from importlib.metadata import distribution
root = distribution("orisys").locate_file("orisys") # site-packages/orisys
```
`build/common/verify/wheel.py` 的内置资产校验就是这么做的，并且会显式断言该目录位于校验用 venv 内，避免被父环境里的旧 `orisys` 顶替。
### 5.2 `--offline` 会被父环境的脏安装污染
`--offline` 通过 venv 的 `--system-site-packages` 复用父环境的 numpy/numba 等依赖，这也顺带暴露了父环境里的 `orisys`。若父环境残留了半卸载的 editable 安装（典型症状是 `pip` 提示 `Ignoring invalid distribution -risys`，且 `site-packages/orisys/__init__.py` 是一个指向死路径的 shim），校验就会读到它。
处置：在构建环境里执行 `pip uninstall -y orisys`，或改用不带 `--offline` 的完全隔离 venv（会联网安装依赖）。
### 5.3 版本号只有一个地方，其他都是推导出来的
版本号的唯一来源是 **`src/orisys/__init__.py::__version__`**。以下全部由它推导，不需要手改：
| 产物 | 推导方式 |
| --- | --- |
| pyproject 的 `[project] version` | `[tool.setuptools.dynamic] version = {attr = "orisys.__version__"}` |
| wheel 文件名里的版本 | `build/wheel/build.py` 调 `get_version()` |
| wheel `METADATA` 的 `Version:` | 生成的 `setup.py` 里写入同一个 `version` |
| `SDK/<version>/` 目录名 | 同上 |
| `importlib.metadata.version("orisys")` | 从 wheel METADATA 读出 |
因此**不要**再往文档、示例、测试里写具体版本号：
- 用法示例写 `SDK/<version>/...`、`--version <version>`，不要写具体版本
- 文档里不要写「版本：x.y.z」；需要时让用户用 `python -c "import orisys; print(orisys.__version__)"` 自己看
- 示例脚本的 docstring 不要再写 `SDK version: x.y.z`
- 测试里需要「一个版本」时用明显是假的版本号（如 `1.x.y` 这种不会与真实版本混淆的值），不要用当前版本 —— 否则每次发版都要改测试，而且一次全局替换就会悄悄改坏断言
`release preflight` 的 `version-strings` 步骤强制这条规则：扫描所有被追踪的文本文件，任何与当前版本不符的版本字面量都会导致失败（`tests/` 与生成物除外；依赖约束如 `numpy>=1.24.4` 由正则排除）。
历史记录属于例外，需显式登记在 `build/release/version.py::VERSION_STRING_ALLOWLIST` 里——登记本身就是「该字面量故意不是包版本」的文档。**不要用全局替换修改含历史版本的文件**：本文件的发布记录表就曾被一次全局替换改错（`0.50` typo tag 被挂到错误的版本上，`0.5.1` 那一行被覆盖丢失）。
### 5.4 wheel 里有什么、没有什么
打包内容由 `pyproject.toml` 的 `[[tool.orisys.bundle.items]]` 单点定义：
| 内容 | 是否进 wheel | 说明 |
| --- | --- | --- |
| `config/*.json` | 是 | `orisys/configs/`，运行时配置解析要用 |
| `sensors/**` | 是 | `orisys/sensors/`，含 finger 的 npz/npy 资产，运行时与 GUI 都要用 |
| `examples/**` | 是 | `orisys/examples/`，用户装完就能跑示例 |
| `docs/**` | **否** | 只随 GitHub 仓库与 PyPI 项目页提供。`docs/publishing.md` 是内部流程文档；`docs/pypi_readme.md` 仅在构建时被读取作为 long description |
因为 docs 不进 wheel，**改文档不会让已构建的 wheel 失效**，不需要重新构建。（改 `examples/`、`config/`、`sensors/` 会——它们确实在 wheel 里，`wheel-inspection` 的 `bundle-fidelity` 会报出来。）
如果以后需要离线 API 参考，加一个只包含 `docs/api.md` 的窄条目即可，不要整个 `docs/` 都塞进去。
---
## 6. 缩短 manylinux 构建时间
`python build/run.py manylinux` 走 `build/wheel/docker/run.ps1` → 容器内 `entrypoint.sh`，对 `cp310/cp311/cp312` **串行**各做一次 Nuitka 编译 + 打 wheel + auditwheel repair + 隔离 venv 校验。
### 6.1 镜像里已经烘好的东西（所以每次构建不必再拉）
`Dockerfile.manylinux` 在每个 ABI 下预置了两类内容：
1. **构建工具**：`pip`/`setuptools`/`wheel`/`nuitka`/`ordered-set`/`zstandard`/`auditwheel`。
因此 `entrypoint.sh` 默认**不再**在每个 ABI 上重跑 `pip install --upgrade pip setuptools wheel auditwheel` —— 那是每次构建 3 次多余的 PyPI 往返，还会让容器里的工具版本与镜像不一致。需要用最新工具时显式开启：
```powershell
docker run ... -e ORISYS_UPDATE_BUILD_TOOLS=1 ...
```
2. **每个 ABI 一个 wheelhouse**（`/opt/orisys-wheelhouse/<tag>`），内含 `numpy`/`scipy`/`opencv-python`/`numba` 及其依赖闭包，外加 `pip`/`setuptools`/`wheel`/`auditwheel` 的 wheel。
`--target pypi-linux` 会**强制**校验：为每个 ABI 建一个一次性 venv，再 `pip install` 刚构建的 wheel 和它声明的运行期依赖。镜像故意不把运行期依赖装进解释器（用户是凭 wheel 元数据装的），所以以前每次构建都要现下约 200 MB × 3。`entrypoint.sh` 现在用 `PIP_NO_INDEX=1` + `PIP_FIND_LINKS=<wheelhouse>` 把 pip 指向本地 wheelhouse：**venv 仍然是隔离的**（没有 `--system-site-packages`），依赖解析与安装依然真实发生，只是不走网络。
找不到 wheelhouse 时会打印警告并回退到联网，不会静默失败。镜像因此增大约 400–500 MB（一次性代价，换每次构建省三次下载）。
### 6.2 不要为了省事直接重建镜像
只改了源码（`src/`、`config/`、`sensors/`、`examples/`、`pyproject.toml`）时加 `-SkipImageBuild`：
```powershell
.\build\wheel\docker\run.ps1 -SkipImageBuild
```
Dockerfile 的 `RUN` 层要装 3 份工具链并下载 3 个 wheelhouse，缓存未命中就是数分钟。**改了 `Dockerfile.manylinux` 或需要更新 wheelhouse 时才重建镜像**。
### 6.3 Nuitka 的排除列表必须完整
`build/wheel/compile.py::NUITKA_NOFOLLOW_IMPORTS` 列出所有**不打进 wheel、运行期从用户环境解析**的第三方包。漏一个意味着 Nuitka 会尝试把它跟进来：编译时间暴涨，而且 wheel 内容开始取决于构建机上恰好装了什么 —— `torch`、`torchvision`（`src/orisys/tracker.py` 的 RAFT 追踪器）和 `pyvkfft`（`backend_availability.py`）就是这样漏掉的，之前只是靠「构建机上没装」才没出事。
`tests/test_nuitka_nofollow.py` 会扫描 `src/orisys` 的全部第三方导入，任何不在排除列表里的都会让测试失败。**新增可选依赖时同步更新列表。**
### 6.4 想要并行？先读这一段
`--jobs` 的默认值是**全部 CPU 数**，单次编译的 C 阶段已经吃满所有核心。真正能靠并行省下的是 Nuitka 的单线程前端（分析/优化/代码生成）、scons 启动、`auditwheel` 与 venv 校验 —— 做法是 3 路并行 + 每个 build 限 `--jobs=<核数/3>`。
但**现在不能直接并行**：`build/common/paths.py` 里的工作目录是写死的共享路径
```
build/nuitka/compile
build/nuitka/wheel
build/nuitka/wheel/repaired
```
而 `build/wheel/build.py` 每次开头都 `_reset_dir()`（`rmtree`）这两个目录，`auditwheel repair` 又会再 `rmtree` 一次 `repaired`。仓库是挂载进容器的（`-v ${RepoRoot}:/src`），所以无论「一个容器 3 路并行」还是「3 个容器并行」，只要共享同一个挂载点，三个 build 就会互相清空暂存目录 —— 表现为随机失败或 wheel 内容错乱。
要并行必须先让工作目录带上 ABI 标签（如 `build/nuitka/<tag>/compile`）。输出路径不用动：`SDK/<version>/`、`dist/repaired/` 的文件名本来就按 ABI 区分，下游 `release preflight` 不受影响。另外 3 个 Nuitka 并行很吃内存，先把 Docker Desktop 内存调到 16 GB 以上再试。
docs/pypi_readme.md
-63
View File
@@ -1,63 +0,0 @@
# Orisys SDK
叠动科技视触觉传感器配套的 Python SDK，提供图像采集、拼接、稠密光流、Helmholtz-Hodge 分解、接触与深度估计，以及可运行的示例。
本文件是 PyPI 项目描述（`long_description`）的源文件：构建时由 `docs/publishing.md` 所述流程从**检出目录**读取（`build/wheel/package.py::PYPI_README_RELPATH`），**不会**被打进 wheel。因此不要在这里写内部仓库布局、构建流水线或发布步骤——那些内容属于仓库内的 `README.md` 与 `docs/publishing.md`。
```bash
python -m pip install orisys
```
支持 Python 3.10 / 3.11 / 3.12，Windows x64 与 Ubuntu x64。
## 快速开始
```bash
python -c "import orisys; print(orisys.__version__)"
python -c "import orisys; print(orisys.list_config())"
```
## 安装选项
- `orisys` — CPU / 基础流程
- `orisys[gui]` — Qt / Open3D 可视化示例
- `orisys[cuda12]` / `orisys[cuda13]` — NVIDIA GPU 加速
- `orisys[gui,cuda12]` — GUI + CUDA 12
`orisys[cuda]` 与 `orisys[cuda12]` 等价。
## 配置
配置解析顺序：
1. 显式 JSON 文件路径
2. 内置配置名（`box`、`mini`、`finger` 等）
3. 产品别名（如 `dd02` -> `dd02-ov`）
4. 向后兼容的 JSON `"name"` 匹配
```python
import orisys
sensor = orisys.Sensor(0, config_name="box")
orisys.export_config("box", "./exported_config.json")
```
## 示例
示例随 wheel 一起分发，位于 `orisys/examples/`。也可以从项目仓库获取：
```bash
python examples/basic_pipeline.py -v 0 -c box
python examples/finger_pipeline.py --camera_id 0 --config finger
```
无相机时可以改用视频文件：
```bash
python examples/finger_pipeline.py --video ./sample.mp4 --config finger
```
## 支持与反馈
- 官网：[https://www.orisys-tech.cn](https://www.orisys-tech.cn)
- 问题反馈：GitHub Issues
examples/basic_pipeline.py
+3 -3
View File
@@ -7,6 +7,7 @@ ORY-BOX：单传感器基础流程
3. 按需计算接触区域
4. 读取并可视化结果
SDK version: 0.5.0
"""
import os
os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0"
@@ -20,7 +21,6 @@ def main():
parser.add_argument("--video","-v", type=str, default="0", help="视频源：摄像头编号或视频文件路径")
parser.add_argument("--config","-c", type=str, default="box", help="配置文件名称或路径")
parser.add_argument("--verbose","-verbose", type=bool, default=True, help="是否输出详细日志")
parser.add_argument("--cal_path","-cal", type=str, default=None, help="标定文件路径")
args = parser.parse_args()
# 步骤 1：创建传感器对象（必需）
@@ -29,9 +29,9 @@ def main():
# 输入既可以是摄像头编号，也可以是视频文件路径
if args.video.isdigit():
sensor = orisys.Sensor(int(args.video), config_name=args.config, verbose=args.verbose, cal_path=args.cal_path)
sensor = orisys.Sensor(int(args.video), config_name=args.config, verbose=args.verbose)
else:
sensor = orisys.Sensor(args.video, config_name=args.config, verbose=args.verbose, cal_path=args.cal_path)
sensor = orisys.Sensor(args.video, config_name=args.config, verbose=args.verbose)
print("\n按键说明：'q' 退出程序，'r' 重置追踪器。")
examples/mini_pipeline.py
+1
View File
@@ -7,6 +7,7 @@ ORY-MINI：单传感器基础流程
3. 按需计算接触区域
4. 读取并可视化结果
SDK version: 0.5.0
"""
import os
os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0"
examples/two_sensor_pipeline.py
+1
View File
@@ -18,6 +18,7 @@
- Windows 下必须在 if __name__ == '__main__' 中调用 mp.freeze_support()
- 每个传感器进程都会占用独立的 GPU 显存
SDK version: 0.5.0
"""
import os
os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0"
pyproject.toml
+5 -8
View File
@@ -99,12 +99,9 @@ target = "examples"
patterns = ["*.py", "README.md", "viz/**/*.py", "pipeline/**/*.py", "gui/**/*.py"]
preserve_tree = true
# NOTE: docs/ is deliberately NOT bundled into the wheel.
# Documentation ships with the GitHub repository / PyPI project page only:
# - docs/publishing.md is an internal process document
# - docs/pypi_readme.md is read from the checkout at build time as the PyPI
# long description (build/wheel/package.py::PYPI_README_RELPATH), never shipped
# - keeping docs out means editing a doc no longer invalidates built wheels
# Should an offline API reference be needed later, add a narrower item that ships
# only docs/api.md.
[[tool.orisys.bundle.items]]
name = "docs"
source = "docs"
target = "docs"
patterns = ["*.md", "*.pdf"]
pytest.ini
-4
View File
@@ -1,9 +1,5 @@
[pytest]
testpaths = tests
python_files = test_*.py
# Run against the checkout without requiring `pip install -e .`, so a stale
# orisys in site-packages cannot silently shadow src/orisys.
pythonpath = src
markers =
slow: long-running tests for release validation
hardware: requires a physical sensor/camera attached to the machine
sensors/box/sensor.json
+1 -1
View File
@@ -29,7 +29,7 @@
"loop_closure_interval": 1,
"algorithm": "dis",
"dis_preset": "medium",
"dis_mode": 1,
"dis_mode": 3,
"algorithm_list": ["raft", "dis"],
"hhd_downsample": 3
},
sensors/finger/sensor.json
+1 -1
View File
@@ -22,7 +22,7 @@
"dis_preset": "medium",
"dis_mode": 1,
"algorithm_list": ["raft", "dis"],
"hhd_downsample": 3
"hhd_downsample": 1
},
"contact": {
"min_contour_area": 3000,
src/orisys/__init__.py
+1 -9
View File
@@ -5,13 +5,12 @@ A Python library for processing and analyzing optical tactile sensor data,
including image stitching, optical flow tracking, and deformation decomposition.
"""
__version__ = "0.5.2"
__version__ = "0.5.0"
__author__ = "Orisys"
from .sensor import Sensor
from . import util
from . import configs
from . import calibration
from . import finger
from . import signal_processing
from . import capture
@@ -23,8 +22,6 @@ CameraSensor = Sensor
# Re-export list_configs as list_config for convenience
list_config = configs.list_config
export_config = configs.export_config
export_sensor_bundle = configs.export_sensor_bundle
export_sensor_bundles = configs.export_sensor_bundles
__all__ = [
# class
@@ -44,14 +41,9 @@ __all__ = [
# capture helpers
'capture',
# force calibration
'calibration',
# configs utilities functions
'list_config',
'export_config',
'export_sensor_bundle',
'export_sensor_bundles',
# test
'FFCFrameProcessor'
src/orisys/calibration.py
-782
View File
@@ -1,782 +0,0 @@
"""Force calibration load, parse, check, build, and write.
Owns ``force_calibration.json`` schema v1/v2. ``Sensor`` loads once and applies
curves in ``_apply_force_filter``; ``read_info(FNORMAL, …)`` returns Newtons.
"""
from __future__ import annotations
import json
import warnings
from copy import deepcopy
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable, Mapping
CURRENT_SCHEMA = 2
SUPPORTED_SCHEMAS = (1, 2)
CHANNEL_IDS: tuple[str, ...] = ("fn", "fx", "fy")
CHANNEL_ALIASES: dict[str, str] = {
"fn": "fn",
"FN": "fn",
"normal": "fn",
"fnormal": "fn",
"fx": "fx",
"FX": "fx",
"shear_x": "fx",
"fshearx": "fx",
"fy": "fy",
"FY": "fy",
"shear_y": "fy",
"fsheary": "fy",
}
_CANONICAL_TO_JSON = {
"fn": "normal",
"fx": "shear_x",
"fy": "shear_y",
}
_JSON_RAW_KEY = {
"normal": "fnormal",
"shear_x": "fshearx",
"shear_y": "fsheary",
}
_KNOWN_TOP_KEYS = frozenset(
{"schema_version", "unit", "channels", "meta", "identity", "pipeline", "provenance", "measurements", "quality"}
)
class CalibrationError(ValueError):
"""Invalid or unusable force calibration document / path."""
def canonicalize_channel(name: str) -> str:
"""Map FN / normal / shear_x / … to canonical ``fn`` / ``fx`` / ``fy``."""
key = str(name or "").strip()
if key in CHANNEL_ALIASES:
return CHANNEL_ALIASES[key]
lower = key.lower()
if lower in CHANNEL_ALIASES:
return CHANNEL_ALIASES[lower]
raise CalibrationError(f"unknown calibration channel: {name!r}")
def _parse_point(point: Any) -> tuple[float, float]:
if isinstance(point, dict):
raw = point.get("fnormal", point.get("raw", point.get("x")))
force_n = point.get("force_n", point.get("n", point.get("y")))
if raw is None or force_n is None:
raise CalibrationError(f"invalid calibration point: {point!r}")
return float(raw), float(force_n)
if isinstance(point, (list, tuple)) and len(point) >= 2:
return float(point[0]), float(point[1])
raise CalibrationError(f"invalid calibration point: {point!r}")
@dataclass(frozen=True)
class ChannelCurve:
"""One force channel: linear coeffs and/or piecewise-linear breakpoints."""
id: str
method: str = "piecewise_linear"
points: tuple[tuple[float, float], ...] = ()
slope: float | None = None
offset: float | None = None
signed: bool = False
extrapolate: bool = True
clamp_output_min: float | None = 0.0
raw_key: str = ""
input_mode: str = "magnitude"
unit: str = "N"
def convert(self, raw: float) -> float:
raw = float(raw)
if self.method in ("linear", "affine"):
if self.slope is None or self.offset is None:
return raw
return float(self.slope) * raw + float(self.offset)
points = self.points
if len(points) < 2:
return raw
if raw <= points[0][0]:
segment = (points[0], points[1])
if not self.extrapolate:
return self._clamp(points[0][1])
elif raw >= points[-1][0]:
segment = (points[-2], points[-1])
if not self.extrapolate:
return self._clamp(points[-1][1])
else:
segment = None
for left, right in zip(points, points[1:]):
if left[0] <= raw <= right[0]:
segment = (left, right)
break
if segment is None:
return self._clamp(points[-1][1])
(raw0, n0), (raw1, n1) = segment
if raw1 == raw0:
value = n0
else:
ratio = (raw - raw0) / (raw1 - raw0)
value = n0 + ratio * (n1 - n0)
return self._clamp(value)
def convert_component(self, raw: float) -> float:
if not self.signed:
return self.convert(raw)
raw = float(raw)
sign = -1.0 if raw < 0.0 else 1.0
return sign * self.convert(abs(raw))
def _clamp(self, value: float) -> float:
if self.clamp_output_min is not None:
return max(float(self.clamp_output_min), float(value))
return float(value)
def to_dict(self) -> dict[str, Any]:
json_name = _CANONICAL_TO_JSON.get(self.id, self.id)
if self.method in ("linear", "affine"):
block: dict[str, Any] = {
"method": "linear",
"slope": self.slope,
"offset": self.offset,
}
else:
block = {
"method": "piecewise_linear",
"raw_key": self.raw_key or _JSON_RAW_KEY.get(json_name, ""),
"extrapolate": self.extrapolate,
"clamp_output_min": self.clamp_output_min,
"points": [list(p) for p in self.points],
}
if self.signed:
block["signed"] = True
if self.input_mode and self.input_mode != "magnitude":
block["input"] = self.input_mode
return block
@dataclass(frozen=True)
class CalSample:
weight_n: float
raw_mean: float
n: int
std: float = 0.0
@dataclass
class CheckResult:
ok: bool
errors: list[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
@dataclass(frozen=True)
class Calibration:
"""Parsed force calibration document."""
schema_version: int
unit: str
origin: str
identity: dict[str, Any]
pipeline: dict[str, Any]
provenance: dict[str, Any]
channels: dict[str, ChannelCurve]
measurements: dict[str, Any]
extras: dict[str, Any] = field(default_factory=dict)
source_path: Path | None = None
# Legacy shared shear (v1) for magnitude-mode fallback in helpers.
_legacy_shear: ChannelCurve | None = None
def get(self, channel: str) -> ChannelCurve | None:
try:
cid = canonicalize_channel(channel)
except CalibrationError:
return self.channels.get(str(channel))
return self.channels.get(cid)
def has(self, channel: str) -> bool:
return self.get(channel) is not None
def convert(self, fn: float, fx: float, fy: float) -> tuple[float, float, float]:
"""Apply present curves; missing channel stays raw."""
out_fn = float(fn)
out_fx = float(fx)
out_fy = float(fy)
cal_fn = self.channels.get("fn")
cal_fx = self.channels.get("fx")
cal_fy = self.channels.get("fy")
if cal_fn is not None:
out_fn = cal_fn.convert(out_fn)
if cal_fx is not None:
out_fx = (
cal_fx.convert_component(out_fx)
if cal_fx.signed
else cal_fx.convert(out_fx)
)
if cal_fy is not None:
out_fy = (
cal_fy.convert_component(out_fy)
if cal_fy.signed
else cal_fy.convert(out_fy)
)
return out_fn, out_fx, out_fy
def to_dict(self) -> dict[str, Any]:
channels: dict[str, Any] = {}
for cid, curve in self.channels.items():
json_name = _CANONICAL_TO_JSON.get(cid, cid)
channels[json_name] = curve.to_dict()
# Prefer emitting v2 axis names even when parsed from v1 shared shear.
doc: dict[str, Any] = {
"schema_version": max(int(self.schema_version), 2) if self.channels else int(self.schema_version),
"unit": self.unit,
"channels": channels,
}
meta: dict[str, Any] = {}
meta.update(self.provenance)
meta.update({k: v for k, v in self.identity.items() if k not in meta})
for key in ("stitch_resolution", "dis_mode", "hhd_downsample", "config_ref"):
if key in self.pipeline and self.pipeline[key] is not None:
meta[key] = self.pipeline[key]
if self.measurements:
meta["samples"] = self.measurements
if self.origin and "origin" not in meta:
meta["origin"] = self.origin
if meta:
doc["meta"] = meta
for key, value in self.extras.items():
if key not in doc:
doc[key] = deepcopy(value)
return doc
def _channel_from_block(
channel_id: str,
block: Any,
*,
unit: str = "N",
default_method: str = "piecewise_linear",
) -> ChannelCurve | None:
if not isinstance(block, dict):
return None
method = str(block.get("method", block.get("model", default_method))).lower()
if method in ("linear", "affine"):
slope = block.get("slope")
offset = block.get("offset")
if slope is None or offset is None:
return None
return ChannelCurve(
id=channel_id,
method="linear",
slope=float(slope),
offset=float(offset),
unit=str(block.get("unit", unit)),
raw_key=str(block.get("raw_key", "")),
)
if method != "piecewise_linear":
return None
raw_points = block.get("points", [])
if not isinstance(raw_points, (list, tuple)) or len(raw_points) < 2:
return None
try:
points = tuple(sorted(_parse_point(p) for p in raw_points))
except (TypeError, ValueError, CalibrationError):
return None
if len(points) < 2:
return None
return ChannelCurve(
id=channel_id,
method="piecewise_linear",
points=points,
extrapolate=bool(block.get("extrapolate", True)),
clamp_output_min=block.get("clamp_output_min", 0.0),
input_mode=str(block.get("input", "magnitude")).lower(),
signed=bool(block.get("signed", False)),
unit=str(block.get("unit", unit)),
raw_key=str(block.get("raw_key", _JSON_RAW_KEY.get(_CANONICAL_TO_JSON.get(channel_id, ""), ""))),
)
def check(doc: dict | Calibration, *, pipeline: dict | None = None) -> CheckResult:
"""Validate a document or Calibration. Errors refuse apply/write."""
errors: list[str] = []
warns: list[str] = []
if isinstance(doc, Calibration):
data = doc.to_dict()
channels_obj = doc.channels
schema = doc.schema_version
identity = doc.identity
elif isinstance(doc, dict):
data = doc
schema_raw = data.get("schema_version", 1)
try:
schema = int(schema_raw)
except (TypeError, ValueError):
errors.append(f"invalid schema_version: {schema_raw!r}")
return CheckResult(ok=False, errors=errors, warnings=warns)
if schema not in SUPPORTED_SCHEMAS:
errors.append(f"unsupported schema_version: {schema}")
channels = data.get("channels")
if not isinstance(channels, dict):
errors.append("missing or invalid channels object")
return CheckResult(ok=False, errors=errors, warnings=warns)
channels_obj = {}
unit = str(data.get("unit", "N"))
for json_name, block in channels.items():
try:
cid = canonicalize_channel(json_name) if json_name != "shear" else "shear"
except CalibrationError:
# Future / unknown channel names: skip strict mapping.
continue
if json_name == "shear":
curve = _channel_from_block("fx", block, unit=unit)
else:
curve = _channel_from_block(cid, block, unit=unit)
if curve is None:
method = None
if isinstance(block, dict):
method = block.get("method", block.get("model"))
if isinstance(block, dict) and method not in (
None,
"piecewise_linear",
"linear",
"affine",
):
errors.append(f"unknown method for {json_name}: {method!r}")
elif isinstance(block, dict):
pts = block.get("points", [])
if method in ("piecewise_linear", None) and (
not isinstance(pts, (list, tuple)) or len(pts) < 2
):
errors.append(f"{json_name}: need at least 2 points")
else:
errors.append(f"invalid channel block: {json_name}")
else:
channels_obj[cid if json_name != "shear" else "fx"] = curve
meta = data.get("meta") if isinstance(data.get("meta"), dict) else {}
identity = {
"sensor_type": meta.get("sensor_type", ""),
"sn": meta.get("sn", ""),
}
else:
return CheckResult(ok=False, errors=["document must be dict or Calibration"])
if not channels_obj and not errors:
errors.append("no usable channel curves")
if "fn" not in channels_obj and "fx" not in channels_obj and "fy" not in channels_obj:
if not errors:
errors.append("no FN/FX/FY channels")
if "fx" not in channels_obj:
warns.append("missing FX channel")
if "fy" not in channels_obj:
warns.append("missing FY channel")
sn = str(identity.get("sn") or "")
origin = str(identity.get("origin") or data.get("meta", {}).get("origin", "") if isinstance(data, dict) else "")
if not sn and origin in ("factory",):
warns.append("empty sn on factory calibration")
if pipeline is not None and isinstance(doc, Calibration):
for key in ("stitch_resolution", "dis_mode", "hhd_downsample"):
if key in doc.pipeline and key in pipeline and doc.pipeline[key] != pipeline[key]:
warns.append(f"pipeline mismatch: {key}")
return CheckResult(ok=not errors, errors=errors, warnings=warns)
def parse(data: dict, *, source_path: Path | None = None) -> Calibration:
"""Parse a force_calibration dict into a Calibration object."""
if not isinstance(data, dict):
raise CalibrationError("calibration document must be a JSON object")
schema_raw = data.get("schema_version", 1)
try:
schema = int(schema_raw)
except (TypeError, ValueError) as exc:
raise CalibrationError(f"invalid schema_version: {schema_raw!r}") from exc
if schema not in SUPPORTED_SCHEMAS:
raise CalibrationError(f"unsupported schema_version: {schema}")
unit = str(data.get("unit", "N"))
channels_raw = data.get("channels")
if not isinstance(channels_raw, dict):
raise CalibrationError("missing or invalid channels object")
channels: dict[str, ChannelCurve] = {}
legacy_shear: ChannelCurve | None = None
# Prefer explicit axis channels (v2).
for json_name, cid in (("normal", "fn"), ("shear_x", "fx"), ("shear_y", "fy")):
if json_name in channels_raw:
curve = _channel_from_block(cid, channels_raw[json_name], unit=unit)
if curve is None:
raise CalibrationError(f"invalid channel block: {json_name}")
channels[cid] = curve
# v1 shared shear → both fx and fy when axes absent.
if "shear" in channels_raw and ("fx" not in channels or "fy" not in channels):
shear_block = channels_raw["shear"]
legacy_shear = _channel_from_block("fx", shear_block, unit=unit)
if legacy_shear is None:
raise CalibrationError("invalid channel block: shear")
# Copy with correct ids for each axis.
shared = ChannelCurve(
id="fx",
method=legacy_shear.method,
points=legacy_shear.points,
slope=legacy_shear.slope,
offset=legacy_shear.offset,
signed=legacy_shear.signed or legacy_shear.input_mode == "components",
extrapolate=legacy_shear.extrapolate,
clamp_output_min=legacy_shear.clamp_output_min,
raw_key=legacy_shear.raw_key,
input_mode=legacy_shear.input_mode,
unit=legacy_shear.unit,
)
if "fx" not in channels:
channels["fx"] = shared
if "fy" not in channels:
channels["fy"] = ChannelCurve(
id="fy",
method=shared.method,
points=shared.points,
slope=shared.slope,
offset=shared.offset,
signed=shared.signed,
extrapolate=shared.extrapolate,
clamp_output_min=shared.clamp_output_min,
raw_key=_JSON_RAW_KEY["shear_y"],
input_mode=shared.input_mode,
unit=shared.unit,
)
legacy_shear = shared
# Legacy single-channel file: {"normal_force": {"model": "linear", ...}}
if "fn" not in channels:
block = data.get("normal_force", data if "slope" in data else None)
if isinstance(block, dict):
curve = _channel_from_block("fn", block, unit=unit, default_method="linear")
if curve is not None:
channels["fn"] = curve
if not channels:
raise CalibrationError("no usable channel curves")
meta = data.get("meta") if isinstance(data.get("meta"), dict) else {}
identity_in = data.get("identity") if isinstance(data.get("identity"), dict) else {}
pipeline_in = data.get("pipeline") if isinstance(data.get("pipeline"), dict) else {}
provenance_in = data.get("provenance") if isinstance(data.get("provenance"), dict) else {}
measurements_in = data.get("measurements") if isinstance(data.get("measurements"), dict) else {}
identity = {
"sensor_type": identity_in.get("sensor_type", meta.get("sensor_type", "")),
"sn": identity_in.get("sn", meta.get("sn", "")),
"hardware_rev": identity_in.get("hardware_rev", meta.get("hardware_rev", "")),
"gel_id": identity_in.get("gel_id", meta.get("gel_id", "")),
}
pipeline = {
"stitch_resolution": pipeline_in.get(
"stitch_resolution", meta.get("stitch_resolution")
),
"dis_mode": pipeline_in.get("dis_mode", meta.get("dis_mode")),
"hhd_downsample": pipeline_in.get("hhd_downsample", meta.get("hhd_downsample")),
"config_ref": pipeline_in.get("config_ref", meta.get("config_ref")),
"config_hash": pipeline_in.get("config_hash", meta.get("config_hash")),
}
provenance = {
"origin": provenance_in.get("origin", meta.get("origin", "")),
"created_at": provenance_in.get(
"created_at", meta.get("date", meta.get("created_at", ""))
),
"calibration_id": provenance_in.get("calibration_id", meta.get("calibration_id", "")),
"tool": provenance_in.get("tool", meta.get("tool")),
"operator": provenance_in.get("operator", meta.get("operator", "")),
"station": provenance_in.get("station", meta.get("station", "")),
"record_duration_s": provenance_in.get(
"record_duration_s", meta.get("record_duration_s")
),
}
measurements = measurements_in or meta.get("samples") or {}
origin = str(provenance.get("origin") or "default")
extras = {k: deepcopy(v) for k, v in data.items() if k not in _KNOWN_TOP_KEYS}
cal = Calibration(
schema_version=schema,
unit=unit,
origin=origin,
identity=identity,
pipeline=pipeline,
provenance=provenance,
channels=channels,
measurements=measurements if isinstance(measurements, dict) else {},
extras=extras,
source_path=Path(source_path).resolve() if source_path is not None else None,
_legacy_shear=legacy_shear,
)
result = check(cal)
if not result.ok:
raise CalibrationError("; ".join(result.errors))
# Missing FX/FY are optional; leave them on CheckResult for callers.
# Only surface non-channel warnings as Python warnings.
for w in result.warnings:
if w.startswith("missing F"):
continue
warnings.warn(f"calibration: {w}", stacklevel=2)
return cal
def load(
source: str | Path | None = None,
*,
sensor_type: str | None = None,
sn: str | None = None, # noqa: ARG001 — reserved for SN files
origin: str | None = None, # noqa: ARG001
strict: bool = False,
) -> Calibration | None:
"""Load calibration from an explicit path or product bundle.
Load order:
1. ``source`` file path when it exists
2. ``sensors/<sensor_type>/force_calibration.json``
3. ``None`` (missing bundle file)
Explicit ``source`` that is missing or invalid raises when the path was
provided (or when ``strict=True``). Missing bundle file returns ``None``.
"""
del sn, origin # reserved
path: Path | None = None
explicit = source is not None and str(source).strip() != ""
if explicit:
path = Path(str(source)).expanduser()
if not path.is_file():
msg = f"calibration file not found: {path}"
if strict or explicit:
raise CalibrationError(msg)
return None
elif sensor_type:
from .configs import resolve_sensor_bundle
bundle = resolve_sensor_bundle(str(sensor_type))
if bundle is not None and bundle.force_calibration_json is not None:
path = bundle.force_calibration_json
elif strict:
raise CalibrationError(
f"no force_calibration.json for sensor_type={sensor_type!r}"
)
else:
return None
else:
if strict:
raise CalibrationError("load() requires source or sensor_type")
return None
assert path is not None
try:
data = json.loads(path.read_text(encoding="utf-8"))
except Exception as exc:
msg = f"failed to read calibration JSON: {path}: {exc}"
if strict or explicit:
raise CalibrationError(msg) from exc
return None
try:
return parse(data, source_path=path)
except CalibrationError:
if strict or explicit:
raise
return None
def pipeline_from_sensor_json(path: str | Path | None) -> dict[str, Any]:
"""Pull stitch / tracker fields from a sensor.json for export pipeline/meta."""
pipeline: dict[str, Any] = {
"stitch_resolution": None,
"dis_mode": None,
"hhd_downsample": None,
"config_ref": None,
}
if path is None:
return pipeline
p = Path(path)
pipeline["config_ref"] = str(p)
if not p.is_file():
return pipeline
try:
data = json.loads(p.read_text(encoding="utf-8"))
except Exception:
return pipeline
if not isinstance(data, dict):
return pipeline
camera = data.get("camera") if isinstance(data.get("camera"), dict) else {}
tracker = data.get("tracker") if isinstance(data.get("tracker"), dict) else {}
stitch = camera.get("resolution_stitch")
if isinstance(stitch, (list, tuple)) and len(stitch) >= 2:
pipeline["stitch_resolution"] = [int(stitch[0]), int(stitch[1])]
if "dis_mode" in tracker:
try:
pipeline["dis_mode"] = int(tracker["dis_mode"])
except (TypeError, ValueError):
pipeline["dis_mode"] = tracker.get("dis_mode")
if "hhd_downsample" in tracker:
try:
pipeline["hhd_downsample"] = int(tracker["hhd_downsample"])
except (TypeError, ValueError):
pipeline["hhd_downsample"] = tracker.get("hhd_downsample")
return pipeline
def _piecewise_from_samples(
channel_id: str,
samples: Iterable[CalSample],
*,
signed: bool,
) -> ChannelCurve | None:
rows = sorted(samples, key=lambda s: abs(float(s.raw_mean)))
if len(rows) < 2:
return None
points = tuple((abs(float(s.raw_mean)), float(s.weight_n)) for s in rows)
json_name = _CANONICAL_TO_JSON[channel_id]
return ChannelCurve(
id=channel_id,
method="piecewise_linear",
points=points,
signed=signed,
extrapolate=True,
clamp_output_min=0.0,
raw_key=_JSON_RAW_KEY[json_name],
input_mode="components" if signed else "magnitude",
unit="N",
)
def build(
samples: Mapping[str, list[CalSample]],
*,
identity: dict[str, Any] | None = None,
pipeline: dict[str, Any] | None = None,
origin: str = "user",
extra_meta: dict[str, Any] | None = None,
) -> Calibration:
"""Fit piecewise curves from samples; omit a channel if fewer than 2 points."""
channels: dict[str, ChannelCurve] = {}
warnings_list: list[str] = []
for key in CHANNEL_IDS:
rows = list(samples.get(key, []) or samples.get(key.upper(), []) or [])
# Also accept FN/FX/FY aliases in the mapping keys.
if not rows:
for alias, cid in CHANNEL_ALIASES.items():
if cid == key and alias in samples:
rows = list(samples[alias] or [])
break
signed = key in ("fx", "fy")
curve = _piecewise_from_samples(key, rows, signed=signed)
if curve is None:
if rows:
warnings_list.append(f"{key}: need at least 2 points (have {len(rows)})")
else:
warnings_list.append(f"{key}: no samples")
continue
channels[key] = curve
if not channels:
raise CalibrationError(
"No complete channels to build (each needs ≥2 points). "
+ "; ".join(warnings_list)
)
identity = dict(identity or {})
pipeline = dict(pipeline or {})
measurements: dict[str, list[dict[str, float | int]]] = {}
for key in CHANNEL_IDS:
rows = list(samples.get(key, []) or [])
if not rows:
for alias, cid in CHANNEL_ALIASES.items():
if cid == key and alias in samples:
rows = list(samples[alias] or [])
break
measurements[key] = [asdict(s) for s in rows]
provenance: dict[str, Any] = {
"origin": origin,
"created_at": datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds"),
}
if extra_meta:
for k, v in extra_meta.items():
if k == "samples":
continue
if k in ("stitch_resolution", "dis_mode", "hhd_downsample", "config_ref"):
if k not in pipeline or pipeline[k] is None:
pipeline[k] = v
elif k in ("sensor_type", "sn", "hardware_rev", "gel_id"):
if k not in identity or not identity[k]:
identity[k] = v
else:
provenance[k] = v
cal = Calibration(
schema_version=CURRENT_SCHEMA,
unit="N",
origin=origin,
identity=identity,
pipeline=pipeline,
provenance=provenance,
channels=channels,
measurements=measurements,
extras={},
source_path=None,
)
for w in warnings_list:
warnings.warn(f"calibration build: {w}", stacklevel=2)
return cal
def write(
cal: Calibration,
path: str | Path,
*,
overwrite: bool = False,
) -> Path:
"""Write calibration JSON. Runs check() first; raises on errors."""
result = check(cal)
if not result.ok:
raise CalibrationError("; ".join(result.errors))
path = Path(path)
if path.exists() and not overwrite:
raise FileExistsError(f"calibration file exists: {path}")
path.parent.mkdir(parents=True, exist_ok=True)
doc = cal.to_dict()
# Always emit schema 2 for new writes.
doc["schema_version"] = CURRENT_SCHEMA
path.write_text(
json.dumps(doc, indent=2, ensure_ascii=False) + "\n",
encoding="utf-8",
)
return path.resolve()
def write_user(
cal: Calibration,
*,
bundle_dir: str | Path,
sn: str,
overwrite: bool = True,
) -> Path:
"""Write a per-SN user calibration under ``bundle_dir/calibrations/<SN>.json``."""
sn_clean = str(sn or "").strip()
if not sn_clean:
raise CalibrationError("write_user requires a non-empty sn")
dest = Path(bundle_dir) / "calibrations" / f"{sn_clean}.json"
return write(cal, dest, overwrite=overwrite)
src/orisys/configs.py
-80
View File
@@ -1,5 +1,4 @@
import json
import shutil
from contextlib import ExitStack
from dataclasses import dataclass
from importlib.resources import as_file, files
@@ -167,85 +166,6 @@ def load_force_calibration(name: str) -> dict | None:
return _load_json(bundle.force_calibration_json)
_BUNDLE_JSON_FILES = ("sensor.json", "force_calibration.json", "manifest.json")
def export_sensor_bundle(
name: str,
dest_dir: str | Path,
*,
include_assets: bool = False,
overwrite: bool = True,
) -> Path:
"""Copy one resolved ``sensors/<name>/`` bundle into ``dest_dir/<name>/``.
By default only JSON metadata is copied (``sensor.json``, and when present
``force_calibration.json`` / ``manifest.json``). Pass ``include_assets=True``
to copy the full tree (including ``assets/``) for packaging finger meshes.
"""
bundle = resolve_sensor_bundle(name)
if bundle is None:
available = ", ".join(list_sensor_bundles()) or "none"
raise FileNotFoundError(
f"Sensor bundle not found: {name!r}. Available: {available}"
)
dest_root = Path(dest_dir).expanduser().resolve()
dest = dest_root / bundle.name
if dest.exists():
if not overwrite:
raise FileExistsError(f"Destination already exists: {dest}")
if dest.is_dir():
shutil.rmtree(dest)
else:
dest.unlink()
if include_assets:
shutil.copytree(bundle.root, dest)
return dest
dest.mkdir(parents=True, exist_ok=True)
for filename in _BUNDLE_JSON_FILES:
src = bundle.root / filename
if src.is_file():
shutil.copy2(src, dest / filename)
if not (dest / "sensor.json").is_file():
raise FileNotFoundError(f"sensor.json missing after export from {bundle.root}")
return dest
def export_sensor_bundles(
dest_dir: str | Path,
*,
names: list[str] | tuple[str, ...] | None = None,
include_assets: bool = False,
overwrite: bool = True,
) -> list[Path]:
"""Export several sensor bundles into ``dest_dir``.
When ``names`` is omitted, exports every name in ``PRODUCT_BUNDLES`` that
currently resolves via :func:`resolve_sensor_bundle`.
"""
if names is None:
selected = [name for name in PRODUCT_BUNDLES if resolve_sensor_bundle(name) is not None]
else:
selected = [str(name) for name in names]
if not selected:
raise FileNotFoundError("No sensor bundles selected for export")
exported: list[Path] = []
for name in selected:
exported.append(
export_sensor_bundle(
name,
dest_dir,
include_assets=include_assets,
overwrite=overwrite,
)
)
return exported
def load_config(config_name_or_path):
"""
Load sensor configuration from a JSON file path or config name.
src/orisys/export_sensors.py
-70
View File
@@ -1,70 +0,0 @@
"""CLI: export packaged sensor bundles to a destination folder.
Examples::
python -m orisys.export_sensors --dest ./out/sensors --names box,mini
python -m orisys.export_sensors --dest ./out/sensors --include-assets
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
from .configs import export_sensor_bundles, list_sensor_bundles
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Export orisys sensors/<name>/ bundles to a destination directory"
)
parser.add_argument(
"--dest",
required=True,
type=Path,
help="Destination directory (creates <dest>/<name>/ for each bundle)",
)
parser.add_argument(
"--names",
default="",
help="Comma-separated bundle names (default: all resolvable product bundles)",
)
parser.add_argument(
"--include-assets",
action="store_true",
help="Copy full bundle trees including assets/",
)
parser.add_argument(
"--no-overwrite",
action="store_true",
help="Fail if a destination bundle folder already exists",
)
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = _parse_args(argv)
names = [part.strip() for part in str(args.names).split(",") if part.strip()] or None
try:
exported = export_sensor_bundles(
args.dest,
names=names,
include_assets=bool(args.include_assets),
overwrite=not bool(args.no_overwrite),
)
except (FileNotFoundError, FileExistsError) as exc:
print(f"error: {exc}", file=sys.stderr)
available = list_sensor_bundles()
if available:
print(f"available bundles: {', '.join(available)}", file=sys.stderr)
return 1
for path in exported:
print(f"exported {path}")
print(f"exported {len(exported)} bundle(s) to {Path(args.dest).resolve()}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
src/orisys/sensor.py
+22 -58
View File
@@ -19,7 +19,6 @@ from typing import Optional, Tuple
from .Stitcher import Stitcher
from .tracker import Tracker
from .hhd import nHHD
from . import calibration as cal_mod
from . import configs
from .get_depth import Contact3D
from .backend_availability import (check_cupy_available, check_opencl_available)
@@ -27,6 +26,8 @@ from .frame_processors import create_frame_processor
from .signal_processing import (
DEFAULT_SAMPLE_RATE_HZ,
LowpassFilter,
calibrate_normal_force,
calibrate_shear_components,
)
from .capture import (
CaptureConfig,
@@ -86,7 +87,7 @@ class Sensor:
isstitch: Enable stitching mode for multi-tile sensor (default: False)
verbose: Print initialization logs (default: True)
config_name: Configuration name (default: "prototype")
cal_path: Explicit force_calibration.json path (overrides bundle default)
cal_path: Path to calibration file (default: None)
override: Skip auto-detection, use provided box (default: False)
box: Manual bounding boxes when override=True (default: None)
backend: Poisson backend (default: "auto")
@@ -151,9 +152,7 @@ class Sensor:
self.n_boxes = None
self.fnormal = self.fshearx = self.fsheary = self.ftor = 0.0
self._force_raw = (0.0, 0.0, 0.0)
self.calibration: Optional[cal_mod.Calibration] = None
self._force_cal_sensor_name: Optional[str] = None
self._init_force_calibration()
self._force_cal_sensor_name = self._resolve_force_calibration_sensor_name()
self._lowpass_enabled = False
self._lowpass_cutoff_hz = 0.0
self._lowpass_sample_rate_hz = float(DEFAULT_SAMPLE_RATE_HZ)
@@ -165,9 +164,7 @@ class Sensor:
n_channels=3,
)
if self.verbose:
if self.calibration is not None and self.calibration.source_path is not None:
print(f"Force calibration: {self.calibration.source_path}")
elif self._force_cal_sensor_name:
if self._force_cal_sensor_name:
print(
f"Force calibration: sensors/{self._force_cal_sensor_name}/force_calibration.json"
)
@@ -805,11 +802,17 @@ class Sensor:
self._update_hhd_mask_cache()
return self
def _bundle_name_candidates(self) -> list[str]:
"""Product bundle names implied by ``config_name``."""
def _resolve_force_calibration_sensor_name(self) -> Optional[str]:
"""Return sensors/<name> when force_calibration.json exists; else None.
Lookup order:
1. ``config_name`` stem / product alias (e.g. ``box``)
2. ``sensors/<name>/...`` path segment in ``config_name``
Missing JSON means ``FNORMAL/FSHEAR*`` stay raw (no hardcoded fallback).
"""
text = str(self.config_name or "").strip().replace("\\", "/")
if not text:
return []
return None
candidates: list[str] = []
stem = Path(text).stem if text else ""
@@ -825,55 +828,15 @@ class Sensor:
candidates.append(name)
seen: set[str] = set()
ordered: list[str] = []
for candidate in candidates:
alias = configs.CONFIG_ALIASES.get(candidate, candidate)
for name in (alias, candidate):
if name in seen:
continue
seen.add(name)
ordered.append(name)
return ordered
def _init_force_calibration(self) -> None:
"""Load force calibration from ``cal_path`` or the product bundle."""
self.calibration = None
self._force_cal_sensor_name = None
if self.cal_path is not None and str(self.cal_path).strip():
# Explicit path: missing/invalid raises (no silent fallback).
self.calibration = cal_mod.load(self.cal_path, strict=True)
return
for name in self._bundle_name_candidates():
cal = cal_mod.load(sensor_type=name)
if cal is not None:
self.calibration = cal
self._force_cal_sensor_name = name
return
@property
def force_calibrated(self) -> bool:
"""True when at least one channel curve is loaded."""
return self.calibration is not None and bool(self.calibration.channels)
@property
def force_unit(self) -> str:
return "N" if self.force_calibrated else "raw"
def set_calibration(
self,
cal: "cal_mod.Calibration | str | Path | None",
) -> None:
"""Attach or replace the active force calibration (live swap after save)."""
if cal is None:
self.calibration = None
self._force_cal_sensor_name = None
return
if isinstance(cal, cal_mod.Calibration):
self.calibration = cal
return
self.calibration = cal_mod.load(cal, strict=True)
if configs.load_force_calibration(name) is not None:
return name
return None
def _apply_force_filter(self):
raw = np.asarray(self._force_raw, dtype=np.float64)
@@ -883,11 +846,12 @@ class Sensor:
else:
fn, fx, fy = float(raw[0]), float(raw[1]), float(raw[2])
# Apply loaded force_calibration.json when present; otherwise keep raw.
if self.calibration is not None:
fn, fx, fy = self.calibration.convert(fn, fx, fy)
# Apply sensors/<name>/force_calibration.json when present; otherwise keep raw.
name = self._force_cal_sensor_name
if name is not None:
fn = float(calibrate_normal_force(fn, sensor_name=name))
fx, fy = calibrate_shear_components(fx, fy, sensor_name=name)
fx, fy = float(fx), float(fy)
fn = float(fn)
self.fnormal, self.fshearx, self.fsheary = fn, fx, fy
src/orisys/signal_processing.py
+147 -82
View File
@@ -3,14 +3,10 @@
Batch ``lowpass_filter`` is the zero-phase Butterworth used by example charts.
``LowpassFilter`` is a causal, stateful filter for per-frame ``Sensor`` outputs.
Force calibration is owned by ``orisys.calibration``. These helpers remain as
convenience wrappers:
1. Bundle JSON via ``calibration.load(sensor_type=...)``
2. Hardcoded linear defaults for normal force when no file is present
3. Shear helpers return raw when no curve is available
Prefer ``Sensor.calibration`` / ``read_info(FNORMAL, …)`` in new code.
Force calibration resolution order (when register support arrives):
1. Sensor register / EEPROM calibration blob
2. ``sensors/<name>/force_calibration.json``
3. Hardcoded linear defaults below
"""
from __future__ import annotations
@@ -30,15 +26,20 @@ DEFAULT_LOWPASS_ORDER = 2
def _parse_calibration_point(point) -> tuple[float, float]:
"""Legacy helper; prefer ``orisys.calibration``."""
from .calibration import _parse_point
return _parse_point(point)
if isinstance(point, dict):
raw = point.get("fnormal", point.get("raw", point.get("x")))
force_n = point.get("force_n", point.get("n", point.get("y")))
if raw is None or force_n is None:
raise ValueError(f"invalid calibration point: {point!r}")
return float(raw), float(force_n)
if isinstance(point, (list, tuple)) and len(point) >= 2:
return float(point[0]), float(point[1])
raise ValueError(f"invalid calibration point: {point!r}")
@dataclass(frozen=True)
class ForceChannelCalibration:
"""Compatibility wrapper around ``calibration.ChannelCurve`` convert math."""
"""One force channel: linear coeffs and/or piecewise-linear breakpoints."""
method: str = "linear"
slope: float | None = None
@@ -62,40 +63,75 @@ class ForceChannelCalibration:
unit: str = "N",
default_method: str = "piecewise_linear",
) -> ForceChannelCalibration | None:
from .calibration import _channel_from_block
curve = _channel_from_block(
"fn", block, unit=unit, default_method=default_method
)
if curve is None:
if not isinstance(block, dict):
return None
method = str(block.get("method", block.get("model", default_method))).lower()
if method in ("linear", "affine"):
slope = block.get("slope")
offset = block.get("offset")
if slope is None or offset is None:
return None
return cls.linear(float(slope), float(offset), unit=str(block.get("unit", unit)))
if method != "piecewise_linear":
return None
raw_points = block.get("points", [])
if not isinstance(raw_points, (list, tuple)) or len(raw_points) < 2:
return None
try:
points = tuple(sorted(_parse_calibration_point(p) for p in raw_points))
except (TypeError, ValueError):
return None
if len(points) < 2:
return None
return cls(
method=curve.method,
slope=curve.slope,
offset=curve.offset,
points=curve.points,
extrapolate=curve.extrapolate,
clamp_output_min=curve.clamp_output_min,
input_mode=curve.input_mode,
signed=curve.signed,
unit=curve.unit,
method="piecewise_linear",
points=points,
extrapolate=bool(block.get("extrapolate", True)),
clamp_output_min=block.get("clamp_output_min", 0.0),
input_mode=str(block.get("input", "magnitude")).lower(),
signed=bool(block.get("signed", False)),
unit=str(block.get("unit", unit)),
)
def convert(self, raw: float) -> float:
from .calibration import ChannelCurve
raw = float(raw)
if self.method == "linear":
slope = NORMAL_FORCE_CAL_SLOPE if self.slope is None else float(self.slope)
offset = NORMAL_FORCE_CAL_OFFSET if self.offset is None else float(self.offset)
return slope * raw + offset
return ChannelCurve(
id="fn",
method=self.method,
points=self.points,
slope=self.slope if self.slope is not None else NORMAL_FORCE_CAL_SLOPE,
offset=self.offset if self.offset is not None else NORMAL_FORCE_CAL_OFFSET,
signed=self.signed,
extrapolate=self.extrapolate,
clamp_output_min=self.clamp_output_min,
input_mode=self.input_mode,
unit=self.unit,
).convert(raw)
points = self.points
if len(points) < 2:
return raw
if raw <= points[0][0]:
segment = (points[0], points[1])
if not self.extrapolate:
return self._clamp(points[0][1])
elif raw >= points[-1][0]:
segment = (points[-2], points[-1])
if not self.extrapolate:
return self._clamp(points[-1][1])
else:
segment = None
for left, right in zip(points, points[1:]):
if left[0] <= raw <= right[0]:
segment = (left, right)
break
if segment is None:
return self._clamp(points[-1][1])
(raw0, n0), (raw1, n1) = segment
if raw1 == raw0:
value = n0
else:
ratio = (raw - raw0) / (raw1 - raw0)
value = n0 + ratio * (n1 - n0)
return self._clamp(value)
def convert_component(self, raw: float) -> float:
if not self.signed:
@@ -104,6 +140,11 @@ class ForceChannelCalibration:
sign = -1.0 if raw < 0.0 else 1.0
return sign * self.convert(abs(raw))
def _clamp(self, value: float) -> float:
if self.clamp_output_min is not None:
return max(float(self.clamp_output_min), float(value))
return float(value)
_DEFAULT_NORMAL = ForceChannelCalibration.linear(
NORMAL_FORCE_CAL_SLOPE,
@@ -111,41 +152,57 @@ _DEFAULT_NORMAL = ForceChannelCalibration.linear(
)
def _load_calibration(sensor_name: str | None):
def _load_force_calibration_data(sensor_name: str | None) -> dict | None:
if not sensor_name:
return None
try:
from .calibration import load
from .configs import load_force_calibration
return load(sensor_type=sensor_name)
data = load_force_calibration(sensor_name)
except Exception:
return None
return data if isinstance(data, dict) else None
def _channel_from_data(data: dict | None, channel: str) -> ForceChannelCalibration | None:
if not isinstance(data, dict):
return None
unit = str(data.get("unit", "N"))
channels = data.get("channels")
if isinstance(channels, dict):
block = channels.get(channel)
cal = ForceChannelCalibration.from_channel_dict(block, unit=unit)
if cal is not None:
return cal
# Legacy single-channel file: {"normal_force": {"model": "linear", ...}}
if channel == "normal":
block = data.get("normal_force", data)
cal = ForceChannelCalibration.from_channel_dict(
block, unit=unit, default_method="linear"
)
if cal is not None:
return cal
if isinstance(block, dict):
slope = block.get("slope")
offset = block.get("offset")
if slope is not None and offset is not None:
return ForceChannelCalibration.linear(
float(slope),
float(offset),
unit=str(block.get("unit", unit)),
)
return None
def _normal_calibration(sensor_name: str | None = None) -> ForceChannelCalibration:
cal = _load_calibration(sensor_name)
if cal is not None:
curve = cal.get("fn")
if curve is not None:
return ForceChannelCalibration(
method=curve.method,
slope=curve.slope,
offset=curve.offset,
points=curve.points,
extrapolate=curve.extrapolate,
clamp_output_min=curve.clamp_output_min,
input_mode=curve.input_mode,
signed=curve.signed,
unit=curve.unit,
)
return _DEFAULT_NORMAL
cal = _channel_from_data(_load_force_calibration_data(sensor_name), "normal")
return cal if cal is not None else _DEFAULT_NORMAL
def _shear_curve(sensor_name: str | None, channel: str):
cal = _load_calibration(sensor_name)
if cal is None:
return None
return cal.get(channel)
def _shear_calibration(sensor_name: str | None = None) -> ForceChannelCalibration | None:
return _channel_from_data(_load_force_calibration_data(sensor_name), "shear")
def _load_normal_force_coeffs(sensor_name: str | None = None) -> tuple[float, float]:
@@ -168,20 +225,12 @@ def calibrate_normal_force(raw_value: float, *, sensor_name: str | None = None)
def calibrate_shear_force(raw_value: float, *, sensor_name: str | None = None) -> float:
"""Convert a raw shear magnitude to Newtons using the shear channel curve.
Prefers the average of FX/FY magnitude maps when present.
Returns ``float(raw_value)`` when unavailable.
Returns ``float(raw_value)`` unchanged when no shear calibration is available.
"""
cal_x = _shear_curve(sensor_name, "fx")
cal_y = _shear_curve(sensor_name, "fy")
if cal_x is not None and cal_y is not None:
return 0.5 * (
cal_x.convert(abs(float(raw_value))) + cal_y.convert(abs(float(raw_value)))
)
if cal_x is not None:
return cal_x.convert(abs(float(raw_value)))
if cal_y is not None:
return cal_y.convert(abs(float(raw_value)))
return float(raw_value)
cal = _shear_calibration(sensor_name)
if cal is None:
return float(raw_value)
return cal.convert(raw_value)
def calibrate_shear_components(
@@ -190,11 +239,27 @@ def calibrate_shear_components(
*,
sensor_name: str | None = None,
) -> tuple[float, float]:
"""Convert raw shear components to Newtons via ``calibration.load`` curves."""
cal = _load_calibration(sensor_name)
"""Convert raw shear components to Newtons.
When the shear channel uses ``input: components`` and ``signed: true``, each
axis is calibrated on absolute magnitude then re-signed. Otherwise the
magnitude is calibrated and components are scaled proportionally.
"""
sx = float(shear_x)
sy = float(shear_y)
cal = _shear_calibration(sensor_name)
if cal is None:
return float(shear_x), float(shear_y)
return cal.convert(0.0, float(shear_x), float(shear_y))[1:]
return sx, sy
if cal.input_mode == "components":
return cal.convert_component(sx), cal.convert_component(sy)
magnitude = (sx * sx + sy * sy) ** 0.5
if magnitude <= 0.0:
return 0.0, 0.0
calibrated = cal.convert(magnitude)
scale = calibrated / magnitude
return sx * scale, sy * scale
def _nyquist_hz(sample_rate_hz: float) -> float:
tests/conftest.py
-162
View File
@@ -1,162 +0,0 @@
"""Shared pytest fixtures for the orisys test suite.
This module was accidentally deleted in commit edc6e42 while four test modules
(``test_backends.py``, ``test_smoke_api.py``, ``test_errors_and_cleanup.py``,
``test_longrun_stability.py``) still depend on its fixtures, which made the
whole suite unrunnable. It is restored here with two fixes:
* the default test video now points at clips that are actually tracked in the
repository (``data/sample/``), instead of the never-committed
``tests/idle_1min.mp4``;
* the stale ``"prototype"`` default config in :func:`make_sensor` is gone, so
callers cannot silently bypass the ``stitcher.box_size`` validation.
``tests/test_harness_guards.py`` guards against a repeat deletion.
"""
import os
import sys
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
SRC_ROOT = REPO_ROOT / "src"
BUILD_ROOT = REPO_ROOT / "build"
EXAMPLES_ROOT = REPO_ROOT / "examples"
# Establish import precedence once, here, before any test module is imported. The
# test modules insert the same directories, but all of those inserts are guarded
# by `if path not in sys.path`, so once the paths are present the guards turn
# them into no-ops and this order sticks.
#
# Order matters and is not incidental:
# * examples/ must win over build/ because `build/gui` is also a package named
# `gui`; with build/ first, `import gui.viewer2d` silently resolves to the
# build tooling and fails.
# * build/ must precede site-packages so `wheel`, `common` and `release` mean
# the repository's build tooling, not the unrelated PyPI `wheel` package.
# * src/ provides the checkout's `orisys`.
# Insert in reverse priority (each insert goes to the front).
for _path in (SRC_ROOT, BUILD_ROOT, EXAMPLES_ROOT):
if str(_path) not in sys.path:
sys.path.insert(0, str(_path))
from orisys import Sensor
from orisys import configs as orisys_configs
from orisys.backend_availability import check_cupy_available, check_opencl_available
# Video clips that are tracked in git, in preference order.
DEFAULT_VIDEO_CANDIDATES = (
Path("tests") / "idle_1min.mp4",
Path("data") / "sample" / "s1_480p.mp4",
Path("data") / "sample" / "s1_720p.mp4",
)
def _resolve_test_video() -> Path:
env_video = os.getenv("ORISYS_TEST_VIDEO")
if env_video:
raw = Path(env_video)
candidates = [
raw, # as provided (absolute or cwd-relative)
REPO_ROOT / raw, # repo-relative
]
# Common typo fallback: "test/..." -> "tests/..."
if env_video.startswith("test/") or env_video.startswith("test\\"):
fixed = Path("tests", *Path(env_video).parts[1:])
candidates.append(REPO_ROOT / fixed)
for candidate in candidates:
if candidate.exists():
return candidate
return raw
for relative in DEFAULT_VIDEO_CANDIDATES:
candidate = REPO_ROOT / relative
if candidate.is_file():
return candidate
# Keep the reported path informative when nothing is available; the
# ``test_video_path`` fixture turns this into a skip.
return REPO_ROOT / DEFAULT_VIDEO_CANDIDATES[-1]
@pytest.fixture(scope="session")
def test_video_path() -> Path:
path = _resolve_test_video()
if not path.exists():
pytest.skip(
"Test video not found. Set ORISYS_TEST_VIDEO to a valid video path."
)
return path
@pytest.fixture(scope="session")
def test_config_name() -> str:
requested = os.getenv("ORISYS_TEST_CONFIG")
def _is_usable_config(name: str) -> bool:
cfg = orisys_configs.load_config(name)
return "box_size" in cfg.get("stitcher", {})
if requested:
if not _is_usable_config(requested):
raise RuntimeError(
f"ORISYS_TEST_CONFIG='{requested}' is missing required key "
"'stitcher.box_size'. Use a config like 'ddjx01' or provide a full path."
)
return requested
# Auto-pick a stable config for tests (avoid prototype variants missing keys).
for name in orisys_configs.list_config():
try:
if _is_usable_config(name):
return name
except Exception:
continue
raise RuntimeError(
"No usable config found for tests (missing 'stitcher.box_size'). "
"Set ORISYS_TEST_CONFIG to a valid config JSON/path."
)
@pytest.fixture(scope="session")
def available_backends() -> list[str]:
backends = ["cpu"]
if check_cupy_available():
backends.append("cupy")
if check_opencl_available():
backends.append("opencl")
return backends
def make_sensor(
video_path: Path,
config_name: str,
backend: str = "cpu",
verbose: bool = False,
) -> Sensor:
use_cuda = backend in ("cupy", "opencl")
return Sensor(
vid_src=str(video_path),
cuda=use_cuda,
isstitch=True,
verbose=verbose,
config_name=config_name,
backend=backend,
)
@pytest.fixture
def sensor_factory(test_video_path, test_config_name):
def _factory(backend: str = "cpu", verbose: bool = False):
return make_sensor(
test_video_path,
config_name=test_config_name,
backend=backend,
verbose=verbose,
)
return _factory
tests/dd06.py
+1
View File
@@ -8,6 +8,7 @@ Flow:
4. Feed the rectified frame into Sensor.get_img(..., already_processed=True)
5. Compute deformation/contact and visualize
SDK version: 0.5.0
"""
from __future__ import annotations
tests/test_calibration.py
-243
View File
@@ -1,243 +0,0 @@
"""Tests for orisys.calibration load/parse/write and Sensor apply."""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from orisys import Sensor
from orisys.calibration import (
CalSample,
CalibrationError,
build,
check,
load,
parse,
write,
)
def _box_cal_path() -> Path:
# Editable checkout: repo_root/sensors/box/force_calibration.json
here = Path(__file__).resolve()
candidate = here.parents[1] / "sensors" / "box" / "force_calibration.json"
assert candidate.is_file(), f"missing shipped cal: {candidate}"
return candidate
def test_parse_box_v1_piecewise():
cal = load(sensor_type="box")
assert cal is not None
assert cal.get("FN") is not None
assert cal.get("FN").convert(10000.0) == pytest.approx(0.377)
assert cal.get("FX").convert_component(36000.0) == pytest.approx(1.19)
assert cal.get("FX").convert_component(-36000.0) == pytest.approx(-1.19)
assert cal.get("FY").convert_component(36000.0) == pytest.approx(1.19)
def test_parse_v2_tmp_file(tmp_path: Path):
doc = {
"schema_version": 2,
"unit": "N",
"channels": {
"normal": {
"method": "piecewise_linear",
"points": [[0.0, 0.0], [1000.0, 1.0]],
"extrapolate": True,
"clamp_output_min": 0.0,
},
"shear_x": {
"method": "piecewise_linear",
"signed": True,
"points": [[0.0, 0.0], [500.0, 2.0]],
"extrapolate": True,
"clamp_output_min": 0.0,
},
"shear_y": {
"method": "piecewise_linear",
"signed": True,
"points": [[0.0, 0.0], [500.0, 3.0]],
"extrapolate": True,
"clamp_output_min": 0.0,
},
},
"meta": {"sensor_type": "box", "sn": "TEST-001"},
}
path = tmp_path / "force_calibration_v2.json"
path.write_text(json.dumps(doc), encoding="utf-8")
cal = load(path)
assert cal is not None
assert check(cal).ok
assert cal.convert(1000.0, 500.0, -500.0) == pytest.approx((1.0, 2.0, -3.0))
def test_build_write_load_roundtrip(tmp_path: Path):
samples = {
"fn": [
CalSample(weight_n=0.0, raw_mean=0.0, n=10),
CalSample(weight_n=1.0, raw_mean=1000.0, n=10),
],
"fx": [
CalSample(weight_n=0.0, raw_mean=0.0, n=10),
CalSample(weight_n=2.0, raw_mean=500.0, n=10),
],
"fy": [
CalSample(weight_n=0.0, raw_mean=0.0, n=10),
CalSample(weight_n=3.0, raw_mean=500.0, n=10),
],
}
cal = build(
samples,
identity={"sensor_type": "box", "sn": "RT-1"},
pipeline={"dis_mode": 1},
origin="user",
)
path = tmp_path / "out.json"
write(cal, path, overwrite=True)
loaded = load(path)
assert loaded is not None
assert loaded.convert(1000.0, 500.0, -500.0) == pytest.approx((1.0, 2.0, -3.0))
def test_check_rejects_bad_schema_and_short_points():
bad_schema = {"schema_version": 99, "channels": {"normal": {"method": "linear", "slope": 1, "offset": 0}}}
result = check(bad_schema)
assert not result.ok
assert any("schema_version" in e for e in result.errors)
short = {
"schema_version": 2,
"channels": {
"normal": {
"method": "piecewise_linear",
"points": [[0.0, 0.0]],
}
},
}
result2 = check(short)
assert not result2.ok
def test_sensor_cal_path_v2_applies_to_read_info(tmp_path: Path):
doc = {
"schema_version": 2,
"unit": "N",
"channels": {
"normal": {
"method": "piecewise_linear",
"points": [[0.0, 0.0], [100.0, 5.0]],
"extrapolate": True,
"clamp_output_min": 0.0,
},
"shear_x": {
"method": "piecewise_linear",
"signed": True,
"points": [[0.0, 0.0], [100.0, 7.0]],
"extrapolate": True,
"clamp_output_min": 0.0,
},
"shear_y": {
"method": "piecewise_linear",
"signed": True,
"points": [[0.0, 0.0], [100.0, 9.0]],
"extrapolate": True,
"clamp_output_min": 0.0,
},
},
}
path = tmp_path / "v2.json"
path.write_text(json.dumps(doc), encoding="utf-8")
sensor = Sensor(
"external",
config_name="box",
cal_path=str(path),
verbose=False,
backend="cpu",
)
try:
assert sensor.force_calibrated
assert sensor.force_unit == "N"
sensor._force_raw = (100.0, 100.0, -100.0)
sensor._apply_force_filter()
fn, fx, fy = sensor.read_info(
sensor.info.FNORMAL,
sensor.info.FSHEARX,
sensor.info.FSHEARY,
)
fn_raw, fx_raw, fy_raw = sensor.read_info(
sensor.info.FNORMAL_RAW,
sensor.info.FSHEARX_RAW,
sensor.info.FSHEARY_RAW,
)
assert (fn_raw, fx_raw, fy_raw) == pytest.approx((100.0, 100.0, -100.0))
assert (fn, fx, fy) == pytest.approx((5.0, 7.0, -9.0))
finally:
sensor.disconnect()
def test_set_calibration_swaps_curve(tmp_path: Path):
path_a = tmp_path / "a.json"
path_b = tmp_path / "b.json"
path_a.write_text(
json.dumps(
{
"schema_version": 2,
"channels": {
"normal": {
"method": "piecewise_linear",
"points": [[0.0, 0.0], [10.0, 1.0]],
}
},
}
),
encoding="utf-8",
)
path_b.write_text(
json.dumps(
{
"schema_version": 2,
"channels": {
"normal": {
"method": "piecewise_linear",
"points": [[0.0, 0.0], [10.0, 2.0]],
}
},
}
),
encoding="utf-8",
)
sensor = Sensor(
"external",
config_name="box",
cal_path=str(path_a),
verbose=False,
backend="cpu",
)
try:
sensor._force_raw = (10.0, 0.0, 0.0)
sensor._apply_force_filter()
assert sensor.fnormal == pytest.approx(1.0)
sensor.set_calibration(path_b)
sensor._apply_force_filter()
assert sensor.fnormal == pytest.approx(2.0)
finally:
sensor.disconnect()
def test_missing_cal_path_raises():
with pytest.raises(CalibrationError):
Sensor(
"external",
config_name="box",
cal_path="D:/no/such/force_calibration.json",
verbose=False,
backend="cpu",
)
def test_parse_shipped_box_file_directly():
cal = parse(json.loads(_box_cal_path().read_text(encoding="utf-8")), source_path=_box_cal_path())
assert cal.schema_version == 1
assert cal.convert(10000.0, 36000.0, -36000.0) == pytest.approx((0.377, 1.19, -1.19))
tests/test_export_sensor_bundle.py
-48
View File
@@ -1,48 +0,0 @@
"""Tests for sensor-bundle export helpers."""
from __future__ import annotations
from pathlib import Path
import pytest
from orisys.configs import (
export_sensor_bundle,
export_sensor_bundles,
list_sensor_bundles,
resolve_sensor_bundle,
)
def test_export_sensor_bundle_json_only(tmp_path: Path) -> None:
assert resolve_sensor_bundle("box") is not None
dest = export_sensor_bundle("box", tmp_path, include_assets=False)
assert dest == tmp_path / "box"
assert (dest / "sensor.json").is_file()
assert (dest / "force_calibration.json").is_file()
assert not (dest / "assets").exists()
def test_export_sensor_bundles_default_names(tmp_path: Path) -> None:
available = list_sensor_bundles()
assert "box" in available
exported = export_sensor_bundles(tmp_path, include_assets=False)
names = {path.name for path in exported}
assert "box" in names
assert "mini" in names
for path in exported:
assert (path / "sensor.json").is_file()
assert not (path / "assets").exists()
def test_export_sensor_bundle_no_overwrite(tmp_path: Path) -> None:
export_sensor_bundle("box", tmp_path, overwrite=True)
with pytest.raises(FileExistsError):
export_sensor_bundle("box", tmp_path, overwrite=False)
def test_export_sensor_bundle_missing_name(tmp_path: Path) -> None:
with pytest.raises(FileNotFoundError):
export_sensor_bundle("not-a-real-sensor", tmp_path)
tests/test_harness_guards.py
-142
View File
@@ -1,142 +0,0 @@
"""Guards for the pytest harness itself.
``tests/conftest.py`` was accidentally deleted in commit edc6e42 while four test
modules still depended on its fixtures, which made the suite unrunnable without
anything failing loudly in CI. These tests turn that class of regression into an
explicit failure, and also catch a stale ``orisys`` install silently shadowing
``src/orisys`` (a real hazard: base/py312 environments have older orisys copies
in site-packages).
"""
from __future__ import annotations
import functools
import importlib.util
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
CONFTEST_PATH = Path(__file__).resolve().parent / "conftest.py"
# Names that must exist in tests/conftest.py.
REQUIRED_CONFTEST_NAMES = (
"DEFAULT_VIDEO_CANDIDATES",
"make_sensor",
"test_video_path",
"test_config_name",
"available_backends",
"sensor_factory",
)
# Fixtures that must be resolvable through pytest.
REQUIRED_FIXTURES = (
"test_video_path",
"test_config_name",
"available_backends",
"sensor_factory",
)
@functools.lru_cache(maxsize=1)
def _load_conftest_module():
"""Load tests/conftest.py by path so a deletion is observable here."""
spec = importlib.util.spec_from_file_location("_orisys_conftest_probe", CONFTEST_PATH)
if spec is None or spec.loader is None:
raise AssertionError(f"Cannot load {CONFTEST_PATH}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_conftest_defines_shared_fixtures():
assert CONFTEST_PATH.is_file(), (
f"{CONFTEST_PATH} is missing. test_backends.py, test_smoke_api.py, "
"test_errors_and_cleanup.py and test_longrun_stability.py depend on its "
"fixtures; it was accidentally deleted in commit edc6e42. Restore it "
"instead of deleting it."
)
module = _load_conftest_module()
missing = [
name for name in REQUIRED_CONFTEST_NAMES if getattr(module, name, None) is None
]
assert not missing, (
f"tests/conftest.py no longer defines: {', '.join(missing)}. "
"The shared sensor-test fixtures are required by the suite."
)
def test_default_test_video_is_reachable():
"""At least one default candidate must exist, or every sensor test skips."""
module = _load_conftest_module()
candidates = [REPO_ROOT / Path(rel) for rel in module.DEFAULT_VIDEO_CANDIDATES]
existing = [path for path in candidates if path.is_file()]
assert existing, (
"None of the default test videos exist:\n "
+ "\n ".join(str(path) for path in candidates)
+ "\nCommit a sample clip under data/sample/ or set ORISYS_TEST_VIDEO."
)
def test_fixtures_are_registered_with_pytest(request):
missing: list[str] = []
for name in REQUIRED_FIXTURES:
try:
request.getfixturevalue(name)
except pytest.FixtureLookupError:
missing.append(name)
assert not missing, (
f"pytest did not register these fixtures: {', '.join(missing)}. "
"Check that tests/conftest.py is present and the @pytest.fixture "
"decorators are intact."
)
def test_import_precedence_is_correct():
"""examples/ must beat build/ for `gui`; build/ must beat site-packages.
Regression guard: build/gui is a package named `gui`, so if build/ precedes
examples/ on sys.path then `import gui.viewer2d` resolves to the build
tooling and tests/test_qt_viewer_backend.py fails at collection.
"""
import common
import gui
import wheel
examples_gui = (REPO_ROOT / "examples" / "gui").resolve()
build_root = (REPO_ROOT / "build").resolve()
assert Path(gui.__file__).resolve().parent == examples_gui, (
f"`gui` resolved to {gui.__file__} instead of {examples_gui}; "
"examples/ must precede build/ on sys.path"
)
for module in (common, wheel):
resolved = Path(module.__file__).resolve()
assert build_root in resolved.parents, (
f"`{module.__name__}` resolved to {resolved}, outside {build_root}; "
"build/ must precede site-packages on sys.path"
)
def test_tests_run_against_checkout_not_stale_install():
"""Fail loudly when an old orisys in site-packages shadows src/orisys."""
src_pkg = (REPO_ROOT / "src" / "orisys").resolve()
if not src_pkg.is_dir():
pytest.skip("src/orisys is not present in this checkout layout")
import orisys
module_file = getattr(orisys, "__file__", None)
assert module_file, (
"import orisys resolved to a namespace package (__file__ is None) "
f"instead of {src_pkg}. A stale orisys in site-packages is shadowing "
"the checkout; run `pip uninstall -y orisys` in this environment."
)
resolved_parent = Path(module_file).resolve().parent
assert resolved_parent == src_pkg, (
f"import orisys resolved to {resolved_parent} instead of {src_pkg}. "
"A stale orisys in site-packages is shadowing the checkout; run "
"`pip uninstall -y orisys` in this environment."
)
tests/test_masked_hhd.py
+3 -8
View File
@@ -70,22 +70,17 @@ def test_sensor_detect_motion_ignores_invalid_region():
backend="cpu",
)
try:
# get_img() resizes the frame to the config's camera.resolution_stitch
# and stores valid_mask at that same resolution, so the flow buffer must
# be sized from sensor.valid_mask rather than from the raw frame.
frame = np.zeros((728, 672, 3), dtype=np.uint8)
mask = np.zeros((728, 672), dtype=bool)
mask[180:540, 160:512] = True
sensor.get_img(frame=frame, already_processed=True, valid_mask=mask)
flow_shape = sensor.valid_mask.shape
assert sensor.valid_mask.any() and not sensor.valid_mask.all()
sensor.flow = np.zeros((*flow_shape, 2), dtype=np.float32)
sensor.flow[~sensor.valid_mask] = 10.0
sensor.flow = np.zeros((728, 672, 2), dtype=np.float32)
sensor.flow[~mask] = 10.0
assert not sensor.detect_motion(threshold=1.0)
sensor.flow[:] = 0.0
sensor.flow[sensor.valid_mask] = 10.0
sensor.flow[mask] = 10.0
assert sensor.detect_motion(threshold=1.0)
finally:
sensor.disconnect()
tests/test_nuitka_nofollow.py
-72
View File
@@ -1,72 +0,0 @@
"""Every third-party import in src/orisys must be excluded from the Nuitka compile.
Omitting one means Nuitka tries to follow it into the compiled extension: compile
time explodes, and the wheel's contents start depending on whatever happens to be
installed in the *build* environment rather than on what is declared. ``torch``,
``torchvision`` and ``pyvkfft`` were imported by the package but missing from the
exclusion list; this test keeps that list complete.
"""
from __future__ import annotations
import ast
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
BUILD_ROOT = REPO_ROOT / "build"
if str(BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(BUILD_ROOT))
from wheel.compile import NUITKA_INCLUDE_PACKAGES, NUITKA_NOFOLLOW_IMPORTS # noqa: E402
PACKAGE_ROOT = REPO_ROOT / "src" / "orisys"
FIRST_PARTY = {"orisys", "orisys_blend"}
def _third_party_imports() -> dict[str, set[str]]:
"""Top-level third-party import names -> the files that import them."""
stdlib = set(sys.stdlib_module_names)
found: dict[str, set[str]] = {}
for path in sorted(PACKAGE_ROOT.rglob("*.py")):
try:
tree = ast.parse(path.read_text(encoding="utf-8", errors="ignore"))
except SyntaxError:
continue
for node in ast.walk(tree):
names: list[str] = []
if isinstance(node, ast.Import):
names = [alias.name for alias in node.names]
elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
names = [node.module]
for name in names:
top = name.split(".")[0]
if top in stdlib or top in FIRST_PARTY:
continue
found.setdefault(top, set()).add(path.relative_to(REPO_ROOT).as_posix())
return found
def test_every_third_party_import_is_nofollowed():
found = _third_party_imports()
assert found, "expected to find third-party imports in src/orisys"
missing = sorted(set(found) - set(NUITKA_NOFOLLOW_IMPORTS))
assert not missing, (
"these third-party modules are imported by src/orisys but missing from "
"build/wheel/compile.py::NUITKA_NOFOLLOW_IMPORTS:\n "
+ "\n ".join(f"{name} (imported by {sorted(found[name])})" for name in missing)
+ "\nNuitka will try to follow them into the compiled module; add them to "
"the exclusion list so they resolve from the user's environment at runtime."
)
def test_included_packages_are_first_party_or_subpackages():
assert "orisys" in NUITKA_INCLUDE_PACKAGES
for name in NUITKA_INCLUDE_PACKAGES:
assert name == "orisys" or name.startswith("orisys."), name
def test_nofollow_list_has_no_duplicates():
assert len(NUITKA_NOFOLLOW_IMPORTS) == len(set(NUITKA_NOFOLLOW_IMPORTS))
tests/test_pyproject.py
+2 -15
View File
@@ -28,25 +28,14 @@ def test_pyproject_dependencies(packaging_utils):
def test_pyproject_dynamic_version(packaging_utils):
"""The dynamic version must resolve to a valid version from one source.
Deliberately does NOT assert a literal version: pinning it here means every
release has to edit this file too. ``src/orisys/__init__.py::__version__`` is
the single source of truth, and build/release/version.py checks that the
wheels agree with it.
"""
import re
import sys
build_dir = REPO_ROOT / "build"
if str(build_dir) not in sys.path:
sys.path.insert(0, str(build_dir))
from common.pyproject_utils import get_version
from release.version import dynamic_version_attr
version = get_version(REPO_ROOT)
assert re.fullmatch(r"\d+(?:\.\d+)+(?:[-+][0-9A-Za-z.]+)?", version), version
assert dynamic_version_attr(REPO_ROOT) == "orisys.__version__"
assert get_version(REPO_ROOT) == "0.5.0"
def test_pyproject_bundle_manifest(packaging_utils):
@@ -59,9 +48,7 @@ def test_pyproject_bundle_manifest(packaging_utils):
items = bundle_items(REPO_ROOT)
names = {item["name"] for item in items}
# docs/ is intentionally not bundled: it ships with the GitHub repo and the
# PyPI project page only, so doc edits do not invalidate built wheels.
assert names == {"configs", "examples", "sensors"}
assert names == {"configs", "examples", "docs", "sensors"}
data_files = dict(bundle_data_files(REPO_ROOT))
assert "orisys/configs" in data_files
tests/test_release_auth.py
-387
View File
@@ -1,387 +0,0 @@
"""Tests for PyPI credential setup (~/.pypirc handling).
The writer does text surgery to preserve the explanatory comments in the file, so
these tests check the result of that surgery rather than trusting it. No test
touches the real ``~/.pypirc``: every path is injected.
"""
from __future__ import annotations
import sys
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
BUILD_ROOT = REPO_ROOT / "build"
if str(BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(BUILD_ROOT))
from release import auth # noqa: E402
TOKEN = "pypi-" + "A" * 30
COMMENTED_TEMPLATE = """\
# keep me: why this file exists
# second comment line
[distutils]
index-servers =
pypi
[pypi]
repository = https://upload.pypi.org/legacy/
username = __token__
password = pypi-PASTE-YOUR-PRODUCTION-TOKEN-HERE
# [testpypi] commented out on purpose
# password = pypi-OLD
"""
def test_parse_pypirc_reads_sections_and_ignores_comments():
sections = auth.parse_pypirc(COMMENTED_TEMPLATE)
assert sections["pypi"]["username"] == "__token__"
assert "testpypi" not in sections, "commented-out sections must stay inactive"
def test_placeholder_is_detected_as_not_configured(tmp_path):
path = tmp_path / ".pypirc"
path.write_text(COMMENTED_TEMPLATE, encoding="utf-8")
state = auth.credential_state("pypi", path=path, env={})
assert not state.configured
assert "placeholder" in state.problem
def test_missing_file_is_reported(tmp_path):
state = auth.credential_state("pypi", path=tmp_path / "nope", env={})
assert not state.configured
assert "does not exist" in state.problem
def test_env_variables_count_as_configured(tmp_path):
state = auth.credential_state(
"pypi", path=tmp_path / "nope", env={"TWINE_PASSWORD": "pypi-x"}
)
assert state.configured
assert state.source == "environment variables"
def test_wrong_username_is_rejected(tmp_path):
path = tmp_path / ".pypirc"
path.write_text(
"[pypi]\nusername = someone@example.com\npassword = pypi-real-token-value\n",
encoding="utf-8",
)
state = auth.credential_state("pypi", path=path, env={})
assert not state.configured
assert "__token__" in state.problem
def test_write_token_replaces_placeholder_and_keeps_comments(tmp_path):
path = tmp_path / ".pypirc"
path.write_text(COMMENTED_TEMPLATE, encoding="utf-8")
auth.write_token("pypi", TOKEN, path=path)
text = path.read_text(encoding="utf-8")
assert "# keep me: why this file exists" in text
assert "# [testpypi] commented out on purpose" in text
assert f"password = {TOKEN}" in text
assert "PASTE-YOUR-PRODUCTION-TOKEN-HERE" not in text
assert text.count("[pypi]") == 1
assert auth.credential_state("pypi", path=path, env={}).configured
def test_write_token_adds_missing_section_and_index_servers(tmp_path):
path = tmp_path / ".pypirc"
path.write_text(COMMENTED_TEMPLATE, encoding="utf-8")
auth.write_token("testpypi", TOKEN, path=path)
text = path.read_text(encoding="utf-8")
assert "[testpypi]" in text
assert auth.index_servers(text) == {"pypi", "testpypi"}
assert auth.credential_state("testpypi", path=path, env={}).configured
# The production entry must be untouched.
assert "PASTE-YOUR-PRODUCTION-TOKEN-HERE" in text
def test_write_token_creates_file_when_absent(tmp_path):
path = tmp_path / "nested" / ".pypirc"
auth.write_token("pypi", TOKEN, path=path)
assert path.is_file()
text = path.read_text(encoding="utf-8")
assert "index-servers" in text
assert auth.credential_state("pypi", path=path, env={}).configured
def test_write_token_updates_existing_password_in_place(tmp_path):
path = tmp_path / ".pypirc"
auth.write_token("pypi", TOKEN, path=path)
replacement = "pypi-" + "B" * 30
auth.write_token("pypi", replacement, path=path)
text = path.read_text(encoding="utf-8")
assert f"password = {replacement}" in text
assert TOKEN not in text
assert text.count("password") == 1
def test_upsert_token_does_not_leak_into_the_next_section():
text = "[pypi]\nusername = __token__\n\n[testpypi]\nusername = __token__\n"
result = auth.upsert_token(text, "pypi", TOKEN)
pypi_block, testpypi_block = result.split("[testpypi]")
assert TOKEN in pypi_block
assert TOKEN not in testpypi_block
def test_validate_token_rejects_empty_and_whitespace():
assert auth.validate_token("")
assert auth.validate_token("pypi-abc def")
assert not auth.validate_token(TOKEN)
def test_token_warnings_flag_suspicious_pastes():
assert any("pypi-" in warning for warning in auth.token_warnings("not-a-token"))
assert any("too short" in warning for warning in auth.token_warnings("pypi-abc"))
assert auth.token_warnings(TOKEN) == []
def test_prompt_never_echoes_the_token(capsys):
token = auth.prompt_for_token("pypi", prompt_fn=lambda label: TOKEN)
captured = capsys.readouterr()
assert token == TOKEN
assert TOKEN not in captured.out
assert "hidden" in captured.out
def test_prompt_abort_returns_none():
def _raise(label):
raise EOFError
assert auth.prompt_for_token("pypi", prompt_fn=_raise) is None
def test_ensure_credentials_prompts_when_interactive(tmp_path, monkeypatch, capsys):
from release import pypi
monkeypatch.setattr(auth, "pypirc_path", lambda: tmp_path / ".pypirc")
monkeypatch.setattr(auth, "prompt_for_token", lambda index, **k: TOKEN)
# _ensure_credentials resolves the path through auth.credential_state, so the
# monkeypatched pypirc_path applies there too.
options = pypi.Options(version="1.2.3", apply=True)
assert pypi._ensure_credentials(["pypi"], options, interactive=True) is True
assert "[ok ] wrote [pypi] credentials" in capsys.readouterr().out
assert auth.credential_state("pypi", path=tmp_path / ".pypirc", env={}).configured
def test_ensure_credentials_fails_without_a_terminal(tmp_path, monkeypatch, capsys):
from release import pypi
monkeypatch.setattr(auth, "pypirc_path", lambda: tmp_path / ".pypirc")
options = pypi.Options(version="1.2.3", apply=True)
assert pypi._ensure_credentials(["pypi"], options, interactive=False) is False
output = capsys.readouterr().out
assert "--set-token pypi" in output
assert "prompted" in output
assert not (tmp_path / ".pypirc").exists(), "must not write anything when it cannot prompt"
def test_ensure_credentials_dry_run_warns_but_continues(tmp_path, monkeypatch, capsys):
from release import pypi
monkeypatch.setattr(auth, "pypirc_path", lambda: tmp_path / ".pypirc")
options = pypi.Options(version="1.2.3", apply=False)
assert pypi._ensure_credentials(["pypi"], options, interactive=False) is True
output = capsys.readouterr().out
assert "[warn]" in output
assert "dry run continues" in output
def test_ensure_credentials_reports_testpypi_alternative(tmp_path, monkeypatch, capsys):
from release import pypi
monkeypatch.setattr(auth, "pypirc_path", lambda: tmp_path / ".pypirc")
options = pypi.Options(version="1.2.3", apply=True)
assert pypi._ensure_credentials(["testpypi"], options, interactive=False) is False
output = capsys.readouterr().out
assert "--set-token testpypi" in output
assert "--pypi-only" in output
def test_piped_token_is_used_only_once(tmp_path, monkeypatch, capsys):
from release import pypi
monkeypatch.setattr(auth, "pypirc_path", lambda: tmp_path / ".pypirc")
options = pypi.Options(version="1.2.3", apply=True, token=TOKEN)
assert pypi._ensure_credentials(["pypi", "testpypi"], options, interactive=True) is False
# Second index had no token left to consume, so it fell back to the prompt,
# which is unavailable in tests; nothing may be written for it.
assert options.token is None
assert auth.credential_state("pypi", path=tmp_path / ".pypirc", env={}).configured
assert not auth.credential_state(
"testpypi", path=tmp_path / ".pypirc", env={}
).configured
def test_section_not_listed_in_index_servers_is_not_configured(tmp_path):
"""twine only honours sections named in [distutils] index-servers.
Regression test for a real failure: a hand-added [testpypi] section looked
configured to us while twine answered "Missing 'testpypi' section".
"""
path = tmp_path / ".pypirc"
path.write_text(
"[distutils]\n"
"index-servers =\n"
" pypi\n"
"\n"
"[pypi]\n"
"username = __token__\n"
"password = pypi-" + "A" * 30 + "\n"
"\n"
"[testpypi]\n"
"username = __token__\n"
"password = pypi-" + "B" * 30 + "\n",
encoding="utf-8",
)
state = auth.credential_state("testpypi", path=path, env={})
assert not state.configured
assert "index-servers" in state.problem
assert "--repair" in state.problem
# The production index is unaffected.
assert auth.credential_state("pypi", path=path, env={}).configured
def test_credential_state_agrees_with_twine(tmp_path):
"""Our verdict must match twine's own resolver for the same file."""
twine_utils = pytest.importorskip("twine.utils")
path = tmp_path / ".pypirc"
path.write_text(
"[distutils]\n"
"index-servers =\n"
" pypi\n"
"\n"
"[pypi]\n"
"username = __token__\n"
"password = pypi-" + "A" * 30 + "\n"
"\n"
"[testpypi]\n"
"username = __token__\n"
"password = pypi-" + "B" * 30 + "\n",
encoding="utf-8",
)
twine_sees = set(twine_utils.get_config(str(path)))
for index in ("pypi", "testpypi"):
ours = auth.credential_state(index, path=path, env={})
# twine is the authority: we may only claim configured when twine agrees.
if ours.configured:
assert index in twine_sees, f"{index} accepted by us but ignored by twine"
def test_missing_index_servers_key_falls_back_to_both_indexes(tmp_path):
"""twine's fallback is 'pypi testpypi' only when the key is absent entirely."""
path = tmp_path / ".pypirc"
text = "[distutils]\nindex-servers =\n pypi\n\n[ pypi ]\n".replace("[ pypi ]", "[pypi]")
path.write_text(text, encoding="utf-8")
assert auth.index_servers(path.read_text(encoding="utf-8")) == {"pypi"}
without_key = path.read_text(encoding="utf-8").replace("index-servers =\n pypi\n", "")
path.write_text(without_key, encoding="utf-8", newline="")
assert auth.index_servers(path.read_text(encoding="utf-8")) == {"pypi", "testpypi"}
def test_empty_index_servers_honours_nothing(tmp_path):
"""A trap: `index-servers =` with no entries means twine honours no sections.
Unlike a missing key, an empty value does not trigger the pypi+testpypi
fallback, so a section that looks configured is ignored.
"""
path = tmp_path / ".pypirc"
path.write_text(
"[distutils]\nindex-servers =\n\n"
"[pypi]\nusername = __token__\npassword = pypi-" + "A" * 30 + "\n",
encoding="utf-8",
)
assert auth.index_servers(path.read_text(encoding="utf-8")) == set()
state = auth.credential_state("pypi", path=path, env={})
assert not state.configured
assert "index-servers" in state.problem
auth.repair_index_servers(path)
assert auth.credential_state("pypi", path=path, env={}).configured
def test_index_servers_reads_inline_and_continuation_forms():
assert auth.index_servers("[distutils]\nindex-servers = pypi testpypi\n") == {
"pypi",
"testpypi",
}
assert auth.index_servers("[distutils]\nindex-servers =\n pypi\n") == {"pypi"}
assert auth.index_servers("") == {"pypi", "testpypi"}
def test_unlisted_sections_reports_the_gap(tmp_path):
path = tmp_path / ".pypirc"
path.write_text(
"[distutils]\nindex-servers =\n pypi\n\n"
"[pypi]\npassword = pypi-x\n\n"
"[testpypi]\npassword = pypi-y\n",
encoding="utf-8",
)
assert auth.unlisted_sections(path) == ["testpypi"]
def test_repair_index_servers_fixes_a_hand_edited_file(tmp_path):
path = tmp_path / ".pypirc"
path.write_text(
"[distutils]\nindex-servers =\n pypi\n\n"
"[pypi]\nusername = __token__\npassword = pypi-" + "A" * 30 + "\n\n"
"[testpypi]\nusername = __token__\npassword = pypi-" + "B" * 30 + "\n",
encoding="utf-8",
)
assert not auth.credential_state("testpypi", path=path, env={}).configured
added = auth.repair_index_servers(path)
assert added == ["testpypi"]
assert auth.index_servers(path.read_text(encoding="utf-8")) == {"pypi", "testpypi"}
assert auth.credential_state("testpypi", path=path, env={}).configured
# Tokens and comments survive the repair.
assert "pypi-" + "B" * 30 in path.read_text(encoding="utf-8")
def test_repair_index_servers_is_a_no_op_when_correct(tmp_path):
path = tmp_path / ".pypirc"
auth.write_token("pypi", "pypi-" + "A" * 30, path=path)
before = path.read_text(encoding="utf-8")
assert auth.repair_index_servers(path) == []
assert path.read_text(encoding="utf-8") == before
def test_write_token_makes_the_result_visible_to_twine(tmp_path):
"""The writer must produce a file twine actually honours, not just one we parse."""
path = tmp_path / ".pypirc"
path.write_text("[distutils]\nindex-servers =\n pypi\n", encoding="utf-8")
auth.write_token("testpypi", "pypi-" + "B" * 30, path=path)
assert auth.credential_state("testpypi", path=path, env={}).configured
twine_utils = pytest.importorskip("twine.utils")
assert "testpypi" in set(twine_utils.get_config(str(path)))
@pytest.mark.parametrize("index", ["pypi", "testpypi"])
def test_write_then_state_round_trips(tmp_path, index):
path = tmp_path / ".pypirc"
auth.write_token(index, TOKEN, path=path)
state = auth.credential_state(index, path=path, env={})
assert state.configured, state.problem
tests/test_release_publish.py
-624
View File
@@ -1,624 +0,0 @@
"""Tests for the publish gate, the GitHub plan and the PyPI command shape.
Nothing here touches the network, ``gh`` or an index: the mutating paths are
verified through their pure planning functions, which is exactly what the CLI
executes after the gate passes.
"""
from __future__ import annotations
import json
import subprocess
import sys
import urllib.error
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
BUILD_ROOT = REPO_ROOT / "build"
if str(BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(BUILD_ROOT))
from release import github, publish, pypi # noqa: E402
def _wheel(tmp_path: Path, name: str, payload: bytes = b"wheel") -> Path:
path = tmp_path / name
path.write_bytes(payload)
return path
def _report_payload(version: str, wheels: list[Path], **overrides) -> dict:
payload = {
"schema": 1,
"version": version,
"ok": True,
"aborted": False,
"git": {"commit": "a" * 40},
"options": {"skip_tests": False, "allow_dirty": False},
"wheels": [
{"name": wheel.name, "sha256": publish.sha256_file(wheel), "size": wheel.stat().st_size}
for wheel in wheels
],
}
payload.update(overrides)
return payload
# --- gate -------------------------------------------------------------------
def test_gate_passes_for_matching_wheels(tmp_path):
wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")]
problems, warnings = publish.evaluate_gate(
_report_payload("1.2.3", wheels), "1.2.3", {w.name: w for w in wheels}
)
assert problems == []
assert warnings == []
def test_gate_rejects_version_mismatch(tmp_path):
wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")]
problems, _ = publish.evaluate_gate(
_report_payload("9.9.9", wheels), "1.2.3", {w.name: w for w in wheels}
)
assert any("report is for version" in problem for problem in problems)
def test_gate_rejects_failed_report(tmp_path):
wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")]
problems, _ = publish.evaluate_gate(
_report_payload("1.2.3", wheels, ok=False), "1.2.3", {w.name: w for w in wheels}
)
assert any("result is FAIL" in problem for problem in problems)
def test_gate_rejects_waived_guard_options(tmp_path):
wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")]
payload = _report_payload("1.2.3", wheels, options={"skip_tests": True, "no_runtime": True})
problems, _ = publish.evaluate_gate(payload, "1.2.3", {w.name: w for w in wheels})
assert any("--skip-tests" in problem and "--no-runtime" in problem for problem in problems)
def test_gate_rejects_changed_wheel(tmp_path):
wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")]
payload = _report_payload("1.2.3", wheels)
wheels[0].write_bytes(b"tampered")
problems, _ = publish.evaluate_gate(payload, "1.2.3", {w.name: w for w in wheels})
assert any("changed after preflight" in problem for problem in problems)
def test_gate_rejects_uncertified_and_missing_wheels(tmp_path):
wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")]
payload = _report_payload("1.2.3", wheels)
extra = _wheel(tmp_path, "orisys-1.2.3-cp311-cp311-win_amd64.whl")
problems, _ = publish.evaluate_gate(
payload, "1.2.3", {w.name: w for w in [*wheels, extra]}
)
assert any("not covered by the report" in problem for problem in problems)
def test_gate_warns_on_commit_drift(tmp_path):
wheels = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")]
payload = _report_payload("1.2.3", wheels)
problems, warnings = publish.evaluate_gate(
payload, "1.2.3", {w.name: w for w in wheels}, head_commit="b" * 40
)
assert problems == []
assert any("HEAD is" in warning for warning in warnings)
def test_load_gate_requires_a_report(tmp_path, monkeypatch):
monkeypatch.setattr(publish, "REPO_ROOT", tmp_path, raising=False)
with pytest.raises(publish.PublishGateError, match="no preflight report"):
publish.load_gate(tmp_path, "1.2.3")
def test_load_gate_explains_waived_report(tmp_path):
report_dir = tmp_path / "build" / "release" / "reports" / "1.2.3"
report_dir.mkdir(parents=True)
wheel_dir = tmp_path / "SDK" / "1.2.3"
wheel_dir.mkdir(parents=True)
wheel = wheel_dir / "orisys-1.2.3-cp310-cp310-win_amd64.whl"
wheel.write_bytes(b"wheel")
(report_dir / "preflight.json").write_text(
json.dumps(_report_payload("1.2.3", [wheel], options={"skip_tests": True})),
encoding="utf-8",
)
with pytest.raises(publish.PublishGateError, match="--skip-tests"):
publish.load_gate(tmp_path, "1.2.3")
def test_load_gate_returns_certified_wheels(tmp_path):
report_dir = tmp_path / "build" / "release" / "reports" / "1.2.3"
report_dir.mkdir(parents=True)
wheel_dir = tmp_path / "SDK" / "1.2.3"
wheel_dir.mkdir(parents=True)
wheel = wheel_dir / "orisys-1.2.3-cp310-cp310-win_amd64.whl"
wheel.write_bytes(b"wheel")
(report_dir / "preflight.json").write_text(
json.dumps(_report_payload("1.2.3", [wheel])), encoding="utf-8"
)
gate = publish.load_gate(tmp_path, "1.2.3")
assert [record.name for record in gate.wheels] == [wheel.name]
assert gate.wheels[0].sha256 == publish.sha256_file(wheel)
assert gate.git_commit == "a" * 40
# --- github -----------------------------------------------------------------
def test_github_steps_shape(tmp_path):
assets = [_wheel(tmp_path, f"orisys-1.2.3-cp3n{n}-cp3n{n}-win_amd64.whl") for n in (0, 1)]
steps = github.build_github_steps(
version="1.2.3",
branch="main",
tag="v1.2.3",
assets=assets,
notes_file=None,
)
by_description = {step.description: step.argv for step in steps}
assert by_description["create tag"] == ["git", "tag", "v1.2.3"]
assert by_description["push branch"] == ["git", "push", "origin", "main"]
assert by_description["push tag"] == ["git", "push", "origin", "v1.2.3"]
release_argv = by_description["create GitHub Release with assets"]
assert release_argv[:2] == ["gh", "release"]
assert "v1.2.3" in release_argv
assert "--verify-tag" in release_argv
assert "--latest" in release_argv
assert release_argv[-2:] == [str(asset) for asset in assets]
def test_github_steps_include_notes_file(tmp_path):
notes = tmp_path / "notes.md"
notes.write_text("hello", encoding="utf-8")
steps = github.build_github_steps(
version="1.2.3", branch="main", tag="v1.2.3", assets=[], notes_file=notes
)
release_argv = next(
step.argv for step in steps if step.description.startswith("create GitHub Release")
)
assert "--notes-file" in release_argv
assert str(notes) in release_argv
def test_release_notes_list_every_wheel():
notes = github.build_release_notes(
"1.2.3", [Path("a.whl"), Path("b.whl")], "c" * 40
)
assert "a.whl" in notes and "b.whl" in notes
assert "cccccccccccc" in notes
def test_manifest_scope_excludes_internal_docs():
"""The public repo must not receive our internal process documents."""
entry = github.manifest_entry()
docs_entry = next(item for item in entry["sync"] if item["from"] == "docs")
assert set(docs_entry["include"]) == {"get_started.md", "api.md"}
assert "publishing.md" not in docs_entry["include"]
assert "pypi_readme.md" not in docs_entry["include"]
def test_manifest_scope_covers_examples_and_sensor_json_only():
entry = github.manifest_entry()
examples_entry = next(item for item in entry["sync"] if item["from"] == "examples")
# Whole tree (no include list), with caches excluded.
assert "include" not in examples_entry
assert set(examples_entry["exclude"]) >= {"__pycache__", "*.pyc"}
sensors_entry = next(item for item in entry["sync"] if item["from"] == "sensors")
assert sensors_entry["include"]
assert all(
name.endswith(".json") for name in sensors_entry["include"]
), "binary sensor assets belong in the wheel, not the public repo"
def test_github_steps_resume_skips_tag_creation(tmp_path):
"""A tag pushed by an earlier failed run must not block the release upload."""
assets = [_wheel(tmp_path, "orisys-1.2.3-cp310-cp310-win_amd64.whl")]
steps = github.build_github_steps(
version="1.2.3",
branch="main",
tag="v1.2.3",
assets=assets,
notes_file=None,
create_tag=False,
)
descriptions = [step.description for step in steps]
assert "create tag" not in descriptions
assert descriptions[-1] == "create GitHub Release with assets"
# The pushes stay: pushing an up-to-date ref is a safe no-op.
assert "push branch" in descriptions and "push tag" in descriptions
def test_non_github_remote_is_reported(tmp_path):
repo = tmp_path / "release"
repo.mkdir()
subprocess.run(["git", "init", "-q"], cwd=repo, check=True, capture_output=True)
subprocess.run(
["git", "remote", "add", "origin", "ssh://git@internal.example:24928/orisys-tech/orisys-sdk.git"],
cwd=repo,
check=True,
capture_output=True,
)
problem = github.non_github_remote_problem(repo)
assert problem is not None and "not a GitHub host" in problem
subprocess.run(
["git", "remote", "set-url", "origin", "git@github.com:orisys-tech/orisys-sdk.git"],
cwd=repo,
check=True,
capture_output=True,
)
assert github.non_github_remote_problem(repo) is None
def _synthetic_publish_layout(tmp_path: Path, version: str, *, tag_exists: bool):
"""Build a fake SDK checkout + release repo and point the module at them."""
sdk = tmp_path / "sdk"
wheel_dir = sdk / "SDK" / version
wheel_dir.mkdir(parents=True)
wheels = [wheel_dir / f"orisys-{version}-cp310-cp310-win_amd64.whl"]
wheels[0].write_bytes(b"wheel")
report_dir = sdk / "build" / "release" / "reports" / version
report_dir.mkdir(parents=True)
(report_dir / "preflight.json").write_text(
json.dumps(_report_payload(version, wheels)), encoding="utf-8"
)
release = tmp_path / "orisys-sdk-release"
release.mkdir()
for command in (
["git", "init", "-q"],
["git", "config", "user.email", "test@example.invalid"],
["git", "config", "user.name", "test"],
["git", "remote", "add", "origin", "git@github.com:orisys-tech/orisys-sdk.git"],
):
subprocess.run(command, cwd=release, check=True, capture_output=True)
(release / "README.md").write_text("x", encoding="utf-8")
subprocess.run(["git", "add", "-A"], cwd=release, check=True, capture_output=True)
subprocess.run(["git", "commit", "-qm", "init"], cwd=release, check=True, capture_output=True)
if tag_exists:
subprocess.run(["git", "tag", f"v{version}"], cwd=release, check=True, capture_output=True)
manifest = sdk / "tools" / "downstream_sync_manifest.json"
manifest.parent.mkdir(parents=True)
manifest.write_text(
json.dumps(
{
"version": 1,
"downstreams": [
{"name": "orisys-sdk-release", "path": "../orisys-sdk-release", "sync": []}
],
}
),
encoding="utf-8",
)
return sdk, release, wheels[0]
def test_publish_dry_run_resumes_when_tag_already_exists(tmp_path, monkeypatch, capsys):
sdk, release, wheel = _synthetic_publish_layout(tmp_path, "1.2.3", tag_exists=True)
monkeypatch.setattr(github, "REPO_ROOT", sdk)
monkeypatch.setattr(github, "MANIFEST_PATH", sdk / "tools" / "downstream_sync_manifest.json")
monkeypatch.setattr(github, "_check_gh", lambda repo: [])
monkeypatch.setattr(github, "_release_exists", lambda repo, tag: False)
exit_code = github.publish(
github.Options(version="1.2.3", apply=False, skip_fetch=True)
)
output = capsys.readouterr().out
assert exit_code == 0
assert "already exist" in output and "skipping tag creation" in output
assert "git tag v1.2.3" not in output
assert "gh release create v1.2.3" in output
def test_publish_dry_run_creates_tag_when_absent(tmp_path, monkeypatch, capsys):
sdk, release, wheel = _synthetic_publish_layout(tmp_path, "1.2.4", tag_exists=False)
monkeypatch.setattr(github, "REPO_ROOT", sdk)
monkeypatch.setattr(github, "MANIFEST_PATH", sdk / "tools" / "downstream_sync_manifest.json")
monkeypatch.setattr(github, "_check_gh", lambda repo: [])
monkeypatch.setattr(github, "_release_exists", lambda repo, tag: False)
exit_code = github.publish(
github.Options(version="1.2.4", apply=False, skip_fetch=True)
)
output = capsys.readouterr().out
assert exit_code == 0
assert "git tag v1.2.4" in output
def test_publish_refuses_when_release_already_exists(tmp_path, monkeypatch, capsys):
sdk, release, wheel = _synthetic_publish_layout(tmp_path, "1.2.5", tag_exists=True)
monkeypatch.setattr(github, "REPO_ROOT", sdk)
monkeypatch.setattr(github, "MANIFEST_PATH", sdk / "tools" / "downstream_sync_manifest.json")
monkeypatch.setattr(github, "_check_gh", lambda repo: [])
monkeypatch.setattr(github, "_release_exists", lambda repo, tag: True)
exit_code = github.publish(
github.Options(version="1.2.5", apply=False, skip_fetch=True)
)
assert exit_code == 1
assert "already exists; nothing to publish" in capsys.readouterr().out
def _synthetic_report_layout(tmp_path: Path, version: str) -> tuple[Path, Path]:
"""A fake SDK checkout with wheels and a passing preflight report."""
sdk = tmp_path / "sdk"
wheel_dir = sdk / "SDK" / version
wheel_dir.mkdir(parents=True)
wheel = wheel_dir / f"orisys-{version}-cp310-cp310-win_amd64.whl"
wheel.write_bytes(b"wheel")
report_dir = sdk / "build" / "release" / "reports" / version
report_dir.mkdir(parents=True)
(report_dir / "preflight.json").write_text(
json.dumps(_report_payload(version, [wheel])), encoding="utf-8"
)
return sdk, wheel
def test_upload_phases_default_is_testpypi_only():
assert pypi.upload_phases(pypi.Options(version="1.2.3")) == ["testpypi"]
def test_upload_phases_confirm_pypi_adds_the_real_index():
options = pypi.Options(version="1.2.3", confirm_pypi=True)
assert pypi.upload_phases(options) == ["testpypi", "pypi"]
def test_upload_phases_pypi_only_skips_the_rehearsal():
# The relevant case: a production token cannot be used against TestPyPI.
assert pypi.upload_phases(pypi.Options(version="1.2.3", pypi_only=True)) == ["pypi"]
def test_pypi_only_dry_run_targets_pypi_only(tmp_path, monkeypatch, capsys):
sdk, wheel = _synthetic_report_layout(tmp_path, "1.2.3")
monkeypatch.setattr(pypi, "REPO_ROOT", sdk)
monkeypatch.setattr(pypi, "index_state", lambda index, **k: pypi.IndexState(versions=[]))
from release import auth
auth.write_token("pypi", "pypi-" + "A" * 30, path=tmp_path / ".pypirc")
monkeypatch.setattr(auth, "pypirc_path", lambda: tmp_path / ".pypirc")
exit_code = pypi.publish(
pypi.Options(version="1.2.3", apply=False, pypi_only=True, skip_check=True)
)
output = capsys.readouterr().out
assert exit_code == 0
assert "--pypi-only" in output
assert "upload 1 wheel(s) to pypi" in output
assert "testpypi" not in output
def test_pypi_only_and_confirm_pypi_are_mutually_exclusive():
from release.cli import _cmd_pypi
with pytest.raises(SystemExit) as excinfo:
_cmd_pypi(["--version", "1.2.3", "--pypi-only", "--confirm-pypi"])
assert excinfo.value.code == 2
def test_manifest_checks_use_the_python_placeholder():
entry = github.manifest_entry()
for command in entry["checks"]:
assert command.startswith("{python} "), command
# --- pypi -------------------------------------------------------------------
def test_twine_upload_argv_targets_the_repository():
argv = pypi.twine_upload_argv([Path("a.whl")], "testpypi")
assert argv[:3] == [sys.executable, "-m", "twine"]
assert "--repository" in argv and "testpypi" in argv
assert argv[-1] == "a.whl"
def test_twine_upload_argv_defaults_to_pypi():
argv = pypi.twine_upload_argv([Path("a.whl")], None)
assert "--repository" not in argv
def test_credentials_hint_points_at_the_setup_command(tmp_path, monkeypatch):
from release import auth
monkeypatch.setattr(auth, "pypirc_path", lambda: tmp_path / ".pypirc")
monkeypatch.delenv("TWINE_PASSWORD", raising=False)
hint = pypi.credentials_hint("pypi")
assert hint is not None
assert "release auth --set-token pypi" in hint
def test_credentials_hint_for_testpypi_mentions_pypi_only(tmp_path, monkeypatch):
from release import auth
monkeypatch.setattr(auth, "pypirc_path", lambda: tmp_path / ".pypirc")
monkeypatch.delenv("TWINE_PASSWORD", raising=False)
hint = pypi.credentials_hint("testpypi")
assert hint is not None
assert "--set-token testpypi" in hint
assert "--pypi-only" in hint
def test_credentials_hint_silent_when_configured(tmp_path, monkeypatch):
from release import auth
path = tmp_path / ".pypirc"
auth.write_token("pypi", "pypi-" + "A" * 30, path=path)
monkeypatch.setattr(auth, "pypirc_path", lambda: path)
monkeypatch.delenv("TWINE_PASSWORD", raising=False)
assert pypi.credentials_hint("pypi") is None
def test_index_state_parses_versions(monkeypatch):
class Response:
def __init__(self, payload):
self._payload = json.dumps(payload).encode()
def read(self, *args):
return self._payload
def __enter__(self):
return self
def __exit__(self, *exc):
return False
monkeypatch.setattr(
pypi.urllib.request,
"urlopen",
lambda *a, **k: Response({"releases": {"1.2.3": [{"filename": "a"}], "0.0.0": []}}),
)
state = pypi.index_state("pypi")
assert state.known
assert state.versions == ["1.2.3"]
def test_index_state_reports_lookup_failure(monkeypatch):
def _raise(*args, **kwargs):
raise urllib.error.URLError("offline")
monkeypatch.setattr(pypi.urllib.request, "urlopen", _raise)
state = pypi.index_state("pypi")
assert not state.known
assert "offline" in state.error
def test_index_files_returns_none_when_version_absent(monkeypatch):
class Response:
def read(self, *args):
return json.dumps({"releases": {}}).encode()
def __enter__(self):
return self
def __exit__(self, *exc):
return False
monkeypatch.setattr(pypi.urllib.request, "urlopen", lambda *a, **k: Response())
assert pypi.index_files("pypi", "1.2.3") == (None, None)
def test_index_files_distinguishes_lookup_failure_from_absence(monkeypatch):
"""A network hiccup must not be reported as "not published yet"."""
def _raise(*args, **kwargs):
raise urllib.error.URLError("connection reset")
monkeypatch.setattr(pypi.urllib.request, "urlopen", _raise)
filenames, error = pypi.index_files("testpypi", "1.2.3")
assert filenames is None
assert error is not None and "connection reset" in error
def test_index_files_returns_filenames(monkeypatch):
class Response:
def read(self, *args):
return json.dumps(
{"releases": {"1.2.3": [{"filename": "b.whl"}, {"filename": "a.whl"}]}}
).encode()
def __enter__(self):
return self
def __exit__(self, *exc):
return False
monkeypatch.setattr(pypi.urllib.request, "urlopen", lambda *a, **k: Response())
assert pypi.index_files("pypi", "1.2.3") == (["a.whl", "b.whl"], None)
def test_wait_for_release_retries_until_indexing_finishes(monkeypatch):
"""TestPyPI lags the upload response; the check must poll, not give up."""
calls = {"n": 0}
slept: list[float] = []
def _fake(index, version, *, timeout=20.0):
calls["n"] += 1
if calls["n"] < 3:
return None, None
return ["a.whl"], None
monkeypatch.setattr(pypi, "index_files", _fake)
filenames, error = pypi.wait_for_release(
"testpypi", "1.2.3", attempts=5, delay=1.0, sleep=slept.append
)
assert filenames == ["a.whl"]
assert error is None
assert calls["n"] == 3
assert slept == [1.0, 1.0]
def test_wait_for_release_reports_the_last_error(monkeypatch):
def _fake(index, version, *, timeout=20.0):
return None, "HTTP 503"
monkeypatch.setattr(pypi, "index_files", _fake)
filenames, error = pypi.wait_for_release(
"pypi", "1.2.3", attempts=2, delay=0.0, sleep=lambda _: None
)
assert filenames is None
assert error == "HTTP 503"
def test_existing_release_state_satisfied_conflict_absent(monkeypatch, capsys):
expected = {"orisys-1.2.3-cp310-cp310-win_amd64.whl"}
monkeypatch.setattr(pypi, "index_files", lambda i, v, **k: (None, None))
assert pypi.existing_release_state("pypi", "1.2.3", expected) == "absent"
monkeypatch.setattr(pypi, "index_files", lambda i, v, **k: (sorted(expected), None))
assert pypi.existing_release_state("pypi", "1.2.3", expected) == "satisfied"
monkeypatch.setattr(
pypi, "index_files", lambda i, v, **k: (["orisys-1.2.3-cp311-cp311-win_amd64.whl"], None)
)
assert pypi.existing_release_state("pypi", "1.2.3", expected) == "conflict"
output = capsys.readouterr().out
assert "different file set" in output and "missing" in output
def test_publish_rerun_after_success_is_not_a_failure(tmp_path, monkeypatch, capsys):
"""Re-running a finished phase must succeed, not look like a new error."""
sdk, wheel = _synthetic_report_layout(tmp_path, "1.2.3")
monkeypatch.setattr(pypi, "REPO_ROOT", sdk)
from release import auth
auth.write_token("pypi", "pypi-" + "A" * 30, path=tmp_path / ".pypirc")
monkeypatch.setattr(auth, "pypirc_path", lambda: tmp_path / ".pypirc")
monkeypatch.setattr(
pypi, "index_files", lambda i, v, **k: ([wheel.name], None)
)
exit_code = pypi.publish(
pypi.Options(version="1.2.3", apply=True, pypi_only=True, skip_check=True)
)
output = capsys.readouterr().out
assert exit_code == 0
assert "already serves orisys 1.2.3" in output
assert "every requested index already has this release" in output
def test_publish_aborts_on_a_conflicting_existing_release(tmp_path, monkeypatch, capsys):
sdk, wheel = _synthetic_report_layout(tmp_path, "1.2.3")
monkeypatch.setattr(pypi, "REPO_ROOT", sdk)
from release import auth
auth.write_token("pypi", "pypi-" + "A" * 30, path=tmp_path / ".pypirc")
monkeypatch.setattr(auth, "pypirc_path", lambda: tmp_path / ".pypirc")
monkeypatch.setattr(pypi, "index_files", lambda i, v, **k: (["other.whl"], None))
exit_code = pypi.publish(
pypi.Options(version="1.2.3", apply=True, pypi_only=True, skip_check=True)
)
assert exit_code == 1
assert "different file set" in capsys.readouterr().out
def test_index_json_urls_cover_both_indexes():
assert pypi.INDEX_JSON["testpypi"].startswith("https://test.pypi.org/")
assert pypi.INDEX_JSON["pypi"].startswith("https://pypi.org/")
tests/test_release_version.py
-405
View File
@@ -1,405 +0,0 @@
"""Tests for build/release version guards, report writing and preflight steps."""
from __future__ import annotations
import io
import json
import subprocess
import sys
import urllib.error
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
BUILD_ROOT = REPO_ROOT / "build"
if str(BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(BUILD_ROOT))
from release import report as report_mod # noqa: E402
from release import version as version_mod # noqa: E402
from release.preflight import Preflight, PreflightOptions # noqa: E402
PYPROJECT = """
[project]
name = "orisys"
description = "synthetic"
[tool.setuptools.dynamic]
version = {attr = "orisys.__version__"}
""".lstrip()
def _make_repo(tmp_path: Path, *, version: str = "1.2.3", with_git: bool = False) -> Path:
(tmp_path / "pyproject.toml").write_text(PYPROJECT, encoding="utf-8")
package = tmp_path / "src" / "orisys"
package.mkdir(parents=True)
(package / "__init__.py").write_text(f'__version__ = "{version}"\n', encoding="utf-8")
if with_git:
(tmp_path / ".git").mkdir()
return tmp_path
def _stage_wheel(repo: Path, version: str, name: str | None = None) -> Path:
directory = repo / "SDK" / version
directory.mkdir(parents=True, exist_ok=True)
wheel = directory / (name or f"orisys-{version}-cp310-cp310-win_amd64.whl")
wheel.write_bytes(b"not really a wheel")
return wheel
class _FakeResponse:
def __init__(self, payload: dict) -> None:
self._buffer = io.BytesIO(json.dumps(payload).encode("utf-8"))
def read(self, *args):
return self._buffer.read(*args)
def __enter__(self):
return self
def __exit__(self, *exc) -> bool:
return False
def test_wheel_version_parsing():
assert version_mod.wheel_version(Path("orisys-1.2.3-cp310-cp310-win_amd64.whl")) == "1.2.3"
assert version_mod.wheel_version(Path("orisys-1.0.0-1-cp312-abi3-manylinux_2_28_x86_64.whl")) == "1.0.0"
assert version_mod.wheel_version(Path("something-else.whl")) is None
def test_sdk_versions_are_numerically_sorted(tmp_path):
for name in ("0.9.0", "0.4.10", "1.2.3"):
(tmp_path / "SDK" / name).mkdir(parents=True)
(tmp_path / "SDK" / "not-a-version").mkdir()
assert version_mod.sdk_versions(tmp_path)[:3] == ["0.4.10", "0.9.0", "1.2.3"]
def test_sdk_versions_empty_without_sdk_dir(tmp_path):
assert version_mod.sdk_versions(tmp_path) == []
def test_sdk_wheels_lists_only_that_version(tmp_path):
repo = _make_repo(tmp_path)
_stage_wheel(repo, "1.2.3")
_stage_wheel(repo, "0.4.3")
wheels = version_mod.sdk_wheels(repo, "1.2.3")
assert [w.name for w in wheels] == ["orisys-1.2.3-cp310-cp310-win_amd64.whl"]
def test_dynamic_version_attr_read_from_pyproject(tmp_path):
repo = _make_repo(tmp_path)
assert version_mod.dynamic_version_attr(repo) == "orisys.__version__"
def test_dynamic_version_attr_none_when_absent(tmp_path):
(tmp_path / "pyproject.toml").write_text("[project]\nname = 'x'\n", encoding="utf-8")
assert version_mod.dynamic_version_attr(tmp_path) is None
def test_pypi_released_versions_parses_response(tmp_path, monkeypatch):
fake = _FakeResponse({"releases": {"0.4.2": [{"filename": "a"}], "1.2.3": [{"filename": "b"}], "0.0.0": []}})
monkeypatch.setattr(version_mod.urllib.request, "urlopen", lambda *a, **k: fake)
versions, error = version_mod.pypi_released_versions()
assert error is None
# An empty file list means nothing was ever uploaded for that version.
assert versions == ["0.4.2", "1.2.3"]
def test_pypi_released_versions_reports_http_404_as_empty(monkeypatch):
def _raise(*args, **kwargs):
raise urllib.error.HTTPError(version_mod.PYPI_JSON_URL, 404, "not found", {}, None)
monkeypatch.setattr(version_mod.urllib.request, "urlopen", _raise)
assert version_mod.pypi_released_versions() == ([], None)
def test_pypi_released_versions_returns_error_on_failure(monkeypatch):
def _raise(*args, **kwargs):
raise urllib.error.URLError("no network")
monkeypatch.setattr(version_mod.urllib.request, "urlopen", _raise)
versions, error = version_mod.pypi_released_versions()
assert versions is None
assert error and "URLError" in error
def test_python_for_tag_uses_envs_root(tmp_path):
executable = tmp_path / "py310" / "python.exe"
executable.parent.mkdir(parents=True)
executable.write_bytes(b"")
assert version_mod.python_for_tag("cp310", envs_root=tmp_path) == executable
assert version_mod.python_for_tag("cp311", envs_root=tmp_path) is None
# --- report -----------------------------------------------------------------
def test_write_report_round_trips(tmp_path):
payload = {
"version": "1.2.3",
"ok": False,
"git": {"commit": "abc", "branch": "dev", "dirty": True},
"interpreter": {"version": "3.10.19", "executable": "python"},
"duration_s": 1.5,
"options": {"skip_tests": True},
"steps": [{"name": "tests", "ok": True, "detail": "skipped", "duration_s": 0.0, "data": {}}],
"wheels": [{"name": "w.whl", "size": 12, "sha256": "deadbeef"}],
}
json_path, text_path = report_mod.write_report(payload, tmp_path / "reports")
loaded = json.loads(json_path.read_text(encoding="utf-8"))
assert loaded["schema"] == report_mod.SCHEMA_VERSION
assert loaded["wheels"][0]["sha256"] == "deadbeef"
text = text_path.read_text(encoding="utf-8")
assert "FAIL" in text
assert "deadbeef" in text
assert "waived guards: skip_tests" in text
# --- preflight steps --------------------------------------------------------
def test_preflight_resolves_version_from_pyproject(tmp_path):
repo = _make_repo(tmp_path, version="0.6.0")
preflight = Preflight(PreflightOptions(), repo_root=repo)
assert preflight.version == "0.6.0"
assert preflight.version_source == "pyproject.toml dynamic attr"
def test_preflight_explicit_version_wins(tmp_path):
repo = _make_repo(tmp_path, version="0.6.0")
preflight = Preflight(PreflightOptions(version="0.7.0"), repo_root=repo)
assert preflight.version == "0.7.0"
assert preflight.version_source == "explicit --version"
def test_repo_state_fails_outside_git(tmp_path, capsys):
repo = _make_repo(tmp_path)
step = Preflight(PreflightOptions(), repo_root=repo)._repo_state()
assert not step.ok
assert "not a git repository" in step.detail
def test_version_consistency_detects_mismatch(tmp_path):
repo = _make_repo(tmp_path, version="1.2.3")
step = Preflight(PreflightOptions(version="9.9.9"), repo_root=repo)._version_consistency()
assert not step.ok
assert "1.2.3" in step.detail and "9.9.9" in step.detail
def test_version_consistency_passes_on_match(tmp_path):
repo = _make_repo(tmp_path, version="1.2.3")
step = Preflight(PreflightOptions(version="1.2.3"), repo_root=repo)._version_consistency()
assert step.ok, step.detail
def test_duplicate_guard_allows_staged_sdk_dir(tmp_path, monkeypatch):
"""A populated SDK/<version>/ is the normal preflight input, not a hazard."""
repo = _make_repo(tmp_path, version="1.2.3")
_stage_wheel(repo, "1.2.3")
monkeypatch.setattr(version_mod, "git_tags", lambda root: [])
step = Preflight(PreflightOptions(version="1.2.3"), repo_root=repo)._duplicate_guard()
assert step.ok, step.detail
assert "staged wheel(s) to verify" in step.detail
def test_duplicate_guard_blocks_released_tag(tmp_path, monkeypatch):
repo = _make_repo(tmp_path, version="1.2.3")
_stage_wheel(repo, "1.2.3")
monkeypatch.setattr(version_mod, "git_tags", lambda root: ["v1.2.3", "v0.5.1"])
step = Preflight(PreflightOptions(version="1.2.3"), repo_root=repo)._duplicate_guard()
assert not step.ok
assert "already mark 1.2.3 as released" in step.detail
assert "--force" in step.detail
def test_duplicate_guard_force_accepts_released_tag(tmp_path, monkeypatch):
repo = _make_repo(tmp_path, version="1.2.3")
monkeypatch.setattr(version_mod, "git_tags", lambda root: ["1.2.3"])
step = Preflight(
PreflightOptions(version="1.2.3", force=True), repo_root=repo
)._duplicate_guard()
assert step.ok, step.detail
assert "accepted via --force" in step.detail
def test_duplicate_guard_reports_unreleased_version(tmp_path, monkeypatch):
repo = _make_repo(tmp_path, version="0.5.3")
monkeypatch.setattr(version_mod, "git_tags", lambda root: ["v1.2.3"])
step = Preflight(PreflightOptions(version="0.5.3"), repo_root=repo)._duplicate_guard()
assert step.ok, step.detail
assert "no local git tag marks 0.5.3" in step.detail
def test_duplicate_guard_flags_published_version(tmp_path, monkeypatch):
repo = _make_repo(tmp_path, version="1.2.3")
monkeypatch.setattr(version_mod, "git_tags", lambda root: [])
monkeypatch.setattr(version_mod, "pypi_released_versions", lambda *a, **k: (["1.2.3"], None))
step = Preflight(
PreflightOptions(version="1.2.3", check_pypi=True), repo_root=repo
)._duplicate_guard()
assert not step.ok
assert "already published on PyPI" in step.detail
def test_released_tags_for_matches_version_spellings():
tags = ["v0.5.1", "0.5.1", "v0.4.3", "0.50"]
assert version_mod.released_tags_for("0.5.1", tags) == ["0.5.1", "v0.5.1"]
assert version_mod.released_tags_for("1.2.3", tags) == []
def test_git_tags_reads_repo_tags(tmp_path):
repo = tmp_path / "repo"
repo.mkdir()
for command in (
["git", "init", "-q"],
["git", "config", "user.email", "test@example.invalid"],
["git", "config", "user.name", "test"],
):
subprocess.run(command, cwd=repo, check=True, capture_output=True)
(repo / "f.txt").write_text("x", encoding="utf-8")
subprocess.run(["git", "add", "f.txt"], cwd=repo, check=True, capture_output=True)
subprocess.run(["git", "commit", "-qm", "init"], cwd=repo, check=True, capture_output=True)
subprocess.run(["git", "tag", "v1.2.3"], cwd=repo, check=True, capture_output=True)
assert "v1.2.3" in version_mod.git_tags(repo)
def test_version_string_scan_flags_stale_literal(tmp_path):
repo = tmp_path / "repo"
(repo / "docs").mkdir(parents=True)
(repo / "docs" / "a.md").write_text("版本：0.4.0\n", encoding="utf-8")
(repo / "README.md").write_text("release 1.2.3\n", encoding="utf-8")
hits = version_mod.find_version_literals(repo, "1.2.3")
assert [(path, line) for path, line, _ in hits] == [("docs/a.md", 1)]
def test_version_string_scan_ignores_dependency_constraints(tmp_path):
repo = tmp_path / "repo"
repo.mkdir()
(repo / "docs.md").write_text(
"pip install numpy>=1.24.4 scipy==1.13 opencv-python>=4.10.0.84\n",
encoding="utf-8",
)
assert version_mod.find_version_literals(repo, "1.2.3") == []
def test_version_string_scan_honours_allowlist(tmp_path, monkeypatch):
monkeypatch.setitem(
version_mod.VERSION_STRING_ALLOWLIST, "docs.md", frozenset({"0.4.0"})
)
repo = tmp_path / "repo"
repo.mkdir()
(repo / "docs.md").write_text("历史版本：0.4.0\n", encoding="utf-8")
assert version_mod.find_version_literals(repo, "1.2.3") == []
def test_version_string_scan_skips_tests_directory(tmp_path):
repo = tmp_path / "repo"
(repo / "tests").mkdir(parents=True)
(repo / "tests" / "test_x.py").write_text('v = "0.1.2"\n', encoding="utf-8")
assert version_mod.find_version_literals(repo, "1.2.3") == []
def test_version_string_scan_accepts_current_version_and_v_prefix(tmp_path):
repo = tmp_path / "repo"
repo.mkdir()
(repo / "README.md").write_text("current 1.2.3 and tag v1.2.3\n", encoding="utf-8")
assert version_mod.find_version_literals(repo, "1.2.3") == []
def test_preflight_version_strings_step_fails_on_leftover(tmp_path):
repo = _make_repo(tmp_path, version="1.2.3")
(repo / "docs").mkdir()
(repo / "docs" / "guide.md").write_text("版本：0.4.0\n", encoding="utf-8")
step = Preflight(PreflightOptions(version="1.2.3"), repo_root=repo)._version_strings()
assert not step.ok
assert "0.4.0" in step.detail
assert "VERSION_STRING_ALLOWLIST" in step.detail
def test_preflight_version_strings_step_passes_on_clean_tree(tmp_path):
repo = _make_repo(tmp_path, version="1.2.3")
(repo / "README.md").write_text("no version literal here\n", encoding="utf-8")
step = Preflight(PreflightOptions(version="1.2.3"), repo_root=repo)._version_strings()
assert step.ok, step.detail
def test_repo_has_no_stale_version_literals():
"""The real repository must satisfy its own version-string gate."""
from common.pyproject_utils import get_version
version = get_version(REPO_ROOT)
assert version_mod.find_version_literals(REPO_ROOT, version) == []
def test_bundle_precheck_fails_when_pattern_resolves_nothing(tmp_path):
repo = _make_repo(tmp_path, version="1.2.3")
(repo / "pyproject.toml").write_text(
PYPROJECT
+ """
[[tool.orisys.bundle.items]]
name = "configs"
source = "config"
target = "configs"
patterns = ["*.json"]
""",
encoding="utf-8",
)
step = Preflight(PreflightOptions(version="1.2.3"), repo_root=repo)._bundle_precheck()
assert not step.ok
assert "resolved 0 files" in step.detail
def test_wheel_inspection_fails_without_wheels(tmp_path):
repo = _make_repo(tmp_path, version="1.2.3")
step = Preflight(PreflightOptions(version="1.2.3"), repo_root=repo)._wheel_inspection()
assert not step.ok
assert "does not exist" in step.detail
def test_wheel_runtime_fails_without_video(tmp_path, monkeypatch):
repo = _make_repo(tmp_path, version="1.2.3")
_stage_wheel(repo, "1.2.3")
monkeypatch.setattr("release.preflight.default_verification_video", lambda root: None)
preflight = Preflight(PreflightOptions(version="1.2.3"), repo_root=repo)
preflight._discover_wheels()
step = preflight._wheel_runtime()
assert not step.ok
assert "never silently skipped" in step.detail
def test_wheel_runtime_skip_is_recorded(tmp_path):
repo = _make_repo(tmp_path, version="1.2.3")
step = Preflight(
PreflightOptions(version="1.2.3", no_runtime=True), repo_root=repo
)._wheel_runtime()
assert step.ok
assert "SKIPPED" in step.detail
def test_tests_skip_is_loud_and_recorded(tmp_path):
repo = _make_repo(tmp_path, version="1.2.3")
step = Preflight(
PreflightOptions(version="1.2.3", skip_tests=True), repo_root=repo
)._tests()
assert step.ok
assert "SKIPPED" in step.detail
assert step.data["status"] == "skipped"
def test_execute_marks_failure_and_aborts(tmp_path, capsys):
repo = _make_repo(tmp_path, version="1.2.3")
preflight = Preflight(PreflightOptions(version="1.2.3"), repo_root=repo)
preflight._execute(preflight._repo_state)
assert preflight.aborted
assert preflight.steps[-1].name == "repo-state"
assert not preflight.steps[-1].ok
def test_execute_keep_going_does_not_abort(tmp_path):
repo = _make_repo(tmp_path, version="1.2.3")
preflight = Preflight(PreflightOptions(version="1.2.3", keep_going=True), repo_root=repo)
preflight._execute(preflight._repo_state)
assert not preflight.aborted
tests/test_signal_processing.py
+2 -3
View File
@@ -224,13 +224,12 @@ def test_box_read_info_applies_force_calibration_json():
def test_read_info_force_stays_raw_without_calibration_json(monkeypatch):
monkeypatch.setattr(
"orisys.calibration.load",
lambda *args, **kwargs: None,
"orisys.sensor.configs.load_force_calibration",
lambda _name: None,
)
sensor = Sensor("external", config_name="box", verbose=False, backend="cpu")
try:
assert sensor._force_cal_sensor_name is None
assert sensor.calibration is None
sensor._force_raw = (111.0, -22.0, 33.0)
sensor._apply_force_filter()
fn, fx, fy = sensor.read_info(
tests/test_wheel_build.py
+6 -6
View File
@@ -19,25 +19,25 @@ def wheel_package():
def test_compliant_manylinux_tags(wheel_package):
assert wheel_package.is_compliant_manylinux_wheel(
Path("orisys-1.2.3-cp310-cp310-manylinux_2_17_x86_64.whl")
Path("orisys-0.5.0-cp310-cp310-manylinux_2_17_x86_64.whl")
)
assert wheel_package.is_compliant_manylinux_wheel(
Path("orisys-1.2.3-cp310-cp310-manylinux2014_x86_64.whl")
Path("orisys-0.5.0-cp310-cp310-manylinux2014_x86_64.whl")
)
assert wheel_package.is_compliant_manylinux_wheel(
Path("orisys-1.2.3-cp310-cp310-manylinux_2_28_x86_64.whl")
Path("orisys-0.5.0-cp310-cp310-manylinux_2_28_x86_64.whl")
)
def test_rejects_illegal_manylinux_tags(wheel_package):
assert not wheel_package.is_compliant_manylinux_wheel(
Path("orisys-1.2.3-cp310-cp310-manylinux_x86_64.whl")
Path("orisys-0.5.0-cp310-cp310-manylinux_x86_64.whl")
)
assert not wheel_package.is_compliant_manylinux_wheel(
Path("orisys-1.2.3-cp310-cp310-linux_x86_64.whl")
Path("orisys-0.5.0-cp310-cp310-linux_x86_64.whl")
)
assert not wheel_package.is_compliant_manylinux_wheel(
Path("orisys-1.2.3-cp310-cp310-win_amd64.whl")
Path("orisys-0.5.0-cp310-cp310-win_amd64.whl")
)
tests/test_wheel_contents.py
-378
View File
@@ -1,378 +0,0 @@
"""Tests for :mod:`common.verify.wheel_contents`.
Every check is exercised in both directions: a well-formed synthetic wheel must
pass, and each specific defect must fail with the check named in the report.
Synthetic wheels keep the tests fast and keep them independent of whatever
happens to sit in ``SDK/``.
"""
from __future__ import annotations
import base64
import hashlib
import sys
import zipfile
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
BUILD_ROOT = REPO_ROOT / "build"
if str(BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(BUILD_ROOT))
from common.verify import wheel_contents # noqa: E402
from common.verify.wheel_contents import inspect_wheel # noqa: E402
VERSION = "1.2.3"
DIST_INFO = f"orisys-{VERSION}.dist-info"
CORE_DEPS = ("numpy>=1.24.4", "opencv-python>=4.10.0.84")
EXTRAS = {"cuda12": ("cupy-cuda12x>=12.3.0",), "gui": ("open3d>=0.17.0",)}
PYPROJECT = f"""
[project]
name = "orisys"
version = "{VERSION}"
requires-python = ">=3.10"
dependencies = [{", ".join(repr(dep) for dep in CORE_DEPS)}]
[project.optional-dependencies]
cuda12 = ["cupy-cuda12x>=12.3.0"]
gui = ["open3d>=0.17.0"]
[[tool.orisys.bundle.items]]
name = "configs"
source = "config"
target = "configs"
patterns = ["*.json"]
[[tool.orisys.bundle.items]]
name = "sensors"
source = "sensors"
target = "sensors"
patterns = ["**/*.json"]
preserve_tree = true
[[tool.orisys.bundle.items]]
name = "docs"
source = "docs"
target = "docs"
patterns = ["*.md"]
[[tool.orisys.bundle.items]]
name = "examples"
source = "examples"
target = "examples"
patterns = ["*.py", "README.md"]
preserve_tree = true
"""
# Source tree files, keyed by path relative to the synthetic repo root.
SOURCE_FILES = {
"config/box.json": '{"name": "box"}',
"sensors/box/sensor.json": '{"name": "box"}',
"sensors/finger/manifest.json": '{"name": "finger"}',
"docs/api.md": "# api\n",
"examples/basic_pipeline.py": "print('hi')\n",
"examples/README.md": "# examples\n",
}
# Where each source file must land inside the wheel.
BUNDLED_ENTRIES = {
"orisys/configs/box.json": SOURCE_FILES["config/box.json"],
"orisys/sensors/box/sensor.json": SOURCE_FILES["sensors/box/sensor.json"],
"orisys/sensors/finger/manifest.json": SOURCE_FILES["sensors/finger/manifest.json"],
"orisys/docs/api.md": SOURCE_FILES["docs/api.md"],
"orisys/examples/basic_pipeline.py": SOURCE_FILES["examples/basic_pipeline.py"],
"orisys/examples/README.md": SOURCE_FILES["examples/README.md"],
}
def _metadata_text(
*,
version: str = VERSION,
requires_python: str | None = ">=3.10",
classifiers: tuple[str, ...] = ("Programming Language :: Python :: 3.10",),
project_urls: tuple[str, ...] = ("Homepage, https://example.invalid",),
license_line: str | None = "License: Proprietary",
core_deps: tuple[str, ...] = CORE_DEPS,
extras: dict[str, tuple[str, ...]] | None = None,
dynamic: tuple[str, ...] = (),
name: str = "orisys",
) -> str:
if extras is None:
extras = EXTRAS
lines = ["Metadata-Version: 2.4", f"Name: {name}", f"Version: {version}", "Summary: synthetic"]
if requires_python:
lines.append(f"Requires-Python: {requires_python}")
if license_line:
lines.append(license_line)
lines.extend(f"Classifier: {value}" for value in classifiers)
lines.extend(f"Project-URL: {value}" for value in project_urls)
lines.extend(f"Dynamic: {value}" for value in dynamic)
lines.extend(f"Requires-Dist: {dep}" for dep in core_deps)
for extra, requirements in extras.items():
lines.append(f"Provides-Extra: {extra}")
lines.extend(f'Requires-Dist: {req}; extra == "{extra}"' for req in requirements)
return "\n".join(lines) + "\n"
def _write_source_tree(root: Path) -> None:
root.mkdir(parents=True, exist_ok=True)
(root / "pyproject.toml").write_text(PYPROJECT, encoding="utf-8")
for relative, text in SOURCE_FILES.items():
path = root / relative
path.parent.mkdir(parents=True, exist_ok=True)
# write_bytes, not write_text: text mode would translate "\n" to "\r\n"
# on Windows and the byte-for-byte bundle comparison would fail.
path.write_bytes(text.encode("utf-8"))
def _record(entries: dict[str, bytes]) -> str:
lines = []
for name, data in sorted(entries.items()):
digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode()
lines.append(f"{name},sha256={digest},{len(data)}")
lines.append(f"{DIST_INFO}/RECORD,,")
return "\n".join(lines) + "\n"
def _make_wheel(
tmp_path: Path,
*,
filename: str | None = None,
metadata: str | None = None,
drop: tuple[str, ...] = (),
extra_entries: dict[str, bytes] | None = None,
override_bytes: dict[str, bytes] | None = None,
drop_record: bool = False,
corrupt_record_hash: bool = False,
) -> tuple[Path, Path]:
"""Build a synthetic wheel plus its matching source tree."""
repo = tmp_path / "repo"
_write_source_tree(repo)
if metadata is None:
metadata = _metadata_text()
entries: dict[str, bytes] = {name: data.encode("utf-8") for name, data in BUNDLED_ENTRIES.items()}
entries["orisys/__init__.cp312-win_amd64.pyd"] = b"compiled"
entries["orisys_blend/__init__.py"] = b"# blend\n"
entries["orisys_blend/kernel.py"] = b"# kernel\n"
entries[f"{DIST_INFO}/METADATA"] = metadata.encode("utf-8")
entries[f"{DIST_INFO}/WHEEL"] = b"Wheel-Version: 1.0\n"
entries.update(extra_entries or {})
entries.update(override_bytes or {})
for name in drop:
entries.pop(name, None)
if not drop_record:
record = _record(entries)
if corrupt_record_hash:
record = record.replace("sha256=", "sha256=AAAA", 1)
entries[f"{DIST_INFO}/RECORD"] = record.encode("utf-8")
wheel = tmp_path / (filename or f"orisys-{VERSION}-cp312-cp312-win_amd64.whl")
with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as zf:
for name, data in entries.items():
zf.writestr(name, data)
return wheel, repo
def _failed_names(result) -> set[str]:
return {check.name for check in result.failures}
@pytest.fixture(autouse=True)
def _isolated_repo_root(monkeypatch, tmp_path):
"""Keep inspect_wheel's default repo root from leaking the real repo in."""
monkeypatch.setattr(wheel_contents, "REPO_ROOT", tmp_path / "unused")
def test_good_wheel_passes(tmp_path):
wheel, repo = _make_wheel(tmp_path)
result = inspect_wheel(wheel, expected_version=VERSION, repo_root=repo)
assert result.ok, result.render()
assert result.version == VERSION
assert result.platform_tag == "win_amd64"
assert not result.warnings, result.warnings
def test_missing_wheel_file_fails(tmp_path):
result = inspect_wheel(tmp_path / "nope-1.2.3-cp312-cp312-win_amd64.whl")
assert not result.ok
assert "wheel-exists" in _failed_names(result)
def test_unparsable_filename_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path, filename="orisys-1.2.3-cp312-cp312.whl")
result = inspect_wheel(wheel, repo_root=repo)
assert "filename" in _failed_names(result)
def test_version_mismatch_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path)
result = inspect_wheel(wheel, expected_version="9.9.9", repo_root=repo)
failed = _failed_names(result)
assert {"filename", "metadata-version"} <= failed
def test_missing_requires_python_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path, metadata=_metadata_text(requires_python=None))
assert "metadata-requires-python" in _failed_names(inspect_wheel(wheel, repo_root=repo))
def test_missing_classifier_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path, metadata=_metadata_text(classifiers=()))
assert "metadata-classifiers" in _failed_names(inspect_wheel(wheel, repo_root=repo))
def test_missing_project_url_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path, metadata=_metadata_text(project_urls=()))
assert "metadata-project-urls" in _failed_names(inspect_wheel(wheel, repo_root=repo))
def test_missing_license_fails(tmp_path):
wheel, repo = _make_wheel(
tmp_path,
metadata=_metadata_text(license_line=None, classifiers=("Intended Audience :: Developers",)),
)
assert "metadata-license" in _failed_names(inspect_wheel(wheel, repo_root=repo))
def test_dynamic_version_is_rejected(tmp_path):
wheel, repo = _make_wheel(tmp_path, metadata=_metadata_text(dynamic=("version", "summary")))
result = inspect_wheel(wheel, repo_root=repo)
assert "metadata-no-dynamic-version" in _failed_names(result)
assert any("Dynamic" in warning for warning in result.warnings)
def test_missing_core_requirement_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path, metadata=_metadata_text(core_deps=("numpy>=1.24.4",)))
assert "metadata-requires-dist" in _failed_names(inspect_wheel(wheel, repo_root=repo))
def test_missing_extra_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path, metadata=_metadata_text(extras={"cuda12": EXTRAS["cuda12"]}))
assert "metadata-extras" in _failed_names(inspect_wheel(wheel, repo_root=repo))
def test_missing_compiled_extension_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path, drop=("orisys/__init__.cp312-win_amd64.pyd",))
assert "compiled-extension" in _failed_names(inspect_wheel(wheel, repo_root=repo))
def test_python_source_leak_fails(tmp_path):
wheel, repo = _make_wheel(
tmp_path, extra_entries={"orisys/sensor.py": b"# leaked\n"}
)
result = inspect_wheel(wheel, repo_root=repo)
assert "no-python-sources" in _failed_names(result)
assert "orisys/sensor.py" in result.render()
def test_example_python_files_are_allowed(tmp_path):
wheel, repo = _make_wheel(
tmp_path, extra_entries={"orisys/examples/plot_data.py": b"# shipped on purpose\n"}
)
assert inspect_wheel(wheel, repo_root=repo).ok
def test_missing_bundled_file_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path, drop=("orisys/sensors/box/sensor.json",))
result = inspect_wheel(wheel, repo_root=repo)
assert "bundle-completeness" in _failed_names(result)
assert "orisys/sensors/box/sensor.json" in result.render()
def test_missing_bundled_file_warns_when_not_strict(tmp_path):
"""A file added to the tree after an old wheel was built is not a defect."""
wheel, repo = _make_wheel(tmp_path, drop=("orisys/sensors/box/sensor.json",))
result = inspect_wheel(wheel, repo_root=repo, strict_bundle=False)
assert result.ok, result.render()
assert any("absent from the wheel" in warning for warning in result.warnings)
def test_bundle_drift_is_fatal_when_strict(tmp_path):
wheel, repo = _make_wheel(
tmp_path, override_bytes={"orisys/docs/api.md": b"# stale\n"}
)
assert "bundle-fidelity" in _failed_names(inspect_wheel(wheel, repo_root=repo, strict_bundle=True))
def test_bundle_drift_warns_when_not_strict(tmp_path):
wheel, repo = _make_wheel(
tmp_path, override_bytes={"orisys/docs/api.md": b"# stale\n"}
)
result = inspect_wheel(wheel, repo_root=repo, strict_bundle=False)
assert result.ok, result.render()
assert any("differ from the source tree" in warning for warning in result.warnings)
def test_pycache_entry_fails(tmp_path):
wheel, repo = _make_wheel(
tmp_path, extra_entries={"orisys/examples/__pycache__/x.pyc": b"junk"}
)
assert "no-forbidden-entries" in _failed_names(inspect_wheel(wheel, repo_root=repo))
def test_scratch_directory_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path, extra_entries={"tests/test_smoke.py": b"x"})
assert "no-forbidden-entries" in _failed_names(inspect_wheel(wheel, repo_root=repo))
def test_traversing_entry_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path, extra_entries={"../escape.txt": b"x"})
assert "archive-paths" in _failed_names(inspect_wheel(wheel, repo_root=repo))
def test_record_hash_mismatch_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path, corrupt_record_hash=True)
assert "record" in _failed_names(inspect_wheel(wheel, repo_root=repo))
def test_missing_record_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path, drop_record=True)
assert "record" in _failed_names(inspect_wheel(wheel, repo_root=repo))
def test_size_limit_fails(tmp_path):
wheel, repo = _make_wheel(tmp_path)
assert "size-limit" in _failed_names(inspect_wheel(wheel, repo_root=repo, max_bytes=1))
def test_bad_manylinux_tag_fails(tmp_path):
wheel, repo = _make_wheel(
tmp_path, filename=f"orisys-{VERSION}-cp312-cp312-linux_x86_64.whl"
)
assert "platform-tag" in _failed_names(inspect_wheel(wheel, repo_root=repo))
def test_good_manylinux_tag_passes(tmp_path):
wheel, repo = _make_wheel(
tmp_path,
filename=f"orisys-{VERSION}-cp312-cp312-manylinux_2_28_x86_64.whl",
)
assert inspect_wheel(wheel, repo_root=repo).ok
def test_wrong_distribution_name_fails(tmp_path):
wheel, repo = _make_wheel(
tmp_path,
filename=f"notorisys-{VERSION}-cp312-cp312-win_amd64.whl",
metadata=_metadata_text(name="notorisys"),
)
failed = _failed_names(inspect_wheel(wheel, repo_root=repo))
assert {"filename", "metadata-name"} <= failed
def test_wheel_layout_module_resolves_to_build_tree():
import wheel
resolved = Path(wheel.__file__).resolve()
assert BUILD_ROOT.resolve() in resolved.parents, (
f"`wheel` resolved to {resolved}; build/ must precede site-packages so the "
"build tooling imports the repository's wheel package."
)
tests/test_wheel_setup_metadata.py
-172
View File
@@ -1,172 +0,0 @@
"""Tests for build/wheel/package.py metadata plumbing.
The Nuitka wheel is built from a generated ``setup.py``, which is exactly where
the early 0.5.x releases silently lost ``Requires-Python``, classifiers, project URLs
and the license. These tests run setuptools for real (in a temp dir, on a stub
package tree) and assert on the METADATA it produces.
"""
from __future__ import annotations
import subprocess
import sys
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
BUILD_ROOT = REPO_ROOT / "build"
if str(BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(BUILD_ROOT))
from wheel.package import metadata_kwargs, read_long_description, write_wheel_setup # noqa: E402
PROJECT = {
"name": "orisys",
"description": "SDK for Orisys Technology Visuotactile Sensors",
"requires-python": ">=3.10",
"license": {"text": "Proprietary"},
"authors": [{"name": "Orisys-Tech"}],
"maintainers": [{"name": "Yijian Zhang", "email": "zhangyijian@orisys-tech.cn"}],
"dependencies": ["numpy>=1.24.4"],
"optional-dependencies": {"gui": ["open3d>=0.17.0"]},
"classifiers": [
"Development Status :: 4 - Beta",
"License :: Other/Proprietary License",
],
"urls": {
"Homepage": "https://www.orisys-tech.cn",
"Documentation": "https://github.com/Orisys-Tech/orisys-sdk",
},
}
LONG_DESCRIPTION = "# Orisys SDK\n\nlong description body.\n"
def _stub_wheel_root(wheel_dir: Path) -> None:
"""Minimal package tree so the generated setup.py can build metadata."""
(wheel_dir / "wheel_root" / "orisys").mkdir(parents=True)
(wheel_dir / "wheel_root" / "orisys" / "__init__.cp312-win_amd64.pyd").write_bytes(b"x")
(wheel_dir / "wheel_root" / "orisys" / "examples").mkdir()
(wheel_dir / "wheel_root" / "orisys" / "examples" / "__init__.py").write_text("", encoding="utf-8")
(wheel_dir / "wheel_root" / "orisys_blend").mkdir()
(wheel_dir / "wheel_root" / "orisys_blend" / "__init__.py").write_text("", encoding="utf-8")
def _generate(wheel_dir: Path, tmp_path: Path) -> Path:
_stub_wheel_root(wheel_dir)
write_wheel_setup(
wheel_dir,
version="1.2.3",
package_data={"orisys": ["*.pyd"]},
install_requires=["numpy>=1.24.4"],
extras_require={"gui": ["open3d>=0.17.0"]},
project=PROJECT,
long_description=LONG_DESCRIPTION,
)
completed = subprocess.run(
[sys.executable, "setup.py", "-q", "dist_info"],
cwd=wheel_dir,
capture_output=True,
text=True,
check=False,
)
assert completed.returncode == 0, (
f"generated setup.py failed:\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}"
)
candidates = sorted(wheel_dir.glob("**/*.dist-info/METADATA"))
assert candidates, f"no METADATA produced under {wheel_dir}"
return candidates[0]
def test_metadata_kwargs_translates_pyproject():
kwargs = metadata_kwargs(PROJECT)
assert kwargs["description"] == PROJECT["description"]
assert kwargs["python_requires"] == ">=3.10"
assert kwargs["license"] == "Proprietary"
assert kwargs["classifiers"] == PROJECT["classifiers"]
assert kwargs["project_urls"] == PROJECT["urls"]
assert kwargs["author"] == "Orisys-Tech"
assert kwargs["maintainer"] == "Yijian Zhang <zhangyijian@orisys-tech.cn>"
def test_metadata_kwargs_accepts_string_license_and_drops_empties():
kwargs = metadata_kwargs({"name": "orisys", "license": "Proprietary"})
assert kwargs["license"] == "Proprietary"
assert "python_requires" not in kwargs
assert "classifiers" not in kwargs
def test_read_long_description_uses_pypi_readme(tmp_path):
(tmp_path / "docs").mkdir()
(tmp_path / "docs" / "pypi_readme.md").write_text(LONG_DESCRIPTION, encoding="utf-8")
assert read_long_description(tmp_path) == LONG_DESCRIPTION
def test_read_long_description_missing_returns_none(tmp_path, capsys):
assert read_long_description(tmp_path) is None
assert "not found" in capsys.readouterr().out
def test_generated_wheel_metadata_is_complete(tmp_path):
wheel_dir = tmp_path / "wheel"
wheel_dir.mkdir()
metadata_path = _generate(wheel_dir, tmp_path)
text = metadata_path.read_text(encoding="utf-8")
assert "Name: orisys" in text
assert "Version: 1.2.3" in text
assert f"Summary: {PROJECT['description']}" in text
assert "Requires-Python: >=3.10" in text
assert "License: Proprietary" in text
assert "Classifier: Development Status :: 4 - Beta" in text
assert "Classifier: License :: Other/Proprietary License" in text
assert "Project-URL: Homepage, https://www.orisys-tech.cn" in text
assert "Project-URL: Documentation, https://github.com/Orisys-Tech/orisys-sdk" in text
assert "Requires-Dist: numpy>=1.24.4" in text
assert 'Provides-Extra: gui' in text
assert "Description-Content-Type: text/markdown" in text
assert "long description body." in text
def test_generated_wheel_metadata_keeps_version_static(tmp_path):
"""``Version`` (and ``Name``) must never be annotated as dynamic.
setuptools marks every field supplied through the legacy ``setup()`` path as
``Dynamic:`` even when the value is present and correct (verified by
``test_generated_wheel_metadata_is_complete``). PEP 643 ``Dynamic`` is
sdist-only and pip/PyPI ignore it in wheels, so only the identity fields are
asserted here.
"""
wheel_dir = tmp_path / "wheel"
wheel_dir.mkdir()
text = _generate(wheel_dir, tmp_path).read_text(encoding="utf-8")
dynamic = {line[len("Dynamic: ") :].strip() for line in text.splitlines() if line.startswith("Dynamic:")}
assert "version" not in dynamic, f"wheel METADATA marks Version as dynamic: {sorted(dynamic)}"
assert "name" not in dynamic, f"wheel METADATA marks Name as dynamic: {sorted(dynamic)}"
def test_write_wheel_setup_without_project_keeps_legacy_shape(tmp_path):
"""Calling without metadata must not crash (backwards compatible)."""
wheel_dir = tmp_path / "wheel"
wheel_dir.mkdir()
setup_py = write_wheel_setup(
wheel_dir,
version="0.0.1",
package_data={"orisys": ["*.pyd"]},
install_requires=["numpy"],
extras_require={},
)
text = setup_py.read_text(encoding="utf-8")
assert 'name="orisys"' in text
assert 'version="0.0.1"' in text
assert "long_description" not in text
def test_pypi_readme_exists_in_repo():
assert (REPO_ROOT / "docs" / "pypi_readme.md").is_file(), (
"docs/pypi_readme.md is the PyPI long description source; without it "
"wheels are published with no project description."
)
tools/downstream_sync_manifest.json
+16 -16
View File
@@ -5,21 +5,21 @@
"name": "orisys-sdk-release",
"path": "../orisys-sdk-release",
"sync": [
{"from": "docs", "to": "docs", "include": ["get_started.md", "api.md"]},
{"from": "examples", "to": "examples", "exclude": ["__pycache__", "*.pyc", "*.pyo"]},
{"from": "config", "to": "config", "include": ["box.json", "finger.json", "mini.json"]},
{"from": "sensors", "to": "sensors", "include": [
"box/sensor.json",
"box/force_calibration.json",
"finger/sensor.json",
"finger/manifest.json",
"mini/sensor.json"
]}
{"from": "docs", "to": "docs"},
{"from": "examples", "to": "examples", "include": [
"README.md",
"basic_pipeline.py",
"mini_pipeline.py",
"finger_pipeline.py",
"finger_pipeline_3dview.py",
"plot_data.py",
"two_sensor_pipeline.py"
]},
{"from": "config", "to": "config", "include": ["box.json", "finger.json", "mini.json"]}
],
"checks": [
"{python} -c \"import pathlib, py_compile; files=sorted(pathlib.Path('examples').rglob('*.py')); [py_compile.compile(str(f), doraise=True) for f in files]; print('compiled %d example module(s)' % len(files))\"",
"{python} -c \"import pathlib, sys; bad=[str(p) for p in pathlib.Path('.').rglob('*') if p.is_file() and p.suffix in ('.md','.py') and 'examples/products' in p.read_text(encoding='utf-8', errors='ignore').replace(chr(92),'/')]; print('stale products paths:' if bad else 'no stale products paths'); [print(' ', x) for x in bad]; sys.exit(1 if bad else 0)\"",
"{python} -c \"import pathlib, sys; leaked=[str(p) for p in pathlib.Path('.').rglob('*') if p.name in ('publishing.md','pypi_readme.md')]; print('internal docs leaked into the public repo:', leaked); sys.exit(1 if leaked else 0)\""
"python -m py_compile examples/basic_pipeline.py examples/finger_pipeline.py examples/finger_pipeline_3dview.py examples/plot_data.py",
"python -c \"import pathlib; p=pathlib.Path('.'); hits=[f for f in p.rglob('*') if f.is_file() and 'examples/products' in f.read_text(encoding='utf-8', errors='ignore').replace('\\\\','/')]; print('stale products paths:' if hits else 'no stale products paths'); [print(x) for x in hits]\""
]
},
{
@@ -29,7 +29,7 @@
{"from": "config", "to": "config", "include": ["box.json", "mini.json", "dd06-ov-0810.json"]}
],
"checks": [
"{python} tests/test_smoke.py"
"python tests/test_smoke.py"
]
},
{
@@ -40,8 +40,8 @@
{"from": "assets/finger", "to": "assets/finger"}
],
"checks": [
"{python} tests/test_smoke.py",
"{python} -c \"import pathlib; t=pathlib.Path('README.md').read_text(encoding='utf-8'); print('README still mentions examples/products' if 'examples/products' in t else 'README products path ok')\""
"python tests/test_smoke.py",
"python -c \"import pathlib; t=pathlib.Path('README.md').read_text(encoding='utf-8'); print('README still mentions examples/products' if 'examples/products' in t else 'README products path ok')\""
]
}
]
tools/sync_downstreams.py
+23 -74
View File
@@ -2,9 +2,7 @@
"""Plan and apply downstream file syncs from tools/downstream_sync_manifest.json.
The ``apply`` command copies files/directories listed in the manifest. It does
not delete extra downstream files and does not run downstream tests by default;
``run-checks`` (or ``release github``) runs the manifest's ``checks`` commands
with ``{python}`` replaced by the running interpreter.
not delete extra downstream files and does not run downstream tests.
"""
from __future__ import annotations
@@ -12,24 +10,13 @@ from __future__ import annotations
import argparse
import json
import shutil
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
MANIFEST_NAME = "downstream_sync_manifest.json"
@dataclass(frozen=True)
class SyncJob:
"""One source -> destination copy, with optional name patterns to skip."""
source: Path
dest: Path
exclude: tuple[str, ...] = ()
def sdk_root() -> Path:
return Path(__file__).resolve().parent.parent
@@ -56,19 +43,18 @@ def selected_downstreams(manifest: dict, target: str | None) -> list[dict]:
return matches
def expand_jobs(root: Path, item: dict) -> list[SyncJob]:
def expand_jobs(root: Path, item: dict) -> list[tuple[Path, Path]]:
dest_root = (root / item["path"]).resolve()
jobs: list[SyncJob] = []
jobs: list[tuple[Path, Path]] = []
for entry in item.get("sync", []):
src_base = root / entry["from"]
dest_base = dest_root / entry["to"]
exclude = tuple(entry.get("exclude") or ())
includes = entry.get("include")
if includes:
for name in includes:
jobs.append(SyncJob(src_base / name, dest_base / name, exclude))
jobs.append((src_base / name, dest_base / name))
else:
jobs.append(SyncJob(src_base, dest_base, exclude))
jobs.append((src_base, dest_base))
return jobs
@@ -82,9 +68,8 @@ def print_plan(root: Path, items: list[dict]) -> None:
jobs = expand_jobs(root, item)
if jobs:
print("sync:")
for job in jobs:
note = f" (excluding {', '.join(job.exclude)})" if job.exclude else ""
print(f" {job.source} -> {job.dest}{note}")
for src, dest in jobs:
print(f" {src} -> {dest}")
else:
print("sync: (none)")
checks = item.get("checks") or []
@@ -109,14 +94,14 @@ def check_paths(root: Path, items: list[dict]) -> int:
else:
print(f"MISS downstream root: {dest_root}")
missing.append(str(dest_root))
for job in expand_jobs(root, item):
src_ok = job.source.exists()
dest_note = "exists" if job.dest.exists() else "missing (ok until apply)"
for src, dest in expand_jobs(root, item):
src_ok = src.exists()
dest_note = "exists" if dest.exists() else "missing (ok until apply)"
status = "ok " if src_ok else "MISS"
print(f"{status} source: {job.source}")
print(f" dest: {job.dest} [{dest_note}]")
print(f"{status} source: {src}")
print(f" dest: {dest} [{dest_note}]")
if not src_ok:
missing.append(str(job.source))
missing.append(str(src))
print()
if missing:
print(f"{len(missing)} missing path(s).")
@@ -125,16 +110,14 @@ def check_paths(root: Path, items: list[dict]) -> int:
return 0
def copy_job(job: SyncJob) -> None:
if not job.source.exists():
raise FileNotFoundError(job.source)
if job.source.is_dir():
# exclude keeps caches out of the public repo (e.g. __pycache__, *.pyc)
ignore = shutil.ignore_patterns(*job.exclude) if job.exclude else None
shutil.copytree(job.source, job.dest, dirs_exist_ok=True, ignore=ignore)
def copy_job(src: Path, dest: Path) -> None:
if not src.exists():
raise FileNotFoundError(src)
if src.is_dir():
shutil.copytree(src, dest, dirs_exist_ok=True)
return
job.dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(job.source, job.dest)
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dest)
def apply_sync(root: Path, items: list[dict]) -> int:
@@ -145,45 +128,16 @@ def apply_sync(root: Path, items: list[dict]) -> int:
copied = 0
for item in items:
print(f"== {item['name']} ==")
for job in expand_jobs(root, item):
copy_job(job)
for src, dest in expand_jobs(root, item):
copy_job(src, dest)
copied += 1
print(f"copied {job.source} -> {job.dest}")
print(f"copied {src} -> {dest}")
print()
print(f"copied {copied} sync job(s).")
print("checks were not run; execute the listed check commands manually.")
return 0
def run_checks(root: Path, items: list[dict], *, python: str | None = None) -> int:
"""Run each manifest ``checks`` command inside the downstream repo.
``{python}`` in a command is replaced with ``python`` (or the interpreter
given). Returns the number of failed checks.
"""
interpreter = python or sys.executable
failures = 0
for item in items:
dest_root = (root / item["path"]).resolve()
checks = item.get("checks") or []
if not checks:
continue
print(f"== {item['name']} ==")
for command in checks:
resolved = command.replace("{python}", interpreter)
print(f"running: {resolved}")
completed = subprocess.run(
resolved, shell=True, cwd=dest_root, check=False
)
if completed.returncode != 0:
print(f"[FAIL] exit {completed.returncode}: {resolved}")
failures += 1
else:
print("[ok ] check passed")
print()
return failures
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest="command", required=True)
@@ -197,9 +151,6 @@ def parse_args(argv: list[str]) -> argparse.Namespace:
apply_cmd = sub.add_parser("apply", help="copy manifest sources to downstreams")
apply_cmd.add_argument("--target", help="limit to one downstream name")
run = sub.add_parser("run-checks", help="run the manifest checks in each downstream")
run.add_argument("--target", help="limit to one downstream name")
return parser.parse_args(argv)
@@ -214,8 +165,6 @@ def main(argv: list[str] | None = None) -> int:
return check_paths(root, items)
if args.command == "apply":
return apply_sync(root, items)
if args.command == "run-checks":
return 1 if run_checks(root, items) else 0
raise SystemExit(f"unknown command: {args.command}")