diff --git a/.gitignore b/.gitignore index dd9df22..4aaeb7d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ __pycache__/ /build/dist/ /build/lib/ /build/pyinstaller_work/ +/build/release/reports/ /build/*.spec /dist/ /dist_obfuscated/ diff --git a/build/README.md b/build/README.md index fce906d..8a9668a 100644 --- a/build/README.md +++ b/build/README.md @@ -12,6 +12,7 @@ build/ README.md common/ shared helpers (paths, pyproject, verify) wheel/ SDK .whl (Nuitka + manylinux Docker) + release/ local release gate (preflight / audit) gui/ common/ shared PyInstaller driver ory_finger/ ORY-FINGER dashboard @@ -41,6 +42,22 @@ build\wheel\win.bat .\build\wheel\docker\run.ps1 ``` +### Release gate + +Local only: never publishes, and makes no network call unless `--check-pypi` is +passed. See `docs/publishing.md` for the full flow. + +```bash +python build/run.py release preflight --version 0.5.2 +python build/run.py release preflight --all-wheel-pythons --check-pypi +python build/run.py release audit SDK\0.5.2\orisys-0.5.2-cp312-cp312-win_amd64.whl +``` + +`preflight` runs: repo state → version consistency → duplicate guard → pytest → +bundle precheck → per-wheel static content/metadata assertions → isolated runtime +verification (bundled assets + real frames, no camera). Reports land in +`build/release/reports//`. + ### GUI ```bash @@ -53,13 +70,16 @@ python build/run.py gui box # scaffold / NotImplemented ```bash python build/run.py verify path\to\orisys.whl +python build/run.py verify path\to\orisys.whl --contents --assets --runtime python build/run.py verify --gui-dist --product ory_finger ``` ## Outputs - Wheels: `SDK//`, `dist/repaired/` +- Preflight reports: `build/release/reports//` - Finger GUI: `dist/OrisysDashboard/` - Future mini/box: `dist/OryMini/`, `dist/OryBox/` -Generated work files under `build/nuitka/` and `build/pyinstaller_work/` are gitignored. +Generated work files under `build/nuitka/`, `build/release/reports/` and +`build/pyinstaller_work/` are gitignored. diff --git a/build/common/verify/__init__.py b/build/common/verify/__init__.py index f9d7fa5..d866817 100644 --- a/build/common/verify/__init__.py +++ b/build/common/verify/__init__.py @@ -1,6 +1,19 @@ """Post-build smoke tests for wheels and product GUI folders.""" from common.verify.gui import GUI_PRODUCTS, verify_gui_dist -from common.verify.wheel import verify_built_wheel +from common.verify.wheel import ( + verify_built_wheel, + verify_wheel_assets, + verify_wheel_runtime, +) +from common.verify.wheel_contents import WheelCheckResult, inspect_wheel -__all__ = ["GUI_PRODUCTS", "verify_built_wheel", "verify_gui_dist"] +__all__ = [ + "GUI_PRODUCTS", + "WheelCheckResult", + "inspect_wheel", + "verify_built_wheel", + "verify_gui_dist", + "verify_wheel_assets", + "verify_wheel_runtime", +] diff --git a/build/common/verify/cli.py b/build/common/verify/cli.py index acf7f29..fa948fa 100644 --- a/build/common/verify/cli.py +++ b/build/common/verify/cli.py @@ -3,9 +3,23 @@ from __future__ import annotations import argparse +import sys from pathlib import Path -from common.verify import GUI_PRODUCTS, verify_built_wheel, verify_gui_dist +_BUILD_ROOT = Path(__file__).resolve().parents[2] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import REPO_ROOT # noqa: E402 +from common.verify import ( # noqa: E402 + GUI_PRODUCTS, + inspect_wheel, + verify_built_wheel, + verify_gui_dist, + verify_wheel_assets, + verify_wheel_runtime, +) +from common.verify.wheel import DEFAULT_RUNTIME_CONFIG, default_verification_video # noqa: E402 def main(argv: list[str] | None = None) -> int: @@ -18,6 +32,31 @@ def main(argv: list[str] | None = None) -> int: help="GUI product profile (default: ory_finger)", ) parser.add_argument("--dist-dir", type=Path, default=None, help="Override GUI dist folder") + parser.add_argument( + "--contents", + action="store_true", + help="Run static metadata/content assertions on the wheel", + ) + parser.add_argument( + "--assets", + action="store_true", + help="Install the wheel in a temp venv and assert bundled assets load", + ) + parser.add_argument( + "--runtime", + action="store_true", + help="Install the wheel and process real frames (no camera required)", + ) + parser.add_argument("--expected-version", default=None, help="Assert the wheel version") + parser.add_argument( + "--strict-bundle", + action="store_true", + help="Treat bundled-file drift from the source tree as a failure", + ) + parser.add_argument("--video", type=Path, default=None, help="Clip for --runtime") + parser.add_argument("--config", default=DEFAULT_RUNTIME_CONFIG, help="Config for --runtime") + parser.add_argument("--frames", type=int, default=40, help="Frames for --runtime") + parser.add_argument("--offline", action="store_true", help="Reuse parent-env dependencies") parser.add_argument("wheel", nargs="?", type=Path, help="Wheel path for import smoke test") args = parser.parse_args(argv) @@ -26,6 +65,49 @@ def main(argv: list[str] | None = None) -> int: return 0 if args.wheel is None: parser.error("Provide a wheel path or pass --gui-dist") + + exit_code = 0 + + if args.contents: + result = inspect_wheel( + args.wheel, + expected_version=args.expected_version, + repo_root=REPO_ROOT, + strict_bundle=args.strict_bundle, + ) + print(result.render()) + if not result.ok: + exit_code = 1 + + if args.assets or args.runtime: + try: + print(verify_wheel_assets(args.wheel)) + except Exception as exc: + print(f"[FAIL] bundled assets: {type(exc).__name__}: {exc}") + exit_code = 1 + + if args.runtime: + video = args.video or default_verification_video(REPO_ROOT) + if video is None: + print("[FAIL] runtime: no clip found; pass --video or set ORISYS_VERIFY_VIDEO") + return 1 + try: + info = verify_wheel_runtime( + args.wheel, + video=video, + config_name=args.config, + frames=args.frames, + offline=args.offline, + ) + print(f"[ok ] runtime: {info}") + except Exception as exc: + print(f"[FAIL] runtime: {type(exc).__name__}: {exc}") + exit_code = 1 + + if args.contents or args.assets or args.runtime: + return exit_code + + # Default behaviour is unchanged: import smoke test in an isolated venv. verify_built_wheel(args.wheel) return 0 diff --git a/build/common/verify/wheel.py b/build/common/verify/wheel.py index a8742f9..afeaef5 100644 --- a/build/common/verify/wheel.py +++ b/build/common/verify/wheel.py @@ -1,7 +1,21 @@ -"""Isolated venv smoke tests for packaged orisys wheels.""" +"""Isolated venv smoke tests for packaged orisys wheels. + +Three levels of confidence, all in a throwaway virtualenv so the active +environment is never touched: + +* :func:`verify_built_wheel` - the public API imports (fast, original check); +* :func:`verify_wheel_assets` - bundled configs/sensors resolve and load via + ``importlib.resources``; +* :func:`verify_wheel_runtime` - the compiled wheel actually processes real + frames through the public pipeline with no camera attached. + +Every level can be bootstrapped from another interpreter so each supported +CPython ABI can be verified against its own wheel. +""" from __future__ import annotations +import os import subprocess import sys import tempfile @@ -38,6 +52,167 @@ assert Stitcher is not None print(f"orisys {orisys.__version__} public API imports OK") """ +# Bundled data that must be resolvable and loadable from inside site-packages. +# The wheel ships the compiled package, so these paths only exist if the bundle +# step actually ran for this build. +_ASSET_SCRIPT = r""" +import sys +from importlib.metadata import distribution +from pathlib import Path + +import numpy as np +import orisys + +# Locate bundled data through the installed distribution, NOT through +# orisys.__file__ / importlib.resources. Nuitka compiles the package into one +# extension module whose __init__ installs nuitka_module_loader, which reports a +# synthetic __file__ (e.g. ...\site-packages\orisys\orisys\__init__.py) and a +# synthetic __path__ ending in '\not_existing'. Neither points at the real data +# directory, so importlib.resources silently finds nothing in a Nuitka wheel. +dist = distribution("orisys") +dist_root = Path(dist.locate_file("")).resolve() +package_root = Path(dist.locate_file("orisys")).resolve() + +_diagnosis = ( + f"orisys distribution resolved to {dist_root} (sys.prefix={sys.prefix!r}, " + f"sys.executable={sys.executable!r}), outside the verification venv. A stale " + "orisys in the parent environment is visible through --system-site-packages; " + "run `pip uninstall -y orisys` there, or re-run without --offline." +) +assert str(Path(sys.prefix).resolve()) in str(dist_root), _diagnosis +assert package_root.is_dir(), f"package directory not found: {package_root}" + +required = [ + "configs/box.json", + "configs/finger.json", + "sensors/box/force_calibration.json", + "sensors/box/sensor.json", + "sensors/finger/manifest.json", + "sensors/finger/sensor.json", + "sensors/mini/sensor.json", + "sensors/finger/assets/30fps-0703/mesh.npz", + "sensors/finger/assets/30fps-0703/camera_intrinsics.npz", + "sensors/finger/assets/30fps-0703/world_to_cam.npy", + "sensors/finger/assets/visualization/mesh.npz", + "sensors/finger/assets/visualization/world_to_cam.npy", +] + +missing = [name for name in required if not (package_root / name).is_file()] +assert not missing, ( + f"bundled assets missing from the wheel: {missing}\n" + f"package root: {package_root}\ninstalled tree: " + f"{sorted(p.name for p in package_root.iterdir()) if package_root.is_dir() else ''}" +) + +for name in required: + path = package_root / name + if name.endswith(".npz"): + data = np.load(str(path)) + assert len(data.files) > 0, f"{name} is an empty npz" + elif name.endswith(".npy"): + arr = np.load(str(path)) + assert arr.size > 0, f"{name} is an empty npy" + +configs = orisys.list_config() +assert configs, "orisys.list_config() returned nothing" + +print(f"bundled assets OK ({len(required)} files, {len(configs)} configs) " + f"from {package_root}") +""" + +# End-to-end pipeline run against a video file: no camera, no GUI, no imshow. +_RUNTIME_SCRIPT = r""" +import json +import os +import sys +from importlib.metadata import distribution +from pathlib import Path + +import numpy as np + +os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0") + +import orisys + +# Provenance via the installed distribution: a Nuitka-compiled package reports a +# synthetic orisys.__file__ (see the asset check) and a stale orisys visible +# through --system-site-packages would otherwise satisfy this check. +dist = distribution("orisys") +dist_root = Path(dist.locate_file("")).resolve() +assert str(Path(sys.prefix).resolve()) in str(dist_root), ( + f"the installed orisys came from {dist_root}, outside the verification venv " + f"{sys.prefix!r}; run `pip uninstall -y orisys` in the parent environment or " + "re-run without --offline." +) + +video, config_name, frames = sys.argv[1], sys.argv[2], int(sys.argv[3]) + +sensor = orisys.Sensor( + vid_src=video, + config_name=config_name, + backend="cpu", + verbose=False, +) +processed = 0 +try: + for _ in range(frames): + img = sensor.get_img() + assert img is not None, "get_img returned None" + assert isinstance(img, np.ndarray) and img.ndim in (2, 3) + + sensor.compute_deformation(check_motion=False) + flow, vnormal, vshear = sensor.read_info( + sensor.info.VRAW, sensor.info.VNORMAL, sensor.info.VSHEAR + ) + for name, array in (("flow", flow), ("vnormal", vnormal), ("vshear", vshear)): + assert np.isfinite(array).all(), f"{name} contains NaN/Inf" + + contact = sensor.compute_contact() + assert isinstance(contact, dict) and "centroid_found" in contact + + fnormal, fsx, fsy = sensor.read_info( + sensor.info.FNORMAL, sensor.info.FSHEARX, sensor.info.FSHEARY + ) + assert np.isfinite([fnormal, fsx, fsy]).all(), "force output contains NaN/Inf" + processed += 1 +finally: + sensor.disconnect() + +assert processed == frames, f"only processed {processed}/{frames} frames" +print(json.dumps({ + "version": orisys.__version__, + "frames": processed, + "flow_shape": list(flow.shape), + "video": video, + "config": config_name, + "package_root": str(Path(dist.locate_file("orisys")).resolve()), +})) +""" + + +# Default runtime-verification inputs. Overridable by CLI flags or environment. +DEFAULT_RUNTIME_CONFIG = os.getenv("ORISYS_VERIFY_CONFIG", "box") +# Clips tracked in the repository, smallest first. +DEFAULT_VIDEO_CANDIDATES = ( + Path("data") / "sample" / "s1_480p.mp4", + Path("data") / "sample" / "s1_720p.mp4", + Path("tests") / "idle_1min.mp4", +) + + +def default_verification_video(repo_root: Path) -> Path | None: + """Resolve the clip used for the end-to-end runtime check, if any.""" + env_video = os.getenv("ORISYS_VERIFY_VIDEO") + repo_root = Path(repo_root) + if env_video: + candidate = Path(env_video) + return candidate if candidate.is_absolute() else repo_root / candidate + for relative in DEFAULT_VIDEO_CANDIDATES: + candidate = repo_root / relative + if candidate.is_file(): + return candidate + return None + def _venv_python(venv_dir: Path) -> Path: if sys.platform == "win32": @@ -45,18 +220,172 @@ def _venv_python(venv_dir: Path) -> Path: return venv_dir / "bin" / "python" -def verify_built_wheel(wheel_path: Path) -> None: - """Install the wheel in a temporary venv and smoke-test public imports.""" - wheel_path = wheel_path.resolve() - with tempfile.TemporaryDirectory(prefix="orisys-wheel-verify-") as tmp: - venv_dir = Path(tmp) / "venv" - venv.create(venv_dir, with_pip=True) - python = _venv_python(venv_dir) +def _hermetic_env() -> dict: + """Environment for the verification venv with PYTHONPATH removed. - subprocess.check_call( - [python, "-m", "pip", "install", "-q", "--upgrade", "pip"], + An inherited PYTHONPATH=src would let the checkout satisfy the checks + instead of the installed wheel, which is precisely the failure mode the + provenance assertions in the scripts exist to catch. + """ + return {key: value for key, value in os.environ.items() if key != "PYTHONPATH"} + + +def _create_venv( + venv_dir: Path, + *, + python_executable: Path | None = None, + system_site_packages: bool = False, +) -> Path: + """Create a throwaway venv, optionally from a different interpreter.""" + if python_executable is None: + venv.create(venv_dir, with_pip=True, system_site_packages=system_site_packages) + else: + command = [str(python_executable), "-m", "venv"] + if system_site_packages: + command.append("--system-site-packages") + command.append(str(venv_dir)) + subprocess.check_call(command, env=_hermetic_env()) + return _venv_python(venv_dir) + + +def _install_wheel( + python: Path, wheel_path: Path, *, offline: bool = False, env: dict | None = None +) -> None: + """Install the wheel under test, guaranteeing it lands in the venv. + + ``--force-reinstall --no-deps`` is not optional: pip reports a same-version + requirement as "already satisfied" and skips the file, which is fatal here + because ``--offline`` also exposes the parent environment's site-packages. + Without it the venv check can silently validate a *different* orisys than + the one passed in (the scripts additionally assert provenance). + """ + if offline: + # Dependencies come from the parent environment via --system-site-packages. + commands = [ + [ + str(python), + "-m", + "pip", + "install", + "-q", + "--no-index", + "--no-deps", + "--force-reinstall", + str(wheel_path), + ] + ] + else: + commands = [ + [str(python), "-m", "pip", "install", "-q", "--upgrade", "pip"], + [str(python), "-m", "pip", "install", "-q", str(wheel_path)], + [ + str(python), + "-m", + "pip", + "install", + "-q", + "--force-reinstall", + "--no-deps", + str(wheel_path), + ], + ] + + for command in commands: + subprocess.check_call(command, stdout=subprocess.DEVNULL, env=env) + + +class WheelVerificationError(RuntimeError): + """A verification step failed inside the throwaway venv. + + Carries the child's stdout/stderr so the real traceback is visible instead of + a bare non-zero exit code. + """ + + +def _run_in_fresh_venv( + wheel_path: Path, + script: str, + args: list[str] | None = None, + *, + python_executable: Path | None = None, + offline: bool = False, + prefix: str = "orisys-wheel-verify-", +) -> str: + wheel_path = wheel_path.resolve() + env = _hermetic_env() + with tempfile.TemporaryDirectory(prefix=prefix) as tmp: + venv_dir = Path(tmp) / "venv" + python = _create_venv( + venv_dir, + python_executable=python_executable, + system_site_packages=offline, ) - subprocess.check_call( - [python, "-m", "pip", "install", "-q", str(wheel_path)], + _install_wheel(python, wheel_path, offline=offline, env=env) + completed = subprocess.run( + [str(python), "-c", script, *(args or [])], + capture_output=True, + text=True, + env=env, + check=False, ) - subprocess.check_call([python, "-c", _SMOKE_IMPORT_SCRIPT]) + if completed.returncode != 0: + command = " ".join(str(part) for part in (args or [])) + raise WheelVerificationError( + f"verification script failed in the venv (exit {completed.returncode}" + + (f", args: {command}" if command else "") + + ").\n--- stdout ---\n" + + (completed.stdout.strip() or "") + + "\n--- stderr ---\n" + + (completed.stderr.strip() or "") + ) + return completed.stdout.strip() + + +def verify_built_wheel(wheel_path: Path, *, python_executable: Path | None = None) -> None: + """Install the wheel in a temporary venv and smoke-test public imports.""" + _run_in_fresh_venv( + wheel_path, _SMOKE_IMPORT_SCRIPT, python_executable=python_executable + ) + + +def verify_wheel_assets(wheel_path: Path, *, python_executable: Path | None = None) -> str: + """Assert the wheel carries loadable bundled configs and sensor assets.""" + return _run_in_fresh_venv( + wheel_path, + _ASSET_SCRIPT, + python_executable=python_executable, + prefix="orisys-wheel-assets-", + ) + + +def verify_wheel_runtime( + wheel_path: Path, + *, + video: Path, + config_name: str, + frames: int = 40, + python_executable: Path | None = None, + offline: bool = False, +) -> dict: + """Run real frames through the installed wheel with no camera attached. + + This is the only check that proves the compiled extension plus the bundled + sensor assets actually produce finite flow/force output end to end. + """ + video = Path(video).resolve() + if not video.is_file(): + raise FileNotFoundError(f"verification video not found: {video}") + if frames < 1: + raise ValueError("frames must be >= 1") + + output = _run_in_fresh_venv( + wheel_path, + _RUNTIME_SCRIPT, + [str(video), config_name, str(frames)], + python_executable=python_executable, + offline=offline, + prefix="orisys-wheel-runtime-", + ) + import json + + return json.loads(output.splitlines()[-1]) diff --git a/build/common/verify/wheel_contents.py b/build/common/verify/wheel_contents.py new file mode 100644 index 0000000..52baa2a --- /dev/null +++ b/build/common/verify/wheel_contents.py @@ -0,0 +1,501 @@ +"""Static content and metadata validation for built orisys wheels. + +Everything here is pure standard library: the wheel is opened as a zip and +inspected in place. No installation, no network, no hardware. + +The checks encode the guarantees a Nuitka-compiled release must satisfy: + +* the filename, ``METADATA`` and ``RECORD`` agree on name/version/tags; +* PyPI-visible metadata (``Requires-Python``, classifiers, project URLs, + license) is present instead of silently dropped by the hand-written + ``setup.py`` template in :mod:`wheel.package`; +* no Python sources ship inside ``orisys/`` (IP containment) while the compiled + extension is present; +* every file declared by ``[[tool.orisys.bundle.items]]`` is present and + byte-identical to the source tree; +* no caches, build leftovers or scratch directories leak in. +""" + +from __future__ import annotations + +import base64 +import hashlib +import re +import sys +import zipfile +from dataclasses import dataclass, field +from email.parser import Parser +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[1] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import REPO_ROOT # noqa: E402 +from common.pyproject_utils import bundle_items, extras_require, install_requires, project_metadata # noqa: E402 +from wheel.layout import resolve_bundle_source_files # noqa: E402 +from wheel.package import is_compliant_manylinux_wheel # noqa: E402 + +# PyPI's default per-project file limit is 100 MB; keep headroom. +MAX_WHEEL_BYTES = 90 * 1024 * 1024 + +_FILENAME_RE = re.compile( + r"^(?P[^-]+)-(?P[^-]+)-(?P[^-]+)-(?P[^-]+)-(?P.+)\.whl$" +) +_COMPILED_EXT_RE = re.compile(r"^orisys/__init__\..+\.(?:pyd|so)$") +_FORBIDDEN_PREFIXES = ( + "test/", + "tests/", + "data/", + "build/", + "dist/", + "sdk/", + ".git", +) +_FORBIDDEN_SUFFIXES = (".pyc", ".pyo", ".c", ".pdb", ".map", ".spec") + + +@dataclass(frozen=True) +class Check: + name: str + ok: bool + detail: str = "" + + def render(self) -> str: + return f"[{'ok' if self.ok else 'FAIL'}] {self.name}: {self.detail}" + + +@dataclass +class WheelCheckResult: + wheel: Path + version: str | None = None + name: str | None = None + platform_tag: str | None = None + entry_count: int = 0 + uncompressed_bytes: int = 0 + file_bytes: int = 0 + checks: list[Check] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + @property + def ok(self) -> bool: + return all(check.ok for check in self.checks) + + @property + def failures(self) -> list[Check]: + return [check for check in self.checks if not check.ok] + + def add(self, name: str, ok: bool, detail: str = "") -> bool: + self.checks.append(Check(name=name, ok=ok, detail=detail)) + return ok + + def warn(self, message: str) -> None: + self.warnings.append(message) + + def render(self) -> str: + lines = [ + f"wheel: {self.wheel.name}", + f" version={self.version} platform={self.platform_tag} " + f"entries={self.entry_count} file_size={self.file_bytes} " + f"uncompressed={self.uncompressed_bytes}", + ] + lines.extend(f" {check.render()}" for check in self.checks) + lines.extend(f" [warn] {message}" for message in self.warnings) + lines.append(f" RESULT: {'PASS' if self.ok else 'FAIL'}") + return "\n".join(lines) + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _requirement_name(spec: str) -> str: + return re.split(r"[<>=!~\[; ]", spec, maxsplit=1)[0].strip().lower().replace("_", "-") + + +def _metadata_message(zf: zipfile.ZipFile, result: WheelCheckResult): + candidates = [name for name in zf.namelist() if name.endswith(".dist-info/METADATA")] + if not candidates: + result.add("metadata-present", False, "no *.dist-info/METADATA entry in the wheel") + return None + raw = zf.read(candidates[0]).decode("utf-8", errors="replace") + message = Parser().parsestr(raw) + result.add("metadata-present", True, candidates[0]) + return message + + +def _check_filename(result: WheelCheckResult, expected_version: str | None) -> None: + match = _FILENAME_RE.match(result.wheel.name) + if match is None: + result.add("filename", False, f"unparsable wheel filename: {result.wheel.name}") + return + result.name = match.group("name") + result.version = match.group("version") + result.platform_tag = match.group("platform") + + problems = [] + if result.name != "orisys": + problems.append(f"distribution name is {result.name!r}, expected 'orisys'") + if expected_version is not None and result.version != expected_version: + problems.append(f"filename version {result.version!r} != expected {expected_version!r}") + result.add("filename", not problems, "; ".join(problems) or result.wheel.name) + + +def _check_metadata( + result: WheelCheckResult, + message, + expected_version: str | None, + repo_root: Path | None, +) -> None: + if message is None: + return + + result.add( + "metadata-name", + message.get("Name") == "orisys", + f"Name={message.get('Name')!r}", + ) + + declared = message.get("Version") + expected = expected_version or result.version + if expected is not None and declared != expected: + result.add( + "metadata-version", + False, + f"METADATA Version={declared!r} != expected {expected!r}", + ) + else: + result.add("metadata-version", True, f"Version={declared!r}") + + requires_python = (message.get("Requires-Python") or "").strip() + result.add( + "metadata-requires-python", + bool(requires_python), + f"Requires-Python={requires_python!r}" if requires_python else "Requires-Python is missing", + ) + + classifiers = message.get_all("Classifier") or [] + result.add( + "metadata-classifiers", + bool(classifiers), + f"{len(classifiers)} classifier(s)" if classifiers else "no Classifier fields", + ) + + project_urls = message.get_all("Project-URL") or [] + result.add( + "metadata-project-urls", + bool(project_urls), + f"{len(project_urls)} Project-URL(s)" if project_urls else "no Project-URL fields", + ) + + license_values = [ + (message.get("License") or "").strip(), + (message.get("License-Expression") or "").strip(), + ] + has_license = any(license_values) or any( + value.startswith("License ::") for value in classifiers + ) + result.add( + "metadata-license", + has_license, + f"License={license_values[0]!r} License-Expression={license_values[1]!r}" + if has_license + else "no License / License-Expression / 'License ::' classifier", + ) + + dynamic = message.get_all("Dynamic") or [] + result.add( + "metadata-no-dynamic-version", + "version" not in dynamic, + f"Dynamic={dynamic!r}" if dynamic else "no Dynamic fields", + ) + if dynamic: + result.warn(f"METADATA still declares Dynamic fields: {dynamic!r}") + + if repo_root is not None and (repo_root / "pyproject.toml").is_file(): + requires_dist = message.get_all("Requires-Dist") or [] + provided = {_requirement_name(entry) for entry in requires_dist} + missing_core = [ + spec for spec in install_requires(repo_root) if _requirement_name(spec) not in provided + ] + result.add( + "metadata-requires-dist", + not missing_core, + f"missing core Requires-Dist entries: {missing_core!r}" + if missing_core + else f"{len(requires_dist)} Requires-Dist entries", + ) + + extras = message.get_all("Provides-Extra") or [] + missing_extras = sorted(set(extras_require(repo_root)) - set(extras)) + result.add( + "metadata-extras", + not missing_extras, + f"missing Provides-Extra: {missing_extras!r}" + if missing_extras + else f"Provides-Extra={sorted(extras)!r}", + ) + + +def _check_compiled_only(result: WheelCheckResult, names: list[str]) -> None: + compiled = [name for name in names if _COMPILED_EXT_RE.match(name)] + result.add( + "compiled-extension", + bool(compiled), + f"{compiled[0]}" if compiled else "no orisys/__init__..pyd|.so entry", + ) + + # orisys/examples/** is shipped as package data on purpose; every other + # orisys/**/*.py means source code leaked past the Nuitka compile step. + sources = [ + name + for name in names + if name.startswith("orisys/") + and name.endswith(".py") + and not name.startswith("orisys/examples/") + ] + result.add( + "no-python-sources", + not sources, + f"python sources leaked into the wheel: {sources[:5]!r}" + if sources + else "compiled package ships no python sources (examples excluded by design)", + ) + + blend = [name for name in names if name.startswith("orisys_blend/")] + result.add( + "orisys-blend-present", + "orisys_blend/__init__.py" in blend and "orisys_blend/kernel.py" in blend, + f"{sorted(blend)!r}", + ) + + +def _check_bundle_fidelity( + result: WheelCheckResult, + zf: zipfile.ZipFile, + names: set[str], + repo_root: Path | None, + strict: bool, +) -> None: + if repo_root is None or not (repo_root / "pyproject.toml").is_file(): + result.warn("pyproject.toml not found; bundle fidelity checks skipped") + return + + missing: list[str] = [] + mismatched: list[str] = [] + checked = 0 + + for item in bundle_items(repo_root): + target = str(item["target"]).replace("\\", "/") + source_dir = repo_root / item["source"] + for source_file in resolve_bundle_source_files(repo_root, item): + if item.get("preserve_tree", False): + relative = source_file.relative_to(source_dir).as_posix() + entry = f"orisys/{target}/{relative}" + else: + entry = f"orisys/{target}/{source_file.name}" + + if entry not in names: + missing.append(entry) + continue + checked += 1 + if _sha256(zf.read(entry)) != _sha256(source_file.read_bytes()): + mismatched.append(entry) + + # Both entries here mean "the wheel does not match this revision of the + # source tree". Against a fresh build that is fatal; against an already + # released wheel it is expected, so ``strict`` (set by preflight, unset by + # audit) decides whether it fails or warns. + if missing: + detail = ( + f"{len(missing)} declared file(s) absent from the wheel (added to the " + f"source tree after this build?): {missing[:5]!r}" + ) + result.add("bundle-completeness", not strict, detail) + if not strict: + result.warn(detail) + else: + result.add( + "bundle-completeness", True, f"{checked} declared bundled file(s) present" + ) + + if mismatched: + detail = ( + f"{len(mismatched)} of {checked} bundled file(s) differ from the source " + f"tree (wheel predates or postdates it): {mismatched[:5]!r}" + ) + result.add("bundle-fidelity", not strict, detail) + if not strict: + result.warn(detail) + else: + result.add( + "bundle-fidelity", + True, + f"{checked} bundled file(s) byte-identical to source", + ) + + +def _check_forbidden_entries(result: WheelCheckResult, names: list[str]) -> None: + offenders = [] + for name in names: + lowered = name.lower() + if lowered.startswith(_FORBIDDEN_PREFIXES): + offenders.append(name) + elif lowered.endswith(_FORBIDDEN_SUFFIXES): + offenders.append(name) + elif "__pycache__" in lowered or ".egg-info" in lowered: + offenders.append(name) + result.add( + "no-forbidden-entries", + not offenders, + f"unexpected entries: {offenders[:5]!r}" if offenders else "no caches/leftovers/scratch paths", + ) + + +def _check_archive_paths(result: WheelCheckResult, names: list[str]) -> None: + unsafe = [ + name + for name in names + if name.startswith(("/", "\\")) or ".." in Path(name.replace("\\", "/")).parts + ] + result.add( + "archive-paths", + not unsafe, + f"absolute or traversing entries: {unsafe[:5]!r}" if unsafe else "all entries are relative", + ) + + +def _check_record(result: WheelCheckResult, zf: zipfile.ZipFile, names: list[str]) -> None: + records = [name for name in names if name.endswith(".dist-info/RECORD")] + if not records: + result.add("record", False, "no *.dist-info/RECORD entry") + return + + bad_paths: list[str] = [] + bad_hashes: list[str] = [] + bad_sizes: list[str] = [] + present = set(names) + + for line in zf.read(records[0]).decode("utf-8").splitlines(): + if not line.strip(): + continue + parts = line.split(",") + while len(parts) < 3: + parts.append("") + path, hash_field, size_field = parts[0], parts[1], parts[2] + + if path not in present: + bad_paths.append(path) + continue + if not hash_field.startswith("sha256="): + continue + + expected = hash_field[len("sha256=") :] + actual = base64.urlsafe_b64encode( + hashlib.sha256(zf.read(path)).digest() + ).rstrip(b"=").decode("ascii") + if actual != expected: + bad_hashes.append(path) + if size_field and size_field.isdigit(): + if int(size_field) != len(zf.read(path)): + bad_sizes.append(path) + + problems = [] + if bad_paths: + problems.append(f"paths not in archive: {bad_paths[:5]!r}") + if bad_hashes: + problems.append(f"sha256 mismatch: {bad_hashes[:5]!r}") + if bad_sizes: + problems.append(f"size mismatch: {bad_sizes[:5]!r}") + result.add( + "record", + not problems, + "; ".join(problems) if problems else f"{records[0]} verifies", + ) + + +def _check_platform_tag(result: WheelCheckResult) -> None: + tag = result.platform_tag + if tag is None: + return + if "win" in tag: + ok = tag == "win_amd64" + result.add("platform-tag", ok, f"{tag!r} (expected 'win_amd64')") + return + ok = is_compliant_manylinux_wheel(result.wheel) + result.add( + "platform-tag", + ok, + f"{tag!r}" if ok else f"{tag!r} is not a PyPI-accepted manylinux x86_64 tag", + ) + + +def _check_size(result: WheelCheckResult, max_bytes: int) -> None: + result.add( + "size-limit", + result.file_bytes <= max_bytes, + f"{result.file_bytes} bytes (limit {max_bytes})", + ) + + +def inspect_wheel( + wheel_path: Path, + *, + expected_version: str | None = None, + repo_root: Path | None = None, + max_bytes: int = MAX_WHEEL_BYTES, + strict_bundle: bool = True, +) -> WheelCheckResult: + """Validate one built wheel without installing it. + + ``strict_bundle`` controls how the wheel's bundled files are compared with the + current source tree: ``True`` (gate a fresh build, used by + ``release preflight``) fails on any missing or changed declared file; + ``False`` (audit an already-released wheel) reports the same findings as + warnings. + """ + wheel_path = Path(wheel_path).resolve() + if repo_root is None: + repo_root = REPO_ROOT if (REPO_ROOT / "pyproject.toml").is_file() else None + else: + repo_root = Path(repo_root).resolve() + + result = WheelCheckResult(wheel=wheel_path) + if not wheel_path.is_file(): + result.add("wheel-exists", False, f"not a file: {wheel_path}") + return result + + result.file_bytes = wheel_path.stat().st_size + result.add("wheel-exists", True, str(wheel_path)) + _check_filename(result, expected_version) + _check_platform_tag(result) + _check_size(result, max_bytes) + + try: + zf = zipfile.ZipFile(wheel_path) + except zipfile.BadZipFile as exc: + result.add("zip-integrity", False, f"not a valid zip: {exc}") + return result + + with zf: + corrupt = zf.testzip() + result.add( + "zip-integrity", + corrupt is None, + "CRC check passed" if corrupt is None else f"corrupt entry: {corrupt}", + ) + + names = zf.namelist() + name_set = set(names) + result.entry_count = len(names) + result.uncompressed_bytes = sum(info.file_size for info in zf.infolist()) + + _check_archive_paths(result, names) + _check_forbidden_entries(result, names) + message = _metadata_message(zf, result) + _check_metadata(result, message, expected_version, repo_root) + _check_compiled_only(result, names) + _check_bundle_fidelity(result, zf, name_set, repo_root, strict_bundle) + _check_record(result, zf, names) + + return result diff --git a/build/release/__init__.py b/build/release/__init__.py new file mode 100644 index 0000000..ffa87ed --- /dev/null +++ b/build/release/__init__.py @@ -0,0 +1,7 @@ +"""Local release tooling: preflight gate and per-wheel audit. + +Entry point is ``python build/run.py release ``; see +:mod:`release.preflight` for the gate steps and :mod:`release.cli` for the CLI. +Imports are intentionally not re-exported here so importing a submodule never +triggers the whole subsystem. +""" diff --git a/build/release/cli.py b/build/release/cli.py new file mode 100644 index 0000000..c511521 --- /dev/null +++ b/build/release/cli.py @@ -0,0 +1,163 @@ +"""Command line for ``python build/run.py release``.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[1] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import REPO_ROOT # noqa: E402 +from common.verify.wheel import ( # noqa: E402 + DEFAULT_RUNTIME_CONFIG, + default_verification_video, + verify_wheel_assets, + verify_wheel_runtime, +) +from common.verify.wheel_contents import inspect_wheel # noqa: E402 +from release.preflight import PreflightOptions, run_preflight # noqa: E402 + + +def _cmd_preflight(argv: list[str]) -> int: + parser = argparse.ArgumentParser( + prog="build/run.py release preflight", + description="Gate a release locally: version, tests, wheel content, runtime.", + ) + parser.add_argument("--version", help="release version (default: pyproject's dynamic version)") + parser.add_argument("--wheel-dir", type=Path, help="directory to inspect (default: SDK/)") + parser.add_argument("--video", type=Path, help="clip for the end-to-end runtime check") + parser.add_argument("--config", default=DEFAULT_RUNTIME_CONFIG, help="config name for that check") + parser.add_argument("--frames", type=int, default=40, help="frames to process (default: 40)") + parser.add_argument("--skip-tests", action="store_true", help="skip pytest (recorded in the report)") + parser.add_argument("--longrun", action="store_true", help="include the slow long-run stability test") + parser.add_argument("--allow-dirty", action="store_true", help="allow a dirty working tree") + parser.add_argument("--force", action="store_true", help="allow re-staging an existing SDK/") + parser.add_argument("--check-pypi", action="store_true", help="read PyPI and fail if the version exists") + parser.add_argument( + "--all-wheel-pythons", + action="store_true", + help="verify cp310/cp311/cp312 wheels, each in its own conda env", + ) + parser.add_argument("--offline", action="store_true", help="reuse parent-env deps instead of PyPI") + parser.add_argument("--no-runtime", action="store_true", help="skip the isolated runtime check") + parser.add_argument("--no-assets", action="store_true", help="skip the bundled-asset check") + parser.add_argument("--keep-going", action="store_true", help="collect every failure instead of stopping") + parser.add_argument("--report-dir", type=Path, help="override build/release/reports") + args = parser.parse_args(argv) + + options = PreflightOptions( + version=args.version, + wheel_dir=args.wheel_dir, + video=args.video, + config_name=args.config, + frames=args.frames, + skip_tests=args.skip_tests, + longrun=args.longrun, + allow_dirty=args.allow_dirty, + force=args.force, + check_pypi=args.check_pypi, + all_wheel_pythons=args.all_wheel_pythons, + offline=args.offline, + keep_going=args.keep_going, + report_dir=args.report_dir, + no_runtime=args.no_runtime, + no_assets=args.no_assets, + ) + return run_preflight(options, repo_root=REPO_ROOT) + + +def _cmd_audit(argv: list[str]) -> int: + parser = argparse.ArgumentParser( + prog="build/run.py release audit", + description="Inspect one built wheel without a release context.", + ) + parser.add_argument("wheel", type=Path, help="path to an .whl") + parser.add_argument("--expected-version", help="assert the wheel reports this version") + parser.add_argument( + "--strict-bundle", + action="store_true", + help="treat bundled-file drift from the current source tree as a failure", + ) + parser.add_argument("--runtime", action="store_true", help="also install and run the wheel") + parser.add_argument("--assets", action="store_true", help="also assert bundled assets load") + parser.add_argument("--video", type=Path, help="clip for --runtime") + parser.add_argument("--config", default=DEFAULT_RUNTIME_CONFIG) + parser.add_argument("--frames", type=int, default=40) + parser.add_argument("--offline", action="store_true") + parser.add_argument( + "--python", + type=Path, + default=None, + help="interpreter to build the verification venv from (default: the running one)", + ) + args = parser.parse_args(argv) + + wheel = args.wheel + if not wheel.is_file(): + print(f"[ERROR] wheel not found: {wheel}", file=sys.stderr) + return 2 + + exit_code = 0 + + result = inspect_wheel( + wheel, + expected_version=args.expected_version, + repo_root=REPO_ROOT, + strict_bundle=args.strict_bundle, + ) + print(result.render()) + if not result.ok: + exit_code = 1 + + if args.assets or args.runtime: + try: + print(verify_wheel_assets(wheel, python_executable=args.python)) + except Exception as exc: + print(f"[FAIL] bundled assets: {type(exc).__name__}: {exc}") + exit_code = 1 + + if args.runtime: + video = args.video or default_verification_video(REPO_ROOT) + if video is None: + print("[FAIL] runtime: no verification clip; pass --video or set ORISYS_VERIFY_VIDEO") + return 1 + try: + info = verify_wheel_runtime( + wheel, + video=video, + config_name=args.config, + frames=args.frames, + python_executable=args.python, + offline=args.offline, + ) + print(f"[ok ] runtime: {info}") + except Exception as exc: + print(f"[FAIL] runtime: {type(exc).__name__}: {exc}") + exit_code = 1 + + return exit_code + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if not argv or argv[0] in ("-h", "--help"): + print(__doc__) + print("Commands: preflight | audit") + return 0 + + command, rest = argv[0], argv[1:] + if command == "preflight": + return _cmd_preflight(rest) + if command == "audit": + return _cmd_audit(rest) + + print(f"Unknown release command: {command!r}", file=sys.stderr) + print("Choose from: preflight, audit", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/build/release/preflight.py b/build/release/preflight.py new file mode 100644 index 0000000..5917153 --- /dev/null +++ b/build/release/preflight.py @@ -0,0 +1,515 @@ +"""Local release gate: everything that must hold before publishing a wheel. + +``release preflight`` answers one question with an exit code: is this checkout, +this version and these wheels safe to publish? It never publishes anything, and +by default it makes no network calls. + +Steps, in order: + +1. ``repo-state`` - git repository, clean working tree, recorded commit +2. ``version-consistency`` - __version__ == pyproject dynamic attr == wheel names +3. ``duplicate-guard`` - SDK// already populated; optionally PyPI +4. ``tests`` - pytest (slow/hardware markers excluded) +5. ``bundle-precheck`` - every bundle item resolves; required assets present +6. ``wheel-inspection`` - static metadata/content assertions per wheel +7. ``wheel-runtime`` - isolated venv: bundled assets load and real frames + flow through the public pipeline + +Output lands in ``build/release/reports//preflight.{json,txt}``. +""" + +from __future__ import annotations + +import hashlib +import json +import platform +import subprocess +import sys +import time +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +_BUILD_ROOT = Path(__file__).resolve().parents[1] +if str(_BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(_BUILD_ROOT)) + +from common.paths import REPO_ROOT # noqa: E402 +from common.pyproject_utils import bundle_items, get_version # noqa: E402 +from common.verify.wheel import ( # noqa: E402 + DEFAULT_RUNTIME_CONFIG, + default_verification_video, + verify_wheel_assets, + verify_wheel_runtime, +) +from common.verify.wheel_contents import inspect_wheel # noqa: E402 +from release import report as report_mod # noqa: E402 +from release import version as version_mod # noqa: E402 +from wheel.layout import resolve_bundle_source_files, verify_required_assets # noqa: E402 + + +@dataclass +class Step: + name: str + ok: bool + detail: str = "" + duration_s: float = 0.0 + data: dict = field(default_factory=dict) + + +@dataclass +class PreflightOptions: + version: str | None = None + wheel_dir: Path | None = None + video: Path | None = None + config_name: str = DEFAULT_RUNTIME_CONFIG + frames: int = 40 + skip_tests: bool = False + longrun: bool = False + allow_dirty: bool = False + force: bool = False + check_pypi: bool = False + all_wheel_pythons: bool = False + offline: bool = False + keep_going: bool = False + report_dir: Path | None = None + no_runtime: bool = False + no_assets: bool = False + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +class Preflight: + def __init__(self, options: PreflightOptions, repo_root: Path | None = None) -> None: + self.options = options + self.repo_root = Path(repo_root or REPO_ROOT) + self.steps: list[Step] = [] + self.git: dict = {"commit": None, "branch": None, "dirty": None} + self.wheels: list[Path] = [] + self.wheel_dir: Path | None = None + self.runtime_records: list[dict] = [] + self.aborted = False + self.ok = False + + if options.version: + self.version = options.version + self.version_source = "explicit --version" + elif (self.repo_root / "pyproject.toml").is_file(): + self.version = get_version(self.repo_root) + self.version_source = "pyproject.toml dynamic attr" + else: + self.version = "0.0.0" + self.version_source = "unresolved" + + self.report_dir = report_mod.reports_root(self.repo_root, options.report_dir) / self.version + + # -- helpers --------------------------------------------------------- + + def _git(self, *args: str) -> str: + try: + completed = subprocess.run( + ["git", *args], + cwd=self.repo_root, + capture_output=True, + text=True, + check=False, + ) + except OSError: + return "" + return (completed.stdout or "").strip() + + def _execute(self, step_fn) -> None: + started = time.perf_counter() + try: + step = step_fn() + except Exception as exc: # a crashed step is a failed step, never a silent pass + step = Step(name=getattr(step_fn, "__name__", "step").lstrip("_").replace("_", "-"), + ok=False, detail=f"crashed: {type(exc).__name__}: {exc}") + step.duration_s = round(time.perf_counter() - started, 3) + self.steps.append(step) + + flag = "ok " if step.ok else "FAIL" + print(f"[{flag}] {step.name} ({step.duration_s:.2f}s)") + for line in str(step.detail).splitlines(): + print(f" {line}") + + if not step.ok and not self.options.keep_going: + self.aborted = True + print(" stopping: pass --keep-going to collect every failure") + + # -- steps ----------------------------------------------------------- + + def _repo_state(self) -> Step: + if not (self.repo_root / ".git").exists(): + return Step("repo-state", False, f"{self.repo_root} is not a git repository") + + commit = self._git("rev-parse", "HEAD") + branch = self._git("rev-parse", "--abbrev-ref", "HEAD") + porcelain = self._git("status", "--porcelain") + dirty = bool(porcelain) + self.git = {"commit": commit, "branch": branch, "dirty": dirty} + + if dirty and not self.options.allow_dirty: + preview = "\n".join(porcelain.splitlines()[:20]) + return Step( + "repo-state", + False, + f"working tree is dirty; commit, stash or pass --allow-dirty:\n{preview}", + data=self.git, + ) + + detail = f"{branch} @ {commit[:12]} dirty={dirty} version={self.version} ({self.version_source})" + if dirty: + detail += " (accepted via --allow-dirty)" + return Step("repo-state", True, detail, data=self.git) + + def _version_consistency(self) -> Step: + problems: list[str] = [] + data: dict = {"version": self.version, "source": self.version_source} + + attr = version_mod.dynamic_version_attr(self.repo_root) + data["dynamic_attr"] = attr + if attr is None: + problems.append( + "pyproject.toml no longer declares tool.setuptools.dynamic.version.attr" + ) + + if (self.repo_root / "pyproject.toml").is_file(): + resolved = get_version(self.repo_root) + data["resolved_version"] = resolved + if resolved != self.version: + problems.append( + f"src/orisys/__init__.py __version__ is {resolved!r} but the release " + f"version is {self.version!r}" + ) + + sdk_dirs = version_mod.sdk_versions(self.repo_root) + data["sdk_versions"] = sdk_dirs + for wheel in version_mod.sdk_wheels(self.repo_root, self.version): + found = version_mod.wheel_version(wheel) + if found != self.version: + problems.append(f"{wheel.name} carries version {found!r}, expected {self.version!r}") + + detail = ( + f"version={self.version} ({self.version_source}) dynamic_attr={attr}\n" + f"SDK/ versions present: {sdk_dirs or 'none'}" + ) + if problems: + detail += "\n" + "\n".join(problems) + return Step("version-consistency", not problems, detail, data=data) + + def _duplicate_guard(self) -> Step: + problems: list[str] = [] + lines: list[str] = [] + + existing = version_mod.sdk_wheels(self.repo_root, self.version) + if existing: + if self.options.force: + lines.append( + f"SDK/{self.version}/ already holds {len(existing)} wheel(s); " + "overwriting because --force was passed" + ) + else: + problems.append( + f"SDK/{self.version}/ already holds {len(existing)} wheel(s): " + f"{[w.name for w in existing]}. Pass --force to rebuild over them." + ) + else: + lines.append(f"SDK/{self.version}/ is empty (no wheel has been staged yet)") + + if self.options.check_pypi: + versions, error = version_mod.pypi_released_versions() + if error: + problems.append(f"PyPI lookup failed: {error}") + elif versions is None: + problems.append("PyPI lookup returned no version list") + elif self.version in versions: + problems.append( + f"orisys {self.version} is already published on PyPI; a PyPI upload " + "cannot be undone. Published: " + ", ".join(versions) + ) + else: + lines.append( + f"PyPI has no {self.version}; published versions: " + (", ".join(versions) or "none") + ) + else: + lines.append("PyPI existence check skipped (pass --check-pypi to enable)") + + detail = "\n".join(lines) + if problems: + detail += "\n" + "\n".join(problems) + return Step("duplicate-guard", not problems, detail) + + def _tests(self) -> Step: + if self.options.skip_tests: + return Step( + "tests", + True, + "WARNING: test gate SKIPPED via --skip-tests; this run cannot certify a release", + data={"status": "skipped"}, + ) + + marker = "not hardware" if self.options.longrun else "not slow and not hardware" + self.report_dir.mkdir(parents=True, exist_ok=True) + junit = self.report_dir / "junit.xml" + command = [ + sys.executable, + "-m", + "pytest", + "-q", + "-m", + marker, + f"--junitxml={junit}", + ] + try: + completed = subprocess.run( + command, cwd=self.repo_root, capture_output=True, text=True, check=False + ) + except FileNotFoundError as exc: + return Step("tests", False, f"cannot run pytest: {exc}") + + ok = completed.returncode == 0 + tail = ((completed.stdout or "") + (completed.stderr or "")).strip().splitlines()[-12:] + detail = f"pytest -m {marker!r} in {sys.executable}\nexit code {completed.returncode}" + if tail: + detail += "\n" + "\n".join(tail) + return Step("tests", ok, detail, data={"marker": marker, "junit": str(junit)}) + + def _bundle_precheck(self) -> Step: + if not (self.repo_root / "pyproject.toml").is_file(): + return Step("bundle-precheck", False, "pyproject.toml not found") + + items = bundle_items(self.repo_root) + problems: list[str] = [] + counts: dict[str, int] = {} + for item in items: + files = resolve_bundle_source_files(self.repo_root, item) + counts[str(item.get("name"))] = len(files) + if not files: + problems.append( + f"bundle item {item.get('name')!r} resolved 0 files; its patterns match nothing" + ) + + try: + verify_required_assets(self.repo_root, items) + except FileNotFoundError as exc: + problems.append(str(exc)) + + detail = "resolved files per bundle item: " + json.dumps(counts) + if problems: + detail += "\n" + "\n".join(problems) + return Step("bundle-precheck", not problems, detail, data={"counts": counts}) + + def _discover_wheels(self) -> str | None: + """Populate self.wheels; return an error message when nothing was found.""" + directory = ( + Path(self.options.wheel_dir) + if self.options.wheel_dir + else self.repo_root / "SDK" / self.version + ) + self.wheel_dir = directory + if not directory.is_dir(): + return ( + f"{directory} does not exist. Build first " + "(`python build/run.py wheel`) or pass --wheel-dir." + ) + + self.wheels = sorted(directory.glob("*.whl")) + if not self.wheels: + return f"no *.whl found in {directory}. Build first or pass --wheel-dir." + return None + + def _wheel_inspection(self) -> Step: + problem = self._discover_wheels() + if problem: + return Step("wheel-inspection", False, problem) + + problems: list[str] = [] + blocks: list[str] = [] + for wheel in self.wheels: + result = inspect_wheel( + wheel, + expected_version=self.version, + repo_root=self.repo_root, + strict_bundle=True, + ) + blocks.append(result.render()) + if not result.ok: + problems.append( + f"{wheel.name}: " + ", ".join(check.name for check in result.failures) + ) + + detail = f"{len(self.wheels)} wheel(s) in {self.wheel_dir}\n" + "\n".join(blocks) + if problems: + detail += "\nfailed checks:\n" + "\n".join(problems) + return Step("wheel-inspection", not problems, detail, data={"count": len(self.wheels)}) + + def _platform_token(self) -> str: + return "win_amd64" if sys.platform == "win32" else "manylinux" + + def _runtime_targets(self) -> list[tuple[str, Path, Path | None]]: + token = self._platform_token() + if self.options.all_wheel_pythons: + targets: list[tuple[str, Path, Path | None]] = [] + for tag in ("cp310", "cp311", "cp312"): + match = next( + (w for w in self.wheels if f"-{tag}-" in w.name and token in w.name), None + ) + if match is None: + raise RuntimeError( + f"no {tag} {token} wheel among {[w.name for w in self.wheels]}" + ) + executable = version_mod.python_for_tag(tag) + if executable is None: + raise RuntimeError( + f"cannot resolve an interpreter for {tag}; create the conda env " + f"py{tag[2:]} or set CONDA_ENVS_ROOT" + ) + targets.append((tag, match, executable)) + return targets + + tag = f"cp{sys.version_info.major}{sys.version_info.minor}" + match = next((w for w in self.wheels if f"-{tag}-" in w.name and token in w.name), None) + if match is None: + raise RuntimeError( + f"no {tag} {token} wheel among {[w.name for w in self.wheels]}; pass " + "--all-wheel-pythons to verify every ABI in its own environment, or run from " + "a matching interpreter" + ) + return [(tag, match, None)] + + def _wheel_runtime(self) -> Step: + if self.options.no_runtime: + return Step( + "wheel-runtime", + True, + "WARNING: runtime verification SKIPPED via --no-runtime", + data={"status": "skipped"}, + ) + + video = self.options.video or default_verification_video(self.repo_root) + if video is None: + return Step( + "wheel-runtime", + False, + "no verification clip found. Pass --video, set ORISYS_VERIFY_VIDEO, or commit a " + "clip under data/sample/. The runtime check is never silently skipped.", + ) + video = Path(video) + if not video.is_file(): + return Step("wheel-runtime", False, f"verification clip not found: {video}") + + try: + targets = self._runtime_targets() + except RuntimeError as exc: + return Step("wheel-runtime", False, str(exc)) + + problems: list[str] = [] + lines: list[str] = [f"clip: {video}", f"config: {self.options.config_name}"] + for tag, wheel, executable in targets: + label = f"{tag} {wheel.name}" + try: + if self.options.no_assets: + assets_out = "asset check skipped (--no-assets)" + else: + assets_out = verify_wheel_assets(wheel, python_executable=executable) + runtime_out = verify_wheel_runtime( + wheel, + video=video, + config_name=self.options.config_name, + frames=self.options.frames, + python_executable=executable, + offline=self.options.offline, + ) + except subprocess.CalledProcessError as exc: + problems.append(f"{label}: command failed ({exc})") + continue + except Exception as exc: + problems.append(f"{label}: {type(exc).__name__}: {exc}") + continue + + lines.append(f"{label}: {assets_out} | frames={runtime_out['frames']} flow={runtime_out['flow_shape']}") + self.runtime_records.append({"tag": tag, "wheel": wheel.name, **runtime_out}) + + detail = "\n".join(lines) + if problems: + detail += "\nfailures:\n" + "\n".join(problems) + return Step("wheel-runtime", not problems, detail, data={"records": self.runtime_records}) + + # -- driver ---------------------------------------------------------- + + def run(self) -> int: + started_at = _now() + started = time.perf_counter() + print(f"=== orisys release preflight: version {self.version} ({self.version_source}) ===") + print(f"repo: {self.repo_root}") + print(f"report: {self.report_dir}") + print() + + try: + for step_fn in ( + self._repo_state, + self._version_consistency, + self._duplicate_guard, + self._tests, + self._bundle_precheck, + self._wheel_inspection, + self._wheel_runtime, + ): + self._execute(step_fn) + if self.aborted: + break + finally: + self.ok = bool(self.steps) and all(step.ok for step in self.steps) and not self.aborted + duration = round(time.perf_counter() - started, 3) + payload = { + "version": self.version, + "version_source": self.version_source, + "ok": self.ok, + "aborted": self.aborted, + "started_at": started_at, + "finished_at": _now(), + "duration_s": duration, + "git": self.git, + "interpreter": { + "executable": sys.executable, + "version": platform.python_version(), + "platform": sys.platform, + }, + "options": {key: value for key, value in asdict(self.options).items() + if not isinstance(value, Path)}, + "steps": [asdict(step) for step in self.steps], + "wheels": [ + { + "name": wheel.name, + "size": wheel.stat().st_size, + "sha256": _sha256_file(wheel), + } + for wheel in self.wheels + ], + } + payload["options"]["wheel_dir"] = str(self.options.wheel_dir) if self.options.wheel_dir else None + payload["options"]["video"] = str(self.options.video) if self.options.video else None + + json_path, text_path = report_mod.write_report(payload, self.report_dir) + print() + print(f"report: {json_path}") + print(f"report: {text_path}") + + print() + print(f"=== preflight {'PASSED' if self.ok else 'FAILED'} ({duration}s) ===") + return 0 if self.ok else 1 + + +def run_preflight(options: PreflightOptions, repo_root: Path | None = None) -> int: + return Preflight(options, repo_root=repo_root).run() diff --git a/build/release/report.py b/build/release/report.py new file mode 100644 index 0000000..145bec6 --- /dev/null +++ b/build/release/report.py @@ -0,0 +1,71 @@ +"""Structured preflight reports. + +A preflight run writes both a machine-readable ``preflight.json`` (the contract a +future publish step will consume and re-verify) and a human ``preflight.txt``. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +REPORTS_RELPATH = Path("build") / "release" / "reports" +SCHEMA_VERSION = 1 + + +def reports_root(repo_root: Path, override: Path | None = None) -> Path: + return Path(override) if override else Path(repo_root) / REPORTS_RELPATH + + +def render_text(payload: dict) -> str: + git = payload.get("git") or {} + interpreter = payload.get("interpreter") or {} + lines = [ + f"orisys release preflight :: version {payload.get('version')}", + f" result: {'PASS' if payload.get('ok') else 'FAIL'}", + f" git: {git.get('branch')} @ {git.get('commit')} (dirty={git.get('dirty')})", + f" interpreter: {interpreter.get('version')} at {interpreter.get('executable')}", + f" duration: {payload.get('duration_s')}s", + "", + ] + + for step in payload.get("steps", []): + flag = "ok " if step.get("ok") else "FAIL" + lines.append(f"[{flag}] {step.get('name')} ({step.get('duration_s')}s)") + for detail_line in str(step.get("detail") or "").splitlines(): + lines.append(f" {detail_line}") + lines.append("") + + wheels = payload.get("wheels") or [] + if wheels: + lines.append("wheels:") + for wheel in wheels: + lines.append( + f" {wheel.get('name')} {wheel.get('size')} bytes sha256={wheel.get('sha256')}" + ) + lines.append("") + + options = payload.get("options") or {} + waived = [ + key + for key, value in options.items() + if value and key in {"skip_tests", "allow_dirty", "force", "no_runtime", "no_assets"} + ] + if waived: + lines.append(f"waived guards: {', '.join(sorted(waived))}") + lines.append("") + + return "\n".join(lines).rstrip() + "\n" + + +def write_report(payload: dict, out_dir: Path) -> tuple[Path, Path]: + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + payload = {**payload, "schema": SCHEMA_VERSION} + json_path = out_dir / "preflight.json" + text_path = out_dir / "preflight.txt" + + json_path.write_text(json.dumps(payload, indent=2, sort_keys=False), encoding="utf-8") + text_path.write_text(render_text(payload), encoding="utf-8") + return json_path, text_path diff --git a/build/release/version.py b/build/release/version.py new file mode 100644 index 0000000..6294baf --- /dev/null +++ b/build/release/version.py @@ -0,0 +1,141 @@ +"""Version consistency and duplicate-release guards for the release gate.""" + +from __future__ import annotations + +import json +import os +import re +import urllib.error +import urllib.request +from pathlib import Path + +PYPI_JSON_URL = "https://pypi.org/pypi/orisys/json" + +# orisys----.whl +_WHEEL_NAME_RE = re.compile(r"^orisys-(?P[^-]+)-[^-]+-[^-]+-.+\.whl$") + + +def wheel_version(wheel_path: Path) -> str | None: + match = _WHEEL_NAME_RE.match(Path(wheel_path).name) + return match.group("version") if match else None + + +def sdk_versions(repo_root: Path) -> list[str]: + """Version directory names present under ``SDK/``, in ascending order. + + Only digit-leading names count as versions, so stray directories (``tmp``, + ``latest``) never appear as releases. + """ + sdk_root = Path(repo_root) / "SDK" + if not sdk_root.is_dir(): + return [] + + return sorted( + ( + entry.name + for entry in sdk_root.iterdir() + if entry.is_dir() and entry.name[:1].isdigit() + ), + key=_version_sort_key, + ) + + +def sdk_wheels(repo_root: Path, version: str) -> list[Path]: + """Wheels already published into ``SDK//``.""" + version_dir = Path(repo_root) / "SDK" / version + if not version_dir.is_dir(): + return [] + return sorted(version_dir.glob("*.whl")) + + +def dynamic_version_attr(repo_root: Path) -> str | None: + """Return pyproject's configured dynamic version attr (e.g. orisys.__version__).""" + from common.pyproject_utils import load_pyproject + + config = ( + load_pyproject(Path(repo_root)) + .get("tool", {}) + .get("setuptools", {}) + .get("dynamic", {}) + .get("version") + ) + if isinstance(config, dict): + attr = config.get("attr") + if isinstance(attr, str): + return attr + return None + + +def pypi_released_versions(timeout: float = 20.0) -> tuple[list[str] | None, str | None]: + """Read the published version list from PyPI. + + Returns ``(versions, error)``; ``versions`` is ``None`` when the lookup + failed. This is the only network call in the release gate and it is + read-only, so it is safe to run before an irreversible upload. + """ + request = urllib.request.Request( + PYPI_JSON_URL, + headers={"User-Agent": "orisys-release-preflight", "Accept": "application/json"}, + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.load(response) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return [], None + return None, f"PyPI returned HTTP {exc.code}" + except Exception as exc: # network, DNS, TLS, JSON + return None, f"{type(exc).__name__}: {exc}" + + releases = payload.get("releases") + if not isinstance(releases, dict): + return None, "unexpected PyPI response shape (no 'releases' mapping)" + listed = [ + str(version) + for version, files in releases.items() + if isinstance(files, list) and files + ] + return sorted(listed, key=_version_sort_key), None + + +def _version_sort_key(name: str): + parts = re.findall(r"\d+", name) + return ([int(part) for part in parts] or [0], name) + + +def conda_envs_root() -> Path | None: + """Locate the conda envs directory the same way build/wheel/win.bat does.""" + override = os.getenv("CONDA_ENVS_ROOT") + if override: + path = Path(override) + return path if path.is_dir() else None + + home = Path.home() + candidates = ( + home / "anaconda3" / "envs", + home / "miniconda3" / "envs", + Path(os.getenv("LOCALAPPDATA", "")) / "anaconda3" / "envs", + Path(os.getenv("LOCALAPPDATA", "")) / "miniconda3" / "envs", + ) + for candidate in candidates: + if candidate.is_dir(): + return candidate + return None + + +def python_for_tag(tag: str, envs_root: Path | None = None) -> Path | None: + """Resolve an interpreter for a wheel tag like ``cp310`` (env ``py310``).""" + digits = tag[2:] if tag.startswith("cp") else tag + env_name = f"py{digits}" + root = envs_root or conda_envs_root() + if root is None: + return None + + for relative in ( + Path(env_name) / "python.exe", + Path(env_name) / "bin" / "python", + ): + candidate = root / relative + if candidate.is_file(): + return candidate + return None diff --git a/build/run.py b/build/run.py index 092c06f..6fdd20d 100644 --- a/build/run.py +++ b/build/run.py @@ -8,6 +8,8 @@ Usage (from repo root): python build/run.py gui finger|mini|box python build/run.py verify [wheel.whl] python build/run.py verify --gui-dist [--product ory_finger] + python build/run.py release preflight [--version V] + python build/run.py release audit See build/README.md for details. """ @@ -74,15 +76,23 @@ def _cmd_verify(argv: list[str]) -> int: return verify_main(argv) +def _cmd_release(argv: list[str]) -> int: + from release.cli import main as release_main + + return release_main(argv) + + def main(argv: list[str] | None = None) -> int: argv = list(sys.argv[1:] if argv is None else argv) if not argv or argv[0] in ("-h", "--help"): print(__doc__) - print("Commands: wheel | wheel-win | manylinux | gui | verify") + print("Commands: wheel | wheel-win | manylinux | gui | verify | release") print("Examples:") print(" python build/run.py wheel --help") print(" python build/run.py gui finger") print(" python build/run.py manylinux -PythonVersions 3.10") + print(" python build/run.py release preflight --version 0.5.2") + print(" python build/run.py release audit SDK\\0.5.2\\orisys-0.5.2-cp312-cp312-win_amd64.whl") return 0 command, rest = argv[0], argv[1:] @@ -92,10 +102,14 @@ def main(argv: list[str] | None = None) -> int: "wheel-manylinux": _cmd_manylinux, "gui": _cmd_gui, "verify": _cmd_verify, + "release": _cmd_release, } if command not in dispatch: print(f"Unknown command: {command!r}", file=sys.stderr) - print("Choose from: wheel, wheel-win, manylinux, gui, verify", file=sys.stderr) + print( + "Choose from: wheel, wheel-win, manylinux, gui, verify, release", + file=sys.stderr, + ) return 2 return dispatch[command](rest) diff --git a/build/wheel/build.py b/build/wheel/build.py index ea2dab0..fba9669 100644 --- a/build/wheel/build.py +++ b/build/wheel/build.py @@ -30,7 +30,13 @@ if str(_BUILD_ROOT) not in sys.path: sys.path.insert(0, str(_BUILD_ROOT)) from common.paths import NUITKA_COMPILE_DIR, NUITKA_WHEEL_DIR, REPO_ROOT, repaired_wheel_dir, repo_path -from common.pyproject_utils import bundle_items, extras_require, get_version, install_requires +from common.pyproject_utils import ( + bundle_items, + extras_require, + get_version, + install_requires, + project_metadata, +) from common.verify.wheel import verify_built_wheel from wheel.compile import compile_orisys, copy_compiled_orisys, install_nuitka from wheel.layout import ( @@ -41,6 +47,7 @@ from wheel.layout import ( from wheel.package import ( is_compliant_manylinux_wheel, publish_wheel_to_sdk, + read_long_description, repair_linux_wheel, run_bdist_wheel, write_wheel_setup, @@ -122,6 +129,8 @@ def build(*, target: str = "internal", verify: bool | None = None) -> Path: package_data=package_data_from_bundle(ext_suffix, bundle), install_requires=install_requires(REPO_ROOT), extras_require=extras_require(REPO_ROOT), + project=project_metadata(REPO_ROOT), + long_description=read_long_description(REPO_ROOT), ) # --- 3. Build wheel, repair on Linux, publish to SDK/ --- diff --git a/build/wheel/package.py b/build/wheel/package.py index 9bcf495..848eaee 100644 --- a/build/wheel/package.py +++ b/build/wheel/package.py @@ -19,6 +19,76 @@ _MANYLINUX_TAG_RE = re.compile( r"(?:manylinux2014|manylinux_2_(?:17|24|27|28|31|34|35))_x86_64" ) +# Source of the PyPI project description (see read_long_description). +PYPI_README_RELPATH = Path("docs") / "pypi_readme.md" + + +def _license_text(project: dict) -> str | None: + """Extract free-text license from either pyproject license form.""" + declared = project.get("license") + if isinstance(declared, str): + return declared + if isinstance(declared, dict): + text = declared.get("text") + if isinstance(text, str): + return text + return None + + +def _person_strings(entries) -> list[str]: + """Render PEP 621 author/maintainer tables as ``Name `` strings.""" + rendered: list[str] = [] + for entry in entries or []: + if not isinstance(entry, dict): + continue + name = (entry.get("name") or "").strip() + email = (entry.get("email") or "").strip() + if name and email: + rendered.append(f"{name} <{email}>") + elif name or email: + rendered.append(name or email) + return rendered + + +def metadata_kwargs(project: dict) -> dict: + """Translate a pyproject ``[project]`` table into ``setup()`` keywords. + + The Nuitka wheel is built by a generated ``setup.py`` outside the normal + setuptools pipeline, so every PyPI-visible field has to be passed through + explicitly. Omitting them is what left the 0.5.2 release with no + ``Requires-Python``, classifiers, project URLs or license on PyPI. + """ + authors = _person_strings(project.get("authors")) + maintainers = _person_strings(project.get("maintainers")) + candidates = { + "description": project.get("description"), + "python_requires": project.get("requires-python"), + "classifiers": list(project.get("classifiers") or []), + "project_urls": dict(project.get("urls") or {}), + "license": _license_text(project), + "author": authors[0] if authors else None, + "author_email": None, + "maintainer": maintainers[0] if maintainers else None, + } + return {key: value for key, value in candidates.items() if value} + + +def read_long_description(repo_root: Path) -> str | None: + """Return the PyPI long description, or ``None`` when it is missing. + + Deliberately not the repository ``README.md``: that file documents the + closed-source build/publishing layout, which should not end up on the PyPI + project page. + """ + path = Path(repo_root) / PYPI_README_RELPATH + if not path.is_file(): + print( + f"[WARN] {PYPI_README_RELPATH.as_posix()} not found; the wheel will be " + "published without a long description." + ) + return None + return path.read_text(encoding="utf-8") + def write_wheel_setup( wheel_dir: Path, @@ -27,7 +97,17 @@ def write_wheel_setup( package_data: dict[str, list[str]], install_requires: list[str], extras_require: dict[str, list[str]], + project: dict | None = None, + long_description: str | None = None, ) -> Path: + metadata_lines = "" + if project: + for key, value in sorted(metadata_kwargs(project).items()): + metadata_lines += f" {key}={value!r},\n" + if long_description: + metadata_lines += " long_description=_LONG_DESCRIPTION,\n" + metadata_lines += ' long_description_content_type="text/markdown",\n' + setup_py_path = wheel_dir / "setup.py" setup_content = f""" from setuptools import setup, Distribution @@ -36,12 +116,12 @@ class BinaryDistribution(Distribution): def has_ext_modules(self): return True +{'_LONG_DESCRIPTION = ' + repr(long_description) if long_description else ''} + setup( name="orisys", version="{version}", - description="Orisys - Optical Tactile Sensor SDK (Nuitka Compiled)", - author="Orisys", - packages=['orisys', 'orisys.examples', 'orisys_blend'], +{metadata_lines} packages=['orisys', 'orisys.examples', 'orisys_blend'], package_dir={{'': 'wheel_root'}}, package_data={package_data}, include_package_data=True, diff --git a/docs/get_started.md b/docs/get_started.md index 2b69c09..b760513 100644 --- a/docs/get_started.md +++ b/docs/get_started.md @@ -5,7 +5,7 @@ description: 安装并运行 Orisys SDK 的基础步骤 # 快速开始 -版本：0.5.1 +版本：0.5.2 ## 简介 本文件介绍如何上手和运行叠动视触觉传感器配套的开发套件。本套件是配合叠动视触觉传感器进行图像处理，计算形变、力以及接触信息。它可以从摄像头或视频文件中读取图像，处理后打印和展示处理后信息。 @@ -84,9 +84,9 @@ pip install --upgrade orisys ### 使用 .whl 文件安装（离线或指定版本） 从官方渠道获取 .whl 文件，文件名格式通常为： orisys-{版本}-{Python版本}-{Python版本}-{操作系统}_{硬件架构}.whl -例如：orisys-0.5.1-cp310-cp310-win_amd64.whl 表示： +例如：orisys-0.5.2-cp310-cp310-win_amd64.whl 表示： -版本：0.5.1 +版本：0.5.2 Python 版本：3.10（cp310） 操作系统：Windows（win） 硬件架构：64位（amd64） @@ -97,7 +97,7 @@ Python 版本：3.10（cp310） ```bash # 请将文件名替换为您实际下载的文件名 -pip install orisys-0.5.1-cp310-cp310-win_amd64.whl +pip install orisys-0.5.2-cp310-cp310-win_amd64.whl ``` ### 安装 CuPy（可选） @@ -139,7 +139,7 @@ python import orisys print(orisys.__version__) ``` -如果输出版本号（例如 0.5.1），则说明安装成功。 +如果输出版本号（例如 0.5.2），则说明安装成功。 ## 使用方法 ### 运行示例程序 diff --git a/docs/publishing.md b/docs/publishing.md new file mode 100644 index 0000000..d169078 --- /dev/null +++ b/docs/publishing.md @@ -0,0 +1,183 @@ +# 发布流程（构建 → 校验 → 发布） + +本文档说明如何构建 orisys 的 Nuitka 二进制 wheel、如何在发布前用一条命令做完整门禁，以及当前仍需人工执行的发布步骤。 + +> 历史说明：`README.md` 与 `build/wheel/docker/run.ps1` 早就引用了本文件，但它此前并不存在。本文件补齐该引用。 + +--- + +## 0. 环境准备 + +推荐用 `py310` conda 环境执行构建与校验（它同时具备 pytest / twine / numba / Nuitka / open3d / PyQt5）： + +```powershell +conda activate py310 +python -m pytest -m "not slow and not hardware" # 开发期快速回归 +``` + +`py311`、`py312` 仅作为 Nuitka 各 ABI 的编译环境按需调用（`build/wheel/win.bat` 会依次使用 `py310`/`py311`/`py312`）。 + +**不要**在同一个环境里既装旧版 `orisys` 又从源码目录跑测试：site-packages 里的旧包会静默遮蔽 `src/orisys`。 +`tests/test_harness_guards.py` 会显式报错提示执行 `pip uninstall -y orisys`。 + +--- + +## 1. 构建 wheel + +```powershell +# Windows：py310 / py311 / py312 三个 ABI +python build/run.py wheel-win + +# 单个解释器（当前环境） +python build/run.py wheel + +# Linux manylinux（Docker Desktop，Linux 容器模式） +python build/run.py manylinux +python build/run.py wheel-manylinux -PythonVersions 3.10 +``` + +产物落点： + +- `SDK//` — 交付用 wheel（git 忽略） +- `dist/repaired/` — 上传候选（git 忽略） + +--- + +## 2. 发布前门禁：`release preflight` + +```powershell +python build/run.py release preflight --version 0.5.2 +``` + +按顺序执行 7 个步骤，任一步失败即整体失败（`--keep-going` 可改为收集全部失败）： + +| 步骤 | 检查内容 | +| --- | --- | +| `repo-state` | 是 git 仓库、工作树干净、记录 HEAD（`--allow-dirty` 可豁免） | +| `version-consistency` | `src/orisys/__init__.py::__version__` == pyproject 的 dynamic attr == `SDK//` 中 wheel 文件名里的版本 | +| `duplicate-guard` | `SDK//` 已有产物则拒绝（`--force` 豁免）；`--check-pypi` 时只读查询 PyPI，版本已存在则拒绝 | +| `tests` | `pytest -m "not slow and not hardware"`（`--longrun` 时包含 slow） | +| `bundle-precheck` | 每个 `[[tool.orisys.bundle.items]]` 至少解析出 1 个文件；finger 必需资产存在 | +| `wheel-inspection` | 逐个 wheel 做静态断言：文件名/版本/METADATA 字段/RECORD 哈希/压缩包路径/体积上限/无 pyc 与缓存/无 `orisys/**/*.py` 源码泄漏/与源码树逐文件字节一致 | +| `wheel-runtime` | 在一次性 venv 中安装该 wheel，断言内置 configs/sensors 资产可加载，并对真实视频跑 40 帧公开流水线（无需相机、无 GUI） | + +产物报告： + +- `build/release/reports//preflight.json` — 机器可读，含每个 wheel 的 sha256，供后续发布步骤复用 +- `build/release/reports//preflight.txt` — 人工阅读 +- `build/release/reports//junit.xml` — pytest 结果 + +（`build/release/reports/` 已加入 `.gitignore`。） + +### 常用参数 + +| 参数 | 作用 | +| --- | --- | +| `--wheel-dir DIR` | 校验指定目录下的 wheel（默认 `SDK/`） | +| `--all-wheel-pythons` | cp310/cp311/cp312 各自在对应 conda 环境里独立校验 | +| `--video PATH` / `--frames N` / `--config NAME` | 覆盖运行时校验输入（默认 `data/sample/s1_480p.mp4` / 40 帧 / `box`） | +| `--offline` | 复用父环境的 numpy/opencv/numba，不去 PyPI 装依赖 | +| `--check-pypi` | 唯一一处网络调用：只读 PyPI，防止重复发布 | +| `--skip-tests` / `--no-runtime` / `--no-assets` | 显式跳过并在报告中留痕；**跳过的运行不能作为发布依据** | +| `--report-dir DIR` | 覆盖报告根目录 | + +`wheel-runtime` **不会静默跳过**：找不到视频时直接失败，提示 `--video` / `ORISYS_VERIFY_VIDEO`。 + +### 单体体检：`release audit` + +对单个 wheel 做静态断言（+ 可选运行时校验），不需要版本上下文。适合体检历史产物： + +```powershell +python build/run.py release audit SDK\0.5.2\orisys-0.5.2-cp312-cp312-win_amd64.whl +python build/run.py release audit dist\repaired\orisys-0.5.2-cp310-cp310-win_amd64.whl --strict-bundle + +# 追加运行时校验 +python build/run.py release audit --assets --runtime --video data/sample/s1_480p.mp4 +``` + +默认对「内置文件与当前源码树不一致」只告警（历史 wheel 与当前分支不一致是正常的）；加 `--strict-bundle` 改成失败（刚构建完的 wheel 应当一致）。 + +旧的导入冒烟测试仍然可用，行为不变： + +```powershell +python build/run.py verify +python build/run.py verify --contents # 静态断言 +python build/run.py verify --assets --runtime # 资产 + 真实帧 +python build/run.py verify --gui-dist --product ory_finger +``` + +--- + +## 3. 发布（目前仍需人工） + +阶段一不包含自动发布。当前人工步骤： + +1. 用 `release preflight` 拿到 PASS（并保留 `preflight.json`）。 +2. 把 `SDK//` 同步到 `orisys-sdk-release` 仓库的 `release//`。 +3. GitHub：打 tag **`v`** 并创建 Release，上传 6 个 wheel 作为附件。 + - 注意历史 tag 里有一个 typo：`0.50`（与 `v0.5.2` 指向同一 commit）。 +4. PyPI：先 TestPyPI 再正式仓库。 + +```powershell +python -m twine upload --repository testpypi SDK/0.5.2/*.whl +python -m twine upload SDK/0.5.2/*.whl +``` + +**不要把 API token 写进仓库**。`release preflight --check-pypi` 只做只读查询，不需要 token。 + +### 已知的历史发布空洞 + +| 版本 | 本地 `SDK/` | PyPI | GitHub Release | +| --- | --- | --- | --- | +| 0.4.2 | — | 有 | 有 | +| 0.4.3 | — | 有 | 有 | +| 0.4.4 | 有 | **无** | **无** | +| 0.5.0 | 有 | **无** | **无** | +| 0.5.2 | 有 | 有 | 有（tag `v0.5.2` + typo tag `0.50`） | + +--- + +## 4. 环境变量 + +| 变量 | 用途 | +| --- | --- | +| `ORISYS_TEST_VIDEO` | pytest 用的视频；默认自动取 `data/sample/s1_480p.mp4` | +| `ORISYS_TEST_CONFIG` | pytest 用的配置名；默认自动挑一个含 `stitcher.box_size` 的配置 | +| `ORISYS_VERIFY_VIDEO` | 运行时校验用的视频 | +| `ORISYS_VERIFY_CONFIG` | 运行时校验用的配置名（默认 `box`） | +| `ORISYS_LONGRUN_BACKEND` / `ORISYS_LONGRUN_FRAMES` | `slow` 长稳测试的参数 | +| `CONDA_ENVS_ROOT` | conda envs 根目录（`--all-wheel-pythons` 用它解析 `py310`/`py311`/`py312`） | + +--- + +## 5. 两个必须记住的坑 + +### 5.1 Nuitka 编译包的 `__file__` 是假的 + +Nuitka 把整个 `orisys` 编译成单个扩展模块，其 `__init__` 会安装 `nuitka_module_loader`。于是： + +```python +>>> import orisys +>>> orisys.__file__ # 合成的、磁盘上并不存在的路径 +'...\\site-packages\\orisys\\orisys\\__init__.py' +>>> orisys.__path__ # 第二项是 Nuitka 的哨兵值 +['...\\site-packages\\orisys\\orisys', '\\not_existing'] +>>> orisys.__loader__.__class__.__name__ +'nuitka_module_loader' +``` + +后果：**在 Nuitka wheel 里不能靠 `orisys.__file__` 或 `importlib.resources.files("orisys")` 定位内置数据**——它们会指向不存在的目录，静默返回「文件不存在」。请改用发行包定位： + +```python +from importlib.metadata import distribution +root = distribution("orisys").locate_file("orisys") # site-packages/orisys +``` + +`build/common/verify/wheel.py` 的内置资产校验就是这么做的，并且会显式断言该目录位于校验用 venv 内，避免被父环境里的旧 `orisys` 顶替。 + +### 5.2 `--offline` 会被父环境的脏安装污染 + +`--offline` 通过 venv 的 `--system-site-packages` 复用父环境的 numpy/numba 等依赖，这也顺带暴露了父环境里的 `orisys`。若父环境残留了半卸载的 editable 安装（典型症状是 `pip` 提示 `Ignoring invalid distribution -risys`，且 `site-packages/orisys/__init__.py` 是一个指向死路径的 shim），校验就会读到它。 + +处置：在构建环境里执行 `pip uninstall -y orisys`，或改用不带 `--offline` 的完全隔离 venv（会联网安装依赖）。 + diff --git a/docs/pypi_readme.md b/docs/pypi_readme.md new file mode 100644 index 0000000..52aa170 --- /dev/null +++ b/docs/pypi_readme.md @@ -0,0 +1,63 @@ +# Orisys SDK + +叠动科技视触觉传感器配套的 Python SDK，提供图像采集、拼接、稠密光流、Helmholtz-Hodge 分解、接触与深度估计，以及可运行的示例。 + +本文件是 PyPI 项目描述（`long_description`）的源文件。它按构建配置 `PYPI_README_RELPATH` 打包进 wheel，因此**不要**在这里写内部仓库布局、构建流水线或发布步骤——那些内容属于仓库内的 `README.md` 与 `docs/publishing.md`。 + +```bash +python -m pip install orisys +``` + +支持 Python 3.10 / 3.11 / 3.12，Windows x64 与 Ubuntu x64。 + +## 快速开始 + +```bash +python -c "import orisys; print(orisys.__version__)" +python -c "import orisys; print(orisys.list_config())" +``` + +## 安装选项 + +- `orisys` — CPU / 基础流程 +- `orisys[gui]` — Qt / Open3D 可视化示例 +- `orisys[cuda12]` / `orisys[cuda13]` — NVIDIA GPU 加速 +- `orisys[gui,cuda12]` — GUI + CUDA 12 + +`orisys[cuda]` 与 `orisys[cuda12]` 等价。 + +## 配置 + +配置解析顺序： + +1. 显式 JSON 文件路径 +2. 内置配置名（`box`、`mini`、`finger` 等） +3. 产品别名（如 `dd02` -> `dd02-ov`） +4. 向后兼容的 JSON `"name"` 匹配 + +```python +import orisys + +sensor = orisys.Sensor(0, config_name="box") +orisys.export_config("box", "./exported_config.json") +``` + +## 示例 + +示例随 wheel 一起分发，位于 `orisys/examples/`。也可以从项目仓库获取： + +```bash +python examples/basic_pipeline.py -v 0 -c box +python examples/finger_pipeline.py --camera_id 0 --config finger +``` + +无相机时可以改用视频文件： + +```bash +python examples/finger_pipeline.py --video ./sample.mp4 --config finger +``` + +## 支持与反馈 + +- 官网：[https://www.orisys-tech.cn](https://www.orisys-tech.cn) +- 问题反馈：GitHub Issues diff --git a/examples/basic_pipeline.py b/examples/basic_pipeline.py index 57daa4e..bc6feeb 100644 --- a/examples/basic_pipeline.py +++ b/examples/basic_pipeline.py @@ -7,7 +7,7 @@ ORY-BOX：单传感器基础流程 3. 按需计算接触区域 4. 读取并可视化结果 -SDK version: 0.5.1 +SDK version: 0.5.2 """ import os os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" diff --git a/examples/mini_pipeline.py b/examples/mini_pipeline.py index 9138e8b..da963d5 100644 --- a/examples/mini_pipeline.py +++ b/examples/mini_pipeline.py @@ -7,7 +7,7 @@ ORY-MINI：单传感器基础流程 3. 按需计算接触区域 4. 读取并可视化结果 -SDK version: 0.5.1 +SDK version: 0.5.2 """ import os os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" diff --git a/examples/two_sensor_pipeline.py b/examples/two_sensor_pipeline.py index aef548e..30c0464 100644 --- a/examples/two_sensor_pipeline.py +++ b/examples/two_sensor_pipeline.py @@ -18,7 +18,7 @@ - Windows 下必须在 if __name__ == '__main__' 中调用 mp.freeze_support() - 每个传感器进程都会占用独立的 GPU 显存 -SDK version: 0.5.1 +SDK version: 0.5.2 """ import os os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" diff --git a/pytest.ini b/pytest.ini index 39aeead..5c10fc6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,9 @@ [pytest] testpaths = tests python_files = test_*.py +# Run against the checkout without requiring `pip install -e .`, so a stale +# orisys in site-packages cannot silently shadow src/orisys. +pythonpath = src markers = slow: long-running tests for release validation + hardware: requires a physical sensor/camera attached to the machine diff --git a/src/orisys/__init__.py b/src/orisys/__init__.py index 0378a92..5e4f86b 100644 --- a/src/orisys/__init__.py +++ b/src/orisys/__init__.py @@ -5,7 +5,7 @@ A Python library for processing and analyzing optical tactile sensor data, including image stitching, optical flow tracking, and deformation decomposition. """ -__version__ = "0.5.1" +__version__ = "0.5.2" __author__ = "Orisys" from .sensor import Sensor diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e75c9c3 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,145 @@ +"""Shared pytest fixtures for the orisys test suite. + +This module was accidentally deleted in commit edc6e42 while four test modules +(``test_backends.py``, ``test_smoke_api.py``, ``test_errors_and_cleanup.py``, +``test_longrun_stability.py``) still depend on its fixtures, which made the +whole suite unrunnable. It is restored here with two fixes: + +* the default test video now points at clips that are actually tracked in the + repository (``data/sample/``), instead of the never-committed + ``tests/idle_1min.mp4``; +* the stale ``"prototype"`` default config in :func:`make_sensor` is gone, so + callers cannot silently bypass the ``stitcher.box_size`` validation. + +``tests/test_harness_guards.py`` guards against a repeat deletion. +""" + +import os +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +SRC_ROOT = REPO_ROOT / "src" +if str(SRC_ROOT) not in sys.path: + sys.path.insert(0, str(SRC_ROOT)) + +from orisys import Sensor +from orisys import configs as orisys_configs +from orisys.backend_availability import check_cupy_available, check_opencl_available + +# Video clips that are tracked in git, in preference order. +DEFAULT_VIDEO_CANDIDATES = ( + Path("tests") / "idle_1min.mp4", + Path("data") / "sample" / "s1_480p.mp4", + Path("data") / "sample" / "s1_720p.mp4", +) + + +def _resolve_test_video() -> Path: + env_video = os.getenv("ORISYS_TEST_VIDEO") + if env_video: + raw = Path(env_video) + candidates = [ + raw, # as provided (absolute or cwd-relative) + REPO_ROOT / raw, # repo-relative + ] + # Common typo fallback: "test/..." -> "tests/..." + if env_video.startswith("test/") or env_video.startswith("test\\"): + fixed = Path("tests", *Path(env_video).parts[1:]) + candidates.append(REPO_ROOT / fixed) + + for candidate in candidates: + if candidate.exists(): + return candidate + return raw + + for relative in DEFAULT_VIDEO_CANDIDATES: + candidate = REPO_ROOT / relative + if candidate.is_file(): + return candidate + + # Keep the reported path informative when nothing is available; the + # ``test_video_path`` fixture turns this into a skip. + return REPO_ROOT / DEFAULT_VIDEO_CANDIDATES[-1] + + +@pytest.fixture(scope="session") +def test_video_path() -> Path: + path = _resolve_test_video() + if not path.exists(): + pytest.skip( + "Test video not found. Set ORISYS_TEST_VIDEO to a valid video path." + ) + return path + + +@pytest.fixture(scope="session") +def test_config_name() -> str: + requested = os.getenv("ORISYS_TEST_CONFIG") + + def _is_usable_config(name: str) -> bool: + cfg = orisys_configs.load_config(name) + return "box_size" in cfg.get("stitcher", {}) + + if requested: + if not _is_usable_config(requested): + raise RuntimeError( + f"ORISYS_TEST_CONFIG='{requested}' is missing required key " + "'stitcher.box_size'. Use a config like 'ddjx01' or provide a full path." + ) + return requested + + # Auto-pick a stable config for tests (avoid prototype variants missing keys). + for name in orisys_configs.list_config(): + try: + if _is_usable_config(name): + return name + except Exception: + continue + + raise RuntimeError( + "No usable config found for tests (missing 'stitcher.box_size'). " + "Set ORISYS_TEST_CONFIG to a valid config JSON/path." + ) + + +@pytest.fixture(scope="session") +def available_backends() -> list[str]: + backends = ["cpu"] + if check_cupy_available(): + backends.append("cupy") + if check_opencl_available(): + backends.append("opencl") + return backends + + +def make_sensor( + video_path: Path, + config_name: str, + backend: str = "cpu", + verbose: bool = False, +) -> Sensor: + use_cuda = backend in ("cupy", "opencl") + return Sensor( + vid_src=str(video_path), + cuda=use_cuda, + isstitch=True, + verbose=verbose, + config_name=config_name, + backend=backend, + ) + + +@pytest.fixture +def sensor_factory(test_video_path, test_config_name): + def _factory(backend: str = "cpu", verbose: bool = False): + return make_sensor( + test_video_path, + config_name=test_config_name, + backend=backend, + verbose=verbose, + ) + + return _factory diff --git a/tests/dd06.py b/tests/dd06.py index a71b0d7..04465be 100644 --- a/tests/dd06.py +++ b/tests/dd06.py @@ -8,7 +8,7 @@ Flow: 4. Feed the rectified frame into Sensor.get_img(..., already_processed=True) 5. Compute deformation/contact and visualize -SDK version: 0.5.1 +SDK version: 0.5.2 """ from __future__ import annotations diff --git a/tests/test_harness_guards.py b/tests/test_harness_guards.py new file mode 100644 index 0000000..d079398 --- /dev/null +++ b/tests/test_harness_guards.py @@ -0,0 +1,116 @@ +"""Guards for the pytest harness itself. + +``tests/conftest.py`` was accidentally deleted in commit edc6e42 while four test +modules still depended on its fixtures, which made the suite unrunnable without +anything failing loudly in CI. These tests turn that class of regression into an +explicit failure, and also catch a stale ``orisys`` install silently shadowing +``src/orisys`` (a real hazard: base/py312 environments have older orisys copies +in site-packages). +""" + +from __future__ import annotations + +import functools +import importlib.util +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +CONFTEST_PATH = Path(__file__).resolve().parent / "conftest.py" + +# Names that must exist in tests/conftest.py. +REQUIRED_CONFTEST_NAMES = ( + "DEFAULT_VIDEO_CANDIDATES", + "make_sensor", + "test_video_path", + "test_config_name", + "available_backends", + "sensor_factory", +) + +# Fixtures that must be resolvable through pytest. +REQUIRED_FIXTURES = ( + "test_video_path", + "test_config_name", + "available_backends", + "sensor_factory", +) + + +@functools.lru_cache(maxsize=1) +def _load_conftest_module(): + """Load tests/conftest.py by path so a deletion is observable here.""" + spec = importlib.util.spec_from_file_location("_orisys_conftest_probe", CONFTEST_PATH) + if spec is None or spec.loader is None: + raise AssertionError(f"Cannot load {CONFTEST_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_conftest_defines_shared_fixtures(): + assert CONFTEST_PATH.is_file(), ( + f"{CONFTEST_PATH} is missing. test_backends.py, test_smoke_api.py, " + "test_errors_and_cleanup.py and test_longrun_stability.py depend on its " + "fixtures; it was accidentally deleted in commit edc6e42. Restore it " + "instead of deleting it." + ) + + module = _load_conftest_module() + missing = [ + name for name in REQUIRED_CONFTEST_NAMES if getattr(module, name, None) is None + ] + assert not missing, ( + f"tests/conftest.py no longer defines: {', '.join(missing)}. " + "The shared sensor-test fixtures are required by the suite." + ) + + +def test_default_test_video_is_reachable(): + """At least one default candidate must exist, or every sensor test skips.""" + module = _load_conftest_module() + candidates = [REPO_ROOT / Path(rel) for rel in module.DEFAULT_VIDEO_CANDIDATES] + existing = [path for path in candidates if path.is_file()] + assert existing, ( + "None of the default test videos exist:\n " + + "\n ".join(str(path) for path in candidates) + + "\nCommit a sample clip under data/sample/ or set ORISYS_TEST_VIDEO." + ) + + +def test_fixtures_are_registered_with_pytest(request): + missing: list[str] = [] + for name in REQUIRED_FIXTURES: + try: + request.getfixturevalue(name) + except pytest.FixtureLookupError: + missing.append(name) + assert not missing, ( + f"pytest did not register these fixtures: {', '.join(missing)}. " + "Check that tests/conftest.py is present and the @pytest.fixture " + "decorators are intact." + ) + + +def test_tests_run_against_checkout_not_stale_install(): + """Fail loudly when an old orisys in site-packages shadows src/orisys.""" + src_pkg = (REPO_ROOT / "src" / "orisys").resolve() + if not src_pkg.is_dir(): + pytest.skip("src/orisys is not present in this checkout layout") + + import orisys + + module_file = getattr(orisys, "__file__", None) + assert module_file, ( + "import orisys resolved to a namespace package (__file__ is None) " + f"instead of {src_pkg}. A stale orisys in site-packages is shadowing " + "the checkout; run `pip uninstall -y orisys` in this environment." + ) + + resolved_parent = Path(module_file).resolve().parent + assert resolved_parent == src_pkg, ( + f"import orisys resolved to {resolved_parent} instead of {src_pkg}. " + "A stale orisys in site-packages is shadowing the checkout; run " + "`pip uninstall -y orisys` in this environment." + ) diff --git a/tests/test_masked_hhd.py b/tests/test_masked_hhd.py index 7340d67..eb2da81 100644 --- a/tests/test_masked_hhd.py +++ b/tests/test_masked_hhd.py @@ -70,17 +70,22 @@ def test_sensor_detect_motion_ignores_invalid_region(): backend="cpu", ) try: + # get_img() resizes the frame to the config's camera.resolution_stitch + # and stores valid_mask at that same resolution, so the flow buffer must + # be sized from sensor.valid_mask rather than from the raw frame. frame = np.zeros((728, 672, 3), dtype=np.uint8) mask = np.zeros((728, 672), dtype=bool) mask[180:540, 160:512] = True sensor.get_img(frame=frame, already_processed=True, valid_mask=mask) - sensor.flow = np.zeros((728, 672, 2), dtype=np.float32) - sensor.flow[~mask] = 10.0 + flow_shape = sensor.valid_mask.shape + assert sensor.valid_mask.any() and not sensor.valid_mask.all() + sensor.flow = np.zeros((*flow_shape, 2), dtype=np.float32) + sensor.flow[~sensor.valid_mask] = 10.0 assert not sensor.detect_motion(threshold=1.0) sensor.flow[:] = 0.0 - sensor.flow[mask] = 10.0 + sensor.flow[sensor.valid_mask] = 10.0 assert sensor.detect_motion(threshold=1.0) finally: sensor.disconnect() diff --git a/tests/test_pyproject.py b/tests/test_pyproject.py index 56d8fe5..5f8cc47 100644 --- a/tests/test_pyproject.py +++ b/tests/test_pyproject.py @@ -35,7 +35,7 @@ def test_pyproject_dynamic_version(packaging_utils): sys.path.insert(0, str(build_dir)) from common.pyproject_utils import get_version - assert get_version(REPO_ROOT) == "0.5.1" + assert get_version(REPO_ROOT) == "0.5.2" def test_pyproject_bundle_manifest(packaging_utils): diff --git a/tests/test_release_version.py b/tests/test_release_version.py new file mode 100644 index 0000000..b4456d7 --- /dev/null +++ b/tests/test_release_version.py @@ -0,0 +1,291 @@ +"""Tests for build/release version guards, report writing and preflight steps.""" + +from __future__ import annotations + +import io +import json +import sys +import urllib.error +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +BUILD_ROOT = REPO_ROOT / "build" +if str(BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(BUILD_ROOT)) + +from release import report as report_mod # noqa: E402 +from release import version as version_mod # noqa: E402 +from release.preflight import Preflight, PreflightOptions # noqa: E402 + +PYPROJECT = """ +[project] +name = "orisys" +description = "synthetic" + +[tool.setuptools.dynamic] +version = {attr = "orisys.__version__"} +""".lstrip() + + +def _make_repo(tmp_path: Path, *, version: str = "0.5.2", with_git: bool = False) -> Path: + (tmp_path / "pyproject.toml").write_text(PYPROJECT, encoding="utf-8") + package = tmp_path / "src" / "orisys" + package.mkdir(parents=True) + (package / "__init__.py").write_text(f'__version__ = "{version}"\n', encoding="utf-8") + if with_git: + (tmp_path / ".git").mkdir() + return tmp_path + + +def _stage_wheel(repo: Path, version: str, name: str | None = None) -> Path: + directory = repo / "SDK" / version + directory.mkdir(parents=True, exist_ok=True) + wheel = directory / (name or f"orisys-{version}-cp310-cp310-win_amd64.whl") + wheel.write_bytes(b"not really a wheel") + return wheel + + +class _FakeResponse: + def __init__(self, payload: dict) -> None: + self._buffer = io.BytesIO(json.dumps(payload).encode("utf-8")) + + def read(self, *args): + return self._buffer.read(*args) + + def __enter__(self): + return self + + def __exit__(self, *exc) -> bool: + return False + + +def test_wheel_version_parsing(): + assert version_mod.wheel_version(Path("orisys-0.5.2-cp310-cp310-win_amd64.whl")) == "0.5.2" + assert version_mod.wheel_version(Path("orisys-1.0.0-1-cp312-abi3-manylinux_2_28_x86_64.whl")) == "1.0.0" + assert version_mod.wheel_version(Path("something-else.whl")) is None + + +def test_sdk_versions_are_numerically_sorted(tmp_path): + for name in ("0.9.0", "0.4.10", "0.5.2"): + (tmp_path / "SDK" / name).mkdir(parents=True) + (tmp_path / "SDK" / "not-a-version").mkdir() + assert version_mod.sdk_versions(tmp_path)[:3] == ["0.4.10", "0.5.2", "0.9.0"] + + +def test_sdk_versions_empty_without_sdk_dir(tmp_path): + assert version_mod.sdk_versions(tmp_path) == [] + + +def test_sdk_wheels_lists_only_that_version(tmp_path): + repo = _make_repo(tmp_path) + _stage_wheel(repo, "0.5.2") + _stage_wheel(repo, "0.4.3") + wheels = version_mod.sdk_wheels(repo, "0.5.2") + assert [w.name for w in wheels] == ["orisys-0.5.2-cp310-cp310-win_amd64.whl"] + + +def test_dynamic_version_attr_read_from_pyproject(tmp_path): + repo = _make_repo(tmp_path) + assert version_mod.dynamic_version_attr(repo) == "orisys.__version__" + + +def test_dynamic_version_attr_none_when_absent(tmp_path): + (tmp_path / "pyproject.toml").write_text("[project]\nname = 'x'\n", encoding="utf-8") + assert version_mod.dynamic_version_attr(tmp_path) is None + + +def test_pypi_released_versions_parses_response(tmp_path, monkeypatch): + fake = _FakeResponse({"releases": {"0.4.2": [{"filename": "a"}], "0.5.2": [{"filename": "b"}], "0.0.0": []}}) + monkeypatch.setattr(version_mod.urllib.request, "urlopen", lambda *a, **k: fake) + versions, error = version_mod.pypi_released_versions() + assert error is None + # An empty file list means nothing was ever uploaded for that version. + assert versions == ["0.4.2", "0.5.2"] + + +def test_pypi_released_versions_reports_http_404_as_empty(monkeypatch): + def _raise(*args, **kwargs): + raise urllib.error.HTTPError(version_mod.PYPI_JSON_URL, 404, "not found", {}, None) + + monkeypatch.setattr(version_mod.urllib.request, "urlopen", _raise) + assert version_mod.pypi_released_versions() == ([], None) + + +def test_pypi_released_versions_returns_error_on_failure(monkeypatch): + def _raise(*args, **kwargs): + raise urllib.error.URLError("no network") + + monkeypatch.setattr(version_mod.urllib.request, "urlopen", _raise) + versions, error = version_mod.pypi_released_versions() + assert versions is None + assert error and "URLError" in error + + +def test_python_for_tag_uses_envs_root(tmp_path): + executable = tmp_path / "py310" / "python.exe" + executable.parent.mkdir(parents=True) + executable.write_bytes(b"") + assert version_mod.python_for_tag("cp310", envs_root=tmp_path) == executable + assert version_mod.python_for_tag("cp311", envs_root=tmp_path) is None + + +# --- report ----------------------------------------------------------------- + + +def test_write_report_round_trips(tmp_path): + payload = { + "version": "0.5.2", + "ok": False, + "git": {"commit": "abc", "branch": "dev", "dirty": True}, + "interpreter": {"version": "3.10.19", "executable": "python"}, + "duration_s": 1.5, + "options": {"skip_tests": True}, + "steps": [{"name": "tests", "ok": True, "detail": "skipped", "duration_s": 0.0, "data": {}}], + "wheels": [{"name": "w.whl", "size": 12, "sha256": "deadbeef"}], + } + json_path, text_path = report_mod.write_report(payload, tmp_path / "reports") + loaded = json.loads(json_path.read_text(encoding="utf-8")) + assert loaded["schema"] == report_mod.SCHEMA_VERSION + assert loaded["wheels"][0]["sha256"] == "deadbeef" + + text = text_path.read_text(encoding="utf-8") + assert "FAIL" in text + assert "deadbeef" in text + assert "waived guards: skip_tests" in text + + +# --- preflight steps -------------------------------------------------------- + + +def test_preflight_resolves_version_from_pyproject(tmp_path): + repo = _make_repo(tmp_path, version="0.6.0") + preflight = Preflight(PreflightOptions(), repo_root=repo) + assert preflight.version == "0.6.0" + assert preflight.version_source == "pyproject.toml dynamic attr" + + +def test_preflight_explicit_version_wins(tmp_path): + repo = _make_repo(tmp_path, version="0.6.0") + preflight = Preflight(PreflightOptions(version="0.7.0"), repo_root=repo) + assert preflight.version == "0.7.0" + assert preflight.version_source == "explicit --version" + + +def test_repo_state_fails_outside_git(tmp_path, capsys): + repo = _make_repo(tmp_path) + step = Preflight(PreflightOptions(), repo_root=repo)._repo_state() + assert not step.ok + assert "not a git repository" in step.detail + + +def test_version_consistency_detects_mismatch(tmp_path): + repo = _make_repo(tmp_path, version="0.5.2") + step = Preflight(PreflightOptions(version="9.9.9"), repo_root=repo)._version_consistency() + assert not step.ok + assert "0.5.2" in step.detail and "9.9.9" in step.detail + + +def test_version_consistency_passes_on_match(tmp_path): + repo = _make_repo(tmp_path, version="0.5.2") + step = Preflight(PreflightOptions(version="0.5.2"), repo_root=repo)._version_consistency() + assert step.ok, step.detail + + +def test_duplicate_guard_blocks_existing_sdk_dir(tmp_path): + repo = _make_repo(tmp_path, version="0.5.2") + _stage_wheel(repo, "0.5.2") + step = Preflight(PreflightOptions(version="0.5.2"), repo_root=repo)._duplicate_guard() + assert not step.ok + assert "--force" in step.detail + + +def test_duplicate_guard_allows_force(tmp_path): + repo = _make_repo(tmp_path, version="0.5.2") + _stage_wheel(repo, "0.5.2") + step = Preflight( + PreflightOptions(version="0.5.2", force=True), repo_root=repo + )._duplicate_guard() + assert step.ok, step.detail + + +def test_duplicate_guard_flags_published_version(tmp_path, monkeypatch): + repo = _make_repo(tmp_path, version="0.5.2") + monkeypatch.setattr(version_mod, "pypi_released_versions", lambda *a, **k: (["0.5.2"], None)) + step = Preflight( + PreflightOptions(version="0.5.2", check_pypi=True), repo_root=repo + )._duplicate_guard() + assert not step.ok + assert "already published on PyPI" in step.detail + + +def test_bundle_precheck_fails_when_pattern_resolves_nothing(tmp_path): + repo = _make_repo(tmp_path, version="0.5.2") + (repo / "pyproject.toml").write_text( + PYPROJECT + + """ +[[tool.orisys.bundle.items]] +name = "configs" +source = "config" +target = "configs" +patterns = ["*.json"] +""", + encoding="utf-8", + ) + step = Preflight(PreflightOptions(version="0.5.2"), repo_root=repo)._bundle_precheck() + assert not step.ok + assert "resolved 0 files" in step.detail + + +def test_wheel_inspection_fails_without_wheels(tmp_path): + repo = _make_repo(tmp_path, version="0.5.2") + step = Preflight(PreflightOptions(version="0.5.2"), repo_root=repo)._wheel_inspection() + assert not step.ok + assert "does not exist" in step.detail + + +def test_wheel_runtime_fails_without_video(tmp_path, monkeypatch): + repo = _make_repo(tmp_path, version="0.5.2") + _stage_wheel(repo, "0.5.2") + monkeypatch.setattr("release.preflight.default_verification_video", lambda root: None) + preflight = Preflight(PreflightOptions(version="0.5.2"), repo_root=repo) + preflight._discover_wheels() + step = preflight._wheel_runtime() + assert not step.ok + assert "never silently skipped" in step.detail + + +def test_wheel_runtime_skip_is_recorded(tmp_path): + repo = _make_repo(tmp_path, version="0.5.2") + step = Preflight( + PreflightOptions(version="0.5.2", no_runtime=True), repo_root=repo + )._wheel_runtime() + assert step.ok + assert "SKIPPED" in step.detail + + +def test_tests_skip_is_loud_and_recorded(tmp_path): + repo = _make_repo(tmp_path, version="0.5.2") + step = Preflight( + PreflightOptions(version="0.5.2", skip_tests=True), repo_root=repo + )._tests() + assert step.ok + assert "SKIPPED" in step.detail + assert step.data["status"] == "skipped" + + +def test_execute_marks_failure_and_aborts(tmp_path, capsys): + repo = _make_repo(tmp_path, version="0.5.2") + preflight = Preflight(PreflightOptions(version="0.5.2"), repo_root=repo) + preflight._execute(preflight._repo_state) + assert preflight.aborted + assert preflight.steps[-1].name == "repo-state" + assert not preflight.steps[-1].ok + + +def test_execute_keep_going_does_not_abort(tmp_path): + repo = _make_repo(tmp_path, version="0.5.2") + preflight = Preflight(PreflightOptions(version="0.5.2", keep_going=True), repo_root=repo) + preflight._execute(preflight._repo_state) + assert not preflight.aborted diff --git a/tests/test_wheel_build.py b/tests/test_wheel_build.py index 28b637e..fec6590 100644 --- a/tests/test_wheel_build.py +++ b/tests/test_wheel_build.py @@ -19,25 +19,25 @@ def wheel_package(): def test_compliant_manylinux_tags(wheel_package): assert wheel_package.is_compliant_manylinux_wheel( - Path("orisys-0.5.1-cp310-cp310-manylinux_2_17_x86_64.whl") + Path("orisys-0.5.2-cp310-cp310-manylinux_2_17_x86_64.whl") ) assert wheel_package.is_compliant_manylinux_wheel( - Path("orisys-0.5.1-cp310-cp310-manylinux2014_x86_64.whl") + Path("orisys-0.5.2-cp310-cp310-manylinux2014_x86_64.whl") ) assert wheel_package.is_compliant_manylinux_wheel( - Path("orisys-0.5.1-cp310-cp310-manylinux_2_28_x86_64.whl") + Path("orisys-0.5.2-cp310-cp310-manylinux_2_28_x86_64.whl") ) def test_rejects_illegal_manylinux_tags(wheel_package): assert not wheel_package.is_compliant_manylinux_wheel( - Path("orisys-0.5.1-cp310-cp310-manylinux_x86_64.whl") + Path("orisys-0.5.2-cp310-cp310-manylinux_x86_64.whl") ) assert not wheel_package.is_compliant_manylinux_wheel( - Path("orisys-0.5.1-cp310-cp310-linux_x86_64.whl") + Path("orisys-0.5.2-cp310-cp310-linux_x86_64.whl") ) assert not wheel_package.is_compliant_manylinux_wheel( - Path("orisys-0.5.1-cp310-cp310-win_amd64.whl") + Path("orisys-0.5.2-cp310-cp310-win_amd64.whl") ) diff --git a/tests/test_wheel_contents.py b/tests/test_wheel_contents.py new file mode 100644 index 0000000..473185c --- /dev/null +++ b/tests/test_wheel_contents.py @@ -0,0 +1,378 @@ +"""Tests for :mod:`common.verify.wheel_contents`. + +Every check is exercised in both directions: a well-formed synthetic wheel must +pass, and each specific defect must fail with the check named in the report. +Synthetic wheels keep the tests fast and keep them independent of whatever +happens to sit in ``SDK/``. +""" + +from __future__ import annotations + +import base64 +import hashlib +import sys +import zipfile +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +BUILD_ROOT = REPO_ROOT / "build" +if str(BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(BUILD_ROOT)) + +from common.verify import wheel_contents # noqa: E402 +from common.verify.wheel_contents import inspect_wheel # noqa: E402 + +VERSION = "0.5.2" +DIST_INFO = f"orisys-{VERSION}.dist-info" + +CORE_DEPS = ("numpy>=1.24.4", "opencv-python>=4.10.0.84") +EXTRAS = {"cuda12": ("cupy-cuda12x>=12.3.0",), "gui": ("open3d>=0.17.0",)} + +PYPROJECT = f""" +[project] +name = "orisys" +version = "{VERSION}" +requires-python = ">=3.10" +dependencies = [{", ".join(repr(dep) for dep in CORE_DEPS)}] + +[project.optional-dependencies] +cuda12 = ["cupy-cuda12x>=12.3.0"] +gui = ["open3d>=0.17.0"] + +[[tool.orisys.bundle.items]] +name = "configs" +source = "config" +target = "configs" +patterns = ["*.json"] + +[[tool.orisys.bundle.items]] +name = "sensors" +source = "sensors" +target = "sensors" +patterns = ["**/*.json"] +preserve_tree = true + +[[tool.orisys.bundle.items]] +name = "docs" +source = "docs" +target = "docs" +patterns = ["*.md"] + +[[tool.orisys.bundle.items]] +name = "examples" +source = "examples" +target = "examples" +patterns = ["*.py", "README.md"] +preserve_tree = true +""" + +# Source tree files, keyed by path relative to the synthetic repo root. +SOURCE_FILES = { + "config/box.json": '{"name": "box"}', + "sensors/box/sensor.json": '{"name": "box"}', + "sensors/finger/manifest.json": '{"name": "finger"}', + "docs/api.md": "# api\n", + "examples/basic_pipeline.py": "print('hi')\n", + "examples/README.md": "# examples\n", +} + +# Where each source file must land inside the wheel. +BUNDLED_ENTRIES = { + "orisys/configs/box.json": SOURCE_FILES["config/box.json"], + "orisys/sensors/box/sensor.json": SOURCE_FILES["sensors/box/sensor.json"], + "orisys/sensors/finger/manifest.json": SOURCE_FILES["sensors/finger/manifest.json"], + "orisys/docs/api.md": SOURCE_FILES["docs/api.md"], + "orisys/examples/basic_pipeline.py": SOURCE_FILES["examples/basic_pipeline.py"], + "orisys/examples/README.md": SOURCE_FILES["examples/README.md"], +} + + +def _metadata_text( + *, + version: str = VERSION, + requires_python: str | None = ">=3.10", + classifiers: tuple[str, ...] = ("Programming Language :: Python :: 3.10",), + project_urls: tuple[str, ...] = ("Homepage, https://example.invalid",), + license_line: str | None = "License: Proprietary", + core_deps: tuple[str, ...] = CORE_DEPS, + extras: dict[str, tuple[str, ...]] | None = None, + dynamic: tuple[str, ...] = (), + name: str = "orisys", +) -> str: + if extras is None: + extras = EXTRAS + lines = ["Metadata-Version: 2.4", f"Name: {name}", f"Version: {version}", "Summary: synthetic"] + if requires_python: + lines.append(f"Requires-Python: {requires_python}") + if license_line: + lines.append(license_line) + lines.extend(f"Classifier: {value}" for value in classifiers) + lines.extend(f"Project-URL: {value}" for value in project_urls) + lines.extend(f"Dynamic: {value}" for value in dynamic) + lines.extend(f"Requires-Dist: {dep}" for dep in core_deps) + for extra, requirements in extras.items(): + lines.append(f"Provides-Extra: {extra}") + lines.extend(f'Requires-Dist: {req}; extra == "{extra}"' for req in requirements) + return "\n".join(lines) + "\n" + + +def _write_source_tree(root: Path) -> None: + root.mkdir(parents=True, exist_ok=True) + (root / "pyproject.toml").write_text(PYPROJECT, encoding="utf-8") + for relative, text in SOURCE_FILES.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + # write_bytes, not write_text: text mode would translate "\n" to "\r\n" + # on Windows and the byte-for-byte bundle comparison would fail. + path.write_bytes(text.encode("utf-8")) + + +def _record(entries: dict[str, bytes]) -> str: + lines = [] + for name, data in sorted(entries.items()): + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + lines.append(f"{name},sha256={digest},{len(data)}") + lines.append(f"{DIST_INFO}/RECORD,,") + return "\n".join(lines) + "\n" + + +def _make_wheel( + tmp_path: Path, + *, + filename: str | None = None, + metadata: str | None = None, + drop: tuple[str, ...] = (), + extra_entries: dict[str, bytes] | None = None, + override_bytes: dict[str, bytes] | None = None, + drop_record: bool = False, + corrupt_record_hash: bool = False, +) -> tuple[Path, Path]: + """Build a synthetic wheel plus its matching source tree.""" + repo = tmp_path / "repo" + _write_source_tree(repo) + + if metadata is None: + metadata = _metadata_text() + + entries: dict[str, bytes] = {name: data.encode("utf-8") for name, data in BUNDLED_ENTRIES.items()} + entries["orisys/__init__.cp312-win_amd64.pyd"] = b"compiled" + entries["orisys_blend/__init__.py"] = b"# blend\n" + entries["orisys_blend/kernel.py"] = b"# kernel\n" + entries[f"{DIST_INFO}/METADATA"] = metadata.encode("utf-8") + entries[f"{DIST_INFO}/WHEEL"] = b"Wheel-Version: 1.0\n" + + entries.update(extra_entries or {}) + entries.update(override_bytes or {}) + for name in drop: + entries.pop(name, None) + + if not drop_record: + record = _record(entries) + if corrupt_record_hash: + record = record.replace("sha256=", "sha256=AAAA", 1) + entries[f"{DIST_INFO}/RECORD"] = record.encode("utf-8") + + wheel = tmp_path / (filename or f"orisys-{VERSION}-cp312-cp312-win_amd64.whl") + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as zf: + for name, data in entries.items(): + zf.writestr(name, data) + return wheel, repo + + +def _failed_names(result) -> set[str]: + return {check.name for check in result.failures} + + +@pytest.fixture(autouse=True) +def _isolated_repo_root(monkeypatch, tmp_path): + """Keep inspect_wheel's default repo root from leaking the real repo in.""" + monkeypatch.setattr(wheel_contents, "REPO_ROOT", tmp_path / "unused") + + +def test_good_wheel_passes(tmp_path): + wheel, repo = _make_wheel(tmp_path) + result = inspect_wheel(wheel, expected_version=VERSION, repo_root=repo) + assert result.ok, result.render() + assert result.version == VERSION + assert result.platform_tag == "win_amd64" + assert not result.warnings, result.warnings + + +def test_missing_wheel_file_fails(tmp_path): + result = inspect_wheel(tmp_path / "nope-0.5.2-cp312-cp312-win_amd64.whl") + assert not result.ok + assert "wheel-exists" in _failed_names(result) + + +def test_unparsable_filename_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path, filename="orisys-0.5.2-cp312-cp312.whl") + result = inspect_wheel(wheel, repo_root=repo) + assert "filename" in _failed_names(result) + + +def test_version_mismatch_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path) + result = inspect_wheel(wheel, expected_version="9.9.9", repo_root=repo) + failed = _failed_names(result) + assert {"filename", "metadata-version"} <= failed + + +def test_missing_requires_python_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path, metadata=_metadata_text(requires_python=None)) + assert "metadata-requires-python" in _failed_names(inspect_wheel(wheel, repo_root=repo)) + + +def test_missing_classifier_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path, metadata=_metadata_text(classifiers=())) + assert "metadata-classifiers" in _failed_names(inspect_wheel(wheel, repo_root=repo)) + + +def test_missing_project_url_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path, metadata=_metadata_text(project_urls=())) + assert "metadata-project-urls" in _failed_names(inspect_wheel(wheel, repo_root=repo)) + + +def test_missing_license_fails(tmp_path): + wheel, repo = _make_wheel( + tmp_path, + metadata=_metadata_text(license_line=None, classifiers=("Intended Audience :: Developers",)), + ) + assert "metadata-license" in _failed_names(inspect_wheel(wheel, repo_root=repo)) + + +def test_dynamic_version_is_rejected(tmp_path): + wheel, repo = _make_wheel(tmp_path, metadata=_metadata_text(dynamic=("version", "summary"))) + result = inspect_wheel(wheel, repo_root=repo) + assert "metadata-no-dynamic-version" in _failed_names(result) + assert any("Dynamic" in warning for warning in result.warnings) + + +def test_missing_core_requirement_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path, metadata=_metadata_text(core_deps=("numpy>=1.24.4",))) + assert "metadata-requires-dist" in _failed_names(inspect_wheel(wheel, repo_root=repo)) + + +def test_missing_extra_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path, metadata=_metadata_text(extras={"cuda12": EXTRAS["cuda12"]})) + assert "metadata-extras" in _failed_names(inspect_wheel(wheel, repo_root=repo)) + + +def test_missing_compiled_extension_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path, drop=("orisys/__init__.cp312-win_amd64.pyd",)) + assert "compiled-extension" in _failed_names(inspect_wheel(wheel, repo_root=repo)) + + +def test_python_source_leak_fails(tmp_path): + wheel, repo = _make_wheel( + tmp_path, extra_entries={"orisys/sensor.py": b"# leaked\n"} + ) + result = inspect_wheel(wheel, repo_root=repo) + assert "no-python-sources" in _failed_names(result) + assert "orisys/sensor.py" in result.render() + + +def test_example_python_files_are_allowed(tmp_path): + wheel, repo = _make_wheel( + tmp_path, extra_entries={"orisys/examples/plot_data.py": b"# shipped on purpose\n"} + ) + assert inspect_wheel(wheel, repo_root=repo).ok + + +def test_missing_bundled_file_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path, drop=("orisys/sensors/box/sensor.json",)) + result = inspect_wheel(wheel, repo_root=repo) + assert "bundle-completeness" in _failed_names(result) + assert "orisys/sensors/box/sensor.json" in result.render() + + +def test_missing_bundled_file_warns_when_not_strict(tmp_path): + """A file added to the tree after an old wheel was built is not a defect.""" + wheel, repo = _make_wheel(tmp_path, drop=("orisys/sensors/box/sensor.json",)) + result = inspect_wheel(wheel, repo_root=repo, strict_bundle=False) + assert result.ok, result.render() + assert any("absent from the wheel" in warning for warning in result.warnings) + + +def test_bundle_drift_is_fatal_when_strict(tmp_path): + wheel, repo = _make_wheel( + tmp_path, override_bytes={"orisys/docs/api.md": b"# stale\n"} + ) + assert "bundle-fidelity" in _failed_names(inspect_wheel(wheel, repo_root=repo, strict_bundle=True)) + + +def test_bundle_drift_warns_when_not_strict(tmp_path): + wheel, repo = _make_wheel( + tmp_path, override_bytes={"orisys/docs/api.md": b"# stale\n"} + ) + result = inspect_wheel(wheel, repo_root=repo, strict_bundle=False) + assert result.ok, result.render() + assert any("differ from the source tree" in warning for warning in result.warnings) + + +def test_pycache_entry_fails(tmp_path): + wheel, repo = _make_wheel( + tmp_path, extra_entries={"orisys/examples/__pycache__/x.pyc": b"junk"} + ) + assert "no-forbidden-entries" in _failed_names(inspect_wheel(wheel, repo_root=repo)) + + +def test_scratch_directory_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path, extra_entries={"tests/test_smoke.py": b"x"}) + assert "no-forbidden-entries" in _failed_names(inspect_wheel(wheel, repo_root=repo)) + + +def test_traversing_entry_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path, extra_entries={"../escape.txt": b"x"}) + assert "archive-paths" in _failed_names(inspect_wheel(wheel, repo_root=repo)) + + +def test_record_hash_mismatch_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path, corrupt_record_hash=True) + assert "record" in _failed_names(inspect_wheel(wheel, repo_root=repo)) + + +def test_missing_record_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path, drop_record=True) + assert "record" in _failed_names(inspect_wheel(wheel, repo_root=repo)) + + +def test_size_limit_fails(tmp_path): + wheel, repo = _make_wheel(tmp_path) + assert "size-limit" in _failed_names(inspect_wheel(wheel, repo_root=repo, max_bytes=1)) + + +def test_bad_manylinux_tag_fails(tmp_path): + wheel, repo = _make_wheel( + tmp_path, filename=f"orisys-{VERSION}-cp312-cp312-linux_x86_64.whl" + ) + assert "platform-tag" in _failed_names(inspect_wheel(wheel, repo_root=repo)) + + +def test_good_manylinux_tag_passes(tmp_path): + wheel, repo = _make_wheel( + tmp_path, + filename=f"orisys-{VERSION}-cp312-cp312-manylinux_2_28_x86_64.whl", + ) + assert inspect_wheel(wheel, repo_root=repo).ok + + +def test_wrong_distribution_name_fails(tmp_path): + wheel, repo = _make_wheel( + tmp_path, + filename=f"notorisys-{VERSION}-cp312-cp312-win_amd64.whl", + metadata=_metadata_text(name="notorisys"), + ) + failed = _failed_names(inspect_wheel(wheel, repo_root=repo)) + assert {"filename", "metadata-name"} <= failed + + +def test_wheel_layout_module_resolves_to_build_tree(): + import wheel + + resolved = Path(wheel.__file__).resolve() + assert BUILD_ROOT.resolve() in resolved.parents, ( + f"`wheel` resolved to {resolved}; build/ must precede site-packages so the " + "build tooling imports the repository's wheel package." + ) diff --git a/tests/test_wheel_setup_metadata.py b/tests/test_wheel_setup_metadata.py new file mode 100644 index 0000000..ab341e5 --- /dev/null +++ b/tests/test_wheel_setup_metadata.py @@ -0,0 +1,172 @@ +"""Tests for build/wheel/package.py metadata plumbing. + +The Nuitka wheel is built from a generated ``setup.py``, which is exactly where +the 0.5.2 release silently lost ``Requires-Python``, classifiers, project URLs +and the license. These tests run setuptools for real (in a temp dir, on a stub +package tree) and assert on the METADATA it produces. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +BUILD_ROOT = REPO_ROOT / "build" +if str(BUILD_ROOT) not in sys.path: + sys.path.insert(0, str(BUILD_ROOT)) + +from wheel.package import metadata_kwargs, read_long_description, write_wheel_setup # noqa: E402 + +PROJECT = { + "name": "orisys", + "description": "SDK for Orisys Technology Visuotactile Sensors", + "requires-python": ">=3.10", + "license": {"text": "Proprietary"}, + "authors": [{"name": "Orisys-Tech"}], + "maintainers": [{"name": "Yijian Zhang", "email": "zhangyijian@orisys-tech.cn"}], + "dependencies": ["numpy>=1.24.4"], + "optional-dependencies": {"gui": ["open3d>=0.17.0"]}, + "classifiers": [ + "Development Status :: 4 - Beta", + "License :: Other/Proprietary License", + ], + "urls": { + "Homepage": "https://www.orisys-tech.cn", + "Documentation": "https://github.com/Orisys-Tech/orisys-sdk", + }, +} + +LONG_DESCRIPTION = "# Orisys SDK\n\nlong description body.\n" + + +def _stub_wheel_root(wheel_dir: Path) -> None: + """Minimal package tree so the generated setup.py can build metadata.""" + (wheel_dir / "wheel_root" / "orisys").mkdir(parents=True) + (wheel_dir / "wheel_root" / "orisys" / "__init__.cp312-win_amd64.pyd").write_bytes(b"x") + (wheel_dir / "wheel_root" / "orisys" / "examples").mkdir() + (wheel_dir / "wheel_root" / "orisys" / "examples" / "__init__.py").write_text("", encoding="utf-8") + (wheel_dir / "wheel_root" / "orisys_blend").mkdir() + (wheel_dir / "wheel_root" / "orisys_blend" / "__init__.py").write_text("", encoding="utf-8") + + +def _generate(wheel_dir: Path, tmp_path: Path) -> Path: + _stub_wheel_root(wheel_dir) + write_wheel_setup( + wheel_dir, + version="0.5.2", + package_data={"orisys": ["*.pyd"]}, + install_requires=["numpy>=1.24.4"], + extras_require={"gui": ["open3d>=0.17.0"]}, + project=PROJECT, + long_description=LONG_DESCRIPTION, + ) + + completed = subprocess.run( + [sys.executable, "setup.py", "-q", "dist_info"], + cwd=wheel_dir, + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0, ( + f"generated setup.py failed:\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + + candidates = sorted(wheel_dir.glob("**/*.dist-info/METADATA")) + assert candidates, f"no METADATA produced under {wheel_dir}" + return candidates[0] + + +def test_metadata_kwargs_translates_pyproject(): + kwargs = metadata_kwargs(PROJECT) + assert kwargs["description"] == PROJECT["description"] + assert kwargs["python_requires"] == ">=3.10" + assert kwargs["license"] == "Proprietary" + assert kwargs["classifiers"] == PROJECT["classifiers"] + assert kwargs["project_urls"] == PROJECT["urls"] + assert kwargs["author"] == "Orisys-Tech" + assert kwargs["maintainer"] == "Yijian Zhang " + + +def test_metadata_kwargs_accepts_string_license_and_drops_empties(): + kwargs = metadata_kwargs({"name": "orisys", "license": "Proprietary"}) + assert kwargs["license"] == "Proprietary" + assert "python_requires" not in kwargs + assert "classifiers" not in kwargs + + +def test_read_long_description_uses_pypi_readme(tmp_path): + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "pypi_readme.md").write_text(LONG_DESCRIPTION, encoding="utf-8") + assert read_long_description(tmp_path) == LONG_DESCRIPTION + + +def test_read_long_description_missing_returns_none(tmp_path, capsys): + assert read_long_description(tmp_path) is None + assert "not found" in capsys.readouterr().out + + +def test_generated_wheel_metadata_is_complete(tmp_path): + wheel_dir = tmp_path / "wheel" + wheel_dir.mkdir() + metadata_path = _generate(wheel_dir, tmp_path) + text = metadata_path.read_text(encoding="utf-8") + + assert "Name: orisys" in text + assert "Version: 0.5.2" in text + assert f"Summary: {PROJECT['description']}" in text + assert "Requires-Python: >=3.10" in text + assert "License: Proprietary" in text + assert "Classifier: Development Status :: 4 - Beta" in text + assert "Classifier: License :: Other/Proprietary License" in text + assert "Project-URL: Homepage, https://www.orisys-tech.cn" in text + assert "Project-URL: Documentation, https://github.com/Orisys-Tech/orisys-sdk" in text + assert "Requires-Dist: numpy>=1.24.4" in text + assert 'Provides-Extra: gui' in text + assert "Description-Content-Type: text/markdown" in text + assert "long description body." in text + + +def test_generated_wheel_metadata_keeps_version_static(tmp_path): + """``Version`` (and ``Name``) must never be annotated as dynamic. + + setuptools marks every field supplied through the legacy ``setup()`` path as + ``Dynamic:`` even when the value is present and correct (verified by + ``test_generated_wheel_metadata_is_complete``). PEP 643 ``Dynamic`` is + sdist-only and pip/PyPI ignore it in wheels, so only the identity fields are + asserted here. + """ + wheel_dir = tmp_path / "wheel" + wheel_dir.mkdir() + text = _generate(wheel_dir, tmp_path).read_text(encoding="utf-8") + dynamic = {line[len("Dynamic: ") :].strip() for line in text.splitlines() if line.startswith("Dynamic:")} + assert "version" not in dynamic, f"wheel METADATA marks Version as dynamic: {sorted(dynamic)}" + assert "name" not in dynamic, f"wheel METADATA marks Name as dynamic: {sorted(dynamic)}" + + +def test_write_wheel_setup_without_project_keeps_legacy_shape(tmp_path): + """Calling without metadata must not crash (backwards compatible).""" + wheel_dir = tmp_path / "wheel" + wheel_dir.mkdir() + setup_py = write_wheel_setup( + wheel_dir, + version="0.0.1", + package_data={"orisys": ["*.pyd"]}, + install_requires=["numpy"], + extras_require={}, + ) + text = setup_py.read_text(encoding="utf-8") + assert 'name="orisys"' in text + assert 'version="0.0.1"' in text + assert "long_description" not in text + + +def test_pypi_readme_exists_in_repo(): + assert (REPO_ROOT / "docs" / "pypi_readme.md").is_file(), ( + "docs/pypi_readme.md is the PyPI long description source; without it " + "wheels are published with no project description." + )