added preflight check
This commit is contained in:
ykcheungab
@@ -1,6 +1,19 @@
|
||||
"""Post-build smoke tests for wheels and product GUI folders."""
|
||||
|
||||
from common.verify.gui import GUI_PRODUCTS, verify_gui_dist
|
||||
from common.verify.wheel import verify_built_wheel
|
||||
from common.verify.wheel import (
|
||||
verify_built_wheel,
|
||||
verify_wheel_assets,
|
||||
verify_wheel_runtime,
|
||||
)
|
||||
from common.verify.wheel_contents import WheelCheckResult, inspect_wheel
|
||||
|
||||
__all__ = ["GUI_PRODUCTS", "verify_built_wheel", "verify_gui_dist"]
|
||||
__all__ = [
|
||||
"GUI_PRODUCTS",
|
||||
"WheelCheckResult",
|
||||
"inspect_wheel",
|
||||
"verify_built_wheel",
|
||||
"verify_gui_dist",
|
||||
"verify_wheel_assets",
|
||||
"verify_wheel_runtime",
|
||||
]
|
||||
|
||||
@@ -3,9 +3,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from common.verify import GUI_PRODUCTS, verify_built_wheel, verify_gui_dist
|
||||
_BUILD_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(_BUILD_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_BUILD_ROOT))
|
||||
|
||||
from common.paths import REPO_ROOT # noqa: E402
|
||||
from common.verify import ( # noqa: E402
|
||||
GUI_PRODUCTS,
|
||||
inspect_wheel,
|
||||
verify_built_wheel,
|
||||
verify_gui_dist,
|
||||
verify_wheel_assets,
|
||||
verify_wheel_runtime,
|
||||
)
|
||||
from common.verify.wheel import DEFAULT_RUNTIME_CONFIG, default_verification_video # noqa: E402
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
@@ -18,6 +32,31 @@ def main(argv: list[str] | None = None) -> int:
|
||||
help="GUI product profile (default: ory_finger)",
|
||||
)
|
||||
parser.add_argument("--dist-dir", type=Path, default=None, help="Override GUI dist folder")
|
||||
parser.add_argument(
|
||||
"--contents",
|
||||
action="store_true",
|
||||
help="Run static metadata/content assertions on the wheel",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--assets",
|
||||
action="store_true",
|
||||
help="Install the wheel in a temp venv and assert bundled assets load",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--runtime",
|
||||
action="store_true",
|
||||
help="Install the wheel and process real frames (no camera required)",
|
||||
)
|
||||
parser.add_argument("--expected-version", default=None, help="Assert the wheel version")
|
||||
parser.add_argument(
|
||||
"--strict-bundle",
|
||||
action="store_true",
|
||||
help="Treat bundled-file drift from the source tree as a failure",
|
||||
)
|
||||
parser.add_argument("--video", type=Path, default=None, help="Clip for --runtime")
|
||||
parser.add_argument("--config", default=DEFAULT_RUNTIME_CONFIG, help="Config for --runtime")
|
||||
parser.add_argument("--frames", type=int, default=40, help="Frames for --runtime")
|
||||
parser.add_argument("--offline", action="store_true", help="Reuse parent-env dependencies")
|
||||
parser.add_argument("wheel", nargs="?", type=Path, help="Wheel path for import smoke test")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
@@ -26,6 +65,49 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return 0
|
||||
if args.wheel is None:
|
||||
parser.error("Provide a wheel path or pass --gui-dist")
|
||||
|
||||
exit_code = 0
|
||||
|
||||
if args.contents:
|
||||
result = inspect_wheel(
|
||||
args.wheel,
|
||||
expected_version=args.expected_version,
|
||||
repo_root=REPO_ROOT,
|
||||
strict_bundle=args.strict_bundle,
|
||||
)
|
||||
print(result.render())
|
||||
if not result.ok:
|
||||
exit_code = 1
|
||||
|
||||
if args.assets or args.runtime:
|
||||
try:
|
||||
print(verify_wheel_assets(args.wheel))
|
||||
except Exception as exc:
|
||||
print(f"[FAIL] bundled assets: {type(exc).__name__}: {exc}")
|
||||
exit_code = 1
|
||||
|
||||
if args.runtime:
|
||||
video = args.video or default_verification_video(REPO_ROOT)
|
||||
if video is None:
|
||||
print("[FAIL] runtime: no clip found; pass --video or set ORISYS_VERIFY_VIDEO")
|
||||
return 1
|
||||
try:
|
||||
info = verify_wheel_runtime(
|
||||
args.wheel,
|
||||
video=video,
|
||||
config_name=args.config,
|
||||
frames=args.frames,
|
||||
offline=args.offline,
|
||||
)
|
||||
print(f"[ok ] runtime: {info}")
|
||||
except Exception as exc:
|
||||
print(f"[FAIL] runtime: {type(exc).__name__}: {exc}")
|
||||
exit_code = 1
|
||||
|
||||
if args.contents or args.assets or args.runtime:
|
||||
return exit_code
|
||||
|
||||
# Default behaviour is unchanged: import smoke test in an isolated venv.
|
||||
verify_built_wheel(args.wheel)
|
||||
return 0
|
||||
|
||||
|
||||
+342
-13
@@ -1,7 +1,21 @@
|
||||
"""Isolated venv smoke tests for packaged orisys wheels."""
|
||||
"""Isolated venv smoke tests for packaged orisys wheels.
|
||||
|
||||
Three levels of confidence, all in a throwaway virtualenv so the active
|
||||
environment is never touched:
|
||||
|
||||
* :func:`verify_built_wheel` - the public API imports (fast, original check);
|
||||
* :func:`verify_wheel_assets` - bundled configs/sensors resolve and load via
|
||||
``importlib.resources``;
|
||||
* :func:`verify_wheel_runtime` - the compiled wheel actually processes real
|
||||
frames through the public pipeline with no camera attached.
|
||||
|
||||
Every level can be bootstrapped from another interpreter so each supported
|
||||
CPython ABI can be verified against its own wheel.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -38,6 +52,167 @@ assert Stitcher is not None
|
||||
print(f"orisys {orisys.__version__} public API imports OK")
|
||||
"""
|
||||
|
||||
# Bundled data that must be resolvable and loadable from inside site-packages.
|
||||
# The wheel ships the compiled package, so these paths only exist if the bundle
|
||||
# step actually ran for this build.
|
||||
_ASSET_SCRIPT = r"""
|
||||
import sys
|
||||
from importlib.metadata import distribution
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import orisys
|
||||
|
||||
# Locate bundled data through the installed distribution, NOT through
|
||||
# orisys.__file__ / importlib.resources. Nuitka compiles the package into one
|
||||
# extension module whose __init__ installs nuitka_module_loader, which reports a
|
||||
# synthetic __file__ (e.g. ...\site-packages\orisys\orisys\__init__.py) and a
|
||||
# synthetic __path__ ending in '\not_existing'. Neither points at the real data
|
||||
# directory, so importlib.resources silently finds nothing in a Nuitka wheel.
|
||||
dist = distribution("orisys")
|
||||
dist_root = Path(dist.locate_file("")).resolve()
|
||||
package_root = Path(dist.locate_file("orisys")).resolve()
|
||||
|
||||
_diagnosis = (
|
||||
f"orisys distribution resolved to {dist_root} (sys.prefix={sys.prefix!r}, "
|
||||
f"sys.executable={sys.executable!r}), outside the verification venv. A stale "
|
||||
"orisys in the parent environment is visible through --system-site-packages; "
|
||||
"run `pip uninstall -y orisys` there, or re-run without --offline."
|
||||
)
|
||||
assert str(Path(sys.prefix).resolve()) in str(dist_root), _diagnosis
|
||||
assert package_root.is_dir(), f"package directory not found: {package_root}"
|
||||
|
||||
required = [
|
||||
"configs/box.json",
|
||||
"configs/finger.json",
|
||||
"sensors/box/force_calibration.json",
|
||||
"sensors/box/sensor.json",
|
||||
"sensors/finger/manifest.json",
|
||||
"sensors/finger/sensor.json",
|
||||
"sensors/mini/sensor.json",
|
||||
"sensors/finger/assets/30fps-0703/mesh.npz",
|
||||
"sensors/finger/assets/30fps-0703/camera_intrinsics.npz",
|
||||
"sensors/finger/assets/30fps-0703/world_to_cam.npy",
|
||||
"sensors/finger/assets/visualization/mesh.npz",
|
||||
"sensors/finger/assets/visualization/world_to_cam.npy",
|
||||
]
|
||||
|
||||
missing = [name for name in required if not (package_root / name).is_file()]
|
||||
assert not missing, (
|
||||
f"bundled assets missing from the wheel: {missing}\n"
|
||||
f"package root: {package_root}\ninstalled tree: "
|
||||
f"{sorted(p.name for p in package_root.iterdir()) if package_root.is_dir() else '<missing>'}"
|
||||
)
|
||||
|
||||
for name in required:
|
||||
path = package_root / name
|
||||
if name.endswith(".npz"):
|
||||
data = np.load(str(path))
|
||||
assert len(data.files) > 0, f"{name} is an empty npz"
|
||||
elif name.endswith(".npy"):
|
||||
arr = np.load(str(path))
|
||||
assert arr.size > 0, f"{name} is an empty npy"
|
||||
|
||||
configs = orisys.list_config()
|
||||
assert configs, "orisys.list_config() returned nothing"
|
||||
|
||||
print(f"bundled assets OK ({len(required)} files, {len(configs)} configs) "
|
||||
f"from {package_root}")
|
||||
"""
|
||||
|
||||
# End-to-end pipeline run against a video file: no camera, no GUI, no imshow.
|
||||
_RUNTIME_SCRIPT = r"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from importlib.metadata import distribution
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0")
|
||||
|
||||
import orisys
|
||||
|
||||
# Provenance via the installed distribution: a Nuitka-compiled package reports a
|
||||
# synthetic orisys.__file__ (see the asset check) and a stale orisys visible
|
||||
# through --system-site-packages would otherwise satisfy this check.
|
||||
dist = distribution("orisys")
|
||||
dist_root = Path(dist.locate_file("")).resolve()
|
||||
assert str(Path(sys.prefix).resolve()) in str(dist_root), (
|
||||
f"the installed orisys came from {dist_root}, outside the verification venv "
|
||||
f"{sys.prefix!r}; run `pip uninstall -y orisys` in the parent environment or "
|
||||
"re-run without --offline."
|
||||
)
|
||||
|
||||
video, config_name, frames = sys.argv[1], sys.argv[2], int(sys.argv[3])
|
||||
|
||||
sensor = orisys.Sensor(
|
||||
vid_src=video,
|
||||
config_name=config_name,
|
||||
backend="cpu",
|
||||
verbose=False,
|
||||
)
|
||||
processed = 0
|
||||
try:
|
||||
for _ in range(frames):
|
||||
img = sensor.get_img()
|
||||
assert img is not None, "get_img returned None"
|
||||
assert isinstance(img, np.ndarray) and img.ndim in (2, 3)
|
||||
|
||||
sensor.compute_deformation(check_motion=False)
|
||||
flow, vnormal, vshear = sensor.read_info(
|
||||
sensor.info.VRAW, sensor.info.VNORMAL, sensor.info.VSHEAR
|
||||
)
|
||||
for name, array in (("flow", flow), ("vnormal", vnormal), ("vshear", vshear)):
|
||||
assert np.isfinite(array).all(), f"{name} contains NaN/Inf"
|
||||
|
||||
contact = sensor.compute_contact()
|
||||
assert isinstance(contact, dict) and "centroid_found" in contact
|
||||
|
||||
fnormal, fsx, fsy = sensor.read_info(
|
||||
sensor.info.FNORMAL, sensor.info.FSHEARX, sensor.info.FSHEARY
|
||||
)
|
||||
assert np.isfinite([fnormal, fsx, fsy]).all(), "force output contains NaN/Inf"
|
||||
processed += 1
|
||||
finally:
|
||||
sensor.disconnect()
|
||||
|
||||
assert processed == frames, f"only processed {processed}/{frames} frames"
|
||||
print(json.dumps({
|
||||
"version": orisys.__version__,
|
||||
"frames": processed,
|
||||
"flow_shape": list(flow.shape),
|
||||
"video": video,
|
||||
"config": config_name,
|
||||
"package_root": str(Path(dist.locate_file("orisys")).resolve()),
|
||||
}))
|
||||
"""
|
||||
|
||||
|
||||
# Default runtime-verification inputs. Overridable by CLI flags or environment.
|
||||
DEFAULT_RUNTIME_CONFIG = os.getenv("ORISYS_VERIFY_CONFIG", "box")
|
||||
# Clips tracked in the repository, smallest first.
|
||||
DEFAULT_VIDEO_CANDIDATES = (
|
||||
Path("data") / "sample" / "s1_480p.mp4",
|
||||
Path("data") / "sample" / "s1_720p.mp4",
|
||||
Path("tests") / "idle_1min.mp4",
|
||||
)
|
||||
|
||||
|
||||
def default_verification_video(repo_root: Path) -> Path | None:
|
||||
"""Resolve the clip used for the end-to-end runtime check, if any."""
|
||||
env_video = os.getenv("ORISYS_VERIFY_VIDEO")
|
||||
repo_root = Path(repo_root)
|
||||
if env_video:
|
||||
candidate = Path(env_video)
|
||||
return candidate if candidate.is_absolute() else repo_root / candidate
|
||||
for relative in DEFAULT_VIDEO_CANDIDATES:
|
||||
candidate = repo_root / relative
|
||||
if candidate.is_file():
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _venv_python(venv_dir: Path) -> Path:
|
||||
if sys.platform == "win32":
|
||||
@@ -45,18 +220,172 @@ def _venv_python(venv_dir: Path) -> Path:
|
||||
return venv_dir / "bin" / "python"
|
||||
|
||||
|
||||
def verify_built_wheel(wheel_path: Path) -> None:
|
||||
"""Install the wheel in a temporary venv and smoke-test public imports."""
|
||||
wheel_path = wheel_path.resolve()
|
||||
with tempfile.TemporaryDirectory(prefix="orisys-wheel-verify-") as tmp:
|
||||
venv_dir = Path(tmp) / "venv"
|
||||
venv.create(venv_dir, with_pip=True)
|
||||
python = _venv_python(venv_dir)
|
||||
def _hermetic_env() -> dict:
|
||||
"""Environment for the verification venv with PYTHONPATH removed.
|
||||
|
||||
subprocess.check_call(
|
||||
[python, "-m", "pip", "install", "-q", "--upgrade", "pip"],
|
||||
An inherited PYTHONPATH=src would let the checkout satisfy the checks
|
||||
instead of the installed wheel, which is precisely the failure mode the
|
||||
provenance assertions in the scripts exist to catch.
|
||||
"""
|
||||
return {key: value for key, value in os.environ.items() if key != "PYTHONPATH"}
|
||||
|
||||
|
||||
def _create_venv(
|
||||
venv_dir: Path,
|
||||
*,
|
||||
python_executable: Path | None = None,
|
||||
system_site_packages: bool = False,
|
||||
) -> Path:
|
||||
"""Create a throwaway venv, optionally from a different interpreter."""
|
||||
if python_executable is None:
|
||||
venv.create(venv_dir, with_pip=True, system_site_packages=system_site_packages)
|
||||
else:
|
||||
command = [str(python_executable), "-m", "venv"]
|
||||
if system_site_packages:
|
||||
command.append("--system-site-packages")
|
||||
command.append(str(venv_dir))
|
||||
subprocess.check_call(command, env=_hermetic_env())
|
||||
return _venv_python(venv_dir)
|
||||
|
||||
|
||||
def _install_wheel(
|
||||
python: Path, wheel_path: Path, *, offline: bool = False, env: dict | None = None
|
||||
) -> None:
|
||||
"""Install the wheel under test, guaranteeing it lands in the venv.
|
||||
|
||||
``--force-reinstall --no-deps`` is not optional: pip reports a same-version
|
||||
requirement as "already satisfied" and skips the file, which is fatal here
|
||||
because ``--offline`` also exposes the parent environment's site-packages.
|
||||
Without it the venv check can silently validate a *different* orisys than
|
||||
the one passed in (the scripts additionally assert provenance).
|
||||
"""
|
||||
if offline:
|
||||
# Dependencies come from the parent environment via --system-site-packages.
|
||||
commands = [
|
||||
[
|
||||
str(python),
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"-q",
|
||||
"--no-index",
|
||||
"--no-deps",
|
||||
"--force-reinstall",
|
||||
str(wheel_path),
|
||||
]
|
||||
]
|
||||
else:
|
||||
commands = [
|
||||
[str(python), "-m", "pip", "install", "-q", "--upgrade", "pip"],
|
||||
[str(python), "-m", "pip", "install", "-q", str(wheel_path)],
|
||||
[
|
||||
str(python),
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"-q",
|
||||
"--force-reinstall",
|
||||
"--no-deps",
|
||||
str(wheel_path),
|
||||
],
|
||||
]
|
||||
|
||||
for command in commands:
|
||||
subprocess.check_call(command, stdout=subprocess.DEVNULL, env=env)
|
||||
|
||||
|
||||
class WheelVerificationError(RuntimeError):
|
||||
"""A verification step failed inside the throwaway venv.
|
||||
|
||||
Carries the child's stdout/stderr so the real traceback is visible instead of
|
||||
a bare non-zero exit code.
|
||||
"""
|
||||
|
||||
|
||||
def _run_in_fresh_venv(
|
||||
wheel_path: Path,
|
||||
script: str,
|
||||
args: list[str] | None = None,
|
||||
*,
|
||||
python_executable: Path | None = None,
|
||||
offline: bool = False,
|
||||
prefix: str = "orisys-wheel-verify-",
|
||||
) -> str:
|
||||
wheel_path = wheel_path.resolve()
|
||||
env = _hermetic_env()
|
||||
with tempfile.TemporaryDirectory(prefix=prefix) as tmp:
|
||||
venv_dir = Path(tmp) / "venv"
|
||||
python = _create_venv(
|
||||
venv_dir,
|
||||
python_executable=python_executable,
|
||||
system_site_packages=offline,
|
||||
)
|
||||
subprocess.check_call(
|
||||
[python, "-m", "pip", "install", "-q", str(wheel_path)],
|
||||
_install_wheel(python, wheel_path, offline=offline, env=env)
|
||||
completed = subprocess.run(
|
||||
[str(python), "-c", script, *(args or [])],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
check=False,
|
||||
)
|
||||
subprocess.check_call([python, "-c", _SMOKE_IMPORT_SCRIPT])
|
||||
if completed.returncode != 0:
|
||||
command = " ".join(str(part) for part in (args or []))
|
||||
raise WheelVerificationError(
|
||||
f"verification script failed in the venv (exit {completed.returncode}"
|
||||
+ (f", args: {command}" if command else "")
|
||||
+ ").\n--- stdout ---\n"
|
||||
+ (completed.stdout.strip() or "<empty>")
|
||||
+ "\n--- stderr ---\n"
|
||||
+ (completed.stderr.strip() or "<empty>")
|
||||
)
|
||||
return completed.stdout.strip()
|
||||
|
||||
|
||||
def verify_built_wheel(wheel_path: Path, *, python_executable: Path | None = None) -> None:
|
||||
"""Install the wheel in a temporary venv and smoke-test public imports."""
|
||||
_run_in_fresh_venv(
|
||||
wheel_path, _SMOKE_IMPORT_SCRIPT, python_executable=python_executable
|
||||
)
|
||||
|
||||
|
||||
def verify_wheel_assets(wheel_path: Path, *, python_executable: Path | None = None) -> str:
|
||||
"""Assert the wheel carries loadable bundled configs and sensor assets."""
|
||||
return _run_in_fresh_venv(
|
||||
wheel_path,
|
||||
_ASSET_SCRIPT,
|
||||
python_executable=python_executable,
|
||||
prefix="orisys-wheel-assets-",
|
||||
)
|
||||
|
||||
|
||||
def verify_wheel_runtime(
|
||||
wheel_path: Path,
|
||||
*,
|
||||
video: Path,
|
||||
config_name: str,
|
||||
frames: int = 40,
|
||||
python_executable: Path | None = None,
|
||||
offline: bool = False,
|
||||
) -> dict:
|
||||
"""Run real frames through the installed wheel with no camera attached.
|
||||
|
||||
This is the only check that proves the compiled extension plus the bundled
|
||||
sensor assets actually produce finite flow/force output end to end.
|
||||
"""
|
||||
video = Path(video).resolve()
|
||||
if not video.is_file():
|
||||
raise FileNotFoundError(f"verification video not found: {video}")
|
||||
if frames < 1:
|
||||
raise ValueError("frames must be >= 1")
|
||||
|
||||
output = _run_in_fresh_venv(
|
||||
wheel_path,
|
||||
_RUNTIME_SCRIPT,
|
||||
[str(video), config_name, str(frames)],
|
||||
python_executable=python_executable,
|
||||
offline=offline,
|
||||
prefix="orisys-wheel-runtime-",
|
||||
)
|
||||
import json
|
||||
|
||||
return json.loads(output.splitlines()[-1])
|
||||
|
||||
@@ -0,0 +1,501 @@
|
||||
"""Static content and metadata validation for built orisys wheels.
|
||||
|
||||
Everything here is pure standard library: the wheel is opened as a zip and
|
||||
inspected in place. No installation, no network, no hardware.
|
||||
|
||||
The checks encode the guarantees a Nuitka-compiled release must satisfy:
|
||||
|
||||
* the filename, ``METADATA`` and ``RECORD`` agree on name/version/tags;
|
||||
* PyPI-visible metadata (``Requires-Python``, classifiers, project URLs,
|
||||
license) is present instead of silently dropped by the hand-written
|
||||
``setup.py`` template in :mod:`wheel.package`;
|
||||
* no Python sources ship inside ``orisys/`` (IP containment) while the compiled
|
||||
extension is present;
|
||||
* every file declared by ``[[tool.orisys.bundle.items]]`` is present and
|
||||
byte-identical to the source tree;
|
||||
* no caches, build leftovers or scratch directories leak in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import re
|
||||
import sys
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from email.parser import Parser
|
||||
from pathlib import Path
|
||||
|
||||
_BUILD_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(_BUILD_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_BUILD_ROOT))
|
||||
|
||||
from common.paths import REPO_ROOT # noqa: E402
|
||||
from common.pyproject_utils import bundle_items, extras_require, install_requires, project_metadata # noqa: E402
|
||||
from wheel.layout import resolve_bundle_source_files # noqa: E402
|
||||
from wheel.package import is_compliant_manylinux_wheel # noqa: E402
|
||||
|
||||
# PyPI's default per-project file limit is 100 MB; keep headroom.
|
||||
MAX_WHEEL_BYTES = 90 * 1024 * 1024
|
||||
|
||||
_FILENAME_RE = re.compile(
|
||||
r"^(?P<name>[^-]+)-(?P<version>[^-]+)-(?P<python>[^-]+)-(?P<abi>[^-]+)-(?P<platform>.+)\.whl$"
|
||||
)
|
||||
_COMPILED_EXT_RE = re.compile(r"^orisys/__init__\..+\.(?:pyd|so)$")
|
||||
_FORBIDDEN_PREFIXES = (
|
||||
"test/",
|
||||
"tests/",
|
||||
"data/",
|
||||
"build/",
|
||||
"dist/",
|
||||
"sdk/",
|
||||
".git",
|
||||
)
|
||||
_FORBIDDEN_SUFFIXES = (".pyc", ".pyo", ".c", ".pdb", ".map", ".spec")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Check:
|
||||
name: str
|
||||
ok: bool
|
||||
detail: str = ""
|
||||
|
||||
def render(self) -> str:
|
||||
return f"[{'ok' if self.ok else 'FAIL'}] {self.name}: {self.detail}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class WheelCheckResult:
|
||||
wheel: Path
|
||||
version: str | None = None
|
||||
name: str | None = None
|
||||
platform_tag: str | None = None
|
||||
entry_count: int = 0
|
||||
uncompressed_bytes: int = 0
|
||||
file_bytes: int = 0
|
||||
checks: list[Check] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return all(check.ok for check in self.checks)
|
||||
|
||||
@property
|
||||
def failures(self) -> list[Check]:
|
||||
return [check for check in self.checks if not check.ok]
|
||||
|
||||
def add(self, name: str, ok: bool, detail: str = "") -> bool:
|
||||
self.checks.append(Check(name=name, ok=ok, detail=detail))
|
||||
return ok
|
||||
|
||||
def warn(self, message: str) -> None:
|
||||
self.warnings.append(message)
|
||||
|
||||
def render(self) -> str:
|
||||
lines = [
|
||||
f"wheel: {self.wheel.name}",
|
||||
f" version={self.version} platform={self.platform_tag} "
|
||||
f"entries={self.entry_count} file_size={self.file_bytes} "
|
||||
f"uncompressed={self.uncompressed_bytes}",
|
||||
]
|
||||
lines.extend(f" {check.render()}" for check in self.checks)
|
||||
lines.extend(f" [warn] {message}" for message in self.warnings)
|
||||
lines.append(f" RESULT: {'PASS' if self.ok else 'FAIL'}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _sha256(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def _requirement_name(spec: str) -> str:
|
||||
return re.split(r"[<>=!~\[; ]", spec, maxsplit=1)[0].strip().lower().replace("_", "-")
|
||||
|
||||
|
||||
def _metadata_message(zf: zipfile.ZipFile, result: WheelCheckResult):
|
||||
candidates = [name for name in zf.namelist() if name.endswith(".dist-info/METADATA")]
|
||||
if not candidates:
|
||||
result.add("metadata-present", False, "no *.dist-info/METADATA entry in the wheel")
|
||||
return None
|
||||
raw = zf.read(candidates[0]).decode("utf-8", errors="replace")
|
||||
message = Parser().parsestr(raw)
|
||||
result.add("metadata-present", True, candidates[0])
|
||||
return message
|
||||
|
||||
|
||||
def _check_filename(result: WheelCheckResult, expected_version: str | None) -> None:
|
||||
match = _FILENAME_RE.match(result.wheel.name)
|
||||
if match is None:
|
||||
result.add("filename", False, f"unparsable wheel filename: {result.wheel.name}")
|
||||
return
|
||||
result.name = match.group("name")
|
||||
result.version = match.group("version")
|
||||
result.platform_tag = match.group("platform")
|
||||
|
||||
problems = []
|
||||
if result.name != "orisys":
|
||||
problems.append(f"distribution name is {result.name!r}, expected 'orisys'")
|
||||
if expected_version is not None and result.version != expected_version:
|
||||
problems.append(f"filename version {result.version!r} != expected {expected_version!r}")
|
||||
result.add("filename", not problems, "; ".join(problems) or result.wheel.name)
|
||||
|
||||
|
||||
def _check_metadata(
|
||||
result: WheelCheckResult,
|
||||
message,
|
||||
expected_version: str | None,
|
||||
repo_root: Path | None,
|
||||
) -> None:
|
||||
if message is None:
|
||||
return
|
||||
|
||||
result.add(
|
||||
"metadata-name",
|
||||
message.get("Name") == "orisys",
|
||||
f"Name={message.get('Name')!r}",
|
||||
)
|
||||
|
||||
declared = message.get("Version")
|
||||
expected = expected_version or result.version
|
||||
if expected is not None and declared != expected:
|
||||
result.add(
|
||||
"metadata-version",
|
||||
False,
|
||||
f"METADATA Version={declared!r} != expected {expected!r}",
|
||||
)
|
||||
else:
|
||||
result.add("metadata-version", True, f"Version={declared!r}")
|
||||
|
||||
requires_python = (message.get("Requires-Python") or "").strip()
|
||||
result.add(
|
||||
"metadata-requires-python",
|
||||
bool(requires_python),
|
||||
f"Requires-Python={requires_python!r}" if requires_python else "Requires-Python is missing",
|
||||
)
|
||||
|
||||
classifiers = message.get_all("Classifier") or []
|
||||
result.add(
|
||||
"metadata-classifiers",
|
||||
bool(classifiers),
|
||||
f"{len(classifiers)} classifier(s)" if classifiers else "no Classifier fields",
|
||||
)
|
||||
|
||||
project_urls = message.get_all("Project-URL") or []
|
||||
result.add(
|
||||
"metadata-project-urls",
|
||||
bool(project_urls),
|
||||
f"{len(project_urls)} Project-URL(s)" if project_urls else "no Project-URL fields",
|
||||
)
|
||||
|
||||
license_values = [
|
||||
(message.get("License") or "").strip(),
|
||||
(message.get("License-Expression") or "").strip(),
|
||||
]
|
||||
has_license = any(license_values) or any(
|
||||
value.startswith("License ::") for value in classifiers
|
||||
)
|
||||
result.add(
|
||||
"metadata-license",
|
||||
has_license,
|
||||
f"License={license_values[0]!r} License-Expression={license_values[1]!r}"
|
||||
if has_license
|
||||
else "no License / License-Expression / 'License ::' classifier",
|
||||
)
|
||||
|
||||
dynamic = message.get_all("Dynamic") or []
|
||||
result.add(
|
||||
"metadata-no-dynamic-version",
|
||||
"version" not in dynamic,
|
||||
f"Dynamic={dynamic!r}" if dynamic else "no Dynamic fields",
|
||||
)
|
||||
if dynamic:
|
||||
result.warn(f"METADATA still declares Dynamic fields: {dynamic!r}")
|
||||
|
||||
if repo_root is not None and (repo_root / "pyproject.toml").is_file():
|
||||
requires_dist = message.get_all("Requires-Dist") or []
|
||||
provided = {_requirement_name(entry) for entry in requires_dist}
|
||||
missing_core = [
|
||||
spec for spec in install_requires(repo_root) if _requirement_name(spec) not in provided
|
||||
]
|
||||
result.add(
|
||||
"metadata-requires-dist",
|
||||
not missing_core,
|
||||
f"missing core Requires-Dist entries: {missing_core!r}"
|
||||
if missing_core
|
||||
else f"{len(requires_dist)} Requires-Dist entries",
|
||||
)
|
||||
|
||||
extras = message.get_all("Provides-Extra") or []
|
||||
missing_extras = sorted(set(extras_require(repo_root)) - set(extras))
|
||||
result.add(
|
||||
"metadata-extras",
|
||||
not missing_extras,
|
||||
f"missing Provides-Extra: {missing_extras!r}"
|
||||
if missing_extras
|
||||
else f"Provides-Extra={sorted(extras)!r}",
|
||||
)
|
||||
|
||||
|
||||
def _check_compiled_only(result: WheelCheckResult, names: list[str]) -> None:
|
||||
compiled = [name for name in names if _COMPILED_EXT_RE.match(name)]
|
||||
result.add(
|
||||
"compiled-extension",
|
||||
bool(compiled),
|
||||
f"{compiled[0]}" if compiled else "no orisys/__init__.<abi>.pyd|.so entry",
|
||||
)
|
||||
|
||||
# orisys/examples/** is shipped as package data on purpose; every other
|
||||
# orisys/**/*.py means source code leaked past the Nuitka compile step.
|
||||
sources = [
|
||||
name
|
||||
for name in names
|
||||
if name.startswith("orisys/")
|
||||
and name.endswith(".py")
|
||||
and not name.startswith("orisys/examples/")
|
||||
]
|
||||
result.add(
|
||||
"no-python-sources",
|
||||
not sources,
|
||||
f"python sources leaked into the wheel: {sources[:5]!r}"
|
||||
if sources
|
||||
else "compiled package ships no python sources (examples excluded by design)",
|
||||
)
|
||||
|
||||
blend = [name for name in names if name.startswith("orisys_blend/")]
|
||||
result.add(
|
||||
"orisys-blend-present",
|
||||
"orisys_blend/__init__.py" in blend and "orisys_blend/kernel.py" in blend,
|
||||
f"{sorted(blend)!r}",
|
||||
)
|
||||
|
||||
|
||||
def _check_bundle_fidelity(
|
||||
result: WheelCheckResult,
|
||||
zf: zipfile.ZipFile,
|
||||
names: set[str],
|
||||
repo_root: Path | None,
|
||||
strict: bool,
|
||||
) -> None:
|
||||
if repo_root is None or not (repo_root / "pyproject.toml").is_file():
|
||||
result.warn("pyproject.toml not found; bundle fidelity checks skipped")
|
||||
return
|
||||
|
||||
missing: list[str] = []
|
||||
mismatched: list[str] = []
|
||||
checked = 0
|
||||
|
||||
for item in bundle_items(repo_root):
|
||||
target = str(item["target"]).replace("\\", "/")
|
||||
source_dir = repo_root / item["source"]
|
||||
for source_file in resolve_bundle_source_files(repo_root, item):
|
||||
if item.get("preserve_tree", False):
|
||||
relative = source_file.relative_to(source_dir).as_posix()
|
||||
entry = f"orisys/{target}/{relative}"
|
||||
else:
|
||||
entry = f"orisys/{target}/{source_file.name}"
|
||||
|
||||
if entry not in names:
|
||||
missing.append(entry)
|
||||
continue
|
||||
checked += 1
|
||||
if _sha256(zf.read(entry)) != _sha256(source_file.read_bytes()):
|
||||
mismatched.append(entry)
|
||||
|
||||
# Both entries here mean "the wheel does not match this revision of the
|
||||
# source tree". Against a fresh build that is fatal; against an already
|
||||
# released wheel it is expected, so ``strict`` (set by preflight, unset by
|
||||
# audit) decides whether it fails or warns.
|
||||
if missing:
|
||||
detail = (
|
||||
f"{len(missing)} declared file(s) absent from the wheel (added to the "
|
||||
f"source tree after this build?): {missing[:5]!r}"
|
||||
)
|
||||
result.add("bundle-completeness", not strict, detail)
|
||||
if not strict:
|
||||
result.warn(detail)
|
||||
else:
|
||||
result.add(
|
||||
"bundle-completeness", True, f"{checked} declared bundled file(s) present"
|
||||
)
|
||||
|
||||
if mismatched:
|
||||
detail = (
|
||||
f"{len(mismatched)} of {checked} bundled file(s) differ from the source "
|
||||
f"tree (wheel predates or postdates it): {mismatched[:5]!r}"
|
||||
)
|
||||
result.add("bundle-fidelity", not strict, detail)
|
||||
if not strict:
|
||||
result.warn(detail)
|
||||
else:
|
||||
result.add(
|
||||
"bundle-fidelity",
|
||||
True,
|
||||
f"{checked} bundled file(s) byte-identical to source",
|
||||
)
|
||||
|
||||
|
||||
def _check_forbidden_entries(result: WheelCheckResult, names: list[str]) -> None:
|
||||
offenders = []
|
||||
for name in names:
|
||||
lowered = name.lower()
|
||||
if lowered.startswith(_FORBIDDEN_PREFIXES):
|
||||
offenders.append(name)
|
||||
elif lowered.endswith(_FORBIDDEN_SUFFIXES):
|
||||
offenders.append(name)
|
||||
elif "__pycache__" in lowered or ".egg-info" in lowered:
|
||||
offenders.append(name)
|
||||
result.add(
|
||||
"no-forbidden-entries",
|
||||
not offenders,
|
||||
f"unexpected entries: {offenders[:5]!r}" if offenders else "no caches/leftovers/scratch paths",
|
||||
)
|
||||
|
||||
|
||||
def _check_archive_paths(result: WheelCheckResult, names: list[str]) -> None:
|
||||
unsafe = [
|
||||
name
|
||||
for name in names
|
||||
if name.startswith(("/", "\\")) or ".." in Path(name.replace("\\", "/")).parts
|
||||
]
|
||||
result.add(
|
||||
"archive-paths",
|
||||
not unsafe,
|
||||
f"absolute or traversing entries: {unsafe[:5]!r}" if unsafe else "all entries are relative",
|
||||
)
|
||||
|
||||
|
||||
def _check_record(result: WheelCheckResult, zf: zipfile.ZipFile, names: list[str]) -> None:
|
||||
records = [name for name in names if name.endswith(".dist-info/RECORD")]
|
||||
if not records:
|
||||
result.add("record", False, "no *.dist-info/RECORD entry")
|
||||
return
|
||||
|
||||
bad_paths: list[str] = []
|
||||
bad_hashes: list[str] = []
|
||||
bad_sizes: list[str] = []
|
||||
present = set(names)
|
||||
|
||||
for line in zf.read(records[0]).decode("utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
parts = line.split(",")
|
||||
while len(parts) < 3:
|
||||
parts.append("")
|
||||
path, hash_field, size_field = parts[0], parts[1], parts[2]
|
||||
|
||||
if path not in present:
|
||||
bad_paths.append(path)
|
||||
continue
|
||||
if not hash_field.startswith("sha256="):
|
||||
continue
|
||||
|
||||
expected = hash_field[len("sha256=") :]
|
||||
actual = base64.urlsafe_b64encode(
|
||||
hashlib.sha256(zf.read(path)).digest()
|
||||
).rstrip(b"=").decode("ascii")
|
||||
if actual != expected:
|
||||
bad_hashes.append(path)
|
||||
if size_field and size_field.isdigit():
|
||||
if int(size_field) != len(zf.read(path)):
|
||||
bad_sizes.append(path)
|
||||
|
||||
problems = []
|
||||
if bad_paths:
|
||||
problems.append(f"paths not in archive: {bad_paths[:5]!r}")
|
||||
if bad_hashes:
|
||||
problems.append(f"sha256 mismatch: {bad_hashes[:5]!r}")
|
||||
if bad_sizes:
|
||||
problems.append(f"size mismatch: {bad_sizes[:5]!r}")
|
||||
result.add(
|
||||
"record",
|
||||
not problems,
|
||||
"; ".join(problems) if problems else f"{records[0]} verifies",
|
||||
)
|
||||
|
||||
|
||||
def _check_platform_tag(result: WheelCheckResult) -> None:
|
||||
tag = result.platform_tag
|
||||
if tag is None:
|
||||
return
|
||||
if "win" in tag:
|
||||
ok = tag == "win_amd64"
|
||||
result.add("platform-tag", ok, f"{tag!r} (expected 'win_amd64')")
|
||||
return
|
||||
ok = is_compliant_manylinux_wheel(result.wheel)
|
||||
result.add(
|
||||
"platform-tag",
|
||||
ok,
|
||||
f"{tag!r}" if ok else f"{tag!r} is not a PyPI-accepted manylinux x86_64 tag",
|
||||
)
|
||||
|
||||
|
||||
def _check_size(result: WheelCheckResult, max_bytes: int) -> None:
|
||||
result.add(
|
||||
"size-limit",
|
||||
result.file_bytes <= max_bytes,
|
||||
f"{result.file_bytes} bytes (limit {max_bytes})",
|
||||
)
|
||||
|
||||
|
||||
def inspect_wheel(
|
||||
wheel_path: Path,
|
||||
*,
|
||||
expected_version: str | None = None,
|
||||
repo_root: Path | None = None,
|
||||
max_bytes: int = MAX_WHEEL_BYTES,
|
||||
strict_bundle: bool = True,
|
||||
) -> WheelCheckResult:
|
||||
"""Validate one built wheel without installing it.
|
||||
|
||||
``strict_bundle`` controls how the wheel's bundled files are compared with the
|
||||
current source tree: ``True`` (gate a fresh build, used by
|
||||
``release preflight``) fails on any missing or changed declared file;
|
||||
``False`` (audit an already-released wheel) reports the same findings as
|
||||
warnings.
|
||||
"""
|
||||
wheel_path = Path(wheel_path).resolve()
|
||||
if repo_root is None:
|
||||
repo_root = REPO_ROOT if (REPO_ROOT / "pyproject.toml").is_file() else None
|
||||
else:
|
||||
repo_root = Path(repo_root).resolve()
|
||||
|
||||
result = WheelCheckResult(wheel=wheel_path)
|
||||
if not wheel_path.is_file():
|
||||
result.add("wheel-exists", False, f"not a file: {wheel_path}")
|
||||
return result
|
||||
|
||||
result.file_bytes = wheel_path.stat().st_size
|
||||
result.add("wheel-exists", True, str(wheel_path))
|
||||
_check_filename(result, expected_version)
|
||||
_check_platform_tag(result)
|
||||
_check_size(result, max_bytes)
|
||||
|
||||
try:
|
||||
zf = zipfile.ZipFile(wheel_path)
|
||||
except zipfile.BadZipFile as exc:
|
||||
result.add("zip-integrity", False, f"not a valid zip: {exc}")
|
||||
return result
|
||||
|
||||
with zf:
|
||||
corrupt = zf.testzip()
|
||||
result.add(
|
||||
"zip-integrity",
|
||||
corrupt is None,
|
||||
"CRC check passed" if corrupt is None else f"corrupt entry: {corrupt}",
|
||||
)
|
||||
|
||||
names = zf.namelist()
|
||||
name_set = set(names)
|
||||
result.entry_count = len(names)
|
||||
result.uncompressed_bytes = sum(info.file_size for info in zf.infolist())
|
||||
|
||||
_check_archive_paths(result, names)
|
||||
_check_forbidden_entries(result, names)
|
||||
message = _metadata_message(zf, result)
|
||||
_check_metadata(result, message, expected_version, repo_root)
|
||||
_check_compiled_only(result, names)
|
||||
_check_bundle_fidelity(result, zf, name_set, repo_root, strict_bundle)
|
||||
_check_record(result, zf, names)
|
||||
|
||||
return result
|
||||
Reference in New Issue
Block a user