orisys-tech/orisys-sdk
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added preflight check

Browse Source
This commit is contained in:
ykcheungab
2026-09-12 15:49:28 +08:00
parent 9c23ba3d50
commit 9537cf91a1
31 changed files with 3346 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/README.md
+21 -1
View File
@@ -12,6 +12,7 @@ build/
README.md
common/ shared helpers (paths, pyproject, verify)
wheel/ SDK .whl (Nuitka + manylinux Docker)
release/ local release gate (preflight / audit)
gui/
common/ shared PyInstaller driver
ory_finger/ ORY-FINGER dashboard
@@ -41,6 +42,22 @@ build\wheel\win.bat
.\build\wheel\docker\run.ps1
```
### Release gate
Local only: never publishes, and makes no network call unless `--check-pypi` is
passed. See `docs/publishing.md` for the full flow.
```bash
python build/run.py release preflight --version 0.5.2
python build/run.py release preflight --all-wheel-pythons --check-pypi
python build/run.py release audit SDK\0.5.2\orisys-0.5.2-cp312-cp312-win_amd64.whl
```
`preflight` runs: repo state → version consistency → duplicate guard → pytest →
bundle precheck → per-wheel static content/metadata assertions → isolated runtime
verification (bundled assets + real frames, no camera). Reports land in
`build/release/reports/<version>/`.
### GUI
```bash
@@ -53,13 +70,16 @@ python build/run.py gui box # scaffold / NotImplemented
```bash
python build/run.py verify path\to\orisys.whl
python build/run.py verify path\to\orisys.whl --contents --assets --runtime
python build/run.py verify --gui-dist --product ory_finger
```
## Outputs
- Wheels: `SDK/<version>/`, `dist/repaired/`
- Preflight reports: `build/release/reports/<version>/`
- Finger GUI: `dist/OrisysDashboard/`
- Future mini/box: `dist/OryMini/`, `dist/OryBox/`
Generated work files under `build/nuitka/` and `build/pyinstaller_work/` are gitignored.
Generated work files under `build/nuitka/`, `build/release/reports/` and
`build/pyinstaller_work/` are gitignored.
build/common/verify/__init__.py
+15 -2
View File
@@ -1,6 +1,19 @@
"""Post-build smoke tests for wheels and product GUI folders."""
from common.verify.gui import GUI_PRODUCTS, verify_gui_dist
from common.verify.wheel import verify_built_wheel
from common.verify.wheel import (
verify_built_wheel,
verify_wheel_assets,
verify_wheel_runtime,
)
from common.verify.wheel_contents import WheelCheckResult, inspect_wheel
__all__ = ["GUI_PRODUCTS", "verify_built_wheel", "verify_gui_dist"]
__all__ = [
"GUI_PRODUCTS",
"WheelCheckResult",
"inspect_wheel",
"verify_built_wheel",
"verify_gui_dist",
"verify_wheel_assets",
"verify_wheel_runtime",
]
build/common/verify/cli.py
+83 -1
View File
@@ -3,9 +3,23 @@
from __future__ import annotations
import argparse
import sys
from pathlib import Path
from common.verify import GUI_PRODUCTS, verify_built_wheel, verify_gui_dist
_BUILD_ROOT = Path(__file__).resolve().parents[2]
if str(_BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(_BUILD_ROOT))
from common.paths import REPO_ROOT # noqa: E402
from common.verify import ( # noqa: E402
GUI_PRODUCTS,
inspect_wheel,
verify_built_wheel,
verify_gui_dist,
verify_wheel_assets,
verify_wheel_runtime,
)
from common.verify.wheel import DEFAULT_RUNTIME_CONFIG, default_verification_video # noqa: E402
def main(argv: list[str] | None = None) -> int:
@@ -18,6 +32,31 @@ def main(argv: list[str] | None = None) -> int:
help="GUI product profile (default: ory_finger)",
)
parser.add_argument("--dist-dir", type=Path, default=None, help="Override GUI dist folder")
parser.add_argument(
"--contents",
action="store_true",
help="Run static metadata/content assertions on the wheel",
)
parser.add_argument(
"--assets",
action="store_true",
help="Install the wheel in a temp venv and assert bundled assets load",
)
parser.add_argument(
"--runtime",
action="store_true",
help="Install the wheel and process real frames (no camera required)",
)
parser.add_argument("--expected-version", default=None, help="Assert the wheel version")
parser.add_argument(
"--strict-bundle",
action="store_true",
help="Treat bundled-file drift from the source tree as a failure",
)
parser.add_argument("--video", type=Path, default=None, help="Clip for --runtime")
parser.add_argument("--config", default=DEFAULT_RUNTIME_CONFIG, help="Config for --runtime")
parser.add_argument("--frames", type=int, default=40, help="Frames for --runtime")
parser.add_argument("--offline", action="store_true", help="Reuse parent-env dependencies")
parser.add_argument("wheel", nargs="?", type=Path, help="Wheel path for import smoke test")
args = parser.parse_args(argv)
@@ -26,6 +65,49 @@ def main(argv: list[str] | None = None) -> int:
return 0
if args.wheel is None:
parser.error("Provide a wheel path or pass --gui-dist")
exit_code = 0
if args.contents:
result = inspect_wheel(
args.wheel,
expected_version=args.expected_version,
repo_root=REPO_ROOT,
strict_bundle=args.strict_bundle,
)
print(result.render())
if not result.ok:
exit_code = 1
if args.assets or args.runtime:
try:
print(verify_wheel_assets(args.wheel))
except Exception as exc:
print(f"[FAIL] bundled assets: {type(exc).__name__}: {exc}")
exit_code = 1
if args.runtime:
video = args.video or default_verification_video(REPO_ROOT)
if video is None:
print("[FAIL] runtime: no clip found; pass --video or set ORISYS_VERIFY_VIDEO")
return 1
try:
info = verify_wheel_runtime(
args.wheel,
video=video,
config_name=args.config,
frames=args.frames,
offline=args.offline,
)
print(f"[ok ] runtime: {info}")
except Exception as exc:
print(f"[FAIL] runtime: {type(exc).__name__}: {exc}")
exit_code = 1
if args.contents or args.assets or args.runtime:
return exit_code
# Default behaviour is unchanged: import smoke test in an isolated venv.
verify_built_wheel(args.wheel)
return 0
build/common/verify/wheel.py
+342 -13
View File
@@ -1,7 +1,21 @@
"""Isolated venv smoke tests for packaged orisys wheels."""
"""Isolated venv smoke tests for packaged orisys wheels.
Three levels of confidence, all in a throwaway virtualenv so the active
environment is never touched:
* :func:`verify_built_wheel` - the public API imports (fast, original check);
* :func:`verify_wheel_assets` - bundled configs/sensors resolve and load via
``importlib.resources``;
* :func:`verify_wheel_runtime` - the compiled wheel actually processes real
frames through the public pipeline with no camera attached.
Every level can be bootstrapped from another interpreter so each supported
CPython ABI can be verified against its own wheel.
"""
from __future__ import annotations
import os
import subprocess
import sys
import tempfile
@@ -38,6 +52,167 @@ assert Stitcher is not None
print(f"orisys {orisys.__version__} public API imports OK")
"""
# Bundled data that must be resolvable and loadable from inside site-packages.
# The wheel ships the compiled package, so these paths only exist if the bundle
# step actually ran for this build.
_ASSET_SCRIPT = r"""
import sys
from importlib.metadata import distribution
from pathlib import Path
import numpy as np
import orisys
# Locate bundled data through the installed distribution, NOT through
# orisys.__file__ / importlib.resources. Nuitka compiles the package into one
# extension module whose __init__ installs nuitka_module_loader, which reports a
# synthetic __file__ (e.g. ...\site-packages\orisys\orisys\__init__.py) and a
# synthetic __path__ ending in '\not_existing'. Neither points at the real data
# directory, so importlib.resources silently finds nothing in a Nuitka wheel.
dist = distribution("orisys")
dist_root = Path(dist.locate_file("")).resolve()
package_root = Path(dist.locate_file("orisys")).resolve()
_diagnosis = (
f"orisys distribution resolved to {dist_root} (sys.prefix={sys.prefix!r}, "
f"sys.executable={sys.executable!r}), outside the verification venv. A stale "
"orisys in the parent environment is visible through --system-site-packages; "
"run `pip uninstall -y orisys` there, or re-run without --offline."
)
assert str(Path(sys.prefix).resolve()) in str(dist_root), _diagnosis
assert package_root.is_dir(), f"package directory not found: {package_root}"
required = [
"configs/box.json",
"configs/finger.json",
"sensors/box/force_calibration.json",
"sensors/box/sensor.json",
"sensors/finger/manifest.json",
"sensors/finger/sensor.json",
"sensors/mini/sensor.json",
"sensors/finger/assets/30fps-0703/mesh.npz",
"sensors/finger/assets/30fps-0703/camera_intrinsics.npz",
"sensors/finger/assets/30fps-0703/world_to_cam.npy",
"sensors/finger/assets/visualization/mesh.npz",
"sensors/finger/assets/visualization/world_to_cam.npy",
]
missing = [name for name in required if not (package_root / name).is_file()]
assert not missing, (
f"bundled assets missing from the wheel: {missing}\n"
f"package root: {package_root}\ninstalled tree: "
f"{sorted(p.name for p in package_root.iterdir()) if package_root.is_dir() else '<missing>'}"
)
for name in required:
path = package_root / name
if name.endswith(".npz"):
data = np.load(str(path))
assert len(data.files) > 0, f"{name} is an empty npz"
elif name.endswith(".npy"):
arr = np.load(str(path))
assert arr.size > 0, f"{name} is an empty npy"
configs = orisys.list_config()
assert configs, "orisys.list_config() returned nothing"
print(f"bundled assets OK ({len(required)} files, {len(configs)} configs) "
f"from {package_root}")
"""
# End-to-end pipeline run against a video file: no camera, no GUI, no imshow.
_RUNTIME_SCRIPT = r"""
import json
import os
import sys
from importlib.metadata import distribution
from pathlib import Path
import numpy as np
os.environ.setdefault("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", "0")
import orisys
# Provenance via the installed distribution: a Nuitka-compiled package reports a
# synthetic orisys.__file__ (see the asset check) and a stale orisys visible
# through --system-site-packages would otherwise satisfy this check.
dist = distribution("orisys")
dist_root = Path(dist.locate_file("")).resolve()
assert str(Path(sys.prefix).resolve()) in str(dist_root), (
f"the installed orisys came from {dist_root}, outside the verification venv "
f"{sys.prefix!r}; run `pip uninstall -y orisys` in the parent environment or "
"re-run without --offline."
)
video, config_name, frames = sys.argv[1], sys.argv[2], int(sys.argv[3])
sensor = orisys.Sensor(
vid_src=video,
config_name=config_name,
backend="cpu",
verbose=False,
)
processed = 0
try:
for _ in range(frames):
img = sensor.get_img()
assert img is not None, "get_img returned None"
assert isinstance(img, np.ndarray) and img.ndim in (2, 3)
sensor.compute_deformation(check_motion=False)
flow, vnormal, vshear = sensor.read_info(
sensor.info.VRAW, sensor.info.VNORMAL, sensor.info.VSHEAR
)
for name, array in (("flow", flow), ("vnormal", vnormal), ("vshear", vshear)):
assert np.isfinite(array).all(), f"{name} contains NaN/Inf"
contact = sensor.compute_contact()
assert isinstance(contact, dict) and "centroid_found" in contact
fnormal, fsx, fsy = sensor.read_info(
sensor.info.FNORMAL, sensor.info.FSHEARX, sensor.info.FSHEARY
)
assert np.isfinite([fnormal, fsx, fsy]).all(), "force output contains NaN/Inf"
processed += 1
finally:
sensor.disconnect()
assert processed == frames, f"only processed {processed}/{frames} frames"
print(json.dumps({
"version": orisys.__version__,
"frames": processed,
"flow_shape": list(flow.shape),
"video": video,
"config": config_name,
"package_root": str(Path(dist.locate_file("orisys")).resolve()),
}))
"""
# Default runtime-verification inputs. Overridable by CLI flags or environment.
DEFAULT_RUNTIME_CONFIG = os.getenv("ORISYS_VERIFY_CONFIG", "box")
# Clips tracked in the repository, smallest first.
DEFAULT_VIDEO_CANDIDATES = (
Path("data") / "sample" / "s1_480p.mp4",
Path("data") / "sample" / "s1_720p.mp4",
Path("tests") / "idle_1min.mp4",
)
def default_verification_video(repo_root: Path) -> Path | None:
"""Resolve the clip used for the end-to-end runtime check, if any."""
env_video = os.getenv("ORISYS_VERIFY_VIDEO")
repo_root = Path(repo_root)
if env_video:
candidate = Path(env_video)
return candidate if candidate.is_absolute() else repo_root / candidate
for relative in DEFAULT_VIDEO_CANDIDATES:
candidate = repo_root / relative
if candidate.is_file():
return candidate
return None
def _venv_python(venv_dir: Path) -> Path:
if sys.platform == "win32":
@@ -45,18 +220,172 @@ def _venv_python(venv_dir: Path) -> Path:
return venv_dir / "bin" / "python"
def verify_built_wheel(wheel_path: Path) -> None:
"""Install the wheel in a temporary venv and smoke-test public imports."""
wheel_path = wheel_path.resolve()
with tempfile.TemporaryDirectory(prefix="orisys-wheel-verify-") as tmp:
venv_dir = Path(tmp) / "venv"
venv.create(venv_dir, with_pip=True)
python = _venv_python(venv_dir)
def _hermetic_env() -> dict:
"""Environment for the verification venv with PYTHONPATH removed.
subprocess.check_call(
[python, "-m", "pip", "install", "-q", "--upgrade", "pip"],
An inherited PYTHONPATH=src would let the checkout satisfy the checks
instead of the installed wheel, which is precisely the failure mode the
provenance assertions in the scripts exist to catch.
"""
return {key: value for key, value in os.environ.items() if key != "PYTHONPATH"}
def _create_venv(
venv_dir: Path,
*,
python_executable: Path | None = None,
system_site_packages: bool = False,
) -> Path:
"""Create a throwaway venv, optionally from a different interpreter."""
if python_executable is None:
venv.create(venv_dir, with_pip=True, system_site_packages=system_site_packages)
else:
command = [str(python_executable), "-m", "venv"]
if system_site_packages:
command.append("--system-site-packages")
command.append(str(venv_dir))
subprocess.check_call(command, env=_hermetic_env())
return _venv_python(venv_dir)
def _install_wheel(
python: Path, wheel_path: Path, *, offline: bool = False, env: dict | None = None
) -> None:
"""Install the wheel under test, guaranteeing it lands in the venv.
``--force-reinstall --no-deps`` is not optional: pip reports a same-version
requirement as "already satisfied" and skips the file, which is fatal here
because ``--offline`` also exposes the parent environment's site-packages.
Without it the venv check can silently validate a *different* orisys than
the one passed in (the scripts additionally assert provenance).
"""
if offline:
# Dependencies come from the parent environment via --system-site-packages.
commands = [
[
str(python),
"-m",
"pip",
"install",
"-q",
"--no-index",
"--no-deps",
"--force-reinstall",
str(wheel_path),
]
]
else:
commands = [
[str(python), "-m", "pip", "install", "-q", "--upgrade", "pip"],
[str(python), "-m", "pip", "install", "-q", str(wheel_path)],
[
str(python),
"-m",
"pip",
"install",
"-q",
"--force-reinstall",
"--no-deps",
str(wheel_path),
],
]
for command in commands:
subprocess.check_call(command, stdout=subprocess.DEVNULL, env=env)
class WheelVerificationError(RuntimeError):
"""A verification step failed inside the throwaway venv.
Carries the child's stdout/stderr so the real traceback is visible instead of
a bare non-zero exit code.
"""
def _run_in_fresh_venv(
wheel_path: Path,
script: str,
args: list[str] | None = None,
*,
python_executable: Path | None = None,
offline: bool = False,
prefix: str = "orisys-wheel-verify-",
) -> str:
wheel_path = wheel_path.resolve()
env = _hermetic_env()
with tempfile.TemporaryDirectory(prefix=prefix) as tmp:
venv_dir = Path(tmp) / "venv"
python = _create_venv(
venv_dir,
python_executable=python_executable,
system_site_packages=offline,
)
subprocess.check_call(
[python, "-m", "pip", "install", "-q", str(wheel_path)],
_install_wheel(python, wheel_path, offline=offline, env=env)
completed = subprocess.run(
[str(python), "-c", script, *(args or [])],
capture_output=True,
text=True,
env=env,
check=False,
)
subprocess.check_call([python, "-c", _SMOKE_IMPORT_SCRIPT])
if completed.returncode != 0:
command = " ".join(str(part) for part in (args or []))
raise WheelVerificationError(
f"verification script failed in the venv (exit {completed.returncode}"
+ (f", args: {command}" if command else "")
+ ").\n--- stdout ---\n"
+ (completed.stdout.strip() or "<empty>")
+ "\n--- stderr ---\n"
+ (completed.stderr.strip() or "<empty>")
)
return completed.stdout.strip()
def verify_built_wheel(wheel_path: Path, *, python_executable: Path | None = None) -> None:
"""Install the wheel in a temporary venv and smoke-test public imports."""
_run_in_fresh_venv(
wheel_path, _SMOKE_IMPORT_SCRIPT, python_executable=python_executable
)
def verify_wheel_assets(wheel_path: Path, *, python_executable: Path | None = None) -> str:
"""Assert the wheel carries loadable bundled configs and sensor assets."""
return _run_in_fresh_venv(
wheel_path,
_ASSET_SCRIPT,
python_executable=python_executable,
prefix="orisys-wheel-assets-",
)
def verify_wheel_runtime(
wheel_path: Path,
*,
video: Path,
config_name: str,
frames: int = 40,
python_executable: Path | None = None,
offline: bool = False,
) -> dict:
"""Run real frames through the installed wheel with no camera attached.
This is the only check that proves the compiled extension plus the bundled
sensor assets actually produce finite flow/force output end to end.
"""
video = Path(video).resolve()
if not video.is_file():
raise FileNotFoundError(f"verification video not found: {video}")
if frames < 1:
raise ValueError("frames must be >= 1")
output = _run_in_fresh_venv(
wheel_path,
_RUNTIME_SCRIPT,
[str(video), config_name, str(frames)],
python_executable=python_executable,
offline=offline,
prefix="orisys-wheel-runtime-",
)
import json
return json.loads(output.splitlines()[-1])
build/common/verify/wheel_contents.py
+501
View File
@@ -0,0 +1,501 @@
"""Static content and metadata validation for built orisys wheels.
Everything here is pure standard library: the wheel is opened as a zip and
inspected in place. No installation, no network, no hardware.
The checks encode the guarantees a Nuitka-compiled release must satisfy:
* the filename, ``METADATA`` and ``RECORD`` agree on name/version/tags;
* PyPI-visible metadata (``Requires-Python``, classifiers, project URLs,
license) is present instead of silently dropped by the hand-written
``setup.py`` template in :mod:`wheel.package`;
* no Python sources ship inside ``orisys/`` (IP containment) while the compiled
extension is present;
* every file declared by ``[[tool.orisys.bundle.items]]`` is present and
byte-identical to the source tree;
* no caches, build leftovers or scratch directories leak in.
"""
from __future__ import annotations
import base64
import hashlib
import re
import sys
import zipfile
from dataclasses import dataclass, field
from email.parser import Parser
from pathlib import Path
_BUILD_ROOT = Path(__file__).resolve().parents[1]
if str(_BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(_BUILD_ROOT))
from common.paths import REPO_ROOT # noqa: E402
from common.pyproject_utils import bundle_items, extras_require, install_requires, project_metadata # noqa: E402
from wheel.layout import resolve_bundle_source_files # noqa: E402
from wheel.package import is_compliant_manylinux_wheel # noqa: E402
# PyPI's default per-project file limit is 100 MB; keep headroom.
MAX_WHEEL_BYTES = 90 * 1024 * 1024
_FILENAME_RE = re.compile(
r"^(?P<name>[^-]+)-(?P<version>[^-]+)-(?P<python>[^-]+)-(?P<abi>[^-]+)-(?P<platform>.+)\.whl$"
)
_COMPILED_EXT_RE = re.compile(r"^orisys/__init__\..+\.(?:pyd|so)$")
_FORBIDDEN_PREFIXES = (
"test/",
"tests/",
"data/",
"build/",
"dist/",
"sdk/",
".git",
)
_FORBIDDEN_SUFFIXES = (".pyc", ".pyo", ".c", ".pdb", ".map", ".spec")
@dataclass(frozen=True)
class Check:
name: str
ok: bool
detail: str = ""
def render(self) -> str:
return f"[{'ok' if self.ok else 'FAIL'}] {self.name}: {self.detail}"
@dataclass
class WheelCheckResult:
wheel: Path
version: str | None = None
name: str | None = None
platform_tag: str | None = None
entry_count: int = 0
uncompressed_bytes: int = 0
file_bytes: int = 0
checks: list[Check] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
@property
def ok(self) -> bool:
return all(check.ok for check in self.checks)
@property
def failures(self) -> list[Check]:
return [check for check in self.checks if not check.ok]
def add(self, name: str, ok: bool, detail: str = "") -> bool:
self.checks.append(Check(name=name, ok=ok, detail=detail))
return ok
def warn(self, message: str) -> None:
self.warnings.append(message)
def render(self) -> str:
lines = [
f"wheel: {self.wheel.name}",
f" version={self.version} platform={self.platform_tag} "
f"entries={self.entry_count} file_size={self.file_bytes} "
f"uncompressed={self.uncompressed_bytes}",
]
lines.extend(f" {check.render()}" for check in self.checks)
lines.extend(f" [warn] {message}" for message in self.warnings)
lines.append(f" RESULT: {'PASS' if self.ok else 'FAIL'}")
return "\n".join(lines)
def _sha256(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def _requirement_name(spec: str) -> str:
return re.split(r"[<>=!~\[; ]", spec, maxsplit=1)[0].strip().lower().replace("_", "-")
def _metadata_message(zf: zipfile.ZipFile, result: WheelCheckResult):
candidates = [name for name in zf.namelist() if name.endswith(".dist-info/METADATA")]
if not candidates:
result.add("metadata-present", False, "no *.dist-info/METADATA entry in the wheel")
return None
raw = zf.read(candidates[0]).decode("utf-8", errors="replace")
message = Parser().parsestr(raw)
result.add("metadata-present", True, candidates[0])
return message
def _check_filename(result: WheelCheckResult, expected_version: str | None) -> None:
match = _FILENAME_RE.match(result.wheel.name)
if match is None:
result.add("filename", False, f"unparsable wheel filename: {result.wheel.name}")
return
result.name = match.group("name")
result.version = match.group("version")
result.platform_tag = match.group("platform")
problems = []
if result.name != "orisys":
problems.append(f"distribution name is {result.name!r}, expected 'orisys'")
if expected_version is not None and result.version != expected_version:
problems.append(f"filename version {result.version!r} != expected {expected_version!r}")
result.add("filename", not problems, "; ".join(problems) or result.wheel.name)
def _check_metadata(
result: WheelCheckResult,
message,
expected_version: str | None,
repo_root: Path | None,
) -> None:
if message is None:
return
result.add(
"metadata-name",
message.get("Name") == "orisys",
f"Name={message.get('Name')!r}",
)
declared = message.get("Version")
expected = expected_version or result.version
if expected is not None and declared != expected:
result.add(
"metadata-version",
False,
f"METADATA Version={declared!r} != expected {expected!r}",
)
else:
result.add("metadata-version", True, f"Version={declared!r}")
requires_python = (message.get("Requires-Python") or "").strip()
result.add(
"metadata-requires-python",
bool(requires_python),
f"Requires-Python={requires_python!r}" if requires_python else "Requires-Python is missing",
)
classifiers = message.get_all("Classifier") or []
result.add(
"metadata-classifiers",
bool(classifiers),
f"{len(classifiers)} classifier(s)" if classifiers else "no Classifier fields",
)
project_urls = message.get_all("Project-URL") or []
result.add(
"metadata-project-urls",
bool(project_urls),
f"{len(project_urls)} Project-URL(s)" if project_urls else "no Project-URL fields",
)
license_values = [
(message.get("License") or "").strip(),
(message.get("License-Expression") or "").strip(),
]
has_license = any(license_values) or any(
value.startswith("License ::") for value in classifiers
)
result.add(
"metadata-license",
has_license,
f"License={license_values[0]!r} License-Expression={license_values[1]!r}"
if has_license
else "no License / License-Expression / 'License ::' classifier",
)
dynamic = message.get_all("Dynamic") or []
result.add(
"metadata-no-dynamic-version",
"version" not in dynamic,
f"Dynamic={dynamic!r}" if dynamic else "no Dynamic fields",
)
if dynamic:
result.warn(f"METADATA still declares Dynamic fields: {dynamic!r}")
if repo_root is not None and (repo_root / "pyproject.toml").is_file():
requires_dist = message.get_all("Requires-Dist") or []
provided = {_requirement_name(entry) for entry in requires_dist}
missing_core = [
spec for spec in install_requires(repo_root) if _requirement_name(spec) not in provided
]
result.add(
"metadata-requires-dist",
not missing_core,
f"missing core Requires-Dist entries: {missing_core!r}"
if missing_core
else f"{len(requires_dist)} Requires-Dist entries",
)
extras = message.get_all("Provides-Extra") or []
missing_extras = sorted(set(extras_require(repo_root)) - set(extras))
result.add(
"metadata-extras",
not missing_extras,
f"missing Provides-Extra: {missing_extras!r}"
if missing_extras
else f"Provides-Extra={sorted(extras)!r}",
)
def _check_compiled_only(result: WheelCheckResult, names: list[str]) -> None:
compiled = [name for name in names if _COMPILED_EXT_RE.match(name)]
result.add(
"compiled-extension",
bool(compiled),
f"{compiled[0]}" if compiled else "no orisys/__init__.<abi>.pyd|.so entry",
)
# orisys/examples/** is shipped as package data on purpose; every other
# orisys/**/*.py means source code leaked past the Nuitka compile step.
sources = [
name
for name in names
if name.startswith("orisys/")
and name.endswith(".py")
and not name.startswith("orisys/examples/")
]
result.add(
"no-python-sources",
not sources,
f"python sources leaked into the wheel: {sources[:5]!r}"
if sources
else "compiled package ships no python sources (examples excluded by design)",
)
blend = [name for name in names if name.startswith("orisys_blend/")]
result.add(
"orisys-blend-present",
"orisys_blend/__init__.py" in blend and "orisys_blend/kernel.py" in blend,
f"{sorted(blend)!r}",
)
def _check_bundle_fidelity(
result: WheelCheckResult,
zf: zipfile.ZipFile,
names: set[str],
repo_root: Path | None,
strict: bool,
) -> None:
if repo_root is None or not (repo_root / "pyproject.toml").is_file():
result.warn("pyproject.toml not found; bundle fidelity checks skipped")
return
missing: list[str] = []
mismatched: list[str] = []
checked = 0
for item in bundle_items(repo_root):
target = str(item["target"]).replace("\\", "/")
source_dir = repo_root / item["source"]
for source_file in resolve_bundle_source_files(repo_root, item):
if item.get("preserve_tree", False):
relative = source_file.relative_to(source_dir).as_posix()
entry = f"orisys/{target}/{relative}"
else:
entry = f"orisys/{target}/{source_file.name}"
if entry not in names:
missing.append(entry)
continue
checked += 1
if _sha256(zf.read(entry)) != _sha256(source_file.read_bytes()):
mismatched.append(entry)
# Both entries here mean "the wheel does not match this revision of the
# source tree". Against a fresh build that is fatal; against an already
# released wheel it is expected, so ``strict`` (set by preflight, unset by
# audit) decides whether it fails or warns.
if missing:
detail = (
f"{len(missing)} declared file(s) absent from the wheel (added to the "
f"source tree after this build?): {missing[:5]!r}"
)
result.add("bundle-completeness", not strict, detail)
if not strict:
result.warn(detail)
else:
result.add(
"bundle-completeness", True, f"{checked} declared bundled file(s) present"
)
if mismatched:
detail = (
f"{len(mismatched)} of {checked} bundled file(s) differ from the source "
f"tree (wheel predates or postdates it): {mismatched[:5]!r}"
)
result.add("bundle-fidelity", not strict, detail)
if not strict:
result.warn(detail)
else:
result.add(
"bundle-fidelity",
True,
f"{checked} bundled file(s) byte-identical to source",
)
def _check_forbidden_entries(result: WheelCheckResult, names: list[str]) -> None:
offenders = []
for name in names:
lowered = name.lower()
if lowered.startswith(_FORBIDDEN_PREFIXES):
offenders.append(name)
elif lowered.endswith(_FORBIDDEN_SUFFIXES):
offenders.append(name)
elif "__pycache__" in lowered or ".egg-info" in lowered:
offenders.append(name)
result.add(
"no-forbidden-entries",
not offenders,
f"unexpected entries: {offenders[:5]!r}" if offenders else "no caches/leftovers/scratch paths",
)
def _check_archive_paths(result: WheelCheckResult, names: list[str]) -> None:
unsafe = [
name
for name in names
if name.startswith(("/", "\\")) or ".." in Path(name.replace("\\", "/")).parts
]
result.add(
"archive-paths",
not unsafe,
f"absolute or traversing entries: {unsafe[:5]!r}" if unsafe else "all entries are relative",
)
def _check_record(result: WheelCheckResult, zf: zipfile.ZipFile, names: list[str]) -> None:
records = [name for name in names if name.endswith(".dist-info/RECORD")]
if not records:
result.add("record", False, "no *.dist-info/RECORD entry")
return
bad_paths: list[str] = []
bad_hashes: list[str] = []
bad_sizes: list[str] = []
present = set(names)
for line in zf.read(records[0]).decode("utf-8").splitlines():
if not line.strip():
continue
parts = line.split(",")
while len(parts) < 3:
parts.append("")
path, hash_field, size_field = parts[0], parts[1], parts[2]
if path not in present:
bad_paths.append(path)
continue
if not hash_field.startswith("sha256="):
continue
expected = hash_field[len("sha256=") :]
actual = base64.urlsafe_b64encode(
hashlib.sha256(zf.read(path)).digest()
).rstrip(b"=").decode("ascii")
if actual != expected:
bad_hashes.append(path)
if size_field and size_field.isdigit():
if int(size_field) != len(zf.read(path)):
bad_sizes.append(path)
problems = []
if bad_paths:
problems.append(f"paths not in archive: {bad_paths[:5]!r}")
if bad_hashes:
problems.append(f"sha256 mismatch: {bad_hashes[:5]!r}")
if bad_sizes:
problems.append(f"size mismatch: {bad_sizes[:5]!r}")
result.add(
"record",
not problems,
"; ".join(problems) if problems else f"{records[0]} verifies",
)
def _check_platform_tag(result: WheelCheckResult) -> None:
tag = result.platform_tag
if tag is None:
return
if "win" in tag:
ok = tag == "win_amd64"
result.add("platform-tag", ok, f"{tag!r} (expected 'win_amd64')")
return
ok = is_compliant_manylinux_wheel(result.wheel)
result.add(
"platform-tag",
ok,
f"{tag!r}" if ok else f"{tag!r} is not a PyPI-accepted manylinux x86_64 tag",
)
def _check_size(result: WheelCheckResult, max_bytes: int) -> None:
result.add(
"size-limit",
result.file_bytes <= max_bytes,
f"{result.file_bytes} bytes (limit {max_bytes})",
)
def inspect_wheel(
wheel_path: Path,
*,
expected_version: str | None = None,
repo_root: Path | None = None,
max_bytes: int = MAX_WHEEL_BYTES,
strict_bundle: bool = True,
) -> WheelCheckResult:
"""Validate one built wheel without installing it.
``strict_bundle`` controls how the wheel's bundled files are compared with the
current source tree: ``True`` (gate a fresh build, used by
``release preflight``) fails on any missing or changed declared file;
``False`` (audit an already-released wheel) reports the same findings as
warnings.
"""
wheel_path = Path(wheel_path).resolve()
if repo_root is None:
repo_root = REPO_ROOT if (REPO_ROOT / "pyproject.toml").is_file() else None
else:
repo_root = Path(repo_root).resolve()
result = WheelCheckResult(wheel=wheel_path)
if not wheel_path.is_file():
result.add("wheel-exists", False, f"not a file: {wheel_path}")
return result
result.file_bytes = wheel_path.stat().st_size
result.add("wheel-exists", True, str(wheel_path))
_check_filename(result, expected_version)
_check_platform_tag(result)
_check_size(result, max_bytes)
try:
zf = zipfile.ZipFile(wheel_path)
except zipfile.BadZipFile as exc:
result.add("zip-integrity", False, f"not a valid zip: {exc}")
return result
with zf:
corrupt = zf.testzip()
result.add(
"zip-integrity",
corrupt is None,
"CRC check passed" if corrupt is None else f"corrupt entry: {corrupt}",
)
names = zf.namelist()
name_set = set(names)
result.entry_count = len(names)
result.uncompressed_bytes = sum(info.file_size for info in zf.infolist())
_check_archive_paths(result, names)
_check_forbidden_entries(result, names)
message = _metadata_message(zf, result)
_check_metadata(result, message, expected_version, repo_root)
_check_compiled_only(result, names)
_check_bundle_fidelity(result, zf, name_set, repo_root, strict_bundle)
_check_record(result, zf, names)
return result
build/release/__init__.py
+7
View File
@@ -0,0 +1,7 @@
"""Local release tooling: preflight gate and per-wheel audit.
Entry point is ``python build/run.py release <command>``; see
:mod:`release.preflight` for the gate steps and :mod:`release.cli` for the CLI.
Imports are intentionally not re-exported here so importing a submodule never
triggers the whole subsystem.
"""
build/release/cli.py
+163
View File
@@ -0,0 +1,163 @@
"""Command line for ``python build/run.py release``."""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
_BUILD_ROOT = Path(__file__).resolve().parents[1]
if str(_BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(_BUILD_ROOT))
from common.paths import REPO_ROOT # noqa: E402
from common.verify.wheel import ( # noqa: E402
DEFAULT_RUNTIME_CONFIG,
default_verification_video,
verify_wheel_assets,
verify_wheel_runtime,
)
from common.verify.wheel_contents import inspect_wheel # noqa: E402
from release.preflight import PreflightOptions, run_preflight # noqa: E402
def _cmd_preflight(argv: list[str]) -> int:
parser = argparse.ArgumentParser(
prog="build/run.py release preflight",
description="Gate a release locally: version, tests, wheel content, runtime.",
)
parser.add_argument("--version", help="release version (default: pyproject's dynamic version)")
parser.add_argument("--wheel-dir", type=Path, help="directory to inspect (default: SDK/<version>)")
parser.add_argument("--video", type=Path, help="clip for the end-to-end runtime check")
parser.add_argument("--config", default=DEFAULT_RUNTIME_CONFIG, help="config name for that check")
parser.add_argument("--frames", type=int, default=40, help="frames to process (default: 40)")
parser.add_argument("--skip-tests", action="store_true", help="skip pytest (recorded in the report)")
parser.add_argument("--longrun", action="store_true", help="include the slow long-run stability test")
parser.add_argument("--allow-dirty", action="store_true", help="allow a dirty working tree")
parser.add_argument("--force", action="store_true", help="allow re-staging an existing SDK/<version>")
parser.add_argument("--check-pypi", action="store_true", help="read PyPI and fail if the version exists")
parser.add_argument(
"--all-wheel-pythons",
action="store_true",
help="verify cp310/cp311/cp312 wheels, each in its own conda env",
)
parser.add_argument("--offline", action="store_true", help="reuse parent-env deps instead of PyPI")
parser.add_argument("--no-runtime", action="store_true", help="skip the isolated runtime check")
parser.add_argument("--no-assets", action="store_true", help="skip the bundled-asset check")
parser.add_argument("--keep-going", action="store_true", help="collect every failure instead of stopping")
parser.add_argument("--report-dir", type=Path, help="override build/release/reports")
args = parser.parse_args(argv)
options = PreflightOptions(
version=args.version,
wheel_dir=args.wheel_dir,
video=args.video,
config_name=args.config,
frames=args.frames,
skip_tests=args.skip_tests,
longrun=args.longrun,
allow_dirty=args.allow_dirty,
force=args.force,
check_pypi=args.check_pypi,
all_wheel_pythons=args.all_wheel_pythons,
offline=args.offline,
keep_going=args.keep_going,
report_dir=args.report_dir,
no_runtime=args.no_runtime,
no_assets=args.no_assets,
)
return run_preflight(options, repo_root=REPO_ROOT)
def _cmd_audit(argv: list[str]) -> int:
parser = argparse.ArgumentParser(
prog="build/run.py release audit",
description="Inspect one built wheel without a release context.",
)
parser.add_argument("wheel", type=Path, help="path to an .whl")
parser.add_argument("--expected-version", help="assert the wheel reports this version")
parser.add_argument(
"--strict-bundle",
action="store_true",
help="treat bundled-file drift from the current source tree as a failure",
)
parser.add_argument("--runtime", action="store_true", help="also install and run the wheel")
parser.add_argument("--assets", action="store_true", help="also assert bundled assets load")
parser.add_argument("--video", type=Path, help="clip for --runtime")
parser.add_argument("--config", default=DEFAULT_RUNTIME_CONFIG)
parser.add_argument("--frames", type=int, default=40)
parser.add_argument("--offline", action="store_true")
parser.add_argument(
"--python",
type=Path,
default=None,
help="interpreter to build the verification venv from (default: the running one)",
)
args = parser.parse_args(argv)
wheel = args.wheel
if not wheel.is_file():
print(f"[ERROR] wheel not found: {wheel}", file=sys.stderr)
return 2
exit_code = 0
result = inspect_wheel(
wheel,
expected_version=args.expected_version,
repo_root=REPO_ROOT,
strict_bundle=args.strict_bundle,
)
print(result.render())
if not result.ok:
exit_code = 1
if args.assets or args.runtime:
try:
print(verify_wheel_assets(wheel, python_executable=args.python))
except Exception as exc:
print(f"[FAIL] bundled assets: {type(exc).__name__}: {exc}")
exit_code = 1
if args.runtime:
video = args.video or default_verification_video(REPO_ROOT)
if video is None:
print("[FAIL] runtime: no verification clip; pass --video or set ORISYS_VERIFY_VIDEO")
return 1
try:
info = verify_wheel_runtime(
wheel,
video=video,
config_name=args.config,
frames=args.frames,
python_executable=args.python,
offline=args.offline,
)
print(f"[ok ] runtime: {info}")
except Exception as exc:
print(f"[FAIL] runtime: {type(exc).__name__}: {exc}")
exit_code = 1
return exit_code
def main(argv: list[str] | None = None) -> int:
argv = list(sys.argv[1:] if argv is None else argv)
if not argv or argv[0] in ("-h", "--help"):
print(__doc__)
print("Commands: preflight | audit")
return 0
command, rest = argv[0], argv[1:]
if command == "preflight":
return _cmd_preflight(rest)
if command == "audit":
return _cmd_audit(rest)
print(f"Unknown release command: {command!r}", file=sys.stderr)
print("Choose from: preflight, audit", file=sys.stderr)
return 2
if __name__ == "__main__":
raise SystemExit(main())
build/release/preflight.py
+515
View File
@@ -0,0 +1,515 @@
"""Local release gate: everything that must hold before publishing a wheel.
``release preflight`` answers one question with an exit code: is this checkout,
this version and these wheels safe to publish? It never publishes anything, and
by default it makes no network calls.
Steps, in order:
1. ``repo-state`` - git repository, clean working tree, recorded commit
2. ``version-consistency`` - __version__ == pyproject dynamic attr == wheel names
3. ``duplicate-guard`` - SDK/<version>/ already populated; optionally PyPI
4. ``tests`` - pytest (slow/hardware markers excluded)
5. ``bundle-precheck`` - every bundle item resolves; required assets present
6. ``wheel-inspection`` - static metadata/content assertions per wheel
7. ``wheel-runtime`` - isolated venv: bundled assets load and real frames
flow through the public pipeline
Output lands in ``build/release/reports/<version>/preflight.{json,txt}``.
"""
from __future__ import annotations
import hashlib
import json
import platform
import subprocess
import sys
import time
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
_BUILD_ROOT = Path(__file__).resolve().parents[1]
if str(_BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(_BUILD_ROOT))
from common.paths import REPO_ROOT # noqa: E402
from common.pyproject_utils import bundle_items, get_version # noqa: E402
from common.verify.wheel import ( # noqa: E402
DEFAULT_RUNTIME_CONFIG,
default_verification_video,
verify_wheel_assets,
verify_wheel_runtime,
)
from common.verify.wheel_contents import inspect_wheel # noqa: E402
from release import report as report_mod # noqa: E402
from release import version as version_mod # noqa: E402
from wheel.layout import resolve_bundle_source_files, verify_required_assets # noqa: E402
@dataclass
class Step:
name: str
ok: bool
detail: str = ""
duration_s: float = 0.0
data: dict = field(default_factory=dict)
@dataclass
class PreflightOptions:
version: str | None = None
wheel_dir: Path | None = None
video: Path | None = None
config_name: str = DEFAULT_RUNTIME_CONFIG
frames: int = 40
skip_tests: bool = False
longrun: bool = False
allow_dirty: bool = False
force: bool = False
check_pypi: bool = False
all_wheel_pythons: bool = False
offline: bool = False
keep_going: bool = False
report_dir: Path | None = None
no_runtime: bool = False
no_assets: bool = False
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with Path(path).open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _now() -> str:
return datetime.now(timezone.utc).isoformat(timespec="seconds")
class Preflight:
def __init__(self, options: PreflightOptions, repo_root: Path | None = None) -> None:
self.options = options
self.repo_root = Path(repo_root or REPO_ROOT)
self.steps: list[Step] = []
self.git: dict = {"commit": None, "branch": None, "dirty": None}
self.wheels: list[Path] = []
self.wheel_dir: Path | None = None
self.runtime_records: list[dict] = []
self.aborted = False
self.ok = False
if options.version:
self.version = options.version
self.version_source = "explicit --version"
elif (self.repo_root / "pyproject.toml").is_file():
self.version = get_version(self.repo_root)
self.version_source = "pyproject.toml dynamic attr"
else:
self.version = "0.0.0"
self.version_source = "unresolved"
self.report_dir = report_mod.reports_root(self.repo_root, options.report_dir) / self.version
# -- helpers ---------------------------------------------------------
def _git(self, *args: str) -> str:
try:
completed = subprocess.run(
["git", *args],
cwd=self.repo_root,
capture_output=True,
text=True,
check=False,
)
except OSError:
return ""
return (completed.stdout or "").strip()
def _execute(self, step_fn) -> None:
started = time.perf_counter()
try:
step = step_fn()
except Exception as exc: # a crashed step is a failed step, never a silent pass
step = Step(name=getattr(step_fn, "__name__", "step").lstrip("_").replace("_", "-"),
ok=False, detail=f"crashed: {type(exc).__name__}: {exc}")
step.duration_s = round(time.perf_counter() - started, 3)
self.steps.append(step)
flag = "ok " if step.ok else "FAIL"
print(f"[{flag}] {step.name} ({step.duration_s:.2f}s)")
for line in str(step.detail).splitlines():
print(f" {line}")
if not step.ok and not self.options.keep_going:
self.aborted = True
print(" stopping: pass --keep-going to collect every failure")
# -- steps -----------------------------------------------------------
def _repo_state(self) -> Step:
if not (self.repo_root / ".git").exists():
return Step("repo-state", False, f"{self.repo_root} is not a git repository")
commit = self._git("rev-parse", "HEAD")
branch = self._git("rev-parse", "--abbrev-ref", "HEAD")
porcelain = self._git("status", "--porcelain")
dirty = bool(porcelain)
self.git = {"commit": commit, "branch": branch, "dirty": dirty}
if dirty and not self.options.allow_dirty:
preview = "\n".join(porcelain.splitlines()[:20])
return Step(
"repo-state",
False,
f"working tree is dirty; commit, stash or pass --allow-dirty:\n{preview}",
data=self.git,
)
detail = f"{branch} @ {commit[:12]} dirty={dirty} version={self.version} ({self.version_source})"
if dirty:
detail += " (accepted via --allow-dirty)"
return Step("repo-state", True, detail, data=self.git)
def _version_consistency(self) -> Step:
problems: list[str] = []
data: dict = {"version": self.version, "source": self.version_source}
attr = version_mod.dynamic_version_attr(self.repo_root)
data["dynamic_attr"] = attr
if attr is None:
problems.append(
"pyproject.toml no longer declares tool.setuptools.dynamic.version.attr"
)
if (self.repo_root / "pyproject.toml").is_file():
resolved = get_version(self.repo_root)
data["resolved_version"] = resolved
if resolved != self.version:
problems.append(
f"src/orisys/__init__.py __version__ is {resolved!r} but the release "
f"version is {self.version!r}"
)
sdk_dirs = version_mod.sdk_versions(self.repo_root)
data["sdk_versions"] = sdk_dirs
for wheel in version_mod.sdk_wheels(self.repo_root, self.version):
found = version_mod.wheel_version(wheel)
if found != self.version:
problems.append(f"{wheel.name} carries version {found!r}, expected {self.version!r}")
detail = (
f"version={self.version} ({self.version_source}) dynamic_attr={attr}\n"
f"SDK/ versions present: {sdk_dirs or 'none'}"
)
if problems:
detail += "\n" + "\n".join(problems)
return Step("version-consistency", not problems, detail, data=data)
def _duplicate_guard(self) -> Step:
problems: list[str] = []
lines: list[str] = []
existing = version_mod.sdk_wheels(self.repo_root, self.version)
if existing:
if self.options.force:
lines.append(
f"SDK/{self.version}/ already holds {len(existing)} wheel(s); "
"overwriting because --force was passed"
)
else:
problems.append(
f"SDK/{self.version}/ already holds {len(existing)} wheel(s): "
f"{[w.name for w in existing]}. Pass --force to rebuild over them."
)
else:
lines.append(f"SDK/{self.version}/ is empty (no wheel has been staged yet)")
if self.options.check_pypi:
versions, error = version_mod.pypi_released_versions()
if error:
problems.append(f"PyPI lookup failed: {error}")
elif versions is None:
problems.append("PyPI lookup returned no version list")
elif self.version in versions:
problems.append(
f"orisys {self.version} is already published on PyPI; a PyPI upload "
"cannot be undone. Published: " + ", ".join(versions)
)
else:
lines.append(
f"PyPI has no {self.version}; published versions: " + (", ".join(versions) or "none")
)
else:
lines.append("PyPI existence check skipped (pass --check-pypi to enable)")
detail = "\n".join(lines)
if problems:
detail += "\n" + "\n".join(problems)
return Step("duplicate-guard", not problems, detail)
def _tests(self) -> Step:
if self.options.skip_tests:
return Step(
"tests",
True,
"WARNING: test gate SKIPPED via --skip-tests; this run cannot certify a release",
data={"status": "skipped"},
)
marker = "not hardware" if self.options.longrun else "not slow and not hardware"
self.report_dir.mkdir(parents=True, exist_ok=True)
junit = self.report_dir / "junit.xml"
command = [
sys.executable,
"-m",
"pytest",
"-q",
"-m",
marker,
f"--junitxml={junit}",
]
try:
completed = subprocess.run(
command, cwd=self.repo_root, capture_output=True, text=True, check=False
)
except FileNotFoundError as exc:
return Step("tests", False, f"cannot run pytest: {exc}")
ok = completed.returncode == 0
tail = ((completed.stdout or "") + (completed.stderr or "")).strip().splitlines()[-12:]
detail = f"pytest -m {marker!r} in {sys.executable}\nexit code {completed.returncode}"
if tail:
detail += "\n" + "\n".join(tail)
return Step("tests", ok, detail, data={"marker": marker, "junit": str(junit)})
def _bundle_precheck(self) -> Step:
if not (self.repo_root / "pyproject.toml").is_file():
return Step("bundle-precheck", False, "pyproject.toml not found")
items = bundle_items(self.repo_root)
problems: list[str] = []
counts: dict[str, int] = {}
for item in items:
files = resolve_bundle_source_files(self.repo_root, item)
counts[str(item.get("name"))] = len(files)
if not files:
problems.append(
f"bundle item {item.get('name')!r} resolved 0 files; its patterns match nothing"
)
try:
verify_required_assets(self.repo_root, items)
except FileNotFoundError as exc:
problems.append(str(exc))
detail = "resolved files per bundle item: " + json.dumps(counts)
if problems:
detail += "\n" + "\n".join(problems)
return Step("bundle-precheck", not problems, detail, data={"counts": counts})
def _discover_wheels(self) -> str | None:
"""Populate self.wheels; return an error message when nothing was found."""
directory = (
Path(self.options.wheel_dir)
if self.options.wheel_dir
else self.repo_root / "SDK" / self.version
)
self.wheel_dir = directory
if not directory.is_dir():
return (
f"{directory} does not exist. Build first "
"(`python build/run.py wheel`) or pass --wheel-dir."
)
self.wheels = sorted(directory.glob("*.whl"))
if not self.wheels:
return f"no *.whl found in {directory}. Build first or pass --wheel-dir."
return None
def _wheel_inspection(self) -> Step:
problem = self._discover_wheels()
if problem:
return Step("wheel-inspection", False, problem)
problems: list[str] = []
blocks: list[str] = []
for wheel in self.wheels:
result = inspect_wheel(
wheel,
expected_version=self.version,
repo_root=self.repo_root,
strict_bundle=True,
)
blocks.append(result.render())
if not result.ok:
problems.append(
f"{wheel.name}: " + ", ".join(check.name for check in result.failures)
)
detail = f"{len(self.wheels)} wheel(s) in {self.wheel_dir}\n" + "\n".join(blocks)
if problems:
detail += "\nfailed checks:\n" + "\n".join(problems)
return Step("wheel-inspection", not problems, detail, data={"count": len(self.wheels)})
def _platform_token(self) -> str:
return "win_amd64" if sys.platform == "win32" else "manylinux"
def _runtime_targets(self) -> list[tuple[str, Path, Path | None]]:
token = self._platform_token()
if self.options.all_wheel_pythons:
targets: list[tuple[str, Path, Path | None]] = []
for tag in ("cp310", "cp311", "cp312"):
match = next(
(w for w in self.wheels if f"-{tag}-" in w.name and token in w.name), None
)
if match is None:
raise RuntimeError(
f"no {tag} {token} wheel among {[w.name for w in self.wheels]}"
)
executable = version_mod.python_for_tag(tag)
if executable is None:
raise RuntimeError(
f"cannot resolve an interpreter for {tag}; create the conda env "
f"py{tag[2:]} or set CONDA_ENVS_ROOT"
)
targets.append((tag, match, executable))
return targets
tag = f"cp{sys.version_info.major}{sys.version_info.minor}"
match = next((w for w in self.wheels if f"-{tag}-" in w.name and token in w.name), None)
if match is None:
raise RuntimeError(
f"no {tag} {token} wheel among {[w.name for w in self.wheels]}; pass "
"--all-wheel-pythons to verify every ABI in its own environment, or run from "
"a matching interpreter"
)
return [(tag, match, None)]
def _wheel_runtime(self) -> Step:
if self.options.no_runtime:
return Step(
"wheel-runtime",
True,
"WARNING: runtime verification SKIPPED via --no-runtime",
data={"status": "skipped"},
)
video = self.options.video or default_verification_video(self.repo_root)
if video is None:
return Step(
"wheel-runtime",
False,
"no verification clip found. Pass --video, set ORISYS_VERIFY_VIDEO, or commit a "
"clip under data/sample/. The runtime check is never silently skipped.",
)
video = Path(video)
if not video.is_file():
return Step("wheel-runtime", False, f"verification clip not found: {video}")
try:
targets = self._runtime_targets()
except RuntimeError as exc:
return Step("wheel-runtime", False, str(exc))
problems: list[str] = []
lines: list[str] = [f"clip: {video}", f"config: {self.options.config_name}"]
for tag, wheel, executable in targets:
label = f"{tag} {wheel.name}"
try:
if self.options.no_assets:
assets_out = "asset check skipped (--no-assets)"
else:
assets_out = verify_wheel_assets(wheel, python_executable=executable)
runtime_out = verify_wheel_runtime(
wheel,
video=video,
config_name=self.options.config_name,
frames=self.options.frames,
python_executable=executable,
offline=self.options.offline,
)
except subprocess.CalledProcessError as exc:
problems.append(f"{label}: command failed ({exc})")
continue
except Exception as exc:
problems.append(f"{label}: {type(exc).__name__}: {exc}")
continue
lines.append(f"{label}: {assets_out} | frames={runtime_out['frames']} flow={runtime_out['flow_shape']}")
self.runtime_records.append({"tag": tag, "wheel": wheel.name, **runtime_out})
detail = "\n".join(lines)
if problems:
detail += "\nfailures:\n" + "\n".join(problems)
return Step("wheel-runtime", not problems, detail, data={"records": self.runtime_records})
# -- driver ----------------------------------------------------------
def run(self) -> int:
started_at = _now()
started = time.perf_counter()
print(f"=== orisys release preflight: version {self.version} ({self.version_source}) ===")
print(f"repo: {self.repo_root}")
print(f"report: {self.report_dir}")
print()
try:
for step_fn in (
self._repo_state,
self._version_consistency,
self._duplicate_guard,
self._tests,
self._bundle_precheck,
self._wheel_inspection,
self._wheel_runtime,
):
self._execute(step_fn)
if self.aborted:
break
finally:
self.ok = bool(self.steps) and all(step.ok for step in self.steps) and not self.aborted
duration = round(time.perf_counter() - started, 3)
payload = {
"version": self.version,
"version_source": self.version_source,
"ok": self.ok,
"aborted": self.aborted,
"started_at": started_at,
"finished_at": _now(),
"duration_s": duration,
"git": self.git,
"interpreter": {
"executable": sys.executable,
"version": platform.python_version(),
"platform": sys.platform,
},
"options": {key: value for key, value in asdict(self.options).items()
if not isinstance(value, Path)},
"steps": [asdict(step) for step in self.steps],
"wheels": [
{
"name": wheel.name,
"size": wheel.stat().st_size,
"sha256": _sha256_file(wheel),
}
for wheel in self.wheels
],
}
payload["options"]["wheel_dir"] = str(self.options.wheel_dir) if self.options.wheel_dir else None
payload["options"]["video"] = str(self.options.video) if self.options.video else None
json_path, text_path = report_mod.write_report(payload, self.report_dir)
print()
print(f"report: {json_path}")
print(f"report: {text_path}")
print()
print(f"=== preflight {'PASSED' if self.ok else 'FAILED'} ({duration}s) ===")
return 0 if self.ok else 1
def run_preflight(options: PreflightOptions, repo_root: Path | None = None) -> int:
return Preflight(options, repo_root=repo_root).run()
build/release/report.py
+71
View File
@@ -0,0 +1,71 @@
"""Structured preflight reports.
A preflight run writes both a machine-readable ``preflight.json`` (the contract a
future publish step will consume and re-verify) and a human ``preflight.txt``.
"""
from __future__ import annotations
import json
from pathlib import Path
REPORTS_RELPATH = Path("build") / "release" / "reports"
SCHEMA_VERSION = 1
def reports_root(repo_root: Path, override: Path | None = None) -> Path:
return Path(override) if override else Path(repo_root) / REPORTS_RELPATH
def render_text(payload: dict) -> str:
git = payload.get("git") or {}
interpreter = payload.get("interpreter") or {}
lines = [
f"orisys release preflight :: version {payload.get('version')}",
f" result: {'PASS' if payload.get('ok') else 'FAIL'}",
f" git: {git.get('branch')} @ {git.get('commit')} (dirty={git.get('dirty')})",
f" interpreter: {interpreter.get('version')} at {interpreter.get('executable')}",
f" duration: {payload.get('duration_s')}s",
"",
]
for step in payload.get("steps", []):
flag = "ok " if step.get("ok") else "FAIL"
lines.append(f"[{flag}] {step.get('name')} ({step.get('duration_s')}s)")
for detail_line in str(step.get("detail") or "").splitlines():
lines.append(f" {detail_line}")
lines.append("")
wheels = payload.get("wheels") or []
if wheels:
lines.append("wheels:")
for wheel in wheels:
lines.append(
f" {wheel.get('name')} {wheel.get('size')} bytes sha256={wheel.get('sha256')}"
)
lines.append("")
options = payload.get("options") or {}
waived = [
key
for key, value in options.items()
if value and key in {"skip_tests", "allow_dirty", "force", "no_runtime", "no_assets"}
]
if waived:
lines.append(f"waived guards: {', '.join(sorted(waived))}")
lines.append("")
return "\n".join(lines).rstrip() + "\n"
def write_report(payload: dict, out_dir: Path) -> tuple[Path, Path]:
out_dir = Path(out_dir)
out_dir.mkdir(parents=True, exist_ok=True)
payload = {**payload, "schema": SCHEMA_VERSION}
json_path = out_dir / "preflight.json"
text_path = out_dir / "preflight.txt"
json_path.write_text(json.dumps(payload, indent=2, sort_keys=False), encoding="utf-8")
text_path.write_text(render_text(payload), encoding="utf-8")
return json_path, text_path
build/release/version.py
+141
View File
@@ -0,0 +1,141 @@
"""Version consistency and duplicate-release guards for the release gate."""
from __future__ import annotations
import json
import os
import re
import urllib.error
import urllib.request
from pathlib import Path
PYPI_JSON_URL = "https://pypi.org/pypi/orisys/json"
# orisys-<version>-<pytag>-<abitag>-<platformtag>.whl
_WHEEL_NAME_RE = re.compile(r"^orisys-(?P<version>[^-]+)-[^-]+-[^-]+-.+\.whl$")
def wheel_version(wheel_path: Path) -> str | None:
match = _WHEEL_NAME_RE.match(Path(wheel_path).name)
return match.group("version") if match else None
def sdk_versions(repo_root: Path) -> list[str]:
"""Version directory names present under ``SDK/``, in ascending order.
Only digit-leading names count as versions, so stray directories (``tmp``,
``latest``) never appear as releases.
"""
sdk_root = Path(repo_root) / "SDK"
if not sdk_root.is_dir():
return []
return sorted(
(
entry.name
for entry in sdk_root.iterdir()
if entry.is_dir() and entry.name[:1].isdigit()
),
key=_version_sort_key,
)
def sdk_wheels(repo_root: Path, version: str) -> list[Path]:
"""Wheels already published into ``SDK/<version>/``."""
version_dir = Path(repo_root) / "SDK" / version
if not version_dir.is_dir():
return []
return sorted(version_dir.glob("*.whl"))
def dynamic_version_attr(repo_root: Path) -> str | None:
"""Return pyproject's configured dynamic version attr (e.g. orisys.__version__)."""
from common.pyproject_utils import load_pyproject
config = (
load_pyproject(Path(repo_root))
.get("tool", {})
.get("setuptools", {})
.get("dynamic", {})
.get("version")
)
if isinstance(config, dict):
attr = config.get("attr")
if isinstance(attr, str):
return attr
return None
def pypi_released_versions(timeout: float = 20.0) -> tuple[list[str] | None, str | None]:
"""Read the published version list from PyPI.
Returns ``(versions, error)``; ``versions`` is ``None`` when the lookup
failed. This is the only network call in the release gate and it is
read-only, so it is safe to run before an irreversible upload.
"""
request = urllib.request.Request(
PYPI_JSON_URL,
headers={"User-Agent": "orisys-release-preflight", "Accept": "application/json"},
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
payload = json.load(response)
except urllib.error.HTTPError as exc:
if exc.code == 404:
return [], None
return None, f"PyPI returned HTTP {exc.code}"
except Exception as exc: # network, DNS, TLS, JSON
return None, f"{type(exc).__name__}: {exc}"
releases = payload.get("releases")
if not isinstance(releases, dict):
return None, "unexpected PyPI response shape (no 'releases' mapping)"
listed = [
str(version)
for version, files in releases.items()
if isinstance(files, list) and files
]
return sorted(listed, key=_version_sort_key), None
def _version_sort_key(name: str):
parts = re.findall(r"\d+", name)
return ([int(part) for part in parts] or [0], name)
def conda_envs_root() -> Path | None:
"""Locate the conda envs directory the same way build/wheel/win.bat does."""
override = os.getenv("CONDA_ENVS_ROOT")
if override:
path = Path(override)
return path if path.is_dir() else None
home = Path.home()
candidates = (
home / "anaconda3" / "envs",
home / "miniconda3" / "envs",
Path(os.getenv("LOCALAPPDATA", "")) / "anaconda3" / "envs",
Path(os.getenv("LOCALAPPDATA", "")) / "miniconda3" / "envs",
)
for candidate in candidates:
if candidate.is_dir():
return candidate
return None
def python_for_tag(tag: str, envs_root: Path | None = None) -> Path | None:
"""Resolve an interpreter for a wheel tag like ``cp310`` (env ``py310``)."""
digits = tag[2:] if tag.startswith("cp") else tag
env_name = f"py{digits}"
root = envs_root or conda_envs_root()
if root is None:
return None
for relative in (
Path(env_name) / "python.exe",
Path(env_name) / "bin" / "python",
):
candidate = root / relative
if candidate.is_file():
return candidate
return None
build/run.py
+16 -2
View File
@@ -8,6 +8,8 @@ Usage (from repo root):
python build/run.py gui finger|mini|box
python build/run.py verify [wheel.whl]
python build/run.py verify --gui-dist [--product ory_finger]
python build/run.py release preflight [--version V]
python build/run.py release audit <wheel.whl>
See build/README.md for details.
"""
@@ -74,15 +76,23 @@ def _cmd_verify(argv: list[str]) -> int:
return verify_main(argv)
def _cmd_release(argv: list[str]) -> int:
from release.cli import main as release_main
return release_main(argv)
def main(argv: list[str] | None = None) -> int:
argv = list(sys.argv[1:] if argv is None else argv)
if not argv or argv[0] in ("-h", "--help"):
print(__doc__)
print("Commands: wheel | wheel-win | manylinux | gui | verify")
print("Commands: wheel | wheel-win | manylinux | gui | verify | release")
print("Examples:")
print(" python build/run.py wheel --help")
print(" python build/run.py gui finger")
print(" python build/run.py manylinux -PythonVersions 3.10")
print(" python build/run.py release preflight --version 0.5.2")
print(" python build/run.py release audit SDK\\0.5.2\\orisys-0.5.2-cp312-cp312-win_amd64.whl")
return 0
command, rest = argv[0], argv[1:]
@@ -92,10 +102,14 @@ def main(argv: list[str] | None = None) -> int:
"wheel-manylinux": _cmd_manylinux,
"gui": _cmd_gui,
"verify": _cmd_verify,
"release": _cmd_release,
}
if command not in dispatch:
print(f"Unknown command: {command!r}", file=sys.stderr)
print("Choose from: wheel, wheel-win, manylinux, gui, verify", file=sys.stderr)
print(
"Choose from: wheel, wheel-win, manylinux, gui, verify, release",
file=sys.stderr,
)
return 2
return dispatch[command](rest)
build/wheel/build.py
+10 -1
View File
@@ -30,7 +30,13 @@ if str(_BUILD_ROOT) not in sys.path:
sys.path.insert(0, str(_BUILD_ROOT))
from common.paths import NUITKA_COMPILE_DIR, NUITKA_WHEEL_DIR, REPO_ROOT, repaired_wheel_dir, repo_path
from common.pyproject_utils import bundle_items, extras_require, get_version, install_requires
from common.pyproject_utils import (
bundle_items,
extras_require,
get_version,
install_requires,
project_metadata,
)
from common.verify.wheel import verify_built_wheel
from wheel.compile import compile_orisys, copy_compiled_orisys, install_nuitka
from wheel.layout import (
@@ -41,6 +47,7 @@ from wheel.layout import (
from wheel.package import (
is_compliant_manylinux_wheel,
publish_wheel_to_sdk,
read_long_description,
repair_linux_wheel,
run_bdist_wheel,
write_wheel_setup,
@@ -122,6 +129,8 @@ def build(*, target: str = "internal", verify: bool | None = None) -> Path:
package_data=package_data_from_bundle(ext_suffix, bundle),
install_requires=install_requires(REPO_ROOT),
extras_require=extras_require(REPO_ROOT),
project=project_metadata(REPO_ROOT),
long_description=read_long_description(REPO_ROOT),
)
# --- 3. Build wheel, repair on Linux, publish to SDK/ ---
build/wheel/package.py
+83 -3
View File
@@ -19,6 +19,76 @@ _MANYLINUX_TAG_RE = re.compile(
r"(?:manylinux2014|manylinux_2_(?:17|24|27|28|31|34|35))_x86_64"
)
# Source of the PyPI project description (see read_long_description).
PYPI_README_RELPATH = Path("docs") / "pypi_readme.md"
def _license_text(project: dict) -> str | None:
"""Extract free-text license from either pyproject license form."""
declared = project.get("license")
if isinstance(declared, str):
return declared
if isinstance(declared, dict):
text = declared.get("text")
if isinstance(text, str):
return text
return None
def _person_strings(entries) -> list[str]:
"""Render PEP 621 author/maintainer tables as ``Name <email>`` strings."""
rendered: list[str] = []
for entry in entries or []:
if not isinstance(entry, dict):
continue
name = (entry.get("name") or "").strip()
email = (entry.get("email") or "").strip()
if name and email:
rendered.append(f"{name} <{email}>")
elif name or email:
rendered.append(name or email)
return rendered
def metadata_kwargs(project: dict) -> dict:
"""Translate a pyproject ``[project]`` table into ``setup()`` keywords.
The Nuitka wheel is built by a generated ``setup.py`` outside the normal
setuptools pipeline, so every PyPI-visible field has to be passed through
explicitly. Omitting them is what left the 0.5.2 release with no
``Requires-Python``, classifiers, project URLs or license on PyPI.
"""
authors = _person_strings(project.get("authors"))
maintainers = _person_strings(project.get("maintainers"))
candidates = {
"description": project.get("description"),
"python_requires": project.get("requires-python"),
"classifiers": list(project.get("classifiers") or []),
"project_urls": dict(project.get("urls") or {}),
"license": _license_text(project),
"author": authors[0] if authors else None,
"author_email": None,
"maintainer": maintainers[0] if maintainers else None,
}
return {key: value for key, value in candidates.items() if value}
def read_long_description(repo_root: Path) -> str | None:
"""Return the PyPI long description, or ``None`` when it is missing.
Deliberately not the repository ``README.md``: that file documents the
closed-source build/publishing layout, which should not end up on the PyPI
project page.
"""
path = Path(repo_root) / PYPI_README_RELPATH
if not path.is_file():
print(
f"[WARN] {PYPI_README_RELPATH.as_posix()} not found; the wheel will be "
"published without a long description."
)
return None
return path.read_text(encoding="utf-8")
def write_wheel_setup(
wheel_dir: Path,
@@ -27,7 +97,17 @@ def write_wheel_setup(
package_data: dict[str, list[str]],
install_requires: list[str],
extras_require: dict[str, list[str]],
project: dict | None = None,
long_description: str | None = None,
) -> Path:
metadata_lines = ""
if project:
for key, value in sorted(metadata_kwargs(project).items()):
metadata_lines += f" {key}={value!r},\n"
if long_description:
metadata_lines += " long_description=_LONG_DESCRIPTION,\n"
metadata_lines += ' long_description_content_type="text/markdown",\n'
setup_py_path = wheel_dir / "setup.py"
setup_content = f"""
from setuptools import setup, Distribution
@@ -36,12 +116,12 @@ class BinaryDistribution(Distribution):
def has_ext_modules(self):
return True
{'_LONG_DESCRIPTION = ' + repr(long_description) if long_description else ''}
setup(
name="orisys",
version="{version}",
description="Orisys - Optical Tactile Sensor SDK (Nuitka Compiled)",
author="Orisys",
packages=['orisys', 'orisys.examples', 'orisys_blend'],
{metadata_lines} packages=['orisys', 'orisys.examples', 'orisys_blend'],
package_dir={{'': 'wheel_root'}},
package_data={package_data},
include_package_data=True,