ykcheungab
4c1ab5c27e
Adds tools/api_audit.py, a read-only static audit of the public API surface, and docs/api-audit.md, its first run against this revision. The audit answers three questions the reference cannot answer about itself: 1. Does every declared export exist, have a docstring, and appear in docs/api.md? It reads the root __all__ plus each submodule's __all__. 2. How complete is each public definition? Docstring presence, detected docstring convention, parameter and return annotations. 3. Which public names do examples and tests actually use but the reference never explains? That set is the real documentation gap. It parses with ast and never imports the package, so it costs no numpy/cv2 import and cannot be affected by the environment. Findings on this revision (full detail in docs/api-audit.md): - 393 public classes/methods/functions; 65% have a docstring, 32% have a fully annotated signature, 38 document their return value. - 243 docstrings are unstructured prose, 137 are absent, 13 use Google or NumPy sections. There is effectively one convention: none. - 9 of the 18 root exports are absent from docs/api.md, including StreamPipeline, StreamRecorder, mosaic_videos and FFCFrameProcessor - the pipelined multi-stream and offline-recording paths. - Six sub-surfaces have zero coverage in the reference: unroll (18 exports), capture (17), hhd + solvers (7), pipeline (3), export (2). - 74 names are used by examples or tests but absent from the reference. - 21 public classes are exported nowhere; LowpassFilter is documented in docs/api.md and importable, but declared in no __all__. No behaviour change: a new tool and a new document, no source file touched.
425 lines
16 KiB
Python
425 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Audit the public API surface of the orisys SDK: what is declared, what is documented.
|
|
|
|
Read-only and static. Never imports the package, so it costs no numpy/cv2 import and cannot be
|
|
affected by the environment.
|
|
|
|
What it reports
|
|
1. Coverage of every declared export (the root ``__all__`` plus each submodule's ``__all__``):
|
|
where it lives, whether it has a docstring, docstring style, whether its parameters and
|
|
return value are annotated, and whether docs/api.md mentions it.
|
|
2. The docstring and annotation coverage of every public class and method, per module.
|
|
3. Public definitions that are *not* exported anywhere — candidates for hiding or for adding to
|
|
an ``__all__``.
|
|
4. Public names used by ``examples/`` and ``tests/`` but absent from docs/api.md. These are the
|
|
APIs users reach for that the reference does not explain.
|
|
|
|
Output is a text report by default, or ``--json`` for machine use.
|
|
|
|
Usage::
|
|
|
|
python tools/api_audit.py # text report
|
|
python tools/api_audit.py --json # machine-readable
|
|
python tools/api_audit.py --quiet # headline numbers only
|
|
|
|
Exit codes: 0 always. This is a report, not a gate; add thresholds when the numbers are known.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import ast
|
|
import io
|
|
import json
|
|
import re
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
# The report contains Chinese headings from the docs it inspects.
|
|
for _stream in (sys.stdout, sys.stderr):
|
|
try:
|
|
_stream.reconfigure(encoding="utf-8", errors="replace")
|
|
except (AttributeError, ValueError): # pragma: no cover
|
|
pass
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
SRC = REPO / "src" / "orisys"
|
|
DOC = REPO / "docs" / "api.md"
|
|
USAGE_DIRS = ("examples", "tests")
|
|
|
|
#: Docstring conventions, detected from the body. Order matters: the first match wins.
|
|
STYLE_PATTERNS = (
|
|
("google", re.compile(r"(?m)^\s*(Args|Arguments|Returns|Raises|Yields|Attributes)\s*:")),
|
|
("numpy", re.compile(r"(?m)^\s*(Parameters|Returns|Raises|Attributes)\s*\n\s*-{3,}")),
|
|
("sphinx", re.compile(r"(?m)^\s*:(param|type|returns?|rtype|raises)\b")),
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class Symbol:
|
|
"""One public definition, keyed by 'module:qualname'."""
|
|
|
|
module: str
|
|
qualname: str
|
|
kind: str # class | function | method
|
|
docstring: str
|
|
style: str
|
|
params: int = 0
|
|
params_annotated: int = 0
|
|
has_return_annotation: bool = False
|
|
has_return_section: bool = False
|
|
|
|
@property
|
|
def documented(self) -> bool:
|
|
return bool(self.docstring.strip())
|
|
|
|
@property
|
|
def signature_complete(self) -> bool:
|
|
return self.params == self.params_annotated and self.has_return_annotation
|
|
|
|
|
|
@dataclass
|
|
class Analysis:
|
|
symbols: dict[str, Symbol] = field(default_factory=dict)
|
|
classes: dict[str, list[str]] = field(default_factory=dict) # qualname -> method names
|
|
exported: dict[str, list[str]] = field(default_factory=dict) # source module -> names
|
|
root_all: list[str] = field(default_factory=list)
|
|
|
|
|
|
def module_name(path: Path) -> str:
|
|
rel = path.relative_to(SRC).as_posix()
|
|
if rel.endswith("/__init__.py"):
|
|
rel = rel[: -len("/__init__.py")]
|
|
elif rel == "__init__.py":
|
|
return "orisys"
|
|
elif rel.endswith(".py"):
|
|
rel = rel[:-3]
|
|
dotted = rel.replace("/", ".").removesuffix(".__init__")
|
|
return f"orisys.{dotted}" if dotted else "orisys"
|
|
|
|
|
|
def detect_style(doc: str) -> str:
|
|
if not doc.strip():
|
|
return "none"
|
|
for name, pattern in STYLE_PATTERNS:
|
|
if pattern.search(doc):
|
|
return name
|
|
return "plain"
|
|
|
|
|
|
def param_names(node: ast.FunctionDef | ast.AsyncFunctionDef) -> list[ast.arg]:
|
|
args = [*node.args.args, *node.args.kwonlyargs]
|
|
if node.args.vararg:
|
|
args.append(node.args.vararg)
|
|
if node.args.kwarg:
|
|
args.append(node.args.kwarg)
|
|
return [a for a in args if a.arg not in ("self", "cls")]
|
|
|
|
|
|
def has_return_section(doc: str) -> bool:
|
|
lowered = doc.lower()
|
|
return "returns:" in lowered or "return:" in lowered or ":returns:" in lowered or ":return:" in lowered
|
|
|
|
|
|
def collect() -> Analysis:
|
|
analysis = Analysis()
|
|
for path in sorted(SRC.rglob("*.py")):
|
|
try:
|
|
tree = ast.parse(path.read_text(encoding="utf-8"))
|
|
except SyntaxError:
|
|
continue
|
|
mod = module_name(path)
|
|
|
|
# __all__ declarations
|
|
for node in tree.body:
|
|
if isinstance(node, ast.Assign) and any(
|
|
getattr(t, "id", "") == "__all__" for t in node.targets
|
|
):
|
|
names = [e.value for e in getattr(node.value, "elts", []) if isinstance(e, ast.Constant)]
|
|
analysis.exported[mod] = names
|
|
if mod == "orisys":
|
|
analysis.root_all = names
|
|
|
|
for node in tree.body:
|
|
if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
if node.name.startswith("_"):
|
|
continue
|
|
doc = ast.get_docstring(node) or ""
|
|
if isinstance(node, ast.ClassDef):
|
|
key = f"{mod}:{node.name}"
|
|
analysis.symbols[key] = Symbol(mod, node.name, "class", doc, detect_style(doc))
|
|
methods: list[str] = []
|
|
for sub in node.body:
|
|
if isinstance(sub, (ast.FunctionDef, ast.AsyncFunctionDef)) and not sub.name.startswith("_"):
|
|
params = param_names(sub)
|
|
mdoc = ast.get_docstring(sub) or ""
|
|
mkey = f"{mod}:{node.name}.{sub.name}"
|
|
analysis.symbols[mkey] = Symbol(
|
|
mod, f"{node.name}.{sub.name}", "method", mdoc, detect_style(mdoc),
|
|
params=len(params),
|
|
params_annotated=sum(1 for a in params if a.annotation is not None),
|
|
has_return_annotation=sub.returns is not None,
|
|
has_return_section=has_return_section(mdoc),
|
|
)
|
|
methods.append(sub.name)
|
|
analysis.classes[node.name] = methods
|
|
else:
|
|
params = param_names(node)
|
|
key = f"{mod}:{node.name}"
|
|
analysis.symbols[key] = Symbol(
|
|
mod, node.name, "function", doc, detect_style(doc),
|
|
params=len(params),
|
|
params_annotated=sum(1 for a in params if a.annotation is not None),
|
|
has_return_annotation=node.returns is not None,
|
|
has_return_section=has_return_section(doc),
|
|
)
|
|
return analysis
|
|
|
|
|
|
def doc_names() -> tuple[set[str], str]:
|
|
"""Identifiers mentioned in docs/api.md, plus the raw text."""
|
|
text = DOC.read_text(encoding="utf-8") if DOC.is_file() else ""
|
|
names = set(re.findall(r"[A-Za-z_][A-Za-z0-9_]*", text))
|
|
return names, text
|
|
|
|
|
|
def usage_names() -> dict[str, set[str]]:
|
|
"""Public names imported or attribute-accessed by examples/ and tests/."""
|
|
used: dict[str, set[str]] = {}
|
|
for sub in USAGE_DIRS:
|
|
base = REPO / sub
|
|
if not base.is_dir():
|
|
continue
|
|
found: set[str] = set()
|
|
for path in base.rglob("*.py"):
|
|
try:
|
|
tree = ast.parse(path.read_text(encoding="utf-8"))
|
|
except (SyntaxError, OSError):
|
|
continue
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
for alias in node.names:
|
|
found.add(alias.name.split(".")[-1])
|
|
for part in alias.name.split("."):
|
|
found.add(part)
|
|
elif isinstance(node, ast.Attribute):
|
|
found.add(node.attr)
|
|
used[sub] = found
|
|
return used
|
|
|
|
|
|
def resolve_export(analysis: Analysis, name: str) -> Symbol | None:
|
|
"""Find where an exported name is actually defined."""
|
|
if name in analysis.classes:
|
|
for key, sym in analysis.symbols.items():
|
|
if sym.qualname == name and sym.kind == "class":
|
|
return sym
|
|
for key, sym in analysis.symbols.items():
|
|
if sym.qualname == name and sym.kind in ("class", "function"):
|
|
return sym
|
|
return None
|
|
|
|
|
|
def build_report(analysis: Analysis, names_in_doc: set[str], used: dict[str, set[str]]) -> dict:
|
|
symbols = analysis.symbols
|
|
|
|
docstrings = sum(1 for s in symbols.values() if s.documented)
|
|
annotated = sum(1 for s in symbols.values() if s.signature_complete)
|
|
returns_documented = sum(1 for s in symbols.values() if s.has_return_section)
|
|
|
|
styles: dict[str, int] = {}
|
|
for s in symbols.values():
|
|
styles[s.style] = styles.get(s.style, 0) + 1
|
|
|
|
per_module: dict[str, dict[str, int]] = {}
|
|
for s in symbols.values():
|
|
bucket = per_module.setdefault(s.module, {"total": 0, "documented": 0, "annotated": 0})
|
|
bucket["total"] += 1
|
|
bucket["documented"] += 1 if s.documented else 0
|
|
bucket["annotated"] += 1 if s.signature_complete else 0
|
|
|
|
# Declared exports. Submodule packages export their own names; the root re-exports some.
|
|
declared: list[dict] = []
|
|
for mod, names in sorted(analysis.exported.items()):
|
|
if mod == "orisys":
|
|
continue
|
|
for name in names:
|
|
sym = resolve_export(analysis, name)
|
|
declared.append({
|
|
"name": name,
|
|
"exported_by": mod,
|
|
"defined_in": sym.module if sym else None,
|
|
"in_api_md": name in names_in_doc,
|
|
"has_docstring": bool(sym and sym.documented),
|
|
"style": sym.style if sym else "n/a",
|
|
})
|
|
|
|
root_exports = []
|
|
for name in analysis.root_all:
|
|
sym = resolve_export(analysis, name)
|
|
root_exports.append({
|
|
"name": name,
|
|
"defined_in": sym.module if sym else "(submodule or re-export)",
|
|
"in_api_md": name in names_in_doc,
|
|
"has_docstring": bool(sym and sym.documented),
|
|
"style": sym.style if sym else "n/a",
|
|
})
|
|
|
|
exported_names = {d["name"] for d in declared} | set(analysis.root_all)
|
|
unexported = [
|
|
s.qualname
|
|
for s in symbols.values()
|
|
if s.kind == "class" and s.qualname not in exported_names
|
|
]
|
|
|
|
# A name used by examples/tests that the reference never explains. Restricted to names this
|
|
# package actually defines, so local variables and stdlib names do not appear.
|
|
defined_names = set(analysis.classes) | {s.qualname for s in symbols.values()}
|
|
used_but_undocumented = sorted({
|
|
name
|
|
for sub in used.values()
|
|
for name in sub
|
|
if name not in names_in_doc and name in defined_names
|
|
})
|
|
|
|
return {
|
|
"totals": {
|
|
"public_symbols": len(symbols),
|
|
"classes": sum(1 for s in symbols.values() if s.kind == "class"),
|
|
"methods": sum(1 for s in symbols.values() if s.kind == "method"),
|
|
"functions": sum(1 for s in symbols.values() if s.kind == "function"),
|
|
"with_docstring": docstrings,
|
|
"documented_pct": round(100 * docstrings / max(len(symbols), 1)),
|
|
"signature_complete": annotated,
|
|
"signature_complete_pct": round(100 * annotated / max(len(symbols), 1)),
|
|
"return_section": returns_documented,
|
|
},
|
|
"docstring_styles": dict(sorted(styles.items(), key=lambda kv: -kv[1])),
|
|
"per_module": dict(sorted(per_module.items())),
|
|
"root_exports": root_exports,
|
|
"declared_exports": declared,
|
|
"unexported_public_classes": sorted(unexported),
|
|
"used_but_undocumented": used_but_undocumented,
|
|
"usage_dirs": {k: len(v) for k, v in used.items()},
|
|
}
|
|
|
|
|
|
def print_text(report: dict) -> None:
|
|
t = report["totals"]
|
|
print("=" * 78)
|
|
print("orisys SDK public API audit")
|
|
print("=" * 78)
|
|
print(f" public classes/methods/functions : {t['public_symbols']} "
|
|
f"({t['classes']} classes, {t['methods']} methods, {t['functions']} functions)")
|
|
print(f" with a docstring : {t['with_docstring']} ({t['documented_pct']}%)")
|
|
print(f" fully annotated signature : {t['signature_complete']} ({t['signature_complete_pct']}%)")
|
|
print(f" docstring documents its return : {t['return_section']}")
|
|
print()
|
|
print(" docstring conventions in use:")
|
|
for style, count in report["docstring_styles"].items():
|
|
print(f" {style:<10} {count}")
|
|
|
|
print()
|
|
print("-" * 78)
|
|
print("ROOT `__all__` — the declared top-level API")
|
|
print("-" * 78)
|
|
print(f" {'name':<24}{'defined in':<22}{'in api.md':<11}{'docstring'}")
|
|
for row in report["root_exports"]:
|
|
print(f" {row['name']:<24}{str(row['defined_in']):<22}"
|
|
f"{'yes' if row['in_api_md'] else 'NO':<11}{'yes' if row['has_docstring'] else 'NO'}")
|
|
missing = [r["name"] for r in report["root_exports"] if not r["in_api_md"]]
|
|
print()
|
|
print(f" not mentioned in docs/api.md: {len(missing)} of {len(report['root_exports'])}")
|
|
if missing:
|
|
print(f" {', '.join(missing)}")
|
|
|
|
print()
|
|
print("-" * 78)
|
|
print("SUB-SURFACE EXPORTS — coverage per declaring module")
|
|
print("-" * 78)
|
|
by_mod: dict[str, list[dict]] = {}
|
|
for row in report["declared_exports"]:
|
|
by_mod.setdefault(row["exported_by"], []).append(row)
|
|
for mod, rows in by_mod.items():
|
|
in_doc = sum(1 for r in rows if r["in_api_md"])
|
|
documented = sum(1 for r in rows if r["has_docstring"])
|
|
print(f" {mod:<22} {len(rows):>3} exports in api.md: {in_doc:>3} "
|
|
f"have docstring: {documented:>3}")
|
|
undocumented = [r for r in report["declared_exports"] if not r["has_docstring"]]
|
|
print()
|
|
print(f" exports with NO docstring: {len(undocumented)} of {len(report['declared_exports'])}")
|
|
for row in undocumented[:20]:
|
|
print(f" {row['exported_by']:<22} {row['name']}")
|
|
if len(undocumented) > 20:
|
|
print(f" ... and {len(undocumented) - 20} more")
|
|
|
|
print()
|
|
print("-" * 78)
|
|
print("DOC STRING / ANNOTATION COVERAGE BY MODULE")
|
|
print("-" * 78)
|
|
print(f" {'module':<34}{'symbols':>8}{'docs':>7}{'docs%':>7}{'annot':>7}{'annot%':>7}")
|
|
for mod, b in report["per_module"].items():
|
|
if b["total"] < 3:
|
|
continue
|
|
print(f" {mod:<34}{b['total']:>8}{b['documented']:>7}"
|
|
f"{round(100*b['documented']/b['total']):>6}%"
|
|
f"{b['annotated']:>7}{round(100*b['annotated']/b['total']):>6}%")
|
|
|
|
print()
|
|
print("-" * 78)
|
|
print("USED BY EXAMPLES/TESTS BUT ABSENT FROM docs/api.md")
|
|
print("-" * 78)
|
|
used_undoc = report["used_but_undocumented"]
|
|
print(f" {len(used_undoc)} name(s)")
|
|
for name in used_undoc[:30]:
|
|
print(f" {name}")
|
|
if len(used_undoc) > 30:
|
|
print(f" ... and {len(used_undoc) - 30} more")
|
|
|
|
print()
|
|
print("-" * 78)
|
|
print("PUBLIC CLASSES NOT EXPORTED ANYWHERE — hide, or add to an `__all__`")
|
|
print("-" * 78)
|
|
unexp = report["unexported_public_classes"]
|
|
print(f" {len(unexp)} class(es)")
|
|
for name in unexp[:30]:
|
|
print(f" {name}")
|
|
if len(unexp) > 30:
|
|
print(f" ... and {len(unexp) - 30} more")
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument("--json", action="store_true", help="machine-readable report on stdout")
|
|
parser.add_argument("--quiet", action="store_true", help="headline numbers only")
|
|
args = parser.parse_args(argv)
|
|
|
|
if not SRC.is_dir():
|
|
print(f"api_audit: source not found at {SRC}", file=sys.stderr)
|
|
return 2
|
|
|
|
analysis = collect()
|
|
names_in_doc, _text = doc_names()
|
|
used = usage_names()
|
|
report = build_report(analysis, names_in_doc, used)
|
|
|
|
if args.json:
|
|
print(json.dumps(report, indent=2, ensure_ascii=False))
|
|
return 0
|
|
|
|
if args.quiet:
|
|
t = report["totals"]
|
|
print(f"public={t['public_symbols']} docs={t['documented_pct']}% "
|
|
f"annotated={t['signature_complete_pct']}% "
|
|
f"root_exports_missing_from_api_md="
|
|
f"{sum(1 for r in report['root_exports'] if not r['in_api_md'])}")
|
|
return 0
|
|
|
|
print_text(report)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|