Read the analysis cache before refusing a pass, and own the cache root
Opening a recording while the session that recorded it was still live never reached ``stored_result()``, so the panel reported no result over a recording whose result was on disk. The guard now follows the cache read, results move to the app's derived-data root (swept at startup, the old in-recording root ignored), and the panel states which of the four no-result states it is in, with the pass's own reason. Verified: apps/studio 462/0/18, shared 431/0/1, autocal 358, ui_check 628/0, both gates clean, real-GL snapshots per state.
This commit is contained in:
ykcheungab
@@ -18,6 +18,17 @@ properties matter more than speed:
|
||||
Storage is not free, so :func:`evict` removes whole results against a stated size cap.
|
||||
It only ever deletes a direct child of the cache root that carries a ``meta.json``, so
|
||||
a wrong path cannot turn into a recursive delete of something else.
|
||||
|
||||
**The root is the app's own derived-data directory, never the recording's folder.**
|
||||
Results used to be written to ``<recording>/analysis``, which put the app's own
|
||||
rebuildable data inside the operator's measurement — and, because the *root* is
|
||||
path-dependent while the *key* is not, made a result invisible as soon as the recording
|
||||
was opened from anywhere else (a copy, another machine, the same session on a different
|
||||
drive letter). ``paths.analysis_cache_dir()`` is the owner's decision: one root per user
|
||||
under ``%LOCALAPPDATA%\\OrisysStudio\\analysis-cache``, addressed by the result key
|
||||
alone. A result written under the old root is **ignored, not migrated and not deleted**:
|
||||
the app never writes into a recording folder again, and a leftover ``analysis``
|
||||
directory is left exactly as the operator's own tooling left it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -36,6 +47,7 @@ from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from orisys_app_shared import paths as paths_module
|
||||
from orisys_app_shared.analysis.model import (
|
||||
ANALYSIS_FORMAT,
|
||||
ANALYSIS_FORMAT_VERSION,
|
||||
@@ -59,14 +71,18 @@ from orisys_app_shared.analysis.model import (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_INCOMPLETE_GRACE_S",
|
||||
"DEFAULT_MAX_BYTES",
|
||||
"INCOMPLETE_MARKER",
|
||||
"LEGACY_RESULT_DIRNAME",
|
||||
"META_NAME",
|
||||
"AnalysisWriter",
|
||||
"ResultCache",
|
||||
"ResultSummary",
|
||||
"SweepReport",
|
||||
"evict",
|
||||
"scan_results",
|
||||
"sweep",
|
||||
]
|
||||
|
||||
#: The file that declares a finished result. Its presence is not enough — the
|
||||
@@ -90,6 +106,18 @@ MAX_CHUNK_FRAMES = 4096
|
||||
#: holds roughly fourteen of them; the operator can lower it per call.
|
||||
DEFAULT_MAX_BYTES = 1 << 30
|
||||
|
||||
#: How long an unfinished result is left alone by :func:`sweep`. A directory is written
|
||||
#: only while a pass is running in *this* process, and the sweep runs at startup before
|
||||
#: any pass exists, but a second copy of the app (or a build script) can share the root,
|
||||
#: and deleting a directory another process is writing would turn its pass into a
|
||||
#: mysterious failure. An hour is far longer than any pass here and still removes the
|
||||
#: garbage of the previous session.
|
||||
DEFAULT_INCOMPLETE_GRACE_S = 3600.0
|
||||
|
||||
#: Where a result used to live: ``<recording>/analysis``. Named so the old location can
|
||||
#: be *recognised* (and reported), never so it can be written to again.
|
||||
LEGACY_RESULT_DIRNAME = RESULT_DIRNAME
|
||||
|
||||
#: Above this many remembered grid digests the writer stops deduplicating and gives a
|
||||
#: deviating frame its own keyframe. Correct either way; the map exists to keep the
|
||||
#: common case (one grid for the whole recording) at one keyframe.
|
||||
@@ -242,6 +270,101 @@ def evict(
|
||||
return removed
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SweepReport:
|
||||
"""What one :func:`sweep` did: the garbage it removed, and what is left."""
|
||||
|
||||
removed: list[Path]
|
||||
#: The subset of :attr:`removed` that was unreadable by construction (an incomplete
|
||||
#: pass). Reported apart because removing these is never news, while removing a
|
||||
#: *complete* result is the size cap doing its job and worth seeing.
|
||||
incomplete: list[Path]
|
||||
freed_bytes: int
|
||||
remaining_bytes: int
|
||||
|
||||
@property
|
||||
def kept(self) -> bool:
|
||||
return not self.removed
|
||||
|
||||
|
||||
def sweep(
|
||||
root: str | Path,
|
||||
*,
|
||||
max_bytes: int = DEFAULT_MAX_BYTES,
|
||||
keep: Iterable[str | Path] = (),
|
||||
incomplete_grace_s: float = DEFAULT_INCOMPLETE_GRACE_S,
|
||||
now: float | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> SweepReport:
|
||||
"""Tidy one cache root: unfinished results first, then the size cap.
|
||||
|
||||
Two passes, in this order, because they answer different questions:
|
||||
|
||||
1. **Garbage.** A result that is not complete is refused by every default reader
|
||||
(Part A §7), so nothing can ever use it again; it is removed once it is older than
|
||||
``incomplete_grace_s``. A cancelled pass is *inspectable*, which is why the grace
|
||||
period exists rather than "delete on sight": the evidence of how far a pass got is
|
||||
worth keeping for a while.
|
||||
2. **Size.** :func:`evict` then removes whole results, incomplete first and oldest
|
||||
first, until the total fits ``max_bytes`` — never touching anything in ``keep``,
|
||||
and only ever a direct child of ``root`` carrying a ``meta.json``.
|
||||
|
||||
Returns what it removed and what is left, so a caller can log a real number instead
|
||||
of "swept". Never raises for a root that does not exist: the first run of an app has
|
||||
no cache, and that is not an error.
|
||||
"""
|
||||
base = Path(root)
|
||||
protected = {Path(item).resolve() for item in keep}
|
||||
if not base.is_dir():
|
||||
return SweepReport(removed=[], incomplete=[], freed_bytes=0, remaining_bytes=0)
|
||||
moment = time.time() if now is None else float(now)
|
||||
grace = max(0.0, float(incomplete_grace_s))
|
||||
start_bytes = sum(item.bytes_used for item in scan_results(base))
|
||||
removed: list[Path] = []
|
||||
incomplete: list[Path] = []
|
||||
for item in scan_results(base):
|
||||
if item.complete:
|
||||
continue
|
||||
if item.directory.resolve() in protected:
|
||||
continue
|
||||
age = moment - float(item.modified or 0.0)
|
||||
if item.modified and age < grace:
|
||||
continue
|
||||
if not (item.directory / META_NAME).is_file():
|
||||
# Same rule as ``evict``: only a result directory is a candidate, so a wrong
|
||||
# root cannot turn into a recursive delete of something else.
|
||||
continue
|
||||
if item.directory.parent.resolve() != base.resolve():
|
||||
continue
|
||||
removed.append(item.directory)
|
||||
incomplete.append(item.directory)
|
||||
if not dry_run:
|
||||
shutil.rmtree(item.directory, ignore_errors=False)
|
||||
evicted = evict(base, max_bytes=max_bytes, keep=keep, dry_run=dry_run)
|
||||
for path in evicted:
|
||||
if path not in removed:
|
||||
removed.append(path)
|
||||
if removed:
|
||||
logger.info(
|
||||
"analysis cache sweep: %s %d result(s) (%d incomplete)",
|
||||
"would remove" if dry_run else "removed",
|
||||
len(removed),
|
||||
len(incomplete),
|
||||
)
|
||||
# Measured after the fact rather than subtracted: ``evict`` decides what fits, and a
|
||||
# second measurement is the only honest answer to "what is left" (a dry run leaves
|
||||
# everything, which is what its number says).
|
||||
remaining = (
|
||||
start_bytes if dry_run else sum(item.bytes_used for item in scan_results(base))
|
||||
)
|
||||
return SweepReport(
|
||||
removed=removed,
|
||||
incomplete=incomplete,
|
||||
freed_bytes=max(0, start_bytes - remaining),
|
||||
remaining_bytes=remaining,
|
||||
)
|
||||
|
||||
|
||||
class AnalysisWriter:
|
||||
"""Append-only writer for one result directory.
|
||||
|
||||
@@ -570,22 +693,36 @@ class AnalysisWriter:
|
||||
|
||||
|
||||
class ResultCache:
|
||||
"""The results belonging to one recording, addressed by key.
|
||||
"""The stored results, addressed by key under the app's own derived-data root.
|
||||
|
||||
``root`` is usually ``<recording>/analysis``: a result then travels with the
|
||||
recording it describes, and deleting the recording deletes its cache. Nothing here
|
||||
writes to the current working directory — that matters because opening a *box*
|
||||
``Sensor`` drops a ~2 MB ``config/compound_eye.png`` relative to it (see
|
||||
``service.py``).
|
||||
``root`` defaults to :func:`~orisys_app_shared.paths.analysis_cache_dir` —
|
||||
``%LOCALAPPDATA%\\OrisysStudio\\analysis-cache`` — because a result is the app's
|
||||
rebuildable data and must not be written into the operator's recording folder. The
|
||||
*key* is what identifies a result (it ignores the recording's path), so the root is
|
||||
shared by every recording and a copied recording finds the same result.
|
||||
|
||||
A directory left under the old ``<recording>/analysis`` root is not read and not
|
||||
touched: see the module docstring.
|
||||
"""
|
||||
|
||||
def __init__(self, root: str | Path, *, max_bytes: int = DEFAULT_MAX_BYTES) -> None:
|
||||
self.root = Path(root)
|
||||
def __init__(self, root: str | Path | None = None, *, max_bytes: int = DEFAULT_MAX_BYTES) -> None:
|
||||
# Resolved at construction through the module, so a caller (or a test) can point
|
||||
# the app's derived-data root elsewhere without patching this class.
|
||||
self.root = (
|
||||
Path(paths_module.analysis_cache_dir()) if root is None else Path(root)
|
||||
)
|
||||
self.max_bytes = int(max_bytes)
|
||||
|
||||
@classmethod
|
||||
def for_recording(cls, recording_dir: str | Path, **kwargs: Any) -> ResultCache:
|
||||
return cls(Path(recording_dir) / RESULT_DIRNAME, **kwargs)
|
||||
"""The cache a recording's results live in.
|
||||
|
||||
The recording is named because that is how every caller thinks about it, and
|
||||
because the *key* — which is what actually identifies the result — is built from
|
||||
the recording's own metadata. It no longer decides the root.
|
||||
"""
|
||||
del recording_dir # the identity is in the key, not in the folder layout
|
||||
return cls(**kwargs)
|
||||
|
||||
def directory(self, key: ResultKey) -> Path:
|
||||
return self.root / key.digest
|
||||
@@ -621,5 +758,26 @@ class ResultCache:
|
||||
) -> list[Path]:
|
||||
return evict(self.root, max_bytes=self.max_bytes, keep=keep, dry_run=dry_run)
|
||||
|
||||
def sweep(
|
||||
self,
|
||||
*,
|
||||
keep: Iterable[str | Path] = (),
|
||||
incomplete_grace_s: float = DEFAULT_INCOMPLETE_GRACE_S,
|
||||
now: float | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> SweepReport:
|
||||
"""Tidy this root: unfinished results first, then the size cap.
|
||||
|
||||
The startup call site for the app's own cache; see :func:`sweep`.
|
||||
"""
|
||||
return sweep(
|
||||
self.root,
|
||||
max_bytes=self.max_bytes,
|
||||
keep=keep,
|
||||
incomplete_grace_s=incomplete_grace_s,
|
||||
now=now,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
def total_bytes(self) -> int:
|
||||
return sum(item.bytes_used for item in scan_results(self.root))
|
||||
|
||||
Reference in New Issue
Block a user