Read the analysis cache before refusing a pass, and own the cache root
Opening a recording while the session that recorded it was still live never reached ``stored_result()``, so the panel reported no result over a recording whose result was on disk. The guard now follows the cache read, results move to the app's derived-data root (swept at startup, the old in-recording root ignored), and the panel states which of the four no-result states it is in, with the pass's own reason. Verified: apps/studio 462/0/18, shared 431/0/1, autocal 358, ui_check 628/0, both gates clean, real-GL snapshots per state.
This commit is contained in:
ykcheungab
@@ -68,6 +68,31 @@ RAW_NOTE_KEY = "panels.replay_raw_note"
|
||||
#: line says which field is absent instead of the view looking broken.
|
||||
MISSING_NOTE_KEY = "panels.frame_missing_note"
|
||||
|
||||
#: The four states of "this recording has no result to draw from", which one line under
|
||||
#: the canvas has to tell apart. They are facts about the *analysis*, not about the
|
||||
#: frame: the same recorded picture carries all four, and the operator's next action is
|
||||
#: different in each. A single "no result yet" line was true in only the first of them,
|
||||
#: and its promise ("opening a recording runs a pass") is false in the other three.
|
||||
RESULT_NONE = "none"
|
||||
#: A pass is in flight: the result is coming, so waiting is the action.
|
||||
RESULT_RUNNING = "running"
|
||||
#: A pass ended without a result, or was refused before it started. ``result_reason``
|
||||
#: carries why — never a bare "failed", because "the chain failed on frame 41" and "a
|
||||
#: session is running so no chain was opened" send the operator to different places.
|
||||
RESULT_FAILED = "failed"
|
||||
#: No pass can produce one for this frame at all: a bare video has no session metadata
|
||||
#: and no analysis source, so the only way to numbers is the recompute pass.
|
||||
RESULT_IMPOSSIBLE = "impossible"
|
||||
RESULT_STATES = (RESULT_NONE, RESULT_RUNNING, RESULT_FAILED, RESULT_IMPOSSIBLE)
|
||||
|
||||
#: The line each state shows. The first is the oldest and the only one whose wording
|
||||
#: may promise the automatic pass, because it is the only state in which one is coming.
|
||||
RESULT_NOTE_KEYS = {
|
||||
RESULT_RUNNING: "panels.replay_analysis_running",
|
||||
RESULT_FAILED: "panels.replay_analysis_failed",
|
||||
RESULT_IMPOSSIBLE: "panels.replay_analysis_unavailable",
|
||||
}
|
||||
|
||||
#: Sources whose frames carry derived fields.
|
||||
DERIVED_SOURCES = (SOURCE_LIVE, SOURCE_RECOMPUTE, SOURCE_ANALYSIS)
|
||||
|
||||
@@ -193,6 +218,13 @@ class ImageFrame:
|
||||
#: that needs derived data is served from the result. ``False`` means no result — the
|
||||
#: only case in which "a recording stores only the raw camera image" is the truth.
|
||||
result_available: bool = False
|
||||
#: Which of the four no-result states the producer is in (see :data:`RESULT_STATES`),
|
||||
#: and why. Read only when there is no result to draw from; ``""`` means "the
|
||||
#: producer did not say", which is read as :data:`RESULT_NONE` — the state whose line
|
||||
#: names the pass that produces one, and the only wording that was ever true before
|
||||
#: the states were told apart.
|
||||
result_state: str = ""
|
||||
result_reason: str = ""
|
||||
|
||||
def has_flow(self) -> bool:
|
||||
if self.flow is None:
|
||||
@@ -224,6 +256,10 @@ class ImagePlan:
|
||||
#: :attr:`note_fields` is non-empty the line takes it as ``fields``.
|
||||
note_key: str = ""
|
||||
note_fields: tuple[str, ...] = ()
|
||||
#: The cause a ``note_key`` that asks for one is formatted with (``{reason}``). Kept
|
||||
#: apart from :attr:`note_fields`, which names the *fields* a view is missing: a
|
||||
#: failure's own words are not a field name and must not be looked up as one.
|
||||
note_reason: str = ""
|
||||
|
||||
def accepts(self, mode: str) -> bool:
|
||||
return str(mode) in self.available_modes
|
||||
@@ -266,7 +302,9 @@ def plan_image(frame: ImageFrame) -> ImagePlan:
|
||||
|
||||
The raw-recording note is therefore an *absence* statement and appears only in the
|
||||
one case where it is true: no result, nothing to draw, and the panel says which
|
||||
action produces one.
|
||||
action produces one. **Which** absence it is comes from ``frame.result_state`` — a
|
||||
pass that is running, one that failed or was refused with a reason, and a frame no
|
||||
pass can serve at all are three different absences, and each has its own line.
|
||||
"""
|
||||
derived = str(frame.source or SOURCE_LIVE) in DERIVED_SOURCES
|
||||
missing: list[str] = []
|
||||
@@ -275,6 +313,9 @@ def plan_image(frame: ImageFrame) -> ImagePlan:
|
||||
if not frame.has_depth():
|
||||
missing.append("depth")
|
||||
if not derived and not frame.result_available:
|
||||
state = str(frame.result_state or RESULT_NONE)
|
||||
note_key = RESULT_NOTE_KEYS.get(state, RAW_NOTE_KEY)
|
||||
reason = str(frame.result_reason or "")
|
||||
return ImagePlan(
|
||||
available_modes=(RECORDED,),
|
||||
default_mode=RECORDED,
|
||||
@@ -283,7 +324,10 @@ def plan_image(frame: ImageFrame) -> ImagePlan:
|
||||
frame_id=int(frame.frame_id),
|
||||
t_rel_s=float(frame.t_rel_s),
|
||||
stale=bool(frame.stale),
|
||||
note_key=RAW_NOTE_KEY,
|
||||
note_key=note_key,
|
||||
# Only the failed/refused line takes the reason: a state that has none must
|
||||
# not render an empty placeholder into a sentence that names a cause.
|
||||
note_reason=reason if note_key == RESULT_NOTE_KEYS[RESULT_FAILED] else "",
|
||||
)
|
||||
if not derived:
|
||||
# A recording with a result waiting behind it. ``RECORDED`` stays the default —
|
||||
|
||||
@@ -18,6 +18,17 @@ properties matter more than speed:
|
||||
Storage is not free, so :func:`evict` removes whole results against a stated size cap.
|
||||
It only ever deletes a direct child of the cache root that carries a ``meta.json``, so
|
||||
a wrong path cannot turn into a recursive delete of something else.
|
||||
|
||||
**The root is the app's own derived-data directory, never the recording's folder.**
|
||||
Results used to be written to ``<recording>/analysis``, which put the app's own
|
||||
rebuildable data inside the operator's measurement — and, because the *root* is
|
||||
path-dependent while the *key* is not, made a result invisible as soon as the recording
|
||||
was opened from anywhere else (a copy, another machine, the same session on a different
|
||||
drive letter). ``paths.analysis_cache_dir()`` is the owner's decision: one root per user
|
||||
under ``%LOCALAPPDATA%\\OrisysStudio\\analysis-cache``, addressed by the result key
|
||||
alone. A result written under the old root is **ignored, not migrated and not deleted**:
|
||||
the app never writes into a recording folder again, and a leftover ``analysis``
|
||||
directory is left exactly as the operator's own tooling left it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -36,6 +47,7 @@ from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from orisys_app_shared import paths as paths_module
|
||||
from orisys_app_shared.analysis.model import (
|
||||
ANALYSIS_FORMAT,
|
||||
ANALYSIS_FORMAT_VERSION,
|
||||
@@ -59,14 +71,18 @@ from orisys_app_shared.analysis.model import (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_INCOMPLETE_GRACE_S",
|
||||
"DEFAULT_MAX_BYTES",
|
||||
"INCOMPLETE_MARKER",
|
||||
"LEGACY_RESULT_DIRNAME",
|
||||
"META_NAME",
|
||||
"AnalysisWriter",
|
||||
"ResultCache",
|
||||
"ResultSummary",
|
||||
"SweepReport",
|
||||
"evict",
|
||||
"scan_results",
|
||||
"sweep",
|
||||
]
|
||||
|
||||
#: The file that declares a finished result. Its presence is not enough — the
|
||||
@@ -90,6 +106,18 @@ MAX_CHUNK_FRAMES = 4096
|
||||
#: holds roughly fourteen of them; the operator can lower it per call.
|
||||
DEFAULT_MAX_BYTES = 1 << 30
|
||||
|
||||
#: How long an unfinished result is left alone by :func:`sweep`. A directory is written
|
||||
#: only while a pass is running in *this* process, and the sweep runs at startup before
|
||||
#: any pass exists, but a second copy of the app (or a build script) can share the root,
|
||||
#: and deleting a directory another process is writing would turn its pass into a
|
||||
#: mysterious failure. An hour is far longer than any pass here and still removes the
|
||||
#: garbage of the previous session.
|
||||
DEFAULT_INCOMPLETE_GRACE_S = 3600.0
|
||||
|
||||
#: Where a result used to live: ``<recording>/analysis``. Named so the old location can
|
||||
#: be *recognised* (and reported), never so it can be written to again.
|
||||
LEGACY_RESULT_DIRNAME = RESULT_DIRNAME
|
||||
|
||||
#: Above this many remembered grid digests the writer stops deduplicating and gives a
|
||||
#: deviating frame its own keyframe. Correct either way; the map exists to keep the
|
||||
#: common case (one grid for the whole recording) at one keyframe.
|
||||
@@ -242,6 +270,101 @@ def evict(
|
||||
return removed
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SweepReport:
|
||||
"""What one :func:`sweep` did: the garbage it removed, and what is left."""
|
||||
|
||||
removed: list[Path]
|
||||
#: The subset of :attr:`removed` that was unreadable by construction (an incomplete
|
||||
#: pass). Reported apart because removing these is never news, while removing a
|
||||
#: *complete* result is the size cap doing its job and worth seeing.
|
||||
incomplete: list[Path]
|
||||
freed_bytes: int
|
||||
remaining_bytes: int
|
||||
|
||||
@property
|
||||
def kept(self) -> bool:
|
||||
return not self.removed
|
||||
|
||||
|
||||
def sweep(
|
||||
root: str | Path,
|
||||
*,
|
||||
max_bytes: int = DEFAULT_MAX_BYTES,
|
||||
keep: Iterable[str | Path] = (),
|
||||
incomplete_grace_s: float = DEFAULT_INCOMPLETE_GRACE_S,
|
||||
now: float | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> SweepReport:
|
||||
"""Tidy one cache root: unfinished results first, then the size cap.
|
||||
|
||||
Two passes, in this order, because they answer different questions:
|
||||
|
||||
1. **Garbage.** A result that is not complete is refused by every default reader
|
||||
(Part A §7), so nothing can ever use it again; it is removed once it is older than
|
||||
``incomplete_grace_s``. A cancelled pass is *inspectable*, which is why the grace
|
||||
period exists rather than "delete on sight": the evidence of how far a pass got is
|
||||
worth keeping for a while.
|
||||
2. **Size.** :func:`evict` then removes whole results, incomplete first and oldest
|
||||
first, until the total fits ``max_bytes`` — never touching anything in ``keep``,
|
||||
and only ever a direct child of ``root`` carrying a ``meta.json``.
|
||||
|
||||
Returns what it removed and what is left, so a caller can log a real number instead
|
||||
of "swept". Never raises for a root that does not exist: the first run of an app has
|
||||
no cache, and that is not an error.
|
||||
"""
|
||||
base = Path(root)
|
||||
protected = {Path(item).resolve() for item in keep}
|
||||
if not base.is_dir():
|
||||
return SweepReport(removed=[], incomplete=[], freed_bytes=0, remaining_bytes=0)
|
||||
moment = time.time() if now is None else float(now)
|
||||
grace = max(0.0, float(incomplete_grace_s))
|
||||
start_bytes = sum(item.bytes_used for item in scan_results(base))
|
||||
removed: list[Path] = []
|
||||
incomplete: list[Path] = []
|
||||
for item in scan_results(base):
|
||||
if item.complete:
|
||||
continue
|
||||
if item.directory.resolve() in protected:
|
||||
continue
|
||||
age = moment - float(item.modified or 0.0)
|
||||
if item.modified and age < grace:
|
||||
continue
|
||||
if not (item.directory / META_NAME).is_file():
|
||||
# Same rule as ``evict``: only a result directory is a candidate, so a wrong
|
||||
# root cannot turn into a recursive delete of something else.
|
||||
continue
|
||||
if item.directory.parent.resolve() != base.resolve():
|
||||
continue
|
||||
removed.append(item.directory)
|
||||
incomplete.append(item.directory)
|
||||
if not dry_run:
|
||||
shutil.rmtree(item.directory, ignore_errors=False)
|
||||
evicted = evict(base, max_bytes=max_bytes, keep=keep, dry_run=dry_run)
|
||||
for path in evicted:
|
||||
if path not in removed:
|
||||
removed.append(path)
|
||||
if removed:
|
||||
logger.info(
|
||||
"analysis cache sweep: %s %d result(s) (%d incomplete)",
|
||||
"would remove" if dry_run else "removed",
|
||||
len(removed),
|
||||
len(incomplete),
|
||||
)
|
||||
# Measured after the fact rather than subtracted: ``evict`` decides what fits, and a
|
||||
# second measurement is the only honest answer to "what is left" (a dry run leaves
|
||||
# everything, which is what its number says).
|
||||
remaining = (
|
||||
start_bytes if dry_run else sum(item.bytes_used for item in scan_results(base))
|
||||
)
|
||||
return SweepReport(
|
||||
removed=removed,
|
||||
incomplete=incomplete,
|
||||
freed_bytes=max(0, start_bytes - remaining),
|
||||
remaining_bytes=remaining,
|
||||
)
|
||||
|
||||
|
||||
class AnalysisWriter:
|
||||
"""Append-only writer for one result directory.
|
||||
|
||||
@@ -570,22 +693,36 @@ class AnalysisWriter:
|
||||
|
||||
|
||||
class ResultCache:
|
||||
"""The results belonging to one recording, addressed by key.
|
||||
"""The stored results, addressed by key under the app's own derived-data root.
|
||||
|
||||
``root`` is usually ``<recording>/analysis``: a result then travels with the
|
||||
recording it describes, and deleting the recording deletes its cache. Nothing here
|
||||
writes to the current working directory — that matters because opening a *box*
|
||||
``Sensor`` drops a ~2 MB ``config/compound_eye.png`` relative to it (see
|
||||
``service.py``).
|
||||
``root`` defaults to :func:`~orisys_app_shared.paths.analysis_cache_dir` —
|
||||
``%LOCALAPPDATA%\\OrisysStudio\\analysis-cache`` — because a result is the app's
|
||||
rebuildable data and must not be written into the operator's recording folder. The
|
||||
*key* is what identifies a result (it ignores the recording's path), so the root is
|
||||
shared by every recording and a copied recording finds the same result.
|
||||
|
||||
A directory left under the old ``<recording>/analysis`` root is not read and not
|
||||
touched: see the module docstring.
|
||||
"""
|
||||
|
||||
def __init__(self, root: str | Path, *, max_bytes: int = DEFAULT_MAX_BYTES) -> None:
|
||||
self.root = Path(root)
|
||||
def __init__(self, root: str | Path | None = None, *, max_bytes: int = DEFAULT_MAX_BYTES) -> None:
|
||||
# Resolved at construction through the module, so a caller (or a test) can point
|
||||
# the app's derived-data root elsewhere without patching this class.
|
||||
self.root = (
|
||||
Path(paths_module.analysis_cache_dir()) if root is None else Path(root)
|
||||
)
|
||||
self.max_bytes = int(max_bytes)
|
||||
|
||||
@classmethod
|
||||
def for_recording(cls, recording_dir: str | Path, **kwargs: Any) -> ResultCache:
|
||||
return cls(Path(recording_dir) / RESULT_DIRNAME, **kwargs)
|
||||
"""The cache a recording's results live in.
|
||||
|
||||
The recording is named because that is how every caller thinks about it, and
|
||||
because the *key* — which is what actually identifies the result — is built from
|
||||
the recording's own metadata. It no longer decides the root.
|
||||
"""
|
||||
del recording_dir # the identity is in the key, not in the folder layout
|
||||
return cls(**kwargs)
|
||||
|
||||
def directory(self, key: ResultKey) -> Path:
|
||||
return self.root / key.digest
|
||||
@@ -621,5 +758,26 @@ class ResultCache:
|
||||
) -> list[Path]:
|
||||
return evict(self.root, max_bytes=self.max_bytes, keep=keep, dry_run=dry_run)
|
||||
|
||||
def sweep(
|
||||
self,
|
||||
*,
|
||||
keep: Iterable[str | Path] = (),
|
||||
incomplete_grace_s: float = DEFAULT_INCOMPLETE_GRACE_S,
|
||||
now: float | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> SweepReport:
|
||||
"""Tidy this root: unfinished results first, then the size cap.
|
||||
|
||||
The startup call site for the app's own cache; see :func:`sweep`.
|
||||
"""
|
||||
return sweep(
|
||||
self.root,
|
||||
max_bytes=self.max_bytes,
|
||||
keep=keep,
|
||||
incomplete_grace_s=incomplete_grace_s,
|
||||
now=now,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
def total_bytes(self) -> int:
|
||||
return sum(item.bytes_used for item in scan_results(self.root))
|
||||
|
||||
@@ -121,6 +121,43 @@ def recordings_dir() -> Path:
|
||||
return path
|
||||
|
||||
|
||||
#: Directory name of the product's per-user data, under the OS's own root for it.
|
||||
DERIVED_DIR_NAME = "OrisysStudio"
|
||||
#: Subdirectory holding the offline analysis results.
|
||||
ANALYSIS_CACHE_DIRNAME = "analysis-cache"
|
||||
|
||||
|
||||
def derived_data_dir() -> Path:
|
||||
"""The OS-correct per-user root for **derived** (rebuildable) data.
|
||||
|
||||
Windows: ``%LOCALAPPDATA%\\OrisysStudio`` (falling back to
|
||||
``%USERPROFILE%\\AppData\\Local`` when the variable is unset — a service or a
|
||||
stripped environment, where the fallback is still the right directory). Everywhere
|
||||
else: ``$XDG_CACHE_HOME/OrisysStudio``, or ``~/.cache/OrisysStudio``.
|
||||
|
||||
Deliberately *not* :func:`outputs_dir`: in a packaged build that is the EXE's own
|
||||
directory, and it holds the recordings. Anything the app can rebuild belongs in the
|
||||
user's own derived-data area, where a reinstall cannot destroy it and where writing
|
||||
it needs no permission on the installation.
|
||||
"""
|
||||
name = DERIVED_DIR_NAME
|
||||
if os.name == "nt":
|
||||
local = os.environ.get("LOCALAPPDATA", "").strip()
|
||||
base = (
|
||||
Path(local)
|
||||
if local
|
||||
else Path(os.environ.get("USERPROFILE", str(Path.home()))) / "AppData" / "Local"
|
||||
)
|
||||
return base / name
|
||||
cache = os.environ.get("XDG_CACHE_HOME", "").strip()
|
||||
return (Path(cache) if cache else Path.home() / ".cache") / name
|
||||
|
||||
|
||||
def analysis_cache_dir() -> Path:
|
||||
"""Where the offline analysis results live: the app's own root, never a recording."""
|
||||
return derived_data_dir() / ANALYSIS_CACHE_DIRNAME
|
||||
|
||||
|
||||
def preference_path() -> Path:
|
||||
return app_root() / "preference.json"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user