ykcheungab
1130 lines
43 KiB
Python
1130 lines
43 KiB
Python
"""Force calibration load, parse, check, build, and write.
|
|
|
|
Owns ``force_calibration.json`` schema v1/v2. ``Sensor`` loads once and applies
|
|
curves in ``_apply_force_filter``; ``read_info(FNORMAL, …)`` returns Newtons.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import warnings
|
|
from collections.abc import Iterable, Mapping
|
|
from copy import deepcopy
|
|
from dataclasses import asdict, dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
CURRENT_SCHEMA = 2
|
|
SUPPORTED_SCHEMAS = (1, 2, 3)
|
|
CHANNEL_IDS: tuple[str, ...] = ("fn", "fx", "fy")
|
|
|
|
CHANNEL_ALIASES: dict[str, str] = {
|
|
"fn": "fn",
|
|
"FN": "fn",
|
|
"normal": "fn",
|
|
"fnormal": "fn",
|
|
"fx": "fx",
|
|
"FX": "fx",
|
|
"shear_x": "fx",
|
|
"fshearx": "fx",
|
|
"fy": "fy",
|
|
"FY": "fy",
|
|
"shear_y": "fy",
|
|
"fsheary": "fy",
|
|
}
|
|
|
|
_CANONICAL_TO_JSON = {
|
|
"fn": "normal",
|
|
"fx": "shear_x",
|
|
"fy": "shear_y",
|
|
}
|
|
_JSON_RAW_KEY = {
|
|
"normal": "fnormal",
|
|
"shear_x": "fshearx",
|
|
"shear_y": "fsheary",
|
|
}
|
|
_KNOWN_TOP_KEYS = frozenset(
|
|
{"schema_version", "unit", "channels", "meta", "identity", "pipeline", "provenance", "measurements", "quality"}
|
|
)
|
|
|
|
#: Filter stages that change **what is measured**: they sit upstream of the
|
|
#: force computation, so two recordings with different values are not comparable.
|
|
#: They belong to the measurement-pipeline identity, for the same reason
|
|
#: ``dis_mode`` and ``hhd_downsample`` do.
|
|
FILTER_MEASUREMENT_KEYS: tuple[str, ...] = ("flow_ema_alpha",)
|
|
#: Filter stages that only smooth the force triple on its way out. They are
|
|
#: recorded so a file explains its own numbers, but a difference between two
|
|
#: files does not invalidate a comparison.
|
|
FILTER_READOUT_KEYS: tuple[str, ...] = (
|
|
"force_median_window",
|
|
"force_lowpass_cutoff_hz",
|
|
"force_lowpass_order",
|
|
"force_lowpass_sample_rate_hz",
|
|
"force_lowpass_rate_auto",
|
|
)
|
|
#: Every recorded filter value, measurement group first.
|
|
FILTER_KEYS: tuple[str, ...] = FILTER_MEASUREMENT_KEYS + FILTER_READOUT_KEYS
|
|
#: What "off" looks like, per stage. ``enabled: false`` means exactly this.
|
|
_OFF_FILTER_VALUES: dict[str, Any] = {
|
|
"flow_ema_alpha": 1.0,
|
|
"force_median_window": 1,
|
|
"force_lowpass_cutoff_hz": 0.0,
|
|
}
|
|
_INT_FILTER_KEYS = frozenset({"force_median_window", "force_lowpass_order"})
|
|
_FLOAT_FILTER_KEYS = frozenset(
|
|
{"flow_ema_alpha", "force_lowpass_cutoff_hz", "force_lowpass_sample_rate_hz"}
|
|
)
|
|
|
|
|
|
class CalibrationError(ValueError):
|
|
"""Invalid or unusable force calibration document / path."""
|
|
|
|
|
|
def _coerce_filter_value(key: str, value: Any) -> Any:
|
|
"""A filter value in its recorded type, or ``None`` when unusable."""
|
|
if value is None:
|
|
return None
|
|
if key == "force_lowpass_rate_auto":
|
|
return bool(value)
|
|
try:
|
|
if key in _INT_FILTER_KEYS:
|
|
return int(value)
|
|
if key in _FLOAT_FILTER_KEYS:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return value
|
|
|
|
|
|
def _filters_from_config(block: Mapping[str, Any] | Any) -> dict[str, dict[str, Any]]:
|
|
"""Best-effort filter groups from a flat ``filters`` block.
|
|
|
|
Only what the mapping states — directly or through a named preset, with an
|
|
explicit key winning over the preset — is reported. Nothing is filled in from
|
|
a default: a default is how a document ends up claiming a filter the device
|
|
is not running. ``enabled: false`` keeps its deprecated meaning of "every
|
|
stage off".
|
|
"""
|
|
flat = dict(block) if isinstance(block, Mapping) else {}
|
|
preset_name = flat.get("preset") if isinstance(flat.get("preset"), str) else None
|
|
merged: dict[str, Any] = {}
|
|
if preset_name:
|
|
try:
|
|
from .sensor import FILTER_PRESETS # deferred: sensor imports this module
|
|
except Exception: # pragma: no cover - defensive
|
|
FILTER_PRESETS = {}
|
|
merged.update(FILTER_PRESETS.get(preset_name, {}) or {})
|
|
for key in FILTER_KEYS:
|
|
if flat.get(key) is not None:
|
|
merged[key] = flat[key]
|
|
if isinstance(flat.get("force_lowpass_sample_rate_hz"), str):
|
|
# "auto" / "measured" is a rate *source*, not a number; keep it as one.
|
|
merged["force_lowpass_rate_auto"] = True
|
|
merged.pop("force_lowpass_sample_rate_hz", None)
|
|
if flat.get("enabled", None) is False:
|
|
order = merged.get("force_lowpass_order")
|
|
rate_auto = merged.get("force_lowpass_rate_auto")
|
|
merged = dict(_OFF_FILTER_VALUES)
|
|
# Order and rate tracking stay meaningful even with the stage off.
|
|
if order is not None:
|
|
merged["force_lowpass_order"] = order
|
|
if rate_auto is not None:
|
|
merged["force_lowpass_rate_auto"] = rate_auto
|
|
return {
|
|
group: {key: _coerce_filter_value(key, merged.get(key)) for key in keys}
|
|
for group, keys in (
|
|
("measurement", FILTER_MEASUREMENT_KEYS),
|
|
("readout", FILTER_READOUT_KEYS),
|
|
)
|
|
}
|
|
|
|
|
|
def _flat_filters_from(*sources: Mapping[str, Any] | Any) -> dict[str, Any]:
|
|
"""The flat form older documents used: filter keys at the top of meta."""
|
|
out: dict[str, Any] = {}
|
|
for source in sources:
|
|
if not isinstance(source, Mapping):
|
|
continue
|
|
for key in FILTER_KEYS:
|
|
if key not in out and source.get(key) is not None:
|
|
out[key] = source[key]
|
|
return out
|
|
|
|
|
|
def filters_fingerprint(state: Mapping[str, Any] | Any) -> dict[str, Any]:
|
|
"""Split a filter state into its ``measurement`` and ``readout`` groups.
|
|
|
|
The honest input is ``Sensor.filters_state()`` — the values that actually
|
|
took effect, which is not necessarily what the config file says. A flat
|
|
mapping is accepted as a fallback (a sensor.json ``filters`` block, or the
|
|
flat form older documents wrote) and read as the intent it is.
|
|
|
|
Values a source does not state are absent from the result rather than
|
|
guessed, so ``{}`` means "nothing recorded" and never "probably off".
|
|
"""
|
|
if not isinstance(state, Mapping) or not state:
|
|
return {}
|
|
measurement = state.get("measurement")
|
|
readout = state.get("readout")
|
|
if isinstance(measurement, Mapping) or isinstance(readout, Mapping):
|
|
groups = {
|
|
"measurement": {
|
|
key: _coerce_filter_value(key, measurement.get(key))
|
|
for key in FILTER_MEASUREMENT_KEYS
|
|
}
|
|
if isinstance(measurement, Mapping)
|
|
else {},
|
|
"readout": {
|
|
key: _coerce_filter_value(key, readout.get(key))
|
|
for key in FILTER_READOUT_KEYS
|
|
}
|
|
if isinstance(readout, Mapping)
|
|
else {},
|
|
}
|
|
source = str(state.get("source") or "sensor.filters_state")
|
|
else:
|
|
groups = _filters_from_config(state)
|
|
source = "sensor.json.filters"
|
|
out: dict[str, Any] = {}
|
|
for group in ("measurement", "readout"):
|
|
values = {
|
|
key: value for key, value in (groups.get(group) or {}).items() if value is not None
|
|
}
|
|
if values:
|
|
out[group] = values
|
|
if not out:
|
|
return {}
|
|
if state.get("preset"):
|
|
out["preset"] = str(state["preset"])
|
|
out["source"] = source
|
|
return out
|
|
|
|
|
|
def filter_mismatches(
|
|
recorded: Mapping[str, Any] | Any, expected: Mapping[str, Any] | Any
|
|
) -> tuple[list[str], list[str]]:
|
|
"""Compare two filter fingerprints → ``(measurement, readout)`` differences.
|
|
|
|
A measurement difference means the two records are not comparable, so it is
|
|
reported as a warning. A readout difference only changes how the same
|
|
measurement was smoothed, so it is a note: recorded, never an obstacle.
|
|
"""
|
|
left = filters_fingerprint(recorded)
|
|
right = filters_fingerprint(expected)
|
|
warns: list[str] = []
|
|
notes: list[str] = []
|
|
for group, keys, bucket, label in (
|
|
("measurement", FILTER_MEASUREMENT_KEYS, warns, "measurement (not comparable)"),
|
|
("readout", FILTER_READOUT_KEYS, notes, "readout"),
|
|
):
|
|
for key in keys:
|
|
if key in (left.get(group) or {}) and key in (right.get(group) or {}):
|
|
if left[group][key] != right[group][key]:
|
|
bucket.append(
|
|
f"filter {label} mismatch: {key} "
|
|
f"{left[group][key]!r} != {right[group][key]!r}"
|
|
)
|
|
return warns, notes
|
|
|
|
|
|
def canonicalize_channel(name: str) -> str:
|
|
"""Map FN / normal / shear_x / … to canonical ``fn`` / ``fx`` / ``fy``."""
|
|
key = str(name or "").strip()
|
|
if key in CHANNEL_ALIASES:
|
|
return CHANNEL_ALIASES[key]
|
|
lower = key.lower()
|
|
if lower in CHANNEL_ALIASES:
|
|
return CHANNEL_ALIASES[lower]
|
|
raise CalibrationError(f"unknown calibration channel: {name!r}")
|
|
|
|
|
|
def _parse_point(point: Any) -> tuple[float, float]:
|
|
if isinstance(point, dict):
|
|
raw = point.get("fnormal", point.get("raw", point.get("x")))
|
|
force_n = point.get("force_n", point.get("n", point.get("y")))
|
|
if raw is None or force_n is None:
|
|
raise CalibrationError(f"invalid calibration point: {point!r}")
|
|
return float(raw), float(force_n)
|
|
if isinstance(point, (list, tuple)) and len(point) >= 2:
|
|
return float(point[0]), float(point[1])
|
|
raise CalibrationError(f"invalid calibration point: {point!r}")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ChannelCurve:
|
|
"""One force channel: linear coeffs and/or piecewise-linear breakpoints."""
|
|
|
|
id: str
|
|
method: str = "piecewise_linear"
|
|
points: tuple[tuple[float, float], ...] = ()
|
|
slope: float | None = None
|
|
offset: float | None = None
|
|
signed: bool = False
|
|
extrapolate: bool = True
|
|
clamp_output_min: float | None = 0.0
|
|
raw_key: str = ""
|
|
input_mode: str = "magnitude"
|
|
unit: str = "N"
|
|
|
|
def convert(self, raw: float) -> float:
|
|
raw = float(raw)
|
|
if self.method in ("linear", "affine"):
|
|
if self.slope is None or self.offset is None:
|
|
return raw
|
|
return float(self.slope) * raw + float(self.offset)
|
|
|
|
points = self.points
|
|
if len(points) < 2:
|
|
return raw
|
|
|
|
if raw <= points[0][0]:
|
|
segment = (points[0], points[1])
|
|
if not self.extrapolate:
|
|
return self._clamp(points[0][1])
|
|
elif raw >= points[-1][0]:
|
|
segment = (points[-2], points[-1])
|
|
if not self.extrapolate:
|
|
return self._clamp(points[-1][1])
|
|
else:
|
|
segment = None
|
|
for left, right in zip(points, points[1:]):
|
|
if left[0] <= raw <= right[0]:
|
|
segment = (left, right)
|
|
break
|
|
if segment is None:
|
|
return self._clamp(points[-1][1])
|
|
|
|
(raw0, n0), (raw1, n1) = segment
|
|
if raw1 == raw0:
|
|
value = n0
|
|
else:
|
|
ratio = (raw - raw0) / (raw1 - raw0)
|
|
value = n0 + ratio * (n1 - n0)
|
|
return self._clamp(value)
|
|
|
|
def convert_component(self, raw: float) -> float:
|
|
if not self.signed:
|
|
return self.convert(raw)
|
|
raw = float(raw)
|
|
sign = -1.0 if raw < 0.0 else 1.0
|
|
return sign * self.convert(abs(raw))
|
|
|
|
def _clamp(self, value: float) -> float:
|
|
if self.clamp_output_min is not None:
|
|
return max(float(self.clamp_output_min), float(value))
|
|
return float(value)
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
json_name = _CANONICAL_TO_JSON.get(self.id, self.id)
|
|
if self.method in ("linear", "affine"):
|
|
block: dict[str, Any] = {
|
|
"method": "linear",
|
|
"slope": self.slope,
|
|
"offset": self.offset,
|
|
}
|
|
else:
|
|
block = {
|
|
"method": "piecewise_linear",
|
|
"raw_key": self.raw_key or _JSON_RAW_KEY.get(json_name, ""),
|
|
"extrapolate": self.extrapolate,
|
|
"clamp_output_min": self.clamp_output_min,
|
|
"points": [list(p) for p in self.points],
|
|
}
|
|
if self.signed:
|
|
block["signed"] = True
|
|
if self.input_mode and self.input_mode != "magnitude":
|
|
block["input"] = self.input_mode
|
|
return block
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CalSample:
|
|
weight_n: float
|
|
raw_mean: float
|
|
n: int
|
|
std: float = 0.0
|
|
|
|
|
|
@dataclass
|
|
class CheckResult:
|
|
ok: bool
|
|
errors: list[str] = field(default_factory=list)
|
|
warnings: list[str] = field(default_factory=list)
|
|
#: Differences worth recording that are not a reason to refuse anything,
|
|
#: such as a readout-only filter change between two documents.
|
|
notes: list[str] = field(default_factory=list)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Calibration:
|
|
"""Parsed force calibration document."""
|
|
|
|
schema_version: int
|
|
unit: str
|
|
origin: str
|
|
identity: dict[str, Any]
|
|
pipeline: dict[str, Any]
|
|
provenance: dict[str, Any]
|
|
channels: dict[str, ChannelCurve]
|
|
measurements: dict[str, Any]
|
|
extras: dict[str, Any] = field(default_factory=dict)
|
|
source_path: Path | None = None
|
|
# Legacy shared shear (v1) for magnitude-mode fallback in helpers.
|
|
_legacy_shear: ChannelCurve | None = None
|
|
|
|
def get(self, channel: str) -> ChannelCurve | None:
|
|
try:
|
|
cid = canonicalize_channel(channel)
|
|
except CalibrationError:
|
|
return self.channels.get(str(channel))
|
|
return self.channels.get(cid)
|
|
|
|
def has(self, channel: str) -> bool:
|
|
return self.get(channel) is not None
|
|
|
|
def convert(self, fn: float, fx: float, fy: float) -> tuple[float, float, float]:
|
|
"""Apply present curves; missing channel stays raw."""
|
|
out_fn = float(fn)
|
|
out_fx = float(fx)
|
|
out_fy = float(fy)
|
|
cal_fn = self.channels.get("fn")
|
|
cal_fx = self.channels.get("fx")
|
|
cal_fy = self.channels.get("fy")
|
|
if cal_fn is not None:
|
|
out_fn = cal_fn.convert(out_fn)
|
|
if cal_fx is not None:
|
|
out_fx = (
|
|
cal_fx.convert_component(out_fx)
|
|
if cal_fx.signed
|
|
else cal_fx.convert(out_fx)
|
|
)
|
|
if cal_fy is not None:
|
|
out_fy = (
|
|
cal_fy.convert_component(out_fy)
|
|
if cal_fy.signed
|
|
else cal_fy.convert(out_fy)
|
|
)
|
|
return out_fn, out_fx, out_fy
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
channels: dict[str, Any] = {}
|
|
for cid, curve in self.channels.items():
|
|
json_name = _CANONICAL_TO_JSON.get(cid, cid)
|
|
channels[json_name] = curve.to_dict()
|
|
# Prefer emitting v2 axis names even when parsed from v1 shared shear.
|
|
doc: dict[str, Any] = {
|
|
"schema_version": max(int(self.schema_version), 2) if self.channels else int(self.schema_version),
|
|
"unit": self.unit,
|
|
"channels": channels,
|
|
}
|
|
meta: dict[str, Any] = {}
|
|
meta.update(self.provenance)
|
|
meta.update({k: v for k, v in self.identity.items() if k not in meta})
|
|
for key in ("stitch_resolution", "dis_mode", "hhd_downsample", "config_ref"):
|
|
if key in self.pipeline and self.pipeline[key] is not None:
|
|
meta[key] = self.pipeline[key]
|
|
fingerprint = filters_fingerprint(self.pipeline.get("filters"))
|
|
if fingerprint:
|
|
# Structured, so a reader can tell a change that makes two records
|
|
# incomparable (measurement) from one that only changes how the same
|
|
# measurement was smoothed (readout).
|
|
meta["filters"] = fingerprint
|
|
# The flat keys are kept as well for readers that look them up by
|
|
# name; the groups above are the authoritative form.
|
|
for group in ("measurement", "readout"):
|
|
meta.update(fingerprint.get(group) or {})
|
|
if self.measurements:
|
|
meta["samples"] = self.measurements
|
|
if self.origin and "origin" not in meta:
|
|
meta["origin"] = self.origin
|
|
if meta:
|
|
doc["meta"] = meta
|
|
for key, value in self.extras.items():
|
|
if key not in doc:
|
|
doc[key] = deepcopy(value)
|
|
return doc
|
|
|
|
|
|
def _channel_from_block(
|
|
channel_id: str,
|
|
block: Any,
|
|
*,
|
|
unit: str = "N",
|
|
default_method: str = "piecewise_linear",
|
|
) -> ChannelCurve | None:
|
|
if not isinstance(block, dict):
|
|
return None
|
|
method = str(block.get("method", block.get("model", default_method))).lower()
|
|
if method in ("linear", "affine"):
|
|
slope = block.get("slope")
|
|
offset = block.get("offset")
|
|
if slope is None or offset is None:
|
|
return None
|
|
return ChannelCurve(
|
|
id=channel_id,
|
|
method="linear",
|
|
slope=float(slope),
|
|
offset=float(offset),
|
|
unit=str(block.get("unit", unit)),
|
|
raw_key=str(block.get("raw_key", "")),
|
|
)
|
|
if method != "piecewise_linear":
|
|
return None
|
|
raw_points = block.get("points", [])
|
|
if not isinstance(raw_points, (list, tuple)) or len(raw_points) < 2:
|
|
return None
|
|
try:
|
|
points = tuple(sorted(_parse_point(p) for p in raw_points))
|
|
except (TypeError, ValueError, CalibrationError):
|
|
return None
|
|
if len(points) < 2:
|
|
return None
|
|
return ChannelCurve(
|
|
id=channel_id,
|
|
method="piecewise_linear",
|
|
points=points,
|
|
extrapolate=bool(block.get("extrapolate", True)),
|
|
clamp_output_min=block.get("clamp_output_min", 0.0),
|
|
input_mode=str(block.get("input", "magnitude")).lower(),
|
|
signed=bool(block.get("signed", False)),
|
|
unit=str(block.get("unit", unit)),
|
|
raw_key=str(block.get("raw_key", _JSON_RAW_KEY.get(_CANONICAL_TO_JSON.get(channel_id, ""), ""))),
|
|
)
|
|
|
|
|
|
def check(doc: dict | Calibration, *, pipeline: dict | None = None) -> CheckResult:
|
|
"""Validate a document or Calibration. Errors refuse apply/write."""
|
|
errors: list[str] = []
|
|
warns: list[str] = []
|
|
notes: list[str] = []
|
|
|
|
if getattr(doc, "schema_version", None) == 3 or (isinstance(doc, dict) and doc.get("schema_version") == 3):
|
|
from .calibration_models import parse_model
|
|
|
|
try:
|
|
selected = parse_model(doc) if isinstance(doc, dict) else doc
|
|
if pipeline is not None:
|
|
selected.validate_for_context(pipeline)
|
|
except (CalibrationError, ValueError, TypeError) as exc:
|
|
return CheckResult(ok=False, errors=[str(exc)])
|
|
return CheckResult(ok=True, errors=[], warnings=[])
|
|
|
|
if isinstance(doc, Calibration):
|
|
data = doc.to_dict()
|
|
channels_obj = doc.channels
|
|
schema = doc.schema_version
|
|
identity = doc.identity
|
|
elif isinstance(doc, dict):
|
|
data = doc
|
|
schema_raw = data.get("schema_version", 1)
|
|
try:
|
|
schema = int(schema_raw)
|
|
except (TypeError, ValueError):
|
|
errors.append(f"invalid schema_version: {schema_raw!r}")
|
|
return CheckResult(ok=False, errors=errors, warnings=warns)
|
|
if schema not in SUPPORTED_SCHEMAS:
|
|
errors.append(f"unsupported schema_version: {schema}")
|
|
channels = data.get("channels")
|
|
if not isinstance(channels, dict):
|
|
errors.append("missing or invalid channels object")
|
|
return CheckResult(ok=False, errors=errors, warnings=warns)
|
|
channels_obj = {}
|
|
unit = str(data.get("unit", "N"))
|
|
for json_name, block in channels.items():
|
|
try:
|
|
cid = canonicalize_channel(json_name) if json_name != "shear" else "shear"
|
|
except CalibrationError:
|
|
# Future / unknown channel names: skip strict mapping.
|
|
continue
|
|
if json_name == "shear":
|
|
curve = _channel_from_block("fx", block, unit=unit)
|
|
else:
|
|
curve = _channel_from_block(cid, block, unit=unit)
|
|
if curve is None:
|
|
method = None
|
|
if isinstance(block, dict):
|
|
method = block.get("method", block.get("model"))
|
|
if isinstance(block, dict) and method not in (
|
|
None,
|
|
"piecewise_linear",
|
|
"linear",
|
|
"affine",
|
|
):
|
|
errors.append(f"unknown method for {json_name}: {method!r}")
|
|
elif isinstance(block, dict):
|
|
pts = block.get("points", [])
|
|
if method in ("piecewise_linear", None) and (
|
|
not isinstance(pts, (list, tuple)) or len(pts) < 2
|
|
):
|
|
errors.append(f"{json_name}: need at least 2 points")
|
|
else:
|
|
errors.append(f"invalid channel block: {json_name}")
|
|
else:
|
|
channels_obj[cid if json_name != "shear" else "fx"] = curve
|
|
meta = data.get("meta") if isinstance(data.get("meta"), dict) else {}
|
|
identity = {
|
|
"sensor_type": meta.get("sensor_type", ""),
|
|
"sn": meta.get("sn", ""),
|
|
}
|
|
else:
|
|
return CheckResult(ok=False, errors=["document must be dict or Calibration"])
|
|
|
|
if not channels_obj and not errors:
|
|
errors.append("no usable channel curves")
|
|
|
|
if "fn" not in channels_obj and "fx" not in channels_obj and "fy" not in channels_obj:
|
|
if not errors:
|
|
errors.append("no FN/FX/FY channels")
|
|
|
|
if "fx" not in channels_obj:
|
|
warns.append("missing FX channel")
|
|
if "fy" not in channels_obj:
|
|
warns.append("missing FY channel")
|
|
|
|
sn = str(identity.get("sn") or "")
|
|
origin = str(identity.get("origin") or data.get("meta", {}).get("origin", "") if isinstance(data, dict) else "")
|
|
if not sn and origin in ("factory",):
|
|
warns.append("empty sn on factory calibration")
|
|
|
|
if pipeline is not None and isinstance(doc, Calibration):
|
|
for key in ("stitch_resolution", "dis_mode", "hhd_downsample"):
|
|
if key in doc.pipeline and key in pipeline and doc.pipeline[key] != pipeline[key]:
|
|
warns.append(f"pipeline mismatch: {key}")
|
|
# The filter groups are compared apart on purpose: ``flow_ema_alpha``
|
|
# changes what was measured, the readout stages only smooth it.
|
|
filter_warns, filter_notes = filter_mismatches(
|
|
doc.pipeline.get("filters"), pipeline.get("filters")
|
|
)
|
|
warns.extend(filter_warns)
|
|
notes.extend(filter_notes)
|
|
|
|
return CheckResult(ok=not errors, errors=errors, warnings=warns, notes=notes)
|
|
|
|
|
|
def parse(data: dict, *, source_path: Path | None = None) -> Calibration:
|
|
"""Parse a force_calibration dict into a Calibration object."""
|
|
if not isinstance(data, dict):
|
|
raise CalibrationError("calibration document must be a JSON object")
|
|
|
|
schema_raw = data.get("schema_version", 1)
|
|
try:
|
|
schema = int(schema_raw)
|
|
except (TypeError, ValueError) as exc:
|
|
raise CalibrationError(f"invalid schema_version: {schema_raw!r}") from exc
|
|
if schema not in SUPPORTED_SCHEMAS:
|
|
raise CalibrationError(f"unsupported schema_version: {schema}")
|
|
|
|
if schema == 3:
|
|
from .calibration_models import parse_model
|
|
|
|
try:
|
|
return parse_model(data, source_path=source_path)
|
|
except (ValueError, TypeError, KeyError) as exc:
|
|
raise CalibrationError(f"invalid schema 3 calibration: {exc}") from exc
|
|
|
|
unit = str(data.get("unit", "N"))
|
|
channels_raw = data.get("channels")
|
|
if not isinstance(channels_raw, dict):
|
|
raise CalibrationError("missing or invalid channels object")
|
|
|
|
channels: dict[str, ChannelCurve] = {}
|
|
legacy_shear: ChannelCurve | None = None
|
|
|
|
# Prefer explicit axis channels (v2).
|
|
for json_name, cid in (("normal", "fn"), ("shear_x", "fx"), ("shear_y", "fy")):
|
|
if json_name in channels_raw:
|
|
curve = _channel_from_block(cid, channels_raw[json_name], unit=unit)
|
|
if curve is None:
|
|
raise CalibrationError(f"invalid channel block: {json_name}")
|
|
channels[cid] = curve
|
|
|
|
# v1 shared shear → both fx and fy when axes absent.
|
|
if "shear" in channels_raw and ("fx" not in channels or "fy" not in channels):
|
|
shear_block = channels_raw["shear"]
|
|
legacy_shear = _channel_from_block("fx", shear_block, unit=unit)
|
|
if legacy_shear is None:
|
|
raise CalibrationError("invalid channel block: shear")
|
|
# Copy with correct ids for each axis.
|
|
shared = ChannelCurve(
|
|
id="fx",
|
|
method=legacy_shear.method,
|
|
points=legacy_shear.points,
|
|
slope=legacy_shear.slope,
|
|
offset=legacy_shear.offset,
|
|
signed=legacy_shear.signed or legacy_shear.input_mode == "components",
|
|
extrapolate=legacy_shear.extrapolate,
|
|
clamp_output_min=legacy_shear.clamp_output_min,
|
|
raw_key=legacy_shear.raw_key,
|
|
input_mode=legacy_shear.input_mode,
|
|
unit=legacy_shear.unit,
|
|
)
|
|
if "fx" not in channels:
|
|
channels["fx"] = shared
|
|
if "fy" not in channels:
|
|
channels["fy"] = ChannelCurve(
|
|
id="fy",
|
|
method=shared.method,
|
|
points=shared.points,
|
|
slope=shared.slope,
|
|
offset=shared.offset,
|
|
signed=shared.signed,
|
|
extrapolate=shared.extrapolate,
|
|
clamp_output_min=shared.clamp_output_min,
|
|
raw_key=_JSON_RAW_KEY["shear_y"],
|
|
input_mode=shared.input_mode,
|
|
unit=shared.unit,
|
|
)
|
|
legacy_shear = shared
|
|
|
|
# Legacy single-channel file: {"normal_force": {"model": "linear", ...}}
|
|
if "fn" not in channels:
|
|
block = data.get("normal_force", data if "slope" in data else None)
|
|
if isinstance(block, dict):
|
|
curve = _channel_from_block("fn", block, unit=unit, default_method="linear")
|
|
if curve is not None:
|
|
channels["fn"] = curve
|
|
|
|
if not channels:
|
|
raise CalibrationError("no usable channel curves")
|
|
|
|
meta = data.get("meta") if isinstance(data.get("meta"), dict) else {}
|
|
identity_in = data.get("identity") if isinstance(data.get("identity"), dict) else {}
|
|
pipeline_in = data.get("pipeline") if isinstance(data.get("pipeline"), dict) else {}
|
|
provenance_in = data.get("provenance") if isinstance(data.get("provenance"), dict) else {}
|
|
measurements_in = data.get("measurements") if isinstance(data.get("measurements"), dict) else {}
|
|
|
|
identity = {
|
|
"sensor_type": identity_in.get("sensor_type", meta.get("sensor_type", "")),
|
|
"sn": identity_in.get("sn", meta.get("sn", "")),
|
|
"hardware_rev": identity_in.get("hardware_rev", meta.get("hardware_rev", "")),
|
|
"gel_id": identity_in.get("gel_id", meta.get("gel_id", "")),
|
|
}
|
|
pipeline = {
|
|
"stitch_resolution": pipeline_in.get(
|
|
"stitch_resolution", meta.get("stitch_resolution")
|
|
),
|
|
"dis_mode": pipeline_in.get("dis_mode", meta.get("dis_mode")),
|
|
"hhd_downsample": pipeline_in.get("hhd_downsample", meta.get("hhd_downsample")),
|
|
"config_ref": pipeline_in.get("config_ref", meta.get("config_ref")),
|
|
"config_hash": pipeline_in.get("config_hash", meta.get("config_hash")),
|
|
# Structured when the document has it, otherwise rebuilt from the flat
|
|
# keys older documents wrote — reading them back is what makes the
|
|
# fingerprint checkable at all.
|
|
"filters": filters_fingerprint(
|
|
pipeline_in.get("filters")
|
|
or meta.get("filters")
|
|
or _flat_filters_from(pipeline_in, meta)
|
|
),
|
|
}
|
|
provenance = {
|
|
"origin": provenance_in.get("origin", meta.get("origin", "")),
|
|
"created_at": provenance_in.get(
|
|
"created_at", meta.get("date", meta.get("created_at", ""))
|
|
),
|
|
"calibration_id": provenance_in.get("calibration_id", meta.get("calibration_id", "")),
|
|
"tool": provenance_in.get("tool", meta.get("tool")),
|
|
"operator": provenance_in.get("operator", meta.get("operator", "")),
|
|
"station": provenance_in.get("station", meta.get("station", "")),
|
|
"record_duration_s": provenance_in.get(
|
|
"record_duration_s", meta.get("record_duration_s")
|
|
),
|
|
}
|
|
measurements = measurements_in or meta.get("samples") or {}
|
|
origin = str(provenance.get("origin") or "default")
|
|
|
|
extras = {k: deepcopy(v) for k, v in data.items() if k not in _KNOWN_TOP_KEYS}
|
|
|
|
cal = Calibration(
|
|
schema_version=schema,
|
|
unit=unit,
|
|
origin=origin,
|
|
identity=identity,
|
|
pipeline=pipeline,
|
|
provenance=provenance,
|
|
channels=channels,
|
|
measurements=measurements if isinstance(measurements, dict) else {},
|
|
extras=extras,
|
|
source_path=Path(source_path).resolve() if source_path is not None else None,
|
|
_legacy_shear=legacy_shear,
|
|
)
|
|
result = check(cal)
|
|
if not result.ok:
|
|
raise CalibrationError("; ".join(result.errors))
|
|
# Missing FX/FY are optional; leave them on CheckResult for callers.
|
|
# Only surface non-channel warnings as Python warnings.
|
|
for w in result.warnings:
|
|
if w.startswith("missing F"):
|
|
continue
|
|
warnings.warn(f"calibration: {w}", stacklevel=2)
|
|
return cal
|
|
|
|
|
|
def load(
|
|
source: str | Path | None = None,
|
|
*,
|
|
sensor_type: str | None = None,
|
|
sn: str | None = None, # noqa: ARG001 — reserved for SN files
|
|
origin: str | None = None, # noqa: ARG001
|
|
strict: bool = False,
|
|
) -> Calibration | None:
|
|
"""Load calibration from an explicit path or product bundle.
|
|
|
|
Load order:
|
|
1. ``source`` file path when it exists
|
|
2. ``sensors/<sensor_type>/force_calibration.json``
|
|
3. ``None`` (missing bundle file)
|
|
|
|
Explicit ``source`` that is missing or invalid raises when the path was
|
|
provided (or when ``strict=True``). Missing bundle file returns ``None``.
|
|
"""
|
|
del sn, origin # reserved
|
|
|
|
path: Path | None = None
|
|
explicit = source is not None and str(source).strip() != ""
|
|
|
|
if explicit:
|
|
path = select_calibration_path(cal_path=source)
|
|
if not path.is_file():
|
|
msg = f"calibration file not found: {path}"
|
|
if strict or explicit:
|
|
raise CalibrationError(msg)
|
|
return None
|
|
elif sensor_type:
|
|
from .configs import resolve_sensor_bundle
|
|
|
|
bundle = resolve_sensor_bundle(str(sensor_type))
|
|
if bundle is not None and bundle.force_calibration_json is not None:
|
|
path = bundle.force_calibration_json
|
|
elif strict:
|
|
raise CalibrationError(
|
|
f"no force_calibration.json for sensor_type={sensor_type!r}"
|
|
)
|
|
else:
|
|
return None
|
|
else:
|
|
if strict:
|
|
raise CalibrationError("load() requires source or sensor_type")
|
|
return None
|
|
|
|
assert path is not None
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
except Exception as exc:
|
|
msg = f"failed to read calibration JSON: {path}: {exc}"
|
|
if strict or explicit:
|
|
raise CalibrationError(msg) from exc
|
|
return None
|
|
|
|
try:
|
|
return parse(data, source_path=path)
|
|
except CalibrationError:
|
|
if strict or explicit or (isinstance(data, dict) and (data.get("schema_version") == 3 or "model" in data)):
|
|
raise
|
|
return None
|
|
|
|
|
|
def load_for_device(
|
|
calibration_dir: str | Path,
|
|
sensor_type: str,
|
|
device_id: str,
|
|
*,
|
|
pipeline: Mapping[str, Any] | None = None,
|
|
) -> Calibration | None:
|
|
"""Load ``<directory>/<sensor_type>/<device_id>.json``; missing file means raw."""
|
|
path = select_calibration_path(calibration_dir=calibration_dir, sensor_type=sensor_type, device_id=device_id)
|
|
if path is None:
|
|
return None
|
|
cal = load(path, strict=True)
|
|
assert cal is not None
|
|
return validate_for_device(cal, sensor_type, device_id, pipeline=pipeline)
|
|
|
|
|
|
def select_calibration_path(*, cal_path=None, calibration_dir=None, sensor_type=None,
|
|
device_id=None, default_path=None) -> Path | None:
|
|
"""Select a file without guessing a model or falling back across device identities."""
|
|
explicit = cal_path is not None and bool(str(cal_path).strip())
|
|
if explicit and calibration_dir is not None:
|
|
raise CalibrationError("cal_path and calibration_dir are mutually exclusive")
|
|
if device_id is not None and not explicit and calibration_dir is None:
|
|
raise CalibrationError("device_id requires calibration_dir or cal_path")
|
|
if explicit:
|
|
return Path(cal_path).expanduser()
|
|
if calibration_dir is None:
|
|
return Path(default_path) if default_path is not None else None
|
|
root = Path(calibration_dir).expanduser()
|
|
if not root.is_dir():
|
|
raise CalibrationError(f"calibration directory not found: {root}")
|
|
for label, value in (("sensor_type", sensor_type), ("device_id", device_id)):
|
|
if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", value):
|
|
raise CalibrationError(f"invalid {label}: {value!r}")
|
|
path = root / sensor_type / f"{device_id}.json"
|
|
if not path.exists():
|
|
if path.is_symlink():
|
|
raise CalibrationError(f"calibration link is broken: {path}")
|
|
return None
|
|
if not path.is_file():
|
|
raise CalibrationError(f"calibration path is not a file: {path}")
|
|
return path
|
|
|
|
|
|
def validate_for_device(
|
|
cal: Calibration,
|
|
sensor_type: str,
|
|
device_id: str,
|
|
*,
|
|
pipeline: Mapping[str, Any] | None = None,
|
|
) -> Calibration:
|
|
"""Reject a calibration for another device or measurement pipeline."""
|
|
path = cal.source_path or "<in-memory calibration>"
|
|
if cal.identity.get("sensor_type") != sensor_type or cal.identity.get("sn") != device_id:
|
|
raise CalibrationError(f"calibration identity mismatch: {path}")
|
|
if cal.schema_version == 3:
|
|
if cal.unit != "N" or not cal.channels:
|
|
raise CalibrationError(f"calibration requires declared force axes in N: {path}")
|
|
# Full feature/geometry validation happens when the processor binds its
|
|
# actual context. Device construction only knows this legacy subset.
|
|
if pipeline is not None:
|
|
for key in ("stitch_resolution", "dis_mode", "hhd_downsample"):
|
|
if pipeline.get(key) is not None and cal.pipeline.get(key) != pipeline[key]:
|
|
raise CalibrationError(f"calibration pipeline mismatch ({key}): {path}")
|
|
measurement, _ = filter_mismatches(cal.pipeline.get("filters"), pipeline.get("filters"))
|
|
if measurement:
|
|
raise CalibrationError(f"calibration pipeline mismatch ({'; '.join(measurement)}): {path}")
|
|
return cal
|
|
if cal.unit != "N" or any(cal.channels[axis].unit != "N" for axis in cal.channels):
|
|
raise CalibrationError(f"calibration unit must be N: {path}")
|
|
if set(CHANNEL_IDS) - cal.channels.keys():
|
|
raise CalibrationError(f"calibration requires FN, FX and FY: {path}")
|
|
if pipeline is not None:
|
|
for key in ("stitch_resolution", "dis_mode", "hhd_downsample"):
|
|
recorded, current = cal.pipeline.get(key), pipeline.get(key)
|
|
if recorded is not None and current is not None and recorded != current:
|
|
raise CalibrationError(f"calibration pipeline mismatch ({key}): {path}")
|
|
measurement, _ = filter_mismatches(
|
|
cal.pipeline.get("filters"), pipeline.get("filters")
|
|
)
|
|
if measurement:
|
|
raise CalibrationError(f"calibration pipeline mismatch ({'; '.join(measurement)}): {path}")
|
|
return cal
|
|
|
|
|
|
def pipeline_from_sensor_json(
|
|
path: str | Path | None,
|
|
*,
|
|
filters_state: Mapping[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Pull stitch / tracker / filter fields from a sensor.json for export pipeline/meta.
|
|
|
|
``filters_state`` is the device's ``Sensor.filters_state()``. Pass it
|
|
whenever a device is available: a config file states an intent that a runtime
|
|
``configure()`` may have moved on from, so only the effective values are
|
|
worth recording. Without it the file's ``filters`` block is reported for what
|
|
it is — the intent — and values it does not state are left out rather than
|
|
filled in with a default.
|
|
|
|
The result carries the filter settings in one structured ``"filters"`` key
|
|
(``measurement`` / ``readout`` / ``preset`` / ``source``), not as loose
|
|
top-level keys: the groups decide whether two records are comparable, so they
|
|
have to survive a round trip intact.
|
|
"""
|
|
pipeline: dict[str, Any] = {
|
|
"stitch_resolution": None,
|
|
"dis_mode": None,
|
|
"hhd_downsample": None,
|
|
"filters": {},
|
|
"config_ref": None,
|
|
}
|
|
if path is None and filters_state is None:
|
|
return pipeline
|
|
p = Path(path) if path is not None else None
|
|
if p is not None:
|
|
pipeline["config_ref"] = str(p)
|
|
data: dict[str, Any] = {}
|
|
if p is not None and p.is_file():
|
|
try:
|
|
loaded = json.loads(p.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
loaded = None
|
|
if isinstance(loaded, dict):
|
|
data = loaded
|
|
camera = data.get("camera") if isinstance(data.get("camera"), dict) else {}
|
|
tracker = data.get("tracker") if isinstance(data.get("tracker"), dict) else {}
|
|
stitch = camera.get("resolution_stitch")
|
|
if isinstance(stitch, (list, tuple)) and len(stitch) >= 2:
|
|
pipeline["stitch_resolution"] = [int(stitch[0]), int(stitch[1])]
|
|
if "dis_mode" in tracker:
|
|
try:
|
|
pipeline["dis_mode"] = int(tracker["dis_mode"])
|
|
except (TypeError, ValueError):
|
|
pipeline["dis_mode"] = tracker.get("dis_mode")
|
|
if "hhd_downsample" in tracker:
|
|
try:
|
|
pipeline["hhd_downsample"] = int(tracker["hhd_downsample"])
|
|
except (TypeError, ValueError):
|
|
pipeline["hhd_downsample"] = tracker.get("hhd_downsample")
|
|
# Signal conditioning is part of the measurement-pipeline identity: these
|
|
# stages change the reported force values, so they are recorded next to
|
|
# dis_mode / hhd_downsample. The effective state wins over the file.
|
|
pipeline["filters"] = filters_fingerprint(
|
|
filters_state if filters_state else data.get("filters")
|
|
)
|
|
return pipeline
|
|
|
|
|
|
def _piecewise_from_samples(
|
|
channel_id: str,
|
|
samples: Iterable[CalSample],
|
|
*,
|
|
signed: bool,
|
|
) -> ChannelCurve | None:
|
|
rows = sorted(samples, key=lambda s: abs(float(s.raw_mean)))
|
|
if len(rows) < 2:
|
|
return None
|
|
points = tuple((abs(float(s.raw_mean)), float(s.weight_n)) for s in rows)
|
|
json_name = _CANONICAL_TO_JSON[channel_id]
|
|
return ChannelCurve(
|
|
id=channel_id,
|
|
method="piecewise_linear",
|
|
points=points,
|
|
signed=signed,
|
|
extrapolate=True,
|
|
clamp_output_min=0.0,
|
|
raw_key=_JSON_RAW_KEY[json_name],
|
|
input_mode="components" if signed else "magnitude",
|
|
unit="N",
|
|
)
|
|
|
|
|
|
def build(
|
|
samples: Mapping[str, list[CalSample]],
|
|
*,
|
|
identity: dict[str, Any] | None = None,
|
|
pipeline: dict[str, Any] | None = None,
|
|
origin: str = "user",
|
|
extra_meta: dict[str, Any] | None = None,
|
|
) -> Calibration:
|
|
"""Fit piecewise curves from samples; omit a channel if fewer than 2 points."""
|
|
channels: dict[str, ChannelCurve] = {}
|
|
warnings_list: list[str] = []
|
|
for key in CHANNEL_IDS:
|
|
rows = list(samples.get(key, []) or samples.get(key.upper(), []) or [])
|
|
# Also accept FN/FX/FY aliases in the mapping keys.
|
|
if not rows:
|
|
for alias, cid in CHANNEL_ALIASES.items():
|
|
if cid == key and alias in samples:
|
|
rows = list(samples[alias] or [])
|
|
break
|
|
signed = key in ("fx", "fy")
|
|
curve = _piecewise_from_samples(key, rows, signed=signed)
|
|
if curve is None:
|
|
if rows:
|
|
warnings_list.append(f"{key}: need at least 2 points (have {len(rows)})")
|
|
else:
|
|
warnings_list.append(f"{key}: no samples")
|
|
continue
|
|
channels[key] = curve
|
|
|
|
if not channels:
|
|
raise CalibrationError(
|
|
"No complete channels to build (each needs ≥2 points). "
|
|
+ "; ".join(warnings_list)
|
|
)
|
|
|
|
identity = dict(identity or {})
|
|
pipeline = dict(pipeline or {})
|
|
measurements: dict[str, list[dict[str, float | int]]] = {}
|
|
for key in CHANNEL_IDS:
|
|
rows = list(samples.get(key, []) or [])
|
|
if not rows:
|
|
for alias, cid in CHANNEL_ALIASES.items():
|
|
if cid == key and alias in samples:
|
|
rows = list(samples[alias] or [])
|
|
break
|
|
measurements[key] = [asdict(s) for s in rows]
|
|
|
|
provenance: dict[str, Any] = {
|
|
"origin": origin,
|
|
"created_at": datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds"),
|
|
}
|
|
if extra_meta:
|
|
for k, v in extra_meta.items():
|
|
if k == "samples":
|
|
continue
|
|
if k in ("stitch_resolution", "dis_mode", "hhd_downsample", "config_ref"):
|
|
if k not in pipeline or pipeline[k] is None:
|
|
pipeline[k] = v
|
|
elif k == "filters":
|
|
if not pipeline.get("filters"):
|
|
pipeline["filters"] = filters_fingerprint(v)
|
|
elif k in FILTER_KEYS:
|
|
# Flat form: fold the loose key into the structured groups.
|
|
merged = deepcopy(pipeline.get("filters") or {})
|
|
group = "measurement" if k in FILTER_MEASUREMENT_KEYS else "readout"
|
|
merged.setdefault(group, {})[k] = v
|
|
pipeline["filters"] = filters_fingerprint(merged)
|
|
elif k in ("sensor_type", "sn", "hardware_rev", "gel_id"):
|
|
if k not in identity or not identity[k]:
|
|
identity[k] = v
|
|
else:
|
|
provenance[k] = v
|
|
|
|
cal = Calibration(
|
|
schema_version=CURRENT_SCHEMA,
|
|
unit="N",
|
|
origin=origin,
|
|
identity=identity,
|
|
pipeline=pipeline,
|
|
provenance=provenance,
|
|
channels=channels,
|
|
measurements=measurements,
|
|
extras={},
|
|
source_path=None,
|
|
)
|
|
for w in warnings_list:
|
|
warnings.warn(f"calibration build: {w}", stacklevel=2)
|
|
return cal
|
|
|
|
|
|
def write(
|
|
cal: Calibration,
|
|
path: str | Path,
|
|
*,
|
|
overwrite: bool = False,
|
|
) -> Path:
|
|
"""Write calibration JSON. Runs check() first; raises on errors."""
|
|
result = check(cal)
|
|
if not result.ok:
|
|
raise CalibrationError("; ".join(result.errors))
|
|
path = Path(path)
|
|
if path.exists() and not overwrite:
|
|
raise FileExistsError(f"calibration file exists: {path}")
|
|
if cal.schema_version == 3:
|
|
# A moved manifest must not acquire unresolved or different attachments.
|
|
parse(cal.to_dict(), source_path=path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
doc = cal.to_dict()
|
|
# Preserve an explicitly selected model format; legacy builders still emit v2.
|
|
doc["schema_version"] = 3 if cal.schema_version == 3 else CURRENT_SCHEMA
|
|
path.write_text(
|
|
json.dumps(doc, indent=2, ensure_ascii=False) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
return path.resolve()
|
|
|
|
|
|
def write_user(
|
|
cal: Calibration,
|
|
*,
|
|
bundle_dir: str | Path,
|
|
sn: str,
|
|
overwrite: bool = True,
|
|
) -> Path:
|
|
"""Write a per-SN user calibration under ``bundle_dir/calibrations/<SN>.json``."""
|
|
sn_clean = str(sn or "").strip()
|
|
if not sn_clean:
|
|
raise CalibrationError("write_user requires a non-empty sn")
|
|
dest = Path(bundle_dir) / "calibrations" / f"{sn_clean}.json"
|
|
return write(cal, dest, overwrite=overwrite)
|