补齐 Autocal 测点质量与独立 trial 验证

按已确认窗口比较重复性、加载卸载差异和零载恢复，并逐 trial 留出验证。55 项定向测试、已录视频复核闭环及代码文档检查通过。
2026-10-11 18:27:35 +08:00
parent 3304b4eda9
commit 161e64278b
7 changed files with 352 additions and 4 deletions
@@ -490,6 +490,40 @@
    "depth_missing": "No valid depth/deformation for this frame",
    "flow_missing": "No valid displacement for this frame"
  },
+  "quality": {
+    "open": "Repeatability and validation…",
+    "title": "Site quality analysis",
+    "notice": "Only operator-confirmed valid windows are used. Results remain exploratory. Replay, stability review and trial holdout do not establish physical accuracy or reference synchronisation.",
+    "repeatability": "Repeatability",
+    "repeatability_note": "For the same branch and exact declared reference force, average within each trial, then report range and sample standard deviation across trials. At least two trials are required; this differs from within-hold variation.",
+    "hysteresis": "Loading/unloading difference",
+    "hysteresis_note": "Pair nonzero loading and unloading windows at exactly the same declared reference force within one trial, without interpolation. Difference is unloading minus loading, in raw units; unmatched levels produce no result.",
+    "zero_return": "Zero return",
+    "zero_return_note": "Final unloaded zero minus initial zero in the same trial, in raw units. Only confirmed windows declaring 0 N are paired.",
+    "validation": "Held-out trial validation",
+    "validation_note": "Hold out each entire trial, fit zero/loading windows from other trials, then predict all confirmed windows in the held-out trial. Windows have equal weight. Cross-validation does not replace another independent experiment.",
+    "validation_summary": "Evaluated windows: {count}; RMSE {rmse} N; MAE {mae} N; maximum absolute error {maximum} N; outside training raw range: {outside}.",
+    "no_data": "No eligible data yet; review the required windows first.",
+    "insufficient_training": "Insufficient training data in other trials",
+    "outside_range": "Outside training raw range",
+    "in_range": "Within training raw range",
+    "col_branch": "Phase",
+    "col_load": "Manual reference (N)",
+    "col_trials": "Trials",
+    "col_range": "Between-trial range (raw)",
+    "col_std": "Between-trial sample std (raw)",
+    "col_trial": "Trial",
+    "col_load_raw": "Loading mean (raw)",
+    "col_unload_raw": "Unloading mean (raw)",
+    "col_delta_raw": "Difference (raw)",
+    "col_initial_zero": "Initial zero (raw)",
+    "col_final_zero": "Final zero (raw)",
+    "col_held_out": "Held-out trial",
+    "col_training": "Training trials",
+    "col_predicted": "Predicted force (N)",
+    "col_error": "Prediction minus reference (N)",
+    "col_range_status": "Range/status"
+  },
  "review": {
    "load_selection": "Load saved review…",
    "confirm_stable": "Reviewed as stable",
@@ -490,6 +490,40 @@
    "depth_missing": "本帧无有效深度／形变信号",
    "flow_missing": "本帧无有效位移信号"
  },
+  "quality": {
+    "open": "重复性与验证…",
+    "title": "测点质量分析",
+    "notice": "仅使用人工已确认的有效窗口；结果仍为初步分析。视频调试、人工稳定确认及 trial 留出验证均不能证明实物准确度或参考力同步。",
+    "repeatability": "重复性",
+    "repeatability_note": "相同分支、相同人工参考力下，先计算各 trial 的均值，再计算 trial 间极差和样本标准差。至少两次 trial 才有重复性统计；这不同于保持段内波动。",
+    "hysteresis": "加载／卸载差异",
+    "hysteresis_note": "只配对同一 trial 内人工参考力完全相同的非零加载与卸载窗口，不插值。差值为卸载减加载，单位 raw；缺少匹配档位时不生成结果。",
+    "zero_return": "零载恢复",
+    "zero_return_note": "同一 trial 的最终卸载零载减初始零载，单位 raw；只配对人工参考力为 0 N 的已确认窗口。",
+    "validation": "独立 trial 验证",
+    "validation_note": "逐次留出一个完整 trial，用其余 trial 的零载／加载窗口拟合，再预测留出 trial 的全部已确认窗口。每窗口等权；这是交叉验证，不能代替另一场独立实验。",
+    "validation_summary": "验证窗口 {count} 个；RMSE {rmse} N；MAE {mae} N；最大绝对误差 {maximum} N；超出训练 raw 范围 {outside} 个。",
+    "no_data": "尚无可用数据，请先复核所需窗口。",
+    "insufficient_training": "其余 trial 的训练数据不足",
+    "outside_range": "超出训练 raw 范围",
+    "in_range": "训练 raw 范围内",
+    "col_branch": "阶段",
+    "col_load": "人工参考力 (N)",
+    "col_trials": "trial 数",
+    "col_range": "trial 间极差 (raw)",
+    "col_std": "trial 间样本标准差 (raw)",
+    "col_trial": "trial",
+    "col_load_raw": "加载均值 (raw)",
+    "col_unload_raw": "卸载均值 (raw)",
+    "col_delta_raw": "差值 (raw)",
+    "col_initial_zero": "初始零载 (raw)",
+    "col_final_zero": "最终零载 (raw)",
+    "col_held_out": "留出验证 trial",
+    "col_training": "训练 trial",
+    "col_predicted": "预测力 (N)",
+    "col_error": "预测减参考 (N)",
+    "col_range_status": "范围／状态"
+  },
  "review": {
    "load_selection": "载入已保存复核…",
    "confirm_stable": "已检查此段稳定",
@@ -103,7 +103,7 @@ finger_F001_<时间>_<标识>/
 | 视频／相机 → 帧采集时间 → finger 处理 → `SdkPacket` | SDK capture、shared acquisition |
 | 手工参考 + 保持状态 → 有效候选窗口及事件 | Autocal trial |
 | session／site／trial 关联 → 根清单；数据包 → 视频及 CSV | Autocal 会话；shared recorder 负责文件写入 |
-| 完整 trial + 帧身份／电脑时间 → 保持窗口原始均值 → 加载／卸载对照及 UV 测点着色 | Autocal 初步分析；稳定性和正式验证仍后置 |
+| 完整 trial + 帧身份／电脑时间 → 人工复核窗口 → 加载／卸载、重复性、trial 留出验证及 UV 着色 | Autocal 初步分析；实机验收后置 |

 主时间是电脑取得该帧时绑定的单调时间，UTC 用于追溯；处理完成时间不能代替采集时间。FT／ECAT 后续接入同源接收时间，按时间关联，不按行号配对；低延迟不等于零延迟。旧视频的 `host_replay` 是本次解码时间，只能调试，不能恢复原实验与参考设备的物理同步。

@@ -117,4 +117,8 @@ finger_F001_<时间>_<标识>/

 分析结果保存 selections、候选与已确认拟合、原始样本轨迹、输入 CSV／manifest 的 SHA256、配置快照及分析包版本；每个已确认初始零载段保存基准图像及帧身份、图像哈希。点击“载入已保存复核…”选 review.json 恢复窗口并重算；输入身份或哈希改变时拒绝载入及保存。新增结果另存为独立目录，不修改 reference.csv 的采集声明。

-正式分析拟合 `F_ref = gain_n_per_raw × raw + offset_n`，保存有效档位、范围、误差及加载／卸载差异，并保留独立 trial 验证。基准图像与算法历史尚未完整快照，因此当前录像不承诺从全新进程精确复算同样信号。FT、实际平台反馈、视频容量与驱动延迟仍须联调；自动加载、剪切、空间外推及实时部署留到后续。实施顺序见[实施计划](implementation-plan.zh.md)。
+“重复性与验证…”只用已确认窗口，结果保存在各 site 的 quality：同分支、相同人工参考力跨 trial 的极差／样本标准差；同 trial 同参考力的卸载减加载 raw 差；最终零载减初始零载。配对要求参考力完全相同，不插值；不足两次 trial 不计算重复性。
+
+独立验证逐次留出一个完整 trial，用其余 trial 的零载／加载窗口拟合，再预测留出 trial 的全部已确认窗口。每窗口等权，报告 RMSE／MAE／最大绝对误差及超出训练 raw 范围的数量；训练档位不足时列明原因。交叉验证仍不能代替另一场独立实验，也不生成正式实时系数。
+
+跟踪器基准与算法历史尚未完整快照，录像不能保证从全新进程精确复算。相机、FT、实际平台反馈和容量／延迟须实机联调；自动加载、剪切、空间外推及实时部署留待后续。见[实施计划](implementation-plan.zh.md)。
@@ -38,6 +38,78 @@ def _fit(windows: list[dict]) -> dict | None:
    }


+def _quality(windows: list[dict]) -> dict:
+    """Compare confirmed window means, never random frames from the same trial."""
+    confirmed = [w for w in windows if w["stability_verified"] and w["status"] == "candidate"]
+    trial_ids = sorted({w["trial_id"] for w in confirmed})
+    groups = {}
+    for window in confirmed:
+        key = (window["branch"], window["reference_n"])
+        groups.setdefault(key, {}).setdefault(window["trial_id"], []).append(window["raw_mean"])
+    repeatability = []
+    for (branch, load), trials in sorted(groups.items()):
+        means = {trial: float(np.mean(values)) for trial, values in trials.items()}
+        values = list(means.values())
+        repeatability.append({
+            "branch": branch, "reference_n": load, "trials": len(means),
+            "trial_means_raw": means,
+            "raw_range": float(np.ptp(values)) if len(values) >= 2 else None,
+            "raw_std_between_trials": float(np.std(values, ddof=1)) if len(values) >= 2 else None,
+        })
+    hysteresis, zero_return, folds = [], [], []
+    errors = []
+    for trial_id in trial_ids:
+        own = [w for w in confirmed if w["trial_id"] == trial_id]
+        # Exact declared reference forces only; no nearest-load interpolation.
+        branches = {}
+        for window in own:
+            branches.setdefault((window["branch"], window["reference_n"]), []).append(window["raw_mean"])
+        branch_means = {key: float(np.mean(values)) for key, values in branches.items()}
+        for (branch, load), raw in sorted(branch_means.items()):
+            if branch == "load" and load > 0 and ("unload", load) in branch_means:
+                unload = branch_means[("unload", load)]
+                hysteresis.append({"trial_id": trial_id, "reference_n": load,
+                                   "load_raw": raw, "unload_raw": unload,
+                                   "unload_minus_load_raw": unload - raw})
+        if ("zero", 0.0) in branch_means and ("unload", 0.0) in branch_means:
+            zero_return.append({"trial_id": trial_id,
+                "initial_zero_raw": branch_means[("zero", 0.0)],
+                "final_zero_raw": branch_means[("unload", 0.0)],
+                "final_minus_initial_raw": branch_means[("unload", 0.0)] - branch_means[("zero", 0.0)]})
+        training = [w for w in confirmed if w["trial_id"] != trial_id and w["branch"] in {"zero", "load"}]
+        fit = _fit(training)
+        fold = {"held_out_trial": trial_id,
+                "training_trials": sorted({w["trial_id"] for w in training}),
+                "fit": fit, "windows": [], "status": "insufficient_training"}
+        if fit is not None:
+            low, high = min(w["raw_mean"] for w in training), max(w["raw_mean"] for w in training)
+            for window in own:
+                predicted = fit["gain_n_per_raw"] * window["raw_mean"] + fit["offset_n"]
+                error = predicted - window["reference_n"]
+                fold["windows"].append({
+                    "step_index": window["step_index"], "branch": window["branch"],
+                    "reference_n": window["reference_n"], "predicted_n": predicted,
+                    "error_n": error, "outside_training_raw_range": not low <= window["raw_mean"] <= high,
+                })
+                errors.append(error)
+            fold["status"] = "available"
+        folds.append(fold)
+    return {
+        "basis": "confirmed_window_means", "status": "exploratory",
+        "reference_alignment_possible": bool(confirmed) and all(w["reference_alignment_possible"] for w in confirmed),
+        "repeatability": repeatability, "hysteresis": hysteresis, "zero_return": zero_return,
+        "validation": {
+            "method": "leave_one_trial_out", "folds": folds,
+            "status": "available" if errors else "insufficient_data",
+            "evaluated_windows": len(errors),
+            "rmse_n": float(np.sqrt(np.mean(np.square(errors)))) if errors else None,
+            "mae_n": float(np.mean(np.abs(errors))) if errors else None,
+            "max_abs_error_n": float(np.max(np.abs(errors))) if errors else None,
+            "outside_training_raw_range": sum(w["outside_training_raw_range"] for f in folds for w in f["windows"]),
+        },
+    }
+
+
 def review_session(directory: Path, *, selections: dict | None = None) -> dict:
    """Join frame identity to host timestamps and inspect only complete windows.

@@ -144,7 +216,8 @@ def review_session(directory: Path, *, selections: dict | None = None) -> dict:
        fit = _fit(training)
        confirmed = [w for w in training if w["stability_verified"]]
        confirmed_fit = _fit(confirmed)
-        sites.append({**site, "windows": selected, "fit": fit, "confirmed_fit": confirmed_fit})
+        sites.append({**site, "windows": selected, "fit": fit,
+                      "confirmed_fit": confirmed_fit, "quality": _quality(selected)})
    return {
        "format_version": "autocal-review-1.0", "session_id": manifest["session_id"],
        "source_directory": str(root), "session_status": manifest["status"],
@@ -0,0 +1,78 @@
+"""Per-site trial comparisons and held-out validation from reviewed data."""
+
+from orisys_app_shared.i18n import t
+from PySide6.QtWidgets import (
+    QDialog, QHeaderView, QLabel, QTableWidget, QTableWidgetItem,
+    QTabWidget, QVBoxLayout, QWidget,
+)
+
+
+def _number(value):
+    return "—" if value is None else f"{value:.6g}"
+
+
+class QualityReviewDialog(QDialog):
+    def __init__(self, site: dict, parent=None):
+        super().__init__(parent)
+        self.setWindowTitle(t("quality.title") + " · " + site["site_id"])
+        self.resize(1060, 560)
+        layout = QVBoxLayout(self)
+        notice = QLabel(t("quality.notice"), self)
+        notice.setWordWrap(True)
+        layout.addWidget(notice)
+        self.tabs = QTabWidget(self)
+        layout.addWidget(self.tabs, 1)
+        quality = site["quality"]
+        self._table("repeatability", ("branch", "load", "trials", "range", "std"), [
+            [t("calibration." + row["branch"]), _number(row["reference_n"]), str(row["trials"]),
+             _number(row["raw_range"]), _number(row["raw_std_between_trials"])]
+            for row in quality["repeatability"]])
+        self._table("hysteresis", ("trial", "load", "load_raw", "unload_raw", "delta_raw"), [
+            [row["trial_id"], _number(row["reference_n"]), _number(row["load_raw"]),
+             _number(row["unload_raw"]), _number(row["unload_minus_load_raw"])]
+            for row in quality["hysteresis"]])
+        self._table("zero_return", ("trial", "initial_zero", "final_zero", "delta_raw"), [
+            [row["trial_id"], _number(row["initial_zero_raw"]), _number(row["final_zero_raw"]),
+             _number(row["final_minus_initial_raw"])] for row in quality["zero_return"]])
+        validation = quality["validation"]
+        rows = []
+        for fold in validation["folds"]:
+            prefix = [fold["held_out_trial"], ", ".join(fold["training_trials"]) or "—"]
+            if not fold["windows"]:
+                rows.append(prefix + ["—"] * 4 + [t("quality.insufficient_training")])
+            for window in fold["windows"]:
+                rows.append(prefix + [t("calibration." + window["branch"]),
+                    _number(window["reference_n"]), _number(window["predicted_n"]),
+                    _number(window["error_n"]),
+                    t("quality.outside_range" if window["outside_training_raw_range"] else "quality.in_range")])
+        summary = t("quality.validation_summary").format(
+            count=validation["evaluated_windows"], rmse=_number(validation["rmse_n"]),
+            mae=_number(validation["mae_n"]), maximum=_number(validation["max_abs_error_n"]),
+            outside=validation["outside_training_raw_range"],
+        )
+        self._table("validation", ("held_out", "training", "branch", "load", "predicted", "error", "range_status"),
+                    rows, summary)
+
+    def _table(self, key, columns, rows, summary=""):
+        page = QWidget(self.tabs)
+        layout = QVBoxLayout(page)
+        note = QLabel(t("quality." + key + "_note"), page)
+        note.setWordWrap(True)
+        layout.addWidget(note)
+        if summary:
+            label = QLabel(summary, page)
+            label.setWordWrap(True)
+            layout.addWidget(label)
+        if not rows:
+            layout.addWidget(QLabel(t("quality.no_data"), page))
+        table = QTableWidget(len(rows), len(columns), page)
+        table.setObjectName("quality_" + key)
+        table.setEditTriggers(QTableWidget.EditTrigger.NoEditTriggers)
+        table.setHorizontalHeaderLabels([t("quality.col_" + column) for column in columns])
+        table.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeMode.ResizeToContents)
+        table.horizontalHeader().setStretchLastSection(True)
+        for row, values in enumerate(rows):
+            for column, value in enumerate(values):
+                table.setItem(row, column, QTableWidgetItem(value))
+        layout.addWidget(table, 1)
+        self.tabs.addTab(page, t("quality." + key))
@@ -43,7 +43,13 @@ class SessionReviewDialog(QDialog):
        layout.addLayout(row)
        self.summary = QLabel(self)
        self.summary.setWordWrap(True)
-        layout.addWidget(self.summary)
+        summary_row = QHBoxLayout()
+        summary_row.addWidget(self.summary, 1)
+        self.quality_button = QPushButton(t("quality.open"), self)
+        self.quality_button.setObjectName("reviewQuality")
+        self.quality_button.clicked.connect(self._quality)
+        summary_row.addWidget(self.quality_button)
+        layout.addLayout(summary_row)
        self.chart = QChart()
        self.chart_view = QChartView(self.chart, self)
        self.chart_view.setRenderHint(QPainter.RenderHint.Antialiasing)
@@ -95,6 +101,7 @@ class SessionReviewDialog(QDialog):

    def _show_site(self) -> None:
        index = self.sites.currentIndex()
+        self.quality_button.setEnabled(index >= 0)
        self.chart.removeAllSeries()
        for axis in self.chart.axes():
            self.chart.removeAxis(axis)
@@ -150,6 +157,13 @@ class SessionReviewDialog(QDialog):
            self.table.selectRow(0)
        self._show_window()

+    def _quality(self):
+        from .quality_review_dialog import QualityReviewDialog
+
+        index = self.sites.currentIndex()
+        if index >= 0:
+            QualityReviewDialog(self.review["sites"][index], self).exec()
+
    def _current_window(self):
        site, row = self.sites.currentIndex(), self.table.currentRow()
        if site < 0 or row < 0 or row >= len(self.review["sites"][site]["windows"]):
@@ -210,3 +210,114 @@ def test_baseline_image_matches_first_reviewed_zero_frame(session):
    assert baseline["status"] == "saved" and baseline["segment_frame"] == 1
    image = cv2.imread(str(saved.parent / baseline["image"]))
    assert 10 < image.mean() < 30
+
+
+@pytest.fixture
+def repeated_session(session):
+    import shutil
+
+    source = session / "trials/trial-001"
+    destination = session / "trials/trial-004"
+    shutil.copytree(source, destination)
+    metadata = json.loads((destination / "session.json").read_text(encoding="utf-8"))
+    metadata["trial_id"] = "trial-004"
+    write_json(destination / "session.json", metadata)
+    with (destination / "forces.csv").open(encoding="utf-8") as handle:
+        forces = list(csv.DictReader(handle))
+    for row in forces:
+        row["FN_raw"] = float(row["FN_raw"]) + 2
+    _csv(destination / "forces.csv", forces)
+    manifest = json.loads((session / "session.json").read_text(encoding="utf-8"))
+    manifest["trials"].append({"trial_id": "trial-004", "site_id": "site-001",
+                               "status": "complete", "directory": "trials/trial-004"})
+    write_json(session / "session.json", manifest)
+    result = review_session(session)
+    selections = {f"{w['trial_id']}:{w['step_index']}": {
+        "t_start_host_mono_ns": w["t_start_host_mono_ns"],
+        "t_end_host_mono_ns": w["t_end_host_mono_ns"], "stability_verified": True,
+    } for site in result["sites"] for w in site["windows"]}
+    return session, selections
+
+
+def test_quality_uses_distinct_trials_and_never_trains_on_held_out_data(repeated_session):
+    import math
+    from orisys_autocal.session_review import load_review
+
+    directory, selections = repeated_session
+    result = review_session(directory, selections=selections)
+    quality = result["sites"][0]["quality"]
+    assert quality["reference_alignment_possible"] is False
+    assert all(row["trials"] == 2 and row["raw_range"] == 2
+               and row["raw_std_between_trials"] == pytest.approx(math.sqrt(2))
+               for row in quality["repeatability"])
+    assert len(quality["hysteresis"]) == 2
+    assert all(row["unload_minus_load_raw"] == 1 for row in quality["hysteresis"])
+    assert all(row["final_minus_initial_raw"] == 0 for row in quality["zero_return"])
+    validation = quality["validation"]
+    assert validation["evaluated_windows"] == 10
+    assert validation["rmse_n"] == pytest.approx(math.sqrt(1.05))
+    assert validation["mae_n"] == pytest.approx(1)
+    assert validation["max_abs_error_n"] == pytest.approx(1.5)
+    assert validation["outside_training_raw_range"] == 3
+    for fold in validation["folds"]:
+        assert fold["held_out_trial"] not in fold["training_trials"]
+        assert len(fold["training_trials"]) == 1
+    assert validation["folds"][0]["fit"]["offset_n"] == pytest.approx(-1.5)
+    assert validation["folds"][1]["fit"]["offset_n"] == pytest.approx(-.5)
+    second = result["sites"][1]["quality"]
+    assert second["validation"]["rmse_n"] is None
+    assert all(row["raw_range"] is None for row in second["repeatability"])
+    saved = save_review(result)
+    assert load_review(directory, saved)["sites"][0]["quality"] == quality
+
+
+def test_quality_ignores_unconfirmed_data_and_unmatched_loads(repeated_session):
+    directory, selections = repeated_session
+    result = review_session(directory)
+    assert result["sites"][0]["quality"]["repeatability"] == []
+    assert result["sites"][0]["quality"]["validation"]["rmse_n"] is None
+    # Keep only one confirmed loading level in the other trial: no usable fit.
+    for key in list(selections):
+        if key.startswith("trial-004:") and key != "trial-004:1":
+            selections[key]["stability_verified"] = False
+    result = review_session(directory, selections=selections)
+    quality = result["sites"][0]["quality"]
+    assert quality["validation"]["folds"][0]["status"] == "insufficient_training"
+    assert len(quality["hysteresis"]) == 1
+    path = directory / "trials/trial-001/reference.csv"
+    with path.open(encoding="utf-8") as handle:
+        rows = list(csv.DictReader(handle))
+    rows[3]["F_ref_normal_N"] = "5.01"
+    _csv(path, rows)
+    quality = review_session(directory, selections=selections)["sites"][0]["quality"]
+    assert quality["hysteresis"] == []
+
+
+def test_quality_dialog_is_reachable_and_shows_validation_tables(qt_app, repeated_session, monkeypatch, tmp_path):
+    from PySide6.QtWidgets import QTableWidget
+    from orisys_autocal.ui.session_review_dialog import SessionReviewDialog
+    from orisys_autocal.ui.quality_review_dialog import QualityReviewDialog
+
+    directory, selections = repeated_session
+    review = SessionReviewDialog(directory)
+    review.review = review_session(directory, selections=selections)
+    review._show_site()
+    opened = []
+    monkeypatch.setattr(QualityReviewDialog, "exec", lambda dialog: opened.append(dialog))
+    review.quality_button.click()
+    assert len(opened) == 1
+    dialog = opened[0]
+    assert dialog.tabs.count() == 4
+    repeat = dialog.findChild(QTableWidget, "quality_repeatability")
+    validation = dialog.findChild(QTableWidget, "quality_validation")
+    assert repeat.rowCount() == 5 and validation.rowCount() == 10
+    assert validation.item(0, 0).text() == "trial-001"
+    assert validation.item(0, 1).text() == "trial-004"
+    dialog.show()
+    qt_app.processEvents()
+    assert dialog.grab().save(str(tmp_path / "repeatability.png"))
+    dialog.tabs.setCurrentIndex(3)
+    qt_app.processEvents()
+    assert dialog.grab().save(str(tmp_path / "validation.png"))
+    dialog.close()
+    review.close()