715 lines
28 KiB
Python
715 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""Evaluate reviewed wearer entry and INT-yield opportunities.
|
|
|
|
The forecasting model emits dense EgoSpeak-compatible probabilities rather
|
|
than event detections. At each nominal horizon, a confirmed positive event is
|
|
therefore represented by exactly one score at its corrected behavior-anchor
|
|
bin. A confirmed hard negative is represented by the maximum erroneous-action
|
|
score inside its labeled half-open opportunity region. Positives are never
|
|
expanded into tolerance windows.
|
|
|
|
Thresholds for recall-matched false-positive rates must come from disjoint
|
|
training-side calibration predictions when Sessions 1--3 are evaluated.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
from dataclasses import dataclass
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
from typing import Any, Iterable, Literal
|
|
|
|
import numpy as np
|
|
|
|
from evaluate_floor_state_changes import (
|
|
CLASS_NAMES,
|
|
NUM_CLASSES,
|
|
NUM_HORIZONS,
|
|
OFFICIAL_TRAIN_SESSIONS,
|
|
OFFICIAL_TEST_SESSIONS,
|
|
STEP_SECONDS,
|
|
WEARER_INDEX,
|
|
_operating_points,
|
|
_safe_ap,
|
|
parse_runs,
|
|
validate_recalls,
|
|
)
|
|
|
|
|
|
SOURCE_FPS = 20.0
|
|
Family = Literal["entry", "int"]
|
|
FAMILIES: tuple[Family, ...] = ("entry", "int")
|
|
PROVISIONAL_LLM_SOURCE = "PROVISIONAL_LLM_AUTO_ACCEPT"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AnnotatedOpportunity:
|
|
uid: str
|
|
session: str
|
|
family: Family
|
|
positive: bool
|
|
subtype: str
|
|
start_row: int
|
|
end_row_exclusive: int
|
|
anchor_seconds: float
|
|
region_start_seconds: float
|
|
region_end_seconds: float
|
|
candidate_source: str = ""
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--reviews",
|
|
type=Path,
|
|
action="append",
|
|
required=True,
|
|
help="Review JSONL. Repeat to combine independently stored files.",
|
|
)
|
|
parser.add_argument(
|
|
"--run",
|
|
action="append",
|
|
required=True,
|
|
metavar="NAME=RESULT_DIR",
|
|
help="Inference directory containing pred/ and valid/. Repeat per method.",
|
|
)
|
|
parser.add_argument(
|
|
"--sessions",
|
|
nargs="+",
|
|
default=list(OFFICIAL_TEST_SESSIONS),
|
|
)
|
|
parser.add_argument(
|
|
"--calibration-reviews",
|
|
type=Path,
|
|
action="append",
|
|
default=None,
|
|
help="Disjoint training-side review JSONL used for thresholds.",
|
|
)
|
|
parser.add_argument(
|
|
"--calibration-run",
|
|
action="append",
|
|
default=None,
|
|
metavar="NAME=RESULT_DIR",
|
|
help="Calibration inference directory for each --run name.",
|
|
)
|
|
parser.add_argument(
|
|
"--calibration-sessions",
|
|
nargs="+",
|
|
default=None,
|
|
)
|
|
parser.add_argument(
|
|
"--matched-recalls",
|
|
nargs="+",
|
|
type=float,
|
|
default=(0.7, 0.8, 0.9),
|
|
)
|
|
parser.add_argument(
|
|
"--allow-in-sample-thresholds",
|
|
action="store_true",
|
|
help="Exploratory only: select thresholds on the evaluated events.",
|
|
)
|
|
parser.add_argument(
|
|
"--output",
|
|
type=Path,
|
|
default=Path("reports/easycom_annotated_floor_events/summary.json"),
|
|
)
|
|
parser.add_argument(
|
|
"--csv-output",
|
|
type=Path,
|
|
default=None,
|
|
)
|
|
parser.add_argument(
|
|
"--event-csv-output",
|
|
type=Path,
|
|
default=None,
|
|
help="Optional scored-opportunity CSV (default: OUTPUT stem + _events.csv).",
|
|
)
|
|
parser.add_argument(
|
|
"--print-event-examples",
|
|
type=int,
|
|
default=0,
|
|
metavar="N",
|
|
help="Print up to N scored examples per family for the first run.",
|
|
)
|
|
parser.add_argument(
|
|
"--allow-provisional-llm-reviews",
|
|
action="store_true",
|
|
help=(
|
|
"Explicitly allow auto-accepted LLM labels. Outputs remain marked "
|
|
"provisional and must not be reported as human-adjudicated ground truth."
|
|
),
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def _read_jsonl(paths: Iterable[Path]) -> list[dict[str, Any]]:
|
|
rows: list[dict[str, Any]] = []
|
|
for path in paths:
|
|
if not path.is_file():
|
|
raise FileNotFoundError(path)
|
|
with path.open("r", encoding="utf-8") as handle:
|
|
for line_number, raw_line in enumerate(handle, 1):
|
|
line = raw_line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
row = json.loads(line)
|
|
except json.JSONDecodeError as error:
|
|
raise ValueError(f"Invalid JSON at {path}:{line_number}: {error}") from error
|
|
if not isinstance(row, dict):
|
|
raise ValueError(f"Expected object at {path}:{line_number}")
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def _integer_frame(value: Any, *, field: str, uid: str) -> int:
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
raise ValueError(f"{uid}: {field} must be a numeric 20-Hz frame")
|
|
rounded = int(round(float(value)))
|
|
if not math.isclose(float(value), rounded, abs_tol=1e-6):
|
|
raise ValueError(f"{uid}: {field} must be integral, got {value!r}")
|
|
if rounded < 0:
|
|
raise ValueError(f"{uid}: {field} must be non-negative")
|
|
return rounded
|
|
|
|
|
|
def _seconds(value: Any, *, field: str, uid: str) -> float:
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
raise ValueError(f"{uid}: {field} must be numeric")
|
|
result = float(value)
|
|
if not math.isfinite(result) or result < 0:
|
|
raise ValueError(f"{uid}: {field} must be finite and non-negative")
|
|
return result
|
|
|
|
|
|
def _frame_to_row(frame: int) -> int:
|
|
"""Use the same 20-Hz-to-5-Hz rasterization as EasyCom event derivation."""
|
|
|
|
return frame // int(round(SOURCE_FPS * STEP_SECONDS))
|
|
|
|
|
|
def _seconds_to_start_row(seconds: float) -> int:
|
|
return int(math.floor(seconds / STEP_SECONDS + 1e-7))
|
|
|
|
|
|
def _seconds_to_end_row_exclusive(seconds: float) -> int:
|
|
return int(math.ceil(seconds / STEP_SECONDS - 1e-7))
|
|
|
|
|
|
def _review_payload(
|
|
row: dict[str, Any], uid: str, *, allow_provisional_llm: bool
|
|
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
review = row.get("review")
|
|
candidate = row.get("candidate")
|
|
if not isinstance(review, dict) or not isinstance(candidate, dict):
|
|
raise ValueError(
|
|
f"{uid}: expected review_floor_events.py JSONL with nested review and candidate objects"
|
|
)
|
|
if review.get("human_status") != "REVIEWED":
|
|
raise ValueError(f"{uid}: human_status must be REVIEWED")
|
|
review_source = row.get("review_source", review.get("review_source"))
|
|
if review_source == PROVISIONAL_LLM_SOURCE and not allow_provisional_llm:
|
|
raise ValueError(
|
|
f"{uid}: provisional LLM auto-accept requires "
|
|
"--allow-provisional-llm-reviews"
|
|
)
|
|
return review, candidate
|
|
|
|
|
|
def load_annotated_opportunities(
|
|
paths: Iterable[Path],
|
|
sessions: Iterable[str],
|
|
*,
|
|
allow_provisional_llm: bool = False,
|
|
) -> tuple[list[AnnotatedOpportunity], dict[str, int]]:
|
|
selected_sessions = set(sessions)
|
|
opportunities: list[AnnotatedOpportunity] = []
|
|
seen: set[str] = set()
|
|
excluded = 0
|
|
provisional_llm = 0
|
|
|
|
for row_number, row in enumerate(_read_jsonl(paths), 1):
|
|
uid = str(row.get("candidate_id") or row.get("candidate_uid") or f"row-{row_number}")
|
|
if uid in seen:
|
|
raise ValueError(f"Duplicate reviewed candidate: {uid}")
|
|
seen.add(uid)
|
|
session = str(row.get("session", ""))
|
|
if session not in selected_sessions:
|
|
continue
|
|
review, candidate = _review_payload(
|
|
row, uid, allow_provisional_llm=allow_provisional_llm
|
|
)
|
|
review_source = row.get("review_source", review.get("review_source"))
|
|
if review_source == PROVISIONAL_LLM_SOURCE:
|
|
provisional_llm += 1
|
|
verdict = review.get("human_verdict")
|
|
if verdict == "EXCLUDE":
|
|
excluded += 1
|
|
continue
|
|
if verdict not in {"POSITIVE", "HARD_NEGATIVE"}:
|
|
raise ValueError(f"{uid}: invalid human_verdict {verdict!r}")
|
|
|
|
family_value = row.get("candidate_family")
|
|
if family_value == "entry":
|
|
family: Family = "entry"
|
|
elif family_value == "int":
|
|
family = "int"
|
|
else:
|
|
raise ValueError(f"{uid}: scoring row has invalid candidate_family {family_value!r}")
|
|
|
|
positive = verdict == "POSITIVE"
|
|
event_type = review.get("human_event_type")
|
|
subtype = review.get("human_subtype")
|
|
expected_event_type = "WEARER_ENTRY" if family == "entry" else "INT_YIELD"
|
|
if positive:
|
|
if event_type != expected_event_type or subtype is not None:
|
|
raise ValueError(f"{uid}: positive {family} must be {expected_event_type}")
|
|
subtype_text = expected_event_type
|
|
else:
|
|
prefix = "ENTRY_" if family == "entry" else "INT_"
|
|
if not isinstance(subtype, str) or (
|
|
subtype != "OTHER_HARD_NEGATIVE" and not subtype.startswith(prefix)
|
|
):
|
|
raise ValueError(f"{uid}: hard-negative subtype does not match {family}")
|
|
subtype_text = subtype
|
|
|
|
correction = review.get("boundary_correction_seconds") or 0.0
|
|
correction = float(correction)
|
|
correction_rows = int(round(correction / STEP_SECONDS))
|
|
if not math.isclose(correction, correction_rows * STEP_SECONDS, abs_tol=1e-7):
|
|
raise ValueError(f"{uid}: boundary correction is not on the 5-Hz grid")
|
|
|
|
evidence = candidate.get("temporal_evidence")
|
|
evidence = evidence if isinstance(evidence, dict) else {}
|
|
if positive:
|
|
raw_frame = evidence.get("behavior_anchor_frame", candidate.get("anchor_dense_frame"))
|
|
frame = _integer_frame(raw_frame, field="behavior_anchor_frame", uid=uid)
|
|
row_index = _frame_to_row(frame) + correction_rows
|
|
if row_index < 0:
|
|
raise ValueError(f"{uid}: corrected behavior anchor precedes the session")
|
|
anchor_seconds = frame / SOURCE_FPS + correction
|
|
start_row = row_index
|
|
end_row_exclusive = row_index + 1
|
|
region_start = anchor_seconds
|
|
region_end = anchor_seconds
|
|
else:
|
|
raw_start = _seconds(
|
|
candidate.get("region_start_seconds", candidate.get("anchor_dense_seconds")),
|
|
field="region_start_seconds",
|
|
uid=uid,
|
|
) + correction
|
|
raw_end = _seconds(
|
|
candidate.get("region_end_seconds", candidate.get("anchor_dense_seconds")),
|
|
field="region_end_seconds",
|
|
uid=uid,
|
|
) + correction
|
|
if raw_start < 0 or raw_end < raw_start:
|
|
raise ValueError(f"{uid}: invalid corrected hard-negative region")
|
|
start_row = _seconds_to_start_row(raw_start)
|
|
end_row_exclusive = max(start_row + 1, _seconds_to_end_row_exclusive(raw_end))
|
|
anchor_seconds = _seconds(
|
|
candidate.get("anchor_dense_seconds", raw_start - correction),
|
|
field="anchor_dense_seconds",
|
|
uid=uid,
|
|
) + correction
|
|
region_start, region_end = raw_start, raw_end
|
|
|
|
opportunities.append(
|
|
AnnotatedOpportunity(
|
|
uid=uid,
|
|
session=session,
|
|
family=family,
|
|
positive=positive,
|
|
subtype=subtype_text,
|
|
start_row=start_row,
|
|
end_row_exclusive=end_row_exclusive,
|
|
anchor_seconds=anchor_seconds,
|
|
region_start_seconds=region_start,
|
|
region_end_seconds=region_end,
|
|
candidate_source=str(candidate.get("candidate_source", "")),
|
|
)
|
|
)
|
|
|
|
diagnostics = {
|
|
"num_review_rows": len(seen),
|
|
"num_scoring_eligible": len(opportunities),
|
|
"num_excluded": excluded,
|
|
"num_provisional_llm": provisional_llm,
|
|
}
|
|
return opportunities, diagnostics
|
|
|
|
|
|
def load_session_predictions(
|
|
run_dir: Path,
|
|
sessions: Iterable[str],
|
|
) -> dict[str, tuple[np.ndarray, np.ndarray]]:
|
|
result: dict[str, tuple[np.ndarray, np.ndarray]] = {}
|
|
for session in sessions:
|
|
pred_path = run_dir / "pred" / f"{session}.npy"
|
|
valid_path = run_dir / "valid" / f"{session}.npy"
|
|
if not pred_path.is_file():
|
|
raise FileNotFoundError(pred_path)
|
|
if not valid_path.is_file():
|
|
raise FileNotFoundError(valid_path)
|
|
prediction = np.asarray(np.load(pred_path, allow_pickle=False), dtype=np.float32)
|
|
valid = np.asarray(np.load(valid_path, allow_pickle=False), dtype=bool)
|
|
if prediction.ndim != 3 or prediction.shape[1:] != (NUM_CLASSES, NUM_HORIZONS):
|
|
raise ValueError(f"Expected [T,{NUM_CLASSES},{NUM_HORIZONS}], got {prediction.shape}: {pred_path}")
|
|
if valid.shape != (len(prediction), NUM_HORIZONS):
|
|
raise ValueError(f"Expected {(len(prediction), NUM_HORIZONS)}, got {valid.shape}: {valid_path}")
|
|
if not np.isfinite(prediction).all() or (prediction < -1e-6).any() or (prediction > 1 + 1e-6).any():
|
|
raise ValueError(f"Prediction must be finite and in [0,1]: {pred_path}")
|
|
result[session] = (np.clip(prediction, 0.0, 1.0), valid)
|
|
return result
|
|
|
|
|
|
def score_opportunity(
|
|
opportunity: AnnotatedOpportunity,
|
|
arrays: dict[str, tuple[np.ndarray, np.ndarray]],
|
|
horizon_index: int,
|
|
) -> float | None:
|
|
prediction, valid = arrays[opportunity.session]
|
|
start = opportunity.start_row
|
|
end = opportunity.end_row_exclusive
|
|
if start < 0 or start >= end or end > len(prediction):
|
|
return None
|
|
usable = valid[start:end, horizon_index]
|
|
if not usable.all():
|
|
return None
|
|
scores = prediction[start:end, WEARER_INDEX, horizon_index]
|
|
if opportunity.family == "int":
|
|
scores = 1.0 - scores
|
|
return float(np.max(scores))
|
|
|
|
|
|
def evaluate_family_horizon(
|
|
opportunities: list[AnnotatedOpportunity],
|
|
arrays: dict[str, tuple[np.ndarray, np.ndarray]],
|
|
family: Family,
|
|
horizon_index: int,
|
|
recalls: list[float],
|
|
*,
|
|
fixed_thresholds: dict[float, float | None] | None = None,
|
|
threshold_source: str = "evaluated_events",
|
|
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
rows: list[dict[str, Any]] = []
|
|
invalid = 0
|
|
for opportunity in opportunities:
|
|
if opportunity.family != family:
|
|
continue
|
|
score = score_opportunity(opportunity, arrays, horizon_index)
|
|
if score is None:
|
|
invalid += 1
|
|
continue
|
|
rows.append(
|
|
{
|
|
"uid": opportunity.uid,
|
|
"session": opportunity.session,
|
|
"family": family,
|
|
"positive": opportunity.positive,
|
|
"subtype": opportunity.subtype,
|
|
"score": score,
|
|
"anchor_seconds": opportunity.anchor_seconds,
|
|
"region_start_seconds": opportunity.region_start_seconds,
|
|
"region_end_seconds": opportunity.region_end_seconds,
|
|
"target_start_row": opportunity.start_row,
|
|
"target_end_row_exclusive": opportunity.end_row_exclusive,
|
|
"source_start_row": opportunity.start_row - horizon_index - 1,
|
|
"source_end_row_exclusive": opportunity.end_row_exclusive - horizon_index - 1,
|
|
"nominal_horizon_seconds": (horizon_index + 1) * STEP_SECONDS,
|
|
}
|
|
)
|
|
|
|
scores = np.asarray([row["score"] for row in rows], dtype=np.float64)
|
|
positive = np.asarray([row["positive"] for row in rows], dtype=bool)
|
|
negative = ~positive
|
|
subtype_names = sorted({str(row["subtype"]) for row in rows if not row["positive"]})
|
|
extra_groups = {
|
|
f"fpr_{name.lower()}": np.asarray(
|
|
[not row["positive"] and row["subtype"] == name for row in rows],
|
|
dtype=bool,
|
|
)
|
|
for name in subtype_names
|
|
}
|
|
labels = positive.astype(np.int8)
|
|
metric = {
|
|
"family": family,
|
|
"horizon_index": horizon_index + 1,
|
|
"nominal_seconds": (horizon_index + 1) * STEP_SECONDS,
|
|
"num_positive": int(positive.sum()),
|
|
"num_hard_negative": int(negative.sum()),
|
|
"num_invalid": invalid,
|
|
"average_precision": _safe_ap(labels, scores),
|
|
"operating_points": _operating_points(
|
|
scores,
|
|
positive,
|
|
negative,
|
|
recalls,
|
|
extra_groups=extra_groups,
|
|
fixed_thresholds=fixed_thresholds,
|
|
threshold_source=threshold_source,
|
|
),
|
|
}
|
|
return metric, rows
|
|
|
|
|
|
def thresholds_from_metric(metric: dict[str, Any]) -> dict[float, float | None]:
|
|
return {
|
|
float(item["requested_positive_recall"]): item["threshold"]
|
|
for item in metric["operating_points"]
|
|
}
|
|
|
|
|
|
def _run_names(runs: list[tuple[str, Path]]) -> list[str]:
|
|
return [name for name, _ in runs]
|
|
|
|
|
|
def _validate_protocol_args(args: argparse.Namespace, runs: list[tuple[str, Path]]) -> tuple[list[tuple[str, Path]], bool]:
|
|
official_test = bool(set(args.sessions) & set(OFFICIAL_TEST_SESSIONS))
|
|
if bool(args.calibration_reviews) != bool(args.calibration_sessions):
|
|
raise ValueError(
|
|
"--calibration-reviews and --calibration-sessions must be supplied together"
|
|
)
|
|
has_calibration = bool(args.calibration_reviews and args.calibration_sessions)
|
|
if has_calibration:
|
|
unknown = sorted(set(args.calibration_sessions) - set(OFFICIAL_TRAIN_SESSIONS))
|
|
if unknown:
|
|
raise ValueError(
|
|
f"Calibration must use EasyCom Sessions 4--12, got {unknown}"
|
|
)
|
|
overlap = sorted(set(args.sessions) & set(args.calibration_sessions))
|
|
if overlap:
|
|
raise ValueError(f"Evaluation and calibration sessions overlap: {overlap}")
|
|
if official_test and not has_calibration and not args.allow_in_sample_thresholds:
|
|
raise ValueError(
|
|
"Official Sessions 1--3 require --calibration-reviews and "
|
|
"--calibration-sessions; use --allow-in-sample-thresholds only for exploration"
|
|
)
|
|
if args.calibration_run:
|
|
calibration_runs = parse_runs(args.calibration_run)
|
|
if _run_names(calibration_runs) != _run_names(runs):
|
|
raise ValueError("--calibration-run names/order must match --run")
|
|
else:
|
|
calibration_runs = runs
|
|
return calibration_runs, has_calibration
|
|
|
|
|
|
def write_csv(path: Path, run_results: list[dict[str, Any]]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
fieldnames = [
|
|
"run", "family", "horizon_index", "nominal_seconds", "num_positive",
|
|
"num_hard_negative", "num_invalid", "average_precision", "requested_recall",
|
|
"threshold", "threshold_source", "achieved_recall", "false_positive_rate",
|
|
"false_positive_count",
|
|
]
|
|
with path.open("w", encoding="utf-8", newline="") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=fieldnames)
|
|
writer.writeheader()
|
|
for run in run_results:
|
|
for metric in run["horizons"]:
|
|
for point in metric["operating_points"]:
|
|
writer.writerow(
|
|
{
|
|
"run": run["name"],
|
|
"family": metric["family"],
|
|
"horizon_index": metric["horizon_index"],
|
|
"nominal_seconds": metric["nominal_seconds"],
|
|
"num_positive": metric["num_positive"],
|
|
"num_hard_negative": metric["num_hard_negative"],
|
|
"num_invalid": metric["num_invalid"],
|
|
"average_precision": metric["average_precision"],
|
|
"requested_recall": point["requested_positive_recall"],
|
|
"threshold": point["threshold"],
|
|
"threshold_source": point["threshold_source"],
|
|
"achieved_recall": point["achieved_positive_recall"],
|
|
"false_positive_rate": point["negative_false_positive_rate"],
|
|
"false_positive_count": point["negative_false_positive_count"],
|
|
}
|
|
)
|
|
|
|
|
|
def write_event_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
fieldnames = [
|
|
"run",
|
|
"uid",
|
|
"session",
|
|
"family",
|
|
"positive",
|
|
"subtype",
|
|
"horizon_index",
|
|
"nominal_horizon_seconds",
|
|
"score",
|
|
"anchor_seconds",
|
|
"region_start_seconds",
|
|
"region_end_seconds",
|
|
"target_start_row",
|
|
"target_end_row_exclusive",
|
|
"source_start_row",
|
|
"source_end_row_exclusive",
|
|
]
|
|
with path.open("w", encoding="utf-8", newline="") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=fieldnames)
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
|
|
def print_examples(rows: list[dict[str, Any]], limit: int) -> None:
|
|
if limit <= 0:
|
|
return
|
|
print("\nAnnotated fixed-horizon examples (first run, 0.2-s nominal horizon)")
|
|
for family in FAMILIES:
|
|
print(f"{family}:")
|
|
shown = 0
|
|
for row in rows:
|
|
if row["family"] != family:
|
|
continue
|
|
label = "POS" if row["positive"] else "HN"
|
|
print(
|
|
f" {label} {row['session']} {row['uid']} "
|
|
f"target_rows=[{row['target_start_row']},{row['target_end_row_exclusive']}) "
|
|
f"source_rows=[{row['source_start_row']},{row['source_end_row_exclusive']}) "
|
|
f"region={row['region_start_seconds']:.2f}--{row['region_end_seconds']:.2f}s "
|
|
f"score={row['score']:.4f} subtype={row['subtype']}"
|
|
)
|
|
shown += 1
|
|
if shown >= limit:
|
|
break
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
recalls = validate_recalls(args.matched_recalls)
|
|
runs = parse_runs(args.run)
|
|
calibration_runs, has_calibration = _validate_protocol_args(args, runs)
|
|
|
|
allow_provisional = bool(getattr(args, "allow_provisional_llm_reviews", False))
|
|
opportunities, review_diagnostics = load_annotated_opportunities(
|
|
args.reviews,
|
|
args.sessions,
|
|
allow_provisional_llm=allow_provisional,
|
|
)
|
|
if not opportunities:
|
|
raise ValueError("No scoring-eligible reviewed opportunities for evaluated sessions")
|
|
|
|
calibration_opportunities: list[AnnotatedOpportunity] | None = None
|
|
calibration_diagnostics: dict[str, int] | None = None
|
|
if has_calibration:
|
|
calibration_opportunities, calibration_diagnostics = load_annotated_opportunities(
|
|
args.calibration_reviews,
|
|
args.calibration_sessions,
|
|
allow_provisional_llm=allow_provisional,
|
|
)
|
|
if not calibration_opportunities:
|
|
raise ValueError("No scoring-eligible calibration opportunities")
|
|
|
|
all_results: list[dict[str, Any]] = []
|
|
all_event_rows: list[dict[str, Any]] = []
|
|
first_rows: list[dict[str, Any]] = []
|
|
for run_index, ((name, run_dir), (_, calibration_dir)) in enumerate(zip(runs, calibration_runs)):
|
|
arrays = load_session_predictions(run_dir, args.sessions)
|
|
calibration_arrays = (
|
|
None
|
|
if calibration_opportunities is None
|
|
else load_session_predictions(calibration_dir, args.calibration_sessions)
|
|
)
|
|
horizons: list[dict[str, Any]] = []
|
|
event_rows: list[dict[str, Any]] = []
|
|
for horizon_index in range(NUM_HORIZONS):
|
|
for family in FAMILIES:
|
|
fixed_thresholds = None
|
|
threshold_source = "evaluated_events_exploratory"
|
|
if calibration_opportunities is not None and calibration_arrays is not None:
|
|
calibration_metric, _ = evaluate_family_horizon(
|
|
calibration_opportunities,
|
|
calibration_arrays,
|
|
family,
|
|
horizon_index,
|
|
recalls,
|
|
threshold_source="training_side_calibration",
|
|
)
|
|
fixed_thresholds = thresholds_from_metric(calibration_metric)
|
|
threshold_source = "training_side_calibration"
|
|
metric, rows = evaluate_family_horizon(
|
|
opportunities,
|
|
arrays,
|
|
family,
|
|
horizon_index,
|
|
recalls,
|
|
fixed_thresholds=fixed_thresholds,
|
|
threshold_source=threshold_source,
|
|
)
|
|
horizons.append(metric)
|
|
event_rows.extend(
|
|
{"run": name, **row, "horizon_index": horizon_index + 1}
|
|
for row in rows
|
|
)
|
|
if run_index == 0 and horizon_index == 0:
|
|
first_rows.extend(rows)
|
|
all_event_rows.extend(event_rows)
|
|
all_results.append(
|
|
{
|
|
"name": name,
|
|
"result_dir": str(run_dir),
|
|
"horizons": horizons,
|
|
"mean_entry_ap": float(np.mean([
|
|
item["average_precision"] for item in horizons
|
|
if item["family"] == "entry" and item["average_precision"] is not None
|
|
])) if any(item["family"] == "entry" and item["average_precision"] is not None for item in horizons) else None,
|
|
"mean_int_ap": float(np.mean([
|
|
item["average_precision"] for item in horizons
|
|
if item["family"] == "int" and item["average_precision"] is not None
|
|
])) if any(item["family"] == "int" and item["average_precision"] is not None for item in horizons) else None,
|
|
}
|
|
)
|
|
|
|
provisional_count = review_diagnostics["num_provisional_llm"] + (
|
|
0 if calibration_diagnostics is None
|
|
else calibration_diagnostics["num_provisional_llm"]
|
|
)
|
|
payload = {
|
|
"schema_version": 1,
|
|
"protocol": (
|
|
"provisional-llm-fixed-horizon-floor-events-v1"
|
|
if provisional_count
|
|
else "human-adjudicated-fixed-horizon-floor-events-v1"
|
|
),
|
|
"semantic_scope": (
|
|
"Provisional LLM auto-accepted wearer entry, INT-yield, and hard-negative "
|
|
"opportunities; no human review is implied."
|
|
if provisional_count
|
|
else "Human-confirmed wearer entry, INT-yield, and hard-negative "
|
|
"opportunities; positive events remain single 5-Hz target bins."
|
|
),
|
|
"provisional_llm": bool(provisional_count),
|
|
"prediction_layout": "[target_time, class, future_horizon]",
|
|
"class_names": list(CLASS_NAMES),
|
|
"sessions": list(args.sessions),
|
|
"calibration_sessions": args.calibration_sessions,
|
|
"matched_recalls": recalls,
|
|
"positive_score": "one corrected behavior-anchor bin at each nominal horizon",
|
|
"hard_negative_score": "maximum erroneous-action score over the labeled half-open region at each nominal horizon",
|
|
"review_diagnostics": review_diagnostics,
|
|
"calibration_review_diagnostics": calibration_diagnostics,
|
|
"runs": all_results,
|
|
}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
csv_path = args.csv_output or args.output.with_name(args.output.stem + "_metrics.csv")
|
|
event_csv_path = args.event_csv_output or args.output.with_name(
|
|
args.output.stem + "_events.csv"
|
|
)
|
|
write_csv(csv_path, all_results)
|
|
write_event_csv(event_csv_path, all_event_rows)
|
|
print_examples(first_rows, args.print_event_examples)
|
|
print(f"Saved {args.output}")
|
|
print(f"Saved {csv_path}")
|
|
print(f"Saved {event_csv_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|