Add opportunity-conditioned EgoSpeak event evaluation
This commit is contained in:
Xin Wang
@@ -0,0 +1,375 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import numpy as np
|
||||
|
||||
import evaluate_egospeak_opportunity_events as evaluator
|
||||
from evaluate_egospeak_opportunity_events import (
|
||||
OpportunityEvent,
|
||||
_first_causal_source_row,
|
||||
_score_event,
|
||||
evaluate_family,
|
||||
load_opportunity_events,
|
||||
)
|
||||
from evaluate_floor_state_changes import NUM_CLASSES, NUM_HORIZONS
|
||||
|
||||
|
||||
def reviewed_row(
|
||||
uid: str,
|
||||
*,
|
||||
family: str,
|
||||
verdict: str,
|
||||
subtype: str | None,
|
||||
anchor: float,
|
||||
region_start: float,
|
||||
region_end: float,
|
||||
selected_release: float | None = None,
|
||||
nearby: list[dict[str, object]] | None = None,
|
||||
candidate_source: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
event_type = None
|
||||
if verdict == "POSITIVE":
|
||||
event_type = "WEARER_ENTRY" if family == "entry" else "INT_YIELD"
|
||||
return {
|
||||
"candidate_id": uid,
|
||||
"session": "Session_4",
|
||||
"candidate_family": family,
|
||||
"review": {
|
||||
"human_status": "REVIEWED",
|
||||
"human_verdict": verdict,
|
||||
"human_event_type": event_type,
|
||||
"human_subtype": subtype,
|
||||
"selected_opportunity_seconds": selected_release,
|
||||
"boundary_correction_seconds": 0,
|
||||
},
|
||||
"candidate": {
|
||||
"candidate_source": candidate_source or (
|
||||
"WEARER_ONSET" if family == "entry" else "WEARER_OFFSET"
|
||||
),
|
||||
"anchor_dense_seconds": anchor,
|
||||
"region_start_seconds": region_start,
|
||||
"region_end_seconds": region_end,
|
||||
"temporal_evidence": {
|
||||
"behavior_anchor_seconds": anchor,
|
||||
"wearer_offset_seconds": anchor,
|
||||
"nearby_other_intervals": nearby or [],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def prediction_arrays(length: int = 80) -> tuple[np.ndarray, np.ndarray]:
|
||||
return (
|
||||
np.zeros((length, NUM_CLASSES, NUM_HORIZONS), dtype=np.float32),
|
||||
np.ones((length, NUM_HORIZONS), dtype=bool),
|
||||
)
|
||||
|
||||
|
||||
class OpportunityEventTests(unittest.TestCase):
|
||||
def write_reviews(self, rows: list[dict[str, object]]) -> tuple[tempfile.TemporaryDirectory[str], Path]:
|
||||
temporary = tempfile.TemporaryDirectory()
|
||||
path = Path(temporary.name) / "reviews.jsonl"
|
||||
path.write_text(
|
||||
"".join(json.dumps(row) + "\n" for row in rows),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return temporary, path
|
||||
|
||||
def test_first_source_row_observes_non_grid_opportunity(self) -> None:
|
||||
source_row, cutoff = _first_causal_source_row(1.05)
|
||||
self.assertEqual(source_row, 5)
|
||||
self.assertAlmostEqual(cutoff, 1.2)
|
||||
self.assertEqual(_first_causal_source_row(1.0), (4, 1.0))
|
||||
|
||||
def test_loader_uses_selected_release_and_excludes_entry_backchannel(self) -> None:
|
||||
rows = [
|
||||
reviewed_row(
|
||||
"entry-positive",
|
||||
family="entry",
|
||||
verdict="POSITIVE",
|
||||
subtype=None,
|
||||
anchor=1.6,
|
||||
region_start=0.0,
|
||||
region_end=3.0,
|
||||
selected_release=1.0,
|
||||
),
|
||||
reviewed_row(
|
||||
"entry-backchannel",
|
||||
family="entry",
|
||||
verdict="HARD_NEGATIVE",
|
||||
subtype="ENTRY_WEARER_BACKCHANNEL",
|
||||
anchor=3.0,
|
||||
region_start=2.0,
|
||||
region_end=4.0,
|
||||
),
|
||||
]
|
||||
temporary, path = self.write_reviews(rows)
|
||||
self.addCleanup(temporary.cleanup)
|
||||
events, diagnostics = load_opportunity_events([path], ["Session_4"])
|
||||
self.assertEqual(len(events), 1)
|
||||
self.assertEqual(events[0].opportunity_anchor_seconds, 1.0)
|
||||
self.assertEqual(events[0].resolution_seconds, 1.6)
|
||||
self.assertEqual(
|
||||
diagnostics["skipped"]["entry_backchannel_separate_diagnostic"], 1
|
||||
)
|
||||
|
||||
def test_loader_selects_latest_crossing_other_for_int(self) -> None:
|
||||
nearby = [
|
||||
{
|
||||
"onset_seconds": 1.0,
|
||||
"offset_seconds": 3.5,
|
||||
"crosses_offset_with_tolerance": True,
|
||||
},
|
||||
{
|
||||
"onset_seconds": 2.0,
|
||||
"offset_seconds": 3.8,
|
||||
"crosses_offset_with_tolerance": True,
|
||||
},
|
||||
]
|
||||
temporary, path = self.write_reviews([
|
||||
reviewed_row(
|
||||
"yield-positive",
|
||||
family="int",
|
||||
verdict="POSITIVE",
|
||||
subtype=None,
|
||||
anchor=3.0,
|
||||
region_start=0.0,
|
||||
region_end=6.0,
|
||||
nearby=nearby,
|
||||
)
|
||||
])
|
||||
self.addCleanup(temporary.cleanup)
|
||||
events, _ = load_opportunity_events([path], ["Session_4"])
|
||||
self.assertEqual(len(events), 1)
|
||||
self.assertEqual(events[0].opportunity_anchor_seconds, 2.0)
|
||||
self.assertEqual(events[0].censor_seconds, 3.8)
|
||||
self.assertEqual(events[0].competing_intervals, 2)
|
||||
|
||||
def test_entry_hn_requires_matching_independently_mined_source(self) -> None:
|
||||
rows = [
|
||||
reviewed_row(
|
||||
"valid",
|
||||
family="entry",
|
||||
verdict="HARD_NEGATIVE",
|
||||
subtype="ENTRY_OTHER_TO_OTHER",
|
||||
anchor=2.0,
|
||||
region_start=2.0,
|
||||
region_end=2.6,
|
||||
candidate_source="OTHER_TO_OTHER",
|
||||
),
|
||||
reviewed_row(
|
||||
"invalid",
|
||||
family="entry",
|
||||
verdict="HARD_NEGATIVE",
|
||||
subtype="ENTRY_OTHER_TO_OTHER",
|
||||
anchor=4.0,
|
||||
region_start=1.0,
|
||||
region_end=7.0,
|
||||
candidate_source="WEARER_ONSET",
|
||||
),
|
||||
]
|
||||
temporary, path = self.write_reviews(rows)
|
||||
self.addCleanup(temporary.cleanup)
|
||||
events, diagnostics = load_opportunity_events([path], ["Session_4"])
|
||||
self.assertEqual([event.uid for event in events], ["valid"])
|
||||
self.assertEqual(
|
||||
diagnostics["skipped"]["entry_hn_without_matching_mined_release"],
|
||||
1,
|
||||
)
|
||||
|
||||
def test_entry_score_uses_max_wearer_probability_before_resolution(self) -> None:
|
||||
event = OpportunityEvent(
|
||||
"entry", "Session_4", "entry", True, "WEARER_ENTRY", "WEARER_ONSET",
|
||||
1.0, 1.6, 1.6, 1.6, "reviewed_release",
|
||||
)
|
||||
prediction, valid = prediction_arrays()
|
||||
# source row 4, cutoff 1.0; h=.2/.4/.6 are admitted. Since the onset
|
||||
# is exactly on-grid, the h=.8 post-onset bin must be excluded.
|
||||
prediction[5, 1, 0] = 0.1
|
||||
prediction[6, 1, 1] = 0.2
|
||||
prediction[7, 1, 2] = 0.8
|
||||
prediction[8, 1, 3] = 0.3
|
||||
scored = _score_event(
|
||||
event,
|
||||
{"Session_4": (prediction, valid)},
|
||||
entry_max_seconds=2.0,
|
||||
yield_max_seconds=0.6,
|
||||
)
|
||||
self.assertIsNotNone(scored)
|
||||
self.assertAlmostEqual(scored[0], 0.8) # type: ignore[index]
|
||||
self.assertAlmostEqual(scored[1]["best_horizon_seconds"], 0.6) # type: ignore[index]
|
||||
self.assertEqual(len(scored[1]["used_horizons_seconds"]), 3) # type: ignore[index]
|
||||
|
||||
def test_entry_positive_admits_bin_containing_off_grid_onset(self) -> None:
|
||||
event = OpportunityEvent(
|
||||
"entry", "Session_4", "entry", True, "WEARER_ENTRY", "WEARER_ONSET",
|
||||
1.0, 1.65, 1.65, 1.65, "reviewed_release",
|
||||
)
|
||||
prediction, valid = prediction_arrays()
|
||||
prediction[8, 1, 3] = 0.9 # target cutoff 1.8 contains onset 1.65.
|
||||
scored = _score_event(
|
||||
event,
|
||||
{"Session_4": (prediction, valid)},
|
||||
entry_max_seconds=2.0,
|
||||
yield_max_seconds=0.6,
|
||||
)
|
||||
self.assertIsNotNone(scored)
|
||||
self.assertAlmostEqual(scored[0], 0.9) # type: ignore[index]
|
||||
self.assertEqual(len(scored[1]["used_horizons_seconds"]), 4) # type: ignore[index]
|
||||
|
||||
def test_entry_hard_negative_stops_before_competing_onset(self) -> None:
|
||||
event = OpportunityEvent(
|
||||
"negative", "Session_4", "entry", False, "ENTRY_OTHER_TO_OTHER",
|
||||
"OTHER_TO_OTHER", 1.0, 1.6, 1.6, None, "mined_release",
|
||||
)
|
||||
prediction, valid = prediction_arrays()
|
||||
prediction[5, 1, 0] = 0.1
|
||||
prediction[6, 1, 1] = 0.2
|
||||
prediction[7, 1, 2] = 0.3
|
||||
prediction[8, 1, 3] = 0.95 # 1.8 s: after competing onset, must be ignored.
|
||||
scored = _score_event(
|
||||
event,
|
||||
{"Session_4": (prediction, valid)},
|
||||
entry_max_seconds=2.0,
|
||||
yield_max_seconds=0.6,
|
||||
)
|
||||
self.assertIsNotNone(scored)
|
||||
self.assertAlmostEqual(scored[0], 0.3) # type: ignore[index]
|
||||
self.assertEqual(scored[1]["used_horizons_seconds"], [0.2, 0.4, 0.6000000000000001]) # type: ignore[index]
|
||||
|
||||
def test_yield_score_is_near_term_inactivity_and_censored(self) -> None:
|
||||
event = OpportunityEvent(
|
||||
"yield", "Session_4", "int", True, "INT_YIELD", "WEARER_OFFSET",
|
||||
2.0, 2.4, 2.5, 2.4, "crossing_other",
|
||||
)
|
||||
prediction, valid = prediction_arrays()
|
||||
prediction[10, 1, 0] = 0.9
|
||||
prediction[11, 1, 1] = 0.2
|
||||
prediction[12, 1, 2] = 0.0 # target 2.6 crosses other offset 2.5.
|
||||
scored = _score_event(
|
||||
event,
|
||||
{"Session_4": (prediction, valid)},
|
||||
entry_max_seconds=2.0,
|
||||
yield_max_seconds=0.6,
|
||||
)
|
||||
self.assertIsNotNone(scored)
|
||||
self.assertAlmostEqual(scored[0], 0.8) # type: ignore[index]
|
||||
self.assertEqual(scored[1]["used_horizons_seconds"], [0.2, 0.4]) # type: ignore[index]
|
||||
|
||||
def test_positive_is_dropped_if_first_causal_state_observes_outcome(self) -> None:
|
||||
event = OpportunityEvent(
|
||||
"fast", "Session_4", "entry", True, "WEARER_ENTRY", "WEARER_ONSET",
|
||||
1.05, 1.1, 1.1, 1.1, "reviewed_release",
|
||||
)
|
||||
prediction, valid = prediction_arrays()
|
||||
self.assertIsNone(
|
||||
_score_event(
|
||||
event,
|
||||
{"Session_4": (prediction, valid)},
|
||||
entry_max_seconds=2.0,
|
||||
yield_max_seconds=0.6,
|
||||
)
|
||||
)
|
||||
|
||||
def test_one_score_per_opportunity_and_ap(self) -> None:
|
||||
events = [
|
||||
OpportunityEvent(
|
||||
"p", "Session_4", "entry", True, "WEARER_ENTRY", "WEARER_ONSET",
|
||||
1.0, 1.6, 1.6, 1.6, "reviewed_release",
|
||||
),
|
||||
OpportunityEvent(
|
||||
"n", "Session_4", "entry", False, "ENTRY_OTHER_TO_OTHER",
|
||||
"OTHER_TO_OTHER", 2.0, 2.6, 2.6, None, "mined_release",
|
||||
),
|
||||
]
|
||||
prediction, valid = prediction_arrays()
|
||||
prediction[7, 1, 2] = 0.9
|
||||
metric, rows = evaluate_family(
|
||||
events,
|
||||
{"Session_4": (prediction, valid)},
|
||||
"entry",
|
||||
[],
|
||||
entry_max_seconds=2.0,
|
||||
yield_max_seconds=0.6,
|
||||
)
|
||||
self.assertEqual(len(rows), 2)
|
||||
self.assertEqual(metric["num_positive"], 1)
|
||||
self.assertEqual(metric["num_hard_negative"], 1)
|
||||
self.assertAlmostEqual(metric["average_precision"], 1.0)
|
||||
|
||||
def test_main_writes_summary_metrics_and_event_audit(self) -> None:
|
||||
nearby = [{
|
||||
"onset_seconds": 5.0,
|
||||
"offset_seconds": 6.0,
|
||||
"crosses_offset_with_tolerance": True,
|
||||
}]
|
||||
rows = [
|
||||
reviewed_row(
|
||||
"ep", family="entry", verdict="POSITIVE", subtype=None,
|
||||
anchor=1.6, region_start=0.0, region_end=3.0,
|
||||
selected_release=1.0,
|
||||
),
|
||||
reviewed_row(
|
||||
"en", family="entry", verdict="HARD_NEGATIVE",
|
||||
subtype="ENTRY_OTHER_TO_OTHER", anchor=3.0,
|
||||
region_start=3.0, region_end=3.8,
|
||||
candidate_source="OTHER_TO_OTHER",
|
||||
),
|
||||
reviewed_row(
|
||||
"ip", family="int", verdict="POSITIVE", subtype=None,
|
||||
anchor=5.4, region_start=3.0, region_end=7.0, nearby=nearby,
|
||||
),
|
||||
reviewed_row(
|
||||
"in", family="int", verdict="HARD_NEGATIVE",
|
||||
subtype="INT_OTHER_BACKCHANNEL", anchor=7.0,
|
||||
region_start=7.0, region_end=7.8,
|
||||
candidate_source="OTHER_BACKCHANNEL",
|
||||
),
|
||||
]
|
||||
temporary, reviews = self.write_reviews(rows)
|
||||
self.addCleanup(temporary.cleanup)
|
||||
root = Path(temporary.name)
|
||||
run_dir = root / "run"
|
||||
(run_dir / "pred").mkdir(parents=True)
|
||||
(run_dir / "valid").mkdir()
|
||||
prediction, valid = prediction_arrays()
|
||||
np.save(run_dir / "pred" / "Session_4.npy", prediction)
|
||||
np.save(run_dir / "valid" / "Session_4.npy", valid)
|
||||
output = root / "summary.json"
|
||||
args = argparse.Namespace(
|
||||
reviews=[reviews],
|
||||
run=[f"egospeak={run_dir}"],
|
||||
sessions=["Session_4"],
|
||||
entry_max_seconds=2.0,
|
||||
yield_max_seconds=0.6,
|
||||
calibration_reviews=None,
|
||||
calibration_run=None,
|
||||
calibration_sessions=None,
|
||||
matched_recalls=[],
|
||||
allow_in_sample_thresholds=False,
|
||||
output=output,
|
||||
csv_output=None,
|
||||
event_csv_output=None,
|
||||
print_event_examples=0,
|
||||
allow_provisional_llm_reviews=False,
|
||||
)
|
||||
with mock.patch.object(evaluator, "parse_args", return_value=args):
|
||||
evaluator.main()
|
||||
payload = json.loads(output.read_text(encoding="utf-8"))
|
||||
self.assertEqual(
|
||||
payload["protocol"],
|
||||
"egospeak-opportunity-conditioned-floor-actions-v1",
|
||||
)
|
||||
self.assertTrue((root / "summary_metrics.csv").is_file())
|
||||
self.assertTrue((root / "summary_events.csv").is_file())
|
||||
self.assertEqual(len(payload["runs"][0]["metrics"]), 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user