from __future__ import annotations import argparse import json import tempfile import unittest from pathlib import Path from unittest import mock import numpy as np import evaluate_egospeak_opportunity_events as evaluator from evaluate_egospeak_opportunity_events import ( OpportunityEvent, _first_causal_source_row, _score_event, evaluate_family, load_opportunity_events, ) from evaluate_floor_state_changes import NUM_CLASSES, NUM_HORIZONS def reviewed_row( uid: str, *, family: str, verdict: str, subtype: str | None, anchor: float, region_start: float, region_end: float, selected_release: float | None = None, selected_release_uid: str | None = None, nearby: list[dict[str, object]] | None = None, opportunity_options: list[dict[str, object]] | None = None, gap_other_intervals: list[dict[str, object]] | None = None, candidate_source: str | None = None, ) -> dict[str, object]: event_type = None if verdict == "POSITIVE": event_type = "WEARER_ENTRY" if family == "entry" else "INT_YIELD" return { "candidate_id": uid, "session": "Session_4", "candidate_family": family, "review": { "human_status": "REVIEWED", "human_verdict": verdict, "human_event_type": event_type, "human_subtype": subtype, "selected_opportunity_uid": selected_release_uid, "selected_opportunity_seconds": selected_release, "boundary_correction_seconds": 0, }, "candidate": { "candidate_source": candidate_source or ( "WEARER_ONSET" if family == "entry" else "WEARER_OFFSET" ), "anchor_dense_seconds": anchor, "region_start_seconds": region_start, "region_end_seconds": region_end, "temporal_evidence": { "behavior_anchor_seconds": anchor, "wearer_offset_seconds": anchor, "nearby_other_intervals": nearby or [], "opportunity_options": opportunity_options or [], "gap_other_intervals": gap_other_intervals or [], }, }, } def prediction_arrays(length: int = 80) -> tuple[np.ndarray, np.ndarray]: return ( np.zeros((length, NUM_CLASSES, NUM_HORIZONS), dtype=np.float32), np.ones((length, NUM_HORIZONS), dtype=bool), ) class OpportunityEventTests(unittest.TestCase): def write_reviews(self, rows: list[dict[str, object]]) -> tuple[tempfile.TemporaryDirectory[str], Path]: temporary = tempfile.TemporaryDirectory() path = Path(temporary.name) / "reviews.jsonl" path.write_text( "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8", ) return temporary, path def test_first_source_row_observes_non_grid_opportunity(self) -> None: source_row, cutoff = _first_causal_source_row(1.05) self.assertEqual(source_row, 5) self.assertAlmostEqual(cutoff, 1.2) self.assertEqual(_first_causal_source_row(1.0), (4, 1.0)) def test_loader_uses_selected_release_and_excludes_entry_backchannel(self) -> None: rows = [ reviewed_row( "entry-positive", family="entry", verdict="POSITIVE", subtype=None, anchor=1.6, region_start=0.0, region_end=3.0, selected_release=1.0, ), reviewed_row( "entry-backchannel", family="entry", verdict="HARD_NEGATIVE", subtype="ENTRY_WEARER_BACKCHANNEL", anchor=3.0, region_start=2.0, region_end=4.0, ), ] temporary, path = self.write_reviews(rows) self.addCleanup(temporary.cleanup) events, diagnostics = load_opportunity_events([path], ["Session_4"]) self.assertEqual(len(events), 1) self.assertEqual(events[0].opportunity_anchor_seconds, 1.0) self.assertEqual(events[0].resolution_seconds, 1.6) self.assertEqual( diagnostics["skipped"]["entry_backchannel_separate_diagnostic"], 1 ) def test_loader_excludes_ambiguous_crossing_others_for_int(self) -> None: nearby = [ { "onset_seconds": 1.0, "offset_seconds": 3.5, "crosses_offset_with_tolerance": True, }, { "onset_seconds": 2.0, "offset_seconds": 3.8, "crosses_offset_with_tolerance": True, }, ] temporary, path = self.write_reviews([ reviewed_row( "yield-positive", family="int", verdict="POSITIVE", subtype=None, anchor=3.0, region_start=0.0, region_end=6.0, nearby=nearby, ) ]) self.addCleanup(temporary.cleanup) events, diagnostics = load_opportunity_events([path], ["Session_4"]) self.assertEqual(events, []) self.assertEqual( diagnostics["skipped"]["int_positive_ambiguous_competing_others"], 1, ) def test_loader_excludes_entry_positive_at_competing_other_onset(self) -> None: rows = [ reviewed_row( "entry-competed", family="entry", verdict="POSITIVE", subtype=None, anchor=1.6, region_start=0.0, region_end=3.0, selected_release=1.0, selected_release_uid="release", opportunity_options=[ { "uid": "release", "anchor_seconds": 1.0, "wearer_active_between_release_and_onset": False, "other_onset_between_release_and_onset": True, } ], gap_other_intervals=[ {"uid": "competitor", "onset_seconds": 1.4} ], ), reviewed_row( "entry-simultaneous", family="entry", verdict="POSITIVE", subtype=None, anchor=4.0, region_start=3.0, region_end=5.0, selected_release=3.2, gap_other_intervals=[ {"uid": "competitor", "onset_seconds": 4.0} ], ), ] temporary, path = self.write_reviews(rows) self.addCleanup(temporary.cleanup) events, diagnostics = load_opportunity_events([path], ["Session_4"]) self.assertEqual(events, []) self.assertEqual( diagnostics["skipped"]["entry_positive_competed_before_behavior"], 2, ) def test_loader_excludes_entry_positive_after_prior_wearer_activity(self) -> None: temporary, path = self.write_reviews([ reviewed_row( "entry-post-outcome", family="entry", verdict="POSITIVE", subtype=None, anchor=2.0, region_start=0.0, region_end=3.0, selected_release=1.0, selected_release_uid="release", opportunity_options=[ { "uid": "release", "anchor_seconds": 1.0, "wearer_active_between_release_and_onset": True, "other_onset_between_release_and_onset": False, } ], ) ]) self.addCleanup(temporary.cleanup) events, diagnostics = load_opportunity_events([path], ["Session_4"]) self.assertEqual(events, []) self.assertEqual( diagnostics["skipped"][ "entry_positive_post_outcome_wearer_activity" ], 1, ) def test_entry_hn_requires_matching_independently_mined_source(self) -> None: rows = [ reviewed_row( "valid", family="entry", verdict="HARD_NEGATIVE", subtype="ENTRY_OTHER_TO_OTHER", anchor=2.0, region_start=2.0, region_end=2.6, candidate_source="OTHER_TO_OTHER", ), reviewed_row( "invalid", family="entry", verdict="HARD_NEGATIVE", subtype="ENTRY_OTHER_TO_OTHER", anchor=4.0, region_start=1.0, region_end=7.0, candidate_source="WEARER_ONSET", ), ] temporary, path = self.write_reviews(rows) self.addCleanup(temporary.cleanup) events, diagnostics = load_opportunity_events([path], ["Session_4"]) self.assertEqual([event.uid for event in events], ["valid"]) self.assertEqual( diagnostics["skipped"]["entry_hn_without_matching_mined_release"], 1, ) def test_zero_duration_mined_opportunity_is_skipped_not_fatal(self) -> None: temporary, path = self.write_reviews([ reviewed_row( "zero-gap", family="entry", verdict="HARD_NEGATIVE", subtype="ENTRY_OTHER_TO_OTHER", anchor=2.0, region_start=2.0, region_end=2.0, candidate_source="OTHER_TO_OTHER", ) ]) self.addCleanup(temporary.cleanup) events, diagnostics = load_opportunity_events([path], ["Session_4"]) self.assertEqual(events, []) self.assertEqual( diagnostics["skipped"]["entry_nonpositive_opportunity_duration"], 1, ) def test_entry_score_uses_max_wearer_probability_before_resolution(self) -> None: event = OpportunityEvent( "entry", "Session_4", "entry", True, "WEARER_ENTRY", "WEARER_ONSET", 1.0, 1.6, 1.6, 1.6, "reviewed_release", ) prediction, valid = prediction_arrays() # source row 4, cutoff 1.0; h=.2/.4/.6 are admitted. Since the onset # is exactly on-grid, the h=.8 post-onset bin must be excluded. prediction[5, 1, 0] = 0.1 prediction[6, 1, 1] = 0.2 prediction[7, 1, 2] = 0.8 prediction[8, 1, 3] = 0.3 scored = _score_event( event, {"Session_4": (prediction, valid)}, entry_max_seconds=2.0, yield_max_seconds=0.6, ) self.assertIsNotNone(scored) self.assertAlmostEqual(scored[0], 0.8) # type: ignore[index] self.assertAlmostEqual(scored[1]["best_horizon_seconds"], 0.6) # type: ignore[index] self.assertEqual(len(scored[1]["used_horizons_seconds"]), 3) # type: ignore[index] def test_entry_positive_admits_bin_containing_off_grid_onset(self) -> None: event = OpportunityEvent( "entry", "Session_4", "entry", True, "WEARER_ENTRY", "WEARER_ONSET", 1.0, 1.65, 1.65, 1.65, "reviewed_release", ) prediction, valid = prediction_arrays() prediction[8, 1, 3] = 0.9 # target cutoff 1.8 contains onset 1.65. scored = _score_event( event, {"Session_4": (prediction, valid)}, entry_max_seconds=2.0, yield_max_seconds=0.6, ) self.assertIsNotNone(scored) self.assertAlmostEqual(scored[0], 0.9) # type: ignore[index] self.assertEqual(len(scored[1]["used_horizons_seconds"]), 4) # type: ignore[index] def test_entry_hard_negative_stops_before_competing_onset(self) -> None: event = OpportunityEvent( "negative", "Session_4", "entry", False, "ENTRY_OTHER_TO_OTHER", "OTHER_TO_OTHER", 1.0, 1.6, 1.6, None, "mined_release", ) prediction, valid = prediction_arrays() prediction[5, 1, 0] = 0.1 prediction[6, 1, 1] = 0.2 prediction[7, 1, 2] = 0.3 prediction[8, 1, 3] = 0.95 # 1.8 s: after competing onset, must be ignored. scored = _score_event( event, {"Session_4": (prediction, valid)}, entry_max_seconds=2.0, yield_max_seconds=0.6, ) self.assertIsNotNone(scored) self.assertAlmostEqual(scored[0], 0.3) # type: ignore[index] self.assertEqual(scored[1]["used_horizons_seconds"], [0.2, 0.4, 0.6000000000000001]) # type: ignore[index] def test_yield_score_is_near_term_inactivity_and_censored(self) -> None: event = OpportunityEvent( "yield", "Session_4", "int", True, "INT_YIELD", "WEARER_OFFSET", 2.0, 2.4, 2.5, 2.4, "crossing_other", ) prediction, valid = prediction_arrays() prediction[10, 1, 0] = 0.9 prediction[11, 1, 1] = 0.2 prediction[12, 1, 2] = 0.0 # target 2.6 crosses other offset 2.5. scored = _score_event( event, {"Session_4": (prediction, valid)}, entry_max_seconds=2.0, yield_max_seconds=0.6, ) self.assertIsNotNone(scored) self.assertAlmostEqual(scored[0], 0.8) # type: ignore[index] self.assertEqual(scored[1]["used_horizons_seconds"], [0.2, 0.4]) # type: ignore[index] def test_positive_is_dropped_if_first_causal_state_observes_outcome(self) -> None: event = OpportunityEvent( "fast", "Session_4", "entry", True, "WEARER_ENTRY", "WEARER_ONSET", 1.05, 1.1, 1.1, 1.1, "reviewed_release", ) prediction, valid = prediction_arrays() self.assertIsNone( _score_event( event, {"Session_4": (prediction, valid)}, entry_max_seconds=2.0, yield_max_seconds=0.6, ) ) def test_one_score_per_opportunity_and_ap(self) -> None: events = [ OpportunityEvent( "p", "Session_4", "entry", True, "WEARER_ENTRY", "WEARER_ONSET", 1.0, 1.6, 1.6, 1.6, "reviewed_release", ), OpportunityEvent( "n", "Session_4", "entry", False, "ENTRY_OTHER_TO_OTHER", "OTHER_TO_OTHER", 2.0, 2.6, 2.6, None, "mined_release", ), ] prediction, valid = prediction_arrays() prediction[7, 1, 2] = 0.9 metric, rows = evaluate_family( events, {"Session_4": (prediction, valid)}, "entry", [], entry_max_seconds=2.0, yield_max_seconds=0.6, ) self.assertEqual(len(rows), 2) self.assertEqual(metric["num_positive"], 1) self.assertEqual(metric["num_hard_negative"], 1) self.assertAlmostEqual(metric["average_precision"], 1.0) def test_main_writes_summary_metrics_and_event_audit(self) -> None: nearby = [{ "onset_seconds": 5.0, "offset_seconds": 6.0, "crosses_offset_with_tolerance": True, }] rows = [ reviewed_row( "ep", family="entry", verdict="POSITIVE", subtype=None, anchor=1.6, region_start=0.0, region_end=3.0, selected_release=1.0, ), reviewed_row( "en", family="entry", verdict="HARD_NEGATIVE", subtype="ENTRY_OTHER_TO_OTHER", anchor=3.0, region_start=3.0, region_end=3.8, candidate_source="OTHER_TO_OTHER", ), reviewed_row( "ip", family="int", verdict="POSITIVE", subtype=None, anchor=5.4, region_start=3.0, region_end=7.0, nearby=nearby, ), reviewed_row( "in", family="int", verdict="HARD_NEGATIVE", subtype="INT_OTHER_BACKCHANNEL", anchor=7.0, region_start=7.0, region_end=7.8, candidate_source="OTHER_BACKCHANNEL", ), ] temporary, reviews = self.write_reviews(rows) self.addCleanup(temporary.cleanup) root = Path(temporary.name) run_dir = root / "run" (run_dir / "pred").mkdir(parents=True) (run_dir / "valid").mkdir() prediction, valid = prediction_arrays() np.save(run_dir / "pred" / "Session_4.npy", prediction) np.save(run_dir / "valid" / "Session_4.npy", valid) output = root / "summary.json" args = argparse.Namespace( reviews=[reviews], run=[f"egospeak={run_dir}"], sessions=["Session_4"], entry_max_seconds=2.0, yield_max_seconds=0.6, calibration_reviews=None, calibration_run=None, calibration_sessions=None, matched_recalls=[], allow_in_sample_thresholds=False, output=output, csv_output=None, event_csv_output=None, print_event_examples=0, allow_provisional_llm_reviews=False, ) with mock.patch.object(evaluator, "parse_args", return_value=args): evaluator.main() payload = json.loads(output.read_text(encoding="utf-8")) self.assertEqual( payload["protocol"], "egospeak-opportunity-conditioned-floor-actions-v2", ) self.assertTrue((root / "summary_metrics.csv").is_file()) self.assertTrue((root / "summary_events.csv").is_file()) self.assertEqual(len(payload["runs"][0]["metrics"]), 2) if __name__ == "__main__": unittest.main()