483 lines
18 KiB
Python
483 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
import numpy as np
|
|
|
|
import evaluate_egospeak_opportunity_events as evaluator
|
|
from evaluate_egospeak_opportunity_events import (
|
|
OpportunityEvent,
|
|
_first_causal_source_row,
|
|
_score_event,
|
|
evaluate_family,
|
|
load_opportunity_events,
|
|
)
|
|
from evaluate_floor_state_changes import NUM_CLASSES, NUM_HORIZONS
|
|
|
|
|
|
def reviewed_row(
|
|
uid: str,
|
|
*,
|
|
family: str,
|
|
verdict: str,
|
|
subtype: str | None,
|
|
anchor: float,
|
|
region_start: float,
|
|
region_end: float,
|
|
selected_release: float | None = None,
|
|
selected_release_uid: str | None = None,
|
|
nearby: list[dict[str, object]] | None = None,
|
|
opportunity_options: list[dict[str, object]] | None = None,
|
|
gap_other_intervals: list[dict[str, object]] | None = None,
|
|
candidate_source: str | None = None,
|
|
) -> dict[str, object]:
|
|
event_type = None
|
|
if verdict == "POSITIVE":
|
|
event_type = "WEARER_ENTRY" if family == "entry" else "INT_YIELD"
|
|
return {
|
|
"candidate_id": uid,
|
|
"session": "Session_4",
|
|
"candidate_family": family,
|
|
"review": {
|
|
"human_status": "REVIEWED",
|
|
"human_verdict": verdict,
|
|
"human_event_type": event_type,
|
|
"human_subtype": subtype,
|
|
"selected_opportunity_uid": selected_release_uid,
|
|
"selected_opportunity_seconds": selected_release,
|
|
"boundary_correction_seconds": 0,
|
|
},
|
|
"candidate": {
|
|
"candidate_source": candidate_source or (
|
|
"WEARER_ONSET" if family == "entry" else "WEARER_OFFSET"
|
|
),
|
|
"anchor_dense_seconds": anchor,
|
|
"region_start_seconds": region_start,
|
|
"region_end_seconds": region_end,
|
|
"temporal_evidence": {
|
|
"behavior_anchor_seconds": anchor,
|
|
"wearer_offset_seconds": anchor,
|
|
"nearby_other_intervals": nearby or [],
|
|
"opportunity_options": opportunity_options or [],
|
|
"gap_other_intervals": gap_other_intervals or [],
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def prediction_arrays(length: int = 80) -> tuple[np.ndarray, np.ndarray]:
|
|
return (
|
|
np.zeros((length, NUM_CLASSES, NUM_HORIZONS), dtype=np.float32),
|
|
np.ones((length, NUM_HORIZONS), dtype=bool),
|
|
)
|
|
|
|
|
|
class OpportunityEventTests(unittest.TestCase):
|
|
def write_reviews(self, rows: list[dict[str, object]]) -> tuple[tempfile.TemporaryDirectory[str], Path]:
|
|
temporary = tempfile.TemporaryDirectory()
|
|
path = Path(temporary.name) / "reviews.jsonl"
|
|
path.write_text(
|
|
"".join(json.dumps(row) + "\n" for row in rows),
|
|
encoding="utf-8",
|
|
)
|
|
return temporary, path
|
|
|
|
def test_first_source_row_observes_non_grid_opportunity(self) -> None:
|
|
source_row, cutoff = _first_causal_source_row(1.05)
|
|
self.assertEqual(source_row, 5)
|
|
self.assertAlmostEqual(cutoff, 1.2)
|
|
self.assertEqual(_first_causal_source_row(1.0), (4, 1.0))
|
|
|
|
def test_loader_uses_selected_release_and_excludes_entry_backchannel(self) -> None:
|
|
rows = [
|
|
reviewed_row(
|
|
"entry-positive",
|
|
family="entry",
|
|
verdict="POSITIVE",
|
|
subtype=None,
|
|
anchor=1.6,
|
|
region_start=0.0,
|
|
region_end=3.0,
|
|
selected_release=1.0,
|
|
),
|
|
reviewed_row(
|
|
"entry-backchannel",
|
|
family="entry",
|
|
verdict="HARD_NEGATIVE",
|
|
subtype="ENTRY_WEARER_BACKCHANNEL",
|
|
anchor=3.0,
|
|
region_start=2.0,
|
|
region_end=4.0,
|
|
),
|
|
]
|
|
temporary, path = self.write_reviews(rows)
|
|
self.addCleanup(temporary.cleanup)
|
|
events, diagnostics = load_opportunity_events([path], ["Session_4"])
|
|
self.assertEqual(len(events), 1)
|
|
self.assertEqual(events[0].opportunity_anchor_seconds, 1.0)
|
|
self.assertEqual(events[0].resolution_seconds, 1.6)
|
|
self.assertEqual(
|
|
diagnostics["skipped"]["entry_backchannel_separate_diagnostic"], 1
|
|
)
|
|
|
|
def test_loader_excludes_ambiguous_crossing_others_for_int(self) -> None:
|
|
nearby = [
|
|
{
|
|
"onset_seconds": 1.0,
|
|
"offset_seconds": 3.5,
|
|
"crosses_offset_with_tolerance": True,
|
|
},
|
|
{
|
|
"onset_seconds": 2.0,
|
|
"offset_seconds": 3.8,
|
|
"crosses_offset_with_tolerance": True,
|
|
},
|
|
]
|
|
temporary, path = self.write_reviews([
|
|
reviewed_row(
|
|
"yield-positive",
|
|
family="int",
|
|
verdict="POSITIVE",
|
|
subtype=None,
|
|
anchor=3.0,
|
|
region_start=0.0,
|
|
region_end=6.0,
|
|
nearby=nearby,
|
|
)
|
|
])
|
|
self.addCleanup(temporary.cleanup)
|
|
events, diagnostics = load_opportunity_events([path], ["Session_4"])
|
|
self.assertEqual(events, [])
|
|
self.assertEqual(
|
|
diagnostics["skipped"]["int_positive_ambiguous_competing_others"],
|
|
1,
|
|
)
|
|
|
|
def test_loader_excludes_entry_positive_at_competing_other_onset(self) -> None:
|
|
rows = [
|
|
reviewed_row(
|
|
"entry-competed",
|
|
family="entry",
|
|
verdict="POSITIVE",
|
|
subtype=None,
|
|
anchor=1.6,
|
|
region_start=0.0,
|
|
region_end=3.0,
|
|
selected_release=1.0,
|
|
selected_release_uid="release",
|
|
opportunity_options=[
|
|
{
|
|
"uid": "release",
|
|
"anchor_seconds": 1.0,
|
|
"wearer_active_between_release_and_onset": False,
|
|
"other_onset_between_release_and_onset": True,
|
|
}
|
|
],
|
|
gap_other_intervals=[
|
|
{"uid": "competitor", "onset_seconds": 1.4}
|
|
],
|
|
),
|
|
reviewed_row(
|
|
"entry-simultaneous",
|
|
family="entry",
|
|
verdict="POSITIVE",
|
|
subtype=None,
|
|
anchor=4.0,
|
|
region_start=3.0,
|
|
region_end=5.0,
|
|
selected_release=3.2,
|
|
gap_other_intervals=[
|
|
{"uid": "competitor", "onset_seconds": 4.0}
|
|
],
|
|
),
|
|
]
|
|
temporary, path = self.write_reviews(rows)
|
|
self.addCleanup(temporary.cleanup)
|
|
events, diagnostics = load_opportunity_events([path], ["Session_4"])
|
|
self.assertEqual(events, [])
|
|
self.assertEqual(
|
|
diagnostics["skipped"]["entry_positive_competed_before_behavior"],
|
|
2,
|
|
)
|
|
|
|
def test_loader_excludes_entry_positive_after_prior_wearer_activity(self) -> None:
|
|
temporary, path = self.write_reviews([
|
|
reviewed_row(
|
|
"entry-post-outcome",
|
|
family="entry",
|
|
verdict="POSITIVE",
|
|
subtype=None,
|
|
anchor=2.0,
|
|
region_start=0.0,
|
|
region_end=3.0,
|
|
selected_release=1.0,
|
|
selected_release_uid="release",
|
|
opportunity_options=[
|
|
{
|
|
"uid": "release",
|
|
"anchor_seconds": 1.0,
|
|
"wearer_active_between_release_and_onset": True,
|
|
"other_onset_between_release_and_onset": False,
|
|
}
|
|
],
|
|
)
|
|
])
|
|
self.addCleanup(temporary.cleanup)
|
|
events, diagnostics = load_opportunity_events([path], ["Session_4"])
|
|
self.assertEqual(events, [])
|
|
self.assertEqual(
|
|
diagnostics["skipped"][
|
|
"entry_positive_post_outcome_wearer_activity"
|
|
],
|
|
1,
|
|
)
|
|
|
|
def test_entry_hn_requires_matching_independently_mined_source(self) -> None:
|
|
rows = [
|
|
reviewed_row(
|
|
"valid",
|
|
family="entry",
|
|
verdict="HARD_NEGATIVE",
|
|
subtype="ENTRY_OTHER_TO_OTHER",
|
|
anchor=2.0,
|
|
region_start=2.0,
|
|
region_end=2.6,
|
|
candidate_source="OTHER_TO_OTHER",
|
|
),
|
|
reviewed_row(
|
|
"invalid",
|
|
family="entry",
|
|
verdict="HARD_NEGATIVE",
|
|
subtype="ENTRY_OTHER_TO_OTHER",
|
|
anchor=4.0,
|
|
region_start=1.0,
|
|
region_end=7.0,
|
|
candidate_source="WEARER_ONSET",
|
|
),
|
|
]
|
|
temporary, path = self.write_reviews(rows)
|
|
self.addCleanup(temporary.cleanup)
|
|
events, diagnostics = load_opportunity_events([path], ["Session_4"])
|
|
self.assertEqual([event.uid for event in events], ["valid"])
|
|
self.assertEqual(
|
|
diagnostics["skipped"]["entry_hn_without_matching_mined_release"],
|
|
1,
|
|
)
|
|
|
|
def test_zero_duration_mined_opportunity_is_skipped_not_fatal(self) -> None:
|
|
temporary, path = self.write_reviews([
|
|
reviewed_row(
|
|
"zero-gap",
|
|
family="entry",
|
|
verdict="HARD_NEGATIVE",
|
|
subtype="ENTRY_OTHER_TO_OTHER",
|
|
anchor=2.0,
|
|
region_start=2.0,
|
|
region_end=2.0,
|
|
candidate_source="OTHER_TO_OTHER",
|
|
)
|
|
])
|
|
self.addCleanup(temporary.cleanup)
|
|
events, diagnostics = load_opportunity_events([path], ["Session_4"])
|
|
self.assertEqual(events, [])
|
|
self.assertEqual(
|
|
diagnostics["skipped"]["entry_nonpositive_opportunity_duration"],
|
|
1,
|
|
)
|
|
|
|
def test_entry_score_uses_max_wearer_probability_before_resolution(self) -> None:
|
|
event = OpportunityEvent(
|
|
"entry", "Session_4", "entry", True, "WEARER_ENTRY", "WEARER_ONSET",
|
|
1.0, 1.6, 1.6, 1.6, "reviewed_release",
|
|
)
|
|
prediction, valid = prediction_arrays()
|
|
# source row 4, cutoff 1.0; h=.2/.4/.6 are admitted. Since the onset
|
|
# is exactly on-grid, the h=.8 post-onset bin must be excluded.
|
|
prediction[5, 1, 0] = 0.1
|
|
prediction[6, 1, 1] = 0.2
|
|
prediction[7, 1, 2] = 0.8
|
|
prediction[8, 1, 3] = 0.3
|
|
scored = _score_event(
|
|
event,
|
|
{"Session_4": (prediction, valid)},
|
|
entry_max_seconds=2.0,
|
|
yield_max_seconds=0.6,
|
|
)
|
|
self.assertIsNotNone(scored)
|
|
self.assertAlmostEqual(scored[0], 0.8) # type: ignore[index]
|
|
self.assertAlmostEqual(scored[1]["best_horizon_seconds"], 0.6) # type: ignore[index]
|
|
self.assertEqual(len(scored[1]["used_horizons_seconds"]), 3) # type: ignore[index]
|
|
|
|
def test_entry_positive_admits_bin_containing_off_grid_onset(self) -> None:
|
|
event = OpportunityEvent(
|
|
"entry", "Session_4", "entry", True, "WEARER_ENTRY", "WEARER_ONSET",
|
|
1.0, 1.65, 1.65, 1.65, "reviewed_release",
|
|
)
|
|
prediction, valid = prediction_arrays()
|
|
prediction[8, 1, 3] = 0.9 # target cutoff 1.8 contains onset 1.65.
|
|
scored = _score_event(
|
|
event,
|
|
{"Session_4": (prediction, valid)},
|
|
entry_max_seconds=2.0,
|
|
yield_max_seconds=0.6,
|
|
)
|
|
self.assertIsNotNone(scored)
|
|
self.assertAlmostEqual(scored[0], 0.9) # type: ignore[index]
|
|
self.assertEqual(len(scored[1]["used_horizons_seconds"]), 4) # type: ignore[index]
|
|
|
|
def test_entry_hard_negative_stops_before_competing_onset(self) -> None:
|
|
event = OpportunityEvent(
|
|
"negative", "Session_4", "entry", False, "ENTRY_OTHER_TO_OTHER",
|
|
"OTHER_TO_OTHER", 1.0, 1.6, 1.6, None, "mined_release",
|
|
)
|
|
prediction, valid = prediction_arrays()
|
|
prediction[5, 1, 0] = 0.1
|
|
prediction[6, 1, 1] = 0.2
|
|
prediction[7, 1, 2] = 0.3
|
|
prediction[8, 1, 3] = 0.95 # 1.8 s: after competing onset, must be ignored.
|
|
scored = _score_event(
|
|
event,
|
|
{"Session_4": (prediction, valid)},
|
|
entry_max_seconds=2.0,
|
|
yield_max_seconds=0.6,
|
|
)
|
|
self.assertIsNotNone(scored)
|
|
self.assertAlmostEqual(scored[0], 0.3) # type: ignore[index]
|
|
self.assertEqual(scored[1]["used_horizons_seconds"], [0.2, 0.4, 0.6000000000000001]) # type: ignore[index]
|
|
|
|
def test_yield_score_is_near_term_inactivity_and_censored(self) -> None:
|
|
event = OpportunityEvent(
|
|
"yield", "Session_4", "int", True, "INT_YIELD", "WEARER_OFFSET",
|
|
2.0, 2.4, 2.5, 2.4, "crossing_other",
|
|
)
|
|
prediction, valid = prediction_arrays()
|
|
prediction[10, 1, 0] = 0.9
|
|
prediction[11, 1, 1] = 0.2
|
|
prediction[12, 1, 2] = 0.0 # target 2.6 crosses other offset 2.5.
|
|
scored = _score_event(
|
|
event,
|
|
{"Session_4": (prediction, valid)},
|
|
entry_max_seconds=2.0,
|
|
yield_max_seconds=0.6,
|
|
)
|
|
self.assertIsNotNone(scored)
|
|
self.assertAlmostEqual(scored[0], 0.8) # type: ignore[index]
|
|
self.assertEqual(scored[1]["used_horizons_seconds"], [0.2, 0.4]) # type: ignore[index]
|
|
|
|
def test_positive_is_dropped_if_first_causal_state_observes_outcome(self) -> None:
|
|
event = OpportunityEvent(
|
|
"fast", "Session_4", "entry", True, "WEARER_ENTRY", "WEARER_ONSET",
|
|
1.05, 1.1, 1.1, 1.1, "reviewed_release",
|
|
)
|
|
prediction, valid = prediction_arrays()
|
|
self.assertIsNone(
|
|
_score_event(
|
|
event,
|
|
{"Session_4": (prediction, valid)},
|
|
entry_max_seconds=2.0,
|
|
yield_max_seconds=0.6,
|
|
)
|
|
)
|
|
|
|
def test_one_score_per_opportunity_and_ap(self) -> None:
|
|
events = [
|
|
OpportunityEvent(
|
|
"p", "Session_4", "entry", True, "WEARER_ENTRY", "WEARER_ONSET",
|
|
1.0, 1.6, 1.6, 1.6, "reviewed_release",
|
|
),
|
|
OpportunityEvent(
|
|
"n", "Session_4", "entry", False, "ENTRY_OTHER_TO_OTHER",
|
|
"OTHER_TO_OTHER", 2.0, 2.6, 2.6, None, "mined_release",
|
|
),
|
|
]
|
|
prediction, valid = prediction_arrays()
|
|
prediction[7, 1, 2] = 0.9
|
|
metric, rows = evaluate_family(
|
|
events,
|
|
{"Session_4": (prediction, valid)},
|
|
"entry",
|
|
[],
|
|
entry_max_seconds=2.0,
|
|
yield_max_seconds=0.6,
|
|
)
|
|
self.assertEqual(len(rows), 2)
|
|
self.assertEqual(metric["num_positive"], 1)
|
|
self.assertEqual(metric["num_hard_negative"], 1)
|
|
self.assertAlmostEqual(metric["average_precision"], 1.0)
|
|
|
|
def test_main_writes_summary_metrics_and_event_audit(self) -> None:
|
|
nearby = [{
|
|
"onset_seconds": 5.0,
|
|
"offset_seconds": 6.0,
|
|
"crosses_offset_with_tolerance": True,
|
|
}]
|
|
rows = [
|
|
reviewed_row(
|
|
"ep", family="entry", verdict="POSITIVE", subtype=None,
|
|
anchor=1.6, region_start=0.0, region_end=3.0,
|
|
selected_release=1.0,
|
|
),
|
|
reviewed_row(
|
|
"en", family="entry", verdict="HARD_NEGATIVE",
|
|
subtype="ENTRY_OTHER_TO_OTHER", anchor=3.0,
|
|
region_start=3.0, region_end=3.8,
|
|
candidate_source="OTHER_TO_OTHER",
|
|
),
|
|
reviewed_row(
|
|
"ip", family="int", verdict="POSITIVE", subtype=None,
|
|
anchor=5.4, region_start=3.0, region_end=7.0, nearby=nearby,
|
|
),
|
|
reviewed_row(
|
|
"in", family="int", verdict="HARD_NEGATIVE",
|
|
subtype="INT_OTHER_BACKCHANNEL", anchor=7.0,
|
|
region_start=7.0, region_end=7.8,
|
|
candidate_source="OTHER_BACKCHANNEL",
|
|
),
|
|
]
|
|
temporary, reviews = self.write_reviews(rows)
|
|
self.addCleanup(temporary.cleanup)
|
|
root = Path(temporary.name)
|
|
run_dir = root / "run"
|
|
(run_dir / "pred").mkdir(parents=True)
|
|
(run_dir / "valid").mkdir()
|
|
prediction, valid = prediction_arrays()
|
|
np.save(run_dir / "pred" / "Session_4.npy", prediction)
|
|
np.save(run_dir / "valid" / "Session_4.npy", valid)
|
|
output = root / "summary.json"
|
|
args = argparse.Namespace(
|
|
reviews=[reviews],
|
|
run=[f"egospeak={run_dir}"],
|
|
sessions=["Session_4"],
|
|
entry_max_seconds=2.0,
|
|
yield_max_seconds=0.6,
|
|
calibration_reviews=None,
|
|
calibration_run=None,
|
|
calibration_sessions=None,
|
|
matched_recalls=[],
|
|
allow_in_sample_thresholds=False,
|
|
output=output,
|
|
csv_output=None,
|
|
event_csv_output=None,
|
|
print_event_examples=0,
|
|
allow_provisional_llm_reviews=False,
|
|
)
|
|
with mock.patch.object(evaluator, "parse_args", return_value=args):
|
|
evaluator.main()
|
|
payload = json.loads(output.read_text(encoding="utf-8"))
|
|
self.assertEqual(
|
|
payload["protocol"],
|
|
"egospeak-opportunity-conditioned-floor-actions-v2",
|
|
)
|
|
self.assertTrue((root / "summary_metrics.csv").is_file())
|
|
self.assertTrue((root / "summary_events.csv").is_file())
|
|
self.assertEqual(len(payload["runs"][0]["metrics"]), 2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|