Fix opportunity competing-risk evaluation
This commit is contained in:
Xin Wang
@@ -8,14 +8,18 @@ event head.
|
||||
|
||||
ENTRY opportunities are anchored at a reviewed preceding non-wearer release.
|
||||
Positive opportunities resolve at the wearer entry; core hard negatives
|
||||
resolve when the same or a different non-wearer speaks next. The score is the
|
||||
maximum wearer probability over available 0.2--2.0 s forecasts up to that
|
||||
competing resolution.
|
||||
resolve when the same or a different non-wearer speaks next. A positive is
|
||||
excluded if another non-wearer onset competes before (or exactly at) its wearer
|
||||
entry, or if wearer activity already occurred after the selected release. The
|
||||
score is the maximum wearer probability over available 0.2--2.0 s forecasts up
|
||||
to the uncontested resolution.
|
||||
|
||||
YIELD opportunities are anchored at a non-wearer onset during wearer speech.
|
||||
The score is the maximum future wearer-inactivity probability over 0.2--0.6 s,
|
||||
censored at that non-wearer's offset. Positive and hard-negative examples
|
||||
therefore begin at the same kind of observed conversational opportunity.
|
||||
censored at that non-wearer's offset. Reviews with multiple eligible
|
||||
competing onsets are excluded until the annotation selects one. Positive and
|
||||
hard-negative examples therefore begin at the same kind of unambiguous
|
||||
observed conversational opportunity.
|
||||
|
||||
Prediction arrays use the repository contract
|
||||
``[target_time, class, future_horizon]``. All scores are read diagonally from
|
||||
@@ -178,17 +182,83 @@ def _candidate_region(
|
||||
return start, end
|
||||
|
||||
|
||||
def _int_competing_interval(
|
||||
def _entry_selected_option(
|
||||
candidate: dict[str, Any],
|
||||
review: dict[str, Any],
|
||||
*,
|
||||
selected_release_seconds: float,
|
||||
uid: str,
|
||||
) -> dict[str, Any] | None:
|
||||
evidence = candidate.get("temporal_evidence")
|
||||
evidence = evidence if isinstance(evidence, dict) else {}
|
||||
options = evidence.get("opportunity_options")
|
||||
if not isinstance(options, list):
|
||||
return None
|
||||
selected_uid = review.get("selected_opportunity_uid")
|
||||
matches: list[dict[str, Any]] = []
|
||||
for index, option in enumerate(options):
|
||||
if not isinstance(option, dict):
|
||||
continue
|
||||
option_seconds = _optional_seconds(
|
||||
option.get("anchor_seconds"),
|
||||
field=f"opportunity_options[{index}].anchor_seconds",
|
||||
uid=uid,
|
||||
)
|
||||
if selected_uid is not None and option.get("uid") == selected_uid:
|
||||
matches.append(option)
|
||||
elif selected_uid is None and option_seconds is not None and math.isclose(
|
||||
option_seconds, selected_release_seconds, abs_tol=1e-7
|
||||
):
|
||||
matches.append(option)
|
||||
if len(matches) > 1:
|
||||
raise ValueError(f"{uid}: selected ENTRY opportunity is not unique")
|
||||
return matches[0] if matches else None
|
||||
|
||||
|
||||
def _entry_competing_onsets(
|
||||
candidate: dict[str, Any],
|
||||
*,
|
||||
selected_release_seconds: float,
|
||||
behavior_anchor_seconds: float,
|
||||
uid: str,
|
||||
) -> list[float]:
|
||||
"""Return other onsets that resolve a release before the wearer entry."""
|
||||
|
||||
evidence = candidate.get("temporal_evidence")
|
||||
evidence = evidence if isinstance(evidence, dict) else {}
|
||||
raw_intervals = evidence.get("gap_other_intervals")
|
||||
if not isinstance(raw_intervals, list):
|
||||
raw_intervals = []
|
||||
onsets: set[float] = set()
|
||||
for index, interval in enumerate(raw_intervals):
|
||||
if not isinstance(interval, dict):
|
||||
continue
|
||||
onset = _optional_seconds(
|
||||
interval.get("onset_seconds"),
|
||||
field=f"gap_other_intervals[{index}].onset_seconds",
|
||||
uid=uid,
|
||||
)
|
||||
if onset is None:
|
||||
continue
|
||||
if (
|
||||
onset >= selected_release_seconds - 1e-7
|
||||
and onset <= behavior_anchor_seconds + 1e-7
|
||||
):
|
||||
onsets.add(onset)
|
||||
return sorted(onsets)
|
||||
|
||||
|
||||
def _int_competing_intervals(
|
||||
candidate: dict[str, Any],
|
||||
*,
|
||||
behavior_anchor_seconds: float,
|
||||
uid: str,
|
||||
) -> tuple[float, float, int] | None:
|
||||
) -> list[tuple[float, float]]:
|
||||
evidence = candidate.get("temporal_evidence")
|
||||
evidence = evidence if isinstance(evidence, dict) else {}
|
||||
raw_intervals = evidence.get("nearby_other_intervals")
|
||||
if not isinstance(raw_intervals, list):
|
||||
return None
|
||||
return []
|
||||
eligible: list[tuple[float, float]] = []
|
||||
for index, interval in enumerate(raw_intervals):
|
||||
if not isinstance(interval, dict):
|
||||
@@ -209,13 +279,7 @@ def _int_competing_interval(
|
||||
continue
|
||||
if onset < behavior_anchor_seconds and offset > onset:
|
||||
eligible.append((onset, offset))
|
||||
if not eligible:
|
||||
return None
|
||||
# A semantic INT review does not currently select one competing onset.
|
||||
# The latest interval crossing the wearer offset is the deterministic,
|
||||
# most proximal causal candidate. Multiplicity is surfaced for audit.
|
||||
onset, offset = max(eligible, key=lambda item: (item[0], item[1]))
|
||||
return onset, offset, len(eligible)
|
||||
return sorted(set(eligible))
|
||||
|
||||
|
||||
def load_opportunity_events(
|
||||
@@ -278,6 +342,38 @@ def load_opportunity_events(
|
||||
if selected_release is None:
|
||||
skipped["entry_positive_without_selected_release"] += 1
|
||||
continue
|
||||
selected_option = _entry_selected_option(
|
||||
candidate,
|
||||
review,
|
||||
selected_release_seconds=selected_release,
|
||||
uid=uid,
|
||||
)
|
||||
if (
|
||||
selected_option is not None
|
||||
and selected_option.get(
|
||||
"wearer_active_between_release_and_onset"
|
||||
) is True
|
||||
):
|
||||
skipped["entry_positive_post_outcome_wearer_activity"] += 1
|
||||
continue
|
||||
competing_onsets = _entry_competing_onsets(
|
||||
candidate,
|
||||
selected_release_seconds=selected_release,
|
||||
behavior_anchor_seconds=behavior,
|
||||
uid=uid,
|
||||
)
|
||||
option_reports_competitor = (
|
||||
selected_option is not None
|
||||
and selected_option.get(
|
||||
"other_onset_between_release_and_onset"
|
||||
) is True
|
||||
)
|
||||
if option_reports_competitor and not competing_onsets:
|
||||
skipped["entry_positive_competing_onset_unresolved"] += 1
|
||||
continue
|
||||
if competing_onsets:
|
||||
skipped["entry_positive_competed_before_behavior"] += 1
|
||||
continue
|
||||
anchor = selected_release
|
||||
resolution = behavior
|
||||
censor = behavior
|
||||
@@ -298,17 +394,21 @@ def load_opportunity_events(
|
||||
competing_intervals = 0
|
||||
else:
|
||||
if positive:
|
||||
interval = _int_competing_interval(
|
||||
intervals = _int_competing_intervals(
|
||||
candidate,
|
||||
behavior_anchor_seconds=behavior,
|
||||
uid=uid,
|
||||
)
|
||||
if interval is None:
|
||||
if not intervals:
|
||||
skipped["int_positive_without_crossing_other"] += 1
|
||||
continue
|
||||
anchor, censor, competing_intervals = interval
|
||||
if len(intervals) > 1:
|
||||
skipped["int_positive_ambiguous_competing_others"] += 1
|
||||
continue
|
||||
anchor, censor = intervals[0]
|
||||
competing_intervals = 1
|
||||
resolution = behavior
|
||||
basis = "latest_other_onset_crossing_wearer_offset"
|
||||
basis = "unique_other_onset_crossing_wearer_offset"
|
||||
elif subtype_value in INT_HARD_NEGATIVES:
|
||||
if source in INT_MINED_SOURCES:
|
||||
anchor, censor = _candidate_region(candidate, uid=uid)
|
||||
@@ -316,17 +416,21 @@ def load_opportunity_events(
|
||||
competing_intervals = 1
|
||||
basis = "mined_competing_other_onset"
|
||||
else:
|
||||
interval = _int_competing_interval(
|
||||
intervals = _int_competing_intervals(
|
||||
candidate,
|
||||
behavior_anchor_seconds=behavior,
|
||||
uid=uid,
|
||||
)
|
||||
if interval is None:
|
||||
if not intervals:
|
||||
skipped["int_hn_without_competing_other"] += 1
|
||||
continue
|
||||
anchor, censor, competing_intervals = interval
|
||||
if len(intervals) > 1:
|
||||
skipped["int_hn_ambiguous_competing_others"] += 1
|
||||
continue
|
||||
anchor, censor = intervals[0]
|
||||
competing_intervals = 1
|
||||
resolution = behavior
|
||||
basis = "latest_other_onset_crossing_non_yield_offset"
|
||||
basis = "unique_other_onset_crossing_non_yield_offset"
|
||||
else:
|
||||
skipped["int_unsupported_hard_negative"] += 1
|
||||
continue
|
||||
@@ -731,8 +835,8 @@ def main() -> None:
|
||||
else calibration_diagnostics["num_provisional_llm"]
|
||||
)
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"protocol": "egospeak-opportunity-conditioned-floor-actions-v1",
|
||||
"schema_version": 2,
|
||||
"protocol": "egospeak-opportunity-conditioned-floor-actions-v2",
|
||||
"semantic_scope": (
|
||||
"LLM-assisted opportunity labels with dense wearer-activity scores; "
|
||||
"this is provisional and no human review is implied."
|
||||
@@ -757,6 +861,13 @@ def main() -> None:
|
||||
"yield_max_seconds": yield_max,
|
||||
"entry_headline_hard_negatives": sorted(ENTRY_CORE_HARD_NEGATIVES),
|
||||
"entry_backchannel_policy": "separate diagnostic; excluded from headline",
|
||||
"entry_competing_risk_policy": (
|
||||
"exclude selected-release positives with an other onset at or before "
|
||||
"the wearer entry, or with intervening wearer activity"
|
||||
),
|
||||
"int_multiple_competitor_policy": (
|
||||
"exclude until one competing onset is selected by review"
|
||||
),
|
||||
"matched_recalls": recalls,
|
||||
"review_diagnostics": review_diagnostics,
|
||||
"calibration_review_diagnostics": calibration_diagnostics,
|
||||
|
||||
Reference in New Issue
Block a user