Revamp EasyCom floor event review pipeline
This commit is contained in:
Xin Wang
+363
-427
@@ -1,18 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare manuscript-aligned EasyCom EOT/INT regions for human review.
|
||||
"""Generate wearer-centric EasyCom EOT/INT candidates for AI and human review.
|
||||
|
||||
The released EasyCom annotations provide participant-resolved speech intervals,
|
||||
not semantic EOT/INT ground truth. This script therefore creates auditable
|
||||
candidate regions from the original 20 Hz timeline. It keeps the two wearer-
|
||||
relative directions distinct:
|
||||
|
||||
* EOT-response: an aggregate non-wearer offset while the wearer is silent;
|
||||
* INT-yield: an aggregate non-wearer onset while the wearer is speaking.
|
||||
|
||||
Observed wearer transitions form timing-positive candidates. Competing other
|
||||
transitions and fully observed no-transition windows form hard-negative
|
||||
candidates. Source-boundary cases are retained as exclusions. Every output
|
||||
row remains an automatic candidate until it is independently human reviewed.
|
||||
This stage never changes EasyCom speech boundaries and never claims semantic
|
||||
ground truth. It emits every wearer transition plus independent participant-
|
||||
resolved hard-negative regions. All timing thresholds are CLI parameters, and
|
||||
raw evidence is retained for DeepSeek pre-labeling and human correction.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -26,22 +18,24 @@ import random
|
||||
from typing import Any, Iterable, Sequence
|
||||
|
||||
from derive_easycom_interaction_events import (
|
||||
CAMERA_WEARER_ID,
|
||||
SOURCE_FPS,
|
||||
Clip,
|
||||
Rules,
|
||||
Utterance,
|
||||
build_clips,
|
||||
is_lexical_feedback,
|
||||
is_pure_nonverbal,
|
||||
load_utterances,
|
||||
CAMERA_WEARER_ID, SOURCE_FPS, Clip, Rules, Utterance, build_clips,
|
||||
is_lexical_feedback, is_pure_nonverbal, load_utterances,
|
||||
)
|
||||
from easycom_manifest import ALL_SESSIONS, TEST_SESSIONS
|
||||
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
PROTOCOL_VERSION = "easycom-manuscript-eot-int-v1"
|
||||
DEFAULT_WINDOW_SECONDS = 2.0
|
||||
SCHEMA_VERSION = 2
|
||||
PROTOCOL_VERSION = "easycom-wearer-centric-eot-int-candidates-v2"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CandidateConfig:
|
||||
context_seconds: float = 3.0
|
||||
relation_window_seconds: float = 3.0
|
||||
same_speaker_pause_max_seconds: float = 2.0
|
||||
other_handover_max_seconds: float = 1.5
|
||||
short_vocalization_max_seconds: float = 1.0
|
||||
continuation_min_seconds: float = 0.4
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -49,40 +43,46 @@ class ProtocolCandidate:
|
||||
schema_version: int
|
||||
protocol_version: str
|
||||
uid: str
|
||||
event_id: str
|
||||
session: str
|
||||
split: str
|
||||
task: str
|
||||
event_type: str
|
||||
candidate_source: str
|
||||
rule_candidate_type: str
|
||||
automatic_protocol_label: str
|
||||
automatic_hard_negative_type: str
|
||||
automatic_hard_negative_type: str | None
|
||||
automatic_confidence: str
|
||||
anchor_type: str
|
||||
anchor_dense_frame: int
|
||||
anchor_dense_seconds: float
|
||||
behavior_anchor_type: str | None
|
||||
behavior_dense_frame: int | None
|
||||
behavior_seconds: float | None
|
||||
event_time: float
|
||||
region_start_seconds: float
|
||||
region_end_seconds: float
|
||||
region_end_reason: str
|
||||
previous_other_participant_ids: tuple[int, ...]
|
||||
incoming_other_participant_ids: tuple[int, ...]
|
||||
clip_stem: str
|
||||
source_start_frame: int
|
||||
previous_other_participant_ids: tuple[int, ...]
|
||||
incoming_other_participant_ids: tuple[int, ...]
|
||||
temporal_evidence: dict[str, Any]
|
||||
speaker_timeline: tuple[dict[str, Any], ...]
|
||||
context: tuple[dict[str, Any], ...]
|
||||
automatic_reason: str
|
||||
ai_verdict: None = None
|
||||
ai_event_type: None = None
|
||||
ai_subtype: None = None
|
||||
ai_confidence: None = None
|
||||
ai_reason: None = None
|
||||
human_status: str = "UNREVIEWED"
|
||||
human_verdict: None = None
|
||||
human_event_type: None = None
|
||||
human_subtype: None = None
|
||||
exclude_reason: None = None
|
||||
|
||||
|
||||
def _fill_state(
|
||||
rows: Sequence[Utterance],
|
||||
*,
|
||||
segment_start: int,
|
||||
segment_end: int,
|
||||
wearer_id: int,
|
||||
) -> tuple[bytearray, bytearray]:
|
||||
def _fill_states(rows: Sequence[Utterance], *, segment_start: int, segment_end: int,
|
||||
wearer_id: int) -> tuple[bytearray, bytearray]:
|
||||
length = segment_end - segment_start
|
||||
wearer = bytearray(length)
|
||||
others = bytearray(length)
|
||||
wearer, others = bytearray(length), bytearray(length)
|
||||
for row in rows:
|
||||
left = max(segment_start, row.dense_start_frame) - segment_start
|
||||
right = min(segment_end, row.dense_end_frame_exclusive) - segment_start
|
||||
@@ -93,371 +93,329 @@ def _fill_state(
|
||||
return wearer, others
|
||||
|
||||
|
||||
def _first_transition(
|
||||
state: bytearray,
|
||||
*,
|
||||
start: int,
|
||||
stop: int,
|
||||
rising: bool,
|
||||
) -> int | None:
|
||||
for index in range(max(1, start), min(stop, len(state))):
|
||||
before = bool(state[index - 1])
|
||||
after = bool(state[index])
|
||||
if (not before and after) if rising else (before and not after):
|
||||
return index
|
||||
return None
|
||||
def _run_after(state: bytearray, index: int, value: bool) -> int:
|
||||
end = index
|
||||
while end < len(state) and bool(state[end]) == value:
|
||||
end += 1
|
||||
return end - index
|
||||
|
||||
|
||||
def _active_other_ids(
|
||||
rows: Sequence[Utterance],
|
||||
dense_frame: int,
|
||||
*,
|
||||
wearer_id: int,
|
||||
) -> tuple[int, ...]:
|
||||
def _run_before(state: bytearray, index: int, value: bool) -> int:
|
||||
start = index - 1
|
||||
while start >= 0 and bool(state[start]) == value:
|
||||
start -= 1
|
||||
return index - start - 1
|
||||
|
||||
|
||||
def _all_state(state: bytearray, start: int, stop: int, value: bool) -> bool:
|
||||
left, right = max(0, start), min(len(state), stop)
|
||||
return right > left and all(bool(state[i]) == value for i in range(left, right))
|
||||
|
||||
|
||||
def _active_other_ids(rows: Sequence[Utterance], frame: int, *, wearer_id: int) -> tuple[int, ...]:
|
||||
return tuple(sorted({
|
||||
row.participant_id
|
||||
for row in rows
|
||||
row.participant_id for row in rows
|
||||
if row.participant_id != wearer_id
|
||||
and row.dense_start_frame <= dense_frame < row.dense_end_frame_exclusive
|
||||
and row.dense_start_frame <= frame < row.dense_end_frame_exclusive
|
||||
}))
|
||||
|
||||
|
||||
def _clip_for_frame(clips: Sequence[Clip], dense_frame: int) -> Clip | None:
|
||||
return next(
|
||||
(
|
||||
clip
|
||||
for clip in clips
|
||||
if clip.dense_start_frame <= dense_frame < clip.dense_start_frame + clip.num_frames
|
||||
def _clip_for_frame(clips: Sequence[Clip], frame: int) -> Clip | None:
|
||||
return next((clip for clip in clips
|
||||
if clip.dense_start_frame <= frame < clip.dense_start_frame + clip.num_frames), None)
|
||||
|
||||
|
||||
def _context_rows(rows: Sequence[Utterance], *, frame: int, context_frames: int,
|
||||
wearer_id: int) -> tuple[dict[str, Any], ...]:
|
||||
left, right = frame - context_frames, frame + context_frames
|
||||
return tuple({
|
||||
"uid": row.uid,
|
||||
"participant_id": row.participant_id,
|
||||
"participant_role": "wearer" if row.participant_id == wearer_id else "other",
|
||||
"start_seconds": row.dense_start_seconds,
|
||||
"end_seconds": row.dense_end_seconds,
|
||||
"duration_seconds": row.duration_seconds,
|
||||
"transcription": row.transcription,
|
||||
"target_of_speech": list(row.target_of_speech),
|
||||
} for row in rows
|
||||
if row.dense_end_frame_exclusive > left and row.dense_start_frame < right)
|
||||
|
||||
|
||||
def _relation_evidence(rows: Sequence[Utterance], *, frame: int, wearer_id: int,
|
||||
window_frames: int) -> dict[str, Any]:
|
||||
previous = [row for row in rows if row.participant_id != wearer_id
|
||||
and row.dense_end_frame_exclusive <= frame
|
||||
and frame - row.dense_end_frame_exclusive <= window_frames]
|
||||
following = [row for row in rows if row.participant_id != wearer_id
|
||||
and row.dense_start_frame >= frame
|
||||
and row.dense_start_frame - frame <= window_frames]
|
||||
prev = max(previous, key=lambda row: row.dense_end_frame_exclusive, default=None)
|
||||
nxt = min(following, key=lambda row: row.dense_start_frame, default=None)
|
||||
active_rows = [row for row in rows if row.participant_id != wearer_id
|
||||
and row.dense_start_frame < frame < row.dense_end_frame_exclusive]
|
||||
overlapping = sorted({row.participant_id for row in active_rows})
|
||||
resumes = [] if prev is None else [row for row in following if row.participant_id == prev.participant_id]
|
||||
resumed = min(resumes, key=lambda row: row.dense_start_frame, default=None)
|
||||
nearby_onsets = [row for row in rows if row.participant_id != wearer_id
|
||||
and abs(row.dense_start_frame - frame) <= window_frames]
|
||||
closest_onset = min(nearby_onsets, key=lambda row: abs(row.dense_start_frame - frame), default=None)
|
||||
return {
|
||||
"previous_speaker_id": None if prev is None else prev.participant_id,
|
||||
"previous_speaker_offset_seconds": None if prev is None else prev.dense_end_seconds,
|
||||
"gap_from_previous_speaker_offset_seconds": (
|
||||
None if prev is None else (frame - prev.dense_end_frame_exclusive) / SOURCE_FPS),
|
||||
"overlapping_other_ids": overlapping,
|
||||
"other_overlap_at_event_seconds": max(
|
||||
[0.0] + [(row.dense_end_frame_exclusive - frame) / SOURCE_FPS for row in active_rows]
|
||||
),
|
||||
None,
|
||||
)
|
||||
"next_speaker_id": None if nxt is None else nxt.participant_id,
|
||||
"next_other_onset_seconds": None if nxt is None else nxt.dense_start_seconds,
|
||||
"previous_speaker_resumes": resumed is not None,
|
||||
"previous_speaker_resume_delay_seconds": (
|
||||
None if resumed is None or prev is None
|
||||
else (resumed.dense_start_frame - prev.dense_end_frame_exclusive) / SOURCE_FPS
|
||||
),
|
||||
"closest_other_onset_relative_seconds": (
|
||||
None if closest_onset is None else (closest_onset.dense_start_frame - frame) / SOURCE_FPS
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _context_rows(
|
||||
rows: Sequence[Utterance],
|
||||
*,
|
||||
anchor_frame: int,
|
||||
context_frames: int,
|
||||
wearer_id: int,
|
||||
) -> tuple[dict[str, Any], ...]:
|
||||
left = anchor_frame - context_frames
|
||||
right = anchor_frame + context_frames
|
||||
selected = []
|
||||
for row in rows:
|
||||
if row.dense_end_frame_exclusive <= left or row.dense_start_frame >= right:
|
||||
continue
|
||||
selected.append(
|
||||
{
|
||||
"uid": row.uid,
|
||||
"participant_id": row.participant_id,
|
||||
"participant_role": "wearer" if row.participant_id == wearer_id else "other",
|
||||
"start_seconds": row.dense_start_seconds,
|
||||
"end_seconds": row.dense_end_seconds,
|
||||
"transcription": row.transcription,
|
||||
"target_of_speech": list(row.target_of_speech),
|
||||
}
|
||||
)
|
||||
return tuple(selected)
|
||||
|
||||
|
||||
def _incoming_is_backchannel(
|
||||
rows: Sequence[Utterance],
|
||||
*,
|
||||
dense_frame: int,
|
||||
wearer_id: int,
|
||||
max_seconds: float,
|
||||
) -> bool:
|
||||
incoming = [
|
||||
row
|
||||
for row in rows
|
||||
if row.participant_id != wearer_id
|
||||
and row.dense_start_frame <= dense_frame < row.dense_end_frame_exclusive
|
||||
and row.dense_start_frame >= dense_frame - 1
|
||||
]
|
||||
return bool(incoming) and all(row.duration_seconds <= max_seconds for row in incoming) and any(
|
||||
is_lexical_feedback(row.transcription) or is_pure_nonverbal(row.transcription)
|
||||
for row in incoming
|
||||
)
|
||||
|
||||
|
||||
def _candidate(
|
||||
*,
|
||||
session: str,
|
||||
task: str,
|
||||
automatic_protocol_label: str,
|
||||
automatic_hard_negative_type: str,
|
||||
anchor_frame: int,
|
||||
behavior_frame: int | None,
|
||||
region_end_frame: int,
|
||||
region_end_reason: str,
|
||||
previous_ids: tuple[int, ...],
|
||||
incoming_ids: tuple[int, ...],
|
||||
clips: Sequence[Clip],
|
||||
rows: Sequence[Utterance],
|
||||
wearer_id: int,
|
||||
context_frames: int,
|
||||
reason: str,
|
||||
) -> ProtocolCandidate:
|
||||
clip = _clip_for_frame(clips, anchor_frame)
|
||||
def _candidate(*, session: str, task: str, source: str, rule_type: str,
|
||||
anchor_type: str, frame: int, region_start: int, region_end: int,
|
||||
label: str, subtype: str | None, previous_ids: tuple[int, ...],
|
||||
incoming_ids: tuple[int, ...], evidence: dict[str, Any], clips: Sequence[Clip],
|
||||
rows: Sequence[Utterance], wearer_id: int, context_frames: int,
|
||||
reason: str) -> ProtocolCandidate | None:
|
||||
clip = _clip_for_frame(clips, frame)
|
||||
if clip is None:
|
||||
raise ValueError(f"No source clip for {session} dense frame {anchor_frame}")
|
||||
event_type = "eot_response_candidate" if task == "eot_response" else "int_yield_candidate"
|
||||
anchor_type = "other_offset" if task == "eot_response" else "other_onset"
|
||||
behavior_type = None
|
||||
if behavior_frame is not None:
|
||||
behavior_type = "wearer_onset" if task == "eot_response" else "wearer_offset"
|
||||
return None
|
||||
participant_key = "prev-" + ("-".join(f"p{x}" for x in previous_ids) or "none")
|
||||
participant_key += "__in-" + ("-".join(f"p{x}" for x in incoming_ids) or "none")
|
||||
uid = f"{session}/{source.lower()}/{frame:08d}/{participant_key}"
|
||||
context = _context_rows(rows, frame=frame, context_frames=context_frames, wearer_id=wearer_id)
|
||||
return ProtocolCandidate(
|
||||
schema_version=SCHEMA_VERSION,
|
||||
protocol_version=PROTOCOL_VERSION,
|
||||
uid=f"{session}/{task}/{anchor_frame:08d}",
|
||||
session=session,
|
||||
split="test" if session in TEST_SESSIONS else "train",
|
||||
task=task,
|
||||
event_type=event_type,
|
||||
automatic_protocol_label=automatic_protocol_label,
|
||||
automatic_hard_negative_type=automatic_hard_negative_type,
|
||||
automatic_confidence="timing_candidate",
|
||||
anchor_type=anchor_type,
|
||||
anchor_dense_frame=anchor_frame,
|
||||
anchor_dense_seconds=anchor_frame / SOURCE_FPS,
|
||||
behavior_anchor_type=behavior_type,
|
||||
behavior_dense_frame=behavior_frame,
|
||||
behavior_seconds=None if behavior_frame is None else behavior_frame / SOURCE_FPS,
|
||||
region_start_seconds=anchor_frame / SOURCE_FPS,
|
||||
region_end_seconds=region_end_frame / SOURCE_FPS,
|
||||
region_end_reason=region_end_reason,
|
||||
schema_version=SCHEMA_VERSION, protocol_version=PROTOCOL_VERSION,
|
||||
uid=uid, event_id=uid, session=session,
|
||||
split="test" if session in TEST_SESSIONS else "train", task=task,
|
||||
event_type=source.lower(), candidate_source=source, rule_candidate_type=rule_type,
|
||||
automatic_protocol_label=label, automatic_hard_negative_type=subtype,
|
||||
automatic_confidence="rule_candidate", anchor_type=anchor_type,
|
||||
anchor_dense_frame=frame, anchor_dense_seconds=frame / SOURCE_FPS,
|
||||
event_time=frame / SOURCE_FPS, region_start_seconds=region_start / SOURCE_FPS,
|
||||
region_end_seconds=region_end / SOURCE_FPS, clip_stem=clip.stem,
|
||||
source_start_frame=frame - clip.dense_start_frame + 1,
|
||||
previous_other_participant_ids=previous_ids,
|
||||
incoming_other_participant_ids=incoming_ids,
|
||||
clip_stem=clip.stem,
|
||||
source_start_frame=anchor_frame - clip.dense_start_frame + 1,
|
||||
context=_context_rows(
|
||||
rows,
|
||||
anchor_frame=anchor_frame,
|
||||
context_frames=context_frames,
|
||||
wearer_id=wearer_id,
|
||||
),
|
||||
automatic_reason=reason,
|
||||
incoming_other_participant_ids=incoming_ids, temporal_evidence=evidence,
|
||||
speaker_timeline=context, context=context, automatic_reason=reason,
|
||||
)
|
||||
|
||||
|
||||
def derive_protocol_candidates(
|
||||
utterances: Sequence[Utterance],
|
||||
clips: Sequence[Clip],
|
||||
*,
|
||||
session: str,
|
||||
wearer_id: int = CAMERA_WEARER_ID,
|
||||
window_seconds: float = DEFAULT_WINDOW_SECONDS,
|
||||
context_seconds: float = 3.0,
|
||||
) -> list[ProtocolCandidate]:
|
||||
"""Construct timing candidates with manuscript-aligned directionality."""
|
||||
|
||||
if window_seconds <= 0 or context_seconds < 0:
|
||||
raise ValueError("window_seconds must be positive and context_seconds non-negative")
|
||||
window_frames = max(1, round(window_seconds * SOURCE_FPS))
|
||||
context_frames = max(0, round(context_seconds * SOURCE_FPS))
|
||||
def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[Clip], *,
|
||||
session: str, wearer_id: int = CAMERA_WEARER_ID,
|
||||
config: CandidateConfig | None = None,
|
||||
window_seconds: float | None = None,
|
||||
context_seconds: float | None = None) -> list[ProtocolCandidate]:
|
||||
"""Return every wearer transition and independent hard-negative regions."""
|
||||
cfg = config or CandidateConfig()
|
||||
values = asdict(cfg)
|
||||
if window_seconds is not None:
|
||||
values["relation_window_seconds"] = window_seconds
|
||||
if context_seconds is not None:
|
||||
values["context_seconds"] = context_seconds
|
||||
cfg = CandidateConfig(**values)
|
||||
if any(value < 0 for value in asdict(cfg).values()) or cfg.relation_window_seconds == 0:
|
||||
raise ValueError("candidate timing parameters must be non-negative and relation window positive")
|
||||
context_frames = round(cfg.context_seconds * SOURCE_FPS)
|
||||
relation_frames = round(cfg.relation_window_seconds * SOURCE_FPS)
|
||||
pause_frames = round(cfg.same_speaker_pause_max_seconds * SOURCE_FPS)
|
||||
handover_frames = round(cfg.other_handover_max_seconds * SOURCE_FPS)
|
||||
short_frames = round(cfg.short_vocalization_max_seconds * SOURCE_FPS)
|
||||
continuation_frames = round(cfg.continuation_min_seconds * SOURCE_FPS)
|
||||
rows_by_segment: dict[int, list[Utterance]] = defaultdict(list)
|
||||
clips_by_segment: dict[int, list[Clip]] = defaultdict(list)
|
||||
for row in utterances:
|
||||
rows_by_segment[row.segment_id].append(row)
|
||||
for clip in clips:
|
||||
clips_by_segment[clip.segment_id].append(clip)
|
||||
|
||||
candidates: list[ProtocolCandidate] = []
|
||||
seen: set[tuple[str, int, tuple[int, ...]]] = set()
|
||||
|
||||
def add(item: ProtocolCandidate | None, key: tuple[str, int, tuple[int, ...]]) -> None:
|
||||
if item is not None and key not in seen:
|
||||
seen.add(key)
|
||||
candidates.append(item)
|
||||
|
||||
for segment_id, segment_clips in sorted(clips_by_segment.items()):
|
||||
segment_clips.sort(key=lambda clip: clip.dense_start_frame)
|
||||
segment_rows = sorted(
|
||||
rows_by_segment.get(segment_id, []),
|
||||
key=lambda row: (row.dense_start_frame, row.dense_end_frame_exclusive, row.uid),
|
||||
)
|
||||
rows = sorted(rows_by_segment.get(segment_id, []),
|
||||
key=lambda row: (row.dense_start_frame, row.dense_end_frame_exclusive, row.uid))
|
||||
segment_start = segment_clips[0].dense_start_frame
|
||||
segment_end = segment_clips[-1].dense_start_frame + segment_clips[-1].num_frames
|
||||
wearer, others = _fill_state(
|
||||
segment_rows,
|
||||
segment_start=segment_start,
|
||||
segment_end=segment_end,
|
||||
wearer_id=wearer_id,
|
||||
)
|
||||
wearer, others = _fill_states(rows, segment_start=segment_start,
|
||||
segment_end=segment_end, wearer_id=wearer_id)
|
||||
|
||||
for index in range(1, len(others)):
|
||||
dense_frame = segment_start + index
|
||||
if others[index - 1] and not others[index] and not wearer[index - 1]:
|
||||
competing = _first_transition(
|
||||
others,
|
||||
start=index + 1,
|
||||
stop=min(len(others), index + window_frames + 1),
|
||||
rising=True,
|
||||
# A: all wearer 0->1 and 1->0 transitions, without semantic guessing.
|
||||
for index in range(1, len(wearer)):
|
||||
frame = segment_start + index
|
||||
if not wearer[index - 1] and wearer[index]:
|
||||
evidence = _relation_evidence(rows, frame=frame, wearer_id=wearer_id,
|
||||
window_frames=relation_frames)
|
||||
evidence.update(
|
||||
wearer_speech_duration_after_onset_seconds=_run_after(wearer, index, True) / SOURCE_FPS,
|
||||
previous_wearer_silence_duration_seconds=_run_before(wearer, index, False) / SOURCE_FPS,
|
||||
original_timestamp_seconds=frame / SOURCE_FPS,
|
||||
)
|
||||
nominal_end = index + window_frames
|
||||
observable_end = min(nominal_end, len(others))
|
||||
effective_end = min(observable_end, competing if competing is not None else observable_end)
|
||||
outcome = _first_transition(
|
||||
wearer,
|
||||
start=index,
|
||||
stop=effective_end,
|
||||
rising=True,
|
||||
)
|
||||
previous_ids = _active_other_ids(
|
||||
segment_rows, dense_frame - 1, wearer_id=wearer_id
|
||||
)
|
||||
incoming_ids = (
|
||||
()
|
||||
if competing is None
|
||||
else _active_other_ids(
|
||||
segment_rows, segment_start + competing, wearer_id=wearer_id
|
||||
)
|
||||
)
|
||||
if outcome is not None:
|
||||
label = "positive"
|
||||
subtype = "not_applicable"
|
||||
end_index = outcome
|
||||
end_reason = "wearer_onset"
|
||||
reason = "wearer onset observed after other offset and before competing other onset"
|
||||
elif competing is not None:
|
||||
label = "hard_negative"
|
||||
subtype = (
|
||||
"eot_midturn_pause"
|
||||
if set(previous_ids) & set(incoming_ids)
|
||||
else "eot_other_to_other"
|
||||
)
|
||||
end_index = competing
|
||||
end_reason = "competing_other_onset"
|
||||
reason = "wearer remains silent until another non-wearer onset"
|
||||
elif nominal_end <= len(others):
|
||||
label = "hard_negative"
|
||||
subtype = "eot_no_response"
|
||||
end_index = nominal_end
|
||||
end_reason = "full_window"
|
||||
reason = "wearer remains silent for the full observable response window"
|
||||
else:
|
||||
label = "exclude"
|
||||
subtype = "other"
|
||||
end_index = len(others)
|
||||
end_reason = "right_censored"
|
||||
reason = "recording segment ends before a wearer outcome or full window"
|
||||
candidates.append(_candidate(
|
||||
session=session,
|
||||
task="eot_response",
|
||||
automatic_protocol_label=label,
|
||||
automatic_hard_negative_type=subtype,
|
||||
anchor_frame=dense_frame,
|
||||
behavior_frame=None if outcome is None else segment_start + outcome,
|
||||
region_end_frame=segment_start + end_index,
|
||||
region_end_reason=end_reason,
|
||||
previous_ids=previous_ids,
|
||||
incoming_ids=incoming_ids,
|
||||
clips=segment_clips,
|
||||
rows=segment_rows,
|
||||
wearer_id=wearer_id,
|
||||
add(_candidate(
|
||||
session=session, task="eot_response", source="WEARER_ONSET",
|
||||
rule_type="ENTRY_CANDIDATE", anchor_type="wearer_onset", frame=frame,
|
||||
region_start=frame, region_end=min(segment_end, frame + relation_frames),
|
||||
label="unsure", subtype=None,
|
||||
previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id),
|
||||
incoming_ids=_active_other_ids(rows, frame, wearer_id=wearer_id),
|
||||
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
||||
context_frames=context_frames,
|
||||
reason=reason,
|
||||
))
|
||||
|
||||
if not others[index - 1] and others[index] and wearer[index - 1]:
|
||||
competing = _first_transition(
|
||||
others,
|
||||
start=index + 1,
|
||||
stop=min(len(others), index + window_frames + 1),
|
||||
rising=False,
|
||||
reason="observed wearer 0->1 transition; semantic EOT-response status is unresolved",
|
||||
), ("WEARER_ONSET", frame, ()))
|
||||
elif wearer[index - 1] and not wearer[index]:
|
||||
evidence = _relation_evidence(rows, frame=frame, wearer_id=wearer_id,
|
||||
window_frames=relation_frames)
|
||||
evidence.update(
|
||||
wearer_speech_duration_before_offset_seconds=_run_before(wearer, index, True) / SOURCE_FPS,
|
||||
wearer_silence_duration_after_offset_seconds=_run_after(wearer, index, False) / SOURCE_FPS,
|
||||
active_other_ids_before_offset=list(_active_other_ids(rows, frame - 1, wearer_id=wearer_id)),
|
||||
active_other_ids_after_offset=list(_active_other_ids(rows, frame, wearer_id=wearer_id)),
|
||||
original_timestamp_seconds=frame / SOURCE_FPS,
|
||||
)
|
||||
nominal_end = index + window_frames
|
||||
observable_end = min(nominal_end, len(others))
|
||||
effective_end = min(observable_end, competing if competing is not None else observable_end)
|
||||
outcome = _first_transition(
|
||||
wearer,
|
||||
start=index,
|
||||
stop=effective_end,
|
||||
rising=False,
|
||||
)
|
||||
previous_ids = _active_other_ids(
|
||||
segment_rows, dense_frame - 1, wearer_id=wearer_id
|
||||
)
|
||||
incoming_ids = _active_other_ids(
|
||||
segment_rows, dense_frame, wearer_id=wearer_id
|
||||
)
|
||||
if outcome is not None:
|
||||
label = "positive"
|
||||
subtype = "not_applicable"
|
||||
end_index = outcome
|
||||
end_reason = "wearer_offset"
|
||||
reason = "wearer offset observed after other onset and before competing other offset"
|
||||
elif competing is not None:
|
||||
label = "hard_negative"
|
||||
subtype = (
|
||||
"int_backchannel"
|
||||
if _incoming_is_backchannel(
|
||||
segment_rows,
|
||||
dense_frame=dense_frame,
|
||||
wearer_id=wearer_id,
|
||||
max_seconds=1.0,
|
||||
)
|
||||
else "int_non_takeover_overlap"
|
||||
)
|
||||
end_index = competing
|
||||
end_reason = "competing_other_offset"
|
||||
reason = "wearer retains the floor until the incoming other activity ends"
|
||||
elif nominal_end <= len(others):
|
||||
label = "hard_negative"
|
||||
subtype = "int_retain_floor"
|
||||
end_index = nominal_end
|
||||
end_reason = "full_window"
|
||||
reason = "wearer remains active through the full observable yield window"
|
||||
else:
|
||||
label = "exclude"
|
||||
subtype = "other"
|
||||
end_index = len(others)
|
||||
end_reason = "right_censored"
|
||||
reason = "recording segment ends before a wearer outcome or full window"
|
||||
candidates.append(_candidate(
|
||||
session=session,
|
||||
task="int_yield",
|
||||
automatic_protocol_label=label,
|
||||
automatic_hard_negative_type=subtype,
|
||||
anchor_frame=dense_frame,
|
||||
behavior_frame=None if outcome is None else segment_start + outcome,
|
||||
region_end_frame=segment_start + end_index,
|
||||
region_end_reason=end_reason,
|
||||
previous_ids=previous_ids,
|
||||
incoming_ids=incoming_ids,
|
||||
clips=segment_clips,
|
||||
rows=segment_rows,
|
||||
wearer_id=wearer_id,
|
||||
add(_candidate(
|
||||
session=session, task="int_yield", source="WEARER_OFFSET",
|
||||
rule_type="YIELD_CANDIDATE", anchor_type="wearer_offset", frame=frame,
|
||||
region_start=max(segment_start, frame - relation_frames),
|
||||
region_end=min(segment_end, frame + relation_frames), label="unsure", subtype=None,
|
||||
previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id),
|
||||
incoming_ids=_active_other_ids(rows, frame, wearer_id=wearer_id),
|
||||
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
||||
context_frames=context_frames,
|
||||
reason=reason,
|
||||
))
|
||||
reason="observed wearer 1->0 transition; semantic INT-yield status is unresolved",
|
||||
), ("WEARER_OFFSET", frame, ()))
|
||||
|
||||
return sorted(candidates, key=lambda row: (row.anchor_dense_frame, row.task, row.uid))
|
||||
other_rows = [row for row in rows if row.participant_id != wearer_id]
|
||||
|
||||
# B1: same speaker pauses and resumes while the wearer remains silent.
|
||||
by_speaker: dict[int, list[Utterance]] = defaultdict(list)
|
||||
for row in other_rows:
|
||||
by_speaker[row.participant_id].append(row)
|
||||
for participant_id, speaker_rows in by_speaker.items():
|
||||
for first, resumed in zip(speaker_rows, speaker_rows[1:]):
|
||||
gap = resumed.dense_start_frame - first.dense_end_frame_exclusive
|
||||
left = first.dense_end_frame_exclusive - segment_start
|
||||
right = resumed.dense_start_frame - segment_start
|
||||
if (gap <= 0 or gap > pause_frames
|
||||
or not _all_state(wearer, left, right, False)
|
||||
or not _all_state(others, left, right, False)):
|
||||
continue
|
||||
frame = first.dense_end_frame_exclusive
|
||||
evidence = {
|
||||
"speaker_id": participant_id, "speaker_offset_seconds": first.dense_end_seconds,
|
||||
"speaker_resume_seconds": resumed.dense_start_seconds,
|
||||
"resume_delay_seconds": gap / SOURCE_FPS, "wearer_active_during_pause": False,
|
||||
"original_timestamp_seconds": frame / SOURCE_FPS,
|
||||
}
|
||||
add(_candidate(
|
||||
session=session, task="eot_response", source="SAME_SPEAKER_PAUSE",
|
||||
rule_type="EOT_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame,
|
||||
region_start=frame, region_end=resumed.dense_start_frame,
|
||||
label="hard_negative", subtype="EOT_MIDTURN_PAUSE",
|
||||
previous_ids=(participant_id,), incoming_ids=(participant_id,), evidence=evidence,
|
||||
clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames,
|
||||
reason="same participant resumes after a configurable pause while wearer stays silent",
|
||||
), ("SAME_SPEAKER_PAUSE", frame, (participant_id,)))
|
||||
|
||||
# B2: participant-resolved non-wearer to non-wearer handover.
|
||||
for first in other_rows:
|
||||
options = [row for row in other_rows if row.participant_id != first.participant_id
|
||||
and row.dense_start_frame >= first.dense_end_frame_exclusive
|
||||
and row.dense_start_frame - first.dense_end_frame_exclusive <= handover_frames]
|
||||
if not options:
|
||||
continue
|
||||
incoming = min(options, key=lambda row: row.dense_start_frame)
|
||||
same_resumes = any(row.participant_id == first.participant_id
|
||||
and first.dense_end_frame_exclusive < row.dense_start_frame <= incoming.dense_start_frame
|
||||
for row in other_rows)
|
||||
left = first.dense_end_frame_exclusive - segment_start
|
||||
right = max(left + 1, incoming.dense_start_frame - segment_start + 1)
|
||||
if same_resumes or not _all_state(wearer, left, right, False):
|
||||
continue
|
||||
frame = first.dense_end_frame_exclusive
|
||||
evidence = {
|
||||
"previous_speaker_id": first.participant_id, "incoming_speaker_id": incoming.participant_id,
|
||||
"previous_offset_seconds": first.dense_end_seconds,
|
||||
"incoming_onset_seconds": incoming.dense_start_seconds,
|
||||
"handover_gap_seconds": (incoming.dense_start_frame - frame) / SOURCE_FPS,
|
||||
"wearer_active_during_handover": False, "original_timestamp_seconds": frame / SOURCE_FPS,
|
||||
}
|
||||
add(_candidate(
|
||||
session=session, task="eot_response", source="OTHER_TO_OTHER",
|
||||
rule_type="EOT_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame,
|
||||
region_start=frame, region_end=incoming.dense_start_frame,
|
||||
label="hard_negative", subtype="EOT_OTHER_TO_OTHER",
|
||||
previous_ids=(first.participant_id,), incoming_ids=(incoming.participant_id,),
|
||||
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
||||
context_frames=context_frames,
|
||||
reason="participant-level IDs show a non-wearer to non-wearer handover",
|
||||
), ("OTHER_TO_OTHER", frame, (first.participant_id, incoming.participant_id)))
|
||||
|
||||
# B3/B4: other activity ends while the wearer continuously holds and continues the floor.
|
||||
for other in other_rows:
|
||||
start = other.dense_start_frame - segment_start
|
||||
end = other.dense_end_frame_exclusive - segment_start
|
||||
if (start < 0 or end + continuation_frames > len(wearer)
|
||||
or not _all_state(wearer, start, end + continuation_frames, True)):
|
||||
continue
|
||||
feedback = (other.dense_end_frame_exclusive - other.dense_start_frame <= short_frames
|
||||
and (is_lexical_feedback(other.transcription) or is_pure_nonverbal(other.transcription)))
|
||||
source = "OTHER_BACKCHANNEL" if feedback else "NON_TAKEOVER_OVERLAP"
|
||||
subtype = "INT_OTHER_BACKCHANNEL" if feedback else "INT_NON_TAKEOVER_OVERLAP"
|
||||
evidence = {
|
||||
"other_speaker_id": other.participant_id, "other_onset_seconds": other.dense_start_seconds,
|
||||
"other_offset_seconds": other.dense_end_seconds, "other_duration_seconds": other.duration_seconds,
|
||||
"wearer_active_through_region": True,
|
||||
"wearer_continuation_after_other_seconds": continuation_frames / SOURCE_FPS,
|
||||
"lexical_or_nonverbal_feedback_hint": feedback,
|
||||
"original_timestamp_seconds": other.dense_start_seconds,
|
||||
}
|
||||
add(_candidate(
|
||||
session=session, task="int_yield", source=source,
|
||||
rule_type="INT_HARD_NEGATIVE_REGION", anchor_type="other_onset",
|
||||
frame=other.dense_start_frame, region_start=other.dense_start_frame,
|
||||
region_end=other.dense_end_frame_exclusive, label="hard_negative", subtype=subtype,
|
||||
previous_ids=_active_other_ids(rows, other.dense_start_frame - 1, wearer_id=wearer_id),
|
||||
incoming_ids=(other.participant_id,), evidence=evidence, clips=segment_clips,
|
||||
rows=rows, wearer_id=wearer_id, context_frames=context_frames,
|
||||
reason=("short feedback-like other activity ends while wearer holds the floor"
|
||||
if feedback else "other overlaps but wearer retains and continues the floor"),
|
||||
), (source, other.dense_start_frame, (other.participant_id,)))
|
||||
|
||||
return sorted(candidates, key=lambda row: (row.anchor_dense_frame, row.candidate_source, row.uid))
|
||||
|
||||
|
||||
def stratified_limit(
|
||||
rows: Sequence[ProtocolCandidate],
|
||||
limit: int | None,
|
||||
*,
|
||||
seed: int,
|
||||
) -> list[ProtocolCandidate]:
|
||||
def stratified_limit(rows: Sequence[ProtocolCandidate], limit: int | None, *, seed: int) -> list[ProtocolCandidate]:
|
||||
if limit is None or limit >= len(rows):
|
||||
return list(rows)
|
||||
if limit < 0:
|
||||
raise ValueError("--max-candidates must be non-negative")
|
||||
rng = random.Random(seed)
|
||||
buckets: dict[tuple[str, str, str], list[ProtocolCandidate]] = defaultdict(list)
|
||||
buckets: dict[tuple[str, str], list[ProtocolCandidate]] = defaultdict(list)
|
||||
for row in rows:
|
||||
buckets[(row.task, row.automatic_protocol_label, row.automatic_hard_negative_type)].append(row)
|
||||
keys = sorted(buckets)
|
||||
for key in keys:
|
||||
rng.shuffle(buckets[key])
|
||||
rng.shuffle(keys)
|
||||
buckets[(row.session, row.candidate_source)].append(row)
|
||||
for bucket in buckets.values():
|
||||
rng.shuffle(bucket)
|
||||
selected: list[ProtocolCandidate] = []
|
||||
while len(selected) < limit:
|
||||
progressed = False
|
||||
for key in keys:
|
||||
if buckets[key]:
|
||||
while len(selected) < limit and any(buckets.values()):
|
||||
for key in sorted(buckets):
|
||||
if buckets[key] and len(selected) < limit:
|
||||
selected.append(buckets[key].pop())
|
||||
progressed = True
|
||||
if len(selected) == limit:
|
||||
break
|
||||
if not progressed:
|
||||
break
|
||||
return sorted(selected, key=lambda row: (row.session, row.anchor_dense_frame, row.task))
|
||||
return sorted(selected, key=lambda row: (row.session, row.anchor_dense_frame, row.candidate_source))
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, rows: Iterable[ProtocolCandidate]) -> None:
|
||||
@@ -473,20 +431,13 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--sessions", nargs="+", default=list(ALL_SESSIONS))
|
||||
parser.add_argument("--camera-wearer-id", type=int, default=CAMERA_WEARER_ID)
|
||||
parser.add_argument("--window-seconds", type=float, default=DEFAULT_WINDOW_SECONDS)
|
||||
parser.add_argument("--context-seconds", type=float, default=3.0)
|
||||
parser.add_argument(
|
||||
"--tasks",
|
||||
nargs="+",
|
||||
choices=("eot_response", "int_yield"),
|
||||
default=("eot_response", "int_yield"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--automatic-labels",
|
||||
nargs="+",
|
||||
choices=("positive", "hard_negative", "exclude"),
|
||||
default=("positive", "hard_negative", "exclude"),
|
||||
)
|
||||
parser.add_argument("--relation-window-seconds", type=float, default=3.0)
|
||||
parser.add_argument("--same-speaker-pause-max-seconds", type=float, default=2.0)
|
||||
parser.add_argument("--other-handover-max-seconds", type=float, default=1.5)
|
||||
parser.add_argument("--short-vocalization-max-seconds", type=float, default=1.0)
|
||||
parser.add_argument("--continuation-min-seconds", type=float, default=0.4)
|
||||
parser.add_argument("--candidate-sources", nargs="+")
|
||||
parser.add_argument("--max-candidates", type=int)
|
||||
parser.add_argument("--selection-seed", type=int, default=2027)
|
||||
return parser.parse_args()
|
||||
@@ -497,74 +448,59 @@ def main() -> None:
|
||||
unknown = sorted(set(args.sessions) - set(ALL_SESSIONS))
|
||||
if unknown:
|
||||
raise ValueError(f"Unknown EasyCom sessions: {unknown}")
|
||||
dataset_root = args.dataset_root.expanduser().resolve()
|
||||
output_dir = args.output_dir.expanduser().resolve()
|
||||
config = CandidateConfig(
|
||||
context_seconds=args.context_seconds,
|
||||
relation_window_seconds=args.relation_window_seconds,
|
||||
same_speaker_pause_max_seconds=args.same_speaker_pause_max_seconds,
|
||||
other_handover_max_seconds=args.other_handover_max_seconds,
|
||||
short_vocalization_max_seconds=args.short_vocalization_max_seconds,
|
||||
continuation_min_seconds=args.continuation_min_seconds,
|
||||
)
|
||||
dataset_root, output_dir = args.dataset_root.expanduser().resolve(), args.output_dir.expanduser().resolve()
|
||||
all_rows: list[ProtocolCandidate] = []
|
||||
session_timelines: dict[str, list[dict[str, Any]]] = {}
|
||||
rules = Rules()
|
||||
for session in args.sessions:
|
||||
clips = build_clips(dataset_root, session, rules)
|
||||
utterances = load_utterances(session, clips)
|
||||
rows = derive_protocol_candidates(
|
||||
utterances,
|
||||
clips,
|
||||
session=session,
|
||||
wearer_id=args.camera_wearer_id,
|
||||
window_seconds=args.window_seconds,
|
||||
context_seconds=args.context_seconds,
|
||||
)
|
||||
session_timelines[session] = [{
|
||||
"uid": row.uid,
|
||||
"participant_id": row.participant_id,
|
||||
"participant_role": "wearer" if row.participant_id == args.camera_wearer_id else "other",
|
||||
"start_seconds": row.dense_start_seconds,
|
||||
"end_seconds": row.dense_end_seconds,
|
||||
} for row in utterances]
|
||||
rows = derive_protocol_candidates(utterances, clips, session=session,
|
||||
wearer_id=args.camera_wearer_id, config=config)
|
||||
all_rows.extend(rows)
|
||||
counts = Counter((row.task, row.automatic_protocol_label) for row in rows)
|
||||
print(f"{session}: candidates={len(rows)} counts={dict(sorted(counts.items()))}")
|
||||
|
||||
eligible = [
|
||||
row
|
||||
for row in all_rows
|
||||
if row.task in set(args.tasks)
|
||||
and row.automatic_protocol_label in set(args.automatic_labels)
|
||||
]
|
||||
print(f"{session}: candidates={len(rows)} sources={dict(sorted(Counter(row.candidate_source for row in rows).items()))}")
|
||||
sources = None if args.candidate_sources is None else set(args.candidate_sources)
|
||||
eligible = [row for row in all_rows if sources is None or row.candidate_source in sources]
|
||||
selected = stratified_limit(eligible, args.max_candidates, seed=args.selection_seed)
|
||||
output_path = output_dir / "easycom_eot_int_candidates.jsonl"
|
||||
_write_jsonl(output_path, selected)
|
||||
counts = Counter(
|
||||
(row.split, row.task, row.automatic_protocol_label, row.automatic_hard_negative_type)
|
||||
for row in selected
|
||||
)
|
||||
counts = Counter((row.split, row.candidate_source, row.rule_candidate_type) for row in selected)
|
||||
summary = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"protocol_version": PROTOCOL_VERSION,
|
||||
"dataset": "EasyCom",
|
||||
"semantic_warning": (
|
||||
"Automatic labels are timing candidates for independent human audit; "
|
||||
"they are not semantic EOT/INT ground truth."
|
||||
),
|
||||
"sessions": list(args.sessions),
|
||||
"camera_wearer_id": args.camera_wearer_id,
|
||||
"window_seconds": args.window_seconds,
|
||||
"context_seconds": args.context_seconds,
|
||||
"selection_seed": args.selection_seed,
|
||||
"schema_version": SCHEMA_VERSION, "protocol_version": PROTOCOL_VERSION, "dataset": "EasyCom",
|
||||
"semantic_warning": "Rules create candidates only; AI labels need human review and never modify dense activity ground truth.",
|
||||
"sessions": list(args.sessions), "camera_wearer_id": args.camera_wearer_id,
|
||||
"candidate_config": asdict(config), "selection_seed": args.selection_seed,
|
||||
"max_candidates": args.max_candidates,
|
||||
"tasks": list(args.tasks),
|
||||
"automatic_labels": list(args.automatic_labels),
|
||||
"num_candidates_before_filter": len(all_rows),
|
||||
"num_candidates_before_limit": len(eligible),
|
||||
"candidate_sources": None if sources is None else sorted(sources),
|
||||
"num_candidates_before_filter": len(all_rows), "num_candidates_before_limit": len(eligible),
|
||||
"num_candidates_written": len(selected),
|
||||
"counts": [
|
||||
{
|
||||
"split": key[0],
|
||||
"task": key[1],
|
||||
"automatic_protocol_label": key[2],
|
||||
"automatic_hard_negative_type": key[3],
|
||||
"count": value,
|
||||
}
|
||||
for key, value in sorted(counts.items())
|
||||
],
|
||||
"counts": [{"split": key[0], "candidate_source": key[1],
|
||||
"rule_candidate_type": key[2], "count": value}
|
||||
for key, value in sorted(counts.items())],
|
||||
}
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "summary.json").write_text(
|
||||
json.dumps(summary, indent=2, ensure_ascii=False) + "\n",
|
||||
json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
(output_dir / "easycom_session_timelines.json").write_text(
|
||||
json.dumps({"schema_version": 1, "sessions": session_timelines}, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"Saved {len(selected)} manuscript-aligned review candidates to {output_path}")
|
||||
print(f"Saved {len(selected)} wearer-centric review candidates to {output_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user