#!/usr/bin/env python3 """Generate wearer-centric EasyCom EOT/INT candidates for AI and human review. This stage never changes EasyCom speech boundaries and never claims semantic ground truth. It emits every wearer transition plus independent participant- resolved hard-negative regions. All timing thresholds are CLI parameters, and raw evidence is retained for DeepSeek pre-labeling and human correction. """ from __future__ import annotations import argparse from collections import Counter, defaultdict from dataclasses import asdict, dataclass import json from pathlib import Path import random from typing import Any, Iterable, Sequence from derive_easycom_interaction_events import ( CAMERA_WEARER_ID, SOURCE_FPS, Clip, Rules, Utterance, build_clips, is_lexical_feedback, is_pure_nonverbal, load_utterances, ) from easycom_manifest import ALL_SESSIONS, TEST_SESSIONS SCHEMA_VERSION = 2 PROTOCOL_VERSION = "easycom-wearer-centric-eot-int-candidates-v2" @dataclass(frozen=True) class CandidateConfig: context_seconds: float = 3.0 relation_window_seconds: float = 3.0 same_speaker_pause_max_seconds: float = 2.0 other_handover_max_seconds: float = 1.5 short_vocalization_max_seconds: float = 1.0 continuation_min_seconds: float = 0.4 @dataclass(frozen=True) class ProtocolCandidate: schema_version: int protocol_version: str uid: str event_id: str session: str split: str task: str event_type: str candidate_source: str rule_candidate_type: str automatic_protocol_label: str automatic_hard_negative_type: str | None automatic_confidence: str anchor_type: str anchor_dense_frame: int anchor_dense_seconds: float event_time: float region_start_seconds: float region_end_seconds: float clip_stem: str source_start_frame: int previous_other_participant_ids: tuple[int, ...] incoming_other_participant_ids: tuple[int, ...] temporal_evidence: dict[str, Any] speaker_timeline: tuple[dict[str, Any], ...] context: tuple[dict[str, Any], ...] automatic_reason: str ai_verdict: None = None ai_event_type: None = None ai_subtype: None = None ai_confidence: None = None ai_reason: None = None human_status: str = "UNREVIEWED" human_verdict: None = None human_event_type: None = None human_subtype: None = None exclude_reason: None = None def _fill_states(rows: Sequence[Utterance], *, segment_start: int, segment_end: int, wearer_id: int) -> tuple[bytearray, bytearray]: length = segment_end - segment_start wearer, others = bytearray(length), bytearray(length) for row in rows: left = max(segment_start, row.dense_start_frame) - segment_start right = min(segment_end, row.dense_end_frame_exclusive) - segment_start if right <= left: continue target = wearer if row.participant_id == wearer_id else others target[left:right] = b"\x01" * (right - left) return wearer, others def _run_after(state: bytearray, index: int, value: bool) -> int: end = index while end < len(state) and bool(state[end]) == value: end += 1 return end - index def _run_before(state: bytearray, index: int, value: bool) -> int: start = index - 1 while start >= 0 and bool(state[start]) == value: start -= 1 return index - start - 1 def _all_state(state: bytearray, start: int, stop: int, value: bool) -> bool: left, right = max(0, start), min(len(state), stop) return right > left and all(bool(state[i]) == value for i in range(left, right)) def _active_other_ids(rows: Sequence[Utterance], frame: int, *, wearer_id: int) -> tuple[int, ...]: return tuple(sorted({ row.participant_id for row in rows if row.participant_id != wearer_id and row.dense_start_frame <= frame < row.dense_end_frame_exclusive })) def _clip_for_frame(clips: Sequence[Clip], frame: int) -> Clip | None: return next((clip for clip in clips if clip.dense_start_frame <= frame < clip.dense_start_frame + clip.num_frames), None) def _context_rows(rows: Sequence[Utterance], *, frame: int, context_frames: int, wearer_id: int) -> tuple[dict[str, Any], ...]: left, right = frame - context_frames, frame + context_frames return tuple({ "uid": row.uid, "participant_id": row.participant_id, "participant_role": "wearer" if row.participant_id == wearer_id else "other", "start_seconds": row.dense_start_seconds, "end_seconds": row.dense_end_seconds, "duration_seconds": row.duration_seconds, "transcription": row.transcription, "target_of_speech": list(row.target_of_speech), } for row in rows if row.dense_end_frame_exclusive > left and row.dense_start_frame < right) def _relation_evidence(rows: Sequence[Utterance], *, frame: int, wearer_id: int, window_frames: int) -> dict[str, Any]: previous = [row for row in rows if row.participant_id != wearer_id and row.dense_end_frame_exclusive <= frame and frame - row.dense_end_frame_exclusive <= window_frames] following = [row for row in rows if row.participant_id != wearer_id and row.dense_start_frame >= frame and row.dense_start_frame - frame <= window_frames] prev = max(previous, key=lambda row: row.dense_end_frame_exclusive, default=None) nxt = min(following, key=lambda row: row.dense_start_frame, default=None) active_rows = [row for row in rows if row.participant_id != wearer_id and row.dense_start_frame < frame < row.dense_end_frame_exclusive] overlapping = sorted({row.participant_id for row in active_rows}) resumes = [] if prev is None else [row for row in following if row.participant_id == prev.participant_id] resumed = min(resumes, key=lambda row: row.dense_start_frame, default=None) nearby_onsets = [row for row in rows if row.participant_id != wearer_id and abs(row.dense_start_frame - frame) <= window_frames] closest_onset = min(nearby_onsets, key=lambda row: abs(row.dense_start_frame - frame), default=None) return { "previous_speaker_id": None if prev is None else prev.participant_id, "previous_speaker_offset_seconds": None if prev is None else prev.dense_end_seconds, "gap_from_previous_speaker_offset_seconds": ( None if prev is None else (frame - prev.dense_end_frame_exclusive) / SOURCE_FPS), "overlapping_other_ids": overlapping, "other_overlap_at_event_seconds": max( [0.0] + [(row.dense_end_frame_exclusive - frame) / SOURCE_FPS for row in active_rows] ), "next_speaker_id": None if nxt is None else nxt.participant_id, "next_other_onset_seconds": None if nxt is None else nxt.dense_start_seconds, "previous_speaker_resumes": resumed is not None, "previous_speaker_resume_delay_seconds": ( None if resumed is None or prev is None else (resumed.dense_start_frame - prev.dense_end_frame_exclusive) / SOURCE_FPS ), "closest_other_onset_relative_seconds": ( None if closest_onset is None else (closest_onset.dense_start_frame - frame) / SOURCE_FPS ), } def _candidate(*, session: str, task: str, source: str, rule_type: str, anchor_type: str, frame: int, region_start: int, region_end: int, label: str, subtype: str | None, previous_ids: tuple[int, ...], incoming_ids: tuple[int, ...], evidence: dict[str, Any], clips: Sequence[Clip], rows: Sequence[Utterance], wearer_id: int, context_frames: int, reason: str) -> ProtocolCandidate | None: clip = _clip_for_frame(clips, frame) if clip is None: return None participant_key = "prev-" + ("-".join(f"p{x}" for x in previous_ids) or "none") participant_key += "__in-" + ("-".join(f"p{x}" for x in incoming_ids) or "none") uid = f"{session}/{source.lower()}/{frame:08d}/{participant_key}" context = _context_rows(rows, frame=frame, context_frames=context_frames, wearer_id=wearer_id) return ProtocolCandidate( schema_version=SCHEMA_VERSION, protocol_version=PROTOCOL_VERSION, uid=uid, event_id=uid, session=session, split="test" if session in TEST_SESSIONS else "train", task=task, event_type=source.lower(), candidate_source=source, rule_candidate_type=rule_type, automatic_protocol_label=label, automatic_hard_negative_type=subtype, automatic_confidence="rule_candidate", anchor_type=anchor_type, anchor_dense_frame=frame, anchor_dense_seconds=frame / SOURCE_FPS, event_time=frame / SOURCE_FPS, region_start_seconds=region_start / SOURCE_FPS, region_end_seconds=region_end / SOURCE_FPS, clip_stem=clip.stem, source_start_frame=frame - clip.dense_start_frame + 1, previous_other_participant_ids=previous_ids, incoming_other_participant_ids=incoming_ids, temporal_evidence=evidence, speaker_timeline=context, context=context, automatic_reason=reason, ) def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[Clip], *, session: str, wearer_id: int = CAMERA_WEARER_ID, config: CandidateConfig | None = None, window_seconds: float | None = None, context_seconds: float | None = None) -> list[ProtocolCandidate]: """Return every wearer transition and independent hard-negative regions.""" cfg = config or CandidateConfig() values = asdict(cfg) if window_seconds is not None: values["relation_window_seconds"] = window_seconds if context_seconds is not None: values["context_seconds"] = context_seconds cfg = CandidateConfig(**values) if any(value < 0 for value in asdict(cfg).values()) or cfg.relation_window_seconds == 0: raise ValueError("candidate timing parameters must be non-negative and relation window positive") context_frames = round(cfg.context_seconds * SOURCE_FPS) relation_frames = round(cfg.relation_window_seconds * SOURCE_FPS) pause_frames = round(cfg.same_speaker_pause_max_seconds * SOURCE_FPS) handover_frames = round(cfg.other_handover_max_seconds * SOURCE_FPS) short_frames = round(cfg.short_vocalization_max_seconds * SOURCE_FPS) continuation_frames = round(cfg.continuation_min_seconds * SOURCE_FPS) rows_by_segment: dict[int, list[Utterance]] = defaultdict(list) clips_by_segment: dict[int, list[Clip]] = defaultdict(list) for row in utterances: rows_by_segment[row.segment_id].append(row) for clip in clips: clips_by_segment[clip.segment_id].append(clip) candidates: list[ProtocolCandidate] = [] seen: set[tuple[str, int, tuple[int, ...]]] = set() def add(item: ProtocolCandidate | None, key: tuple[str, int, tuple[int, ...]]) -> None: if item is not None and key not in seen: seen.add(key) candidates.append(item) for segment_id, segment_clips in sorted(clips_by_segment.items()): segment_clips.sort(key=lambda clip: clip.dense_start_frame) rows = sorted(rows_by_segment.get(segment_id, []), key=lambda row: (row.dense_start_frame, row.dense_end_frame_exclusive, row.uid)) segment_start = segment_clips[0].dense_start_frame segment_end = segment_clips[-1].dense_start_frame + segment_clips[-1].num_frames wearer, others = _fill_states(rows, segment_start=segment_start, segment_end=segment_end, wearer_id=wearer_id) # A: all wearer 0->1 and 1->0 transitions, without semantic guessing. for index in range(1, len(wearer)): frame = segment_start + index if not wearer[index - 1] and wearer[index]: evidence = _relation_evidence(rows, frame=frame, wearer_id=wearer_id, window_frames=relation_frames) evidence.update( wearer_speech_duration_after_onset_seconds=_run_after(wearer, index, True) / SOURCE_FPS, previous_wearer_silence_duration_seconds=_run_before(wearer, index, False) / SOURCE_FPS, original_timestamp_seconds=frame / SOURCE_FPS, ) add(_candidate( session=session, task="eot_response", source="WEARER_ONSET", rule_type="ENTRY_CANDIDATE", anchor_type="wearer_onset", frame=frame, region_start=frame, region_end=min(segment_end, frame + relation_frames), label="unsure", subtype=None, previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id), incoming_ids=_active_other_ids(rows, frame, wearer_id=wearer_id), evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames, reason="observed wearer 0->1 transition; semantic EOT-response status is unresolved", ), ("WEARER_ONSET", frame, ())) elif wearer[index - 1] and not wearer[index]: evidence = _relation_evidence(rows, frame=frame, wearer_id=wearer_id, window_frames=relation_frames) evidence.update( wearer_speech_duration_before_offset_seconds=_run_before(wearer, index, True) / SOURCE_FPS, wearer_silence_duration_after_offset_seconds=_run_after(wearer, index, False) / SOURCE_FPS, active_other_ids_before_offset=list(_active_other_ids(rows, frame - 1, wearer_id=wearer_id)), active_other_ids_after_offset=list(_active_other_ids(rows, frame, wearer_id=wearer_id)), original_timestamp_seconds=frame / SOURCE_FPS, ) add(_candidate( session=session, task="int_yield", source="WEARER_OFFSET", rule_type="YIELD_CANDIDATE", anchor_type="wearer_offset", frame=frame, region_start=max(segment_start, frame - relation_frames), region_end=min(segment_end, frame + relation_frames), label="unsure", subtype=None, previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id), incoming_ids=_active_other_ids(rows, frame, wearer_id=wearer_id), evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames, reason="observed wearer 1->0 transition; semantic INT-yield status is unresolved", ), ("WEARER_OFFSET", frame, ())) other_rows = [row for row in rows if row.participant_id != wearer_id] # B1: same speaker pauses and resumes while the wearer remains silent. by_speaker: dict[int, list[Utterance]] = defaultdict(list) for row in other_rows: by_speaker[row.participant_id].append(row) for participant_id, speaker_rows in by_speaker.items(): for first, resumed in zip(speaker_rows, speaker_rows[1:]): gap = resumed.dense_start_frame - first.dense_end_frame_exclusive left = first.dense_end_frame_exclusive - segment_start right = resumed.dense_start_frame - segment_start if (gap <= 0 or gap > pause_frames or not _all_state(wearer, left, right, False) or not _all_state(others, left, right, False)): continue frame = first.dense_end_frame_exclusive evidence = { "speaker_id": participant_id, "speaker_offset_seconds": first.dense_end_seconds, "speaker_resume_seconds": resumed.dense_start_seconds, "resume_delay_seconds": gap / SOURCE_FPS, "wearer_active_during_pause": False, "original_timestamp_seconds": frame / SOURCE_FPS, } add(_candidate( session=session, task="eot_response", source="SAME_SPEAKER_PAUSE", rule_type="EOT_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame, region_start=frame, region_end=resumed.dense_start_frame, label="hard_negative", subtype="EOT_MIDTURN_PAUSE", previous_ids=(participant_id,), incoming_ids=(participant_id,), evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames, reason="same participant resumes after a configurable pause while wearer stays silent", ), ("SAME_SPEAKER_PAUSE", frame, (participant_id,))) # B2: participant-resolved non-wearer to non-wearer handover. for first in other_rows: options = [row for row in other_rows if row.participant_id != first.participant_id and row.dense_start_frame >= first.dense_end_frame_exclusive and row.dense_start_frame - first.dense_end_frame_exclusive <= handover_frames] if not options: continue incoming = min(options, key=lambda row: row.dense_start_frame) same_resumes = any(row.participant_id == first.participant_id and first.dense_end_frame_exclusive < row.dense_start_frame <= incoming.dense_start_frame for row in other_rows) left = first.dense_end_frame_exclusive - segment_start right = max(left + 1, incoming.dense_start_frame - segment_start + 1) if same_resumes or not _all_state(wearer, left, right, False): continue frame = first.dense_end_frame_exclusive evidence = { "previous_speaker_id": first.participant_id, "incoming_speaker_id": incoming.participant_id, "previous_offset_seconds": first.dense_end_seconds, "incoming_onset_seconds": incoming.dense_start_seconds, "handover_gap_seconds": (incoming.dense_start_frame - frame) / SOURCE_FPS, "wearer_active_during_handover": False, "original_timestamp_seconds": frame / SOURCE_FPS, } add(_candidate( session=session, task="eot_response", source="OTHER_TO_OTHER", rule_type="EOT_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame, region_start=frame, region_end=incoming.dense_start_frame, label="hard_negative", subtype="EOT_OTHER_TO_OTHER", previous_ids=(first.participant_id,), incoming_ids=(incoming.participant_id,), evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames, reason="participant-level IDs show a non-wearer to non-wearer handover", ), ("OTHER_TO_OTHER", frame, (first.participant_id, incoming.participant_id))) # B3/B4: other activity ends while the wearer continuously holds and continues the floor. for other in other_rows: start = other.dense_start_frame - segment_start end = other.dense_end_frame_exclusive - segment_start if (start < 0 or end + continuation_frames > len(wearer) or not _all_state(wearer, start, end + continuation_frames, True)): continue feedback = (other.dense_end_frame_exclusive - other.dense_start_frame <= short_frames and (is_lexical_feedback(other.transcription) or is_pure_nonverbal(other.transcription))) source = "OTHER_BACKCHANNEL" if feedback else "NON_TAKEOVER_OVERLAP" subtype = "INT_OTHER_BACKCHANNEL" if feedback else "INT_NON_TAKEOVER_OVERLAP" evidence = { "other_speaker_id": other.participant_id, "other_onset_seconds": other.dense_start_seconds, "other_offset_seconds": other.dense_end_seconds, "other_duration_seconds": other.duration_seconds, "wearer_active_through_region": True, "wearer_continuation_after_other_seconds": continuation_frames / SOURCE_FPS, "lexical_or_nonverbal_feedback_hint": feedback, "original_timestamp_seconds": other.dense_start_seconds, } add(_candidate( session=session, task="int_yield", source=source, rule_type="INT_HARD_NEGATIVE_REGION", anchor_type="other_onset", frame=other.dense_start_frame, region_start=other.dense_start_frame, region_end=other.dense_end_frame_exclusive, label="hard_negative", subtype=subtype, previous_ids=_active_other_ids(rows, other.dense_start_frame - 1, wearer_id=wearer_id), incoming_ids=(other.participant_id,), evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames, reason=("short feedback-like other activity ends while wearer holds the floor" if feedback else "other overlaps but wearer retains and continues the floor"), ), (source, other.dense_start_frame, (other.participant_id,))) return sorted(candidates, key=lambda row: (row.anchor_dense_frame, row.candidate_source, row.uid)) def stratified_limit(rows: Sequence[ProtocolCandidate], limit: int | None, *, seed: int) -> list[ProtocolCandidate]: if limit is None or limit >= len(rows): return list(rows) if limit < 0: raise ValueError("--max-candidates must be non-negative") rng = random.Random(seed) buckets: dict[tuple[str, str], list[ProtocolCandidate]] = defaultdict(list) for row in rows: buckets[(row.session, row.candidate_source)].append(row) for bucket in buckets.values(): rng.shuffle(bucket) selected: list[ProtocolCandidate] = [] while len(selected) < limit and any(buckets.values()): for key in sorted(buckets): if buckets[key] and len(selected) < limit: selected.append(buckets[key].pop()) return sorted(selected, key=lambda row: (row.session, row.anchor_dense_frame, row.candidate_source)) def _write_jsonl(path: Path, rows: Iterable[ProtocolCandidate]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as handle: for row in rows: handle.write(json.dumps(asdict(row), ensure_ascii=False) + "\n") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--dataset-root", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--sessions", nargs="+", default=list(ALL_SESSIONS)) parser.add_argument("--camera-wearer-id", type=int, default=CAMERA_WEARER_ID) parser.add_argument("--context-seconds", type=float, default=3.0) parser.add_argument("--relation-window-seconds", type=float, default=3.0) parser.add_argument("--same-speaker-pause-max-seconds", type=float, default=2.0) parser.add_argument("--other-handover-max-seconds", type=float, default=1.5) parser.add_argument("--short-vocalization-max-seconds", type=float, default=1.0) parser.add_argument("--continuation-min-seconds", type=float, default=0.4) parser.add_argument("--candidate-sources", nargs="+") parser.add_argument("--max-candidates", type=int) parser.add_argument("--selection-seed", type=int, default=2027) return parser.parse_args() def main() -> None: args = parse_args() unknown = sorted(set(args.sessions) - set(ALL_SESSIONS)) if unknown: raise ValueError(f"Unknown EasyCom sessions: {unknown}") config = CandidateConfig( context_seconds=args.context_seconds, relation_window_seconds=args.relation_window_seconds, same_speaker_pause_max_seconds=args.same_speaker_pause_max_seconds, other_handover_max_seconds=args.other_handover_max_seconds, short_vocalization_max_seconds=args.short_vocalization_max_seconds, continuation_min_seconds=args.continuation_min_seconds, ) dataset_root, output_dir = args.dataset_root.expanduser().resolve(), args.output_dir.expanduser().resolve() all_rows: list[ProtocolCandidate] = [] session_timelines: dict[str, list[dict[str, Any]]] = {} rules = Rules() for session in args.sessions: clips = build_clips(dataset_root, session, rules) utterances = load_utterances(session, clips) session_timelines[session] = [{ "uid": row.uid, "participant_id": row.participant_id, "participant_role": "wearer" if row.participant_id == args.camera_wearer_id else "other", "start_seconds": row.dense_start_seconds, "end_seconds": row.dense_end_seconds, } for row in utterances] rows = derive_protocol_candidates(utterances, clips, session=session, wearer_id=args.camera_wearer_id, config=config) all_rows.extend(rows) print(f"{session}: candidates={len(rows)} sources={dict(sorted(Counter(row.candidate_source for row in rows).items()))}") sources = None if args.candidate_sources is None else set(args.candidate_sources) eligible = [row for row in all_rows if sources is None or row.candidate_source in sources] selected = stratified_limit(eligible, args.max_candidates, seed=args.selection_seed) output_path = output_dir / "easycom_eot_int_candidates.jsonl" _write_jsonl(output_path, selected) counts = Counter((row.split, row.candidate_source, row.rule_candidate_type) for row in selected) summary = { "schema_version": SCHEMA_VERSION, "protocol_version": PROTOCOL_VERSION, "dataset": "EasyCom", "semantic_warning": "Rules create candidates only; AI labels need human review and never modify dense activity ground truth.", "sessions": list(args.sessions), "camera_wearer_id": args.camera_wearer_id, "candidate_config": asdict(config), "selection_seed": args.selection_seed, "max_candidates": args.max_candidates, "candidate_sources": None if sources is None else sorted(sources), "num_candidates_before_filter": len(all_rows), "num_candidates_before_limit": len(eligible), "num_candidates_written": len(selected), "counts": [{"split": key[0], "candidate_source": key[1], "rule_candidate_type": key[2], "count": value} for key, value in sorted(counts.items())], } output_dir.mkdir(parents=True, exist_ok=True) (output_dir / "summary.json").write_text( json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") (output_dir / "easycom_session_timelines.json").write_text( json.dumps({"schema_version": 1, "sessions": session_timelines}, ensure_ascii=False) + "\n", encoding="utf-8", ) print(f"Saved {len(selected)} wearer-centric review candidates to {output_path}") if __name__ == "__main__": main()