#!/usr/bin/env python3 """Prepare manuscript-aligned EasyCom EOT/INT regions for human review. The released EasyCom annotations provide participant-resolved speech intervals, not semantic EOT/INT ground truth. This script therefore creates auditable candidate regions from the original 20 Hz timeline. It keeps the two wearer- relative directions distinct: * EOT-response: an aggregate non-wearer offset while the wearer is silent; * INT-yield: an aggregate non-wearer onset while the wearer is speaking. Observed wearer transitions form timing-positive candidates. Competing other transitions and fully observed no-transition windows form hard-negative candidates. Source-boundary cases are retained as exclusions. Every output row remains an automatic candidate until it is independently human reviewed. """ from __future__ import annotations import argparse from collections import Counter, defaultdict from dataclasses import asdict, dataclass import json from pathlib import Path import random from typing import Any, Iterable, Sequence from derive_easycom_interaction_events import ( CAMERA_WEARER_ID, SOURCE_FPS, Clip, Rules, Utterance, build_clips, is_lexical_feedback, is_pure_nonverbal, load_utterances, ) from easycom_manifest import ALL_SESSIONS, TEST_SESSIONS SCHEMA_VERSION = 1 PROTOCOL_VERSION = "easycom-manuscript-eot-int-v1" DEFAULT_WINDOW_SECONDS = 2.0 @dataclass(frozen=True) class ProtocolCandidate: schema_version: int protocol_version: str uid: str session: str split: str task: str event_type: str automatic_protocol_label: str automatic_hard_negative_type: str automatic_confidence: str anchor_type: str anchor_dense_frame: int anchor_dense_seconds: float behavior_anchor_type: str | None behavior_dense_frame: int | None behavior_seconds: float | None region_start_seconds: float region_end_seconds: float region_end_reason: str previous_other_participant_ids: tuple[int, ...] incoming_other_participant_ids: tuple[int, ...] clip_stem: str source_start_frame: int context: tuple[dict[str, Any], ...] automatic_reason: str def _fill_state( rows: Sequence[Utterance], *, segment_start: int, segment_end: int, wearer_id: int, ) -> tuple[bytearray, bytearray]: length = segment_end - segment_start wearer = bytearray(length) others = bytearray(length) for row in rows: left = max(segment_start, row.dense_start_frame) - segment_start right = min(segment_end, row.dense_end_frame_exclusive) - segment_start if right <= left: continue target = wearer if row.participant_id == wearer_id else others target[left:right] = b"\x01" * (right - left) return wearer, others def _first_transition( state: bytearray, *, start: int, stop: int, rising: bool, ) -> int | None: for index in range(max(1, start), min(stop, len(state))): before = bool(state[index - 1]) after = bool(state[index]) if (not before and after) if rising else (before and not after): return index return None def _active_other_ids( rows: Sequence[Utterance], dense_frame: int, *, wearer_id: int, ) -> tuple[int, ...]: return tuple(sorted({ row.participant_id for row in rows if row.participant_id != wearer_id and row.dense_start_frame <= dense_frame < row.dense_end_frame_exclusive })) def _clip_for_frame(clips: Sequence[Clip], dense_frame: int) -> Clip | None: return next( ( clip for clip in clips if clip.dense_start_frame <= dense_frame < clip.dense_start_frame + clip.num_frames ), None, ) def _context_rows( rows: Sequence[Utterance], *, anchor_frame: int, context_frames: int, wearer_id: int, ) -> tuple[dict[str, Any], ...]: left = anchor_frame - context_frames right = anchor_frame + context_frames selected = [] for row in rows: if row.dense_end_frame_exclusive <= left or row.dense_start_frame >= right: continue selected.append( { "uid": row.uid, "participant_id": row.participant_id, "participant_role": "wearer" if row.participant_id == wearer_id else "other", "start_seconds": row.dense_start_seconds, "end_seconds": row.dense_end_seconds, "transcription": row.transcription, "target_of_speech": list(row.target_of_speech), } ) return tuple(selected) def _incoming_is_backchannel( rows: Sequence[Utterance], *, dense_frame: int, wearer_id: int, max_seconds: float, ) -> bool: incoming = [ row for row in rows if row.participant_id != wearer_id and row.dense_start_frame <= dense_frame < row.dense_end_frame_exclusive and row.dense_start_frame >= dense_frame - 1 ] return bool(incoming) and all(row.duration_seconds <= max_seconds for row in incoming) and any( is_lexical_feedback(row.transcription) or is_pure_nonverbal(row.transcription) for row in incoming ) def _candidate( *, session: str, task: str, automatic_protocol_label: str, automatic_hard_negative_type: str, anchor_frame: int, behavior_frame: int | None, region_end_frame: int, region_end_reason: str, previous_ids: tuple[int, ...], incoming_ids: tuple[int, ...], clips: Sequence[Clip], rows: Sequence[Utterance], wearer_id: int, context_frames: int, reason: str, ) -> ProtocolCandidate: clip = _clip_for_frame(clips, anchor_frame) if clip is None: raise ValueError(f"No source clip for {session} dense frame {anchor_frame}") event_type = "eot_response_candidate" if task == "eot_response" else "int_yield_candidate" anchor_type = "other_offset" if task == "eot_response" else "other_onset" behavior_type = None if behavior_frame is not None: behavior_type = "wearer_onset" if task == "eot_response" else "wearer_offset" return ProtocolCandidate( schema_version=SCHEMA_VERSION, protocol_version=PROTOCOL_VERSION, uid=f"{session}/{task}/{anchor_frame:08d}", session=session, split="test" if session in TEST_SESSIONS else "train", task=task, event_type=event_type, automatic_protocol_label=automatic_protocol_label, automatic_hard_negative_type=automatic_hard_negative_type, automatic_confidence="timing_candidate", anchor_type=anchor_type, anchor_dense_frame=anchor_frame, anchor_dense_seconds=anchor_frame / SOURCE_FPS, behavior_anchor_type=behavior_type, behavior_dense_frame=behavior_frame, behavior_seconds=None if behavior_frame is None else behavior_frame / SOURCE_FPS, region_start_seconds=anchor_frame / SOURCE_FPS, region_end_seconds=region_end_frame / SOURCE_FPS, region_end_reason=region_end_reason, previous_other_participant_ids=previous_ids, incoming_other_participant_ids=incoming_ids, clip_stem=clip.stem, source_start_frame=anchor_frame - clip.dense_start_frame + 1, context=_context_rows( rows, anchor_frame=anchor_frame, context_frames=context_frames, wearer_id=wearer_id, ), automatic_reason=reason, ) def derive_protocol_candidates( utterances: Sequence[Utterance], clips: Sequence[Clip], *, session: str, wearer_id: int = CAMERA_WEARER_ID, window_seconds: float = DEFAULT_WINDOW_SECONDS, context_seconds: float = 3.0, ) -> list[ProtocolCandidate]: """Construct timing candidates with manuscript-aligned directionality.""" if window_seconds <= 0 or context_seconds < 0: raise ValueError("window_seconds must be positive and context_seconds non-negative") window_frames = max(1, round(window_seconds * SOURCE_FPS)) context_frames = max(0, round(context_seconds * SOURCE_FPS)) rows_by_segment: dict[int, list[Utterance]] = defaultdict(list) clips_by_segment: dict[int, list[Clip]] = defaultdict(list) for row in utterances: rows_by_segment[row.segment_id].append(row) for clip in clips: clips_by_segment[clip.segment_id].append(clip) candidates: list[ProtocolCandidate] = [] for segment_id, segment_clips in sorted(clips_by_segment.items()): segment_clips.sort(key=lambda clip: clip.dense_start_frame) segment_rows = sorted( rows_by_segment.get(segment_id, []), key=lambda row: (row.dense_start_frame, row.dense_end_frame_exclusive, row.uid), ) segment_start = segment_clips[0].dense_start_frame segment_end = segment_clips[-1].dense_start_frame + segment_clips[-1].num_frames wearer, others = _fill_state( segment_rows, segment_start=segment_start, segment_end=segment_end, wearer_id=wearer_id, ) for index in range(1, len(others)): dense_frame = segment_start + index if others[index - 1] and not others[index] and not wearer[index - 1]: competing = _first_transition( others, start=index + 1, stop=min(len(others), index + window_frames + 1), rising=True, ) nominal_end = index + window_frames observable_end = min(nominal_end, len(others)) effective_end = min(observable_end, competing if competing is not None else observable_end) outcome = _first_transition( wearer, start=index, stop=effective_end, rising=True, ) previous_ids = _active_other_ids( segment_rows, dense_frame - 1, wearer_id=wearer_id ) incoming_ids = ( () if competing is None else _active_other_ids( segment_rows, segment_start + competing, wearer_id=wearer_id ) ) if outcome is not None: label = "positive" subtype = "not_applicable" end_index = outcome end_reason = "wearer_onset" reason = "wearer onset observed after other offset and before competing other onset" elif competing is not None: label = "hard_negative" subtype = ( "eot_midturn_pause" if set(previous_ids) & set(incoming_ids) else "eot_other_to_other" ) end_index = competing end_reason = "competing_other_onset" reason = "wearer remains silent until another non-wearer onset" elif nominal_end <= len(others): label = "hard_negative" subtype = "eot_no_response" end_index = nominal_end end_reason = "full_window" reason = "wearer remains silent for the full observable response window" else: label = "exclude" subtype = "other" end_index = len(others) end_reason = "right_censored" reason = "recording segment ends before a wearer outcome or full window" candidates.append(_candidate( session=session, task="eot_response", automatic_protocol_label=label, automatic_hard_negative_type=subtype, anchor_frame=dense_frame, behavior_frame=None if outcome is None else segment_start + outcome, region_end_frame=segment_start + end_index, region_end_reason=end_reason, previous_ids=previous_ids, incoming_ids=incoming_ids, clips=segment_clips, rows=segment_rows, wearer_id=wearer_id, context_frames=context_frames, reason=reason, )) if not others[index - 1] and others[index] and wearer[index - 1]: competing = _first_transition( others, start=index + 1, stop=min(len(others), index + window_frames + 1), rising=False, ) nominal_end = index + window_frames observable_end = min(nominal_end, len(others)) effective_end = min(observable_end, competing if competing is not None else observable_end) outcome = _first_transition( wearer, start=index, stop=effective_end, rising=False, ) previous_ids = _active_other_ids( segment_rows, dense_frame - 1, wearer_id=wearer_id ) incoming_ids = _active_other_ids( segment_rows, dense_frame, wearer_id=wearer_id ) if outcome is not None: label = "positive" subtype = "not_applicable" end_index = outcome end_reason = "wearer_offset" reason = "wearer offset observed after other onset and before competing other offset" elif competing is not None: label = "hard_negative" subtype = ( "int_backchannel" if _incoming_is_backchannel( segment_rows, dense_frame=dense_frame, wearer_id=wearer_id, max_seconds=1.0, ) else "int_non_takeover_overlap" ) end_index = competing end_reason = "competing_other_offset" reason = "wearer retains the floor until the incoming other activity ends" elif nominal_end <= len(others): label = "hard_negative" subtype = "int_retain_floor" end_index = nominal_end end_reason = "full_window" reason = "wearer remains active through the full observable yield window" else: label = "exclude" subtype = "other" end_index = len(others) end_reason = "right_censored" reason = "recording segment ends before a wearer outcome or full window" candidates.append(_candidate( session=session, task="int_yield", automatic_protocol_label=label, automatic_hard_negative_type=subtype, anchor_frame=dense_frame, behavior_frame=None if outcome is None else segment_start + outcome, region_end_frame=segment_start + end_index, region_end_reason=end_reason, previous_ids=previous_ids, incoming_ids=incoming_ids, clips=segment_clips, rows=segment_rows, wearer_id=wearer_id, context_frames=context_frames, reason=reason, )) return sorted(candidates, key=lambda row: (row.anchor_dense_frame, row.task, row.uid)) def stratified_limit( rows: Sequence[ProtocolCandidate], limit: int | None, *, seed: int, ) -> list[ProtocolCandidate]: if limit is None or limit >= len(rows): return list(rows) if limit < 0: raise ValueError("--max-candidates must be non-negative") rng = random.Random(seed) buckets: dict[tuple[str, str, str], list[ProtocolCandidate]] = defaultdict(list) for row in rows: buckets[(row.task, row.automatic_protocol_label, row.automatic_hard_negative_type)].append(row) keys = sorted(buckets) for key in keys: rng.shuffle(buckets[key]) rng.shuffle(keys) selected: list[ProtocolCandidate] = [] while len(selected) < limit: progressed = False for key in keys: if buckets[key]: selected.append(buckets[key].pop()) progressed = True if len(selected) == limit: break if not progressed: break return sorted(selected, key=lambda row: (row.session, row.anchor_dense_frame, row.task)) def _write_jsonl(path: Path, rows: Iterable[ProtocolCandidate]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as handle: for row in rows: handle.write(json.dumps(asdict(row), ensure_ascii=False) + "\n") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--dataset-root", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--sessions", nargs="+", default=list(ALL_SESSIONS)) parser.add_argument("--camera-wearer-id", type=int, default=CAMERA_WEARER_ID) parser.add_argument("--window-seconds", type=float, default=DEFAULT_WINDOW_SECONDS) parser.add_argument("--context-seconds", type=float, default=3.0) parser.add_argument( "--tasks", nargs="+", choices=("eot_response", "int_yield"), default=("eot_response", "int_yield"), ) parser.add_argument( "--automatic-labels", nargs="+", choices=("positive", "hard_negative", "exclude"), default=("positive", "hard_negative", "exclude"), ) parser.add_argument("--max-candidates", type=int) parser.add_argument("--selection-seed", type=int, default=2027) return parser.parse_args() def main() -> None: args = parse_args() unknown = sorted(set(args.sessions) - set(ALL_SESSIONS)) if unknown: raise ValueError(f"Unknown EasyCom sessions: {unknown}") dataset_root = args.dataset_root.expanduser().resolve() output_dir = args.output_dir.expanduser().resolve() all_rows: list[ProtocolCandidate] = [] rules = Rules() for session in args.sessions: clips = build_clips(dataset_root, session, rules) utterances = load_utterances(session, clips) rows = derive_protocol_candidates( utterances, clips, session=session, wearer_id=args.camera_wearer_id, window_seconds=args.window_seconds, context_seconds=args.context_seconds, ) all_rows.extend(rows) counts = Counter((row.task, row.automatic_protocol_label) for row in rows) print(f"{session}: candidates={len(rows)} counts={dict(sorted(counts.items()))}") eligible = [ row for row in all_rows if row.task in set(args.tasks) and row.automatic_protocol_label in set(args.automatic_labels) ] selected = stratified_limit(eligible, args.max_candidates, seed=args.selection_seed) output_path = output_dir / "easycom_eot_int_candidates.jsonl" _write_jsonl(output_path, selected) counts = Counter( (row.split, row.task, row.automatic_protocol_label, row.automatic_hard_negative_type) for row in selected ) summary = { "schema_version": SCHEMA_VERSION, "protocol_version": PROTOCOL_VERSION, "dataset": "EasyCom", "semantic_warning": ( "Automatic labels are timing candidates for independent human audit; " "they are not semantic EOT/INT ground truth." ), "sessions": list(args.sessions), "camera_wearer_id": args.camera_wearer_id, "window_seconds": args.window_seconds, "context_seconds": args.context_seconds, "selection_seed": args.selection_seed, "max_candidates": args.max_candidates, "tasks": list(args.tasks), "automatic_labels": list(args.automatic_labels), "num_candidates_before_filter": len(all_rows), "num_candidates_before_limit": len(eligible), "num_candidates_written": len(selected), "counts": [ { "split": key[0], "task": key[1], "automatic_protocol_label": key[2], "automatic_hard_negative_type": key[3], "count": value, } for key, value in sorted(counts.items()) ], } output_dir.mkdir(parents=True, exist_ok=True) (output_dir / "summary.json").write_text( json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", ) print(f"Saved {len(selected)} manuscript-aligned review candidates to {output_path}") if __name__ == "__main__": main()