wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update llm assisted annotation

Browse Source
This commit is contained in:
Xin Wang
2026-09-14 23:04:21 +08:00
parent 068b7ab9e6
commit 06b5e6c385
12 changed files with 1348 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files
prepare_easycom_eot_int_review.py
+352 -42
View File
@@ -2,9 +2,12 @@
"""Generate wearer-centric EasyCom EOT/INT candidates for AI and human review.
This stage never changes EasyCom speech boundaries and never claims semantic
ground truth. It emits every wearer transition plus independent participant-
resolved hard-negative regions. All timing thresholds are CLI parameters, and
raw evidence is retained for DeepSeek pre-labeling and human correction.
ground truth. It groups short wearer resumptions into floor-level episodes,
pairs EOT candidates with a preceding non-wearer release, requires a new
non-wearer onset inside the wearer floor episode for INT candidates, and keeps
unpaired or ineligible raw boundaries as deterministic controls. All timing
thresholds are CLI parameters, and raw evidence is retained for DeepSeek
pre-labeling and human correction.
"""
from __future__ import annotations
@@ -24,8 +27,8 @@ from derive_easycom_interaction_events import (
from easycom_manifest import ALL_SESSIONS, TEST_SESSIONS
SCHEMA_VERSION = 2
PROTOCOL_VERSION = "easycom-wearer-centric-eot-int-candidates-v2"
SCHEMA_VERSION = 3
PROTOCOL_VERSION = "easycom-wearer-centric-eot-int-candidates-v5"
@dataclass(frozen=True)
@@ -36,6 +39,8 @@ class CandidateConfig:
other_handover_max_seconds: float = 1.5
short_vocalization_max_seconds: float = 1.0
continuation_min_seconds: float = 0.4
int_offset_tolerance_seconds: float = 0.05
wearer_resume_merge_max_seconds: float = 0.8
@dataclass(frozen=True)
@@ -79,6 +84,32 @@ class ProtocolCandidate:
exclude_reason: None = None
def requires_semantic_adjudication(row: dict[str, Any]) -> bool:
"""Return whether a stored row belongs in the primary AI/human queue.
Explicit v3--v5 control sources are authoritative. For old v2 candidate
files, reconstruct the same-speaker crossing requirement from the stored
activity IDs. Missing legacy evidence remains reviewable rather than being
silently discarded.
"""
source = row.get("candidate_source")
if source in {"WEARER_ONSET_CONTROL", "WEARER_OFFSET_CONTROL", "WEARER_RESUME"}:
return False
if source != "WEARER_OFFSET":
return True
evidence = row.get("temporal_evidence")
if not isinstance(evidence, dict):
return True
explicit = evidence.get("other_crosses_wearer_offset")
if isinstance(explicit, bool):
return explicit
before = evidence.get("active_other_ids_before_offset")
after = evidence.get("active_other_ids_after_offset")
if isinstance(before, list) and isinstance(after, list):
return bool(set(before) & set(after))
return True
def _fill_states(rows: Sequence[Utterance], *, segment_start: int, segment_end: int,
wearer_id: int) -> tuple[bytearray, bytearray]:
length = segment_end - segment_start
@@ -120,6 +151,68 @@ def _active_other_ids(rows: Sequence[Utterance], frame: int, *, wearer_id: int)
}))
def _other_ids_crossing_offset(
rows: Sequence[Utterance], *, frame: int, wearer_id: int, tolerance_frames: int
) -> tuple[int, ...]:
"""Return other speakers whose annotated interval crosses a wearer offset.
With zero tolerance this encodes the strict prerequisite
``other.start < wearer_offset < other.end``. A one-frame tolerance accepts
an annotation boundary at the same 20 Hz frame, but not an other onset one
or more frames after the wearer offset.
"""
return tuple(sorted({
row.participant_id for row in rows
if row.participant_id != wearer_id
and row.dense_start_frame < frame + tolerance_frames
and row.dense_end_frame_exclusive > frame - tolerance_frames
}))
def _wearer_gap_competition(
rows: Sequence[Utterance], *, gap_start_frame: int, onset_frame: int,
wearer_id: int, short_frames: int, continuation_frames: int,
) -> dict[str, Any]:
"""Summarize plausible competing floor activity inside a short wearer gap.
Feedback-like short lexical/nonverbal activity is retained as evidence but
does not by itself split the wearer floor episode. Any other contribution
that occupies the gap for ``continuation_frames`` or continues across the
resumed wearer onset conservatively preserves a new ENTRY candidate.
"""
gap_rows = [
row for row in rows
if row.participant_id != wearer_id
and row.dense_start_frame < onset_frame
and row.dense_end_frame_exclusive > gap_start_frame
]
feedback_ids: set[int] = set()
competing_ids: set[int] = set()
max_gap_coverage = 0
for row in gap_rows:
coverage = max(
0,
min(row.dense_end_frame_exclusive, onset_frame)
- max(row.dense_start_frame, gap_start_frame),
)
max_gap_coverage = max(max_gap_coverage, coverage)
feedback_like = (
row.dense_end_frame_exclusive - row.dense_start_frame <= short_frames
and (is_lexical_feedback(row.transcription) or is_pure_nonverbal(row.transcription))
)
if feedback_like:
feedback_ids.add(row.participant_id)
elif coverage >= continuation_frames or row.dense_end_frame_exclusive > onset_frame:
competing_ids.add(row.participant_id)
return {
"gap_other_participant_ids": sorted({row.participant_id for row in gap_rows}),
"feedback_like_other_ids_during_gap": sorted(feedback_ids),
"sustained_competing_other_ids_during_gap": sorted(competing_ids),
"max_other_gap_coverage_seconds": max_gap_coverage / SOURCE_FPS,
"sustained_other_takeover_during_gap": bool(competing_ids),
}
def _clip_for_frame(clips: Sequence[Clip], frame: int) -> Clip | None:
return next((clip for clip in clips
if clip.dense_start_frame <= frame < clip.dense_start_frame + clip.num_frames), None)
@@ -216,7 +309,7 @@ def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[
config: CandidateConfig | None = None,
window_seconds: float | None = None,
context_seconds: float | None = None) -> list[ProtocolCandidate]:
"""Return every wearer transition and independent hard-negative regions."""
"""Return primary semantic candidates plus separate offset controls."""
cfg = config or CandidateConfig()
values = asdict(cfg)
if window_seconds is not None:
@@ -232,6 +325,8 @@ def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[
handover_frames = round(cfg.other_handover_max_seconds * SOURCE_FPS)
short_frames = round(cfg.short_vocalization_max_seconds * SOURCE_FPS)
continuation_frames = round(cfg.continuation_min_seconds * SOURCE_FPS)
int_offset_tolerance_frames = round(cfg.int_offset_tolerance_seconds * SOURCE_FPS)
wearer_resume_merge_frames = round(cfg.wearer_resume_merge_max_seconds * SOURCE_FPS)
rows_by_segment: dict[int, list[Utterance]] = defaultdict(list)
clips_by_segment: dict[int, list[Clip]] = defaultdict(list)
for row in utterances:
@@ -254,52 +349,256 @@ def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[
segment_end = segment_clips[-1].dense_start_frame + segment_clips[-1].num_frames
wearer, others = _fill_states(rows, segment_start=segment_start,
segment_end=segment_end, wearer_id=wearer_id)
other_rows = [row for row in rows if row.participant_id != wearer_id]
# A: all wearer 0->1 and 1->0 transitions, without semantic guessing.
# A: classify raw wearer onsets before proposing any behavioral event.
# Short uncompeted gaps are internal to a floor episode. All other raw
# onsets begin a new floor episode, but become EOT candidates only when
# they can be paired with a preceding non-wearer release opportunity.
onset_records: dict[int, dict[str, Any]] = {}
raw_offsets: list[int] = []
for index in range(1, len(wearer)):
frame = segment_start + index
if not wearer[index - 1] and wearer[index]:
gap_frames = _run_before(wearer, index, False)
previous_wearer_index = index - gap_frames - 1
has_previous_wearer = previous_wearer_index >= 0 and bool(wearer[previous_wearer_index])
previous_wearer_offset = segment_start + index - gap_frames
competition = _wearer_gap_competition(
rows,
gap_start_frame=previous_wearer_offset,
onset_frame=frame,
wearer_id=wearer_id,
short_frames=short_frames,
continuation_frames=continuation_frames,
) if has_previous_wearer else {
"gap_other_participant_ids": [],
"feedback_like_other_ids_during_gap": [],
"sustained_competing_other_ids_during_gap": [],
"max_other_gap_coverage_seconds": 0.0,
"sustained_other_takeover_during_gap": False,
}
within_turn_resume = (
has_previous_wearer
and gap_frames <= wearer_resume_merge_frames
and not competition["sustained_other_takeover_during_gap"]
)
evidence = _relation_evidence(rows, frame=frame, wearer_id=wearer_id,
window_frames=relation_frames)
evidence.update(
wearer_speech_duration_after_onset_seconds=_run_after(wearer, index, True) / SOURCE_FPS,
previous_wearer_silence_duration_seconds=_run_before(wearer, index, False) / SOURCE_FPS,
previous_wearer_silence_duration_seconds=gap_frames / SOURCE_FPS,
previous_wearer_offset_seconds=(
previous_wearer_offset / SOURCE_FPS if has_previous_wearer else None
),
wearer_resume_merge_max_seconds=cfg.wearer_resume_merge_max_seconds,
floor_episode_merged=within_turn_resume,
original_timestamp_seconds=frame / SOURCE_FPS,
**competition,
)
add(_candidate(
session=session, task="eot_response", source="WEARER_ONSET",
rule_type="ENTRY_CANDIDATE", anchor_type="wearer_onset", frame=frame,
region_start=frame, region_end=min(segment_end, frame + relation_frames),
label="unsure", subtype=None,
previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id),
incoming_ids=_active_other_ids(rows, frame, wearer_id=wearer_id),
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
context_frames=context_frames,
reason="observed wearer 0->1 transition; semantic EOT-response status is unresolved",
), ("WEARER_ONSET", frame, ()))
onset_records[frame] = {
"within_turn_resume": within_turn_resume,
"evidence": evidence,
}
elif wearer[index - 1] and not wearer[index]:
evidence = _relation_evidence(rows, frame=frame, wearer_id=wearer_id,
window_frames=relation_frames)
evidence.update(
wearer_speech_duration_before_offset_seconds=_run_before(wearer, index, True) / SOURCE_FPS,
wearer_silence_duration_after_offset_seconds=_run_after(wearer, index, False) / SOURCE_FPS,
active_other_ids_before_offset=list(_active_other_ids(rows, frame - 1, wearer_id=wearer_id)),
active_other_ids_after_offset=list(_active_other_ids(rows, frame, wearer_id=wearer_id)),
original_timestamp_seconds=frame / SOURCE_FPS,
)
add(_candidate(
session=session, task="int_yield", source="WEARER_OFFSET",
rule_type="YIELD_CANDIDATE", anchor_type="wearer_offset", frame=frame,
region_start=max(segment_start, frame - relation_frames),
region_end=min(segment_end, frame + relation_frames), label="unsure", subtype=None,
previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id),
incoming_ids=_active_other_ids(rows, frame, wearer_id=wearer_id),
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
context_frames=context_frames,
reason="observed wearer 1->0 transition; semantic INT-yield status is unresolved",
), ("WEARER_OFFSET", frame, ()))
raw_offsets.append(frame)
other_rows = [row for row in rows if row.participant_id != wearer_id]
resume_onsets = {
frame for frame, record in onset_records.items() if record["within_turn_resume"]
}
floor_onsets = sorted(set(onset_records) - resume_onsets)
for frame in sorted(resume_onsets):
record = onset_records[frame]
add(_candidate(
session=session, task="eot_response_control", source="WEARER_RESUME",
rule_type="WITHIN_TURN_RESUME_CONTROL", anchor_type="wearer_onset", frame=frame,
region_start=frame, region_end=min(segment_end, frame + relation_frames),
label="not_applicable", subtype=None,
previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id),
incoming_ids=_active_other_ids(rows, frame, wearer_id=wearer_id),
evidence=record["evidence"], clips=segment_clips, rows=rows,
wearer_id=wearer_id, context_frames=context_frames,
reason="short raw wearer pause without sustained competing other takeover; merged into the existing floor episode",
), ("WEARER_RESUME", frame, ()))
# A1: open an opportunity at a non-wearer release and pair it with the
# subsequent floor-level wearer entry only if no competing other onset
# or wearer activity intervenes. Long silence alone does not invalidate
# the opportunity. Unpaired floor onsets are retained as controls.
paired_floor_onsets: set[int] = set()
for release in other_rows:
opportunity_frame = release.dense_end_frame_exclusive
onset_frame = next(
(value for value in floor_onsets if value >= opportunity_frame), None
)
if onset_frame is None:
continue
wearer_silent = (
opportunity_frame == onset_frame
or _all_state(
wearer,
opportunity_frame - segment_start,
onset_frame - segment_start,
False,
)
)
competing_onsets = [
row for row in other_rows
if opportunity_frame <= row.dense_start_frame < onset_frame
]
other_already_holds = bool(
_active_other_ids(rows, opportunity_frame, wearer_id=wearer_id)
)
if not wearer_silent or competing_onsets or other_already_holds:
continue
evidence = dict(onset_records[onset_frame]["evidence"])
evidence.update(
opportunity_anchor_type="other_offset",
opportunity_anchor_frame=opportunity_frame,
opportunity_anchor_seconds=opportunity_frame / SOURCE_FPS,
opportunity_speaker_id=release.participant_id,
opportunity_source_uid=release.uid,
opportunity_source_transcription=release.transcription,
opportunity_source_target_of_speech=list(release.target_of_speech),
behavior_anchor_type="wearer_floor_onset",
behavior_anchor_frame=onset_frame,
behavior_anchor_seconds=onset_frame / SOURCE_FPS,
response_delay_seconds=(onset_frame - opportunity_frame) / SOURCE_FPS,
competing_other_onsets_before_behavior=[],
eot_opportunity_paired=True,
)
add(_candidate(
session=session, task="eot_response", source="WEARER_ONSET",
rule_type="EOT_OPPORTUNITY_CANDIDATE", anchor_type="wearer_onset",
frame=onset_frame, region_start=opportunity_frame,
region_end=min(segment_end, onset_frame + relation_frames),
label="unsure", subtype=None, previous_ids=(release.participant_id,),
incoming_ids=_active_other_ids(rows, onset_frame, wearer_id=wearer_id),
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
context_frames=context_frames,
reason="non-wearer release opportunity paired with a subsequent floor-level wearer entry",
), ("WEARER_ONSET", onset_frame, (release.participant_id,)))
paired_floor_onsets.add(onset_frame)
for frame in floor_onsets:
if frame in paired_floor_onsets:
continue
evidence = dict(onset_records[frame]["evidence"])
evidence.update(
opportunity_anchor_type=None,
opportunity_anchor_frame=None,
opportunity_anchor_seconds=None,
behavior_anchor_type="wearer_floor_onset",
behavior_anchor_frame=frame,
behavior_anchor_seconds=frame / SOURCE_FPS,
eot_opportunity_paired=False,
)
add(_candidate(
session=session, task="eot_response_control", source="WEARER_ONSET_CONTROL",
rule_type="UNPAIRED_WEARER_ENTRY_CONTROL", anchor_type="wearer_onset", frame=frame,
region_start=frame, region_end=min(segment_end, frame + relation_frames),
label="not_applicable", subtype=None,
previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id),
incoming_ids=_active_other_ids(rows, frame, wearer_id=wearer_id),
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
context_frames=context_frames,
reason="floor-level wearer entry has no surviving preceding non-wearer release opportunity",
), ("WEARER_ONSET_CONTROL", frame, ()))
# A2: a raw wearer offset is an INT candidate only at the end of a
# floor-level wearer episode and only if a new non-wearer onset obeys
# t_w < t_o < t_y and that same interval crosses the yield boundary.
# All other offsets remain auditable controls.
ordered_raw_onsets = sorted(onset_records)
current_floor_start = segment_start if wearer and wearer[0] else None
floor_start_by_offset: dict[int, tuple[int | None, bool]] = {}
for index in range(1, len(wearer)):
frame = segment_start + index
if not wearer[index - 1] and wearer[index] and frame not in resume_onsets:
current_floor_start = frame
elif wearer[index - 1] and not wearer[index]:
next_onset = next((value for value in ordered_raw_onsets if value > frame), None)
internal_pause = next_onset in resume_onsets
floor_start_by_offset[frame] = (current_floor_start, internal_pause)
if not internal_pause:
current_floor_start = None
for frame in raw_offsets:
index = frame - segment_start
floor_start, internal_pause = floor_start_by_offset[frame]
strict_crossing_ids = _other_ids_crossing_offset(
other_rows, frame=frame, wearer_id=wearer_id, tolerance_frames=0
)
tolerance_crossing_ids = _other_ids_crossing_offset(
other_rows, frame=frame, wearer_id=wearer_id,
tolerance_frames=int_offset_tolerance_frames,
)
eligible_rows = [
row for row in other_rows
if floor_start is not None
and floor_start < row.dense_start_frame < frame
and row.dense_end_frame_exclusive > frame - int_offset_tolerance_frames
]
eligible_ids = tuple(sorted({row.participant_id for row in eligible_rows}))
preexisting_ids = tuple(sorted({
row.participant_id for row in other_rows
if floor_start is not None
and row.dense_start_frame <= floor_start
and row.dense_end_frame_exclusive > frame - int_offset_tolerance_frames
}))
is_int_candidate = bool(eligible_ids) and not internal_pause
evidence = _relation_evidence(
rows, frame=frame, wearer_id=wearer_id, window_frames=relation_frames
)
evidence.update(
wearer_floor_episode_start_frame=floor_start,
wearer_floor_episode_start_seconds=(
None if floor_start is None else floor_start / SOURCE_FPS
),
wearer_speech_duration_before_offset_seconds=_run_before(wearer, index, True) / SOURCE_FPS,
wearer_silence_duration_after_offset_seconds=_run_after(wearer, index, False) / SOURCE_FPS,
raw_offset_is_within_merged_floor_episode=internal_pause,
active_other_ids_before_offset=list(_active_other_ids(rows, frame - 1, wearer_id=wearer_id)),
active_other_ids_after_offset=list(_active_other_ids(rows, frame, wearer_id=wearer_id)),
strict_crossing_other_ids=list(strict_crossing_ids),
tolerance_crossing_other_ids=list(tolerance_crossing_ids),
eligible_new_other_onset_ids=list(eligible_ids),
eligible_new_other_onsets=[{
"participant_id": row.participant_id,
"onset_seconds": row.dense_start_seconds,
"offset_seconds": row.dense_end_seconds,
"uid": row.uid,
} for row in eligible_rows],
preexisting_crossing_other_ids=list(preexisting_ids),
other_crosses_wearer_offset=bool(tolerance_crossing_ids),
int_ordering_satisfied=bool(eligible_ids),
int_offset_tolerance_seconds=cfg.int_offset_tolerance_seconds,
original_timestamp_seconds=frame / SOURCE_FPS,
)
control_reason = (
"raw wearer offset is internal to a merged floor episode"
if internal_pause
else "no new non-wearer onset satisfies t_w < t_o < t_y and crosses the wearer offset"
)
add(_candidate(
session=session,
task="int_yield" if is_int_candidate else "int_yield_control",
source="WEARER_OFFSET" if is_int_candidate else "WEARER_OFFSET_CONTROL",
rule_type="YIELD_CANDIDATE" if is_int_candidate else "WEARER_OFFSET_CONTROL",
anchor_type="wearer_offset", frame=frame,
region_start=max(segment_start, frame - relation_frames),
region_end=min(segment_end, frame + relation_frames),
label="unsure" if is_int_candidate else "not_applicable", subtype=None,
previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id),
incoming_ids=eligible_ids,
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
context_frames=context_frames,
reason=(
"new non-wearer onset occurs inside the wearer floor episode and crosses the yield boundary; semantic takeover status is unresolved"
if is_int_candidate else control_reason
),
), ("WEARER_OFFSET" if is_int_candidate else "WEARER_OFFSET_CONTROL", frame, eligible_ids))
# B1: same speaker pauses and resumes while the wearer remains silent.
by_speaker: dict[int, list[Utterance]] = defaultdict(list)
@@ -437,6 +736,8 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--other-handover-max-seconds", type=float, default=1.5)
parser.add_argument("--short-vocalization-max-seconds", type=float, default=1.0)
parser.add_argument("--continuation-min-seconds", type=float, default=0.4)
parser.add_argument("--int-offset-tolerance-seconds", type=float, default=0.05)
parser.add_argument("--wearer-resume-merge-max-seconds", type=float, default=0.8)
parser.add_argument("--candidate-sources", nargs="+")
parser.add_argument("--max-candidates", type=int)
parser.add_argument("--selection-seed", type=int, default=2027)
@@ -455,6 +756,8 @@ def main() -> None:
other_handover_max_seconds=args.other_handover_max_seconds,
short_vocalization_max_seconds=args.short_vocalization_max_seconds,
continuation_min_seconds=args.continuation_min_seconds,
int_offset_tolerance_seconds=args.int_offset_tolerance_seconds,
wearer_resume_merge_max_seconds=args.wearer_resume_merge_max_seconds,
)
dataset_root, output_dir = args.dataset_root.expanduser().resolve(), args.output_dir.expanduser().resolve()
all_rows: list[ProtocolCandidate] = []
@@ -475,10 +778,14 @@ def main() -> None:
all_rows.extend(rows)
print(f"{session}: candidates={len(rows)} sources={dict(sorted(Counter(row.candidate_source for row in rows).items()))}")
sources = None if args.candidate_sources is None else set(args.candidate_sources)
eligible = [row for row in all_rows if sources is None or row.candidate_source in sources]
controls = [row for row in all_rows if not requires_semantic_adjudication(asdict(row))]
primary = [row for row in all_rows if requires_semantic_adjudication(asdict(row))]
eligible = [row for row in primary if sources is None or row.candidate_source in sources]
selected = stratified_limit(eligible, args.max_candidates, seed=args.selection_seed)
output_path = output_dir / "easycom_eot_int_candidates.jsonl"
_write_jsonl(output_path, selected)
controls_path = output_dir / "easycom_eot_int_controls.jsonl"
_write_jsonl(controls_path, controls)
counts = Counter((row.split, row.candidate_source, row.rule_candidate_type) for row in selected)
summary = {
"schema_version": SCHEMA_VERSION, "protocol_version": PROTOCOL_VERSION, "dataset": "EasyCom",
@@ -487,8 +794,10 @@ def main() -> None:
"candidate_config": asdict(config), "selection_seed": args.selection_seed,
"max_candidates": args.max_candidates,
"candidate_sources": None if sources is None else sorted(sources),
"num_candidates_before_filter": len(all_rows), "num_candidates_before_limit": len(eligible),
"num_candidates_before_filter": len(primary), "num_candidates_before_limit": len(eligible),
"num_candidates_written": len(selected),
"num_controls_written": len(controls),
"controls_file": controls_path.name,
"counts": [{"split": key[0], "candidate_source": key[1],
"rule_candidate_type": key[2], "count": value}
for key, value in sorted(counts.items())],
@@ -501,6 +810,7 @@ def main() -> None:
encoding="utf-8",
)
print(f"Saved {len(selected)} wearer-centric review candidates to {output_path}")
print(f"Saved {len(controls)} deterministic boundary controls to {controls_path}")
if __name__ == "__main__":