From 65347dd75d41df8fe5242011195db835fa86270e Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Mon, 14 Sep 2026 23:30:31 +0800 Subject: [PATCH] Make EasyCom event review wearer anchor first --- PROJECT_CONTEXT.md | 2 +- adjudicate_easycom_eot_int_candidates.py | 69 ++++++++- docs/easycom_event_review_tool.md | 35 +++-- floor_event_review_web/app.css | 4 +- floor_event_review_web/app.js | 40 ++++- floor_event_review_web/index.html | 1 + manuscript/main/main.tex | 29 ++-- prepare_easycom_eot_int_review.py | 139 ++++++++++-------- review_floor_events.py | 48 +++++- test_adjudicate_easycom_eot_int_candidates.py | 21 ++- test_prepare_easycom_eot_int_review.py | 24 ++- test_review_floor_events.py | 31 ++++ 12 files changed, 336 insertions(+), 107 deletions(-) diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md index 0e5e576..31547b8 100644 --- a/PROJECT_CONTEXT.md +++ b/PROJECT_CONTEXT.md @@ -304,7 +304,7 @@ Do not reorganize them physically without compatibility wrappers. | `derive_easycom_interaction_events.py` | Deterministic transcript/timing proxies | | `adjudicate_easycom_interaction_events.py` | Optional ambiguous-event adjudication | | `audit_easycom_floor_events.py` | Count EOT/takeover/backchannel candidates | -| `prepare_easycom_eot_int_review.py` | Group short non-competed wearer resumptions into floor episodes, pair EOT release/opportunity and wearer-entry behavior anchors, require `wearer_floor_start < other_onset < wearer_offset` plus boundary crossing for INT, preserve ineligible boundaries as controls, and scan independent hard negatives; semantic labels remain AI/human review outputs | +| `prepare_easycom_eot_int_review.py` | Group short non-competed wearer resumptions into floor episodes; start from wearer onsets/offsets, retrieve constrained preceding release options for EOT, require `wearer_floor_start < other_onset < wearer_offset` plus boundary crossing for INT, preserve ineligible boundaries as controls, and scan independent hard negatives; semantic labels and the selected EOT trigger remain AI/human review outputs | | `adjudicate_easycom_eot_int_candidates.py` | Add optional DeepSeek transcript/timing suggestions; never human ground truth | | `review_floor_events.py` | Serve the local video/transcript workspace and persist human event reviews | | `prepare_easycom_oracle_features.py` | Build explicit past-only oracle person-activity sidecars and analysis-only addressee controls | diff --git a/adjudicate_easycom_eot_int_candidates.py b/adjudicate_easycom_eot_int_candidates.py index 55c507d..67d8a34 100644 --- a/adjudicate_easycom_eot_int_candidates.py +++ b/adjudicate_easycom_eot_int_candidates.py @@ -26,7 +26,7 @@ import urllib.request from prepare_easycom_eot_int_review import requires_semantic_adjudication -PROMPT_VERSION = "easycom-wearer-centric-eot-int-prelabel-v6" +PROMPT_VERSION = "easycom-wearer-anchor-first-eot-int-prelabel-v7" DEFAULT_MODEL = "deepseek-v4-flash" DEFAULT_BASE_URL = "https://api.deepseek.com" VERDICTS = {"POSITIVE", "HARD_NEGATIVE", "NOT_APPLICABLE", "EXCLUDE", "UNSURE"} @@ -93,7 +93,7 @@ Evidence rules: 2. Never alter or propose an event timestamp. Do not infer prosody, gaze, intention, or acoustic cues that were not supplied. 3. A wearer onset is not automatically a floor take; a wearer offset is not automatically a yield; an other onset is not automatically a takeover. A pause is not automatically EOT, overlap is not automatically interruption, and words such as "yeah" are not automatically backchannels. Check who speaks sustainably afterward and who resumes. 4. Use UNSURE when the evidence does not establish conversational function. Never force a label for coverage. -5. Every supplied WEARER_ONSET has two immutable anchors: a non-wearer release opportunity and a later floor-level wearer entry. Do not reject it solely because response_delay_seconds is long. Decide whether the release remained available, whether the wearer contribution is a genuine response, or whether it is a semantic hard negative such as feedback. +5. Every supplied WEARER_ONSET was found by starting from an observed floor-level wearer entry and looking backward. Its immutable behavior anchor is fixed. Select the true preceding release only from temporal_evidence.opportunity_options; never invent a trigger or timestamp. A POSITIVE EOT_RESPONSE must return that option's uid as selected_opportunity_uid. Do not reject an option solely because its response delay is long. If none is a defensible trigger, return another verdict and null. 6. Every supplied WEARER_OFFSET has already passed both deterministic INT prerequisites: a new non-wearer onset occurs strictly inside the current wearer floor episode (t_w < t_o < t_y), and that same non-wearer interval crosses the wearer yield boundary within the recorded annotation tolerance. Decide only whether it is a genuine takeover or a semantic hard negative. Pre-existing other activity, post-completion other onset, within-turn wearer pauses, and completion-to-silence are controls outside this queue. 7. Transcript event markers are evidence, not lexical turns. In EasyCom, a transcription consisting of [L] denotes laughter-only vocalization. Do not treat [L] as a sustained floor take, an EOT response, or an interruption takeover merely because its activity interval creates an onset, offset, or overlap candidate. Specifically: - wearer [L] during a non-wearer's retained turn is normally HARD_NEGATIVE / EOT_WEARER_BACKCHANNEL, not EOT_RESPONSE; @@ -113,7 +113,7 @@ POSITIVE event_type must be exactly EOT_RESPONSE or INT_YIELD. EOT_RESPONSE mean HARD_NEGATIVE subtype must be exactly one of EOT_MIDTURN_PAUSE, EOT_OTHER_TO_OTHER, EOT_WEARER_BACKCHANNEL, INT_OTHER_BACKCHANNEL, INT_NON_TAKEOVER_OVERLAP, INT_WEARER_HOLDS_FLOOR, or OTHER_HARD_NEGATIVE. Return exactly one JSON object with this shape and no markdown: -{"items":[{"uid":"...","verdict":"UNSURE","event_type":null,"subtype":null,"confidence":0.25,"reason":"brief evidence-based sentence"}]} +{"items":[{"uid":"...","verdict":"UNSURE","event_type":null,"subtype":null,"selected_opportunity_uid":null,"confidence":0.25,"reason":"brief evidence-based sentence"}]} Return every supplied uid exactly once. Copy uid byte-for-byte and never add a backslash before an underscore. confidence must be a number from 0 to 1.""" @@ -211,7 +211,11 @@ def _transport_rows(rows: Sequence[dict[str, Any]]) -> tuple[list[dict[str, Any] return transported, original_by_transport -def validate_decisions(payload: Any, expected_uids: Sequence[str]) -> list[dict[str, Any]]: +def validate_decisions( + payload: Any, + expected_uids: Sequence[str], + allowed_opportunities: dict[str, set[str]] | None = None, +) -> list[dict[str, Any]]: if not isinstance(payload, dict) or not isinstance(payload.get("items"), list): raise ValueError("Response must be an object with an items list") expected = set(expected_uids) @@ -224,6 +228,7 @@ def validate_decisions(payload: Any, expected_uids: Sequence[str]) -> list[dict[ verdict = item.get("verdict") event_type = item.get("event_type") subtype = item.get("subtype") + selected_opportunity_uid = item.get("selected_opportunity_uid") confidence = item.get("confidence") reason = item.get("reason") if uid not in expected or uid in decisions: @@ -238,6 +243,14 @@ def validate_decisions(payload: Any, expected_uids: Sequence[str]) -> list[dict[ raise ValueError(f"HARD_NEGATIVE requires a valid subtype for {uid}") if verdict != "HARD_NEGATIVE" and subtype is not None: raise ValueError(f"Only HARD_NEGATIVE may have subtype for {uid}") + allowed = set() if allowed_opportunities is None else allowed_opportunities.get(str(uid), set()) + if verdict == "POSITIVE" and event_type == "EOT_RESPONSE" and allowed: + if selected_opportunity_uid not in allowed: + raise ValueError( + f"EOT_RESPONSE must select a supplied opportunity uid for {uid}" + ) + elif selected_opportunity_uid is not None: + raise ValueError(f"selected_opportunity_uid is only valid for EOT_RESPONSE: {uid}") if isinstance(confidence, bool) or not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1: raise ValueError(f"confidence must be in [0, 1] for {uid}") if not isinstance(reason, str) or not reason.strip(): @@ -247,6 +260,7 @@ def validate_decisions(payload: Any, expected_uids: Sequence[str]) -> list[dict[ "verdict": str(verdict), "event_type": event_type, "subtype": subtype, + "selected_opportunity_uid": selected_opportunity_uid, "confidence": float(confidence), "reason": reason.strip(), # Every AI decision enters the human queue. This is deliberately @@ -295,7 +309,20 @@ def _post( envelope = json.loads(response.read().decode("utf-8")) content = envelope["choices"][0]["message"]["content"] parsed = json.loads(content) - decisions = validate_decisions(parsed, list(original_by_transport)) + allowed_opportunities = {} + for row in transported: + evidence = row.get("temporal_evidence") or {} + options = evidence.get("opportunity_options") or [] + allowed_opportunities[row["uid"]] = { + str(option["uid"]) + for option in options + if isinstance(option, dict) and option.get("uid") is not None + } + if not allowed_opportunities[row["uid"]] and evidence.get("opportunity_source_uid"): + allowed_opportunities[row["uid"]].add(str(evidence["opportunity_source_uid"])) + decisions = validate_decisions( + parsed, list(original_by_transport), allowed_opportunities + ) for decision in decisions: decision["uid"] = original_by_transport[decision["uid"]] return decisions, envelope.get("usage", {}) @@ -376,23 +403,47 @@ def add_suggestions( output: list[dict[str, Any]] = [] for source in rows: decisions = by_uid[source["uid"]] - labels = {(d["verdict"], d["event_type"], d["subtype"]) for d in decisions} + labels = { + (d["verdict"], d["event_type"], d["subtype"], d["selected_opportunity_uid"]) + for d in decisions + } unanimous = len(labels) == 1 if unanimous: - verdict, event_type, subtype = next(iter(labels)) + verdict, event_type, subtype, selected_opportunity_uid = next(iter(labels)) confidence = min(float(item["confidence"]) for item in decisions) reason = " | ".join(item["reason"] for item in decisions) needs_human_review = any(item["needs_human_review"] for item in decisions) else: - verdict, event_type, subtype = "UNSURE", None, None + verdict, event_type, subtype, selected_opportunity_uid = "UNSURE", None, None, None confidence = min(float(item["confidence"]) for item in decisions) reason = "The independent DeepSeek passes disagreed." needs_human_review = True + opportunity_options = (source.get("temporal_evidence") or {}).get("opportunity_options") or [] + selected_opportunity = next( + ( + option for option in opportunity_options + if isinstance(option, dict) and option.get("uid") == selected_opportunity_uid + ), + None, + ) + selected_opportunity_seconds = ( + None if selected_opportunity is None else selected_opportunity.get("anchor_seconds") + ) + behavior_seconds = (source.get("temporal_evidence") or {}).get( + "behavior_anchor_seconds", source.get("anchor_dense_seconds") + ) enriched = dict(source) enriched["llm_suggestion"] = { "verdict": verdict, "event_type": event_type, "subtype": subtype, + "selected_opportunity_uid": selected_opportunity_uid, + "selected_opportunity_seconds": selected_opportunity_seconds, + "valid_prediction_interval_seconds": ( + None + if selected_opportunity_seconds is None + else [selected_opportunity_seconds, behavior_seconds] + ), "confidence": confidence, "reason": reason, "needs_human_review": needs_human_review, @@ -402,6 +453,8 @@ def add_suggestions( ai_verdict=verdict, ai_event_type=event_type, ai_subtype=subtype, + ai_selected_opportunity_uid=selected_opportunity_uid, + ai_selected_opportunity_seconds=selected_opportunity_seconds, ai_confidence=confidence, ai_reason=reason, human_status="UNREVIEWED", diff --git a/docs/easycom_event_review_tool.md b/docs/easycom_event_review_tool.md index 21d1dd3..5f64c4d 100644 --- a/docs/easycom_event_review_tool.md +++ b/docs/easycom_event_review_tool.md @@ -24,8 +24,8 @@ python3 prepare_easycom_eot_int_review.py \ The generator emits two independent files: -- `easycom_eot_int_candidates.jsonl` contains EOT opportunities that pair a - non-wearer release with a later floor-level wearer `0->1` behavior anchor +- `easycom_eot_int_candidates.jsonl` contains floor-level wearer `0->1` + behavior anchors with one or more admissible preceding non-wearer releases (`WEARER_ONSET`, `EOT_OPPORTUNITY_CANDIDATE`), only episode-ending wearer `1->0` boundaries where a new non-wearer onset satisfies `wearer_floor_start < other_onset < wearer_offset` and the same activity @@ -49,11 +49,15 @@ continues across the resumed onset conservatively preserves a new wearer-entry candidate. Every decision is stored in `temporal_evidence`, including the raw gap, feedback IDs, competing IDs, threshold, and `floor_episode_merged` flag. -After each non-wearer release, EOT construction scans forward until a -floor-level wearer onset or a competing non-wearer onset. Silence length alone -does not invalidate the opportunity. Each primary EOT candidate stores both -`opportunity_anchor_seconds` and `behavior_anchor_seconds`, plus their response -delay. +EOT construction iterates floor-level wearer onsets, then searches backward for +preceding non-wearer releases. A release is an admissible option only when the +wearer stays silent and no new non-wearer onset or already-active other floor +invalidates the interval before the wearer onset. Silence length alone does not +invalidate an option. Each wearer onset produces at most one primary row. Its +`opportunity_options` list contains the only release anchors that DeepSeek or a +human may select; neither may invent a timestamp. The selected release is the +opportunity anchor, the wearer onset is the behavior anchor, and their interval +deterministically defines the valid EOT prediction interval. Paired wearer onsets and strictly qualified offsets are initially `unsure`; timing alone is not a semantic positive label. The INT prerequisite is @@ -85,14 +89,15 @@ timeline; it is not sent to DeepSeek. ## 2. Add DeepSeek pre-labels -Prompt v6 treats the EasyCom transcript marker `[L]` as laughter-only evidence, +Prompt v7 treats the EasyCom transcript marker `[L]` as laughter-only evidence, not as a lexical floor take. Wearer laughter during another participant's retained turn is directed toward `EOT_WEARER_BACKCHANNEL`; non-wearer laughter overlapping a retained wearer turn is directed toward `INT_OTHER_BACKCHANNEL`. Ambiguous laughter remains `UNSURE`, and `[L]` alone is neither an exclusion reason nor evidence of a visible smile. The prompt also treats the two EOT -anchors and strict INT ordering as deterministic prerequisites rather than -asking the model to reconstruct them. +anchor options and strict INT ordering as deterministic prerequisites. For a +positive EOT response it must return `selected_opportunity_uid` from the supplied +options; all other decisions use `null`. `needs_human_review` is not requested from DeepSeek: every pre-label enters the human queue by protocol, so the adjudicator assigns that boolean locally. This @@ -125,8 +130,9 @@ DeepSeek sees candidate source, structured timing, per-speaker activity, and transcript together. It returns one of `POSITIVE`, `HARD_NEGATIVE`, `NOT_APPLICABLE`, `EXCLUDE`, or `UNSURE`; positives additionally receive `EOT_RESPONSE` or `INT_YIELD`, and hard negatives receive a named subtype. It -cannot modify timestamps. The output preserves both the original rule candidate -and the complete AI decision/provenance. +cannot modify timestamps. The output preserves the original rule candidate, +the selected EOT opportunity UID when applicable, and the complete AI +decision/provenance. During live adjudication, the CLI displays a dependency-free progress bar over all candidate/pass work, including the current pass, batch, percentage, and @@ -154,8 +160,9 @@ question, then presents the exact AI verdict/subtype with a plain-language meaning; the Accept control repeats that exact label so its effect is explicit. The common path is one operation: -- **Accept** keeps the AI verdict/type; -- **Relabel** expands verdict and subtype controls; +- **Accept** keeps the AI verdict/type and its selected EOT opportunity anchor; +- **Relabel** expands verdict and subtype controls and, for a positive EOT, + requires one of the supplied release anchors; - **Exclude** requires a concrete data-quality reason. The current primary-review UI deliberately has no second-review action. Older diff --git a/floor_event_review_web/app.css b/floor_event_review_web/app.css index 51012ed..340a813 100644 --- a/floor_event_review_web/app.css +++ b/floor_event_review_web/app.css @@ -274,7 +274,9 @@ textarea { resize: vertical; line-height: 1.45; } .speaker-lane { display: grid; grid-template-columns: 58px 1fr; align-items: center; gap: 8px; } .speaker-lane > strong { color: var(--muted); text-align: right; font-size: .72rem; } .speaker-track { height: 19px; position: relative; border: 1px solid var(--line); border-radius: 5px; overflow: hidden; background: #07101d; } -.speaker-track > i { position: absolute; top: 0; bottom: 0; width: 1px; background: var(--amber); z-index: 3; } +.speaker-track > i { position: absolute; top: 0; bottom: 0; width: 1px; z-index: 3; } +.speaker-track > i.behavior-anchor { background: var(--amber); } +.speaker-track > i.opportunity-anchor { width: 2px; background: #60a5fa; opacity: .9; } .speech-segment { position: absolute; top: 3px; height: 11px; min-width: 4px; padding: 0; border: 1px solid transparent; border-radius: 3px; background: #60a5fa; cursor: pointer; } .speech-segment.wearer { background: var(--accent); } .speech-segment:hover, .speech-segment:focus-visible, .speech-segment.active { border-color: #fff; box-shadow: 0 0 0 2px rgba(255,255,255,.28); z-index: 4; outline: none; } diff --git a/floor_event_review_web/app.js b/floor_event_review_web/app.js index fdd8162..df7ec64 100644 --- a/floor_event_review_web/app.js +++ b/floor_event_review_web/app.js @@ -26,7 +26,7 @@ const elements = Object.fromEntries( "eventVideo", "mediaEmpty", "jumpEvent", "timeReadout", "clipLabel", "transcriptList", "rawCandidate", "reviewForm", "formError", "saveButton", "saveState", "toast", "relabelFields", - "excludeFields", "eventTypeField", "subtypeField", + "excludeFields", "eventTypeField", "opportunityField", "opportunitySelect", "subtypeField", ].map((id) => [id, document.getElementById(id)]) ); @@ -262,6 +262,18 @@ function renderDetail(detail) { elements.eventSubtitle.textContent = detail.uid; elements.clipLabel.textContent = detail.clip_stem ? `${detail.session} / ${detail.clip_stem}` : detail.session; elements.rawCandidate.textContent = JSON.stringify(detail.candidate, null, 2); + const opportunityOptions = detail.candidate?.temporal_evidence?.opportunity_options || []; + elements.opportunitySelect.replaceChildren(); + const emptyOption = document.createElement("option"); + emptyOption.value = ""; + emptyOption.textContent = "请选择 preceding release"; + elements.opportunitySelect.append(emptyOption); + for (const option of opportunityOptions) { + const control = document.createElement("option"); + control.value = option.uid; + control.textContent = `${formatTime(option.anchor_seconds)} · P${option.participant_id ?? "?"} · ${option.transcription || "无转写"}`; + elements.opportunitySelect.append(control); + } elements.saveState.textContent = detail.review ? "已保存" : "未保存"; elements.saveState.classList.toggle("saved", Boolean(detail.review)); elements.reviewQuestion.textContent = reviewQuestions[detail.family] || "核对该候选是否属于目标 floor event。"; @@ -278,7 +290,10 @@ function renderDetail(detail) { elements.aiRationale.textContent = view.rationale; elements.acceptRadio.disabled = false; elements.acceptTitle.textContent = `接受 ${view.verdict.replaceAll("_", " ")}`; - elements.acceptSummary.textContent = `${view.label} · ${view.meaning}`; + const selected = suggestion.selected_opportunity_uid; + const selectedOption = opportunityOptions.find((option) => option.uid === selected); + const anchorText = selectedOption ? ` · trigger ${formatTime(selectedOption.anchor_seconds)} P${selectedOption.participant_id}` : ""; + elements.acceptSummary.textContent = `${view.label}${anchorText} · ${view.meaning}`; } else { elements.acceptRadio.disabled = true; elements.acceptTitle.textContent = "没有 AI 结论"; @@ -435,6 +450,7 @@ function renderSpeakerTimeline(detail) { return; } const eventTime = Number(detail.candidate.event_time ?? detail.candidate.anchor_dense_seconds); + const opportunityOptions = detail.candidate?.temporal_evidence?.opportunity_options || []; const left = Math.min(...rows.map((row) => Number(row.start_seconds)), eventTime - 3); const right = Math.max(...rows.map((row) => Number(row.end_seconds)), eventTime + 3); const span = Math.max(0.001, right - left); @@ -458,7 +474,18 @@ function renderSpeakerTimeline(detail) { segment.addEventListener("click", () => showSpeakerEvidence(row, segment, true)); track.append(segment); } - const anchor = document.createElement("i"); anchor.style.left = `${100 * (eventTime - left) / span}%`; track.append(anchor); + for (const option of opportunityOptions) { + const opportunity = document.createElement("i"); + opportunity.className = "opportunity-anchor"; + opportunity.style.left = `${100 * (Number(option.anchor_seconds) - left) / span}%`; + opportunity.title = `可选 release · ${formatTime(option.anchor_seconds)} · P${option.participant_id ?? "?"}`; + track.append(opportunity); + } + const anchor = document.createElement("i"); + anchor.className = "behavior-anchor"; + anchor.style.left = `${100 * (eventTime - left) / span}%`; + anchor.title = `wearer behavior · ${formatTime(eventTime)}`; + track.append(anchor); lane.append(label, track); elements.speakerTimeline.append(lane); } const defaultRow = rows.find((row) => @@ -496,6 +523,7 @@ function formPayload() { human_verdict: nullable("human_verdict"), human_event_type: nullable("human_event_type"), human_subtype: nullable("human_subtype"), + selected_opportunity_uid: nullable("selected_opportunity_uid"), exclude_reason: nullable("exclude_reason"), boundary_correction_seconds: correction === "" ? null : Number(correction), confidence: data.get("confidence"), @@ -508,6 +536,8 @@ function validateReview(review) { if (!review.human_action) missing.push("主操作"); if (review.human_action === "relabel" && !review.human_verdict) missing.push("一级 verdict"); if (review.human_action === "relabel" && review.human_verdict === "POSITIVE" && !review.human_event_type) missing.push("positive event"); + const options = state.currentDetail?.candidate?.temporal_evidence?.opportunity_options || []; + if (review.human_action === "relabel" && review.human_verdict === "POSITIVE" && review.human_event_type === "EOT_RESPONSE" && options.length && !review.selected_opportunity_uid) missing.push("EOT opportunity anchor"); if (review.human_action === "relabel" && review.human_verdict === "HARD_NEGATIVE" && !review.human_subtype) missing.push("hard-negative subtype"); if (review.human_action === "exclude" && !review.exclude_reason) missing.push("排除原因"); if (review.human_action === "accept" && !state.currentDetail?.candidate?.ai_verdict && !state.currentDetail?.llm_suggestion?.verdict) missing.push("AI 预标注（尚未运行，需 Relabel）"); @@ -517,9 +547,12 @@ function validateReview(review) { function updateConditionalFields() { const action = elements.reviewForm.elements.human_action.value; const verdict = elements.reviewForm.elements.human_verdict.value; + const eventType = elements.reviewForm.elements.human_event_type.value; + const hasOpportunityOptions = Boolean(state.currentDetail?.candidate?.temporal_evidence?.opportunity_options?.length); elements.relabelFields.hidden = action !== "relabel"; elements.excludeFields.hidden = action !== "exclude"; elements.eventTypeField.hidden = action !== "relabel" || verdict !== "POSITIVE"; + elements.opportunityField.hidden = action !== "relabel" || verdict !== "POSITIVE" || eventType !== "EOT_RESPONSE" || !hasOpportunityOptions; elements.subtypeField.hidden = action !== "relabel" || verdict !== "HARD_NEGATIVE"; } @@ -619,6 +652,7 @@ function bindEvents() { }); elements.reviewForm.elements.human_action.forEach((control) => control.addEventListener("change", updateConditionalFields)); elements.reviewForm.elements.human_verdict.addEventListener("change", updateConditionalFields); + elements.reviewForm.elements.human_event_type.addEventListener("change", updateConditionalFields); elements.timelineZoomIn.addEventListener("click", () => changeTimelineZoom(2)); elements.timelineZoomOut.addEventListener("click", () => changeTimelineZoom(0.5)); elements.timelinePanLeft.addEventListener("click", () => panTimeline(-1)); diff --git a/floor_event_review_web/index.html b/floor_event_review_web/index.html index c719b42..b651d73 100644 --- a/floor_event_review_web/index.html +++ b/floor_event_review_web/index.html @@ -84,6 +84,7 @@ diff --git a/manuscript/main/main.tex b/manuscript/main/main.tex index 1e85ff7..637021d 100644 --- a/manuscript/main/main.tex +++ b/manuscript/main/main.tex @@ -543,14 +543,18 @@ contribution is retained as evidence but does not by itself split the wearer episode. These rules group events only; they do not modify the source annotations. -Following each candidate non-wearer release, the generator opens an EOT -opportunity and scans forward until a floor-level wearer entry or a competing -non-wearer onset terminates it. A retained \textsc{Wearer-Onset} candidate -therefore contains two immutable anchors: the non-wearer release -(\emph{opportunity anchor}) and the subsequent floor-level wearer onset -(\emph{behavior anchor}). The response delay is stored explicitly. Silence -duration alone does not cancel the opportunity; an unpaired wearer onset is -kept only as an audit control and does not enter the primary EOT queue. +EOT candidate construction starts from each observed floor-level wearer onset, +which is retained as the immutable \emph{behavior anchor}. The generator then +searches backward for preceding non-wearer releases. A release remains an +admissible \emph{opportunity-anchor} option only when the wearer stays silent +and no new non-wearer onset or already-active other floor invalidates the +intervening interval. Silence duration alone does not reject an option. Thus +each wearer onset contributes at most one primary review item rather than one +item per preceding release. The semantic annotator must either select one of +the supplied release options as the trigger or reject the EOT interpretation; +it cannot introduce a new timestamp. The selected opportunity and observed +behavior anchors deterministically define the valid prediction interval. A +wearer onset without an admissible release is retained only as an audit control. \textbf{EOT hard negatives.} Two participant-resolved timing rules independently propose regions in which @@ -598,7 +602,8 @@ treat timing as factual, not to invent prosody, gaze, gesture, intention, or other unavailable cues, and not to equate onset with floor entry, offset with yield, silence with EOT, or overlap with interruption. EasyCom's \texttt{[L]} marker is treated as evidence of laughter only: it is not by itself a sustained -floor take or takeover. +floor take or takeover. For EOT, a positive decision must name one supplied +opportunity-anchor option; for all other EOT decisions no trigger is selected. The allowed decisions are \textsc{Positive}, \textsc{Hard-Negative}, \textsc{Not-Applicable}, \textsc{Exclude}, and \textsc{Unsure}. Positive @@ -616,9 +621,11 @@ candidate. Every AI-pre-labeled candidate enters an event-centered interface containing the original video, local participant timeline, transcript, deterministic timing evidence, and the AI suggestion. The adjudicator performs one primary -action: accept the complete AI label, relabel it, or exclude an unscorable +action: accept the complete AI label and selected EOT trigger, relabel it, or +exclude an unscorable sample. Relabeling exposes the same constrained verdict, event-type, and -hard-negative subtype vocabulary. A confidence level is required; notes are +hard-negative subtype vocabulary. Relabeling a positive EOT also requires one +of the displayed release options. A confidence level is required; notes are optional, and an optional $\pm0.2$~s boundary correction is available only for an evident source-annotation error. The original event time is always retained. diff --git a/prepare_easycom_eot_int_review.py b/prepare_easycom_eot_int_review.py index 9ed314b..19a512b 100644 --- a/prepare_easycom_eot_int_review.py +++ b/prepare_easycom_eot_int_review.py @@ -3,7 +3,8 @@ This stage never changes EasyCom speech boundaries and never claims semantic ground truth. It groups short wearer resumptions into floor-level episodes, -pairs EOT candidates with a preceding non-wearer release, requires a new +starts EOT candidates from observed wearer entries and retrieves preceding +non-wearer release options, requires a new non-wearer onset inside the wearer floor episode for INT candidates, and keeps unpaired or ineligible raw boundaries as deterministic controls. All timing thresholds are CLI parameters, and raw evidence is retained for DeepSeek @@ -27,8 +28,8 @@ from derive_easycom_interaction_events import ( from easycom_manifest import ALL_SESSIONS, TEST_SESSIONS -SCHEMA_VERSION = 3 -PROTOCOL_VERSION = "easycom-wearer-centric-eot-int-candidates-v5" +SCHEMA_VERSION = 4 +PROTOCOL_VERSION = "easycom-wearer-anchor-first-eot-int-candidates-v6" @dataclass(frozen=True) @@ -422,70 +423,84 @@ def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[ reason="short raw wearer pause without sustained competing other takeover; merged into the existing floor episode", ), ("WEARER_RESUME", frame, ())) - # A1: open an opportunity at a non-wearer release and pair it with the - # subsequent floor-level wearer entry only if no competing other onset - # or wearer activity intervenes. Long silence alone does not invalidate - # the opportunity. Unpaired floor onsets are retained as controls. - paired_floor_onsets: set[int] = set() - for release in other_rows: - opportunity_frame = release.dense_end_frame_exclusive - onset_frame = next( - (value for value in floor_onsets if value >= opportunity_frame), None - ) - if onset_frame is None: - continue - wearer_silent = ( - opportunity_frame == onset_frame - or _all_state( - wearer, - opportunity_frame - segment_start, - onset_frame - segment_start, - False, + # A1: start from each observed floor-level wearer entry, then search + # backward for release anchors that survive deterministic timing checks. + # This yields at most one review item per wearer behavior. The semantic + # annotator may select only one of the stored options; it cannot invent + # a trigger or move either timestamp. Long silence alone is not a veto. + for onset_frame in floor_onsets: + release_options: list[tuple[Utterance, int]] = [] + for release in other_rows: + opportunity_frame = release.dense_end_frame_exclusive + if opportunity_frame > onset_frame: + continue + wearer_silent = ( + opportunity_frame == onset_frame + or _all_state( + wearer, + opportunity_frame - segment_start, + onset_frame - segment_start, + False, + ) ) - ) - competing_onsets = [ - row for row in other_rows - if opportunity_frame <= row.dense_start_frame < onset_frame - ] - other_already_holds = bool( - _active_other_ids(rows, opportunity_frame, wearer_id=wearer_id) - ) - if not wearer_silent or competing_onsets or other_already_holds: - continue - evidence = dict(onset_records[onset_frame]["evidence"]) - evidence.update( - opportunity_anchor_type="other_offset", - opportunity_anchor_frame=opportunity_frame, - opportunity_anchor_seconds=opportunity_frame / SOURCE_FPS, - opportunity_speaker_id=release.participant_id, - opportunity_source_uid=release.uid, - opportunity_source_transcription=release.transcription, - opportunity_source_target_of_speech=list(release.target_of_speech), - behavior_anchor_type="wearer_floor_onset", - behavior_anchor_frame=onset_frame, - behavior_anchor_seconds=onset_frame / SOURCE_FPS, - response_delay_seconds=(onset_frame - opportunity_frame) / SOURCE_FPS, - competing_other_onsets_before_behavior=[], - eot_opportunity_paired=True, - ) - add(_candidate( - session=session, task="eot_response", source="WEARER_ONSET", - rule_type="EOT_OPPORTUNITY_CANDIDATE", anchor_type="wearer_onset", - frame=onset_frame, region_start=opportunity_frame, - region_end=min(segment_end, onset_frame + relation_frames), - label="unsure", subtype=None, previous_ids=(release.participant_id,), - incoming_ids=_active_other_ids(rows, onset_frame, wearer_id=wearer_id), - evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id, - context_frames=context_frames, - reason="non-wearer release opportunity paired with a subsequent floor-level wearer entry", - ), ("WEARER_ONSET", onset_frame, (release.participant_id,))) - paired_floor_onsets.add(onset_frame) + competing_onset = any( + opportunity_frame <= row.dense_start_frame < onset_frame + for row in other_rows + ) + other_already_holds = bool( + _active_other_ids(rows, opportunity_frame, wearer_id=wearer_id) + ) + if wearer_silent and not competing_onset and not other_already_holds: + release_options.append((release, opportunity_frame)) - for frame in floor_onsets: - if frame in paired_floor_onsets: + release_options.sort( + key=lambda item: (item[1], item[0].participant_id, item[0].uid) + ) + if release_options: + _, latest_opportunity_frame = release_options[-1] + earliest_opportunity_frame = release_options[0][1] + option_rows = [{ + "uid": release.uid, + "anchor_type": "other_offset", + "anchor_frame": frame, + "anchor_seconds": frame / SOURCE_FPS, + "participant_id": release.participant_id, + "transcription": release.transcription, + "target_of_speech": list(release.target_of_speech), + "response_delay_seconds": (onset_frame - frame) / SOURCE_FPS, + } for release, frame in release_options] + evidence = dict(onset_records[onset_frame]["evidence"]) + evidence.update( + opportunity_options=option_rows, + opportunity_anchor_type=None, + opportunity_anchor_frame=None, + opportunity_anchor_seconds=None, + behavior_anchor_type="wearer_floor_onset", + behavior_anchor_frame=onset_frame, + behavior_anchor_seconds=onset_frame / SOURCE_FPS, + minimum_response_delay_seconds=(onset_frame - latest_opportunity_frame) / SOURCE_FPS, + maximum_response_delay_seconds=(onset_frame - earliest_opportunity_frame) / SOURCE_FPS, + competing_other_onsets_before_behavior=[], + eot_opportunity_paired=True, + ) + previous_ids = tuple(sorted({row.participant_id for row, _ in release_options})) + add(_candidate( + session=session, task="eot_response", source="WEARER_ONSET", + rule_type="EOT_OPPORTUNITY_CANDIDATE", anchor_type="wearer_onset", + frame=onset_frame, region_start=earliest_opportunity_frame, + region_end=min(segment_end, onset_frame + relation_frames), + label="unsure", subtype=None, previous_ids=previous_ids, + incoming_ids=_active_other_ids(rows, onset_frame, wearer_id=wearer_id), + evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id, + context_frames=context_frames, + reason="floor-level wearer entry with one or more surviving preceding release options", + ), ("WEARER_ONSET", onset_frame, previous_ids)) continue + + frame = onset_frame evidence = dict(onset_records[frame]["evidence"]) evidence.update( + opportunity_options=[], opportunity_anchor_type=None, opportunity_anchor_frame=None, opportunity_anchor_seconds=None, diff --git a/review_floor_events.py b/review_floor_events.py index 797d5f6..bbe2365 100644 --- a/review_floor_events.py +++ b/review_floor_events.py @@ -34,7 +34,7 @@ SOURCE_FPS = 20.0 STATIC_ROOT = Path(__file__).resolve().parent / "floor_event_review_web" VIDEO_ROOT_PARTS = ("Main", "Video_Compressed") TRANSCRIPT_ROOT_PARTS = ("Main", "Speech_Transcriptions") -REVIEW_SCHEMA_VERSION = 3 +REVIEW_SCHEMA_VERSION = 4 MAX_REQUEST_BYTES = 128 * 1024 PROTOCOL_DISPOSITION_VALUES = {"positive", "hard_negative", "exclude", "uncertain"} @@ -459,14 +459,20 @@ class ReviewStore: verdict = candidate.get("ai_verdict") or (candidate.get("llm_suggestion") or {}).get("verdict") event_type = candidate.get("ai_event_type") or (candidate.get("llm_suggestion") or {}).get("event_type") subtype = candidate.get("ai_subtype") or (candidate.get("llm_suggestion") or {}).get("subtype") + selected_opportunity_uid = candidate.get("ai_selected_opportunity_uid") or ( + candidate.get("llm_suggestion") or {} + ).get("selected_opportunity_uid") if verdict not in HUMAN_VERDICT_VALUES: raise ValueError("Accept requires an AI pre-label; use Relabel when no AI label is available") elif action == "exclude": verdict, event_type, subtype = "EXCLUDE", None, None + selected_opportunity_uid = None elif action == "second_review": verdict, event_type, subtype = "UNSURE", None, None + selected_opportunity_uid = None else: verdict, event_type, subtype = payload.get("human_verdict"), payload.get("human_event_type"), payload.get("human_subtype") + selected_opportunity_uid = payload.get("selected_opportunity_uid") if verdict not in HUMAN_VERDICT_VALUES: raise ValueError(f"Invalid human_verdict: {verdict!r}") if verdict == "POSITIVE" and event_type not in HUMAN_EVENT_TYPE_VALUES: @@ -477,6 +483,35 @@ class ReviewStore: raise ValueError("HARD_NEGATIVE requires a valid human_subtype") if verdict != "HARD_NEGATIVE" and subtype is not None: raise ValueError("Only HARD_NEGATIVE may have a human_subtype") + evidence = candidate.get("temporal_evidence") or {} + opportunity_options = evidence.get("opportunity_options") or [] + allowed_opportunity_uids = { + str(option["uid"]) + for option in opportunity_options + if isinstance(option, dict) and option.get("uid") is not None + } + if verdict == "POSITIVE" and event_type == "EOT_RESPONSE" and allowed_opportunity_uids: + if selected_opportunity_uid not in allowed_opportunity_uids: + raise ValueError("EOT_RESPONSE requires one supplied opportunity anchor") + elif selected_opportunity_uid is not None: + raise ValueError("selected_opportunity_uid is only valid for EOT_RESPONSE") + selected_opportunity = next( + ( + option for option in opportunity_options + if isinstance(option, dict) and str(option.get("uid")) == selected_opportunity_uid + ), + None, + ) + selected_opportunity_seconds = ( + None if selected_opportunity is None else selected_opportunity.get("anchor_seconds") + ) + behavior_anchor_seconds = evidence.get( + "behavior_anchor_seconds", + candidate.get( + "anchor_dense_seconds", + candidate.get("event_time", candidate.get("dense_start_seconds")), + ), + ) exclude_reason = payload.get("exclude_reason") if action == "exclude" and exclude_reason not in EXCLUDE_REASON_VALUES: raise ValueError("Exclude requires an exclude_reason") @@ -490,6 +525,13 @@ class ReviewStore: human_verdict=verdict, human_event_type=event_type, human_subtype=subtype, + selected_opportunity_uid=selected_opportunity_uid, + selected_opportunity_seconds=selected_opportunity_seconds, + valid_prediction_interval_seconds=( + None + if selected_opportunity_seconds is None + else [selected_opportunity_seconds, behavior_anchor_seconds] + ), exclude_reason=exclude_reason, relabeled=action == "relabel", boundary_correction_seconds=correction, @@ -607,7 +649,9 @@ class ReviewStore: "automatic_hard_negative_type", "ai_verdict", "ai_event_type", "ai_subtype", "human_action", "human_status", "human_verdict", "human_event_type", "human_subtype", - "exclude_reason", "relabeled", "boundary_correction_seconds", "original_event_time", + "selected_opportunity_uid", "selected_opportunity_seconds", + "valid_prediction_interval_seconds", "exclude_reason", "relabeled", + "boundary_correction_seconds", "original_event_time", "confidence", "reviewer", "notes", diff --git a/test_adjudicate_easycom_eot_int_candidates.py b/test_adjudicate_easycom_eot_int_candidates.py index 8b9a43f..7efa07d 100644 --- a/test_adjudicate_easycom_eot_int_candidates.py +++ b/test_adjudicate_easycom_eot_int_candidates.py @@ -32,8 +32,8 @@ class EOTINTAdjudicationTests(unittest.TestCase): self.assertIn("Yes.", prompt) def test_system_prompt_treats_laughter_as_nonlexical_floor_feedback(self) -> None: - self.assertEqual(PROMPT_VERSION, "easycom-wearer-centric-eot-int-prelabel-v6") - self.assertIn("two immutable anchors", SYSTEM_PROMPT) + self.assertEqual(PROMPT_VERSION, "easycom-wearer-anchor-first-eot-int-prelabel-v7") + self.assertIn("opportunity_options", SYSTEM_PROMPT) self.assertIn("t_w < t_o < t_y", SYSTEM_PROMPT) self.assertIn("a transcription consisting of [L] denotes laughter-only vocalization", SYSTEM_PROMPT) self.assertIn("EOT_WEARER_BACKCHANNEL", SYSTEM_PROMPT) @@ -72,6 +72,23 @@ class EOTINTAdjudicationTests(unittest.TestCase): decision = validate_decisions(payload, ["u1"])[0] self.assertTrue(decision["needs_human_review"]) + def test_eot_positive_must_select_one_supplied_release(self) -> None: + item = { + "uid": "u1", "verdict": "POSITIVE", "event_type": "EOT_RESPONSE", + "subtype": None, "selected_opportunity_uid": "release-2", + "confidence": 0.9, "reason": "The wearer answers the second release.", + } + decision = validate_decisions( + {"items": [item]}, ["u1"], {"u1": {"release-1", "release-2"}} + )[0] + self.assertEqual(decision["selected_opportunity_uid"], "release-2") + with self.assertRaisesRegex(ValueError, "select a supplied opportunity"): + validate_decisions( + {"items": [{**item, "selected_opportunity_uid": "invented"}]}, + ["u1"], + {"u1": {"release-1", "release-2"}}, + ) + def test_markdown_escaped_uid_is_normalized_but_still_unique(self) -> None: item = { "uid": "Session\\_4/wearer\\_onset/00000024", diff --git a/test_prepare_easycom_eot_int_review.py b/test_prepare_easycom_eot_int_review.py index 8627e61..7954fc6 100644 --- a/test_prepare_easycom_eot_int_review.py +++ b/test_prepare_easycom_eot_int_review.py @@ -60,9 +60,12 @@ class EOTINTReviewCandidateTests(unittest.TestCase): self.assertEqual(event.rule_candidate_type, "EOT_OPPORTUNITY_CANDIDATE") self.assertEqual(event.anchor_type, "wearer_onset") self.assertAlmostEqual(event.event_time, 1.2) - self.assertAlmostEqual(event.temporal_evidence["opportunity_anchor_seconds"], 1.0) + self.assertIsNone(event.temporal_evidence["opportunity_anchor_seconds"]) + self.assertAlmostEqual( + event.temporal_evidence["opportunity_options"][0]["anchor_seconds"], 1.0 + ) self.assertAlmostEqual(event.temporal_evidence["behavior_anchor_seconds"], 1.2) - self.assertAlmostEqual(event.temporal_evidence["response_delay_seconds"], 0.2) + self.assertAlmostEqual(event.temporal_evidence["minimum_response_delay_seconds"], 0.2) self.assertAlmostEqual(event.temporal_evidence["wearer_speech_duration_after_onset_seconds"], 0.8) def test_long_silence_does_not_cancel_eot_opportunity(self) -> None: @@ -71,7 +74,22 @@ class EOTINTReviewCandidateTests(unittest.TestCase): utterance("wearer", 2, 3.5, 4.5, "Delayed response."), ] event = next(row for row in self.derive(rows) if row.candidate_source == "WEARER_ONSET") - self.assertAlmostEqual(event.temporal_evidence["response_delay_seconds"], 3.0) + self.assertAlmostEqual(event.temporal_evidence["maximum_response_delay_seconds"], 3.0) + + def test_one_wearer_onset_has_one_row_with_constrained_release_options(self) -> None: + rows = [ + utterance("other-4", 4, 0.0, 1.0, "Question one."), + utterance("other-3", 3, 0.2, 1.0, "Question two."), + utterance("wearer", 2, 1.4, 2.4, "Wearer answers."), + ] + events = [ + row for row in self.derive(rows) + if row.candidate_source == "WEARER_ONSET" and row.anchor_dense_seconds == 1.4 + ] + self.assertEqual(len(events), 1) + options = events[0].temporal_evidence["opportunity_options"] + self.assertEqual({option["uid"] for option in options}, {"other-4", "other-3"}) + self.assertEqual(events[0].temporal_evidence["behavior_anchor_seconds"], 1.4) def test_competing_other_onset_terminates_eot_opportunity(self) -> None: rows = [ diff --git a/test_review_floor_events.py b/test_review_floor_events.py index 8ce7c3a..936cd97 100644 --- a/test_review_floor_events.py +++ b/test_review_floor_events.py @@ -194,6 +194,37 @@ class FloorEventReviewTests(unittest.TestCase): self.assertEqual(record["review"]["human_subtype"], "EOT_WEARER_BACKCHANNEL") self.assertTrue(record["review"]["relabeled"]) + def test_eot_review_selects_only_a_supplied_opportunity_anchor(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + _, events = self._fixture(root) + candidates = load_candidates([events]) + candidate = candidates[0] + candidate.payload["temporal_evidence"] = { + "opportunity_options": [ + {"uid": "release-1", "anchor_seconds": 1.0, "participant_id": 4} + ] + } + store = ReviewStore(root / "reviews.jsonl", candidates) + base = { + "human_action": "relabel", "human_verdict": "POSITIVE", + "human_event_type": "EOT_RESPONSE", "human_subtype": None, + "exclude_reason": None, "boundary_correction_seconds": None, + "confidence": "high", "notes": "clear response", + } + record = store.save( + candidate.candidate_id, {**base, "selected_opportunity_uid": "release-1"} + ) + self.assertEqual(record["review"]["selected_opportunity_uid"], "release-1") + self.assertEqual( + record["review"]["valid_prediction_interval_seconds"], [1.0, 1.2] + ) + with self.assertRaisesRegex(ValueError, "requires one supplied"): + store.save( + candidate.candidate_id, + {**base, "selected_opportunity_uid": "invented"}, + ) + def test_byte_ranges(self) -> None: self.assertEqual(parse_byte_range("bytes=2-5", 10), (2, 5)) self.assertEqual(parse_byte_range("bytes=7-", 10), (7, 9))