Delegate wearer boundary semantics to review
This commit is contained in:
Xin Wang
@@ -23,10 +23,11 @@ from typing import Any, Sequence, TextIO
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from easycom_event_taxonomy import EXCLUDE_REASONS, exclude_reason_group
|
||||
from prepare_easycom_eot_int_review import requires_semantic_adjudication
|
||||
|
||||
|
||||
PROMPT_VERSION = "easycom-wearer-entry-int-prelabel-v12"
|
||||
PROMPT_VERSION = "easycom-wearer-boundary-semantic-prelabel-v14"
|
||||
DEFAULT_MODEL = "deepseek-v4-flash"
|
||||
DEFAULT_BASE_URL = "https://api.deepseek.com"
|
||||
VERDICTS = {"POSITIVE", "HARD_NEGATIVE", "EXCLUDE"}
|
||||
@@ -40,8 +41,6 @@ HARD_NEGATIVE_TYPES = {
|
||||
"INT_WEARER_HOLDS_FLOOR",
|
||||
"OTHER_HARD_NEGATIVE",
|
||||
}
|
||||
|
||||
|
||||
def _progress_line(
|
||||
completed: int, total: int, *, label: str, elapsed_seconds: float, width: int = 26
|
||||
) -> str:
|
||||
@@ -91,10 +90,10 @@ SYSTEM_PROMPT = """You conservatively pre-label candidate regions for a scientif
|
||||
Evidence rules:
|
||||
1. Reconstruct the event chronologically before assigning semantics. Use exact start/end timestamps to identify (a) the proposed release or competing onset, (b) the wearer behavior anchor, (c) which same participant continues across the anchor, and (d) who produces the next substantive turn. Then use transcript content and Target_of_Speech, when supplied, to decide conversational function. Timing has priority over speculative transcript interpretation, but duration alone never determines floor function.
|
||||
2. Never alter or propose an event timestamp. Do not infer prosody, gaze, intention, or acoustic cues that were not supplied.
|
||||
3. A raw wearer onset is not automatically a floor entry; a wearer offset is not automatically a yield; an other onset is not automatically a takeover. A pause is not automatically a release, overlap is not automatically interruption, and words such as "yeah" are not automatically backchannels. Determine what the utterance does in context, not only how long it lasts.
|
||||
3. Every supplied WEARER_ONSET and WEARER_OFFSET is a raw observed wearer activity boundary. Candidate generation has made no floor-entry, within-turn-resumption, interruption, or takeover decision. A raw wearer onset is not automatically a floor entry; a wearer offset is not automatically a yield; an other onset is not automatically a takeover. A pause is not automatically a release, overlap is not automatically interruption, and words such as "yeah" are not automatically backchannels. Determine what the utterance does in context, not only how long it lasts.
|
||||
4. EXCLUDE is the sole non-scoring outcome. Use it both for insufficient/unresolvable evidence and for clear out-of-scope non-events that should not enter evaluation, including a normal wearer completion that merely coincides with laughter or feedback. Never force such a region into HARD_NEGATIVE for coverage.
|
||||
5. Every supplied WEARER_ONSET is an observed floor-level onset after deterministic within-turn pause merging. Decide whether the wearer genuinely enters or re-enters the conversational floor. A POSITIVE WEARER_ENTRY does not require a preceding non-wearer release: it may occur after a release, from silence, or while another speaker is active. temporal_evidence.opportunity_options and entry_context_timing are context metadata only. If one supplied release clearly prompted the entry, selected_opportunity_uid may contain its uid; otherwise return null. Never invent a trigger or timestamp. A direct, semantically complete answer can be a genuine entry even when short. Do not call it a backchannel merely because it lasts less than one second or because another participant speaks later. For example, "Nope." beginning 0.15 s after an addressed question ends is normally WEARER_ENTRY, not ENTRY_WEARER_BACKCHANNEL. Later conversation does not retroactively cancel a completed entry.
|
||||
6. Every supplied WEARER_OFFSET has already passed both deterministic INT prerequisites: a new non-wearer onset occurs strictly inside the current wearer floor episode (t_w < t_o < t_y), and that same non-wearer interval crosses the wearer yield boundary within the recorded annotation tolerance. This is only a retrieval gate. Verify that the same incoming non-wearer actually takes and sustains the next floor after the wearer offset. A different participant's later substantive turn is not evidence that the incoming speaker took over. Do not label INT_WEARER_HOLDS_FLOOR merely because the wearer speaks again later: if the competing other establishes an intervening substantive floor and the wearer has an intervening silent interval, the later wearer onset is a new WEARER_ENTRY and does not invalidate the earlier INT_YIELD. If the wearer reaches a semantically complete natural ending while the overlapping contribution is only [L] or feedback, and that contributor does not take the next substantive floor, return EXCLUDE: out-of-scope normal completion, not INT_YIELD and not a scored hard negative. Pre-existing other activity, post-completion other onset, within-turn wearer pauses, and completion-to-silence are controls outside this queue.
|
||||
5. For every WEARER_ONSET, decide whether the wearer genuinely enters or re-enters the conversational floor. First inspect the previous wearer offset and silence gap, gap_other_intervals, nearby other activity, transcript, and Target_of_Speech. A short within-turn pause followed by continuation of the same wearer contribution is not a new entry; use EXCLUDE unless the region is a valid named false-positive opportunity. A POSITIVE WEARER_ENTRY does not require a preceding non-wearer release: it may occur after a release, from silence, or while another speaker is active. temporal_evidence.opportunity_options contains nearby preceding other offsets with factual relation flags, not pre-approved causes. If one supplied release clearly prompted the entry, selected_opportunity_uid may contain its uid; otherwise return null. Never invent a trigger or timestamp. A direct, semantically complete answer can be a genuine entry even when short. Do not call it a backchannel merely because it lasts less than one second or because another participant speaks later. For example, "Nope." beginning 0.15 s after an addressed question ends is normally WEARER_ENTRY, not ENTRY_WEARER_BACKCHANNEL. Later conversation does not retroactively cancel a completed entry.
|
||||
6. For every WEARER_OFFSET, answer the central question: did the wearer stop because a non-wearer genuinely took over the conversational floor? Reconstruct the wearer activity-run start and offset, every nearby other onset and offset, which others are active immediately before and after the offset, whether each other is pre-existing or starts during wearer speech, whether it crosses the offset, overlap duration, who speaks next, transcript content, and Target_of_Speech. All temporal booleans are descriptive evidence, not prerequisite labels. A valid INT_YIELD may involve a newly entering or pre-existing competitor if the chronology and semantics establish genuine takeover and wearer relinquishment. Conversely, crossing the offset or satisfying t_w < t_o < t_y never proves takeover by itself. Verify that the relevant non-wearer actually takes and sustains the next floor. A different participant's later substantive turn is not evidence that the proposed competitor took over. Do not label INT_WEARER_HOLDS_FLOOR merely because the wearer speaks again later: if a competitor establishes an intervening substantive floor and the wearer has an intervening silent interval, the later wearer onset is a new WEARER_ENTRY and does not invalidate the earlier INT_YIELD. Natural completion, completion-to-silence, a within-turn wearer pause, or a post-completion other onset that could not have caused the stop is normally EXCLUDE unless the region is a valid named hard-negative opportunity. If the wearer reaches a semantically complete natural ending while overlapping activity is only [L] or feedback and that contributor does not take the next substantive floor, return EXCLUDE.
|
||||
7. Transcript event markers are evidence, not lexical turns. In EasyCom, a transcription consisting of [L] denotes laughter-only vocalization. Do not treat [L] as a sustained floor entry or interruption takeover merely because its activity interval creates an onset, offset, or overlap candidate. Specifically:
|
||||
- wearer [L] during a non-wearer's retained turn is normally HARD_NEGATIVE / ENTRY_WEARER_BACKCHANNEL, not WEARER_ENTRY;
|
||||
- non-wearer [L] overlapping a wearer's retained turn is normally HARD_NEGATIVE / INT_OTHER_BACKCHANNEL, not INT_YIELD;
|
||||
@@ -106,6 +105,19 @@ Top-level verdicts:
|
||||
- HARD_NEGATIVE: a deliberately retained, valid false-positive opportunity matching one of the behavioral counterexamples below; it enters evaluation as a negative.
|
||||
- EXCLUDE: a non-scoring region. This includes clear out-of-scope/non-target opportunities such as natural wearer completion coincident with feedback, as well as insufficient evidence, annotation corruption, a recording boundary, missing context, a non-speech artifact, timestamp corruption, or another unresolvable defect.
|
||||
|
||||
EXCLUDE requires exactly one exclude_reason from this closed taxonomy:
|
||||
- OUT_OF_SCOPE_NORMAL_COMPLETION: the wearer naturally completes; no interruption-yield target occurs.
|
||||
- OUT_OF_SCOPE_WITHIN_TURN_CONTINUATION: a raw boundary is only a pause/resumption within the same wearer contribution.
|
||||
- OUT_OF_SCOPE_POST_COMPLETION_ACTIVITY: another contribution begins only after wearer completion and cannot have caused the boundary.
|
||||
- OUT_OF_SCOPE_NON_EVENT: another clear, interpretable non-target boundary not covered above.
|
||||
- INSUFFICIENT_EVIDENCE: the supplied valid evidence cannot resolve conversational function.
|
||||
- DATA_ANNOTATION_ERROR: participant, speech, transcript, or target annotation is evidently erroneous or contradictory.
|
||||
- DATA_RECORDING_BOUNDARY: the recording/clip boundary prevents reliable interpretation.
|
||||
- DATA_MISSING_CONTEXT: required context is absent from the supplied annotation window or source.
|
||||
- DATA_NON_SPEECH_ARTIFACT: the boundary is caused by an artifact rather than an annotated conversational vocalization.
|
||||
- DATA_TIMESTAMP_CORRUPTION: timestamps are corrupted or irreconcilably misaligned.
|
||||
The OUT_OF_SCOPE_* reasons describe valid data outside the evaluation target. DATA_* reasons describe bad or unusable data. Never substitute one family for the other.
|
||||
|
||||
POSITIVE event_type must be exactly WEARER_ENTRY or INT_YIELD. WEARER_ENTRY means the wearer genuinely enters or re-enters the floor with a contextually complete response or substantive contribution; no prior release is required and the contribution need not be long. INT_YIELD means a non-wearer genuinely takes over during wearer speech and the wearer relinquishes the floor. A later WEARER_ENTRY after an intervening other floor is a new event and must not retroactively invalidate the preceding INT_YIELD.
|
||||
|
||||
HARD_NEGATIVE subtypes are behavioral counterexamples, not data-quality labels:
|
||||
@@ -124,12 +136,12 @@ Subtype precedence when descriptions overlap:
|
||||
4. Use OTHER_HARD_NEGATIVE only after the named subtypes have been ruled out. Never use it merely because confidence is low.
|
||||
|
||||
Return exactly one JSON object with this shape and no markdown:
|
||||
{"items":[{"uid":"...","verdict":"EXCLUDE","event_type":null,"subtype":null,"selected_opportunity_uid":null,"confidence":0.25,"reason":"brief evidence-based sentence"}]}
|
||||
{"items":[{"uid":"...","verdict":"EXCLUDE","event_type":null,"subtype":null,"exclude_reason":"INSUFFICIENT_EVIDENCE","selected_opportunity_uid":null,"confidence":0.25,"reason":"brief evidence-based sentence"}]}
|
||||
Field combinations are strict:
|
||||
- POSITIVE / WEARER_ENTRY: subtype=null; selected_opportunity_uid is one supplied opportunity uid or null.
|
||||
- POSITIVE / INT_YIELD: subtype=null; selected_opportunity_uid=null.
|
||||
- HARD_NEGATIVE: event_type=null; selected_opportunity_uid=null; subtype is one valid subtype from the candidate's ENTRY or INT family.
|
||||
- EXCLUDE: event_type=null; subtype=null; selected_opportunity_uid=null.
|
||||
- POSITIVE / WEARER_ENTRY: subtype=null; exclude_reason=null; selected_opportunity_uid is one supplied opportunity uid or null.
|
||||
- POSITIVE / INT_YIELD: subtype=null; exclude_reason=null; selected_opportunity_uid=null.
|
||||
- HARD_NEGATIVE: event_type=null; exclude_reason=null; selected_opportunity_uid=null; subtype is one valid subtype from the candidate's ENTRY or INT family.
|
||||
- EXCLUDE: event_type=null; subtype=null; selected_opportunity_uid=null; exclude_reason is exactly one taxonomy value above.
|
||||
Return every supplied uid exactly once. Copy uid byte-for-byte and never add a backslash before an underscore. confidence must be a number from 0 to 1."""
|
||||
|
||||
|
||||
@@ -245,6 +257,7 @@ def validate_decisions(
|
||||
verdict = item.get("verdict")
|
||||
event_type = item.get("event_type")
|
||||
subtype = item.get("subtype")
|
||||
exclude_reason = item.get("exclude_reason")
|
||||
selected_opportunity_uid = item.get("selected_opportunity_uid")
|
||||
confidence = item.get("confidence")
|
||||
reason = item.get("reason")
|
||||
@@ -260,6 +273,10 @@ def validate_decisions(
|
||||
raise ValueError(f"HARD_NEGATIVE requires a valid subtype for {uid}")
|
||||
if verdict != "HARD_NEGATIVE" and subtype is not None:
|
||||
raise ValueError(f"Only HARD_NEGATIVE may have subtype for {uid}")
|
||||
if verdict == "EXCLUDE" and exclude_reason not in EXCLUDE_REASONS:
|
||||
raise ValueError(f"EXCLUDE requires a valid exclude_reason for {uid}")
|
||||
if verdict != "EXCLUDE" and exclude_reason is not None:
|
||||
raise ValueError(f"Only EXCLUDE may have exclude_reason for {uid}")
|
||||
family = None if expected_families is None else expected_families.get(str(uid))
|
||||
if verdict == "POSITIVE" and family in {"ENTRY", "INT"}:
|
||||
required_event_type = "WEARER_ENTRY" if family == "ENTRY" else "INT_YIELD"
|
||||
@@ -287,6 +304,8 @@ def validate_decisions(
|
||||
"verdict": str(verdict),
|
||||
"event_type": event_type,
|
||||
"subtype": subtype,
|
||||
"exclude_reason": exclude_reason,
|
||||
"exclude_reason_group": exclude_reason_group(exclude_reason),
|
||||
"selected_opportunity_uid": selected_opportunity_uid,
|
||||
"confidence": float(confidence),
|
||||
"reason": reason.strip(),
|
||||
@@ -437,12 +456,12 @@ def add_suggestions(
|
||||
for source in rows:
|
||||
decisions = by_uid[source["uid"]]
|
||||
labels = {
|
||||
(d["verdict"], d["event_type"], d["subtype"])
|
||||
(d["verdict"], d["event_type"], d["subtype"], d["exclude_reason"])
|
||||
for d in decisions
|
||||
}
|
||||
unanimous = len(labels) == 1
|
||||
if unanimous:
|
||||
verdict, event_type, subtype = next(iter(labels))
|
||||
verdict, event_type, subtype, exclude_reason = next(iter(labels))
|
||||
selected_options = {d["selected_opportunity_uid"] for d in decisions}
|
||||
selected_opportunity_uid = (
|
||||
next(iter(selected_options)) if len(selected_options) == 1 else None
|
||||
@@ -451,7 +470,9 @@ def add_suggestions(
|
||||
reason = " | ".join(item["reason"] for item in decisions)
|
||||
needs_human_review = any(item["needs_human_review"] for item in decisions)
|
||||
else:
|
||||
verdict, event_type, subtype, selected_opportunity_uid = "EXCLUDE", None, None, None
|
||||
verdict, event_type, subtype, exclude_reason, selected_opportunity_uid = (
|
||||
"EXCLUDE", None, None, "INSUFFICIENT_EVIDENCE", None
|
||||
)
|
||||
confidence = min(float(item["confidence"]) for item in decisions)
|
||||
reason = "The independent DeepSeek passes disagreed."
|
||||
needs_human_review = True
|
||||
@@ -474,6 +495,8 @@ def add_suggestions(
|
||||
"verdict": verdict,
|
||||
"event_type": event_type,
|
||||
"subtype": subtype,
|
||||
"exclude_reason": exclude_reason,
|
||||
"exclude_reason_group": exclude_reason_group(exclude_reason),
|
||||
"selected_opportunity_uid": selected_opportunity_uid,
|
||||
"selected_opportunity_seconds": selected_opportunity_seconds,
|
||||
"valid_prediction_interval_seconds": (
|
||||
@@ -490,6 +513,8 @@ def add_suggestions(
|
||||
ai_verdict=verdict,
|
||||
ai_event_type=event_type,
|
||||
ai_subtype=subtype,
|
||||
ai_exclude_reason=exclude_reason,
|
||||
ai_exclude_reason_group=exclude_reason_group(exclude_reason),
|
||||
ai_selected_opportunity_uid=selected_opportunity_uid,
|
||||
ai_selected_opportunity_seconds=selected_opportunity_seconds,
|
||||
ai_confidence=confidence,
|
||||
|
||||
Reference in New Issue
Block a user