wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revise wearer entry and yield annotation protocol

Browse Source
This commit is contained in:
Xin Wang
2026-09-15 01:25:39 +08:00
parent 026d89978f
commit 71dfd8808a
12 changed files with 478 additions and 248 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test_adjudicate_easycom_eot_int_candidates.py
+81 -11
View File
@@ -1,4 +1,4 @@
"""Tests for EOT/INT DeepSeek suggestion validation without network calls."""
"""Tests for wearer-entry/INT DeepSeek suggestion validation without network calls."""
from __future__ import annotations
@@ -13,7 +13,7 @@ from adjudicate_easycom_eot_int_candidates import (
)
class EOTINTAdjudicationTests(unittest.TestCase):
class EntryINTAdjudicationTests(unittest.TestCase):
def test_progress_line_reports_items_percent_pass_and_elapsed_time(self) -> None:
line = _progress_line(25, 100, label="pass 1/2 · batch 4/13", elapsed_seconds=65)
self.assertIn("25/100 ( 25.0%)", line)
@@ -22,7 +22,7 @@ class EOTINTAdjudicationTests(unittest.TestCase):
def test_prompt_contains_task_and_context(self) -> None:
prompt = build_user_prompt(
[{"uid": "u1", "candidate_source": "WEARER_ONSET", "task": "eot_response",
[{"uid": "u1", "candidate_source": "WEARER_ONSET", "task": "wearer_entry",
"temporal_evidence": {"wearer_duration": 2.0},
"context": [{"transcription": "Yes."}]}],
0,
@@ -32,13 +32,26 @@ class EOTINTAdjudicationTests(unittest.TestCase):
self.assertIn("Yes.", prompt)
def test_system_prompt_treats_laughter_as_nonlexical_floor_feedback(self) -> None:
self.assertEqual(PROMPT_VERSION, "easycom-wearer-anchor-first-eot-int-prelabel-v7")
self.assertEqual(PROMPT_VERSION, "easycom-wearer-entry-int-prelabel-v12")
self.assertIn("opportunity_options", SYSTEM_PROMPT)
self.assertIn("t_w < t_o < t_y", SYSTEM_PROMPT)
self.assertIn("a transcription consisting of [L] denotes laughter-only vocalization", SYSTEM_PROMPT)
self.assertIn("EOT_WEARER_BACKCHANNEL", SYSTEM_PROMPT)
self.assertIn("ENTRY_WEARER_BACKCHANNEL", SYSTEM_PROMPT)
self.assertIn("INT_OTHER_BACKCHANNEL", SYSTEM_PROMPT)
self.assertIn("Subtype precedence", SYSTEM_PROMPT)
self.assertIn("same non-wearer resumes", SYSTEM_PROMPT)
self.assertIn("different non-wearer takes", SYSTEM_PROMPT)
self.assertIn("This is not an uncertainty fallback", SYSTEM_PROMPT)
self.assertIn('"Nope." beginning 0.15 s after an addressed question ends', SYSTEM_PROMPT)
self.assertIn("does not require a preceding non-wearer release", SYSTEM_PROMPT)
self.assertIn("duration alone never determines floor function", SYSTEM_PROMPT)
self.assertIn("out-of-scope normal completion", SYSTEM_PROMPT)
self.assertIn("same incoming non-wearer actually takes", SYSTEM_PROMPT)
self.assertIn("must not retroactively invalidate the preceding INT_YIELD", SYSTEM_PROMPT)
self.assertIn("do not infer that a visible smile occurred", SYSTEM_PROMPT)
self.assertIn("POSITIVE / INT_YIELD: subtype=null; selected_opportunity_uid=null", SYSTEM_PROMPT)
self.assertNotIn("NOT_APPLICABLE", SYSTEM_PROMPT)
self.assertNotIn("UNSURE", SYSTEM_PROMPT)
self.assertNotIn('"needs_human_review"', SYSTEM_PROMPT)
def test_decisions_require_exact_uid_and_vocabulary(self) -> None:
@@ -64,7 +77,7 @@ class EOTINTAdjudicationTests(unittest.TestCase):
def test_model_needs_human_review_type_drift_is_ignored(self) -> None:
payload = {
"items": [{
"uid": "u1", "verdict": "UNSURE", "event_type": None,
"uid": "u1", "verdict": "EXCLUDE", "event_type": None,
"subtype": None, "confidence": 0.4, "reason": "Ambiguous context.",
"needs_human_review": "true",
}]
@@ -72,9 +85,18 @@ class EOTINTAdjudicationTests(unittest.TestCase):
decision = validate_decisions(payload, ["u1"])[0]
self.assertTrue(decision["needs_human_review"])
def test_eot_positive_must_select_one_supplied_release(self) -> None:
def test_removed_fallback_verdicts_are_rejected(self) -> None:
for verdict in ("UNSURE", "NOT_APPLICABLE"):
payload = {"items": [{
"uid": "u1", "verdict": verdict, "event_type": None,
"subtype": None, "confidence": 0.4, "reason": "Ambiguous context.",
}]}
with self.subTest(verdict=verdict), self.assertRaisesRegex(ValueError, "Invalid verdict"):
validate_decisions(payload, ["u1"])
def test_entry_positive_may_select_only_a_supplied_release(self) -> None:
item = {
"uid": "u1", "verdict": "POSITIVE", "event_type": "EOT_RESPONSE",
"uid": "u1", "verdict": "POSITIVE", "event_type": "WEARER_ENTRY",
"subtype": None, "selected_opportunity_uid": "release-2",
"confidence": 0.9, "reason": "The wearer answers the second release.",
}
@@ -82,18 +104,66 @@ class EOTINTAdjudicationTests(unittest.TestCase):
{"items": [item]}, ["u1"], {"u1": {"release-1", "release-2"}}
)[0]
self.assertEqual(decision["selected_opportunity_uid"], "release-2")
with self.assertRaisesRegex(ValueError, "select a supplied opportunity"):
with self.assertRaisesRegex(ValueError, "only a supplied opportunity"):
validate_decisions(
{"items": [{**item, "selected_opportunity_uid": "invented"}]},
["u1"],
{"u1": {"release-1", "release-2"}},
)
optional = validate_decisions(
{"items": [{**item, "selected_opportunity_uid": None}]},
["u1"], {"u1": {"release-1", "release-2"}},
)[0]
self.assertIsNone(optional["selected_opportunity_uid"])
def test_irrelevant_selected_opportunity_is_cleared_without_retrying_label(self) -> None:
for verdict, event_type, subtype in (
("POSITIVE", "INT_YIELD", None),
("HARD_NEGATIVE", None, "INT_OTHER_BACKCHANNEL"),
("EXCLUDE", None, None),
):
with self.subTest(verdict=verdict):
item = {
"uid": "u1",
"verdict": verdict,
"event_type": event_type,
"subtype": subtype,
"selected_opportunity_uid": "copied-release",
"confidence": 0.8,
"reason": "Evidence-based decision.",
}
decision = validate_decisions(
{"items": [item]}, ["u1"],
{"u1": {"copied-release"}},
{"u1": "INT"},
)[0]
self.assertIsNone(decision["selected_opportunity_uid"])
def test_event_and_hard_negative_subtypes_match_candidate_family(self) -> None:
positive = {
"uid": "u1", "verdict": "POSITIVE", "event_type": "INT_YIELD",
"subtype": None, "selected_opportunity_uid": None,
"confidence": 0.8, "reason": "Wrong family.",
}
with self.assertRaisesRegex(ValueError, "ENTRY candidate requires WEARER_ENTRY"):
validate_decisions(
{"items": [positive]}, ["u1"], expected_families={"u1": "ENTRY"}
)
hard_negative = {
"uid": "u1", "verdict": "HARD_NEGATIVE", "event_type": None,
"subtype": "ENTRY_OTHER_MIDTURN_PAUSE", "selected_opportunity_uid": None,
"confidence": 0.8, "reason": "Wrong family.",
}
with self.assertRaisesRegex(ValueError, "INT candidate requires an INT_"):
validate_decisions(
{"items": [hard_negative]}, ["u1"], expected_families={"u1": "INT"}
)
def test_markdown_escaped_uid_is_normalized_but_still_unique(self) -> None:
item = {
"uid": "Session\\_4/wearer\\_onset/00000024",
"verdict": "POSITIVE",
"event_type": "EOT_RESPONSE",
"event_type": "WEARER_ENTRY",
"subtype": None,
"confidence": 0.9,
"reason": "Sustained wearer response after the other speaker stops.",
@@ -108,7 +178,7 @@ class EOTINTAdjudicationTests(unittest.TestCase):
def test_duplicate_rule_uids_are_disambiguated_with_provenance(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
path = Path(temporary) / "candidates.jsonl"
base = {"uid": "Session_4/other_to_other/00003600", "task": "eot_response"}
base = {"uid": "Session_4/other_to_other/00003600", "task": "wearer_entry"}
rows = [
{**base, "previous_other_participant_ids": [6], "incoming_other_participant_ids": [4]},
{**base, "previous_other_participant_ids": [3], "incoming_other_participant_ids": [6]},