Add manuscript-aligned EasyCom EOT INT review workflow
This commit is contained in:
Xin Wang
@@ -0,0 +1,339 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Add conservative DeepSeek suggestions to EOT/INT review candidates.
|
||||
|
||||
Only transcript excerpts and timing metadata are sent. Suggestions are never
|
||||
treated as gold labels: the output remains input to ``review_floor_events.py``
|
||||
and requires an identified human reviewer. The API key is read only from the
|
||||
``DEEPSEEK_API_KEY`` environment variable and is never persisted.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import time
|
||||
from typing import Any, Sequence
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
|
||||
PROMPT_VERSION = "easycom-manuscript-eot-int-adjudicator-v1"
|
||||
DEFAULT_MODEL = "deepseek-v4-flash"
|
||||
DEFAULT_BASE_URL = "https://api.deepseek.com"
|
||||
DISPOSITIONS = {"positive", "hard_negative", "exclude", "uncertain"}
|
||||
HARD_NEGATIVE_TYPES = {
|
||||
"eot_midturn_pause",
|
||||
"eot_other_to_other",
|
||||
"eot_no_response",
|
||||
"int_backchannel",
|
||||
"int_non_takeover_overlap",
|
||||
"int_retain_floor",
|
||||
"other",
|
||||
"not_applicable",
|
||||
"uncertain",
|
||||
}
|
||||
CONFIDENCES = {"high", "medium", "low"}
|
||||
|
||||
SYSTEM_PROMPT = """You audit candidate regions for a scientific study of wearer-relative floor control.
|
||||
Use only the supplied participant-resolved transcript and timing evidence. Do not infer prosody, gaze, intention, or acoustic interruption cues that are not supplied.
|
||||
|
||||
Tasks:
|
||||
- eot_response positive: a non-wearer releases the floor and the wearer makes a genuine floor-taking response.
|
||||
- int_yield positive: a non-wearer takes over during wearer speech and the wearer genuinely yields the floor.
|
||||
|
||||
Hard negatives:
|
||||
- eot_midturn_pause: the same non-wearer resumes after a within-turn pause.
|
||||
- eot_other_to_other: a different non-wearer takes the floor while the wearer remains silent.
|
||||
- eot_no_response: the wearer correctly remains silent for the labeled region.
|
||||
- int_backchannel: brief feedback during wearer speech that does not take the floor.
|
||||
- int_non_takeover_overlap: overlapping other speech without takeover.
|
||||
- int_retain_floor: the wearer correctly retains the floor for the labeled region.
|
||||
|
||||
Use exclude for annotation corruption or an unusable boundary. Use uncertain whenever transcript/timing evidence cannot establish the semantic function. Timing alone does not prove EOT, interruption, backchannel, or takeover.
|
||||
|
||||
Return exactly one JSON object with this shape and no markdown:
|
||||
{"items":[{"uid":"...","protocol_disposition":"uncertain","hard_negative_type":"uncertain","confidence":"low","rationale":"brief evidence-based sentence"}]}
|
||||
Return every supplied uid exactly once."""
|
||||
|
||||
|
||||
def read_candidates(path: Path) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
for line_number, line in enumerate(handle, 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
row = json.loads(line)
|
||||
if not isinstance(row, dict) or not isinstance(row.get("uid"), str):
|
||||
raise ValueError(f"Expected candidate object with uid at {path}:{line_number}")
|
||||
if row.get("task") not in {"eot_response", "int_yield"}:
|
||||
raise ValueError(f"Unsupported task at {path}:{line_number}: {row.get('task')!r}")
|
||||
rows.append(row)
|
||||
if not rows:
|
||||
raise ValueError(f"No candidates in {path}")
|
||||
return rows
|
||||
|
||||
|
||||
def candidate_for_prompt(row: dict[str, Any]) -> dict[str, Any]:
|
||||
fields = (
|
||||
"uid",
|
||||
"task",
|
||||
"automatic_protocol_label",
|
||||
"automatic_hard_negative_type",
|
||||
"anchor_type",
|
||||
"anchor_dense_seconds",
|
||||
"behavior_anchor_type",
|
||||
"behavior_seconds",
|
||||
"region_start_seconds",
|
||||
"region_end_seconds",
|
||||
"region_end_reason",
|
||||
"previous_other_participant_ids",
|
||||
"incoming_other_participant_ids",
|
||||
"automatic_reason",
|
||||
"context",
|
||||
)
|
||||
return {field: row.get(field) for field in fields}
|
||||
|
||||
|
||||
def build_user_prompt(rows: Sequence[dict[str, Any]], pass_index: int) -> str:
|
||||
instruction = (
|
||||
"Primary pass: classify conservatively."
|
||||
if pass_index == 0
|
||||
else "Independent skeptical pass: seek alternative explanations and choose uncertain when semantics are unsupported."
|
||||
)
|
||||
return (
|
||||
instruction
|
||||
+ "\nCandidate JSON:\n"
|
||||
+ json.dumps([candidate_for_prompt(row) for row in rows], ensure_ascii=False, separators=(",", ":"))
|
||||
)
|
||||
|
||||
|
||||
def validate_decisions(payload: Any, expected_uids: Sequence[str]) -> list[dict[str, str]]:
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("items"), list):
|
||||
raise ValueError("Response must be an object with an items list")
|
||||
expected = set(expected_uids)
|
||||
decisions: dict[str, dict[str, str]] = {}
|
||||
for item in payload["items"]:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("Every decision must be an object")
|
||||
uid = item.get("uid")
|
||||
disposition = item.get("protocol_disposition")
|
||||
negative_type = item.get("hard_negative_type")
|
||||
confidence = item.get("confidence")
|
||||
rationale = item.get("rationale")
|
||||
if uid not in expected or uid in decisions:
|
||||
raise ValueError(f"Unexpected or duplicate uid: {uid!r}")
|
||||
if disposition not in DISPOSITIONS:
|
||||
raise ValueError(f"Invalid protocol_disposition for {uid}: {disposition!r}")
|
||||
if negative_type not in HARD_NEGATIVE_TYPES:
|
||||
raise ValueError(f"Invalid hard_negative_type for {uid}: {negative_type!r}")
|
||||
if confidence not in CONFIDENCES:
|
||||
raise ValueError(f"Invalid confidence for {uid}: {confidence!r}")
|
||||
if not isinstance(rationale, str) or not rationale.strip():
|
||||
raise ValueError(f"Missing rationale for {uid}")
|
||||
decisions[str(uid)] = {
|
||||
"uid": str(uid),
|
||||
"protocol_disposition": str(disposition),
|
||||
"hard_negative_type": str(negative_type),
|
||||
"confidence": str(confidence),
|
||||
"rationale": rationale.strip(),
|
||||
}
|
||||
if set(decisions) != expected:
|
||||
raise ValueError(f"Response uid mismatch; missing={sorted(expected - set(decisions))}")
|
||||
return [decisions[uid] for uid in expected_uids]
|
||||
|
||||
|
||||
def _post(
|
||||
*,
|
||||
rows: Sequence[dict[str, Any]],
|
||||
pass_index: int,
|
||||
api_key: str,
|
||||
model: str,
|
||||
base_url: str,
|
||||
timeout_seconds: float,
|
||||
max_retries: int,
|
||||
) -> tuple[list[dict[str, str]], dict[str, Any]]:
|
||||
body = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": build_user_prompt(rows, pass_index)},
|
||||
],
|
||||
"response_format": {"type": "json_object"},
|
||||
"thinking": {"type": "disabled"},
|
||||
"temperature": 0,
|
||||
"max_tokens": 2048,
|
||||
"stream": False,
|
||||
}
|
||||
request = urllib.request.Request(
|
||||
base_url.rstrip("/") + "/chat/completions",
|
||||
data=json.dumps(body, ensure_ascii=False).encode("utf-8"),
|
||||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
|
||||
envelope = json.loads(response.read().decode("utf-8"))
|
||||
content = envelope["choices"][0]["message"]["content"]
|
||||
parsed = json.loads(content)
|
||||
return validate_decisions(parsed, [row["uid"] for row in rows]), envelope.get("usage", {})
|
||||
except (
|
||||
urllib.error.HTTPError,
|
||||
urllib.error.URLError,
|
||||
TimeoutError,
|
||||
KeyError,
|
||||
TypeError,
|
||||
json.JSONDecodeError,
|
||||
ValueError,
|
||||
) as error:
|
||||
last_error = error
|
||||
if attempt >= max_retries:
|
||||
break
|
||||
time.sleep(min(30.0, 2.0**attempt))
|
||||
raise RuntimeError(f"DeepSeek request failed after {max_retries + 1} attempts: {last_error}")
|
||||
|
||||
|
||||
def add_suggestions(
|
||||
rows: Sequence[dict[str, Any]],
|
||||
*,
|
||||
api_key: str,
|
||||
model: str,
|
||||
base_url: str,
|
||||
passes: int,
|
||||
timeout_seconds: float,
|
||||
max_retries: int,
|
||||
) -> tuple[list[dict[str, Any]], Counter[str]]:
|
||||
by_uid: dict[str, list[dict[str, str]]] = {row["uid"]: [] for row in rows}
|
||||
usage: Counter[str] = Counter()
|
||||
for pass_index in range(passes):
|
||||
decisions, pass_usage = _post(
|
||||
rows=rows,
|
||||
pass_index=pass_index,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
timeout_seconds=timeout_seconds,
|
||||
max_retries=max_retries,
|
||||
)
|
||||
for decision in decisions:
|
||||
by_uid[decision["uid"]].append(decision)
|
||||
for key, value in pass_usage.items():
|
||||
if isinstance(value, int):
|
||||
usage[key] += value
|
||||
|
||||
output: list[dict[str, Any]] = []
|
||||
for source in rows:
|
||||
decisions = by_uid[source["uid"]]
|
||||
labels = {
|
||||
(decision["protocol_disposition"], decision["hard_negative_type"])
|
||||
for decision in decisions
|
||||
}
|
||||
unanimous = len(labels) == 1
|
||||
if unanimous:
|
||||
disposition, negative_type = next(iter(labels))
|
||||
confidence = (
|
||||
"high" if all(item["confidence"] == "high" for item in decisions) else "medium"
|
||||
)
|
||||
rationale = " | ".join(item["rationale"] for item in decisions)
|
||||
else:
|
||||
disposition = "uncertain"
|
||||
negative_type = "uncertain"
|
||||
confidence = "low"
|
||||
rationale = "The independent DeepSeek passes disagreed."
|
||||
prompt_payload = {
|
||||
"prompt_version": PROMPT_VERSION,
|
||||
"model": model,
|
||||
"passes": passes,
|
||||
"candidate": candidate_for_prompt(source),
|
||||
}
|
||||
enriched = dict(source)
|
||||
enriched["llm_suggestion"] = {
|
||||
"protocol_disposition": disposition,
|
||||
"hard_negative_type": negative_type,
|
||||
"confidence": confidence,
|
||||
"rationale": rationale,
|
||||
"unanimous": unanimous,
|
||||
}
|
||||
enriched["llm_adjudication"] = {
|
||||
"model": model,
|
||||
"prompt_version": PROMPT_VERSION,
|
||||
"passes": passes,
|
||||
"prompt_sha256": hashlib.sha256(
|
||||
json.dumps(prompt_payload, sort_keys=True, ensure_ascii=False).encode("utf-8")
|
||||
).hexdigest(),
|
||||
"decisions": decisions,
|
||||
"human_review_required": True,
|
||||
}
|
||||
output.append(enriched)
|
||||
return output, usage
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--input", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--model", default=os.environ.get("DEEPSEEK_MODEL", DEFAULT_MODEL))
|
||||
parser.add_argument("--base-url", default=os.environ.get("DEEPSEEK_BASE_URL", DEFAULT_BASE_URL))
|
||||
parser.add_argument("--passes", type=int, default=2)
|
||||
parser.add_argument("--max-events", type=int)
|
||||
parser.add_argument("--selection-seed", type=int, default=2027)
|
||||
parser.add_argument("--timeout-seconds", type=float, default=120.0)
|
||||
parser.add_argument("--max-retries", type=int, default=3)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument("--allow-external-transcript-upload", action="store_true")
|
||||
parser.add_argument("--overwrite", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
if args.passes <= 0 or args.max_retries < 0:
|
||||
raise ValueError("passes must be positive and max-retries non-negative")
|
||||
rows = read_candidates(args.input.expanduser().resolve())
|
||||
if args.max_events is not None:
|
||||
if args.max_events < 0:
|
||||
raise ValueError("--max-events must be non-negative")
|
||||
rng = random.Random(args.selection_seed)
|
||||
rows = rng.sample(rows, min(args.max_events, len(rows)))
|
||||
rows.sort(key=lambda row: row["uid"])
|
||||
print(f"Candidates={len(rows)} model={args.model} passes={args.passes}")
|
||||
if args.dry_run:
|
||||
print("DRY RUN: no API request will be made.\n")
|
||||
print(build_user_prompt(rows, 0))
|
||||
return
|
||||
if not args.allow_external_transcript_upload:
|
||||
raise PermissionError(
|
||||
"Refusing to upload transcript excerpts; inspect --dry-run, then pass "
|
||||
"--allow-external-transcript-upload when authorized."
|
||||
)
|
||||
api_key = os.environ.get("DEEPSEEK_API_KEY")
|
||||
if not api_key:
|
||||
raise RuntimeError("Set DEEPSEEK_API_KEY in the environment")
|
||||
output = args.output.expanduser().resolve()
|
||||
if output.exists() and not args.overwrite:
|
||||
raise FileExistsError(f"Refusing to overwrite {output}; pass --overwrite")
|
||||
enriched, usage = add_suggestions(
|
||||
rows,
|
||||
api_key=api_key,
|
||||
model=args.model,
|
||||
base_url=args.base_url,
|
||||
passes=args.passes,
|
||||
timeout_seconds=args.timeout_seconds,
|
||||
max_retries=args.max_retries,
|
||||
)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with output.open("w", encoding="utf-8") as handle:
|
||||
for row in enriched:
|
||||
handle.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||
print(f"Saved {len(enriched)} suggested candidates to {output}")
|
||||
print("Usage:", dict(usage))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user