340 lines
13 KiB
Python
340 lines
13 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Add conservative DeepSeek suggestions to EOT/INT review candidates.
|
||
|
|
|
||
|
|
Only transcript excerpts and timing metadata are sent. Suggestions are never
|
||
|
|
treated as gold labels: the output remains input to ``review_floor_events.py``
|
||
|
|
and requires an identified human reviewer. The API key is read only from the
|
||
|
|
``DEEPSEEK_API_KEY`` environment variable and is never persisted.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
from collections import Counter
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
from pathlib import Path
|
||
|
|
import random
|
||
|
|
import time
|
||
|
|
from typing import Any, Sequence
|
||
|
|
import urllib.error
|
||
|
|
import urllib.request
|
||
|
|
|
||
|
|
|
||
|
|
PROMPT_VERSION = "easycom-manuscript-eot-int-adjudicator-v1"
|
||
|
|
DEFAULT_MODEL = "deepseek-v4-flash"
|
||
|
|
DEFAULT_BASE_URL = "https://api.deepseek.com"
|
||
|
|
DISPOSITIONS = {"positive", "hard_negative", "exclude", "uncertain"}
|
||
|
|
HARD_NEGATIVE_TYPES = {
|
||
|
|
"eot_midturn_pause",
|
||
|
|
"eot_other_to_other",
|
||
|
|
"eot_no_response",
|
||
|
|
"int_backchannel",
|
||
|
|
"int_non_takeover_overlap",
|
||
|
|
"int_retain_floor",
|
||
|
|
"other",
|
||
|
|
"not_applicable",
|
||
|
|
"uncertain",
|
||
|
|
}
|
||
|
|
CONFIDENCES = {"high", "medium", "low"}
|
||
|
|
|
||
|
|
SYSTEM_PROMPT = """You audit candidate regions for a scientific study of wearer-relative floor control.
|
||
|
|
Use only the supplied participant-resolved transcript and timing evidence. Do not infer prosody, gaze, intention, or acoustic interruption cues that are not supplied.
|
||
|
|
|
||
|
|
Tasks:
|
||
|
|
- eot_response positive: a non-wearer releases the floor and the wearer makes a genuine floor-taking response.
|
||
|
|
- int_yield positive: a non-wearer takes over during wearer speech and the wearer genuinely yields the floor.
|
||
|
|
|
||
|
|
Hard negatives:
|
||
|
|
- eot_midturn_pause: the same non-wearer resumes after a within-turn pause.
|
||
|
|
- eot_other_to_other: a different non-wearer takes the floor while the wearer remains silent.
|
||
|
|
- eot_no_response: the wearer correctly remains silent for the labeled region.
|
||
|
|
- int_backchannel: brief feedback during wearer speech that does not take the floor.
|
||
|
|
- int_non_takeover_overlap: overlapping other speech without takeover.
|
||
|
|
- int_retain_floor: the wearer correctly retains the floor for the labeled region.
|
||
|
|
|
||
|
|
Use exclude for annotation corruption or an unusable boundary. Use uncertain whenever transcript/timing evidence cannot establish the semantic function. Timing alone does not prove EOT, interruption, backchannel, or takeover.
|
||
|
|
|
||
|
|
Return exactly one JSON object with this shape and no markdown:
|
||
|
|
{"items":[{"uid":"...","protocol_disposition":"uncertain","hard_negative_type":"uncertain","confidence":"low","rationale":"brief evidence-based sentence"}]}
|
||
|
|
Return every supplied uid exactly once."""
|
||
|
|
|
||
|
|
|
||
|
|
def read_candidates(path: Path) -> list[dict[str, Any]]:
|
||
|
|
rows: list[dict[str, Any]] = []
|
||
|
|
with path.open("r", encoding="utf-8") as handle:
|
||
|
|
for line_number, line in enumerate(handle, 1):
|
||
|
|
if not line.strip():
|
||
|
|
continue
|
||
|
|
row = json.loads(line)
|
||
|
|
if not isinstance(row, dict) or not isinstance(row.get("uid"), str):
|
||
|
|
raise ValueError(f"Expected candidate object with uid at {path}:{line_number}")
|
||
|
|
if row.get("task") not in {"eot_response", "int_yield"}:
|
||
|
|
raise ValueError(f"Unsupported task at {path}:{line_number}: {row.get('task')!r}")
|
||
|
|
rows.append(row)
|
||
|
|
if not rows:
|
||
|
|
raise ValueError(f"No candidates in {path}")
|
||
|
|
return rows
|
||
|
|
|
||
|
|
|
||
|
|
def candidate_for_prompt(row: dict[str, Any]) -> dict[str, Any]:
|
||
|
|
fields = (
|
||
|
|
"uid",
|
||
|
|
"task",
|
||
|
|
"automatic_protocol_label",
|
||
|
|
"automatic_hard_negative_type",
|
||
|
|
"anchor_type",
|
||
|
|
"anchor_dense_seconds",
|
||
|
|
"behavior_anchor_type",
|
||
|
|
"behavior_seconds",
|
||
|
|
"region_start_seconds",
|
||
|
|
"region_end_seconds",
|
||
|
|
"region_end_reason",
|
||
|
|
"previous_other_participant_ids",
|
||
|
|
"incoming_other_participant_ids",
|
||
|
|
"automatic_reason",
|
||
|
|
"context",
|
||
|
|
)
|
||
|
|
return {field: row.get(field) for field in fields}
|
||
|
|
|
||
|
|
|
||
|
|
def build_user_prompt(rows: Sequence[dict[str, Any]], pass_index: int) -> str:
|
||
|
|
instruction = (
|
||
|
|
"Primary pass: classify conservatively."
|
||
|
|
if pass_index == 0
|
||
|
|
else "Independent skeptical pass: seek alternative explanations and choose uncertain when semantics are unsupported."
|
||
|
|
)
|
||
|
|
return (
|
||
|
|
instruction
|
||
|
|
+ "\nCandidate JSON:\n"
|
||
|
|
+ json.dumps([candidate_for_prompt(row) for row in rows], ensure_ascii=False, separators=(",", ":"))
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def validate_decisions(payload: Any, expected_uids: Sequence[str]) -> list[dict[str, str]]:
|
||
|
|
if not isinstance(payload, dict) or not isinstance(payload.get("items"), list):
|
||
|
|
raise ValueError("Response must be an object with an items list")
|
||
|
|
expected = set(expected_uids)
|
||
|
|
decisions: dict[str, dict[str, str]] = {}
|
||
|
|
for item in payload["items"]:
|
||
|
|
if not isinstance(item, dict):
|
||
|
|
raise ValueError("Every decision must be an object")
|
||
|
|
uid = item.get("uid")
|
||
|
|
disposition = item.get("protocol_disposition")
|
||
|
|
negative_type = item.get("hard_negative_type")
|
||
|
|
confidence = item.get("confidence")
|
||
|
|
rationale = item.get("rationale")
|
||
|
|
if uid not in expected or uid in decisions:
|
||
|
|
raise ValueError(f"Unexpected or duplicate uid: {uid!r}")
|
||
|
|
if disposition not in DISPOSITIONS:
|
||
|
|
raise ValueError(f"Invalid protocol_disposition for {uid}: {disposition!r}")
|
||
|
|
if negative_type not in HARD_NEGATIVE_TYPES:
|
||
|
|
raise ValueError(f"Invalid hard_negative_type for {uid}: {negative_type!r}")
|
||
|
|
if confidence not in CONFIDENCES:
|
||
|
|
raise ValueError(f"Invalid confidence for {uid}: {confidence!r}")
|
||
|
|
if not isinstance(rationale, str) or not rationale.strip():
|
||
|
|
raise ValueError(f"Missing rationale for {uid}")
|
||
|
|
decisions[str(uid)] = {
|
||
|
|
"uid": str(uid),
|
||
|
|
"protocol_disposition": str(disposition),
|
||
|
|
"hard_negative_type": str(negative_type),
|
||
|
|
"confidence": str(confidence),
|
||
|
|
"rationale": rationale.strip(),
|
||
|
|
}
|
||
|
|
if set(decisions) != expected:
|
||
|
|
raise ValueError(f"Response uid mismatch; missing={sorted(expected - set(decisions))}")
|
||
|
|
return [decisions[uid] for uid in expected_uids]
|
||
|
|
|
||
|
|
|
||
|
|
def _post(
|
||
|
|
*,
|
||
|
|
rows: Sequence[dict[str, Any]],
|
||
|
|
pass_index: int,
|
||
|
|
api_key: str,
|
||
|
|
model: str,
|
||
|
|
base_url: str,
|
||
|
|
timeout_seconds: float,
|
||
|
|
max_retries: int,
|
||
|
|
) -> tuple[list[dict[str, str]], dict[str, Any]]:
|
||
|
|
body = {
|
||
|
|
"model": model,
|
||
|
|
"messages": [
|
||
|
|
{"role": "system", "content": SYSTEM_PROMPT},
|
||
|
|
{"role": "user", "content": build_user_prompt(rows, pass_index)},
|
||
|
|
],
|
||
|
|
"response_format": {"type": "json_object"},
|
||
|
|
"thinking": {"type": "disabled"},
|
||
|
|
"temperature": 0,
|
||
|
|
"max_tokens": 2048,
|
||
|
|
"stream": False,
|
||
|
|
}
|
||
|
|
request = urllib.request.Request(
|
||
|
|
base_url.rstrip("/") + "/chat/completions",
|
||
|
|
data=json.dumps(body, ensure_ascii=False).encode("utf-8"),
|
||
|
|
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||
|
|
method="POST",
|
||
|
|
)
|
||
|
|
last_error: Exception | None = None
|
||
|
|
for attempt in range(max_retries + 1):
|
||
|
|
try:
|
||
|
|
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
|
||
|
|
envelope = json.loads(response.read().decode("utf-8"))
|
||
|
|
content = envelope["choices"][0]["message"]["content"]
|
||
|
|
parsed = json.loads(content)
|
||
|
|
return validate_decisions(parsed, [row["uid"] for row in rows]), envelope.get("usage", {})
|
||
|
|
except (
|
||
|
|
urllib.error.HTTPError,
|
||
|
|
urllib.error.URLError,
|
||
|
|
TimeoutError,
|
||
|
|
KeyError,
|
||
|
|
TypeError,
|
||
|
|
json.JSONDecodeError,
|
||
|
|
ValueError,
|
||
|
|
) as error:
|
||
|
|
last_error = error
|
||
|
|
if attempt >= max_retries:
|
||
|
|
break
|
||
|
|
time.sleep(min(30.0, 2.0**attempt))
|
||
|
|
raise RuntimeError(f"DeepSeek request failed after {max_retries + 1} attempts: {last_error}")
|
||
|
|
|
||
|
|
|
||
|
|
def add_suggestions(
|
||
|
|
rows: Sequence[dict[str, Any]],
|
||
|
|
*,
|
||
|
|
api_key: str,
|
||
|
|
model: str,
|
||
|
|
base_url: str,
|
||
|
|
passes: int,
|
||
|
|
timeout_seconds: float,
|
||
|
|
max_retries: int,
|
||
|
|
) -> tuple[list[dict[str, Any]], Counter[str]]:
|
||
|
|
by_uid: dict[str, list[dict[str, str]]] = {row["uid"]: [] for row in rows}
|
||
|
|
usage: Counter[str] = Counter()
|
||
|
|
for pass_index in range(passes):
|
||
|
|
decisions, pass_usage = _post(
|
||
|
|
rows=rows,
|
||
|
|
pass_index=pass_index,
|
||
|
|
api_key=api_key,
|
||
|
|
model=model,
|
||
|
|
base_url=base_url,
|
||
|
|
timeout_seconds=timeout_seconds,
|
||
|
|
max_retries=max_retries,
|
||
|
|
)
|
||
|
|
for decision in decisions:
|
||
|
|
by_uid[decision["uid"]].append(decision)
|
||
|
|
for key, value in pass_usage.items():
|
||
|
|
if isinstance(value, int):
|
||
|
|
usage[key] += value
|
||
|
|
|
||
|
|
output: list[dict[str, Any]] = []
|
||
|
|
for source in rows:
|
||
|
|
decisions = by_uid[source["uid"]]
|
||
|
|
labels = {
|
||
|
|
(decision["protocol_disposition"], decision["hard_negative_type"])
|
||
|
|
for decision in decisions
|
||
|
|
}
|
||
|
|
unanimous = len(labels) == 1
|
||
|
|
if unanimous:
|
||
|
|
disposition, negative_type = next(iter(labels))
|
||
|
|
confidence = (
|
||
|
|
"high" if all(item["confidence"] == "high" for item in decisions) else "medium"
|
||
|
|
)
|
||
|
|
rationale = " | ".join(item["rationale"] for item in decisions)
|
||
|
|
else:
|
||
|
|
disposition = "uncertain"
|
||
|
|
negative_type = "uncertain"
|
||
|
|
confidence = "low"
|
||
|
|
rationale = "The independent DeepSeek passes disagreed."
|
||
|
|
prompt_payload = {
|
||
|
|
"prompt_version": PROMPT_VERSION,
|
||
|
|
"model": model,
|
||
|
|
"passes": passes,
|
||
|
|
"candidate": candidate_for_prompt(source),
|
||
|
|
}
|
||
|
|
enriched = dict(source)
|
||
|
|
enriched["llm_suggestion"] = {
|
||
|
|
"protocol_disposition": disposition,
|
||
|
|
"hard_negative_type": negative_type,
|
||
|
|
"confidence": confidence,
|
||
|
|
"rationale": rationale,
|
||
|
|
"unanimous": unanimous,
|
||
|
|
}
|
||
|
|
enriched["llm_adjudication"] = {
|
||
|
|
"model": model,
|
||
|
|
"prompt_version": PROMPT_VERSION,
|
||
|
|
"passes": passes,
|
||
|
|
"prompt_sha256": hashlib.sha256(
|
||
|
|
json.dumps(prompt_payload, sort_keys=True, ensure_ascii=False).encode("utf-8")
|
||
|
|
).hexdigest(),
|
||
|
|
"decisions": decisions,
|
||
|
|
"human_review_required": True,
|
||
|
|
}
|
||
|
|
output.append(enriched)
|
||
|
|
return output, usage
|
||
|
|
|
||
|
|
|
||
|
|
def parse_args() -> argparse.Namespace:
|
||
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
||
|
|
parser.add_argument("--input", type=Path, required=True)
|
||
|
|
parser.add_argument("--output", type=Path, required=True)
|
||
|
|
parser.add_argument("--model", default=os.environ.get("DEEPSEEK_MODEL", DEFAULT_MODEL))
|
||
|
|
parser.add_argument("--base-url", default=os.environ.get("DEEPSEEK_BASE_URL", DEFAULT_BASE_URL))
|
||
|
|
parser.add_argument("--passes", type=int, default=2)
|
||
|
|
parser.add_argument("--max-events", type=int)
|
||
|
|
parser.add_argument("--selection-seed", type=int, default=2027)
|
||
|
|
parser.add_argument("--timeout-seconds", type=float, default=120.0)
|
||
|
|
parser.add_argument("--max-retries", type=int, default=3)
|
||
|
|
parser.add_argument("--dry-run", action="store_true")
|
||
|
|
parser.add_argument("--allow-external-transcript-upload", action="store_true")
|
||
|
|
parser.add_argument("--overwrite", action="store_true")
|
||
|
|
return parser.parse_args()
|
||
|
|
|
||
|
|
|
||
|
|
def main() -> None:
|
||
|
|
args = parse_args()
|
||
|
|
if args.passes <= 0 or args.max_retries < 0:
|
||
|
|
raise ValueError("passes must be positive and max-retries non-negative")
|
||
|
|
rows = read_candidates(args.input.expanduser().resolve())
|
||
|
|
if args.max_events is not None:
|
||
|
|
if args.max_events < 0:
|
||
|
|
raise ValueError("--max-events must be non-negative")
|
||
|
|
rng = random.Random(args.selection_seed)
|
||
|
|
rows = rng.sample(rows, min(args.max_events, len(rows)))
|
||
|
|
rows.sort(key=lambda row: row["uid"])
|
||
|
|
print(f"Candidates={len(rows)} model={args.model} passes={args.passes}")
|
||
|
|
if args.dry_run:
|
||
|
|
print("DRY RUN: no API request will be made.\n")
|
||
|
|
print(build_user_prompt(rows, 0))
|
||
|
|
return
|
||
|
|
if not args.allow_external_transcript_upload:
|
||
|
|
raise PermissionError(
|
||
|
|
"Refusing to upload transcript excerpts; inspect --dry-run, then pass "
|
||
|
|
"--allow-external-transcript-upload when authorized."
|
||
|
|
)
|
||
|
|
api_key = os.environ.get("DEEPSEEK_API_KEY")
|
||
|
|
if not api_key:
|
||
|
|
raise RuntimeError("Set DEEPSEEK_API_KEY in the environment")
|
||
|
|
output = args.output.expanduser().resolve()
|
||
|
|
if output.exists() and not args.overwrite:
|
||
|
|
raise FileExistsError(f"Refusing to overwrite {output}; pass --overwrite")
|
||
|
|
enriched, usage = add_suggestions(
|
||
|
|
rows,
|
||
|
|
api_key=api_key,
|
||
|
|
model=args.model,
|
||
|
|
base_url=args.base_url,
|
||
|
|
passes=args.passes,
|
||
|
|
timeout_seconds=args.timeout_seconds,
|
||
|
|
max_retries=args.max_retries,
|
||
|
|
)
|
||
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
with output.open("w", encoding="utf-8") as handle:
|
||
|
|
for row in enriched:
|
||
|
|
handle.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||
|
|
print(f"Saved {len(enriched)} suggested candidates to {output}")
|
||
|
|
print("Usage:", dict(usage))
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|