icassp2027-experiment/materialize_provisional_llm_reviews.py

#!/usr/bin/env python3
"""Materialize unanimous LLM pre-labels as explicitly provisional review rows.

The output is schema-compatible with ``review_floor_events.py`` and
``evaluate_annotated_floor_events.py``, but it is not evidence of human review.
Every row carries machine-readable provenance stating that no human reviewed
the label.  This command is intended for pipeline smoke tests and preliminary
analysis only.
"""

from __future__ import annotations

import argparse
from collections import Counter
from datetime import datetime, timezone
import json
from pathlib import Path
from typing import Any, Iterable

from review_floor_events import (
    REVIEW_SCHEMA_VERSION,
    ReviewStore,
    expand_candidate_paths,
    load_candidates,
)


LABEL_SOURCE = "PROVISIONAL_LLM_AUTO_ACCEPT"


def _confidence(value: Any) -> str:
    if isinstance(value, bool) or not isinstance(value, (int, float)):
        return "low"
    if value >= 0.8:
        return "high"
    if value >= 0.5:
        return "medium"
    return "low"


def materialize(
    event_paths: Iterable[Path],
    output_path: Path,
    *,
    include_nonunanimous: bool = False,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
    candidate_paths = expand_candidate_paths(event_paths, output_path=output_path)
    candidates = load_candidates(candidate_paths)
    store = ReviewStore(output_path, candidates)
    records: list[dict[str, Any]] = []
    skipped: Counter[str] = Counter()
    counts: Counter[str] = Counter()
    timestamp = datetime.now(timezone.utc).isoformat()

    for candidate in candidates:
        suggestion = candidate.payload.get("llm_suggestion")
        if not isinstance(suggestion, dict) or not suggestion.get("verdict"):
            if candidate.payload.get("llm_adjudication_failure"):
                skipped["pending_llm_failure"] += 1
            else:
                skipped["missing_llm_suggestion"] += 1
            continue
        if suggestion.get("unanimous") is not True and not include_nonunanimous:
            skipped["nonunanimous"] += 1
            continue

        review = store._validated_review(
            candidate.candidate_id,
            {
                "human_action": "accept",
                "confidence": _confidence(suggestion.get("confidence")),
                "boundary_correction_seconds": 0,
                "reviewer": "",
                "notes": (
                    "Automatically accepted from an LLM pre-label for a provisional "
                    "pipeline smoke test; no human reviewed this item."
                ),
            },
        )
        review["review_source"] = LABEL_SOURCE
        review["human_reviewed"] = False
        record = {
            "schema_version": REVIEW_SCHEMA_VERSION,
            "candidate_id": candidate.candidate_id,
            "candidate_uid": candidate.uid,
            "source_file": candidate.source_path.name,
            "source_row": candidate.source_row,
            "session": candidate.session,
            "split": candidate.split,
            "candidate_family": candidate.family,
            "task": candidate.task,
            "automatic_label": candidate.automatic_label,
            "automatic_protocol_label": candidate.payload.get("automatic_protocol_label"),
            "automatic_hard_negative_type": candidate.payload.get(
                "automatic_hard_negative_type"
            ),
            "ai_verdict": candidate.payload.get("ai_verdict"),
            "ai_event_type": candidate.payload.get("ai_event_type"),
            "ai_subtype": candidate.payload.get("ai_subtype"),
            "ai_exclude_reason": candidate.payload.get("ai_exclude_reason"),
            "ai_exclude_reason_group": candidate.payload.get("ai_exclude_reason_group"),
            "review_source": LABEL_SOURCE,
            "human_reviewed": False,
            "review": review,
            "updated_at": timestamp,
            "candidate": candidate.payload,
        }
        records.append(record)
        counts[f"session/{candidate.session}"] += 1
        counts[f"family/{candidate.family}"] += 1
        counts[f"verdict/{review['human_verdict']}"] += 1
        if review.get("human_subtype"):
            counts[f"subtype/{review['human_subtype']}"] += 1
        if review.get("exclude_reason_group"):
            counts[f"exclude_group/{review['exclude_reason_group']}"] += 1

    store.reviews = {str(row["candidate_id"]): row for row in records}
    store._write_locked()
    summary = {
        "schema_version": 1,
        "label_source": LABEL_SOURCE,
        "human_reviewed": False,
        "warning": "Provisional LLM auto-accepts are not human-adjudicated ground truth.",
        "input_files": [str(path) for path in candidate_paths],
        "output": str(output_path),
        "num_input_candidates": len(candidates),
        "num_materialized": len(records),
        "num_skipped": sum(skipped.values()),
        "skipped": dict(sorted(skipped.items())),
        "counts": dict(sorted(counts.items())),
        "include_nonunanimous": include_nonunanimous,
        "created_at": timestamp,
    }
    summary_path = output_path.with_suffix(".summary.json")
    summary_path.write_text(
        json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
    )
    return records, summary


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--events",
        type=Path,
        nargs="+",
        required=True,
        help="LLM-enriched candidate JSONL files or candidate directories.",
    )
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument(
        "--include-nonunanimous",
        action="store_true",
        help="Also accept pass-disagreement fallbacks; discouraged even for smoke tests.",
    )
    parser.add_argument(
        "--acknowledge-provisional-llm-auto-accept",
        action="store_true",
        help="Required acknowledgement that the output is not human-reviewed ground truth.",
    )
    parser.add_argument("--overwrite", action="store_true")
    return parser.parse_args()


def main() -> None:
    args = parse_args()
    if not args.acknowledge_provisional_llm_auto_accept:
        raise PermissionError(
            "Pass --acknowledge-provisional-llm-auto-accept; generated rows are "
            "for provisional smoke tests, not human-reviewed ground truth."
        )
    output = args.output.expanduser().resolve()
    if output.exists() and not args.overwrite:
        raise FileExistsError(f"Refusing to overwrite {output}; pass --overwrite")
    records, summary = materialize(
        args.events,
        output,
        include_nonunanimous=args.include_nonunanimous,
    )
    print(f"Saved {len(records)} provisional review rows to {output}")
    print(f"CSV: {output.with_suffix('.csv')}")
    print(f"Summary: {output.with_suffix('.summary.json')}")
    print("Skipped:", summary["skipped"])
    print("WARNING: no row in this output was human reviewed.")


if __name__ == "__main__":
    main()