Add manuscript-aligned EasyCom EOT INT review workflow
This commit is contained in:
Xin Wang
@@ -0,0 +1,793 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Serve a local, event-centered EasyCom annotation review workspace.
|
||||
|
||||
The server reads JSONL candidates produced by
|
||||
``derive_easycom_interaction_events.py`` or ``audit_easycom_floor_events.py``.
|
||||
It serves only media inside the supplied EasyCom dataset root and stores human
|
||||
reviews outside the source dataset as an atomically rewritten JSONL file plus
|
||||
a CSV companion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tempfile
|
||||
from threading import Lock
|
||||
from typing import Any, Iterable
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
|
||||
SOURCE_FPS = 20.0
|
||||
STATIC_ROOT = Path(__file__).resolve().parent / "floor_event_review_web"
|
||||
VIDEO_ROOT_PARTS = ("Main", "Video_Compressed")
|
||||
TRANSCRIPT_ROOT_PARTS = ("Main", "Speech_Transcriptions")
|
||||
REVIEW_SCHEMA_VERSION = 2
|
||||
MAX_REQUEST_BYTES = 128 * 1024
|
||||
|
||||
PROTOCOL_DISPOSITION_VALUES = {"positive", "hard_negative", "exclude", "uncertain"}
|
||||
HARD_NEGATIVE_TYPE_VALUES = {
|
||||
"eot_midturn_pause",
|
||||
"eot_other_to_other",
|
||||
"eot_no_response",
|
||||
"int_backchannel",
|
||||
"int_non_takeover_overlap",
|
||||
"int_retain_floor",
|
||||
"other",
|
||||
"not_applicable",
|
||||
"uncertain",
|
||||
}
|
||||
VALIDITY_VALUES = {"valid", "artifact", "uncertain"}
|
||||
ONSET_FUNCTION_VALUES = {
|
||||
"floor_taking",
|
||||
"backchannel",
|
||||
"within_turn_resume",
|
||||
"nonverbal",
|
||||
"not_applicable",
|
||||
"uncertain",
|
||||
}
|
||||
CONTEXT_VALUES = {"addressed_wearer", "third_party", "group", "unknown"}
|
||||
OFFSET_OUTCOME_VALUES = {
|
||||
"yield_to_other",
|
||||
"completion_to_silence",
|
||||
"temporary_pause",
|
||||
"backchannel_end",
|
||||
"cutoff",
|
||||
"not_applicable",
|
||||
"uncertain",
|
||||
}
|
||||
CONFIDENCE_VALUES = {"high", "medium", "low"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Candidate:
|
||||
candidate_id: str
|
||||
source_path: Path
|
||||
source_row: int
|
||||
payload: dict[str, Any]
|
||||
|
||||
@property
|
||||
def uid(self) -> str:
|
||||
return str(
|
||||
self.payload.get("uid")
|
||||
or self.payload.get("holder_uid")
|
||||
or f"{self.source_path.name}:{self.source_row}"
|
||||
)
|
||||
|
||||
@property
|
||||
def session(self) -> str:
|
||||
return str(self.payload.get("session", "unknown"))
|
||||
|
||||
@property
|
||||
def split(self) -> str:
|
||||
return str(self.payload.get("split", "unknown"))
|
||||
|
||||
@property
|
||||
def automatic_label(self) -> str:
|
||||
return str(
|
||||
self.payload.get("event_type")
|
||||
or self.payload.get("category")
|
||||
or self.payload.get("proxy_group")
|
||||
or "unclassified"
|
||||
)
|
||||
|
||||
@property
|
||||
def task(self) -> str | None:
|
||||
value = self.payload.get("task")
|
||||
return None if value is None else str(value)
|
||||
|
||||
@property
|
||||
def family(self) -> str:
|
||||
if self.task == "eot_response":
|
||||
return "eot"
|
||||
if self.task == "int_yield":
|
||||
return "int"
|
||||
if "holder_uid" in self.payload or self.automatic_label.startswith("eot_"):
|
||||
return "offset"
|
||||
if self.payload.get("participant_role") == "wearer":
|
||||
return "onset"
|
||||
return "interaction"
|
||||
|
||||
|
||||
def _json_default(value: Any) -> Any:
|
||||
if isinstance(value, Path):
|
||||
return str(value)
|
||||
raise TypeError(f"Cannot serialize {type(value).__name__}")
|
||||
|
||||
|
||||
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
for line_number, raw_line in enumerate(handle, 1):
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ValueError(f"Invalid JSON at {path}:{line_number}: {error}") from error
|
||||
if not isinstance(row, dict):
|
||||
raise ValueError(f"Expected a JSON object at {path}:{line_number}")
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def expand_candidate_paths(values: Iterable[Path], *, output_path: Path) -> list[Path]:
|
||||
paths: list[Path] = []
|
||||
output_resolved = output_path.expanduser().resolve()
|
||||
for raw_path in values:
|
||||
path = raw_path.expanduser().resolve()
|
||||
if path.is_dir():
|
||||
paths.extend(sorted(path.glob("Session_*.events.jsonl")))
|
||||
paths.extend(sorted(path.glob("easycom_*_candidates.jsonl")))
|
||||
elif path.is_file():
|
||||
paths.append(path)
|
||||
else:
|
||||
raise FileNotFoundError(path)
|
||||
unique: list[Path] = []
|
||||
seen: set[Path] = set()
|
||||
for path in paths:
|
||||
resolved = path.resolve()
|
||||
if resolved == output_resolved or resolved in seen:
|
||||
continue
|
||||
if resolved.suffix.lower() != ".jsonl":
|
||||
raise ValueError(f"Candidate input must be JSONL: {resolved}")
|
||||
seen.add(resolved)
|
||||
unique.append(resolved)
|
||||
if not unique:
|
||||
raise FileNotFoundError("No candidate JSONL files found")
|
||||
return unique
|
||||
|
||||
|
||||
def _candidate_digest(path: Path, source_row: int, payload: dict[str, Any]) -> str:
|
||||
identity = {
|
||||
"source_name": path.name,
|
||||
"source_row": source_row,
|
||||
"uid": payload.get("uid") or payload.get("holder_uid"),
|
||||
"session": payload.get("session"),
|
||||
"event_type": payload.get("event_type"),
|
||||
"task": payload.get("task"),
|
||||
"category": payload.get("category"),
|
||||
"dense_start_seconds": payload.get("dense_start_seconds"),
|
||||
"anchor_dense_seconds": payload.get("anchor_dense_seconds"),
|
||||
"boundary_seconds": payload.get("boundary_seconds"),
|
||||
}
|
||||
encoded = json.dumps(identity, sort_keys=True, ensure_ascii=False).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()[:20]
|
||||
|
||||
|
||||
def load_candidates(paths: Iterable[Path]) -> list[Candidate]:
|
||||
candidates: list[Candidate] = []
|
||||
ids: set[str] = set()
|
||||
for path in paths:
|
||||
for source_row, payload in enumerate(_read_jsonl(path), 1):
|
||||
candidate_id = _candidate_digest(path, source_row, payload)
|
||||
if candidate_id in ids:
|
||||
raise ValueError(f"Duplicate candidate identity in {path}:{source_row}")
|
||||
ids.add(candidate_id)
|
||||
candidates.append(Candidate(candidate_id, path, source_row, payload))
|
||||
if not candidates:
|
||||
raise ValueError("Candidate files contain no rows")
|
||||
return candidates
|
||||
|
||||
|
||||
def _uid_parts(value: Any) -> tuple[str, str, int] | None:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
parts = value.split("/")
|
||||
if len(parts) < 3 or not parts[-1].isdigit():
|
||||
return None
|
||||
return parts[-3], parts[-2], int(parts[-1])
|
||||
|
||||
|
||||
def _within_root(path: Path, root: Path) -> bool:
|
||||
try:
|
||||
path.resolve().relative_to(root.resolve())
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
class CandidateRepository:
|
||||
def __init__(
|
||||
self,
|
||||
candidates: list[Candidate],
|
||||
dataset_root: Path,
|
||||
*,
|
||||
context_before_seconds: float,
|
||||
context_after_seconds: float,
|
||||
) -> None:
|
||||
self.candidates = candidates
|
||||
self.by_id = {candidate.candidate_id: candidate for candidate in candidates}
|
||||
self.dataset_root = dataset_root.expanduser().resolve()
|
||||
self.context_before_seconds = context_before_seconds
|
||||
self.context_after_seconds = context_after_seconds
|
||||
self._transcript_cache: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
||||
|
||||
def _transcript(self, session: str, clip_stem: str) -> list[dict[str, Any]]:
|
||||
key = (session, clip_stem)
|
||||
if key not in self._transcript_cache:
|
||||
path = self.dataset_root.joinpath(
|
||||
*TRANSCRIPT_ROOT_PARTS, session, f"{clip_stem}.json"
|
||||
)
|
||||
if not path.is_file() or not _within_root(path, self.dataset_root):
|
||||
self._transcript_cache[key] = []
|
||||
else:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, list):
|
||||
raise ValueError(f"Expected transcript list: {path}")
|
||||
self._transcript_cache[key] = [
|
||||
row for row in payload if isinstance(row, dict)
|
||||
]
|
||||
return self._transcript_cache[key]
|
||||
|
||||
def _location(self, candidate: Candidate) -> tuple[str | None, float | None, int | None]:
|
||||
row = candidate.payload
|
||||
clip_stem = row.get("clip_stem")
|
||||
transcript_index: int | None = None
|
||||
uid = _uid_parts(row.get("uid")) or _uid_parts(row.get("holder_uid"))
|
||||
if uid is not None:
|
||||
uid_session, uid_stem, transcript_index = uid
|
||||
if uid_session == candidate.session:
|
||||
clip_stem = clip_stem or uid_stem
|
||||
|
||||
local_seconds: float | None = None
|
||||
if row.get("source_start_frame") is not None:
|
||||
local_seconds = max(0.0, (float(row["source_start_frame"]) - 1.0) / SOURCE_FPS)
|
||||
elif clip_stem and transcript_index is not None:
|
||||
transcript = self._transcript(candidate.session, str(clip_stem))
|
||||
if 0 <= transcript_index < len(transcript):
|
||||
source_row = transcript[transcript_index]
|
||||
frame_key = "End_Frame" if candidate.family == "offset" else "Start_Frame"
|
||||
if source_row.get(frame_key) is not None:
|
||||
frame = float(source_row[frame_key])
|
||||
local_seconds = max(
|
||||
0.0,
|
||||
frame / SOURCE_FPS if frame_key == "End_Frame" else (frame - 1.0) / SOURCE_FPS,
|
||||
)
|
||||
return (None if clip_stem is None else str(clip_stem), local_seconds, transcript_index)
|
||||
|
||||
def media_path(self, candidate: Candidate) -> Path | None:
|
||||
clip_stem, _, _ = self._location(candidate)
|
||||
if clip_stem is None:
|
||||
return None
|
||||
path = self.dataset_root.joinpath(
|
||||
*VIDEO_ROOT_PARTS, candidate.session, f"{clip_stem}.mp4"
|
||||
).resolve()
|
||||
if not _within_root(path, self.dataset_root) or not path.is_file():
|
||||
return None
|
||||
return path
|
||||
|
||||
def summary(self, candidate: Candidate, reviewed: bool) -> dict[str, Any]:
|
||||
row = candidate.payload
|
||||
event_time = row.get(
|
||||
"anchor_dense_seconds",
|
||||
row.get("dense_start_seconds", row.get("boundary_seconds")),
|
||||
)
|
||||
return {
|
||||
"candidate_id": candidate.candidate_id,
|
||||
"uid": candidate.uid,
|
||||
"session": candidate.session,
|
||||
"split": candidate.split,
|
||||
"family": candidate.family,
|
||||
"task": candidate.task,
|
||||
"automatic_label": candidate.automatic_label,
|
||||
"automatic_protocol_label": row.get("automatic_protocol_label"),
|
||||
"automatic_hard_negative_type": row.get("automatic_hard_negative_type"),
|
||||
"automatic_confidence": row.get("automatic_confidence", row.get("confidence")),
|
||||
"llm_suggestion": row.get("llm_suggestion"),
|
||||
"participant_role": row.get("participant_role") or row.get("holder_role"),
|
||||
"event_time_seconds": event_time,
|
||||
"reviewed": reviewed,
|
||||
}
|
||||
|
||||
def detail(self, candidate: Candidate) -> dict[str, Any]:
|
||||
clip_stem, local_seconds, transcript_index = self._location(candidate)
|
||||
transcript_rows: list[dict[str, Any]] = []
|
||||
if clip_stem is not None:
|
||||
for index, row in enumerate(self._transcript(candidate.session, clip_stem)):
|
||||
try:
|
||||
start = max(0.0, (float(row.get("Start_Frame", 1)) - 1.0) / SOURCE_FPS)
|
||||
end = max(start, float(row.get("End_Frame", 1)) / SOURCE_FPS)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if local_seconds is not None and (
|
||||
end < local_seconds - self.context_before_seconds
|
||||
or start > local_seconds + self.context_after_seconds
|
||||
):
|
||||
continue
|
||||
transcript_rows.append(
|
||||
{
|
||||
"index": index,
|
||||
"start_seconds": start,
|
||||
"end_seconds": end,
|
||||
"participant_id": row.get("Participant_ID"),
|
||||
"target_of_speech": row.get("Target_of_Speech"),
|
||||
"transcription": str(row.get("Transcription", "")),
|
||||
"is_anchor": (
|
||||
index == transcript_index
|
||||
or (
|
||||
transcript_index is None
|
||||
and local_seconds is not None
|
||||
and start <= local_seconds <= end
|
||||
)
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
media_path = self.media_path(candidate)
|
||||
return {
|
||||
**self.summary(candidate, reviewed=False),
|
||||
"source_file": candidate.source_path.name,
|
||||
"source_row": candidate.source_row,
|
||||
"clip_stem": clip_stem,
|
||||
"event_local_seconds": local_seconds,
|
||||
"context_start_seconds": (
|
||||
None
|
||||
if local_seconds is None
|
||||
else max(0.0, local_seconds - self.context_before_seconds)
|
||||
),
|
||||
"context_end_seconds": (
|
||||
None
|
||||
if local_seconds is None
|
||||
else local_seconds + self.context_after_seconds
|
||||
),
|
||||
"media_available": media_path is not None,
|
||||
"transcript_available": bool(transcript_rows),
|
||||
"transcript": transcript_rows,
|
||||
"candidate": candidate.payload,
|
||||
}
|
||||
|
||||
|
||||
class ReviewStore:
|
||||
def __init__(self, output_path: Path, candidates: list[Candidate]) -> None:
|
||||
self.output_path = output_path.expanduser().resolve()
|
||||
self.csv_path = self.output_path.with_suffix(".csv")
|
||||
self.candidates = {candidate.candidate_id: candidate for candidate in candidates}
|
||||
self.order = {candidate.candidate_id: index for index, candidate in enumerate(candidates)}
|
||||
self.lock = Lock()
|
||||
self.reviews: dict[str, dict[str, Any]] = {}
|
||||
if self.output_path.is_file():
|
||||
for row in _read_jsonl(self.output_path):
|
||||
candidate_id = row.get("candidate_id")
|
||||
if isinstance(candidate_id, str) and candidate_id in self.candidates:
|
||||
self.reviews[candidate_id] = row
|
||||
|
||||
def _validated_review(self, candidate_id: str, payload: Any) -> dict[str, Any]:
|
||||
if candidate_id not in self.candidates:
|
||||
raise ValueError("Unknown candidate_id")
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("review must be an object")
|
||||
|
||||
required_choices = {
|
||||
"protocol_disposition": PROTOCOL_DISPOSITION_VALUES,
|
||||
"hard_negative_type": HARD_NEGATIVE_TYPE_VALUES,
|
||||
"validity": VALIDITY_VALUES,
|
||||
"onset_function": ONSET_FUNCTION_VALUES,
|
||||
"context": CONTEXT_VALUES,
|
||||
"offset_outcome": OFFSET_OUTCOME_VALUES,
|
||||
"confidence": CONFIDENCE_VALUES,
|
||||
}
|
||||
normalized: dict[str, Any] = {}
|
||||
for key, allowed in required_choices.items():
|
||||
value = payload.get(key)
|
||||
if value not in allowed:
|
||||
raise ValueError(f"Invalid {key}: {value!r}")
|
||||
normalized[key] = value
|
||||
|
||||
for key in (
|
||||
"anchor_adjustment_bins",
|
||||
"behavior_adjustment_bins",
|
||||
"region_end_adjustment_bins",
|
||||
):
|
||||
adjustment = payload.get(key)
|
||||
if adjustment not in {-1, 0, 1, None}:
|
||||
raise ValueError(f"{key} must be -1, 0, 1, or null")
|
||||
normalized[key] = adjustment
|
||||
|
||||
reviewer = str(payload.get("reviewer", "")).strip()
|
||||
notes = str(payload.get("notes", "")).strip()
|
||||
if len(reviewer) > 100:
|
||||
raise ValueError("reviewer is too long")
|
||||
if not reviewer:
|
||||
raise ValueError("reviewer is required")
|
||||
if len(notes) > 4000:
|
||||
raise ValueError("notes are too long")
|
||||
normalized["reviewer"] = reviewer
|
||||
normalized["notes"] = notes
|
||||
return normalized
|
||||
|
||||
def save(self, candidate_id: str, review: Any) -> dict[str, Any]:
|
||||
normalized = self._validated_review(candidate_id, review)
|
||||
candidate = self.candidates[candidate_id]
|
||||
record = {
|
||||
"schema_version": REVIEW_SCHEMA_VERSION,
|
||||
"candidate_id": candidate_id,
|
||||
"candidate_uid": candidate.uid,
|
||||
"source_file": candidate.source_path.name,
|
||||
"source_row": candidate.source_row,
|
||||
"session": candidate.session,
|
||||
"split": candidate.split,
|
||||
"candidate_family": candidate.family,
|
||||
"task": candidate.task,
|
||||
"automatic_label": candidate.automatic_label,
|
||||
"automatic_protocol_label": candidate.payload.get("automatic_protocol_label"),
|
||||
"automatic_hard_negative_type": candidate.payload.get(
|
||||
"automatic_hard_negative_type"
|
||||
),
|
||||
"review": normalized,
|
||||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"candidate": candidate.payload,
|
||||
}
|
||||
with self.lock:
|
||||
self.reviews[candidate_id] = record
|
||||
self._write_locked()
|
||||
return record
|
||||
|
||||
def _ordered(self) -> list[dict[str, Any]]:
|
||||
return sorted(
|
||||
self.reviews.values(),
|
||||
key=lambda row: self.order.get(str(row.get("candidate_id")), 10**12),
|
||||
)
|
||||
|
||||
def _atomic_text(self, path: Path, write: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary_name: str | None = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
"w", encoding="utf-8", newline="", dir=path.parent, delete=False
|
||||
) as handle:
|
||||
temporary_name = handle.name
|
||||
write(handle)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary_name, path)
|
||||
finally:
|
||||
if temporary_name is not None and Path(temporary_name).exists():
|
||||
Path(temporary_name).unlink()
|
||||
|
||||
def _write_locked(self) -> None:
|
||||
rows = self._ordered()
|
||||
|
||||
def write_jsonl(handle: Any) -> None:
|
||||
for row in rows:
|
||||
handle.write(json.dumps(row, ensure_ascii=False, default=_json_default) + "\n")
|
||||
|
||||
self._atomic_text(self.output_path, write_jsonl)
|
||||
|
||||
fieldnames = [
|
||||
"candidate_id",
|
||||
"candidate_uid",
|
||||
"source_file",
|
||||
"source_row",
|
||||
"session",
|
||||
"split",
|
||||
"candidate_family",
|
||||
"task",
|
||||
"automatic_label",
|
||||
"automatic_protocol_label",
|
||||
"automatic_hard_negative_type",
|
||||
"protocol_disposition",
|
||||
"hard_negative_type",
|
||||
"validity",
|
||||
"onset_function",
|
||||
"context",
|
||||
"offset_outcome",
|
||||
"anchor_adjustment_bins",
|
||||
"behavior_adjustment_bins",
|
||||
"region_end_adjustment_bins",
|
||||
"confidence",
|
||||
"reviewer",
|
||||
"notes",
|
||||
"updated_at",
|
||||
]
|
||||
|
||||
def write_csv(handle: Any) -> None:
|
||||
writer = csv.DictWriter(handle, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
review = row["review"]
|
||||
writer.writerow(
|
||||
{
|
||||
key: review.get(key) if key in review else row.get(key)
|
||||
for key in fieldnames
|
||||
}
|
||||
)
|
||||
|
||||
self._atomic_text(self.csv_path, write_csv)
|
||||
|
||||
|
||||
def parse_byte_range(value: str | None, size: int) -> tuple[int, int] | None:
|
||||
if not value:
|
||||
return None
|
||||
match = re.fullmatch(r"bytes=(\d*)-(\d*)", value.strip())
|
||||
if match is None:
|
||||
raise ValueError("Invalid Range header")
|
||||
start_text, end_text = match.groups()
|
||||
if not start_text and not end_text:
|
||||
raise ValueError("Invalid empty Range header")
|
||||
if not start_text:
|
||||
suffix = int(end_text)
|
||||
if suffix <= 0:
|
||||
raise ValueError("Invalid suffix Range")
|
||||
return max(0, size - suffix), size - 1
|
||||
start = int(start_text)
|
||||
end = size - 1 if not end_text else int(end_text)
|
||||
if start >= size or start < 0 or end < start:
|
||||
raise ValueError("Unsatisfiable Range")
|
||||
return start, min(end, size - 1)
|
||||
|
||||
|
||||
class ReviewHTTPServer(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
address: tuple[str, int],
|
||||
repository: CandidateRepository,
|
||||
store: ReviewStore,
|
||||
) -> None:
|
||||
super().__init__(address, ReviewRequestHandler)
|
||||
self.repository = repository
|
||||
self.store = store
|
||||
|
||||
|
||||
class ReviewRequestHandler(BaseHTTPRequestHandler):
|
||||
server: ReviewHTTPServer
|
||||
|
||||
def log_message(self, format: str, *args: Any) -> None:
|
||||
print(f"[{self.log_date_time_string()}] {format % args}")
|
||||
|
||||
def _json(self, payload: Any, status: int = HTTPStatus.OK) -> None:
|
||||
body = json.dumps(payload, ensure_ascii=False, default=_json_default).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
|
||||
def _error(self, message: str, status: int = HTTPStatus.BAD_REQUEST) -> None:
|
||||
self._json({"error": message}, status)
|
||||
|
||||
def _candidate_from_query(self, query: dict[str, list[str]]) -> Candidate | None:
|
||||
candidate_id = query.get("id", [""])[0]
|
||||
return self.server.repository.by_id.get(candidate_id)
|
||||
|
||||
def do_GET(self) -> None:
|
||||
parsed = urlparse(self.path)
|
||||
query = parse_qs(parsed.query)
|
||||
if parsed.path == "/api/bootstrap":
|
||||
store = self.server.store
|
||||
repository = self.server.repository
|
||||
self._json(
|
||||
{
|
||||
"schema_version": REVIEW_SCHEMA_VERSION,
|
||||
"candidate_count": len(repository.candidates),
|
||||
"reviewed_count": len(store.reviews),
|
||||
"output_name": store.output_path.name,
|
||||
"candidates": [
|
||||
repository.summary(
|
||||
candidate, candidate.candidate_id in store.reviews
|
||||
)
|
||||
for candidate in repository.candidates
|
||||
],
|
||||
"reviews": {
|
||||
candidate_id: row["review"]
|
||||
for candidate_id, row in store.reviews.items()
|
||||
},
|
||||
}
|
||||
)
|
||||
return
|
||||
if parsed.path == "/api/event":
|
||||
candidate = self._candidate_from_query(query)
|
||||
if candidate is None:
|
||||
self._error("Unknown candidate", HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
detail = self.server.repository.detail(candidate)
|
||||
detail["review"] = self.server.store.reviews.get(candidate.candidate_id, {}).get(
|
||||
"review"
|
||||
)
|
||||
self._json(detail)
|
||||
return
|
||||
if parsed.path == "/api/export.jsonl":
|
||||
self._send_download(self.server.store.output_path, "application/x-ndjson")
|
||||
return
|
||||
if parsed.path == "/api/export.csv":
|
||||
self._send_download(self.server.store.csv_path, "text/csv; charset=utf-8")
|
||||
return
|
||||
if parsed.path == "/media":
|
||||
candidate = self._candidate_from_query(query)
|
||||
if candidate is None:
|
||||
self._error("Unknown candidate", HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
media_path = self.server.repository.media_path(candidate)
|
||||
if media_path is None:
|
||||
self._error("Media unavailable", HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
self._send_media(media_path)
|
||||
return
|
||||
self._send_static(parsed.path)
|
||||
|
||||
def do_HEAD(self) -> None:
|
||||
self.do_GET()
|
||||
|
||||
def do_POST(self) -> None:
|
||||
parsed = urlparse(self.path)
|
||||
if parsed.path != "/api/review":
|
||||
self._error("Unknown endpoint", HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
except ValueError:
|
||||
self._error("Invalid Content-Length")
|
||||
return
|
||||
if length <= 0 or length > MAX_REQUEST_BYTES:
|
||||
self._error("Invalid request size")
|
||||
return
|
||||
try:
|
||||
payload = json.loads(self.rfile.read(length).decode("utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("Payload must be an object")
|
||||
candidate_id = str(payload.get("candidate_id", ""))
|
||||
record = self.server.store.save(candidate_id, payload.get("review"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as error:
|
||||
self._error(str(error))
|
||||
return
|
||||
self._json({"saved": True, "record": record})
|
||||
|
||||
def _send_static(self, request_path: str) -> None:
|
||||
name = "index.html" if request_path in {"", "/"} else request_path.lstrip("/")
|
||||
if name not in {"index.html", "app.css", "app.js"}:
|
||||
self.send_error(HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
path = (STATIC_ROOT / name).resolve()
|
||||
if not _within_root(path, STATIC_ROOT) or not path.is_file():
|
||||
self.send_error(HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
|
||||
body = path.read_bytes()
|
||||
self.send_response(HTTPStatus.OK)
|
||||
self.send_header("Content-Type", f"{content_type}; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-cache")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send_download(self, path: Path, content_type: str) -> None:
|
||||
if not path.is_file():
|
||||
self._error("No reviews have been saved yet", HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
body = path.read_bytes()
|
||||
self.send_response(HTTPStatus.OK)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Disposition", f'attachment; filename="{path.name}"')
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send_media(self, path: Path) -> None:
|
||||
size = path.stat().st_size
|
||||
try:
|
||||
byte_range = parse_byte_range(self.headers.get("Range"), size)
|
||||
except ValueError:
|
||||
self.send_response(HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
|
||||
self.send_header("Content-Range", f"bytes */{size}")
|
||||
self.end_headers()
|
||||
return
|
||||
start, end = byte_range or (0, size - 1)
|
||||
status = HTTPStatus.PARTIAL_CONTENT if byte_range is not None else HTTPStatus.OK
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", mimetypes.guess_type(path.name)[0] or "video/mp4")
|
||||
self.send_header("Accept-Ranges", "bytes")
|
||||
self.send_header("Content-Length", str(end - start + 1))
|
||||
if byte_range is not None:
|
||||
self.send_header("Content-Range", f"bytes {start}-{end}/{size}")
|
||||
self.end_headers()
|
||||
if self.command == "HEAD":
|
||||
return
|
||||
with path.open("rb") as handle:
|
||||
handle.seek(start)
|
||||
remaining = end - start + 1
|
||||
while remaining:
|
||||
chunk = handle.read(min(64 * 1024, remaining))
|
||||
if not chunk:
|
||||
break
|
||||
self.wfile.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--events",
|
||||
type=Path,
|
||||
nargs="+",
|
||||
required=True,
|
||||
help="Candidate JSONL files or directories containing generated candidates.",
|
||||
)
|
||||
parser.add_argument("--dataset-root", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=Path("reports/easycom_event_reviews/reviews.jsonl"),
|
||||
)
|
||||
parser.add_argument("--host", default="127.0.0.1")
|
||||
parser.add_argument("--port", type=int, default=8765)
|
||||
parser.add_argument("--context-before-seconds", type=float, default=3.0)
|
||||
parser.add_argument("--context-after-seconds", type=float, default=3.0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
if not STATIC_ROOT.is_dir():
|
||||
raise FileNotFoundError(f"Review UI assets are missing: {STATIC_ROOT}")
|
||||
dataset_root = args.dataset_root.expanduser().resolve()
|
||||
if not dataset_root.is_dir():
|
||||
raise FileNotFoundError(dataset_root)
|
||||
if not 1 <= args.port <= 65535:
|
||||
raise ValueError("--port must be in [1, 65535]")
|
||||
if args.context_before_seconds < 0 or args.context_after_seconds < 0:
|
||||
raise ValueError("Context durations must be non-negative")
|
||||
|
||||
output_path = args.output.expanduser().resolve()
|
||||
candidate_paths = expand_candidate_paths(args.events, output_path=output_path)
|
||||
candidates = load_candidates(candidate_paths)
|
||||
repository = CandidateRepository(
|
||||
candidates,
|
||||
dataset_root,
|
||||
context_before_seconds=args.context_before_seconds,
|
||||
context_after_seconds=args.context_after_seconds,
|
||||
)
|
||||
store = ReviewStore(output_path, candidates)
|
||||
server = ReviewHTTPServer((args.host, args.port), repository, store)
|
||||
actual_host, actual_port = server.server_address[:2]
|
||||
print(f"Loaded {len(candidates)} candidates from {len(candidate_paths)} JSONL file(s)")
|
||||
print(f"Resuming {len(store.reviews)} saved reviews")
|
||||
print(f"Review workspace: http://{actual_host}:{actual_port}")
|
||||
print(f"Review output: {store.output_path}")
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print("\nStopping review workspace")
|
||||
finally:
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user