wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Ego4D AV assisted annotation review workflow

Browse Source
This commit is contained in:
Xin Wang
2026-09-15 13:23:59 +08:00
parent 74b502f770
commit 86aa124a46
9 changed files with 953 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
review_floor_events.py
+76 -12
View File
@@ -1,11 +1,10 @@
#!/usr/bin/env python3
"""Serve a local, event-centered EasyCom annotation review workspace.
"""Serve a local, event-centered EasyCom or Ego4D AV review workspace.
The server reads JSONL candidates produced by
``derive_easycom_interaction_events.py`` or ``audit_easycom_floor_events.py``.
It serves only media inside the supplied EasyCom dataset root and stores human
reviews outside the source dataset as an atomically rewritten JSONL file plus
a CSV companion.
The server reads JSONL candidates produced by the EasyCom or Ego4D AV
candidate generators. It serves only media inside the supplied dataset root
and stores human reviews outside the source dataset as an atomically rewritten
JSONL file plus a CSV companion.
"""
from __future__ import annotations
@@ -94,7 +93,7 @@ class Candidate:
@property
def session(self) -> str:
return str(self.payload.get("session", "unknown"))
return str(self.payload.get("session") or self.payload.get("clip_uid") or "unknown")
@property
def split(self) -> str:
@@ -161,6 +160,7 @@ def expand_candidate_paths(values: Iterable[Path], *, output_path: Path) -> list
if path.is_dir():
paths.extend(sorted(path.glob("Session_*.events.jsonl")))
paths.extend(sorted(path.glob("easycom_*_candidates.jsonl")))
paths.extend(sorted(path.glob("ego4d*_candidates.jsonl")))
elif path.is_file():
paths.append(path)
else:
@@ -239,17 +239,35 @@ class CandidateRepository:
candidates: list[Candidate],
dataset_root: Path,
*,
dataset: str = "auto",
context_before_seconds: float,
context_after_seconds: float,
) -> None:
self.candidates = candidates
self.by_id = {candidate.candidate_id: candidate for candidate in candidates}
self.dataset_root = dataset_root.expanduser().resolve()
if dataset not in {"auto", "easycom", "ego4d"}:
raise ValueError(f"Unknown dataset: {dataset}")
declared = {str(candidate.payload.get("dataset", "")) for candidate in candidates}
kinds = {"ego4d" if value.startswith("ego4d") else "easycom" for value in declared}
if dataset == "auto" and len(kinds) != 1:
raise ValueError(f"Candidate inputs mix dataset layouts: {sorted(declared)}")
inferred = next(iter(kinds))
self.dataset = inferred if dataset == "auto" else dataset
self.context_before_seconds = context_before_seconds
self.context_after_seconds = context_after_seconds
self._transcript_cache: dict[tuple[str, str], list[dict[str, Any]]] = {}
self.session_timelines: dict[str, list[dict[str, Any]]] = {}
for path in sorted({candidate.source_path.parent / "easycom_session_timelines.json" for candidate in candidates}):
timeline_names = ("easycom_session_timelines.json", "ego4d_av_clip_timelines.json")
timeline_paths = {
candidate.source_path.parent / name
for candidate in candidates for name in timeline_names
}
timeline_paths.update(
candidate.source_path.with_name(candidate.source_path.stem + "_timelines.json")
for candidate in candidates
)
for path in sorted(timeline_paths):
if not path.is_file():
continue
payload = json.loads(path.read_text(encoding="utf-8"))
@@ -279,6 +297,14 @@ class CandidateRepository:
def _location(self, candidate: Candidate) -> tuple[str | None, float | None, int | None]:
row = candidate.payload
clip_stem = row.get("clip_stem")
if self.dataset == "ego4d":
clip_uid = row.get("clip_uid") or clip_stem or candidate.session
event_time = row.get("event_time", row.get("anchor_dense_seconds"))
return (
None if clip_uid is None else str(clip_uid),
None if event_time is None else max(0.0, float(event_time)),
None,
)
transcript_index: int | None = None
uid = None if row.get("candidate_source") else (
_uid_parts(row.get("uid")) or _uid_parts(row.get("holder_uid"))
@@ -308,9 +334,12 @@ class CandidateRepository:
clip_stem, _, _ = self._location(candidate)
if clip_stem is None:
return None
path = self.dataset_root.joinpath(
*VIDEO_ROOT_PARTS, candidate.session, f"{clip_stem}.mp4"
).resolve()
if self.dataset == "ego4d":
path = self.dataset_root.joinpath("v1", "clips", f"{clip_stem}.mp4").resolve()
else:
path = self.dataset_root.joinpath(
*VIDEO_ROOT_PARTS, candidate.session, f"{clip_stem}.mp4"
).resolve()
if not _within_root(path, self.dataset_root) or not path.is_file():
return None
return path
@@ -358,7 +387,37 @@ class CandidateRepository:
for row in evidence_rows
if isinstance(row, dict) and row.get("participant_role") == "wearer"
}
if clip_stem is not None:
if self.dataset == "ego4d":
for index, row in enumerate(evidence_rows):
if not isinstance(row, dict):
continue
try:
start = max(0.0, float(row["start_seconds"]))
end = max(start, float(row["end_seconds"]))
except (KeyError, TypeError, ValueError):
continue
if local_seconds is not None and (
end < local_seconds - self.context_before_seconds
or start > local_seconds + self.context_after_seconds
):
continue
transcript_rows.append({
"uid": row.get("uid", f"{candidate.session}/context/{index:04d}"),
"index": index,
"start_seconds": start,
"end_seconds": end,
"participant_id": row.get("participant_id"),
"participant_role": row.get("participant_role", "other"),
"target_of_speech": (
"camera_wearer" if row.get("addressed_to_wearer") is True
else "not_camera_wearer" if row.get("addressed_to_wearer") is False
else None
),
"addressed_to_wearer": row.get("addressed_to_wearer"),
"transcription": str(row.get("transcription", "")),
"is_anchor": start <= (local_seconds or -1) <= end,
})
elif clip_stem is not None:
for index, row in enumerate(self._transcript(candidate.session, clip_stem)):
try:
start = max(0.0, (float(row.get("Start_Frame", 1)) - 1.0) / SOURCE_FPS)
@@ -915,6 +974,10 @@ def parse_args() -> argparse.Namespace:
help="Candidate JSONL files or directories containing generated candidates.",
)
parser.add_argument("--dataset-root", type=Path, required=True)
parser.add_argument(
"--dataset", choices=("auto", "easycom", "ego4d"), default="auto",
help="Dataset layout; auto reads the candidate dataset field (default: %(default)s).",
)
parser.add_argument(
"--output",
type=Path,
@@ -953,6 +1016,7 @@ def main() -> None:
repository = CandidateRepository(
candidates,
dataset_root,
dataset=args.dataset,
context_before_seconds=args.context_before_seconds,
context_after_seconds=args.context_after_seconds,
)