Add causal person fusion and MuVAP stage-3 baselines

third_party/MuVAP/preprocess/asd/__init__.py

@@ -0,0 +1 @@
+"""Canonical active-speaker preprocessing for MuVAP+."""

third_party/MuVAP/preprocess/asd/adapters.py

@@ -0,0 +1,112 @@
+"""Dataset-specific metadata adapters.
+
+Adapters do not normalize signals. They only convert source layouts into
+`SourceRecord`; the shared writer owns every media transformation.
+"""
+
+import ast
+import csv
+import json
+from pathlib import Path
+from typing import Iterable
+
+from preprocess.asd.schema import SAMPLE_RATE, SourceRecord
+
+
+def _labels(text: str) -> list[int]:
+    values = ast.literal_eval(text)
+    if not isinstance(values, (list, tuple)):
+        raise ValueError("labels must be a list")
+    result = [int(value) for value in values]
+    if any(value not in (0, 1) for value in result):
+        raise ValueError("labels must contain only 0 and 1")
+    return result
+
+
+def _existing(path: Path, sample_id: str) -> Path:
+    if not path.exists():
+        raise FileNotFoundError(f"{sample_id}: missing {path}")
+    return path
+
+
+def clip_directory_records(
+    loader_path: Path,
+    visual_root: Path,
+    audio_root: Path,
+    split: str,
+    dataset: str = "ava",
+    skip_missing: bool = False,
+) -> Iterable[SourceRecord]:
+    """Read the AVA-style loader TSV and nested JPEG/WAV layout.
+
+    AVA and WASD ship the identical layout - `clips_videos/<split>/<clip>/
+    <entity>/<timestamp>.jpg` beside `clips_audios/<split>/<clip>/<entity>.wav`
+    - and the identical five-column loader TSV, so one reader serves both.
+    """
+    face_by_id = {
+        path.name: path for path in (visual_root / split).glob("*/*") if path.is_dir()
+    }
+    audio_by_id = {path.stem: path for path in (audio_root / split).glob("*/*.wav")}
+    with loader_path.open(newline="") as handle:
+        for row_number, row in enumerate(csv.reader(handle, delimiter="\t"), start=1):
+            if len(row) < 4:
+                raise ValueError(
+                    f"{loader_path}:{row_number}: expected at least 4 columns"
+                )
+            sample_id, _, fps, label_text = row[:4]
+            face_path = face_by_id.get(sample_id)
+            audio_path = audio_by_id.get(sample_id)
+            if face_path is None or audio_path is None:
+                if skip_missing:
+                    continue
+                raise FileNotFoundError(
+                    f"{sample_id}: missing face directory or clip-local WAV"
+                )
+            yield SourceRecord(
+                dataset=dataset,
+                sample_id=sample_id,
+                visual_path=face_path,
+                audio_path=audio_path,
+                source_fps=float(fps),
+                labels=_labels(label_text),
+                visual_kind="jpeg_directory",
+            )
+
+
+def manifest_records(
+    manifest_path: Path,
+    dataset: str,
+    root: Path | None = None,
+    default_npy_audio_sample_rate: int = SAMPLE_RATE,
+) -> Iterable[SourceRecord]:
+    """Read the canonical TSV adapter used for AVCC and future datasets.
+
+    Required header fields: sample_id, visual_path, audio_path, source_fps,
+    labels. Optional: start_sec, visual_kind, audio_sample_rate.
+    """
+    root = root or manifest_path.parent
+    with manifest_path.open(newline="") as handle:
+        reader = csv.DictReader(handle, delimiter="\t")
+        required = {"sample_id", "visual_path", "audio_path", "source_fps", "labels"}
+        missing = required.difference(reader.fieldnames or [])
+        if missing:
+            raise ValueError(f"{manifest_path}: missing columns {sorted(missing)}")
+        for row_number, row in enumerate(reader, start=2):
+            try:
+                yield SourceRecord(
+                    dataset=dataset,
+                    sample_id=row["sample_id"],
+                    visual_path=_existing(root / row["visual_path"], row["sample_id"]),
+                    audio_path=_existing(root / row["audio_path"], row["sample_id"]),
+                    source_fps=float(row["source_fps"]),
+                    labels=json.loads(row["labels"]),
+                    start_sec=float(row.get("start_sec") or 0.0),
+                    npy_audio_sample_rate=int(
+                        row.get("audio_sample_rate") or default_npy_audio_sample_rate
+                    ),
+                    visual_kind=row.get("visual_kind") or "npy",
+                )
+            except (TypeError, ValueError, json.JSONDecodeError) as error:
+                raise ValueError(
+                    f"{manifest_path}:{row_number}: invalid row"
+                ) from error

third_party/MuVAP/preprocess/asd/prep_msdwild.py

@@ -0,0 +1,270 @@
+"""Crop raw MSDWild videos into aligned 30-second speaker-track samples."""
+
+import argparse
+import csv
+import shutil
+import sys
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+
+import cv2
+import numpy as np
+
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from data.utils import load_audio  # noqa: E402
+from preprocess.asd.video import crop_face, load_visual  # noqa: E402
+from preprocess.asd.writer import ShardWriter  # noqa: E402
+
+
+@dataclass(frozen=True)
+class Detection:
+    bbox: tuple[float, float, float, float]
+
+
+def read_annotations(path: Path) -> dict[int, dict[str, Detection]]:
+    annotations: dict[int, dict[str, Detection]] = defaultdict(dict)
+    with path.open(newline="") as handle:
+        for row_number, row in enumerate(csv.reader(handle), start=1):
+            if len(row) < 8:
+                raise ValueError(f"{path}:{row_number}: expected 8 columns")
+            try:
+                frame = int(row[0])
+                speaker_id = row[2]
+                bbox = tuple(float(value) for value in row[3:7])
+                int(row[7])
+            except ValueError as error:
+                raise ValueError(f"{path}:{row_number}: invalid annotation") from error
+            annotations[frame][speaker_id] = Detection(bbox)
+    return dict(annotations)
+
+
+def read_rttm(path: Path) -> dict[str, dict[str, list[tuple[float, float]]]]:
+    activity: dict[str, dict[str, list[tuple[float, float]]]] = defaultdict(
+        lambda: defaultdict(list)
+    )
+    with path.open() as handle:
+        for line_number, line in enumerate(handle, start=1):
+            parts = line.split()
+            if not parts:
+                continue
+            if len(parts) < 8 or parts[0] != "SPEAKER":
+                raise ValueError(f"{path}:{line_number}: invalid RTTM row")
+            try:
+                start = float(parts[3])
+                duration = float(parts[4])
+            except ValueError as error:
+                raise ValueError(f"{path}:{line_number}: invalid time") from error
+            activity[parts[1]][parts[7]].append((start, start + duration))
+    return {
+        video_id: {speaker: sorted(intervals) for speaker, intervals in speakers.items()}
+        for video_id, speakers in activity.items()
+    }
+
+
+def retained_speakers(
+    annotations: dict[int, dict[str, Detection]],
+    start_frame: int,
+    frame_count: int,
+    min_face_ratio: float,
+) -> list[str]:
+    counts: dict[str, int] = defaultdict(int)
+    for frame in range(start_frame, start_frame + frame_count):
+        for speaker_id in annotations.get(frame, {}):
+            counts[speaker_id] += 1
+    minimum = min_face_ratio * frame_count
+    return sorted(speaker_id for speaker_id, count in counts.items() if count >= minimum)
+
+
+def clip_starts(first_face_frame: int, last_face_frame: int, stride_frames: int):
+    return range(first_face_frame, last_face_frame + 1, stride_frames)
+
+
+def video_metadata(path: Path) -> tuple[float, int]:
+    cap = cv2.VideoCapture(str(path))
+    if not cap.isOpened():
+        raise FileNotFoundError(f"could not open video: {path}")
+    fps = float(cap.get(cv2.CAP_PROP_FPS))
+    frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    cap.release()
+    if fps <= 0 or frame_count <= 0:
+        raise ValueError(f"invalid video metadata: {path}")
+    return fps, frame_count
+
+
+def process_video(
+    video_path: Path,
+    annotation_path: Path,
+    activity: dict[str, list[tuple[float, float]]],
+    writer: ShardWriter,
+    clip_seconds: float,
+    overlap_seconds: float,
+    min_face_ratio: float,
+    min_clip_seconds: float,
+) -> int:
+    annotations = read_annotations(annotation_path)
+    if not annotations:
+        return 0
+
+    fps, video_frames = video_metadata(video_path)
+    clip_frames = round(clip_seconds * fps)
+    stride_frames = round((clip_seconds - overlap_seconds) * fps)
+    min_clip_frames = max(1, round(min_clip_seconds * fps))
+    if clip_frames <= 0 or stride_frames <= 0:
+        raise ValueError("clip and stride must each contain at least one frame")
+    waveform, _ = load_audio(video_path)
+    first_face = min(annotations)
+    last_face = min(max(annotations), video_frames - 1)
+    written = 0
+
+    for segment_id, start_frame in enumerate(
+        clip_starts(first_face, last_face, stride_frames)
+    ):
+        # Take whatever is left at the tail. Windows used to be dropped unless
+        # they were complete, which discarded every video shorter than one clip
+        # - about a third of MSDWild. The loader pads and chunks, so a short
+        # sample costs nothing.
+        window_frames = min(clip_frames, video_frames - start_frame)
+        if window_frames < min_clip_frames:
+            break
+        speakers = retained_speakers(
+            annotations, start_frame, window_frames, min_face_ratio
+        )
+        if not speakers:
+            continue
+
+        frames, decoded_fps = load_visual(video_path, start_frame, window_frames)
+        if not np.isclose(decoded_fps, fps):
+            raise ValueError(f"frame-rate changed while decoding {video_path}")
+        if len(frames) < min_clip_frames:
+            continue
+        window_frames = len(frames)
+
+        start_time = start_frame / fps
+        start_sample = round(start_time * 16_000)
+        target_samples = round(window_frames / fps * 16_000)
+        audio = waveform[:, start_sample : start_sample + target_samples]
+
+        for speaker_id in speakers:
+            faces = []
+            labels = np.zeros(window_frames, dtype=np.uint8)
+            for offset, frame in enumerate(frames):
+                detection = annotations.get(start_frame + offset, {}).get(speaker_id)
+                faces.append(crop_face(frame, detection.bbox if detection else None))
+            intervals = activity.get(speaker_id, [])
+            for offset in range(window_frames):
+                timestamp = start_time + (offset + 0.5) / fps
+                labels[offset] = any(start <= timestamp < end for start, end in intervals)
+            faces = np.stack(faces).astype(np.uint8, copy=False)
+            sample_id = f"{video_path.stem}_{segment_id:05d}_{speaker_id}"
+            writer.add_arrays(
+                dataset="msdwild",
+                sample_id=sample_id,
+                faces=faces,
+                waveform=audio,
+                labels=labels,
+                source_fps=fps,
+                start_sec=start_time,
+            )
+            written += 1
+    return written
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input", type=Path, required=True)
+    parser.add_argument("--rttm", type=Path)
+    parser.add_argument(
+        "--output",
+        type=Path,
+        help="defaults to <dataset root>/packed/<split>, beside the source media",
+    )
+    parser.add_argument(
+        "--split",
+        default="all",
+        help="official split to pack: all, few.train, few.val, many.val",
+    )
+    parser.add_argument("--clip-seconds", type=float, default=30.0)
+    # Windows no longer overlap: the loader computes projection targets over a
+    # whole track before chunking, so duplicated frames buy nothing and would
+    # both inflate the pack and over-weight this corpus in an epoch.
+    parser.add_argument("--overlap-seconds", type=float, default=0.0)
+    parser.add_argument("--min-clip-seconds", type=float, default=2.0)
+    parser.add_argument("--min-face-ratio", type=float, default=0.9)
+    parser.add_argument("--max-shard-gb", type=float, default=1.0)
+    parser.add_argument("--overwrite", action="store_true")
+    return parser
+
+
+def main() -> None:
+    args = build_parser().parse_args()
+    if args.clip_seconds <= 0:
+        raise ValueError("clip-seconds must be positive")
+    if not 0 <= args.overlap_seconds < args.clip_seconds:
+        raise ValueError("overlap-seconds must be in [0, clip-seconds)")
+    if not 0 < args.min_face_ratio <= 1:
+        raise ValueError("min-face-ratio must be in (0, 1]")
+    if args.max_shard_gb <= 0:
+        raise ValueError("max-shard-gb must be positive")
+    if not 0 < args.min_clip_seconds <= args.clip_seconds:
+        raise ValueError("min-clip-seconds must be in (0, clip-seconds]")
+    if args.output is None:
+        args.output = args.input.parent / "packed" / args.split
+        print(f"packing into {args.output}")
+    if args.output.exists():
+        if not args.overwrite and any(args.output.iterdir()):
+            raise FileExistsError(f"output directory is not empty: {args.output}")
+        if args.overwrite:
+            shutil.rmtree(args.output)
+
+    args.output.mkdir(parents=True)
+    videos = sorted(args.input.glob("*.mp4"))
+    if not videos:
+        raise FileNotFoundError(f"no MP4 files found in {args.input}")
+
+    rttm_path = args.rttm or args.input.parent / "all.rttm"
+    if not rttm_path.exists():
+        raise FileNotFoundError(f"missing RTTM labels: {rttm_path}")
+    activity = read_rttm(rttm_path)
+
+    if args.split != "all":
+        # The official split lists are themselves RTTMs; their video ids are
+        # the split membership.
+        split_path = args.input.parent / f"{args.split}.rttm"
+        if not split_path.exists():
+            raise FileNotFoundError(f"missing split list: {split_path}")
+        wanted = set(read_rttm(split_path))
+        videos = [video for video in videos if video.stem in wanted]
+        if not videos:
+            raise ValueError(f"no videos matched split {args.split}")
+        print(f"{args.split}: {len(videos)} videos")
+    total = 0
+    writer = ShardWriter(
+        args.output, max_shard_bytes=round(args.max_shard_gb * 1_000_000_000)
+    )
+    try:
+        for index, video_path in enumerate(videos, start=1):
+            annotation_path = video_path.with_suffix(".csv")
+            if not annotation_path.exists():
+                raise FileNotFoundError(f"missing annotations: {annotation_path}")
+            total += process_video(
+                video_path,
+                annotation_path,
+                activity.get(video_path.stem, {}),
+                writer,
+                args.clip_seconds,
+                args.overlap_seconds,
+                args.min_face_ratio,
+                args.min_clip_seconds,
+            )
+            print(f"[{index}/{len(videos)}] {video_path.stem}: {total} samples")
+    finally:
+        writer.close()
+    print(f"Wrote {total} samples to {args.output}")
+
+
+if __name__ == "__main__":
+    main()

third_party/MuVAP/preprocess/asd/prepare.py

@@ -0,0 +1,93 @@
+"""Build the canonical fast-training format for AVA, WASD, and AVCC."""
+
+import argparse
+from pathlib import Path
+
+from preprocess.asd.adapters import clip_directory_records, manifest_records
+from preprocess.asd.writer import write_records
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("dataset", choices=["ava", "wasd", "msdwild", "avcc"])
+    parser.add_argument(
+        "--loader", type=Path, required=True, help="source TSV/manifest"
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        help="defaults to <dataset root>/packed/<split>, beside the source media",
+    )
+    parser.add_argument("--visual-root", type=Path)
+    parser.add_argument("--audio-root", type=Path)
+    parser.add_argument("--root", type=Path, help="base for AVCC manifest paths")
+    parser.add_argument("--split", default="train")
+    parser.add_argument("--audio-sample-rate", type=int, default=16_000)
+    parser.add_argument("--max-shard-gb", type=float, default=1.0)
+    parser.add_argument(
+        "--skip-missing",
+        action="store_true",
+        help="development subsets only; production preprocessing should fail on missing files",
+    )
+    return parser
+
+
+def require_roots(args: argparse.Namespace) -> None:
+    if args.visual_root is None or args.audio_root is None:
+        raise ValueError(f"{args.dataset} requires --visual-root and --audio-root")
+
+
+def default_output(args: argparse.Namespace) -> Path:
+    """Keep each corpus's pack beside that corpus.
+
+    `<root>/clips_videos/<split>` becomes `<root>/packed/<split>`, so a dataset
+    stays self-contained and can be copied to fast local storage on its own.
+    """
+    if args.visual_root is not None:
+        return args.visual_root.parent / "packed" / args.split
+    if args.root is not None:
+        return args.root / "packed" / args.split
+    raise ValueError("--output is required when no dataset root is given")
+
+
+def main() -> None:
+    args = build_parser().parse_args()
+    if args.output is None:
+        args.output = default_output(args)
+        print(f"packing into {args.output}")
+    if args.output.exists() and any(args.output.iterdir()):
+        raise FileExistsError(f"output directory is not empty: {args.output}")
+
+    if args.dataset in ("ava", "wasd"):
+        require_roots(args)
+        records = clip_directory_records(
+            args.loader,
+            args.visual_root,
+            args.audio_root,
+            args.split,
+            dataset=args.dataset,
+            skip_missing=args.skip_missing,
+        )
+    elif args.dataset == "msdwild":
+        raise SystemExit(
+            "MSDWild is packed from raw media by preprocess/asd/prep_msdwild.py, "
+            "which needs all.rttm for labels; there is no loader TSV to read here"
+        )
+    else:
+        records = manifest_records(
+            args.loader,
+            dataset="avcc",
+            root=args.root,
+            default_npy_audio_sample_rate=args.audio_sample_rate,
+        )
+
+    count = write_records(
+        records,
+        args.output,
+        max_shard_bytes=round(args.max_shard_gb * 1_000_000_000),
+    )
+    print(f"Wrote {count} {args.dataset} samples to {args.output}")
+
+
+if __name__ == "__main__":
+    main()

third_party/MuVAP/preprocess/asd/schema.py

@@ -0,0 +1,44 @@
+"""Shared source record and packed-shard schema.
+
+The packed format is a *decoded mirror* of the source corpus, not a
+model-ready tensor set. Faces, labels, and audio are stored on the native
+timeline with no gain applied, so any codebase can consume a pack without
+inheriting MuVAP+'s 25 Hz model timeline or its loudness convention. Every
+model-specific transform lives in `data.dataloaders.packed` instead.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Sequence
+
+FORMAT_NAME = "muvap-asd-mmap"
+FORMAT_VERSION = 4
+SUPPORTED_VERSIONS = (4,)
+
+# The model timeline. It belongs to the loader, not to the stored data, and is
+# repeated here only because preprocessing reports frame counts against it.
+MODEL_FPS = 25.0
+
+SAMPLE_RATE = 16_000
+FACE_SIZE = 112
+
+
+@dataclass(frozen=True)
+class SourceRecord:
+    dataset: str
+    sample_id: str
+    visual_path: Path
+    audio_path: Path
+    source_fps: float
+    labels: Sequence[int]
+    start_sec: float = 0.0
+    npy_audio_sample_rate: int | None = None
+    visual_kind: str = "npy"
+
+    def __post_init__(self) -> None:
+        if self.source_fps <= 0:
+            raise ValueError(f"{self.sample_id}: source_fps must be positive")
+        if self.start_sec < 0:
+            raise ValueError(f"{self.sample_id}: start_sec cannot be negative")
+        if self.visual_kind not in {"npy", "jpeg_directory"}:
+            raise ValueError(f"unsupported visual_kind: {self.visual_kind}")

third_party/MuVAP/preprocess/asd/validate.py

@@ -0,0 +1,93 @@
+"""Validate packed samples and report per-dataset signal statistics."""
+
+import argparse
+import json
+import math
+from collections import defaultdict
+from pathlib import Path
+from typing import Sequence
+
+import numpy as np
+
+from data.alignment import model_frame_count
+from data.dataloaders.packed import PackedASDDataset
+from data.media import WaveformStats, normalization_gain
+from preprocess.asd.schema import MODEL_FPS
+from preprocess.asd.writer import native_audio_samples
+
+
+def percentile(values: list[float], points=(0, 1, 50, 95, 99, 100)) -> dict:
+    return {str(point): float(np.percentile(values, point)) for point in points}
+
+
+def validate(roots: Path | Sequence[Path]) -> None:
+    dataset = PackedASDDataset(roots)
+    grouped = defaultdict(lambda: defaultdict(list))
+
+    for index in range(len(dataset)):
+        audio, visual, vad, _, metadata = dataset[index]
+        sample_id = metadata["sample_id"]
+        expected = model_frame_count(
+            metadata["source_frames"], metadata["source_fps"], MODEL_FPS
+        )
+        frames = visual.shape[0]
+        if frames == 0:
+            raise ValueError(f"{sample_id}: empty sample")
+        if frames != expected:
+            raise ValueError(
+                f"{sample_id}: {metadata['source_frames']} source frames at "
+                f"{metadata['source_fps']} fps imply {expected} model frames, "
+                f"loader produced {frames}"
+            )
+        if vad.shape[0] != frames or audio.shape[-1] != frames * 640:
+            raise ValueError(f"{sample_id}: stream-length mismatch")
+        if visual.shape[1:] != (112, 112) or visual.min() < 0 or visual.max() > 255:
+            raise ValueError(f"{sample_id}: invalid visual tensor")
+        if not np.isfinite(metadata["audio_peak"]):
+            raise ValueError(f"{sample_id}: invalid audio statistics")
+        if metadata["audio_samples"] != native_audio_samples(
+            metadata["source_frames"], metadata["source_fps"]
+        ):
+            raise ValueError(f"{sample_id}: stored audio does not span the faces")
+
+        stats = grouped[metadata["dataset"]]
+        stats["source_audio_peak"].append(metadata["audio_peak"])
+        stats["source_audio_rms"].append(metadata["audio_rms"])
+        stats["source_audio_mean"].append(metadata["audio_mean"])
+        stats["speaking_ratio"].append(metadata["speaking_ratio"])
+        stats["normalized_rms"].append(
+            audio.square().mean().sqrt().item()
+        )
+        stats["normalized_peak"].append(audio.abs().max().item())
+        gain = normalization_gain(
+            WaveformStats(
+                mean=metadata["audio_mean"],
+                rms=metadata["audio_rms"],
+                peak=metadata["audio_peak"],
+            )
+        )
+        stats["gain_db"].append(20 * math.log10(max(gain, 1e-8)))
+
+    report = {"samples": len(dataset), "datasets": {}}
+    for name, stats in grouped.items():
+        report["datasets"][name] = {
+            "samples": len(stats["source_audio_peak"]),
+            "source_audio_peak_percentiles": percentile(stats["source_audio_peak"]),
+            "source_audio_rms_percentiles": percentile(stats["source_audio_rms"]),
+            "normalized_rms_percentiles": percentile(stats["normalized_rms"]),
+            "normalized_peak_percentiles": percentile(stats["normalized_peak"]),
+            "gain_db_percentiles": percentile(stats["gain_db"]),
+            "max_abs_source_mean": max(abs(v) for v in stats["source_audio_mean"]),
+            "speaking_ratio_percentiles": percentile(stats["speaking_ratio"]),
+        }
+    print(json.dumps(report, indent=2))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("roots", type=Path, nargs="+")
+    validate(parser.parse_args().roots)
+
+
+if __name__ == "__main__":
+    main()

third_party/MuVAP/preprocess/asd/video.py

@@ -0,0 +1,84 @@
+"""Reading face crops off a source video.
+
+Shared by the dataset adapters that pack from video rather than from
+pre-extracted frames.
+"""
+
+import cv2
+import numpy as np
+
+
+def crop_face(frame, bbox, gray=True, dim=(112, 112)):
+    if bbox is None:
+        return np.full(dim if gray else (dim[1], dim[0], 3), 128, dtype=np.uint8)
+
+    x1, y1, x2, y2 = int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3])
+
+    if x1 >= x2 or y1 >= y2:
+        return np.full(dim if gray else (dim[1], dim[0], 3), 128, dtype=np.uint8)
+
+    h, w = frame.shape[:2]
+    cx, cy = (x1 + x2) // 2, (y1 + y2) // 2
+    size = max(x2 - x1, y2 - y1)
+    half = size // 2
+
+    x1s, y1s = cx - half, cy - half
+    x2s, y2s = x1s + size, y1s + size
+
+    x1c, y1c = max(0, x1s), max(0, y1s)
+    x2c, y2c = min(w, x2s), min(h, y2s)
+
+    if x1c >= x2c or y1c >= y2c:
+        return np.full(dim if gray else (dim[1], dim[0], 3), 128, dtype=np.uint8)
+
+    cropped = frame[y1c:y2c, x1c:x2c]
+
+    if gray:
+        cropped = cv2.cvtColor(cropped, cv2.COLOR_BGR2GRAY)
+        border_val = 128
+    else:
+        cropped = cv2.cvtColor(cropped, cv2.COLOR_BGR2RGB)
+        border_val = (128, 128, 128)
+
+    pad_top, pad_left = y1c - y1s, x1c - x1s
+    pad_bottom, pad_right = y2s - y2c, x2s - x2c
+
+    if pad_top > 0 or pad_bottom > 0 or pad_left > 0 or pad_right > 0:
+        cropped = cv2.copyMakeBorder(
+            cropped,
+            pad_top,
+            pad_bottom,
+            pad_left,
+            pad_right,
+            cv2.BORDER_CONSTANT,
+            value=border_val,
+        )
+
+    return cv2.resize(cropped, dim)
+
+
+def load_visual(video_path, start_frame=0, target_frames=None):
+    """Decode a native-FPS video interval without changing its timeline."""
+    if start_frame < 0:
+        raise ValueError("start_frame cannot be negative")
+    if target_frames is not None and target_frames < 0:
+        raise ValueError("target_frames cannot be negative")
+
+    cap = cv2.VideoCapture(str(video_path))
+    if not cap.isOpened():
+        raise FileNotFoundError(f"could not open video: {video_path}")
+
+    fps = float(cap.get(cv2.CAP_PROP_FPS))
+    if fps <= 0:
+        cap.release()
+        raise ValueError(f"video has an invalid frame rate: {video_path}")
+
+    cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame)
+    frames = []
+    while target_frames is None or len(frames) < target_frames:
+        ok, frame = cap.read()
+        if not ok:
+            break
+        frames.append(frame)
+    cap.release()
+    return frames, fps

third_party/MuVAP/preprocess/asd/writer.py

@@ -0,0 +1,204 @@
+"""Pack decoded native-rate samples into memory-mapped NumPy shards.
+
+Nothing here is specific to the model that will read the shards. Faces and
+labels keep the source frame rate, audio keeps its full duration and its
+original gain, and the per-recording level statistics needed to reproduce a
+normalization travel in the index. Resampling onto a model timeline and
+applying loudness normalization are the reader's job.
+"""
+
+import json
+from pathlib import Path
+from typing import Iterable
+
+import cv2
+import numpy as np
+import torch
+
+from data.alignment import model_frame_count
+from data.media import (
+    AudioSpec,
+    load_waveform,
+    slice_audio,
+    validate_visual,
+    waveform_stats,
+)
+from preprocess.asd.schema import (
+    FACE_SIZE,
+    FORMAT_NAME,
+    FORMAT_VERSION,
+    MODEL_FPS,
+    SAMPLE_RATE,
+    SourceRecord,
+)
+
+
+def _load_faces(record: SourceRecord, face_size: int = FACE_SIZE) -> np.ndarray:
+    if record.visual_kind == "npy":
+        return validate_visual(np.load(record.visual_path, allow_pickle=False))
+
+    paths = sorted(record.visual_path.glob("*.jpg"), key=lambda path: float(path.stem))
+    if not paths:
+        raise FileNotFoundError(f"{record.sample_id}: no JPEG faces")
+    frames = []
+    for path in paths:
+        frame = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE)
+        if frame is None:
+            raise ValueError(f"{record.sample_id}: could not decode {path}")
+        frames.append(
+            cv2.resize(frame, (face_size, face_size), interpolation=cv2.INTER_LINEAR)
+        )
+    return np.stack(frames).astype(np.uint8, copy=False)
+
+
+def _pcm16(waveform: torch.Tensor) -> np.ndarray:
+    values = waveform.squeeze(0).clamp(-1.0, 1.0)
+    return (values * 32767.0).round().to(torch.int16).cpu().numpy()
+
+
+def native_audio_samples(source_frames: int, source_fps: float) -> int:
+    """Audio length covering exactly the span of the stored face frames."""
+    return round(source_frames / source_fps * SAMPLE_RATE)
+
+
+class ShardWriter:
+    def __init__(self, output_dir: Path, max_shard_bytes: int = 1_000_000_000):
+        self.output_dir = output_dir
+        self.max_shard_bytes = max_shard_bytes
+        self.shard_index = 0
+        self.samples_written = 0
+        self._reset()
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+    def _reset(self) -> None:
+        self.faces: list[np.ndarray] = []
+        self.audio: list[np.ndarray] = []
+        self.labels: list[np.ndarray] = []
+        self.entries: list[dict] = []
+        self.byte_count = 0
+        self.face_count = 0
+        self.audio_count = 0
+        self.label_count = 0
+
+    def add_arrays(
+        self,
+        *,
+        dataset: str,
+        sample_id: str,
+        faces: np.ndarray,
+        waveform: torch.Tensor,
+        labels,
+        source_fps: float,
+        start_sec: float = 0.0,
+    ) -> None:
+        faces = validate_visual(faces)
+        labels = np.asarray(labels, dtype=np.uint8)
+        if len(faces) != len(labels):
+            raise ValueError(
+                f"{sample_id}: {len(faces)} faces but {len(labels)} labels"
+            )
+        source_frames = len(labels)
+        if source_frames == 0:
+            raise ValueError(f"{sample_id}: sample is empty")
+
+        waveform = slice_audio(
+            waveform,
+            SAMPLE_RATE,
+            target_samples=native_audio_samples(source_frames, source_fps),
+        )
+        stats = waveform_stats(waveform)
+        audio = _pcm16(waveform)
+
+        sample_bytes = faces.nbytes + audio.nbytes + labels.nbytes
+        if self.entries and self.byte_count + sample_bytes > self.max_shard_bytes:
+            self.flush()
+
+        self.entries.append(
+            {
+                "dataset": dataset,
+                "sample_id": sample_id,
+                "source_fps": source_fps,
+                "source_frames": source_frames,
+                "start_sec": start_sec,
+                # Reported for convenience; the loader recomputes it.
+                "model_frames": model_frame_count(source_frames, source_fps, MODEL_FPS),
+                "face_offset": self.face_count,
+                "audio_offset": self.audio_count,
+                "audio_samples": len(audio),
+                "label_offset": self.label_count,
+                "audio_mean": stats.mean,
+                "audio_rms": stats.rms,
+                "audio_peak": stats.peak,
+                "speaking_ratio": float(labels.mean()),
+            }
+        )
+        self.faces.append(faces)
+        self.audio.append(audio)
+        self.labels.append(labels)
+        self.byte_count += sample_bytes
+        self.face_count += len(faces)
+        self.audio_count += len(audio)
+        self.label_count += len(labels)
+
+    def add(self, record: SourceRecord) -> None:
+        faces = _load_faces(record)
+        waveform, sample_rate = load_waveform(
+            record.audio_path,
+            AudioSpec(sample_rate=SAMPLE_RATE, mono=True),
+            npy_sample_rate=record.npy_audio_sample_rate,
+        )
+        waveform = slice_audio(waveform, sample_rate, start_time=record.start_sec)
+        self.add_arrays(
+            dataset=record.dataset,
+            sample_id=record.sample_id,
+            faces=faces,
+            waveform=waveform,
+            labels=record.labels,
+            source_fps=record.source_fps,
+            start_sec=record.start_sec,
+        )
+
+    def flush(self) -> None:
+        if not self.entries:
+            return
+        shard_dir = self.output_dir / f"shard-{self.shard_index:06d}"
+        shard_dir.mkdir(parents=False, exist_ok=False)
+        np.save(shard_dir / "faces.npy", np.concatenate(self.faces), allow_pickle=False)
+        np.save(shard_dir / "audio.npy", np.concatenate(self.audio), allow_pickle=False)
+        np.save(
+            shard_dir / "labels.npy", np.concatenate(self.labels), allow_pickle=False
+        )
+        with (shard_dir / "index.json").open("w") as handle:
+            json.dump(self.entries, handle, separators=(",", ":"))
+        self.samples_written += len(self.entries)
+        self.shard_index += 1
+        self._reset()
+
+    def close(self) -> None:
+        self.flush()
+        manifest = {
+            "format": FORMAT_NAME,
+            "version": FORMAT_VERSION,
+            "timeline": "native",
+            "audio_sample_rate": SAMPLE_RATE,
+            "audio_normalization": None,
+            "face_shape": [FACE_SIZE, FACE_SIZE],
+            "face_color": "grayscale",
+            "face_dtype": "uint8",
+            "audio_dtype": "int16",
+            "label_dtype": "uint8",
+            "shards": self.shard_index,
+            "samples": self.samples_written,
+        }
+        with (self.output_dir / "dataset.json").open("w") as handle:
+            json.dump(manifest, handle, indent=2)
+
+
+def write_records(
+    records: Iterable[SourceRecord], output_dir: Path, max_shard_bytes: int
+) -> int:
+    writer = ShardWriter(output_dir, max_shard_bytes)
+    for record in records:
+        writer.add(record)
+    writer.close()
+    return writer.samples_written

third_party/MuVAP/preprocess/vap/00_prep_fisher.py

@@ -0,0 +1,196 @@
+"""Cut Fisher into fixed segments of stereo audio beside their raw VAD.
+
+Class labels are *not* baked in here. Each segment stores the two-speaker VAD
+padded with `--context-sec` of extra frames on both sides, which is everything
+`ProjectionWindow` needs to produce role-based or speaker-based labels in the
+dataloader. Switching projection setups therefore never requires re-running
+this script.
+"""
+
+import argparse
+import os
+from dataclasses import dataclass
+from glob import glob
+
+import numpy as np
+from tqdm import tqdm
+
+from config.setup import init_yaml_config
+from data.utils import load_audio, load_fisher_vad
+from preprocess.vap.utils import (
+    SAMPLE_RATE,
+    TEST_GROUPS,
+    VAL_GROUPS,
+    ProcessConfig,
+    get_dataset_split,
+)
+
+SEG_SEC: int = 30
+OVERLAP: int = 10
+VAD_HZ: int = 25
+CONTEXT_SEC: float = 2.0
+
+STRIDE_SEC: int = SEG_SEC - OVERLAP
+SEG_SAMPLES: int = int(SEG_SEC * SAMPLE_RATE)
+STRIDE_SAMPLES: int = int(STRIDE_SEC * SAMPLE_RATE)
+SEG_VAD_FRAMES: int = int(SEG_SEC * VAD_HZ)
+
+
+@dataclass()
+class SegmentMeta:
+    part: str
+    group: str
+    cid: str
+    seg_id: int
+    start_t: float
+    end_t: float
+    nframes: int
+
+
+def format_meta(s: SegmentMeta) -> str:
+    return f"{s.part} {s.group} {s.cid} {s.seg_id} {s.start_t:.2f} {s.end_t:.2f} {s.nframes}\n"
+
+
+def extract_and_save_chunks(
+    audio: np.ndarray,
+    vad: np.ndarray,
+    part: str,
+    group: str,
+    cid: str,
+    cfg: ProcessConfig,
+    context_frames: int,
+):
+    total_samples = audio.shape[-1]
+    total_vad_frames = vad.shape[-1]
+
+    audio_out = os.path.join(cfg.fisher_path, part, "seg", "audio", group, cid)
+    vad_out = os.path.join(cfg.fisher_path, part, "seg", "vad", group, cid)
+    os.makedirs(audio_out, exist_ok=True)
+    os.makedirs(vad_out, exist_ok=True)
+
+    segments = []
+    start_sample = 0
+    seg_idx = 0
+
+    while start_sample + SEG_SAMPLES <= total_samples:
+        end_sample = start_sample + SEG_SAMPLES
+
+        chunk_audio = audio[:, start_sample:end_sample]
+        pcm16 = np.round(np.clip(chunk_audio, -1.0, 1.0) * 32767).astype(np.int16)
+        audio_file = os.path.join(audio_out, f"{seg_idx}.npy")
+        if not os.path.exists(audio_file):
+            np.save(audio_file, pcm16)
+
+        curr_vad_start = int((start_sample / SAMPLE_RATE) * VAD_HZ)
+        curr_vad_end = curr_vad_start + SEG_VAD_FRAMES
+
+        ext_start = max(0, curr_vad_start - context_frames)
+        ext_end = min(total_vad_frames, curr_vad_end + context_frames)
+
+        chunk_vad = vad[:, ext_start:ext_end]
+
+        pad_left = max(0, context_frames - curr_vad_start)
+        pad_right = max(0, (curr_vad_end + context_frames) - total_vad_frames)
+
+        if pad_left > 0 or pad_right > 0:
+            chunk_vad = np.pad(
+                chunk_vad, ((0, 0), (pad_left, pad_right)), mode="constant"
+            )
+
+        vad_file = os.path.join(vad_out, f"{seg_idx}.npy")
+        if not os.path.exists(vad_file):
+            np.save(vad_file, chunk_vad.astype(np.float32))
+
+        segments.append(
+            SegmentMeta(
+                part=part,
+                group=group,
+                cid=cid,
+                seg_id=seg_idx,
+                start_t=start_sample / SAMPLE_RATE,
+                end_t=end_sample / SAMPLE_RATE,
+                nframes=total_vad_frames,
+            )
+        )
+
+        start_sample += STRIDE_SAMPLES
+        seg_idx += 1
+
+    return segments
+
+
+def process_entry(
+    part: str,
+    group: str,
+    cid: str,
+    cfg: ProcessConfig,
+    context_frames: int,
+):
+    audio_path = os.path.join(cfg.fisher_path, part, "audio", group, f"{cid}.wav")
+    vad_dir = os.path.join(cfg.fisher_path, part, "regions_vap", group)
+
+    if not os.path.exists(audio_path):
+        return []
+
+    # Both channels are kept: the speaker-based setup needs one per speaker and
+    # the mono setups downmix in the dataloader.
+    audio_tensor, duration = load_audio(audio_path, mono=False)
+    audio = audio_tensor.numpy()
+
+    nframes = int(round(duration * VAD_HZ))
+    vad_tensor = load_fisher_vad(vad_dir, cid, nframes)
+    vad = vad_tensor.numpy()
+
+    return extract_and_save_chunks(audio, vad, part, group, cid, cfg, context_frames)
+
+
+def run_pipeline(cfg: ProcessConfig, context_frames: int):
+    pattern = os.path.join(cfg.fisher_path, "*", "regions_vap", "*", "*_words.txt")
+    all_files = glob(pattern)
+
+    entries = {"train": [], "val": [], "test": []}
+    seen = set()
+
+    for vad_file in tqdm(all_files, desc="Building Dataset"):
+        parts = vad_file.split(os.sep)
+        part, group = parts[-4], parts[-2]
+        cid = "_".join(os.path.basename(vad_file).split("_")[:3])
+
+        if (part, group, cid) in seen:
+            continue
+        seen.add((part, group, cid))
+
+        split = get_dataset_split(group, cfg)
+        if split not in cfg.active_splits:
+            continue
+
+        segments = process_entry(part, group, cid, cfg, context_frames)
+        entries[split].extend(format_meta(s) for s in segments)
+
+    for split in cfg.active_splits:
+        lines = entries.get(split, [])
+        if lines:
+            out_file = os.path.join(cfg.fisher_path, f"{split}.txt")
+            with open(out_file, "w") as f:
+                f.writelines(lines)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", default="config/yaml/vap_role.yaml")
+    parser.add_argument(
+        "--context-sec",
+        type=float,
+        default=CONTEXT_SEC,
+        help="VAD kept on each side of a segment; must cover the widest projection window",
+    )
+    args = parser.parse_args()
+    yaml_cfg = init_yaml_config(args.config)
+
+    config = ProcessConfig(
+        fisher_path=yaml_cfg["fisher_path"],
+        val_groups=VAL_GROUPS,
+        test_groups=TEST_GROUPS,
+        active_splits={"train", "val", "test"},
+    )
+    run_pipeline(config, int(round(args.context_sec * VAD_HZ)))

third_party/MuVAP/preprocess/vap/01_create_event.py

@@ -0,0 +1,268 @@
+import argparse
+import os
+from dataclasses import dataclass
+from glob import glob
+from typing import List, Set, Tuple
+
+from tqdm import tqdm
+
+from config.setup import init_yaml_config
+from preprocess.vap.utils import print_dataset_summary
+
+
+@dataclass
+class LogicConfig:
+    pause_threshold: float = 0.1
+    long_pause_threshold: float = 1.0
+    max_pause_length: float = 3.0
+    active_window: float = 0.5
+    purity_win: float = 1.0
+    min_speech_density: float = 1.0
+
+
+@dataclass
+class SpeechSegment:
+    start_t: float
+    end_t: float
+    speaker: str
+
+
+@dataclass
+class TurnEvent:
+    part: str
+    group: str
+    conv_id: str
+    region_start: float
+    region_end: float
+    prev_spk: str
+    next_spk: str
+    label: str
+    timing: str
+    gap_type: str
+
+
+def parse_vad_file(file_path: str, speaker_id: str):
+    segments = []
+    with open(file_path, "r") as f:
+        for line in f:
+            parts = line.strip().split()
+            if len(parts) >= 2:
+                segments.append(
+                    SpeechSegment(float(parts[0]), float(parts[1]), speaker_id)
+                )
+    return segments
+
+
+def merge_into_ipus(segments: List[SpeechSegment], cfg: LogicConfig):
+    if not segments:
+        return []
+
+    sorted_segs = sorted(segments, key=lambda s: s.start_t)
+    ipus = []
+
+    first = sorted_segs[0]
+    c_start, c_end, c_spk = first.start_t, first.end_t, first.speaker
+
+    for seg in sorted_segs[1:]:
+        if seg.speaker == c_spk and (seg.start_t - c_end) <= cfg.pause_threshold:
+            c_end = max(c_end, seg.end_t)
+        else:
+            ipus.append(SpeechSegment(c_start, c_end, c_spk))
+            c_start, c_end, c_spk = seg.start_t, seg.end_t, seg.speaker
+
+    ipus.append(SpeechSegment(c_start, c_end, c_spk))
+    return ipus
+
+
+def get_pure_speaker(
+    ipus: List[SpeechSegment], win_start: float, win_end: float, min_density: float
+):
+    speaker_time = {}
+
+    for ipu in ipus:
+        if ipu.end_t <= win_start:
+            continue
+        if ipu.start_t >= win_end:
+            break
+
+        o_start = max(ipu.start_t, win_start)
+        o_end = min(ipu.end_t, win_end)
+
+        if o_end > o_start:
+            speaker_time[ipu.speaker] = speaker_time.get(ipu.speaker, 0.0) + (
+                o_end - o_start
+            )
+
+    active_speakers = [spk for spk, duration in speaker_time.items() if duration > 0]
+
+    if len(active_speakers) == 1:
+        spk = active_speakers[0]
+        if speaker_time[spk] >= min_density:
+            return spk
+
+    return None
+
+
+def get_mutual_silences(ipus: List[SpeechSegment]):
+    if not ipus:
+        return []
+
+    sorted_all = sorted(ipus, key=lambda x: x.start_t)
+    merged_speech = []
+
+    c_s, c_e = sorted_all[0].start_t, sorted_all[0].end_t
+    for ipu in sorted_all[1:]:
+        if ipu.start_t <= c_e:
+            c_e = max(c_e, ipu.end_t)
+        else:
+            merged_speech.append((c_s, c_e))
+            c_s, c_e = ipu.start_t, ipu.end_t
+    merged_speech.append((c_s, c_e))
+
+    gaps = []
+    for i in range(len(merged_speech) - 1):
+        gaps.append((merged_speech[i][1], merged_speech[i + 1][0]))
+
+    return gaps
+
+
+def get_overlaps(ipus: List[SpeechSegment]):
+    overlaps = []
+    sorted_ipus = sorted(ipus, key=lambda x: x.start_t)
+    active = []
+
+    for ipu in sorted_ipus:
+        active = [a for a in active if a.end_t > ipu.start_t]
+
+        for a in active:
+            if a.speaker != ipu.speaker:
+                o_start = ipu.start_t
+                o_end = min(a.end_t, ipu.end_t)
+                if o_end > o_start:
+                    overlaps.append((o_start, o_end))
+        active.append(ipu)
+
+    return overlaps
+
+
+def extract_events(
+    ipus: List[SpeechSegment], part: str, group: str, conv_id: str, cfg: LogicConfig
+):
+    events = []
+    sorted_ipus = sorted(ipus, key=lambda s: s.start_t)
+
+    gaps = get_mutual_silences(sorted_ipus)
+
+    for g_start, g_end in gaps:
+        g_dur = g_end - g_start
+
+        prev_spk = get_pure_speaker(
+            sorted_ipus, g_start - cfg.purity_win, g_start, cfg.min_speech_density
+        )
+        next_spk = get_pure_speaker(
+            sorted_ipus, g_end, g_end + cfg.purity_win, cfg.min_speech_density
+        )
+
+        if prev_spk is not None and next_spk is not None:
+            label = "HOLD" if prev_spk == next_spk else "SHIFT"
+            g_type = "LONG" if g_dur >= cfg.long_pause_threshold else "SHORT"
+
+            if g_dur <= cfg.max_pause_length:
+                events.append(
+                    TurnEvent(
+                        part,
+                        group,
+                        conv_id,
+                        g_start,
+                        g_end,
+                        prev_spk,
+                        next_spk,
+                        label,
+                        "SILENT",
+                        g_type,
+                    )
+                )
+
+            region_start = g_start - cfg.active_window
+            region_end = g_start
+            predicted_gap_len = g_dur + cfg.active_window
+
+            if predicted_gap_len <= cfg.max_pause_length:
+                events.append(
+                    TurnEvent(
+                        part,
+                        group,
+                        conv_id,
+                        region_start,
+                        region_end,
+                        prev_spk,
+                        next_spk,
+                        label,
+                        "ACTIVE",
+                        g_type,
+                    )
+                )
+
+    return events
+
+
+def collect_vad(fisher_path: str, part: str):
+    seen: Set[Tuple[str, str]] = set()
+    conv = []
+    pattern = os.path.join(fisher_path, part, "regions_vap", "*", "*_words.txt")
+
+    for vad_file in glob(pattern):
+        group = vad_file.split(os.sep)[-2]
+        file_name = os.path.basename(vad_file)
+        if not file_name.endswith("_words.txt"):
+            continue
+
+        conv_id = "_".join(file_name.split("_")[:3])
+        key = (group, conv_id)
+        if key not in seen:
+            seen.add(key)
+            conv.append(key)
+
+    return conv
+
+
+def run_pipeline(fisher_path: str):
+    cfg = LogicConfig()
+    lines = []
+    all_events = []
+
+    for partition in ["p1", "p2"]:
+        conversations = collect_vad(fisher_path, partition)
+
+        for group, conv_id in tqdm(conversations, desc=f"Gen labels {partition}"):
+            vad_dir = os.path.join(fisher_path, partition, "regions_vap", group)
+            vad_files = sorted(glob(os.path.join(vad_dir, f"{conv_id}_*_words.txt")))
+            if not vad_files:
+                continue
+
+            all_ipus = []
+            for i, file in enumerate(vad_files):
+                raw_segments = parse_vad_file(file, str(i))
+                all_ipus.extend(merge_into_ipus(raw_segments, cfg))
+
+            events = extract_events(all_ipus, partition, group, conv_id, cfg)
+            all_events.extend(events)
+
+            for e in events:
+                lines.append(
+                    f"{e.part} {e.group} {e.conv_id} {e.region_start:.2f} {e.region_end:.2f} "
+                    f"{e.prev_spk} {e.next_spk} {e.timing} {e.gap_type} {e.label}"
+                )
+
+    out_path = os.path.join(fisher_path, "turn_events.txt")
+    with open(out_path, "w") as f:
+        f.write("\n".join(lines))
+
+    print_dataset_summary(all_events)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Create Fisher turn events")
+    parser.add_argument("--config", default="config/yaml/vap_role.yaml")
+    args = parser.parse_args()
+    run_pipeline(init_yaml_config(args.config)["fisher_path"])

third_party/MuVAP/preprocess/vap/02_prep_event.py

@@ -0,0 +1,217 @@
+import argparse
+import os
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import List
+
+import numpy as np
+import torch
+from tqdm import tqdm
+
+from config.setup import init_yaml_config
+from data.utils import load_audio
+from preprocess.vap.utils import (
+    EVENT_PADDING_SEC,
+    EVENT_SEG_LEN_SEC,
+    SAMPLE_RATE,
+    TEST_GROUPS,
+    VAL_GROUPS,
+    ProcessConfig,
+    event_crop,
+    get_dataset_split,
+    print_dataset_summary,
+)
+
+SEG_LEN_SEC: int = EVENT_SEG_LEN_SEC
+PADDING_SEC: float = EVENT_PADDING_SEC
+
+
+AUDIO_DIRS = {1: "audio", 2: "audio_stereo"}
+
+
+@dataclass(slots=True)
+class TurnEvent:
+    part: str
+    group: str
+    conv_id: str
+    region_start: float
+    region_end: float
+    prev_spk: str
+    next_spk: str
+    timing: str
+    gap_type: str
+    label: str
+    crop_start: float = 0.0
+    crop_end: float = 0.0
+
+
+def parse_line(line: str) -> TurnEvent:
+    tokens = line.strip().split()
+    if len(tokens) != 10:
+        return None
+
+    entry = TurnEvent(
+        part=tokens[0],
+        group=tokens[1],
+        conv_id=tokens[2],
+        region_start=float(tokens[3]),
+        region_end=float(tokens[4]),
+        prev_spk=tokens[5],
+        next_spk=tokens[6],
+        timing=tokens[7],
+        gap_type=tokens[8],
+        label=tokens[9],
+    )
+
+    crop = event_crop(entry.timing, entry.region_start, entry.region_end)
+    if crop is None:
+        return None
+    entry.crop_start, entry.crop_end = crop
+
+    return entry
+
+
+def group_by_conversation(entries: List[TurnEvent]):
+    grouped = defaultdict(list)
+    for entry in entries:
+        grouped[(entry.part, entry.group, entry.conv_id)].append(entry)
+    return dict(grouped)
+
+
+def format_output_line(entry: TurnEvent):
+    location = f"{entry.part} {entry.group} {entry.conv_id} {entry.region_start:.2f} {entry.region_end:.2f}"
+    event = f"{entry.prev_spk} {entry.next_spk} {entry.timing} {entry.gap_type} {entry.label}"
+    return f"{location} {event}"
+
+
+def load_entries_from_disk(cfg: ProcessConfig):
+    splits = {"train": [], "val": [], "test": []}
+    path = os.path.join(cfg.fisher_path, cfg.input_file)
+
+    with open(path, "r") as f:
+        parsed_entries = (parse_line(line) for line in f)
+
+        valid_entries = (e for e in parsed_entries if e is not None)
+
+        for entry in valid_entries:
+            split_name = get_dataset_split(entry.group, cfg)
+            if split_name in cfg.active_splits:
+                splits[split_name].append(entry)
+
+    return splits
+
+
+def save_segment(entry: TurnEvent, audio: np.ndarray, cfg: ProcessConfig, channels: int):
+    seg_samples = int(SEG_LEN_SEC * SAMPLE_RATE)
+    start_sample = int(entry.crop_start * SAMPLE_RATE)
+    end_sample = start_sample + seg_samples
+
+    if end_sample > audio.shape[1]:
+        return False
+
+    seg = audio[:, start_sample:end_sample]
+    if seg.shape[1] != seg_samples:
+        return False
+
+    out_dir = os.path.join(
+        cfg.fisher_path,
+        entry.part,
+        "tune",
+        AUDIO_DIRS[channels],
+        entry.group,
+        entry.conv_id,
+    )
+    os.makedirs(out_dir, exist_ok=True)
+
+    out_path = os.path.join(out_dir, f"{entry.region_start:.2f}.npy")
+    if not os.path.exists(out_path):
+        pcm16 = (seg.clamp(-1.0, 1.0) * 32767).round().to(torch.int16)
+        np.save(out_path, pcm16.numpy())
+
+    return True
+
+
+def process_split(
+    split_name: str, entries: List[TurnEvent], cfg: ProcessConfig, channels: int
+):
+    if not entries:
+        return [], []
+
+    out_lines: List[str] = []
+    successful_events: List[TurnEvent] = []
+    grouped_convos = group_by_conversation(entries)
+
+    for (part, group, cid), convo_entries in tqdm(
+        grouped_convos.items(), desc=f"Processing {split_name}"
+    ):
+        audio_sub_dir = "p1" if part == "p1" else "p2"
+        audio_path = os.path.join(
+            cfg.fisher_path, audio_sub_dir, "audio", group, f"{cid}.wav"
+        )
+
+        if not os.path.exists(audio_path):
+            continue
+
+        audio, _ = load_audio(audio_path, mono=channels == 1)
+
+        for entry in convo_entries:
+            if save_segment(entry, audio, cfg, channels):
+                out_lines.append(format_output_line(entry))
+                successful_events.append(entry)
+
+    return out_lines, successful_events
+
+
+def run_pipeline(cfg: ProcessConfig, channels: int = 1):
+    splits = load_entries_from_disk(cfg)
+    all_saved_events = []
+
+    for split_name in cfg.active_splits:
+        entries = splits.get(split_name, [])
+        if not entries:
+            continue
+
+        out_lines, success_events = process_split(split_name, entries, cfg, channels)
+        all_saved_events.extend(success_events)
+
+        # The manifest lists which events have audio, so it is written once per
+        # channel layout and stays identical between them.
+        if out_lines:
+            out_path = os.path.join(cfg.fisher_path, f"{split_name}_events.txt")
+            with open(out_path, "w") as f:
+                f.write("\n".join(out_lines) + "\n")
+
+    if all_saved_events:
+        print_dataset_summary(all_saved_events)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", default="config/yaml/vap_role.yaml")
+    parser.add_argument(
+        "--splits",
+        nargs="+",
+        choices=("train", "val", "test"),
+        default=["val", "test"],
+        help="splits to cut clips for; training reads segments, not events, so "
+        "train clips are only needed to enlarge the probe's tuning pool",
+    )
+    parser.add_argument(
+        "--channels",
+        type=int,
+        choices=(1, 2),
+        default=1,
+        help="1 writes tune/audio for the mono setups, 2 writes tune/audio_stereo for original VAP",
+    )
+    args = parser.parse_args()
+    yaml_cfg = init_yaml_config(args.config)
+
+    config = ProcessConfig(
+        fisher_path=yaml_cfg["fisher_path"],
+        input_file="turn_events.txt",
+        val_groups=VAL_GROUPS,
+        test_groups=TEST_GROUPS,
+        active_splits=set(args.splits),
+    )
+
+    run_pipeline(config, channels=args.channels)

third_party/MuVAP/preprocess/vap/03_prep_vad_rate.py

@@ -0,0 +1,112 @@
+"""Write the segment VAD again at a different frame rate.
+
+`00_prep_fisher.py` rasterizes the word annotations at `VAD_HZ` and stores one
+`.npy` per segment beside the audio. A projection window running at another rate
+needs its labels on that grid: resampling the 25 Hz array would inherit its
+quantisation, so this rasterizes the annotations again from their original
+continuous times.
+
+Only the VAD is rewritten. The audio is rate-independent and stays where it is,
+so this touches ~1 GB rather than the 1.12 TB of segments, and it writes to
+`seg/vad{rate}/` rather than over `seg/vad/`.
+
+    python preprocess/vap/03_prep_vad_rate.py --frame-hz 12.5 \
+        --fisher-path /path/to/fisher --splits train val test
+"""
+
+import argparse
+from collections import defaultdict
+from pathlib import Path
+
+import numpy as np
+
+from preprocess.vap.utils import SAMPLE_RATE
+
+SEG_SEC = 30
+CONTEXT_SEC = 2.0
+
+
+def rasterize(words_dir, conversation, frames, frame_hz):
+    """`[2, frames]` activity for a whole conversation, from the word times."""
+    vad = np.zeros((2, frames), dtype=np.float32)
+    paths = sorted(words_dir.glob(f"{conversation}_*_words.txt"))
+    if len(paths) != 2:
+        raise FileNotFoundError(
+            f"expected two *_words.txt for {conversation} in {words_dir}, found {len(paths)}"
+        )
+    for speaker, path in enumerate(paths):
+        with open(path) as handle:
+            for line in handle:
+                if not line.strip():
+                    continue
+                begin, end, *_ = line.split()
+                first = max(0, int(round(float(begin) * frame_hz)))
+                last = min(frames, int(round(float(end) * frame_hz)))
+                if last > first:
+                    vad[speaker, first:last] = 1.0
+    return vad
+
+
+def segment_window(vad, start_sec, frame_hz, context_frames):
+    """The stored slice for one segment: the segment plus context, zero-padded.
+
+    Mirrors `extract_and_save_chunks`, so a reader can treat these files exactly
+    as it treats the 25 Hz ones.
+    """
+    seg_frames = int(round(SEG_SEC * frame_hz))
+    start = int(round(start_sec * frame_hz))
+    end = start + seg_frames
+    lo, hi = max(0, start - context_frames), min(vad.shape[-1], end + context_frames)
+    chunk = vad[:, lo:hi]
+    pad_left = max(0, context_frames - start)
+    pad_right = max(0, (end + context_frames) - vad.shape[-1])
+    if pad_left or pad_right:
+        chunk = np.pad(chunk, ((0, 0), (pad_left, pad_right)))
+    return chunk
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--fisher-path", required=True)
+    parser.add_argument("--frame-hz", type=float, required=True)
+    parser.add_argument("--splits", nargs="+", default=["train", "val", "test"])
+    parser.add_argument("--overwrite", action="store_true")
+    args = parser.parse_args()
+
+    root = Path(args.fisher_path)
+    label = f"{args.frame_hz:g}".replace(".", "_")
+    context_frames = int(round(CONTEXT_SEC * args.frame_hz))
+
+    # One rasterization per conversation, then slice every segment out of it.
+    by_conversation = defaultdict(list)
+    for split in args.splits:
+        for line in open(root / f"{split}.txt"):
+            if not line.strip():
+                continue
+            part, group, conversation, seg_id, start, *_ = line.split()
+            by_conversation[(part, group, conversation)].append((int(seg_id), float(start)))
+
+    written = skipped = 0
+    for index, ((part, group, conversation), segments) in enumerate(sorted(by_conversation.items()), 1):
+        out = root / part / "seg" / f"vad{label}" / group / conversation
+        out.mkdir(parents=True, exist_ok=True)
+        targets = [(s, p) for s, p in ((s, out / f"{s}.npy") for s, _ in segments)
+                   if args.overwrite or not p.exists()]
+        if not targets:
+            skipped += len(segments)
+            continue
+        last_start = max(start for _, start in segments)
+        frames = int(round((last_start + SEG_SEC + CONTEXT_SEC) * args.frame_hz)) + 1
+        vad = rasterize(root / part / "regions_vap" / group, conversation, frames, args.frame_hz)
+        starts = dict(segments)
+        for seg_id, path in targets:
+            np.save(path, segment_window(vad, starts[seg_id], args.frame_hz, context_frames))
+            written += 1
+        if index % 500 == 0:
+            print(f"  {index}/{len(by_conversation)} conversations, {written} written", flush=True)
+    print(f"done: {written} written, {skipped} already present, "
+          f"{len(by_conversation)} conversations -> seg/vad{label}/")
+
+
+if __name__ == "__main__":
+    main()

third_party/MuVAP/preprocess/vap/utils.py

@@ -0,0 +1,98 @@
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import List
+
+SAMPLE_RATE: int = 16000
+VAL_GROUPS = {"000", "016", "032", "048", "096", "112"}
+TEST_GROUPS = {"064", "080"}
+
+
+@dataclass(slots=True)
+class ProcessConfig:
+    fisher_path: str
+    val_groups: set[str]
+    test_groups: set[str]
+    active_splits: set[str]
+    input_file: str = ""
+
+
+def get_dataset_split(group: str, cfg: ProcessConfig):
+    if group in cfg.val_groups:
+        return "val"
+    if group in cfg.test_groups:
+        return "test"
+    return "train"
+
+
+def print_dataset_summary(events: List, is_turn_event=True):
+    stats = defaultdict(lambda: defaultdict(int))
+    categories = set()
+
+    for e in events:
+        split = get_dataset_split(
+            e.group, ProcessConfig("", VAL_GROUPS, TEST_GROUPS, set())
+        )
+
+        cat = f"{e.timing} {e.label}" if is_turn_event else e.label
+        categories.add(cat)
+        stats[split][cat] += 1
+        stats["TOTAL"][cat] += 1
+
+    sorted_cats = sorted(list(categories))
+
+    header = f"{'SPLIT':<10}"
+    for cat in sorted_cats:
+        header += f" | {cat:<15}"
+    header += f" | {'TOTAL'}"
+
+    width = len(header)
+    print(f"\n{'=' * width}")
+    print(f"{'DATASET EVENT SUMMARY':^{width}}")
+    print(f"{'=' * width}")
+    print(header)
+    print(f"{'-' * width}")
+
+    for split in ["train", "val", "test", "TOTAL"]:
+        row = f"{split.upper():<10}"
+        total = 0
+        for cat in sorted_cats:
+            val = stats[split][cat]
+            total += val
+            row += f" | {val:<15}"
+        row += f" | {total}"
+        print(row)
+    print(f"{'=' * width}\n")
+
+
+EVENT_SEG_LEN_SEC: int = 10
+EVENT_PADDING_SEC: float = 0.1
+
+
+def event_crop(
+    timing: str,
+    region_start: float,
+    region_end: float,
+    seg_len_sec: float = EVENT_SEG_LEN_SEC,
+    padding_sec: float = EVENT_PADDING_SEC,
+):
+    """Audio window an evaluation clip covers, or None if the event has none.
+
+    The window ends just before the decision point: a hair into the silence for
+    a SILENT event, a hair before the speech ends for an ACTIVE one. This is the
+    single definition of that window - `02_prep_event.py` writes clips with it
+    and the raw dataloader reads the same span straight from the source audio,
+    so the two cannot drift apart.
+    """
+    if timing == "SILENT":
+        crop_end = region_start + padding_sec
+        if crop_end > region_end:
+            return None
+    elif timing == "ACTIVE":
+        crop_end = region_end - padding_sec
+    else:
+        return None
+
+    crop_start = crop_end - seg_len_sec
+    if crop_start < 0:
+        return None
+    return crop_start, crop_end