Add scripts for preparing and extracting 5 FPS audio-video clips, including prepare_ego4d_2fps.py, run_extract_5fps_2pergpu.sh, and run_prepare_all_5fps.sh. Introduce streaming_turn_model_directly.py for direct extraction from original media. Update README.md to reflect changes in Qwen's streaming design and causal extraction modes. Add new AGENTS.md file for documentation.

streaming_turn_model.py

@@ -190,10 +190,10 @@ class FixedGridAV:
     def __init__(self, path: str | Path, expected_fps: float) -> None:
         try:
             import decord
-            import librosa
+            import numpy as np
         except ImportError as exc:
             raise RuntimeError(
-                "Fixed-grid AV extraction requires decord and librosa"
+                "Fixed-grid AV extraction requires decord and numpy"
             ) from exc
 
         self.path = Path(path).resolve()
@@ -204,9 +204,43 @@ class FixedGridAV:
                 f"Prepared video is {self.fps:g} FPS, expected {expected_fps:g}"
             )
         self.num_frames = len(self.reader)
-        self.audio, _ = librosa.load(
-            str(self.path), sr=self.sample_rate, mono=True
+        # SoundFile cannot decode the AAC track in MP4, causing librosa to
+        # fall back to its deprecated and much slower audioread path. Decode
+        # directly to the exact float32/16-kHz/mono representation consumed by
+        # Qwen. Keeping the complete track in memory also makes repeated
+        # 200-ms slicing deterministic and inexpensive.
+        completed = subprocess.run(
+            [
+                "ffmpeg",
+                "-nostdin",
+                "-v",
+                "error",
+                "-i",
+                str(self.path),
+                "-map",
+                "0:a:0",
+                "-vn",
+                "-ac",
+                "1",
+                "-ar",
+                str(self.sample_rate),
+                "-c:a",
+                "pcm_f32le",
+                "-f",
+                "f32le",
+                "pipe:1",
+            ],
+            check=False,
+            capture_output=True,
         )
+        if completed.returncode != 0:
+            message = completed.stderr.decode("utf-8", errors="replace").strip()
+            raise RuntimeError(
+                f"Failed to decode embedded audio from {self.path}: {message}"
+            )
+        self.audio = np.frombuffer(completed.stdout, dtype="<f4").copy()
+        if not len(self.audio):
+            raise RuntimeError(f"Decoded audio is empty: {self.path}")
 
     def frames(self, start: float, end: float) -> list[Any]:
         """Return fixed-grid frames whose timestamps lie in [start, end)."""