Add scripts for preparing and extracting 5 FPS audio-video clips, including prepare_ego4d_2fps.py, run_extract_5fps_2pergpu.sh, and run_prepare_all_5fps.sh. Introduce streaming_turn_model_directly.py for direct extraction from original media. Update README.md to reflect changes in Qwen's streaming design and causal extraction modes. Add new AGENTS.md file for documentation.
This commit is contained in:
Xin Wang
@@ -0,0 +1,913 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract causal Qwen2.5-Omni states directly from original media.
|
||||
|
||||
Unlike ``prepare_easycom_session.py`` and ``prepare_ego4d_2fps.py``, this
|
||||
entry point does not create a re-encoded fixed-frame-rate MP4. It builds a
|
||||
global visual sampling lattice in Python, selects the newest decoded source
|
||||
frame whose presentation timestamp is not later than each lattice time, and
|
||||
passes explicit PIL frame lists plus exact 16-kHz audio slices to Qwen.
|
||||
|
||||
The default mode follows the released Qwen2.5-Omni streaming design as closely
|
||||
as the public Transformers implementation permits:
|
||||
|
||||
* 2 FPS causal visual sampling (configurable for ablations);
|
||||
* two-frame temporal patches, with the last observed frame duplicated when an
|
||||
incomplete input contains an odd number of frames;
|
||||
* a single open media turn with completed 2-second AV payload commits;
|
||||
* persistent Thinker KV for the complete extraction (``--memory-seconds 0``);
|
||||
* one temporary partial-block branch and readout every 200 ms.
|
||||
|
||||
Examples
|
||||
--------
|
||||
Ego4D, directly from one original clip::
|
||||
|
||||
python streaming_turn_model_directly.py \
|
||||
--model-id /data/wangx/Models/Qwen2.5-Omni-3B \
|
||||
--video /data/wangx/Datasets/ego4d/v1/clips/UID.mp4 \
|
||||
--target /data/wangx/Datasets/ego4d/ego4d_target_frame/UID.npy \
|
||||
--mode open-stream-replay \
|
||||
--memory-seconds 20 \
|
||||
--video-fps 2 \
|
||||
--flash-attention-2 \
|
||||
--device cuda:0 \
|
||||
--output features/ego4d_direct/UID.pt
|
||||
|
||||
EasyCom, directly from the retained one-minute source clips::
|
||||
|
||||
python streaming_turn_model_directly.py \
|
||||
--model-id /path/to/Qwen2.5-Omni-3B \
|
||||
--easycom-session-dir /path/to/Main/Video_Compressed/Session_1 \
|
||||
--target target_frame/Session_1.npy \
|
||||
--mode open-stream-replay \
|
||||
--memory-seconds 20 \
|
||||
--video-fps 2 \
|
||||
--flash-attention-2 \
|
||||
--device cuda:0 \
|
||||
--output features/easycom_direct/Session_1.pt
|
||||
|
||||
``--media-only`` performs source/timestamp/audio validation without loading
|
||||
Qwen. This is useful before launching a full extraction on a remote machine.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import shutil
|
||||
import subprocess
|
||||
from collections import OrderedDict
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
import streaming_turn_model as core
|
||||
from easycom_manifest import parse_timecode
|
||||
|
||||
|
||||
EPSILON = 1e-7
|
||||
SAMPLE_RATE = 16_000
|
||||
EASYCOM_WIDTH = 1920
|
||||
EASYCOM_HEIGHT = 1080
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceClipSpec:
|
||||
"""One source file in a dense, gap-free media timeline."""
|
||||
|
||||
path: Path
|
||||
duration: float | None = None
|
||||
crop: tuple[int, int, int, int] | None = None
|
||||
crop_rightmost_width: int | None = None
|
||||
audio_path: Path | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class IndexedSourceClip:
|
||||
"""Decoded indexing information for one source clip."""
|
||||
|
||||
path: Path
|
||||
audio_path: Path
|
||||
dense_start: float
|
||||
dense_end: float
|
||||
duration: float
|
||||
crop: tuple[int, int, int, int] | None
|
||||
reader: Any
|
||||
source_frame_starts: np.ndarray
|
||||
source_frame_ends: np.ndarray
|
||||
source_fps: float
|
||||
width: int
|
||||
height: int
|
||||
|
||||
@property
|
||||
def num_frames(self) -> int:
|
||||
return int(len(self.source_frame_starts))
|
||||
|
||||
|
||||
def _grid_indices(start: float, end: float, fps: float) -> list[int]:
|
||||
"""Indices of global samples ``n/fps`` in the half-open interval."""
|
||||
|
||||
first = max(0, int(math.ceil(start * fps - EPSILON)))
|
||||
last_exclusive = int(math.ceil(end * fps - EPSILON))
|
||||
if first >= last_exclusive:
|
||||
# Qwen's visual tower needs at least one observed image before its
|
||||
# temporal-patch duplication. Reuse the latest global tick strictly
|
||||
# before ``end``; this is identical to FixedGridAV's short-tail rule.
|
||||
return [max(0, last_exclusive - 1)]
|
||||
return list(range(first, last_exclusive))
|
||||
|
||||
|
||||
def _timestamp_array(reader: Any) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Return monotonically increasing, zero-origin frame PTS intervals."""
|
||||
|
||||
# Decord releases differ here: some accept an ndarray, while older ones
|
||||
# require a Python list. A list works with both APIs.
|
||||
indices = list(range(len(reader)))
|
||||
raw = reader.get_frame_timestamp(indices)
|
||||
if hasattr(raw, "asnumpy"):
|
||||
raw = raw.asnumpy()
|
||||
timestamps = np.asarray(raw, dtype=np.float64)
|
||||
if timestamps.ndim == 1:
|
||||
starts = timestamps
|
||||
nominal = 1.0 / float(reader.get_avg_fps())
|
||||
ends = np.concatenate((starts[1:], starts[-1:] + nominal))
|
||||
elif timestamps.ndim == 2 and timestamps.shape[1] >= 2:
|
||||
starts = timestamps[:, 0]
|
||||
ends = timestamps[:, 1]
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Unexpected decord frame-timestamp shape: {timestamps.shape}"
|
||||
)
|
||||
if not len(starts):
|
||||
raise RuntimeError("Video contains no decoded frames")
|
||||
origin = float(starts[0])
|
||||
starts = starts - origin
|
||||
ends = ends - origin
|
||||
if np.any(np.diff(starts) < -1e-9):
|
||||
raise RuntimeError("Decoded video frame PTS values are not monotonic")
|
||||
return starts, ends
|
||||
|
||||
|
||||
def _decode_audio_exact(path: Path, duration: float) -> np.ndarray:
|
||||
"""Decode, then pad/trim one clip to its dense video duration."""
|
||||
|
||||
completed = subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-nostdin",
|
||||
"-v",
|
||||
"error",
|
||||
"-i",
|
||||
str(path),
|
||||
"-map",
|
||||
"0:a:0",
|
||||
"-vn",
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
str(SAMPLE_RATE),
|
||||
"-c:a",
|
||||
"pcm_f32le",
|
||||
"-f",
|
||||
"f32le",
|
||||
"pipe:1",
|
||||
],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
message = completed.stderr.decode("utf-8", errors="replace").strip()
|
||||
raise RuntimeError(f"Failed to decode audio from {path}: {message}")
|
||||
|
||||
decoded = np.frombuffer(completed.stdout, dtype="<f4")
|
||||
expected = int(round(duration * SAMPLE_RATE))
|
||||
output = np.zeros(expected, dtype=np.float32)
|
||||
used = min(expected, len(decoded))
|
||||
if used:
|
||||
output[:used] = decoded[:used]
|
||||
return output
|
||||
|
||||
|
||||
class CausalSourceAV:
|
||||
"""Direct PTS-aware reader implementing the existing extractor AV API."""
|
||||
|
||||
sample_rate = SAMPLE_RATE
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
specs: list[SourceClipSpec],
|
||||
*,
|
||||
output_fps: float,
|
||||
required_duration: float,
|
||||
frame_cache_size: int = 8,
|
||||
) -> None:
|
||||
if not specs:
|
||||
raise ValueError("At least one source clip is required")
|
||||
if output_fps <= 0:
|
||||
raise ValueError("output_fps must be positive")
|
||||
if required_duration <= 0:
|
||||
raise ValueError("required_duration must be positive")
|
||||
if frame_cache_size < 0:
|
||||
raise ValueError("frame_cache_size must be non-negative")
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise RuntimeError("ffmpeg is required to decode source audio")
|
||||
|
||||
try:
|
||||
import decord
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"Direct extraction requires decord; install "
|
||||
"qwen-omni-utils[decord]"
|
||||
) from exc
|
||||
|
||||
self.fps = float(output_fps)
|
||||
self.required_duration = float(required_duration)
|
||||
self.frame_cache_size = int(frame_cache_size)
|
||||
self._frame_cache: OrderedDict[tuple[int, int], Any] = OrderedDict()
|
||||
self._request_count = 0
|
||||
self._cache_hits = 0
|
||||
self._odd_frame_duplications = 0
|
||||
self._short_interval_fallbacks = 0
|
||||
self._tail_frame_reuses = 0
|
||||
|
||||
indexed: list[IndexedSourceClip] = []
|
||||
audio_parts: list[np.ndarray] = []
|
||||
dense_cursor = 0.0
|
||||
|
||||
for spec_index, spec in enumerate(specs):
|
||||
path = spec.path.expanduser().resolve()
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"Source video does not exist: {path}")
|
||||
print(
|
||||
f"direct-media: indexing [{spec_index + 1}/{len(specs)}] "
|
||||
f"{path.name}",
|
||||
flush=True,
|
||||
)
|
||||
reader = decord.VideoReader(str(path))
|
||||
if not len(reader):
|
||||
raise RuntimeError(f"No video frames decoded from {path}")
|
||||
starts, ends = _timestamp_array(reader)
|
||||
source_fps = float(reader.get_avg_fps())
|
||||
first_shape = tuple(int(value) for value in reader[0].shape)
|
||||
if len(first_shape) != 3:
|
||||
raise RuntimeError(
|
||||
f"Unexpected decoded frame shape {first_shape}: {path}"
|
||||
)
|
||||
height, width = first_shape[:2]
|
||||
|
||||
inferred_duration = len(reader) / source_fps
|
||||
duration = (
|
||||
float(spec.duration)
|
||||
if spec.duration is not None
|
||||
else float(inferred_duration)
|
||||
)
|
||||
if duration <= 0:
|
||||
raise RuntimeError(f"Non-positive source duration for {path}")
|
||||
|
||||
crop = spec.crop
|
||||
if crop is None and spec.crop_rightmost_width is not None:
|
||||
keep_width = int(spec.crop_rightmost_width)
|
||||
if width < keep_width:
|
||||
raise RuntimeError(
|
||||
f"Cannot keep rightmost {keep_width}px from {width}px: {path}"
|
||||
)
|
||||
crop = (width - keep_width, 0, keep_width, height)
|
||||
if crop is not None:
|
||||
x, y, crop_width, crop_height = crop
|
||||
if (
|
||||
x < 0
|
||||
or y < 0
|
||||
or crop_width <= 0
|
||||
or crop_height <= 0
|
||||
or x + crop_width > width
|
||||
or y + crop_height > height
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"Invalid crop {crop} for {width}x{height}: {path}"
|
||||
)
|
||||
|
||||
audio_path = (spec.audio_path or path).expanduser().resolve()
|
||||
if not audio_path.is_file():
|
||||
raise FileNotFoundError(f"Audio source does not exist: {audio_path}")
|
||||
|
||||
clip = IndexedSourceClip(
|
||||
path=path,
|
||||
audio_path=audio_path,
|
||||
dense_start=dense_cursor,
|
||||
dense_end=dense_cursor + duration,
|
||||
duration=duration,
|
||||
crop=crop,
|
||||
reader=reader,
|
||||
source_frame_starts=starts,
|
||||
source_frame_ends=ends,
|
||||
source_fps=source_fps,
|
||||
width=width,
|
||||
height=height,
|
||||
)
|
||||
indexed.append(clip)
|
||||
print(
|
||||
f"direct-media: decoding audio [{spec_index + 1}/{len(specs)}] "
|
||||
f"{audio_path.name}",
|
||||
flush=True,
|
||||
)
|
||||
audio_parts.append(_decode_audio_exact(audio_path, duration))
|
||||
dense_cursor = clip.dense_end
|
||||
|
||||
# No later source can be observed before the requested endpoint.
|
||||
if dense_cursor >= required_duration - EPSILON:
|
||||
break
|
||||
|
||||
self.clips = indexed
|
||||
self.dense_starts = np.asarray(
|
||||
[clip.dense_start for clip in indexed], dtype=np.float64
|
||||
)
|
||||
self.dense_duration = float(dense_cursor)
|
||||
if self.dense_duration + EPSILON < self.required_duration:
|
||||
raise RuntimeError(
|
||||
f"Dense source duration is {self.dense_duration:.6f}s, shorter "
|
||||
f"than required {self.required_duration:.6f}s"
|
||||
)
|
||||
self.audio = np.concatenate(audio_parts)
|
||||
required_samples = int(round(required_duration * SAMPLE_RATE))
|
||||
if len(self.audio) < required_samples:
|
||||
self.audio = np.pad(self.audio, (0, required_samples - len(self.audio)))
|
||||
elif len(self.audio) > required_samples:
|
||||
self.audio = self.audio[:required_samples]
|
||||
|
||||
max_grid_index = int(
|
||||
math.ceil(required_duration * self.fps - EPSILON)
|
||||
) - 1
|
||||
self.expected_grid_frames = max(0, max_grid_index + 1)
|
||||
if max_grid_index >= 0:
|
||||
self._source_frame_for_grid(max_grid_index, count_tail_reuse=False)
|
||||
|
||||
def _clip_for_time(self, global_time: float) -> tuple[int, IndexedSourceClip, float]:
|
||||
clip_index = int(np.searchsorted(self.dense_starts, global_time, side="right") - 1)
|
||||
clip_index = max(0, min(clip_index, len(self.clips) - 1))
|
||||
clip = self.clips[clip_index]
|
||||
local_time = global_time - clip.dense_start
|
||||
if local_time < 0:
|
||||
local_time = 0.0
|
||||
return clip_index, clip, local_time
|
||||
|
||||
def _source_frame_for_grid(
|
||||
self,
|
||||
grid_index: int,
|
||||
*,
|
||||
count_tail_reuse: bool = True,
|
||||
) -> tuple[int, int, float, float]:
|
||||
"""Map one global output tick to a source frame at or before it."""
|
||||
|
||||
global_time = grid_index / self.fps
|
||||
clip_index, clip, local_time = self._clip_for_time(global_time)
|
||||
frame_index = int(
|
||||
np.searchsorted(clip.source_frame_starts, local_time + EPSILON, side="right")
|
||||
- 1
|
||||
)
|
||||
frame_index = max(0, min(frame_index, clip.num_frames - 1))
|
||||
source_time = float(clip.source_frame_starts[frame_index])
|
||||
if source_time > local_time + 2 * EPSILON:
|
||||
raise RuntimeError(
|
||||
"Causal frame-selection violation: "
|
||||
f"source PTS {source_time:.9f}s > grid-local time {local_time:.9f}s"
|
||||
)
|
||||
if count_tail_reuse and local_time > float(clip.source_frame_starts[-1]) + EPSILON:
|
||||
self._tail_frame_reuses += 1
|
||||
return clip_index, frame_index, global_time, source_time
|
||||
|
||||
def _load_frame(self, clip_index: int, frame_index: int) -> Any:
|
||||
from PIL import Image
|
||||
|
||||
key = (clip_index, frame_index)
|
||||
cached = self._frame_cache.get(key)
|
||||
if cached is not None:
|
||||
self._cache_hits += 1
|
||||
self._frame_cache.move_to_end(key)
|
||||
return cached.copy()
|
||||
|
||||
clip = self.clips[clip_index]
|
||||
array = clip.reader[frame_index].asnumpy()
|
||||
if clip.crop is not None:
|
||||
x, y, width, height = clip.crop
|
||||
array = array[y : y + height, x : x + width]
|
||||
image = Image.fromarray(np.asarray(array)).copy()
|
||||
if self.frame_cache_size:
|
||||
self._frame_cache[key] = image
|
||||
self._frame_cache.move_to_end(key)
|
||||
while len(self._frame_cache) > self.frame_cache_size:
|
||||
self._frame_cache.popitem(last=False)
|
||||
return image.copy()
|
||||
|
||||
def frames(self, start: float, end: float) -> list[Any]:
|
||||
"""Return causal global-grid frames in ``[start, end)``."""
|
||||
|
||||
if end <= start:
|
||||
raise ValueError(f"Empty video interval [{start}, {end}]")
|
||||
raw_first = int(math.ceil(start * self.fps - EPSILON))
|
||||
raw_last_exclusive = int(math.ceil(end * self.fps - EPSILON))
|
||||
grid_indices = _grid_indices(start, end, self.fps)
|
||||
if raw_first >= raw_last_exclusive:
|
||||
self._short_interval_fallbacks += 1
|
||||
|
||||
output = []
|
||||
for grid_index in grid_indices:
|
||||
clip_index, frame_index, _, _ = self._source_frame_for_grid(grid_index)
|
||||
output.append(self._load_frame(clip_index, frame_index))
|
||||
self._request_count += 1
|
||||
if len(output) % 2:
|
||||
output.append(output[-1].copy())
|
||||
self._odd_frame_duplications += 1
|
||||
return output
|
||||
|
||||
def audio_slice(self, start: float, end: float) -> np.ndarray:
|
||||
"""Return the exact 16-kHz mono samples in ``[start, end)``."""
|
||||
|
||||
first = max(0, int(math.floor(start * SAMPLE_RATE + EPSILON)))
|
||||
last = min(
|
||||
len(self.audio), int(math.floor(end * SAMPLE_RATE + EPSILON))
|
||||
)
|
||||
if last <= first:
|
||||
raise ValueError(f"Empty audio interval [{start}, {end}]")
|
||||
return self.audio[first:last]
|
||||
|
||||
def sample_mapping(self, grid_index: int) -> dict[str, Any]:
|
||||
"""Describe one visual-grid mapping without decoding its pixels."""
|
||||
|
||||
clip_index, frame_index, grid_time, source_time = self._source_frame_for_grid(
|
||||
grid_index, count_tail_reuse=False
|
||||
)
|
||||
clip = self.clips[clip_index]
|
||||
return {
|
||||
"grid_index": int(grid_index),
|
||||
"grid_time": float(grid_time),
|
||||
"source_clip": str(clip.path),
|
||||
"source_frame_index": int(frame_index),
|
||||
"source_frame_pts_seconds": float(source_time),
|
||||
"dense_source_frame_time": float(clip.dense_start + source_time),
|
||||
}
|
||||
|
||||
def metadata(self) -> dict[str, Any]:
|
||||
return {
|
||||
"decoder": "decord_with_frame_pts_and_ffmpeg_pcm_audio",
|
||||
"visual_sampling": "global_grid_latest_source_pts_not_after_tick",
|
||||
"interval_convention": "half_open_[start,end)",
|
||||
"output_video_fps": self.fps,
|
||||
"expected_grid_frames": self.expected_grid_frames,
|
||||
"required_duration_seconds": self.required_duration,
|
||||
"dense_source_duration_seconds": self.dense_duration,
|
||||
"audio_sample_rate": SAMPLE_RATE,
|
||||
"audio_samples": int(len(self.audio)),
|
||||
"temporal_patch_padding": "duplicate_last_observed_frame_when_odd",
|
||||
"source_clips": [
|
||||
{
|
||||
"path": str(clip.path),
|
||||
"audio_path": str(clip.audio_path),
|
||||
"dense_start": clip.dense_start,
|
||||
"dense_end": clip.dense_end,
|
||||
"duration": clip.duration,
|
||||
"source_fps": clip.source_fps,
|
||||
"source_frames": clip.num_frames,
|
||||
"source_size": [clip.width, clip.height],
|
||||
"crop": list(clip.crop) if clip.crop is not None else None,
|
||||
"first_source_pts": float(clip.source_frame_starts[0]),
|
||||
"last_source_pts": float(clip.source_frame_starts[-1]),
|
||||
}
|
||||
for clip in self.clips
|
||||
],
|
||||
"runtime_counters": {
|
||||
"frame_requests": self._request_count,
|
||||
"frame_cache_hits": self._cache_hits,
|
||||
"odd_frame_duplications": self._odd_frame_duplications,
|
||||
"short_interval_fallbacks": self._short_interval_fallbacks,
|
||||
"tail_frame_reuses": self._tail_frame_reuses,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _manifest_session_specs(
|
||||
manifest_path: Path,
|
||||
session_name: str,
|
||||
) -> tuple[list[SourceClipSpec], dict[str, Any], Path | None]:
|
||||
payload = json.loads(manifest_path.read_text())
|
||||
matches = [
|
||||
session
|
||||
for session in payload.get("sessions", [])
|
||||
if session.get("session") == session_name
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError(
|
||||
f"Expected exactly one {session_name!r} in {manifest_path}, "
|
||||
f"found {len(matches)}"
|
||||
)
|
||||
session = matches[0]
|
||||
specs = []
|
||||
for record in session.get("clips", []):
|
||||
duration = record.get("video_duration_seconds")
|
||||
crop_record = record.get("egospeak_crop")
|
||||
crop = None
|
||||
if crop_record is not None:
|
||||
crop = (
|
||||
int(crop_record["x"]),
|
||||
int(crop_record["y"]),
|
||||
int(crop_record["width"]),
|
||||
int(crop_record["height"]),
|
||||
)
|
||||
specs.append(
|
||||
SourceClipSpec(
|
||||
path=Path(record["path"]),
|
||||
duration=float(duration) if duration is not None else None,
|
||||
crop=crop,
|
||||
)
|
||||
)
|
||||
if not specs:
|
||||
raise RuntimeError(f"No source clips recorded for {session_name}")
|
||||
target_record = session.get("target") or {}
|
||||
target_path = Path(target_record["path"]) if target_record.get("path") else None
|
||||
return specs, session, target_path
|
||||
|
||||
|
||||
def _easycom_directory_specs(session_dir: Path) -> list[SourceClipSpec]:
|
||||
session_dir = session_dir.expanduser().resolve()
|
||||
if not session_dir.is_dir():
|
||||
raise FileNotFoundError(f"EasyCom session directory does not exist: {session_dir}")
|
||||
paths = sorted(session_dir.glob("*.mp4"), key=lambda path: parse_timecode(path.stem))
|
||||
if not paths:
|
||||
raise RuntimeError(f"No MP4 clips found in {session_dir}")
|
||||
return [
|
||||
SourceClipSpec(path=path, crop_rightmost_width=EASYCOM_WIDTH)
|
||||
for path in paths
|
||||
]
|
||||
|
||||
|
||||
def _parse_crop(value: str | None) -> tuple[int, int, int, int] | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
values = tuple(int(item) for item in value.split(","))
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError("crop must be x,y,width,height") from exc
|
||||
if len(values) != 4:
|
||||
raise argparse.ArgumentTypeError("crop must be x,y,width,height")
|
||||
return values
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
source = parser.add_mutually_exclusive_group(required=True)
|
||||
source.add_argument("--video", type=Path, help="One original video clip")
|
||||
source.add_argument(
|
||||
"--easycom-session-dir",
|
||||
type=Path,
|
||||
help="Original EasyCom Session_N directory containing retained MP4 clips",
|
||||
)
|
||||
source.add_argument(
|
||||
"--easycom-manifest",
|
||||
type=Path,
|
||||
help="easycom_manifest.py JSON; also requires --session",
|
||||
)
|
||||
parser.add_argument("--session", help="Session_N selected from --easycom-manifest")
|
||||
parser.add_argument(
|
||||
"--audio",
|
||||
type=Path,
|
||||
help="Optional separate audio for --video; defaults to its embedded track",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--crop",
|
||||
help="Optional crop for --video as x,y,width,height",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--use-audio-in-video",
|
||||
action="store_true",
|
||||
help="Accepted for command compatibility; direct extraction always uses AV interleaving",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target",
|
||||
type=Path,
|
||||
help="target_perframe [T,3]; T is the authoritative full duration",
|
||||
)
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
choices=("causal-prefix", "open-stream-replay"),
|
||||
default="open-stream-replay",
|
||||
)
|
||||
parser.add_argument("--model-id", default=core.MODEL_ID)
|
||||
parser.add_argument("--start", type=float, default=0.0)
|
||||
parser.add_argument("--end", type=float)
|
||||
parser.add_argument("--step-seconds", type=float, default=0.2)
|
||||
parser.add_argument("--native-block-seconds", type=float, default=2.0)
|
||||
parser.add_argument(
|
||||
"--memory-seconds",
|
||||
type=float,
|
||||
default=0.0,
|
||||
help="0 keeps Thinker KV for the full extraction; positive values use a rolling window",
|
||||
)
|
||||
parser.add_argument("--video-fps", type=float, default=2.0)
|
||||
parser.add_argument("--video-min-pixels", type=int, default=64 * 28 * 28)
|
||||
parser.add_argument("--video-max-pixels", type=int, default=128 * 28 * 28)
|
||||
parser.add_argument("--frame-cache-size", type=int, default=8)
|
||||
parser.add_argument("--verify-every", type=int, default=0)
|
||||
parser.add_argument("--verify-continuous-every", type=int, default=0)
|
||||
parser.add_argument("--no-progress", action="store_true")
|
||||
parser.add_argument("--head-checkpoint", type=Path)
|
||||
parser.add_argument(
|
||||
"--dtype",
|
||||
choices=("auto", "bfloat16", "float16", "float32"),
|
||||
default="bfloat16",
|
||||
)
|
||||
parser.add_argument("--flash-attention-2", action="store_true")
|
||||
parser.add_argument("--device", default="cuda:0")
|
||||
parser.add_argument(
|
||||
"--media-only",
|
||||
action="store_true",
|
||||
help="Validate/index direct media and exit before loading Qwen",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _resolve_duration_and_target(
|
||||
args: argparse.Namespace,
|
||||
manifest_target: Path | None,
|
||||
) -> tuple[Path | None, int | None, float]:
|
||||
target_path = args.target
|
||||
if target_path is None and manifest_target is not None and manifest_target.is_file():
|
||||
target_path = manifest_target
|
||||
target_steps = None
|
||||
if target_path is not None:
|
||||
target_path = target_path.expanduser().resolve()
|
||||
target_steps = core.target_num_steps(target_path)
|
||||
if not math.isclose(args.start, 0.0, rel_tol=0.0, abs_tol=1e-9):
|
||||
raise SystemExit("Full target extraction requires --start 0")
|
||||
target_end = target_steps * args.step_seconds
|
||||
if args.end is not None and not math.isclose(
|
||||
args.end, target_end, rel_tol=0.0, abs_tol=1e-6
|
||||
):
|
||||
raise SystemExit(
|
||||
f"--end {args.end} conflicts with target length {target_steps} "
|
||||
f"at {args.step_seconds}s ({target_end}s)"
|
||||
)
|
||||
end = target_end
|
||||
else:
|
||||
if args.end is None:
|
||||
raise SystemExit("Pass --target for a full run or --end for a bounded test")
|
||||
end = float(args.end)
|
||||
if end <= args.start:
|
||||
raise SystemExit("--end must be greater than --start")
|
||||
return target_path, target_steps, end
|
||||
|
||||
|
||||
def _effective_memory_seconds(
|
||||
requested: float,
|
||||
*,
|
||||
duration: float,
|
||||
block_seconds: float,
|
||||
) -> float:
|
||||
if requested < 0:
|
||||
raise SystemExit("--memory-seconds must be >= 0")
|
||||
if requested > 0:
|
||||
return requested
|
||||
blocks = max(1, int(math.ceil(duration / block_seconds - EPSILON)))
|
||||
return blocks * block_seconds
|
||||
|
||||
|
||||
def _print_media_summary(reader: CausalSourceAV, start: float, end: float) -> None:
|
||||
print(
|
||||
f"Direct media valid: {len(reader.clips)} source clip(s), "
|
||||
f"required={end - start:.3f}s, grid={reader.expected_grid_frames} "
|
||||
f"frames at {reader.fps:g} FPS, audio={len(reader.audio)} samples"
|
||||
)
|
||||
candidates = [0]
|
||||
if reader.expected_grid_frames > 1:
|
||||
candidates.append(reader.expected_grid_frames - 1)
|
||||
for grid_index in candidates:
|
||||
mapping = reader.sample_mapping(grid_index)
|
||||
print(
|
||||
" grid "
|
||||
f"{mapping['grid_time']:.3f}s -> "
|
||||
f"{Path(mapping['source_clip']).name} "
|
||||
f"frame={mapping['source_frame_index']} "
|
||||
f"pts={mapping['source_frame_pts_seconds']:.6f}s"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
core._configure_qwen_logging()
|
||||
if args.verify_every < 0 or args.verify_continuous_every < 0:
|
||||
raise SystemExit("verification intervals must be >= 0")
|
||||
if args.frame_cache_size < 0:
|
||||
raise SystemExit("--frame-cache-size must be >= 0")
|
||||
if args.output is None and not args.media_only:
|
||||
raise SystemExit("--output is required unless --media-only is used")
|
||||
if args.audio is not None and args.video is None:
|
||||
raise SystemExit("--audio is supported only together with --video")
|
||||
if args.session is not None and args.easycom_manifest is None:
|
||||
raise SystemExit("--session is used only with --easycom-manifest")
|
||||
if args.easycom_manifest is not None and args.session is None:
|
||||
raise SystemExit("--easycom-manifest requires --session Session_N")
|
||||
|
||||
manifest_record = None
|
||||
manifest_target = None
|
||||
if args.video is not None:
|
||||
specs = [
|
||||
SourceClipSpec(
|
||||
path=args.video,
|
||||
crop=_parse_crop(args.crop),
|
||||
audio_path=args.audio,
|
||||
)
|
||||
]
|
||||
source_kind = "single_original_video"
|
||||
elif args.easycom_session_dir is not None:
|
||||
if args.crop is not None:
|
||||
raise SystemExit("EasyCom direct extraction uses the fixed rightmost-1920 crop")
|
||||
specs = _easycom_directory_specs(args.easycom_session_dir)
|
||||
source_kind = "easycom_dense_original_clips"
|
||||
else:
|
||||
if args.crop is not None:
|
||||
raise SystemExit("EasyCom manifest supplies its crop metadata")
|
||||
specs, manifest_record, manifest_target = _manifest_session_specs(
|
||||
args.easycom_manifest.expanduser().resolve(), args.session
|
||||
)
|
||||
source_kind = "easycom_manifest_original_clips"
|
||||
|
||||
target_path, target_steps, end = _resolve_duration_and_target(
|
||||
args, manifest_target
|
||||
)
|
||||
required_duration = end
|
||||
reader = CausalSourceAV(
|
||||
specs,
|
||||
output_fps=args.video_fps,
|
||||
required_duration=required_duration,
|
||||
frame_cache_size=args.frame_cache_size,
|
||||
)
|
||||
is_easycom = (
|
||||
args.easycom_session_dir is not None
|
||||
or args.easycom_manifest is not None
|
||||
)
|
||||
if is_easycom:
|
||||
for clip in reader.clips:
|
||||
if clip.height != EASYCOM_HEIGHT:
|
||||
raise RuntimeError(
|
||||
f"Expected EasyCom height {EASYCOM_HEIGHT}, got {clip.height}: "
|
||||
f"{clip.path}"
|
||||
)
|
||||
if not math.isclose(
|
||||
clip.source_fps, 20.0, rel_tol=0.0, abs_tol=1e-6
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"Expected EasyCom source FPS 20, got {clip.source_fps:g}: "
|
||||
f"{clip.path}"
|
||||
)
|
||||
if target_steps is not None:
|
||||
source_frames = sum(clip.num_frames for clip in reader.clips)
|
||||
media_steps = source_frames // 4
|
||||
if media_steps != target_steps:
|
||||
raise RuntimeError(
|
||||
f"EasyCom dense source gives {media_steps} complete 5-Hz "
|
||||
f"steps from {source_frames} frames, but target has "
|
||||
f"{target_steps}"
|
||||
)
|
||||
_print_media_summary(reader, args.start, end)
|
||||
if args.media_only:
|
||||
return
|
||||
|
||||
duration = end - args.start
|
||||
effective_memory = _effective_memory_seconds(
|
||||
args.memory_seconds,
|
||||
duration=duration,
|
||||
block_seconds=args.native_block_seconds,
|
||||
)
|
||||
if args.memory_seconds == 0 and duration > 120:
|
||||
print(
|
||||
"WARNING: persistent Thinker KV grows for the entire media "
|
||||
f"duration ({duration:.1f}s). Long EasyCom sessions can exceed "
|
||||
"the model context or GPU memory; a positive --memory-seconds is "
|
||||
"a bounded-memory experiment rather than native persistent-KV "
|
||||
"semantics.",
|
||||
flush=True,
|
||||
)
|
||||
config = core.StreamConfig(
|
||||
step_seconds=args.step_seconds,
|
||||
native_block_seconds=args.native_block_seconds,
|
||||
memory_seconds=effective_memory,
|
||||
video_fps=args.video_fps,
|
||||
video_min_pixels=args.video_min_pixels,
|
||||
video_max_pixels=args.video_max_pixels,
|
||||
)
|
||||
config.validate()
|
||||
|
||||
model, processor = core.load_qwen_omni(
|
||||
args.model_id,
|
||||
dtype=args.dtype,
|
||||
device_map={"": args.device},
|
||||
use_flash_attention_2=args.flash_attention_2,
|
||||
)
|
||||
model.requires_grad_(False)
|
||||
media_path_for_api = reader.clips[0].path
|
||||
|
||||
if args.mode == "causal-prefix":
|
||||
extractor = core.CausalPrefixExtractor(
|
||||
model, processor, config, fixed_grid_av=reader
|
||||
)
|
||||
result = extractor.extract(
|
||||
video_path=media_path_for_api,
|
||||
audio_path=None,
|
||||
start=args.start,
|
||||
end=end,
|
||||
use_audio_in_video=True,
|
||||
show_progress=not args.no_progress,
|
||||
)
|
||||
else:
|
||||
extractor = core.OpenStreamReplayExtractor(
|
||||
model, processor, config, fixed_grid_av=reader
|
||||
)
|
||||
result = extractor.extract(
|
||||
video_path=media_path_for_api,
|
||||
audio_path=None,
|
||||
start=args.start,
|
||||
end=end,
|
||||
use_audio_in_video=True,
|
||||
verify_every=args.verify_every,
|
||||
verify_continuous_every=args.verify_continuous_every,
|
||||
show_progress=not args.no_progress,
|
||||
)
|
||||
|
||||
if target_steps is not None and len(result.times) != target_steps:
|
||||
raise RuntimeError(
|
||||
f"Extractor returned {len(result.times)} states, expected {target_steps}"
|
||||
)
|
||||
|
||||
persistent_memory = args.memory_seconds == 0
|
||||
payload: dict[str, Any] = {
|
||||
"model_id": args.model_id,
|
||||
"mode": args.mode,
|
||||
"stream_representation": (
|
||||
"single_causal_prefix"
|
||||
if args.mode == "causal-prefix"
|
||||
else "single_open_media_turn_with_2s_payload_commits"
|
||||
),
|
||||
"config": asdict(config),
|
||||
"requested_memory_seconds": args.memory_seconds,
|
||||
"persistent_thinker_kv": persistent_memory,
|
||||
"start": args.start,
|
||||
"end": end,
|
||||
"num_steps": len(result.times),
|
||||
"label_indices": torch.arange(len(result.times), dtype=torch.long),
|
||||
"valid_10_horizon_steps": max(0, len(result.times) - 10),
|
||||
"target_path": str(target_path) if target_path is not None else None,
|
||||
"source_kind": source_kind,
|
||||
"source_manifest_record": manifest_record,
|
||||
"direct_media": reader.metadata(),
|
||||
"video_sampling_grid": (
|
||||
"direct_pts_causal_global_grid_half_open_with_tail_duplication"
|
||||
),
|
||||
"open_stream_policy": (
|
||||
{
|
||||
"commit": "completed_2s_av_payload_only",
|
||||
"temporary": "partial_payload_plus_media_closure_plus_readout",
|
||||
"tmrope_positions_per_chunk": core._tmrope_positions_per_chunk(model),
|
||||
"memory": (
|
||||
"persistent_full_extraction_thinker_kv"
|
||||
if persistent_memory
|
||||
else "rolling_complete_blocks_plus_current_partial_block"
|
||||
),
|
||||
}
|
||||
if args.mode == "open-stream-replay"
|
||||
else None
|
||||
),
|
||||
"times": result.times,
|
||||
"states": result.states,
|
||||
"verification": result.verification,
|
||||
}
|
||||
if args.head_checkpoint is not None:
|
||||
head = core._load_head(
|
||||
args.head_checkpoint, core._hidden_size(model), model.device
|
||||
)
|
||||
with torch.inference_mode():
|
||||
logits = head(
|
||||
result.states.to(model.device, dtype=model.dtype)
|
||||
).float().cpu()
|
||||
payload["logits"] = logits
|
||||
payload["probabilities"] = logits.softmax(dim=-1)
|
||||
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
torch.save(payload, args.output)
|
||||
print(f"Saved {len(result.times)} causal states to {args.output}")
|
||||
print(f"states: {tuple(result.states.shape)}")
|
||||
if result.verification:
|
||||
kinds = sorted({item.get("kind", "unspecified") for item in result.verification})
|
||||
print(f"verification checks: {len(result.verification)}")
|
||||
for kind in kinds:
|
||||
checks = [item for item in result.verification if item.get("kind") == kind]
|
||||
print(
|
||||
f" {kind}: n={len(checks)}, "
|
||||
f"worst max_abs={max(item['max_abs'] for item in checks):.6g}, "
|
||||
f"lowest cosine={min(item['cosine'] for item in checks):.8f}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user