Update README and streaming_turn_model.py to implement half-open interval ownership for fixed-grid AV frames. Adjust verification parameters to 1 for testing, and enhance documentation for clarity on audio slicing and frame selection. Add runtime checks to prevent overlapping temporal coordinates in open-stream intervals.
This commit is contained in:
+35
-3
@@ -167,7 +167,13 @@ class FixedGridAV:
|
||||
)
|
||||
|
||||
def frames(self, start: float, end: float) -> list[Any]:
|
||||
"""Return fixed-grid frames whose timestamps lie in [start, end]."""
|
||||
"""Return fixed-grid frames whose timestamps lie in ``[start, end)``.
|
||||
|
||||
Half-open ownership is essential at native block boundaries. For
|
||||
example, the frame at exactly 2.0 s belongs to ``[2.0, 4.0)`` and must
|
||||
not also be committed in ``[0.0, 2.0)``. Audio slicing below follows
|
||||
the same convention.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
@@ -175,7 +181,7 @@ class FixedGridAV:
|
||||
first = max(0, int(math.ceil(start * self.fps - 1e-7)))
|
||||
last = min(
|
||||
self.num_frames - 1,
|
||||
int(math.floor(end * self.fps + 1e-7)),
|
||||
int(math.ceil(end * self.fps - 1e-7)) - 1,
|
||||
)
|
||||
if first > last:
|
||||
# This can occur only when a caller starts between global visual
|
||||
@@ -1255,11 +1261,32 @@ class OpenStreamReplayExtractor:
|
||||
manual_audios=manual_audios,
|
||||
video_fps=self.config.video_fps,
|
||||
)
|
||||
return _open_media_pieces(
|
||||
pieces = _open_media_pieces(
|
||||
self.model,
|
||||
inputs,
|
||||
use_audio_in_video=use_audio_in_video,
|
||||
)
|
||||
# A standalone tail is at most one native 2 s block. Its temporal
|
||||
# TMRoPE coordinates must therefore stay inside one 50-position chunk.
|
||||
# This catches endpoint-inclusive frame selection (or another
|
||||
# preprocessing regression) before overlapping coordinates are
|
||||
# committed to the next block.
|
||||
duration = interval[1] - interval[0]
|
||||
if duration <= self.config.native_block_seconds + 1e-6:
|
||||
temporal_positions = pieces.payload_position_ids[0]
|
||||
temporal_span = int(
|
||||
temporal_positions.max().item()
|
||||
- temporal_positions.min().item()
|
||||
)
|
||||
positions_per_chunk = _tmrope_positions_per_chunk(self.model)
|
||||
if temporal_span >= positions_per_chunk:
|
||||
raise RuntimeError(
|
||||
"One open-stream AV interval spans multiple native TMRoPE "
|
||||
f"chunks: interval={interval}, temporal_span={temporal_span}, "
|
||||
f"chunk_size={positions_per_chunk}. Check frame-boundary "
|
||||
"ownership and audio preprocessing."
|
||||
)
|
||||
return pieces
|
||||
|
||||
def state_from_scratch(
|
||||
self,
|
||||
@@ -1812,6 +1839,11 @@ def main() -> None:
|
||||
if fixed_grid_av is not None
|
||||
else "qwen_utility_adaptive_sampling"
|
||||
),
|
||||
"media_interval_policy": (
|
||||
"half_open_[start,end)"
|
||||
if fixed_grid_av is not None
|
||||
else "qwen_utility_defined"
|
||||
),
|
||||
"open_stream_policy": (
|
||||
{
|
||||
"commit": "completed_2s_av_payload_only",
|
||||
|
||||
Reference in New Issue
Block a user