From f6dbce30e4587dafccb3f1d32d5389caef14b5f0 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Mon, 24 Aug 2026 11:17:39 +0800 Subject: [PATCH] Reuse global history frames in overlap working memory --- README.md | 45 ++++++++++++++++------------- streaming_turn_model.py | 5 ++-- streaming_turn_model_directly.py | 49 +++++++++++++++++++------------- 3 files changed, 58 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 7a19993..193c512 100644 --- a/README.md +++ b/README.md @@ -1273,23 +1273,26 @@ The overlap is intentional. It presents recent evidence twice but does not add AV-token duration beyond the matched 8-second budget. Thus this ablation trades older unique context for stronger recent evidence instead of silently giving the candidate more input tokens. Fixed blocks retain the global half-open 2 FPS -grid. The working recap uses an endpoint-aligned 2 FPS grid and includes the -current causal frame: +grid. The working recap reuses three causal global-grid history frames and +replaces the oldest visual slot with the current causal frame: ```text fixed [18,20): 18.0 18.5 19.0 19.5 -working [18.4,20.4]: 18.9 19.4 19.9 20.4 +working [18.4,20.4]: 18.5 19.0 19.5 20.4 ``` Keeping four frames means the oldest visual tick is replaced by the current frame rather than adding a fifth frame; this isolates endpoint recency without -increasing visual tokens. Audio remains the full, exactly half-open -`[start,end)` slice. Consequently, call this an endpoint-aligned recent-video -intervention, not an exact timestamp-preserving resampling of both modalities. -The recap is serialized as the next TMRoPE chunk; it is deliberately a -recent-evidence recap, not a claim that duplicated samples occur later in -physical time. This mode is an ablation, not native Qwen streaming semantics, -and the Omni backbone stays frozen. +increasing visual tokens. Historical target times are causally floored to the +session-global 2 FPS grid. This lets them share the complete-block frame cache +and avoids making Decord seek backward through a long-GOP H.264 stream for four +new endpoint-shifted timestamps every 200 ms. Audio remains the full, exactly +half-open `[start,end)` slice. Consequently, call this a global-history plus +current-frame recent-video intervention, not an exact timestamp-preserving +resampling of both modalities. The recap is serialized as the next TMRoPE +chunk; it is deliberately a recent-evidence recap, not a claim that duplicated +samples occur later in physical time. This mode is an ablation, not native Qwen +streaming semantics, and the Omni backbone stays frozen. One deliberate side effect is a 2-second phase cycle: overlap falls toward 0.2 seconds just before a fixed boundary and jumps to 2.0 seconds at the @@ -1332,24 +1335,28 @@ python streaming_turn_model_directly.py \ --dtype bfloat16 \ --flash-attention-2 \ --device cuda:0 \ - --verify-every 10 \ - --output features/easycom_overlap_working_8s_smoke/Session_7_24s.pt + --verify-every 0 \ + --output features/easycom_overlap_working_8s_smoke/Session_7_24s_global_history_current.pt ``` -Expected shape is `(120, 2048)`. `overlap_cache_replay` should be numerically -exact or effectively exact. Inspect the saved timing policy with: +Expected shape is `(120, 2048)`. This speed smoke test disables replay +verification; after its throughput is acceptable, rerun a short four-second +check with `--verify-every 5`, where `overlap_cache_replay` should be +numerically exact or effectively exact. Inspect the saved timing policy and +decoder cache counters with: ```bash python - <<'PY' import torch p = torch.load( - "features/easycom_overlap_working_8s_smoke/Session_7_24s.pt", + "features/easycom_overlap_working_8s_smoke/Session_7_24s_global_history_current.pt", map_location="cpu", weights_only=False, ) print(p["mode"], tuple(p["states"].shape)) print(p["overlap_working_memory_policy"]["working_media"]) +print(p["direct_media"]["runtime_counters"]) for row in p["overlap_working_memory_policy"]["layout"][98:102]: print(row) PY @@ -1359,7 +1366,7 @@ For one complete session, replace `--end 24` with its target and use a session-level output filename: ```bash -mkdir -p /data1/yfzx/wangx/Datasets/easycom/omni_features/overlap_working_2fps_8s +mkdir -p /data1/yfzx/wangx/Datasets/easycom/omni_features/overlap_global_history_current_2fps_8s CUDA_VISIBLE_DEVICES=3 \ OMP_NUM_THREADS=4 \ @@ -1383,7 +1390,7 @@ python streaming_turn_model_directly.py \ --dtype bfloat16 \ --flash-attention-2 \ --device cuda:0 \ - --output /data1/yfzx/wangx/Datasets/easycom/omni_features/overlap_working_2fps_8s/Session_7.pt + --output /data1/yfzx/wangx/Datasets/easycom/omni_features/overlap_global_history_current_2fps_8s/Session_7.pt ``` After all train/test sessions have been extracted, train the unchanged frozen @@ -1391,7 +1398,7 @@ head by setting only the expected feature mode: ```bash python train_turn_head.py \ - --feature-root /data1/yfzx/wangx/Datasets/easycom/omni_features/overlap_working_2fps_8s \ + --feature-root /data1/yfzx/wangx/Datasets/easycom/omni_features/overlap_global_history_current_2fps_8s \ --target-root /data1/yfzx/wangx/Datasets/easycom/easycom_target_frame \ --output checkpoints/easycom_overlap_working_frozen_8s.pt \ --expected-mode overlap-working-memory \ @@ -1400,7 +1407,7 @@ python train_turn_head.py \ --device cuda:0 python infer_turn_head.py \ - --feature-root /data1/yfzx/wangx/Datasets/easycom/omni_features/overlap_working_2fps_8s \ + --feature-root /data1/yfzx/wangx/Datasets/easycom/omni_features/overlap_global_history_current_2fps_8s \ --target-root /data1/yfzx/wangx/Datasets/easycom/easycom_target_frame \ --checkpoint checkpoints/easycom_overlap_working_frozen_8s.pt \ --output-dir results/easycom_overlap_working_frozen_8s \ diff --git a/streaming_turn_model.py b/streaming_turn_model.py index 9a92431..00c40e3 100644 --- a/streaming_turn_model.py +++ b/streaming_turn_model.py @@ -1659,8 +1659,9 @@ class OverlappingWorkingMemoryExtractor: The final fixed block and the recap intentionally overlap. Fixed and working media readers may implement different visual grids. In - the direct-media experiment the fixed blocks remain half-open/global-grid, - while the recap contains a frame sampled exactly at current time ``t``. + the direct-media experiment the fixed blocks remain half-open/global-grid. + The recap reuses global-grid historical frames but replaces its oldest + visual slot with a frame sampled exactly at current time ``t``. """ def __init__( diff --git a/streaming_turn_model_directly.py b/streaming_turn_model_directly.py index 44342fa..0f50b0d 100644 --- a/streaming_turn_model_directly.py +++ b/streaming_turn_model_directly.py @@ -826,15 +826,17 @@ class CausalSourceAV: } -class EndpointAlignedAV: - """Working-window view whose final visual sample is exactly current time. +class GlobalHistoryCurrentAV: + """Working-window view with cached global history plus the current frame. Complete cached blocks use the source reader's fixed global half-open grid, for example ``[18.0, 18.5, 19.0, 19.5]`` for ``[18,20)`` at 2 FPS. A - two-second working interval ending at ``t`` instead uses - ``[t-1.5, t-1.0, t-0.5, t]``. The number of visual samples therefore - stays matched while the working branch explicitly observes the current - causal frame. Audio remains the exact half-open slice ``[start,end)``. + two-second working interval ending at ``t`` uses the latest global-grid + samples at or before ``t-1.5``, ``t-1.0``, and ``t-0.5``, followed by the + current causal frame at ``t``. At ``t=20.4`` this is + ``[18.5, 19.0, 19.5, 20.4]``. Historical pixels therefore reuse the same + source-frame cache as complete blocks and only the current frame advances + at 5 Hz. Audio remains the exact half-open slice ``[start,end)``. """ def __init__( @@ -868,13 +870,16 @@ class EndpointAlignedAV: ) count = max(1, int(math.ceil(duration * self.fps - EPSILON))) count = min(count, self.max_frames) - first = end - (count - 1) / self.fps - if first < start - EPSILON: - raise RuntimeError( - f"Endpoint-aligned grid starts before its interval: " - f"{first:g} < {start:g}" - ) - return [first + index / self.fps for index in range(count)] + history_times = [] + for slots_before_current in range(count - 1, 0, -1): + desired_time = end - slots_before_current / self.fps + # Quantize only historical samples to the session-global grid. + # During the short initial warm-up, clamp to the causal interval + # start rather than selecting media from before the requested + # working window. + grid_index = int(math.floor(desired_time * self.fps + EPSILON)) + history_times.append(max(start, grid_index / self.fps)) + return [*history_times, end] def frames(self, start: float, end: float) -> list[Any]: frames = [ @@ -891,19 +896,23 @@ class EndpointAlignedAV: def metadata(self) -> dict[str, Any]: return { - "visual_sampling": "endpoint_aligned_latest_source_pts_not_after_tick", + "visual_sampling": "global_grid_history_plus_current_causal_frame", "audio_interval": "half_open_[start,end)", "working_memory_seconds": self.max_interval_seconds, "output_video_fps": self.fps, "full_window_frame_count": self.max_frames, - "full_window_relative_video_times": [ + "desired_history_offsets_seconds": [ -(self.max_frames - 1 - index) / self.fps - for index in range(self.max_frames) + for index in range(self.max_frames - 1) ], + "current_frame_offset_seconds": 0.0, + "history_timestamp_quantization": "floor_to_session_global_grid", "includes_current_video_frame": True, "oldest_visual_tick_replaced_by_current": True, "timestamp_note": ( - "video is endpoint-aligned while audio retains the full " + "historical video targets are quantized causally to the " + "global grid for decoder/cache reuse; the final video sample " + "is exactly current time while audio retains the full " "half-open working interval" ), } @@ -1384,7 +1393,7 @@ def main( show_progress=not args.no_progress, ) else: - working_grid_av = EndpointAlignedAV( + working_grid_av = GlobalHistoryCurrentAV( reader, max_interval_seconds=args.working_memory_seconds, ) @@ -1427,7 +1436,7 @@ def main( if args.mode == "open-stream-replay" else ( "serialized_cached_complete_blocks_then_overlapping_" - "endpoint_aligned_recent_recap" + "global_history_current_frame_recent_recap" ) ) ), @@ -1444,7 +1453,7 @@ def main( "source_manifest_record": manifest_record, "direct_media": reader.metadata(), "video_sampling_grid": ( - "fixed_global_half_open_plus_endpoint_aligned_working_recap" + "fixed_global_half_open_plus_global_history_current_frame_recap" if args.mode == "overlap-working-memory" else "direct_pts_causal_global_grid_half_open_with_tail_duplication" ),