Add more preview

docs/ego4d_local_baselines.md

@@ -77,6 +77,30 @@ python extract_turn_local_features.py "${LOCAL_ARGS[@]}" \
 4s smoke 应得到 V/A 各 `[20,2048]`、`partial=True`；此目录不能用于正式训练。
 V-valid 不必是 100%，Ego4D 有无可见人物和缺框的时刻；若有明显人物却长期无有效框，先停下来检查标注。
 
+### 前 4s 没有人物：增加预览范围和数量
+
+旧预览固定最多 4 张；`--max-steps 20` 又把范围限制到前 4s。
+现在可指定数量和时间范围，并仅从有有效人物框的网格帧中均匀抽样：
+
+```bash
+# 前三条 clip：在整条 clip 中各取最多 40 张有效带框图片。
+# 不要再带 --max-steps 20；不传 preview-end 就使用完整可用时间范围。
+python extract_turn_local_features.py "${LOCAL_ARGS[@]}" \
+  --limit 3 --preview-only --preview-count 40 --preview-with-boxes \
+  --output-root features/ego4d_local_preview_people
+
+# 也可聚焦第一条 clip 的 30–120s；实际不足 120s 时截到 clip 结束。
+python extract_turn_local_features.py "${LOCAL_ARGS[@]}" \
+  --limit 1 --preview-only --preview-count 60 \
+  --preview-start 30 --preview-end 120 --preview-with-boxes \
+  --output-root features/ego4d_local_preview_30_120s
+```
+
+每张图标注时间、源帧号和有效人数；`preview.json` 记录候选帧数、有效帧数、实际输出数，
+`boxes.json` 记录当前这次输出的图片和框。无有效框时明确报告 0 张，可换片段或核对帧号对齐，
+不会伪造框或强行当作成功。`--preview-with-boxes` 只筛选预览，**不删除特征提取或评测中的无框帧**。
+默认仍最多 4 张。改变范围时建议换输出目录；旧 JPG 不会自动删除，以当前 JSON 清单为准。
+
 ## 3. 提取完整 manifest：前台、模型只加载一次
 
 ```bash

extract_turn_local_features.py

@@ -264,15 +264,57 @@ def validate_source_timeline(reference: dict[str, Any], media) -> None:
             raise ValueError(f"Local/reference source timeline differs at {new['path']}")
 
 
+def preview_time_bounds(args: argparse.Namespace, available_end: float) -> tuple[float, float]:
+    """Preview-only range on the original timeline, never a rebased feature grid."""
+    start = float(getattr(args, "preview_start", 0.))
+    requested_end = getattr(args, "preview_end", None)
+    end = min(available_end, requested_end) if requested_end is not None else available_end
+    if start >= end:
+        raise ValueError(f"Preview start {start:g}s is outside the available prefix ending at {end:g}s; "
+                         "remove/increase --max-steps or choose an earlier --preview-start")
+    return start, end
+
+
+def preview_ticks(media, annotations, args: argparse.Namespace, end: float) -> tuple[list[int], int, int]:
+    start, stop = preview_time_bounds(args, end)
+    ticks = [tick for tick in causal_frame_ticks(stop, args.video_fps)
+             if tick / args.video_fps >= start - 1e-8]
+    grid_count = len(ticks)
+    if getattr(args, "preview_with_boxes", False):
+        eligible = []
+        # Scan annotation metadata, not every RGB frame. Use the same source
+        # frame mapping, coordinate correction and crop validity as extraction.
+        for tick in ticks:
+            clip_index, frame_index, _ = media._source_frame_for_time(tick / args.video_fps)
+            clip = media.clips[clip_index]
+            width, height = clip.crop[2:] if clip.crop is not None else (clip.width, clip.height)
+            offset_x, offset_y = box_coordinate_offset(clip, args.box_coordinate_space)
+            if any(crop_box(person["xyxy"], width, height, offset_x=offset_x, offset_y=offset_y,
+                            expansion=args.crop_expansion) is not None
+                   for person in annotations[clip_index].get(frame_index, [])):
+                eligible.append(tick)
+        ticks = eligible
+    eligible_count = len(ticks)
+    count = min(getattr(args, "preview_count", 4), eligible_count)
+    indices = np.linspace(0, eligible_count - 1, count, dtype=int) if count else []
+    return [ticks[index] for index in indices], grid_count, eligible_count
+
+
 def preview_boxes(media, annotations, args: argparse.Namespace, name: str, end: float) -> None:
     from PIL import ImageDraw
 
     folder = args.output_root / f"{name}_preview"
     folder.mkdir(parents=True, exist_ok=True)
-    ticks = list(causal_frame_ticks(end, args.video_fps))
+    start, stop = preview_time_bounds(args, end)
+    ticks, grid_count, eligible_count = preview_ticks(media, annotations, args, end)
+    print(f"Preview [{start:g},{stop:g})s: {grid_count} grid frames, {eligible_count} eligible, "
+          f"saving {len(ticks)} images", flush=True)
+    if not ticks:
+        print("No eligible preview frames. Try a longer range or another clip. "
+              "If people are visible but no valid boxes exist, check annotation/frame alignment.", flush=True)
     audit = []
-    for index in np.linspace(0, len(ticks) - 1, min(4, len(ticks)), dtype=int):
+    for tick in ticks:
-        time = ticks[index] / args.video_fps
+        time = tick / args.video_fps
         image, _, records = frame_crops(media, annotations, time, args)
         clip_index, frame_index, _ = media._source_frame_for_time(time)
         clip = media.clips[clip_index]
@@ -281,7 +323,10 @@ def preview_boxes(media, annotations, args: argparse.Namespace, name: str, end:
         for record in records:
             draw.rectangle(record["box"], outline="lime", width=4)
             draw.text(record["box"][:2], record["id"], fill="red")
-        filename = f"frame_{ticks[index]:06d}.jpg"
+        caption = f"t={time:.2f}s | source frame={frame_index} | people={len(records)}"
+        draw.rectangle(draw.textbbox((8, 8), caption), fill="black")
+        draw.text((8, 8), caption, fill="white")
+        filename = f"frame_{tick:06d}.jpg"
         image.save(folder / filename)
         audit.append({
             "image": filename, "time": time, "source_clip": str(clip.path),
@@ -291,9 +336,18 @@ def preview_boxes(media, annotations, args: argparse.Namespace, name: str, end:
             "coordinate_offset": list(offset), "boxes": records,
         })
         print(f"preview t={time:g}s: {clip.width}x{clip.height} -> {image.width}x{image.height}; "
-              f"boxes={args.box_coordinate_space}, subtract offset={offset}")
+              f"boxes={args.box_coordinate_space}, subtract offset={offset}, people={len(records)}")
     (folder / "boxes.json").write_text(json.dumps(audit, indent=2) + "\n")
+    (folder / "preview.json").write_text(json.dumps({
+        "start": start, "end": stop, "video_fps": args.video_fps,
+        "with_boxes_only": getattr(args, "preview_with_boxes", False),
+        "requested_count": getattr(args, "preview_count", 4),
+        "grid_frame_count": grid_count, "eligible_frame_count": eligible_count,
+        "saved_count": len(audit), "saved_with_boxes": sum(bool(row["boxes"]) for row in audit),
+        "images": [row["image"] for row in audit],
+    }, indent=2) + "\n")
     print(f"Box previews: {folder} (inspect before extracting all sessions)")
+    print("boxes.json lists this run's images; older images in the directory are not removed.")
 
 
 def extract_local(media, annotations, encoder, times: torch.Tensor, args: argparse.Namespace) -> dict[str, Any]:
@@ -425,6 +479,11 @@ def parse_args(argv=None):
     p.add_argument("--decoder-gpu-id", type=int, default=0, help="Visible-device index, not physical GPU number")
     p.add_argument("--max-steps", type=int, help="Smoke test only; partial sidecars are rejected by training")
     p.add_argument("--preview-only", action="store_true", help="Save box overlays without loading the model")
+    p.add_argument("--preview-count", type=int, default=4, help="Maximum images per clip (preview only)")
+    p.add_argument("--preview-start", type=float, default=0., help="Start time in seconds on the clip/dense timeline")
+    p.add_argument("--preview-end", type=float, help="Exclusive end time; default: entire available prefix")
+    p.add_argument("--preview-with-boxes", action="store_true",
+                   help="Sample only grid frames with valid provided crops; preview selection only, not feature filtering")
     existing = p.add_mutually_exclusive_group()
     existing.add_argument("--overwrite", action="store_true")
     existing.add_argument("--reuse-existing", action="store_true", help="Validate and skip complete matching sidecars")
@@ -444,6 +503,19 @@ def parse_args(argv=None):
         p.error("--reuse-existing is for full extraction, not smoke tests or previews")
     if args.dry_run and args.preview_only:
         p.error("--dry-run and --preview-only are separate checks")
+    if args.preview_count <= 0:
+        p.error("--preview-count must be positive")
+    if not np.isfinite(args.preview_start) or args.preview_start < 0:
+        p.error("--preview-start must be finite and nonnegative")
+    if args.preview_end is not None and (not np.isfinite(args.preview_end) or args.preview_end <= args.preview_start):
+        p.error("--preview-end must be finite and greater than --preview-start")
+    if not args.preview_only and (args.preview_count != 4 or args.preview_start != 0.
+                                   or args.preview_end is not None or args.preview_with_boxes):
+        p.error("Preview selection options require --preview-only")
+    if args.preview_only and args.max_steps is not None:
+        if args.preview_start >= args.max_steps / 5 or (args.preview_end is not None
+                                                       and args.preview_end > args.max_steps / 5):
+            p.error("Preview range exceeds --max-steps; remove --max-steps to inspect later frames")
     if args.box_coordinate_space is None:
         args.box_coordinate_space = "encoded" if args.dataset == "easycom" else "camera"
     for name in ("video_fps", "audio_window_seconds", "visual_window_seconds", "crop_expansion"):
@@ -523,7 +595,10 @@ def main(argv=None) -> None:
         full_steps = len(times)
         if args.max_steps is not None:
             times, indices = times[:args.max_steps], indices[:args.max_steps]
-        media, annotations = build_media_and_boxes(args, name, len(times) / 5, ego_boxes)
+        media_end = len(times) / 5
+        if args.preview_only:
+            _, media_end = preview_time_bounds(args, media_end)
+        media, annotations = build_media_and_boxes(args, name, media_end, ego_boxes)
         validate_source_timeline(reference, media)
         if (args.dataset == "easycom" and args.box_coordinate_space == "camera"
                 and any(clip.crop is not None and clip.crop[0] for clip in media.clips)):

test_local_turn_features.py

@@ -23,7 +23,7 @@ from extract_turn_local_features import (
     FrozenQwenLocalEncoder, ego4d_clip_boxes, extract_local,
     validate_source_timeline,
     model_config_digest,
-    box_coordinate_offset, frame_crops, parse_args, preview_boxes,
+    box_coordinate_offset, frame_crops, parse_args, preview_boxes, preview_ticks, preview_time_bounds,
     TESTED_TRANSFORMERS_VERSIONS, validate_local_transformers_version,
     main as extract_main,
 )
@@ -160,6 +160,100 @@ class LocalTests(unittest.TestCase):
             self.assertEqual(audit[0]["boxes"][0]["box"], [100, 80, 160, 140])
             self.assertTrue((folder / audit[0]["image"]).is_file())
 
+    def test_preview_can_save_more_than_four_images_in_a_later_range(self):
+        clip = SimpleNamespace(path=Path("source.mp4"), width=640, height=360,
+                               crop=None, dense_start=0.)
+        media = SimpleNamespace(clips=[clip],
+                                _source_frame_for_time=lambda t: (0, round(t * 2), t),
+                                frame_at_or_before=lambda t: Image.new("RGB", (640, 360)))
+        with tempfile.TemporaryDirectory() as directory:
+            args = SimpleNamespace(box_coordinate_space="camera", crop_expansion=1., video_fps=2.,
+                                   output_root=Path(directory), preview_count=8,
+                                   preview_start=2., preview_end=6., preview_with_boxes=False)
+            preview_boxes(media, [{}], args, "clip", 10.)
+            folder = Path(directory) / "clip_preview"
+            audit = json.loads((folder / "boxes.json").read_text())
+            self.assertEqual([row["time"] for row in audit], [2., 2.5, 3., 3.5, 4., 4.5, 5., 5.5])
+            self.assertEqual(audit[0]["image"], "frame_000004.jpg")
+            self.assertEqual(len(list(folder.glob("*.jpg"))), 8)
+            report = json.loads((folder / "preview.json").read_text())
+            self.assertEqual(report["saved_count"], 8)
+            self.assertEqual(report["saved_with_boxes"], 0)
+
+    def test_boxed_preview_filters_valid_corrected_crops_without_decoding(self):
+        # Encoded boxes are corrected by the crop origin, as in feature extraction.
+        clip = SimpleNamespace(crop=(204, 0, 640, 360), width=844, height=360)
+        media = SimpleNamespace(clips=[clip], _source_frame_for_time=lambda t: (0, round(t * 2), t))
+        valid = {"id": "p", "xyxy": [304, 80, 364, 140]}
+        invalid = {"id": "p", "xyxy": [4, 80, 64, 140]}  # entirely inside removed banner
+        annotations = [{1: [valid], 3: [invalid], 5: [valid], 7: [valid]}]
+        args = SimpleNamespace(box_coordinate_space="encoded", crop_expansion=1., video_fps=2.,
+                               preview_count=2, preview_start=0., preview_end=None, preview_with_boxes=True)
+        selected, grid_count, eligible_count = preview_ticks(media, annotations, args, 4.)
+        self.assertEqual(selected, [1, 7])  # uniform among valid frames, not first N
+        self.assertEqual((grid_count, eligible_count), (8, 3))
+        # No RGB reader is needed by the scan, and annotations are not changed.
+        self.assertEqual(len(annotations[0]), 4)
+
+    def test_empty_boxed_preview_writes_an_explicit_empty_report(self):
+        clip = SimpleNamespace(crop=None, width=640, height=360)
+        media = SimpleNamespace(clips=[clip], _source_frame_for_time=lambda t: (0, round(t * 2), t))
+        with tempfile.TemporaryDirectory() as directory:
+            args = SimpleNamespace(box_coordinate_space="camera", crop_expansion=1., video_fps=2.,
+                                   output_root=Path(directory), preview_count=40,
+                                   preview_start=0., preview_end=None, preview_with_boxes=True)
+            preview_boxes(media, [{}], args, "empty_clip", 4.)
+            folder = Path(directory) / "empty_clip_preview"
+            self.assertEqual(json.loads((folder / "boxes.json").read_text()), [])
+            report = json.loads((folder / "preview.json").read_text())
+            self.assertEqual(report["eligible_frame_count"], 0)
+            self.assertEqual(report["saved_count"], 0)
+            self.assertEqual(list(folder.glob("*.jpg")), [])
+
+    def test_preview_range_clips_to_available_end_and_excludes_endpoint(self):
+        args = SimpleNamespace(video_fps=2., preview_start=.1, preview_end=100., preview_count=100)
+        self.assertEqual(preview_time_bounds(args, 1.), (.1, 1.))
+        self.assertEqual(preview_ticks(None, [], args, 1.), ([1], 1, 1))
+        args.preview_start = 2.
+        with self.assertRaisesRegex(ValueError, "outside the available prefix"):
+            preview_time_bounds(args, 1.)
+
+    def test_preview_cli_validation(self):
+        import contextlib
+        import io
+
+        required = ["--data-root", "/raw", "--feature-root", "/global", "--target-root", "/targets",
+                    "--output-root", "/preview", "--samples", "4"]
+        valid = parse_args(required + ["--preview-only", "--preview-count", "40", "--preview-start", "30",
+                                       "--preview-end", "120", "--preview-with-boxes"])
+        self.assertEqual((valid.preview_count, valid.preview_start, valid.preview_end), (40, 30., 120.))
+        for extra in (["--preview-only", "--preview-count", "0"],
+                      ["--preview-only", "--preview-start", "-1"],
+                      ["--preview-only", "--preview-end", "nan"],
+                      ["--preview-only", "--preview-start", "10", "--preview-end", "10"],
+                      ["--preview-only", "--max-steps", "20", "--preview-end", "60"],
+                      ["--preview-count", "40"], ["--preview-with-boxes"]):
+            with self.subTest(extra=extra), contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit):
+                parse_args(required + extra)
+
+    def test_preview_end_limits_media_reader_without_loading_encoder(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            reference, _, labels = fixture()
+            torch.save(reference, root / "Session_4.pt")
+            np.save(root / "Session_4.npy", labels)
+            args = ["--data-root", str(root), "--feature-root", str(root), "--target-root", str(root),
+                    "--output-root", str(root / "preview"), "--samples", "4",
+                    "--preview-only", "--preview-end", "2", "--preview-count", "20"]
+            media = SimpleNamespace(clips=[], metadata=lambda: {})
+            with patch("extract_turn_local_features.build_media_and_boxes", return_value=(media, [])) as build, \
+                 patch("extract_turn_local_features.preview_boxes") as preview, \
+                 patch("extract_turn_local_features.FrozenQwenLocalEncoder") as encoder:
+                extract_main(args)
+                self.assertEqual(build.call_args.args[2], 2.)
+                preview.assert_called_once()
+                encoder.assert_not_called()
+
     def test_sidecar_alignment_contract_and_modalities(self):
         reference, side, _ = fixture()
         with tempfile.TemporaryDirectory() as directory: