From 668996bfa822dca86f6ab0cd71532c4531b4e1b7 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Fri, 28 Aug 2026 01:11:51 +0800 Subject: [PATCH] Handle verified Ego4D terminal bounding-box records --- docs/ego4d_local_baselines.md | 21 ++++- extract_turn_local_features.py | 61 +++++++++++- test_ego4d_box_frames.py | 163 +++++++++++++++++++++++++++++++++ test_ego4d_local_workflow.py | 7 +- 4 files changed, 246 insertions(+), 6 deletions(-) create mode 100644 test_ego4d_box_frames.py diff --git a/docs/ego4d_local_baselines.md b/docs/ego4d_local_baselines.md index 032a563..cd1141b 100644 --- a/docs/ego4d_local_baselines.md +++ b/docs/ego4d_local_baselines.md @@ -173,6 +173,24 @@ worker 及其进程组，不杀无关训练。已完成的 `.pt` 保留，重跑 8 workers 也会增加 CPU、主机 RAM、磁盘 I/O 和标注 JSON 的解析开销；不要预设能达到 8 倍速度。 主机内存不足时改为 `--gpus 0 1 2 3` 并重跑，完成的文件仍可复用；不同次分片也不会改写已完成特征。 +#### 已核实的末尾 bbox 记录 + +2026-08-28 在 `31488ff6-05ef-4362-9786-35981b06f307` 上核对到： +原视频 8999 帧、30 FPS，有效索引为 `0..8998`；标注从 0 开始， +`clip_end_frame=8999`、`clip_end_sec=299.9666666666666`，唯一越界索引是 `8999`。 +这是无对应解码帧的末尾记录，不应把该框移到 `8998`，也不应将所有框减 1。 + +提取器只在以下条件同时满足时忽略末尾记录：唯一越界索引为 `N`， +`N=视频帧数=clip_end_frame`，标注帧起点为 0、时间起点在 1 微秒内为 0， +且标注时长与 `N / source_fps` 相差不超过 0.1 毫秒。 +缺少确认边界、较大越界或时长不匹配仍会报错，并打印具体范围；不自动跳过 clip。 + +日志显示忽略的末尾索引和框数；新 `.pt` 的 `box_frame_audit` 与预览目录的 +`box_frame_audit.json` 保存原始范围、视频帧数、标注边界和忽略记录。 +不修改原视频、JSON、target 或已完成的特征。本例仍输出 1499 行，对应截止时间 +`0.2..299.8s`，2 FPS 的最后一个输入采样时刻仍为 `299.5s`。 +该处理不改编码器/池化配置，之前完成且通过校验的特征可以继续复用；同步修复后重跑原 8 卡命令即可。 + 提取完成后，原训练入口也直接用同一个原始全局特征根，无需 flat 索引： ```bash @@ -247,10 +265,11 @@ python summarize_local_baselines.py \ ## 回归测试 ```bash -python -m unittest -q test_easycom_head_objectives test_local_turn_features test_ego4d_local_workflow test_ego4d_local_multigpu +python -m unittest -q test_easycom_head_objectives test_local_turn_features test_ego4d_local_workflow test_ego4d_local_multigpu test_ego4d_box_frames ``` CPU 回归包括 manifest 顺序/防交叉重复、完整批量提取的模型复用、断点文件校验、 单层/多层全局特征、测试集输入预检，以及合成 Ego4D 上四组各 4 epochs 的训练/推理与增量汇总。 另测试原 train/test 目录、433 clips 的 8 卡完整互斥分片、GPU mask、独立日志、失败/中断时清理自有进程。 +末尾框回归覆盖严格边界验证、其它越界拒绝、1499 行完整提取、审计保存和旧特征续跑。 真实 Ego4D 像素、提供框和 3B/L20 提取仍需按上面的 preview/smoke 步骤验证。 diff --git a/extract_turn_local_features.py b/extract_turn_local_features.py index e0ee3ff..f2d85f1 100644 --- a/extract_turn_local_features.py +++ b/extract_turn_local_features.py @@ -175,7 +175,14 @@ def load_ego4d_boxes(paths: list[Path], names: list[str]) -> dict[str, Any]: if uid in selected: if uid in result: raise ValueError(f"Duplicate Ego4D annotation UID: {uid}") - result[uid] = ego4d_clip_boxes(clip) + # Retain declared bounds so a terminal record can be + # distinguished from a truncated video or frame mismatch. + result[uid] = { + "frames": ego4d_clip_boxes(clip), + "bounds": {key: clip.get(key) for key in ( + "clip_start_frame", "clip_end_frame", "clip_start_sec", "clip_end_sec", + )}, + } del payload missing = set(names) - result.keys() if missing: @@ -183,6 +190,45 @@ def load_ego4d_boxes(paths: list[Path], names: list[str]) -> dict[str, Any]: return result +def validate_ego4d_box_frames(name: str, annotation: dict[str, Any], source): + """Ignore only a confirmed exclusive-end record; never shift/clamp boxes.""" + frames, bounds = annotation["frames"], annotation["bounds"] + count = source.num_frames + outside = sorted(frame for frame in frames if not 0 <= frame < count) + audit = { + "policy": "clip-local-zero-based; ignore-confirmed-exclusive-end-only", + "source_frames": count, "source_fps": source.source_fps, + "declared_bounds": dict(bounds), + "annotation_frame_range": [min(frames), max(frames)] if frames else None, + "ignored_frame_indices": [], "ignored_box_count": 0, + } + if not outside: + return frames, audit + start_sec, end_sec = bounds.get("clip_start_sec"), bounds.get("clip_end_sec") + duration_matches = ( + isinstance(start_sec, (int, float)) and isinstance(end_sec, (int, float)) + and np.isfinite([start_sec, end_sec, source.source_fps]).all() + and source.source_fps > 0 and abs(start_sec) <= 1e-6 + and abs((end_sec - start_sec) - count / source.source_fps) <= 1e-4 + ) + if (outside != [count] or bounds.get("clip_start_frame") != 0 + or bounds.get("clip_end_frame") != count or not duration_matches): + raise ValueError( + f"Ego4D annotation frame exceeds source clip: {name}; " + f"source_frames={count}, source_fps={source.source_fps}, " + f"annotation_range={audit['annotation_frame_range']}, " + f"out_of_range_count={len(outside)}, first_10={outside[:10]}, " + f"declared_bounds={bounds}. Only frame == source_frames == clip_end_frame " + "with zero-origin, matching duration is an allowed terminal record; " + "check annotation/video alignment instead of clamping or skipping the clip." + ) + audit.update(ignored_frame_indices=[count], ignored_box_count=len(frames[count])) + print(f"{name}: ignoring exclusive-end bbox record at frame {count} " + f"({len(frames[count])} boxes); decoded frames are 0..{count - 1}. " + "No frame shift or feature/target row removal.", flush=True) + return {frame: boxes for frame, boxes in frames.items() if frame != count}, audit + + def build_media_and_boxes(args: argparse.Namespace, name: str, end: float, ego_boxes: dict[str, Any]): # Lazy imports allow --help and pure tests without Transformers/decoders. from streaming_turn_model_directly import CausalSourceAV, SourceClipSpec, _easycom_directory_specs @@ -209,9 +255,9 @@ def build_media_and_boxes(args: argparse.Namespace, name: str, end: float, ego_b raise ValueError(f"Box/video frame inventory mismatch: {path}") annotations.append(mapping) else: - annotations = [ego_boxes[name]] - if annotations[0] and max(annotations[0]) >= media.clips[0].num_frames: - raise ValueError(f"Ego4D annotation frame exceeds source clip: {name}") + mapping, audit = validate_ego4d_box_frames(name, ego_boxes[name], media.clips[0]) + annotations = [mapping] + ego_boxes[name]["frame_audit"] = audit return media, annotations @@ -622,6 +668,9 @@ def main(argv=None) -> None: "annotations use --box-coordinate-space encoded; otherwise boxes shift right.", flush=True) if args.preview_only: preview_boxes(media, annotations, args, name, len(times) / 5) + if args.dataset == "ego4d": + audit_path = args.output_root / f"{name}_preview" / "box_frame_audit.json" + audit_path.write_text(json.dumps(ego_boxes[name]["frame_audit"], indent=2) + "\n") ego_boxes.pop(name, None) del media, annotations, reference continue @@ -643,6 +692,10 @@ def main(argv=None) -> None: "reference_features": str(path.resolve()), "direct_media": media.metadata(), "config": config, }) + if args.dataset == "ego4d": + # Per-clip diagnostics, not encoder config: existing valid sidecars + # retain the same reuse contract and need not be re-extracted. + output["box_frame_audit"] = ego_boxes[name]["frame_audit"] destination = args.output_root / f"{name}.pt" # Atomic replacement: a failed extraction never looks like a ready file. with tempfile.NamedTemporaryFile(dir=args.output_root, prefix=f".{name}.", suffix=".tmp", delete=False) as handle: diff --git a/test_ego4d_box_frames.py b/test_ego4d_box_frames.py new file mode 100644 index 0000000..185ae5f --- /dev/null +++ b/test_ego4d_box_frames.py @@ -0,0 +1,163 @@ +"""CPU regressions for a verified Ego4D exclusive-end bbox record.""" + +from __future__ import annotations + +import contextlib +import copy +import io +import json +from pathlib import Path +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import numpy as np +from PIL import Image +import torch + +import extract_turn_local_features as extract +from local_turn_features import load_local_features +from test_local_turn_features import fixture + + +UID = "31488ff6-05ef-4362-9786-35981b06f307" + + +def terminal_clip(): + return { + "clip_uid": UID, "clip_start_frame": 0, "clip_end_frame": 8999, + "clip_start_sec": 0, "clip_end_sec": 299.9666666666666, + "persons": [{"person_id": "p1", "tracking_paths": [{"track": [ + {"frame": frame, "clip_frame": None, "x": 0, "y": 0, "width": 2, "height": 2} + for frame in (0, 8985, 8998, 8999) + ]}]}], + } + + +def terminal_annotation(): + clip = terminal_clip() + return {"frames": extract.ego4d_clip_boxes(clip), + "bounds": {key: clip[key] for key in ( + "clip_start_frame", "clip_end_frame", "clip_start_sec", "clip_end_sec", + )}} + + +class Ego4DBoxFrameTests(unittest.TestCase): + def setUp(self): + torch.set_num_threads(1) + self.source = SimpleNamespace(num_frames=8999, source_fps=30.) + + def test_confirmed_terminal_only_is_removed_without_mutating_input(self): + annotation = terminal_annotation() + original = copy.deepcopy(annotation) + with contextlib.redirect_stdout(io.StringIO()) as log: + frames, audit = extract.validate_ego4d_box_frames(UID, annotation, self.source) + self.assertEqual(frames, {frame: original["frames"][frame] for frame in (0, 8985, 8998)}) + self.assertEqual(annotation, original) + self.assertEqual(audit["ignored_frame_indices"], [8999]) + self.assertEqual(audit["ignored_box_count"], 1) + self.assertEqual(audit["annotation_frame_range"], [0, 8999]) + self.assertIn("No frame shift or feature/target row removal", log.getvalue()) + + def test_valid_and_empty_mappings_are_unchanged_even_without_bounds(self): + for frames in ({}, {0: [], 8998: [{"id": "valid"}]}): + with self.subTest(frames=frames): + result, audit = extract.validate_ego4d_box_frames( + UID, {"frames": frames, "bounds": {}}, self.source) + self.assertIs(result, frames) + self.assertEqual(audit["ignored_frame_indices"], []) + + def test_larger_overrun_negative_and_truncated_source_still_fail(self): + for extra in (-1, 9000, 12000): + annotation = terminal_annotation() + annotation["frames"][extra] = [] + with self.subTest(frame=extra), self.assertRaisesRegex(ValueError, "source_frames=8999"): + extract.validate_ego4d_box_frames(UID, annotation, self.source) + with self.assertRaisesRegex(ValueError, "source_frames=8998"): + extract.validate_ego4d_box_frames( + UID, terminal_annotation(), SimpleNamespace(num_frames=8998, source_fps=30.)) + + def test_terminal_requires_zero_origin_matching_declared_end_and_duration(self): + for key, value in (("clip_start_frame", 1), ("clip_end_frame", 9000), + ("clip_start_sec", .1), ("clip_end_sec", 300.), + ("clip_end_sec", float("nan"))): + annotation = terminal_annotation() + annotation["bounds"][key] = value + with self.subTest(key=key, value=value), self.assertRaisesRegex(ValueError, "declared_bounds"): + extract.validate_ego4d_box_frames(UID, annotation, self.source) + for key in terminal_annotation()["bounds"]: + annotation = terminal_annotation() + del annotation["bounds"][key] + with self.subTest(missing=key), self.assertRaises(ValueError): + extract.validate_ego4d_box_frames(UID, annotation, self.source) + for fps in (29.97, 15., 0., float("nan")): + with self.subTest(fps=fps), self.assertRaises(ValueError): + extract.validate_ego4d_box_frames( + UID, terminal_annotation(), SimpleNamespace(num_frames=8999, source_fps=fps)) + + def test_full_extraction_keeps_1499_rows_saves_audit_and_reuses(self): + image = Image.new("RGB", (2, 2)) + source = SimpleNamespace(num_frames=8999, source_fps=30., crop=None, dense_start=0., + path=Path(f"{UID}.mp4"), width=2, height=2) + media = SimpleNamespace( + clips=[source], metadata=lambda: {}, + _source_frame_for_time=lambda t: (0, int(t * 30 + 1e-8), int(t * 30 + 1e-8) / 30), + frame_at_or_before=lambda t: image, + audio_slice=lambda start, end: np.array([start, end]), + ) + fake_module = SimpleNamespace(CausalSourceAV=lambda *a, **kw: media, + SourceClipSpec=lambda path: path, _easycom_directory_specs=None) + encoder = SimpleNamespace( + visual_dim=6, audio_dim=5, transformers_version="4.57.6", model_config_sha256="test", + images=lambda crops: torch.ones(len(crops), 6), + audio=lambda waveform: torch.full((5,), float(waveform[-1])), + ) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for folder in ("global", "targets", "v1/clips", "v1/annotations"): + (root / folder).mkdir(parents=True) + reference, _, targets = fixture(n=1499, name=UID, dataset="ego4d") + torch.save(reference, root / "global" / f"{UID}.pt") + np.save(root / "targets" / f"{UID}.npy", targets) + (root / "v1/clips" / f"{UID}.mp4").write_bytes(b"fake-video") + for split, clips in (("train", [terminal_clip()]), ("val", [])): + (root / f"v1/annotations/av_{split}.json").write_text(json.dumps({"videos": [{"clips": clips}]})) + loaded = extract.load_ego4d_boxes([root / "v1/annotations/av_train.json"], [UID]) + self.assertEqual(loaded[UID], terminal_annotation()) + argv = ["--dataset", "ego4d", "--data-root", str(root), "--feature-root", str(root / "global"), + "--target-root", str(root / "targets"), "--output-root", str(root / "local"), + "--samples", UID, "--no-progress"] + with patch.dict(sys.modules, {"transformers": SimpleNamespace(__version__="4.57.6"), + "streaming_turn_model_directly": fake_module}), \ + patch.object(extract, "FrozenQwenLocalEncoder", return_value=encoder) as model, \ + contextlib.redirect_stdout(io.StringIO()): + extract.main(argv) + model.assert_called_once() + destination = root / "local" / f"{UID}.pt" + result = torch.load(destination, weights_only=False) + self.assertEqual(result["visual"].shape, (1499, 6)) + self.assertEqual(result["audio"].shape, (1499, 5)) + self.assertTrue(torch.equal(result["times"], reference["times"])) + self.assertTrue(torch.equal(result["label_indices"], reference["label_indices"])) + self.assertAlmostEqual(result["times"][-1].item(), 299.8) + self.assertFalse(result["partial"]) + self.assertEqual(result["box_frame_audit"]["ignored_frame_indices"], [8999]) + self.assertEqual(result["persons"]["frame_times"][-1].item(), 299.5) + self.assertEqual(result["persons"]["records"][-1]["source_frame_index"], 8985) + load_local_features(destination, reference, sample_id=UID, dataset="ego4d", modalities="av") + # Previously completed valid sidecars have no audit. Their + # vectors/config remain reusable: only unsampled boxes changed. + del result["box_frame_audit"] + torch.save(result, destination) + model.reset_mock() + extract.main(argv + ["--reuse-existing"]) + model.assert_not_called() + extract.main(argv + ["--preview-only", "--preview-count", "1"]) + audit = json.loads((root / "local" / f"{UID}_preview/box_frame_audit.json").read_text()) + self.assertEqual(audit["ignored_frame_indices"], [8999]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test_ego4d_local_workflow.py b/test_ego4d_local_workflow.py index a44b621..d538a8e 100644 --- a/test_ego4d_local_workflow.py +++ b/test_ego4d_local_workflow.py @@ -138,9 +138,14 @@ class Ego4DLocalWorkflowTests(unittest.TestCase): model_config_sha256="same-checkpoint") _, side, _ = fixture(dataset="ego4d") media = SimpleNamespace(clips=[], metadata=lambda: {}) + + def fake_build(args, name, end, annotations): + annotations[name]["frame_audit"] = {"ignored_frame_indices": []} + return media, [] + with patch.dict(sys.modules, {"transformers": SimpleNamespace(__version__="4.57.6")}), \ patch("extract_turn_local_features.FrozenQwenLocalEncoder", return_value=fake_encoder) as encoder, \ - patch("extract_turn_local_features.build_media_and_boxes", return_value=(media, [])) as build, \ + patch("extract_turn_local_features.build_media_and_boxes", side_effect=fake_build) as build, \ patch("extract_turn_local_features.extract_local", side_effect=lambda *a: copy.deepcopy(side)): extract.main(self.extraction_args()) encoder.assert_called_once()