From 694473b4756a0ccef2bd46cd204400ecb5662d2e Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Thu, 27 Aug 2026 15:08:38 +0800 Subject: [PATCH] Fix bbox offset --- docs/local_feature_baseline.md | 19 +++++++-- extract_turn_local_features.py | 52 ++++++++++++++++++++++--- test_local_turn_features.py | 70 +++++++++++++++++++++++++++++++++- 3 files changed, 130 insertions(+), 11 deletions(-) diff --git a/docs/local_feature_baseline.md b/docs/local_feature_baseline.md index c669030..5ed114a 100644 --- a/docs/local_feature_baseline.md +++ b/docs/local_feature_baseline.md @@ -49,7 +49,7 @@ LOCAL_ARGS=( --feature-root "$FEATURE_ROOT" --target-root "$TARGET_ROOT" --model-id "$MODEL" --device cuda:0 --dtype bfloat16 --flash-attention-2 --video-fps 2 --visual-window-seconds 1 --audio-window-seconds 1 - --box-kind face --box-coordinate-space camera + --box-kind face --box-coordinate-space encoded ) # 不加载模型；输出四张带框图片，请先检查框是否真在人脸上。 @@ -62,11 +62,22 @@ python extract_turn_local_features.py "${LOCAL_ARGS[@]}" \ ``` EasyCom face 标注首帧是 1，head 首帧是 0；按**原始解码帧号**取框。 -视频裁去左侧身份 banner，保留右侧 1920 像素；默认框坐标属于裁后的相机图像。 -如果你的导出文件的框坐标含 banner，需要改成 `--box-coordinate-space encoded`。 -必须通过 preview 检查这个约定，不能只看最终特征维度。 +视频裁去左侧身份 banner，保留右侧 1920 像素；这份 EasyCom 标注的框坐标包含 banner。 +EasyCom 默认使用 `encoded`，先减去当前源视频的裁剪起点，再裁框： +`x_camera = x_annotation - (source_width - 1920)`。例如已核对的 Session_4 原视频宽度 +为 2124，应减去 204；不同视频可能不同，不能统一硬编码。y 也按实际裁剪起点转换。 +`camera` 仅用于已经手动换算到裁后图像的标注；Ego4D 无此 banner 裁剪，默认仍为 `camera`。 +之前示例中的 `camera` 参数会导致这份原始 EasyCom 标注统一右偏，现已更正。 +若当前 shell 的 `LOCAL_ARGS` 还含旧参数，请修改它，或将 `--box-coordinate-space encoded` +放在 `"${LOCAL_ARGS[@]}"` **后面**覆盖旧值。仅更新代码不会覆盖显式传入的旧参数。 +预览会打印 source/cropped 尺寸与实际扣除偏移，同时在预览目录生成 `boxes.json`， +记录原框、裁后框与源帧号。必须检查框确实落在人脸上，不能只看最终特征维度。 官方帧号与 banner 说明见 [EasyCom 数据说明](https://github.com/facebookresearch/EasyComDataset#notes-and-known-issues)。 +若已经用错误坐标提取局部视觉特征，必须重新从图像提取，不能靠修改 `.pt` 中的框坐标修复。 +推荐输出到新目录（例如 `local_qwen_1s_encoded`），并把训练的 `LOCAL_ROOT` 指向新目录。 +原全局 Omni 特征和 baseline 无需重算；旧局部 V/AV 训练结果不可与修正后的特征混用。 + ```bash python - <<'PY' import torch diff --git a/extract_turn_local_features.py b/extract_turn_local_features.py index 5eeb39a..4313caa 100644 --- a/extract_turn_local_features.py +++ b/extract_turn_local_features.py @@ -194,20 +194,34 @@ def build_media_and_boxes(args: argparse.Namespace, name: str, end: float, ego_b return media, annotations +def box_coordinate_offset(clip, coordinate_space: str) -> tuple[int, int]: + """Map source/encoded boxes into the image actually returned by the reader.""" + if coordinate_space == "encoded": + return tuple(clip.crop[:2]) if clip.crop is not None else (0, 0) + if coordinate_space == "camera": + # Only for annotations already translated into the cropped image. + return (0, 0) + raise ValueError(f"Unknown box coordinate space: {coordinate_space}") + + def frame_crops(media, annotations, time: float, args: argparse.Namespace): clip_index, frame_index, source_pts = media._source_frame_for_time(time) clip = media.clips[clip_index] image = media.frame_at_or_before(time) - # Dataset coordinates normally refer to the 1920px camera image (after - # removing EasyCom's left banner). Explicit override for alternate exports. - offset_x, offset_y = (clip.crop[:2] if clip.crop and args.box_coordinate_space == "encoded" else (0, 0)) + # EasyCom's supplied boxes refer to the encoded image, including the left + # banner. The reader has already removed that banner. Subtract this clip's + # actual crop origin, not a hard-coded offset shared by all sessions. + offset_x, offset_y = box_coordinate_offset(clip, args.box_coordinate_space) crops, records = [], [] for person in annotations[clip_index].get(frame_index, []): box = crop_box(person["xyxy"], *image.size, offset_x=offset_x, offset_y=offset_y, expansion=args.crop_expansion) if box is not None: crops.append(image.crop(box)) - records.append({"id": person["id"], "box": box, "source_clip_index": clip_index, + records.append({"id": person["id"], "box": box, + "annotation_box": list(person["xyxy"]), + "coordinate_offset": [offset_x, offset_y], + "source_clip_index": clip_index, "source_frame_index": frame_index, "source_time": clip.dense_start + source_pts}) return image, crops, records @@ -235,14 +249,29 @@ def preview_boxes(media, annotations, args: argparse.Namespace, name: str, end: folder = args.output_root / f"{name}_preview" folder.mkdir(parents=True, exist_ok=True) ticks = list(causal_frame_ticks(end, args.video_fps)) + audit = [] for index in np.linspace(0, len(ticks) - 1, min(4, len(ticks)), dtype=int): time = ticks[index] / args.video_fps image, _, records = frame_crops(media, annotations, time, args) + clip_index, frame_index, _ = media._source_frame_for_time(time) + clip = media.clips[clip_index] + offset = box_coordinate_offset(clip, args.box_coordinate_space) draw = ImageDraw.Draw(image) for record in records: draw.rectangle(record["box"], outline="lime", width=4) draw.text(record["box"][:2], record["id"], fill="red") - image.save(folder / f"frame_{ticks[index]:06d}.jpg") + filename = f"frame_{ticks[index]:06d}.jpg" + image.save(folder / filename) + audit.append({ + "image": filename, "time": time, "source_clip": str(clip.path), + "source_frame_index": frame_index, "source_size": [clip.width, clip.height], + "crop": list(clip.crop) if clip.crop is not None else None, + "box_coordinate_space": args.box_coordinate_space, + "coordinate_offset": list(offset), "boxes": records, + }) + print(f"preview t={time:g}s: {clip.width}x{clip.height} -> {image.width}x{image.height}; " + f"boxes={args.box_coordinate_space}, subtract offset={offset}") + (folder / "boxes.json").write_text(json.dumps(audit, indent=2) + "\n") print(f"Box previews: {folder} (inspect before extracting all sessions)") @@ -307,7 +336,12 @@ def parse_args(argv=None): p.add_argument("--audio-window-seconds", type=float, default=1.) p.add_argument("--visual-window-seconds", type=float, default=1.) p.add_argument("--box-kind", choices=("face", "head"), default="face") - p.add_argument("--box-coordinate-space", choices=("camera", "encoded"), default="camera") + p.add_argument( + "--box-coordinate-space", choices=("camera", "encoded"), default=None, + help=("Box coordinates before or after video cropping. Default: encoded for EasyCom " + "(subtract the left banner), camera for uncropped Ego4D. Use camera only for " + "annotations already translated to the cropped image."), + ) p.add_argument("--crop-expansion", type=float, default=1.) p.add_argument("--max-pixels", type=int, default=224 * 224) p.add_argument("--image-batch-size", type=int, default=8) @@ -319,6 +353,8 @@ def parse_args(argv=None): p.add_argument("--overwrite", action="store_true") p.add_argument("--no-progress", action="store_true") args = p.parse_args(argv) + if args.box_coordinate_space is None: + args.box_coordinate_space = "encoded" if args.dataset == "easycom" else "camera" for name in ("video_fps", "audio_window_seconds", "visual_window_seconds", "crop_expansion"): if not np.isfinite(getattr(args, name)) or getattr(args, name) <= 0: p.error(f"--{name.replace('_', '-')} must be finite and positive") @@ -363,6 +399,10 @@ def main() -> None: times, indices = times[:args.max_steps], indices[:args.max_steps] media, annotations = build_media_and_boxes(args, name, len(times) / 5, ego_boxes) validate_source_timeline(reference, media) + if (args.dataset == "easycom" and args.box_coordinate_space == "camera" + and any(clip.crop is not None and clip.crop[0] for clip in media.clips)): + print("NOTE: camera boxes receive NO banner-offset correction. For original EasyCom " + "annotations use --box-coordinate-space encoded; otherwise boxes shift right.", flush=True) if args.preview_only: preview_boxes(media, annotations, args, name, len(times) / 5) continue diff --git a/test_local_turn_features.py b/test_local_turn_features.py index 2726a0d..aab353e 100644 --- a/test_local_turn_features.py +++ b/test_local_turn_features.py @@ -15,13 +15,14 @@ from types import SimpleNamespace import numpy as np import torch -from PIL import Image +from PIL import Image, ImageDraw from easycom_head import TurnPredictionHead, checkpoint_head_config from extract_turn_local_features import ( FrozenQwenLocalEncoder, ego4d_clip_boxes, extract_local, validate_source_timeline, model_config_digest, + box_coordinate_offset, frame_crops, parse_args, preview_boxes, ) from local_turn_features import ( LOCAL_FORMAT, causal_frame_ticks, crop_box, easycom_boxes, load_local_features, @@ -82,6 +83,73 @@ class LocalTests(unittest.TestCase): actual = ego4d_clip_boxes({"persons": [wearer, person]}) self.assertEqual(actual, {3: [{"id": "x", "xyxy": [1., 2., 4., 6.]}]}) + def test_dataset_specific_coordinate_defaults_and_explicit_overrides(self): + required = ["--data-root", "/raw", "--feature-root", "/global", + "--target-root", "/targets", "--output-root", "/local", "--samples", "4"] + self.assertEqual(parse_args(required).box_coordinate_space, "encoded") + self.assertEqual(parse_args(required + ["--dataset", "ego4d"]).box_coordinate_space, "camera") + self.assertEqual(parse_args(required + ["--box-coordinate-space", "camera"]).box_coordinate_space, "camera") + # An appended correction overrides an old LOCAL_ARGS array in the shell. + self.assertEqual(parse_args(required + ["--box-coordinate-space", "camera", + "--box-coordinate-space", "encoded"]).box_coordinate_space, "encoded") + + def test_frame_crops_subtract_each_clips_banner_before_encoding(self): + expected = (100, 80, 160, 140) + image = Image.new("RGB", (1920, 1080)) + ImageDraw.Draw(image).rectangle(expected, fill="red") + clips, annotations = [], [] + for width in (2123, 2124, 2203, 1920): + offset = width - 1920 + clips.append(SimpleNamespace(crop=(offset, 0, 1920, 1080), dense_start=float(len(clips)))) + annotations.append({0: [{"id": "3", "xyxy": [100 + offset, 80, 160 + offset, 140]}]}) + media = SimpleNamespace(clips=clips, + _source_frame_for_time=lambda t: (int(t), 0, 0.), + frame_at_or_before=lambda t: image.copy()) + args = SimpleNamespace(box_coordinate_space="encoded", crop_expansion=1.) + for index, clip in enumerate(clips): + with self.subTest(crop=clip.crop): + _, crops, records = frame_crops(media, annotations, float(index), args) + self.assertEqual(records[0]["box"], expected) + self.assertEqual(records[0]["coordinate_offset"], [clip.crop[0], 0]) + self.assertEqual(records[0]["annotation_box"], annotations[index][0][0]["xyxy"]) + self.assertEqual(crops[0].size, (60, 60)) + self.assertEqual(crops[0].getpixel((30, 30)), (255, 0, 0)) + + def test_cropped_camera_annotations_and_uncropped_ego4d_are_not_shifted_twice(self): + clip = SimpleNamespace(crop=(204, 12, 1920, 1080), dense_start=0.) + self.assertEqual(box_coordinate_offset(clip, "encoded"), (204, 12)) + self.assertEqual(box_coordinate_offset(clip, "camera"), (0, 0)) + self.assertEqual(box_coordinate_offset(SimpleNamespace(crop=None), "encoded"), (0, 0)) + with self.assertRaisesRegex(ValueError, "coordinate space"): + box_coordinate_offset(clip, "typo") + media = SimpleNamespace(clips=[clip], _source_frame_for_time=lambda t: (0, 0, 0.), + frame_at_or_before=lambda t: Image.new("RGB", (1920, 1080))) + annotations = [{0: [{"id": "3", "xyxy": [100, 80, 160, 140]}]}] + args = SimpleNamespace(box_coordinate_space="camera", crop_expansion=1.) + _, _, records = frame_crops(media, annotations, .1, args) + self.assertEqual(records[0]["box"], (100, 80, 160, 140)) + # Encoded boxes translate on BOTH axes, then clip at the visible edge. + annotations = [{0: [{"id": "3", "xyxy": [190, 4, 230, 50]}]}] + args.box_coordinate_space = "encoded" + _, _, records = frame_crops(media, annotations, .1, args) + self.assertEqual(records[0]["box"], (0, 0, 26, 38)) + + def test_preview_records_the_same_transformation_used_for_extraction(self): + clip = SimpleNamespace(path=Path("source.mp4"), width=2124, height=1080, + crop=(204, 0, 1920, 1080), dense_start=0.) + media = SimpleNamespace(clips=[clip], _source_frame_for_time=lambda t: (0, 0, 0.), + frame_at_or_before=lambda t: Image.new("RGB", (1920, 1080))) + annotations = [{0: [{"id": "3", "xyxy": [304, 80, 364, 140]}]}] + with tempfile.TemporaryDirectory() as directory: + args = SimpleNamespace(box_coordinate_space="encoded", crop_expansion=1., + video_fps=2., output_root=Path(directory)) + preview_boxes(media, annotations, args, "Session_4", .2) + folder = Path(directory) / "Session_4_preview" + audit = json.loads((folder / "boxes.json").read_text()) + self.assertEqual(audit[0]["coordinate_offset"], [204, 0]) + self.assertEqual(audit[0]["boxes"][0]["box"], [100, 80, 160, 140]) + self.assertTrue((folder / audit[0]["image"]).is_file()) + def test_sidecar_alignment_contract_and_modalities(self): reference, side, _ = fixture() with tempfile.TemporaryDirectory() as directory: