wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix bbox offset

Browse Source
This commit is contained in:
Xin Wang
2026-08-27 15:08:38 +08:00
parent 11f197724f
commit 694473b475
3 changed files with 130 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/local_feature_baseline.md
+15 -4
View File
@@ -49,7 +49,7 @@ LOCAL_ARGS=(
--feature-root "$FEATURE_ROOT" --target-root "$TARGET_ROOT"
--model-id "$MODEL" --device cuda:0 --dtype bfloat16 --flash-attention-2
--video-fps 2 --visual-window-seconds 1 --audio-window-seconds 1
--box-kind face --box-coordinate-space camera
--box-kind face --box-coordinate-space encoded
)
# 不加载模型；输出四张带框图片，请先检查框是否真在人脸上。
@@ -62,11 +62,22 @@ python extract_turn_local_features.py "${LOCAL_ARGS[@]}" \
```
EasyCom face 标注首帧是 1，head 首帧是 0；按**原始解码帧号**取框。
视频裁去左侧身份 banner，保留右侧 1920 像素；默认框坐标属于裁后的相机图像。
如果你的导出文件的框坐标含 banner，需要改成 `--box-coordinate-space encoded`。
必须通过 preview 检查这个约定，不能只看最终特征维度。
视频裁去左侧身份 banner，保留右侧 1920 像素；这份 EasyCom 标注的框坐标包含 banner。
EasyCom 默认使用 `encoded`，先减去当前源视频的裁剪起点，再裁框：
`x_camera = x_annotation - (source_width - 1920)`。例如已核对的 Session_4 原视频宽度
为 2124，应减去 204；不同视频可能不同，不能统一硬编码。y 也按实际裁剪起点转换。
`camera` 仅用于已经手动换算到裁后图像的标注；Ego4D 无此 banner 裁剪，默认仍为 `camera`。
之前示例中的 `camera` 参数会导致这份原始 EasyCom 标注统一右偏，现已更正。
若当前 shell 的 `LOCAL_ARGS` 还含旧参数，请修改它，或将 `--box-coordinate-space encoded`
放在 `"${LOCAL_ARGS[@]}"` **后面**覆盖旧值。仅更新代码不会覆盖显式传入的旧参数。
预览会打印 source/cropped 尺寸与实际扣除偏移，同时在预览目录生成 `boxes.json`，
记录原框、裁后框与源帧号。必须检查框确实落在人脸上，不能只看最终特征维度。
官方帧号与 banner 说明见 [EasyCom 数据说明](https://github.com/facebookresearch/EasyComDataset#notes-and-known-issues)。
若已经用错误坐标提取局部视觉特征，必须重新从图像提取，不能靠修改 `.pt` 中的框坐标修复。
推荐输出到新目录（例如 `local_qwen_1s_encoded`），并把训练的 `LOCAL_ROOT` 指向新目录。
原全局 Omni 特征和 baseline 无需重算；旧局部 V/AV 训练结果不可与修正后的特征混用。
```bash
python - <<'PY'
import torch
extract_turn_local_features.py
+46 -6
View File
@@ -194,20 +194,34 @@ def build_media_and_boxes(args: argparse.Namespace, name: str, end: float, ego_b
return media, annotations
def box_coordinate_offset(clip, coordinate_space: str) -> tuple[int, int]:
"""Map source/encoded boxes into the image actually returned by the reader."""
if coordinate_space == "encoded":
return tuple(clip.crop[:2]) if clip.crop is not None else (0, 0)
if coordinate_space == "camera":
# Only for annotations already translated into the cropped image.
return (0, 0)
raise ValueError(f"Unknown box coordinate space: {coordinate_space}")
def frame_crops(media, annotations, time: float, args: argparse.Namespace):
clip_index, frame_index, source_pts = media._source_frame_for_time(time)
clip = media.clips[clip_index]
image = media.frame_at_or_before(time)
# Dataset coordinates normally refer to the 1920px camera image (after
# removing EasyCom's left banner). Explicit override for alternate exports.
offset_x, offset_y = (clip.crop[:2] if clip.crop and args.box_coordinate_space == "encoded" else (0, 0))
# EasyCom's supplied boxes refer to the encoded image, including the left
# banner. The reader has already removed that banner. Subtract this clip's
# actual crop origin, not a hard-coded offset shared by all sessions.
offset_x, offset_y = box_coordinate_offset(clip, args.box_coordinate_space)
crops, records = [], []
for person in annotations[clip_index].get(frame_index, []):
box = crop_box(person["xyxy"], *image.size, offset_x=offset_x, offset_y=offset_y,
expansion=args.crop_expansion)
if box is not None:
crops.append(image.crop(box))
records.append({"id": person["id"], "box": box, "source_clip_index": clip_index,
records.append({"id": person["id"], "box": box,
"annotation_box": list(person["xyxy"]),
"coordinate_offset": [offset_x, offset_y],
"source_clip_index": clip_index,
"source_frame_index": frame_index,
"source_time": clip.dense_start + source_pts})
return image, crops, records
@@ -235,14 +249,29 @@ def preview_boxes(media, annotations, args: argparse.Namespace, name: str, end:
folder = args.output_root / f"{name}_preview"
folder.mkdir(parents=True, exist_ok=True)
ticks = list(causal_frame_ticks(end, args.video_fps))
audit = []
for index in np.linspace(0, len(ticks) - 1, min(4, len(ticks)), dtype=int):
time = ticks[index] / args.video_fps
image, _, records = frame_crops(media, annotations, time, args)
clip_index, frame_index, _ = media._source_frame_for_time(time)
clip = media.clips[clip_index]
offset = box_coordinate_offset(clip, args.box_coordinate_space)
draw = ImageDraw.Draw(image)
for record in records:
draw.rectangle(record["box"], outline="lime", width=4)
draw.text(record["box"][:2], record["id"], fill="red")
image.save(folder / f"frame_{ticks[index]:06d}.jpg")
filename = f"frame_{ticks[index]:06d}.jpg"
image.save(folder / filename)
audit.append({
"image": filename, "time": time, "source_clip": str(clip.path),
"source_frame_index": frame_index, "source_size": [clip.width, clip.height],
"crop": list(clip.crop) if clip.crop is not None else None,
"box_coordinate_space": args.box_coordinate_space,
"coordinate_offset": list(offset), "boxes": records,
})
print(f"preview t={time:g}s: {clip.width}x{clip.height} -> {image.width}x{image.height}; "
f"boxes={args.box_coordinate_space}, subtract offset={offset}")
(folder / "boxes.json").write_text(json.dumps(audit, indent=2) + "\n")
print(f"Box previews: {folder} (inspect before extracting all sessions)")
@@ -307,7 +336,12 @@ def parse_args(argv=None):
p.add_argument("--audio-window-seconds", type=float, default=1.)
p.add_argument("--visual-window-seconds", type=float, default=1.)
p.add_argument("--box-kind", choices=("face", "head"), default="face")
p.add_argument("--box-coordinate-space", choices=("camera", "encoded"), default="camera")
p.add_argument(
"--box-coordinate-space", choices=("camera", "encoded"), default=None,
help=("Box coordinates before or after video cropping. Default: encoded for EasyCom "
"(subtract the left banner), camera for uncropped Ego4D. Use camera only for "
"annotations already translated to the cropped image."),
)
p.add_argument("--crop-expansion", type=float, default=1.)
p.add_argument("--max-pixels", type=int, default=224 * 224)
p.add_argument("--image-batch-size", type=int, default=8)
@@ -319,6 +353,8 @@ def parse_args(argv=None):
p.add_argument("--overwrite", action="store_true")
p.add_argument("--no-progress", action="store_true")
args = p.parse_args(argv)
if args.box_coordinate_space is None:
args.box_coordinate_space = "encoded" if args.dataset == "easycom" else "camera"
for name in ("video_fps", "audio_window_seconds", "visual_window_seconds", "crop_expansion"):
if not np.isfinite(getattr(args, name)) or getattr(args, name) <= 0:
p.error(f"--{name.replace('_', '-')} must be finite and positive")
@@ -363,6 +399,10 @@ def main() -> None:
times, indices = times[:args.max_steps], indices[:args.max_steps]
media, annotations = build_media_and_boxes(args, name, len(times) / 5, ego_boxes)
validate_source_timeline(reference, media)
if (args.dataset == "easycom" and args.box_coordinate_space == "camera"
and any(clip.crop is not None and clip.crop[0] for clip in media.clips)):
print("NOTE: camera boxes receive NO banner-offset correction. For original EasyCom "
"annotations use --box-coordinate-space encoded; otherwise boxes shift right.", flush=True)
if args.preview_only:
preview_boxes(media, annotations, args, name, len(times) / 5)
continue
test_local_turn_features.py
+69 -1
View File
@@ -15,13 +15,14 @@ from types import SimpleNamespace
import numpy as np
import torch
from PIL import Image
from PIL import Image, ImageDraw
from easycom_head import TurnPredictionHead, checkpoint_head_config
from extract_turn_local_features import (
FrozenQwenLocalEncoder, ego4d_clip_boxes, extract_local,
validate_source_timeline,
model_config_digest,
box_coordinate_offset, frame_crops, parse_args, preview_boxes,
)
from local_turn_features import (
LOCAL_FORMAT, causal_frame_ticks, crop_box, easycom_boxes, load_local_features,
@@ -82,6 +83,73 @@ class LocalTests(unittest.TestCase):
actual = ego4d_clip_boxes({"persons": [wearer, person]})
self.assertEqual(actual, {3: [{"id": "x", "xyxy": [1., 2., 4., 6.]}]})
def test_dataset_specific_coordinate_defaults_and_explicit_overrides(self):
required = ["--data-root", "/raw", "--feature-root", "/global",
"--target-root", "/targets", "--output-root", "/local", "--samples", "4"]
self.assertEqual(parse_args(required).box_coordinate_space, "encoded")
self.assertEqual(parse_args(required + ["--dataset", "ego4d"]).box_coordinate_space, "camera")
self.assertEqual(parse_args(required + ["--box-coordinate-space", "camera"]).box_coordinate_space, "camera")
# An appended correction overrides an old LOCAL_ARGS array in the shell.
self.assertEqual(parse_args(required + ["--box-coordinate-space", "camera",
"--box-coordinate-space", "encoded"]).box_coordinate_space, "encoded")
def test_frame_crops_subtract_each_clips_banner_before_encoding(self):
expected = (100, 80, 160, 140)
image = Image.new("RGB", (1920, 1080))
ImageDraw.Draw(image).rectangle(expected, fill="red")
clips, annotations = [], []
for width in (2123, 2124, 2203, 1920):
offset = width - 1920
clips.append(SimpleNamespace(crop=(offset, 0, 1920, 1080), dense_start=float(len(clips))))
annotations.append({0: [{"id": "3", "xyxy": [100 + offset, 80, 160 + offset, 140]}]})
media = SimpleNamespace(clips=clips,
_source_frame_for_time=lambda t: (int(t), 0, 0.),
frame_at_or_before=lambda t: image.copy())
args = SimpleNamespace(box_coordinate_space="encoded", crop_expansion=1.)
for index, clip in enumerate(clips):
with self.subTest(crop=clip.crop):
_, crops, records = frame_crops(media, annotations, float(index), args)
self.assertEqual(records[0]["box"], expected)
self.assertEqual(records[0]["coordinate_offset"], [clip.crop[0], 0])
self.assertEqual(records[0]["annotation_box"], annotations[index][0][0]["xyxy"])
self.assertEqual(crops[0].size, (60, 60))
self.assertEqual(crops[0].getpixel((30, 30)), (255, 0, 0))
def test_cropped_camera_annotations_and_uncropped_ego4d_are_not_shifted_twice(self):
clip = SimpleNamespace(crop=(204, 12, 1920, 1080), dense_start=0.)
self.assertEqual(box_coordinate_offset(clip, "encoded"), (204, 12))
self.assertEqual(box_coordinate_offset(clip, "camera"), (0, 0))
self.assertEqual(box_coordinate_offset(SimpleNamespace(crop=None), "encoded"), (0, 0))
with self.assertRaisesRegex(ValueError, "coordinate space"):
box_coordinate_offset(clip, "typo")
media = SimpleNamespace(clips=[clip], _source_frame_for_time=lambda t: (0, 0, 0.),
frame_at_or_before=lambda t: Image.new("RGB", (1920, 1080)))
annotations = [{0: [{"id": "3", "xyxy": [100, 80, 160, 140]}]}]
args = SimpleNamespace(box_coordinate_space="camera", crop_expansion=1.)
_, _, records = frame_crops(media, annotations, .1, args)
self.assertEqual(records[0]["box"], (100, 80, 160, 140))
# Encoded boxes translate on BOTH axes, then clip at the visible edge.
annotations = [{0: [{"id": "3", "xyxy": [190, 4, 230, 50]}]}]
args.box_coordinate_space = "encoded"
_, _, records = frame_crops(media, annotations, .1, args)
self.assertEqual(records[0]["box"], (0, 0, 26, 38))
def test_preview_records_the_same_transformation_used_for_extraction(self):
clip = SimpleNamespace(path=Path("source.mp4"), width=2124, height=1080,
crop=(204, 0, 1920, 1080), dense_start=0.)
media = SimpleNamespace(clips=[clip], _source_frame_for_time=lambda t: (0, 0, 0.),
frame_at_or_before=lambda t: Image.new("RGB", (1920, 1080)))
annotations = [{0: [{"id": "3", "xyxy": [304, 80, 364, 140]}]}]
with tempfile.TemporaryDirectory() as directory:
args = SimpleNamespace(box_coordinate_space="encoded", crop_expansion=1.,
video_fps=2., output_root=Path(directory))
preview_boxes(media, annotations, args, "Session_4", .2)
folder = Path(directory) / "Session_4_preview"
audit = json.loads((folder / "boxes.json").read_text())
self.assertEqual(audit[0]["coordinate_offset"], [204, 0])
self.assertEqual(audit[0]["boxes"][0]["box"], [100, 80, 160, 140])
self.assertTrue((folder / audit[0]["image"]).is_file())
def test_sidecar_alignment_contract_and_modalities(self):
reference, side, _ = fixture()
with tempfile.TemporaryDirectory() as directory: