wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix bbox offset

Browse Source
This commit is contained in:
Xin Wang
2026-08-27 15:08:38 +08:00
parent 11f197724f
commit 694473b475
3 changed files with 130 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extract_turn_local_features.py
+46 -6
View File
@@ -194,20 +194,34 @@ def build_media_and_boxes(args: argparse.Namespace, name: str, end: float, ego_b
return media, annotations
def box_coordinate_offset(clip, coordinate_space: str) -> tuple[int, int]:
"""Map source/encoded boxes into the image actually returned by the reader."""
if coordinate_space == "encoded":
return tuple(clip.crop[:2]) if clip.crop is not None else (0, 0)
if coordinate_space == "camera":
# Only for annotations already translated into the cropped image.
return (0, 0)
raise ValueError(f"Unknown box coordinate space: {coordinate_space}")
def frame_crops(media, annotations, time: float, args: argparse.Namespace):
clip_index, frame_index, source_pts = media._source_frame_for_time(time)
clip = media.clips[clip_index]
image = media.frame_at_or_before(time)
# Dataset coordinates normally refer to the 1920px camera image (after
# removing EasyCom's left banner). Explicit override for alternate exports.
offset_x, offset_y = (clip.crop[:2] if clip.crop and args.box_coordinate_space == "encoded" else (0, 0))
# EasyCom's supplied boxes refer to the encoded image, including the left
# banner. The reader has already removed that banner. Subtract this clip's
# actual crop origin, not a hard-coded offset shared by all sessions.
offset_x, offset_y = box_coordinate_offset(clip, args.box_coordinate_space)
crops, records = [], []
for person in annotations[clip_index].get(frame_index, []):
box = crop_box(person["xyxy"], *image.size, offset_x=offset_x, offset_y=offset_y,
expansion=args.crop_expansion)
if box is not None:
crops.append(image.crop(box))
records.append({"id": person["id"], "box": box, "source_clip_index": clip_index,
records.append({"id": person["id"], "box": box,
"annotation_box": list(person["xyxy"]),
"coordinate_offset": [offset_x, offset_y],
"source_clip_index": clip_index,
"source_frame_index": frame_index,
"source_time": clip.dense_start + source_pts})
return image, crops, records
@@ -235,14 +249,29 @@ def preview_boxes(media, annotations, args: argparse.Namespace, name: str, end:
folder = args.output_root / f"{name}_preview"
folder.mkdir(parents=True, exist_ok=True)
ticks = list(causal_frame_ticks(end, args.video_fps))
audit = []
for index in np.linspace(0, len(ticks) - 1, min(4, len(ticks)), dtype=int):
time = ticks[index] / args.video_fps
image, _, records = frame_crops(media, annotations, time, args)
clip_index, frame_index, _ = media._source_frame_for_time(time)
clip = media.clips[clip_index]
offset = box_coordinate_offset(clip, args.box_coordinate_space)
draw = ImageDraw.Draw(image)
for record in records:
draw.rectangle(record["box"], outline="lime", width=4)
draw.text(record["box"][:2], record["id"], fill="red")
image.save(folder / f"frame_{ticks[index]:06d}.jpg")
filename = f"frame_{ticks[index]:06d}.jpg"
image.save(folder / filename)
audit.append({
"image": filename, "time": time, "source_clip": str(clip.path),
"source_frame_index": frame_index, "source_size": [clip.width, clip.height],
"crop": list(clip.crop) if clip.crop is not None else None,
"box_coordinate_space": args.box_coordinate_space,
"coordinate_offset": list(offset), "boxes": records,
})
print(f"preview t={time:g}s: {clip.width}x{clip.height} -> {image.width}x{image.height}; "
f"boxes={args.box_coordinate_space}, subtract offset={offset}")
(folder / "boxes.json").write_text(json.dumps(audit, indent=2) + "\n")
print(f"Box previews: {folder} (inspect before extracting all sessions)")
@@ -307,7 +336,12 @@ def parse_args(argv=None):
p.add_argument("--audio-window-seconds", type=float, default=1.)
p.add_argument("--visual-window-seconds", type=float, default=1.)
p.add_argument("--box-kind", choices=("face", "head"), default="face")
p.add_argument("--box-coordinate-space", choices=("camera", "encoded"), default="camera")
p.add_argument(
"--box-coordinate-space", choices=("camera", "encoded"), default=None,
help=("Box coordinates before or after video cropping. Default: encoded for EasyCom "
"(subtract the left banner), camera for uncropped Ego4D. Use camera only for "
"annotations already translated to the cropped image."),
)
p.add_argument("--crop-expansion", type=float, default=1.)
p.add_argument("--max-pixels", type=int, default=224 * 224)
p.add_argument("--image-batch-size", type=int, default=8)
@@ -319,6 +353,8 @@ def parse_args(argv=None):
p.add_argument("--overwrite", action="store_true")
p.add_argument("--no-progress", action="store_true")
args = p.parse_args(argv)
if args.box_coordinate_space is None:
args.box_coordinate_space = "encoded" if args.dataset == "easycom" else "camera"
for name in ("video_fps", "audio_window_seconds", "visual_window_seconds", "crop_expansion"):
if not np.isfinite(getattr(args, name)) or getattr(args, name) <= 0:
p.error(f"--{name.replace('_', '-')} must be finite and positive")
@@ -363,6 +399,10 @@ def main() -> None:
times, indices = times[:args.max_steps], indices[:args.max_steps]
media, annotations = build_media_and_boxes(args, name, len(times) / 5, ego_boxes)
validate_source_timeline(reference, media)
if (args.dataset == "easycom" and args.box_coordinate_space == "camera"
and any(clip.crop is not None and clip.crop[0] for clip in media.clips)):
print("NOTE: camera boxes receive NO banner-offset correction. For original EasyCom "
"annotations use --box-coordinate-space encoded; otherwise boxes shift right.", flush=True)
if args.preview_only:
preview_boxes(media, annotations, args, name, len(times) / 5)
continue