wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add more preview

Browse Source
This commit is contained in:
Xin Wang
2026-08-28 00:46:05 +08:00
parent b5ce64702a
commit 9bd204b468
3 changed files with 200 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extract_turn_local_features.py
+81 -6
View File
@@ -264,15 +264,57 @@ def validate_source_timeline(reference: dict[str, Any], media) -> None:
raise ValueError(f"Local/reference source timeline differs at {new['path']}")
def preview_time_bounds(args: argparse.Namespace, available_end: float) -> tuple[float, float]:
"""Preview-only range on the original timeline, never a rebased feature grid."""
start = float(getattr(args, "preview_start", 0.))
requested_end = getattr(args, "preview_end", None)
end = min(available_end, requested_end) if requested_end is not None else available_end
if start >= end:
raise ValueError(f"Preview start {start:g}s is outside the available prefix ending at {end:g}s; "
"remove/increase --max-steps or choose an earlier --preview-start")
return start, end
def preview_ticks(media, annotations, args: argparse.Namespace, end: float) -> tuple[list[int], int, int]:
start, stop = preview_time_bounds(args, end)
ticks = [tick for tick in causal_frame_ticks(stop, args.video_fps)
if tick / args.video_fps >= start - 1e-8]
grid_count = len(ticks)
if getattr(args, "preview_with_boxes", False):
eligible = []
# Scan annotation metadata, not every RGB frame. Use the same source
# frame mapping, coordinate correction and crop validity as extraction.
for tick in ticks:
clip_index, frame_index, _ = media._source_frame_for_time(tick / args.video_fps)
clip = media.clips[clip_index]
width, height = clip.crop[2:] if clip.crop is not None else (clip.width, clip.height)
offset_x, offset_y = box_coordinate_offset(clip, args.box_coordinate_space)
if any(crop_box(person["xyxy"], width, height, offset_x=offset_x, offset_y=offset_y,
expansion=args.crop_expansion) is not None
for person in annotations[clip_index].get(frame_index, [])):
eligible.append(tick)
ticks = eligible
eligible_count = len(ticks)
count = min(getattr(args, "preview_count", 4), eligible_count)
indices = np.linspace(0, eligible_count - 1, count, dtype=int) if count else []
return [ticks[index] for index in indices], grid_count, eligible_count
def preview_boxes(media, annotations, args: argparse.Namespace, name: str, end: float) -> None:
from PIL import ImageDraw
folder = args.output_root / f"{name}_preview"
folder.mkdir(parents=True, exist_ok=True)
ticks = list(causal_frame_ticks(end, args.video_fps))
start, stop = preview_time_bounds(args, end)
ticks, grid_count, eligible_count = preview_ticks(media, annotations, args, end)
print(f"Preview [{start:g},{stop:g})s: {grid_count} grid frames, {eligible_count} eligible, "
f"saving {len(ticks)} images", flush=True)
if not ticks:
print("No eligible preview frames. Try a longer range or another clip. "
"If people are visible but no valid boxes exist, check annotation/frame alignment.", flush=True)
audit = []
for index in np.linspace(0, len(ticks) - 1, min(4, len(ticks)), dtype=int):
time = ticks[index] / args.video_fps
for tick in ticks:
time = tick / args.video_fps
image, _, records = frame_crops(media, annotations, time, args)
clip_index, frame_index, _ = media._source_frame_for_time(time)
clip = media.clips[clip_index]
@@ -281,7 +323,10 @@ def preview_boxes(media, annotations, args: argparse.Namespace, name: str, end:
for record in records:
draw.rectangle(record["box"], outline="lime", width=4)
draw.text(record["box"][:2], record["id"], fill="red")
filename = f"frame_{ticks[index]:06d}.jpg"
caption = f"t={time:.2f}s | source frame={frame_index} | people={len(records)}"
draw.rectangle(draw.textbbox((8, 8), caption), fill="black")
draw.text((8, 8), caption, fill="white")
filename = f"frame_{tick:06d}.jpg"
image.save(folder / filename)
audit.append({
"image": filename, "time": time, "source_clip": str(clip.path),
@@ -291,9 +336,18 @@ def preview_boxes(media, annotations, args: argparse.Namespace, name: str, end:
"coordinate_offset": list(offset), "boxes": records,
})
print(f"preview t={time:g}s: {clip.width}x{clip.height} -> {image.width}x{image.height}; "
f"boxes={args.box_coordinate_space}, subtract offset={offset}")
f"boxes={args.box_coordinate_space}, subtract offset={offset}, people={len(records)}")
(folder / "boxes.json").write_text(json.dumps(audit, indent=2) + "\n")
(folder / "preview.json").write_text(json.dumps({
"start": start, "end": stop, "video_fps": args.video_fps,
"with_boxes_only": getattr(args, "preview_with_boxes", False),
"requested_count": getattr(args, "preview_count", 4),
"grid_frame_count": grid_count, "eligible_frame_count": eligible_count,
"saved_count": len(audit), "saved_with_boxes": sum(bool(row["boxes"]) for row in audit),
"images": [row["image"] for row in audit],
}, indent=2) + "\n")
print(f"Box previews: {folder} (inspect before extracting all sessions)")
print("boxes.json lists this run's images; older images in the directory are not removed.")
def extract_local(media, annotations, encoder, times: torch.Tensor, args: argparse.Namespace) -> dict[str, Any]:
@@ -425,6 +479,11 @@ def parse_args(argv=None):
p.add_argument("--decoder-gpu-id", type=int, default=0, help="Visible-device index, not physical GPU number")
p.add_argument("--max-steps", type=int, help="Smoke test only; partial sidecars are rejected by training")
p.add_argument("--preview-only", action="store_true", help="Save box overlays without loading the model")
p.add_argument("--preview-count", type=int, default=4, help="Maximum images per clip (preview only)")
p.add_argument("--preview-start", type=float, default=0., help="Start time in seconds on the clip/dense timeline")
p.add_argument("--preview-end", type=float, help="Exclusive end time; default: entire available prefix")
p.add_argument("--preview-with-boxes", action="store_true",
help="Sample only grid frames with valid provided crops; preview selection only, not feature filtering")
existing = p.add_mutually_exclusive_group()
existing.add_argument("--overwrite", action="store_true")
existing.add_argument("--reuse-existing", action="store_true", help="Validate and skip complete matching sidecars")
@@ -444,6 +503,19 @@ def parse_args(argv=None):
p.error("--reuse-existing is for full extraction, not smoke tests or previews")
if args.dry_run and args.preview_only:
p.error("--dry-run and --preview-only are separate checks")
if args.preview_count <= 0:
p.error("--preview-count must be positive")
if not np.isfinite(args.preview_start) or args.preview_start < 0:
p.error("--preview-start must be finite and nonnegative")
if args.preview_end is not None and (not np.isfinite(args.preview_end) or args.preview_end <= args.preview_start):
p.error("--preview-end must be finite and greater than --preview-start")
if not args.preview_only and (args.preview_count != 4 or args.preview_start != 0.
or args.preview_end is not None or args.preview_with_boxes):
p.error("Preview selection options require --preview-only")
if args.preview_only and args.max_steps is not None:
if args.preview_start >= args.max_steps / 5 or (args.preview_end is not None
and args.preview_end > args.max_steps / 5):
p.error("Preview range exceeds --max-steps; remove --max-steps to inspect later frames")
if args.box_coordinate_space is None:
args.box_coordinate_space = "encoded" if args.dataset == "easycom" else "camera"
for name in ("video_fps", "audio_window_seconds", "visual_window_seconds", "crop_expansion"):
@@ -523,7 +595,10 @@ def main(argv=None) -> None:
full_steps = len(times)
if args.max_steps is not None:
times, indices = times[:args.max_steps], indices[:args.max_steps]
media, annotations = build_media_and_boxes(args, name, len(times) / 5, ego_boxes)
media_end = len(times) / 5
if args.preview_only:
_, media_end = preview_time_bounds(args, media_end)
media, annotations = build_media_and_boxes(args, name, media_end, ego_boxes)
validate_source_timeline(reference, media)
if (args.dataset == "easycom" and args.box_coordinate_space == "camera"
and any(clip.crop is not None and clip.crop[0] for clip in media.clips)):