wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add no-box full-frame Qwen AV baselines

Browse Source
This commit is contained in:
Xin Wang
2026-09-17 02:51:26 +08:00
parent a9f748d6a6
commit 62134a0fa0
10 changed files with 1164 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
train_turn_head.py
+27 -11
View File
@@ -24,6 +24,8 @@ from tqdm.auto import tqdm
from easycom_head import (
CLASS_NAMES,
DATASET_NAMES,
FULLFRAME_FUSION_MODES,
HISTORY_VISUAL_SET_FUSION_MODES,
HIGHRATE_PERSON_AV_FUSION_MODES,
LAYER_FUSION_MODES,
LOCAL_FUSION_MODES,
@@ -87,6 +89,9 @@ def parse_args() -> argparse.Namespace:
"highrate-person-av-attention lets global Omni plus one shared "
"traditional CPC audio stream query causal 25 Hz person visual/ASD "
"states sampled at the 5 Hz cutoffs; "
"fullframe-spatial-attention is F2 over the latest strictly "
"pre-cutoff full-frame grid; fullframe-spatiotemporal-attention "
"is F3 with a shared causal grid-position GRU; "
"audio-person-frame-set and gated-person-state-adapter are "
"current-sidecar F5/F6 screening variants, not full F5/F6."
),
@@ -237,7 +242,7 @@ def _validate_args(args: argparse.Namespace) -> None:
args.local_feature_root is None or args.local_modalities != "av"
):
raise ValueError(
"Person-set --local-fusion modes require --local-feature-root and "
"Set/token --local-fusion modes require --local-feature-root and "
"--local-modalities av"
)
if args.local_fusion in INITIALIZED_PERSON_RESIDUAL_MODES:
@@ -528,7 +533,7 @@ def _evaluate_valid_horizons(
person_mask = None
if people is not None:
rows = torch.arange(start, min(start + batch_size, len(states)))
if head.local_fusion in TRACK_PERSON_FUSION_MODES:
if head.local_fusion in HISTORY_VISUAL_SET_FUSION_MODES:
person_batch, person_mask = people.padded_history(
rows,
max_observations=head.person_history_frames,
@@ -1004,14 +1009,25 @@ def main() -> None:
f"projection={args.local_projection_dim or 'identity'}"
)
if person_features is not None:
print(
"frame sets: "
f"{len(person_features.frame_offsets) - 1} source frames, "
f"{len(person_features.features)} person observations, "
f"max_people={person_features.max_people}, "
f"attention={args.person_attention_dim}D/"
f"{args.person_attention_heads} heads"
)
if args.local_fusion in FULLFRAME_FUSION_MODES:
print(
"full-frame token sets: "
f"{len(person_features.frame_offsets) - 1} source frames, "
f"{len(person_features.features)} spatial tokens, "
f"max_tokens_per_frame={person_features.max_people}, "
f"history_frames={args.person_history_frames}, "
f"attention={args.person_attention_dim}D/"
f"{args.person_attention_heads} heads"
)
else:
print(
"frame sets: "
f"{len(person_features.frame_offsets) - 1} source frames, "
f"{len(person_features.features)} person observations, "
f"max_people={person_features.max_people}, "
f"attention={args.person_attention_dim}D/"
f"{args.person_attention_heads} heads"
)
trainable_parameters = sum(
parameter.numel() for parameter in head.parameters() if parameter.requires_grad
)
@@ -1132,7 +1148,7 @@ def main() -> None:
person_batch = None
person_mask = None
if person_features is not None:
if head.local_fusion in TRACK_PERSON_FUSION_MODES:
if head.local_fusion in HISTORY_VISUAL_SET_FUSION_MODES:
person_batch, person_mask = person_features.padded_history(
example_index_batch,
max_observations=head.person_history_frames,