Add causal person fusion and MuVAP stage-3 baselines
This commit is contained in:
Xin Wang
Executable
+159
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env bash
|
||||
# Global Omni with Qwen audio, traditional audio, and high-rate person evidence.
|
||||
# Usage: bash run_easycom_highrate_person_av.sh \
|
||||
# [all|global|qwen_audio|traditional_audio|highrate|test] [seed ...]
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
FEATURE_ROOT=${FEATURE_ROOT:-/data1/yfzx/wangx/Datasets/easycom/omni_features/open_stream_2fps_8s_last4_stack}
|
||||
TARGET_ROOT=${TARGET_ROOT:-/data1/yfzx/wangx/Datasets/easycom/easycom_target_frame}
|
||||
QWEN_LOCAL_ROOT=${QWEN_LOCAL_ROOT:-/data1/yfzx/wangx/Datasets/easycom/omni_features/local_qwen_1s_encoded_tf4576}
|
||||
HIGHRATE_LOCAL_ROOT=${HIGHRATE_LOCAL_ROOT:-/data1/yfzx/wangx/Datasets/easycom/omni_features/local_muvap_asd_25hz_v2}
|
||||
CHECKPOINT_ROOT=${CHECKPOINT_ROOT:-checkpoints/easycom_highrate_person_av}
|
||||
RESULT_ROOT=${RESULT_ROOT:-results/easycom_highrate_person_av_screen}
|
||||
TEST_RESULT_ROOT=${TEST_RESULT_ROOT:-results/easycom_highrate_person_av_test}
|
||||
EPOCHS=${EPOCHS:-4}
|
||||
WARMUP_EPOCHS=${WARMUP_EPOCHS:-1.6}
|
||||
LOCAL_PROJECTION_DIM=${LOCAL_PROJECTION_DIM:-256}
|
||||
PERSON_ATTENTION_DIM=${PERSON_ATTENTION_DIM:-256}
|
||||
PERSON_ATTENTION_HEADS=${PERSON_ATTENTION_HEADS:-4}
|
||||
SUMMARIZE=${SUMMARIZE:-1}
|
||||
read -r -a TRAIN_SESSION_ARGS <<< "${TRAIN_SESSIONS:-4 5 6 7 8 9 10 11 12}"
|
||||
read -r -a DEV_SESSION_ARGS <<< "${DEV_SESSIONS:-12}"
|
||||
read -r -a SCREEN_SESSION_ARGS <<< "${SCREEN_SESSIONS:-12}"
|
||||
export OMP_NUM_THREADS=${OMP_NUM_THREADS:-4}
|
||||
export MKL_NUM_THREADS=${MKL_NUM_THREADS:-4}
|
||||
export OPENBLAS_NUM_THREADS=${OPENBLAS_NUM_THREADS:-4}
|
||||
|
||||
variant=${1:-all}
|
||||
if (( $# )); then shift; fi
|
||||
if (( $# )); then seeds=("$@"); else seeds=(0 1 2); fi
|
||||
case "$variant" in
|
||||
all) variants=(global qwen_audio traditional_audio highrate) ;;
|
||||
global|qwen_audio|traditional_audio|highrate) variants=("$variant") ;;
|
||||
test) variants=(global qwen_audio traditional_audio highrate) ;;
|
||||
*) echo "Expected all, global, qwen_audio, traditional_audio, highrate, or test" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
mkdir -p "$CHECKPOINT_ROOT" "$RESULT_ROOT"
|
||||
|
||||
run_inference() {
|
||||
local checkpoint=$1 local_root=$2 output=$3
|
||||
shift 3
|
||||
if [[ -e "$output/metrics.json" ]]; then
|
||||
echo "Refusing to overwrite $output/metrics.json" >&2
|
||||
return 1
|
||||
fi
|
||||
local local_args=()
|
||||
if [[ -n "$local_root" ]]; then
|
||||
local_args=(--local-feature-root "$local_root")
|
||||
fi
|
||||
python infer_turn_head.py \
|
||||
--feature-root "$FEATURE_ROOT" \
|
||||
--target-root "$TARGET_ROOT" \
|
||||
"${local_args[@]}" \
|
||||
--checkpoint "$checkpoint" \
|
||||
--output-dir "$output" \
|
||||
--sessions "$@" \
|
||||
--expected-mode open-stream-replay \
|
||||
--device cuda:0
|
||||
}
|
||||
|
||||
if [[ "$variant" == test ]]; then
|
||||
mkdir -p "$TEST_RESULT_ROOT"
|
||||
for selected in "${variants[@]}"; do
|
||||
for seed in "${seeds[@]}"; do
|
||||
checkpoint="$CHECKPOINT_ROOT/${selected}_seed${seed}.pth"
|
||||
[[ -s "$checkpoint" ]] || { echo "Missing checkpoint: $checkpoint" >&2; exit 1; }
|
||||
if [[ "$selected" == global ]]; then
|
||||
local_root=
|
||||
elif [[ "$selected" == qwen_audio ]]; then
|
||||
local_root=$QWEN_LOCAL_ROOT
|
||||
else
|
||||
local_root=$HIGHRATE_LOCAL_ROOT
|
||||
fi
|
||||
run_inference "$checkpoint" "$local_root" \
|
||||
"$TEST_RESULT_ROOT/${selected}_seed${seed}" 3 2 1
|
||||
done
|
||||
done
|
||||
summary_root=$TEST_RESULT_ROOT
|
||||
else
|
||||
for selected in "${variants[@]}"; do
|
||||
for seed in "${seeds[@]}"; do
|
||||
checkpoint="$CHECKPOINT_ROOT/${selected}_seed${seed}.pth"
|
||||
result="$RESULT_ROOT/${selected}_seed${seed}"
|
||||
initialization_args=()
|
||||
local_args=()
|
||||
if [[ -e "$checkpoint" || -e "$result/metrics.json" ]]; then
|
||||
echo "Refusing to overwrite $checkpoint or $result" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$selected" == global ]]; then
|
||||
local_root=
|
||||
modalities=av
|
||||
fusion=pooled
|
||||
elif [[ "$selected" == qwen_audio ]]; then
|
||||
local_root=$QWEN_LOCAL_ROOT
|
||||
modalities=audio
|
||||
fusion=pooled
|
||||
elif [[ "$selected" == traditional_audio ]]; then
|
||||
local_root=$HIGHRATE_LOCAL_ROOT
|
||||
modalities=audio
|
||||
fusion=pooled
|
||||
else
|
||||
local_root=$HIGHRATE_LOCAL_ROOT
|
||||
modalities=av
|
||||
fusion=highrate-person-av-attention
|
||||
audio_checkpoint="$CHECKPOINT_ROOT/traditional_audio_seed${seed}.pth"
|
||||
[[ -s "$audio_checkpoint" ]] || {
|
||||
echo "Missing matched audio checkpoint: $audio_checkpoint" >&2
|
||||
echo "Run: bash run_easycom_highrate_person_av.sh traditional_audio $seed" >&2
|
||||
exit 1
|
||||
}
|
||||
initialization_args=(--initialize-pooled-checkpoint "$audio_checkpoint")
|
||||
fi
|
||||
if [[ -n "$local_root" ]]; then
|
||||
local_args=(
|
||||
--local-feature-root "$local_root"
|
||||
--local-modalities "$modalities"
|
||||
)
|
||||
fi
|
||||
echo "$(date): ${selected}, seed=${seed}"
|
||||
python train_turn_head.py \
|
||||
--feature-root "$FEATURE_ROOT" \
|
||||
--target-root "$TARGET_ROOT" \
|
||||
"${local_args[@]}" \
|
||||
--local-fusion "$fusion" \
|
||||
--local-projection-dim "$LOCAL_PROJECTION_DIM" \
|
||||
--person-attention-dim "$PERSON_ATTENTION_DIM" \
|
||||
--person-attention-heads "$PERSON_ATTENTION_HEADS" \
|
||||
"${initialization_args[@]}" \
|
||||
--output "$checkpoint" \
|
||||
--sessions "${TRAIN_SESSION_ARGS[@]}" \
|
||||
--dev-sessions "${DEV_SESSION_ARGS[@]}" \
|
||||
--epochs "$EPOCHS" \
|
||||
--batch-size 512 \
|
||||
--learning-rate 7e-5 \
|
||||
--weight-decay 5e-5 \
|
||||
--warmup-epochs "$WARMUP_EPOCHS" \
|
||||
--warmup-factor 0.3 \
|
||||
--dropout 0.2 \
|
||||
--loss lstr-mce \
|
||||
--transition-objective none \
|
||||
--layer-fusion concat-mlp \
|
||||
--expected-mode open-stream-replay \
|
||||
--seed "$seed" \
|
||||
--device cuda:0
|
||||
run_inference "$checkpoint" "$local_root" "$result" "${SCREEN_SESSION_ARGS[@]}"
|
||||
done
|
||||
done
|
||||
summary_root=$RESULT_ROOT
|
||||
fi
|
||||
|
||||
if [[ "$SUMMARIZE" == 1 ]]; then
|
||||
python summarize_local_baselines.py \
|
||||
--root "$summary_root" \
|
||||
--dataset easycom \
|
||||
--protocol causal_valid_only \
|
||||
--paired-deltas
|
||||
fi
|
||||
Reference in New Issue
Block a user