oracle: add fixed four-epoch final protocol
This commit is contained in:
@@ -167,6 +167,10 @@ The detailed architecture, controls and stopping gates for P3 are frozen in
|
|||||||
- EasyCom targets are dense `[T, 3]`; predictions are
|
- EasyCom targets are dense `[T, 3]`; predictions are
|
||||||
`[target_time, class, future_horizon]`.
|
`[target_time, class, future_horizon]`.
|
||||||
- EasyCom split: train Session 4--12; test Session 1--3.
|
- EasyCom split: train Session 4--12; test Session 1--3.
|
||||||
|
- Locked final head protocol: exactly 4 epochs on all Sessions 4--12, no
|
||||||
|
development checkpoint selection, followed by one evaluation on Sessions
|
||||||
|
1--3. Historical Session-12 screens remain development diagnostics and are
|
||||||
|
not the final training protocol.
|
||||||
|
|
||||||
### Global feature sidecar
|
### Global feature sidecar
|
||||||
|
|
||||||
|
|||||||
@@ -71,17 +71,39 @@ Sessions 1--3.
|
|||||||
If preparation reports any transcript/target mismatch, stop and inspect it;
|
If preparation reports any transcript/target mismatch, stop and inspect it;
|
||||||
the preparation command has no mismatch-tolerance option.
|
the preparation command has no mismatch-tolerance option.
|
||||||
|
|
||||||
|
## Locked four-epoch train-all protocol
|
||||||
|
|
||||||
|
The final protocol fixes the training budget at four epochs, trains on all
|
||||||
|
official EasyCom training sessions (Sessions 4--12), performs no development
|
||||||
|
checkpoint selection, and evaluates Sessions 1--3 once:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
CUDA_VISIBLE_DEVICES=0 bash run_easycom_oracle_information.sh final 0 1 2
|
||||||
|
```
|
||||||
|
|
||||||
|
The `final` stage enforces `EPOCHS=4` and exactly Sessions 4--12, clears the
|
||||||
|
development-session list internally, and saves the last epoch. Its artifacts
|
||||||
|
use separate `_train4_12_fixed4` checkpoint/result roots by default, so they
|
||||||
|
cannot overwrite the historical Session-12 development screen. Override those
|
||||||
|
roots only with `FINAL_CHECKPOINT_ROOT` and `FINAL_RESULT_ROOT`.
|
||||||
|
|
||||||
|
mAP itself does not require threshold calibration. Nevertheless, the official
|
||||||
|
test results must not be used to choose a variant, epoch, loss, fusion design,
|
||||||
|
or any later hyperparameter. Run `final` only after those decisions are frozen;
|
||||||
|
repeatedly inspecting Sessions 1--3 and modifying the method would contaminate
|
||||||
|
the official test split.
|
||||||
|
|
||||||
## Follow-up
|
## Follow-up
|
||||||
|
|
||||||
If `per-person` is consistently better on Session 12, add seeds 1 and 2 using
|
To reproduce the historical Session-12 development screen for seeds 1 and 2,
|
||||||
the already prepared sidecars:
|
use the already prepared sidecars:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
CUDA_VISIBLE_DEVICES=0 bash run_easycom_oracle_information.sh screen 1 2
|
CUDA_VISIBLE_DEVICES=0 bash run_easycom_oracle_information.sh screen 1 2
|
||||||
```
|
```
|
||||||
|
|
||||||
Only after freezing the method and event evaluator should selected variants be
|
The older two-stage equivalent of final test inference is retained for
|
||||||
run once on Sessions 1--3:
|
reproducibility, but the locked protocol above is preferred for new final runs:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
VARIANTS="audio aggregate-visible per-person shuffled-id delayed" \
|
VARIANTS="audio aggregate-visible per-person shuffled-id delayed" \
|
||||||
|
|||||||
@@ -72,18 +72,13 @@ variant_local_args() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
train_and_screen() {
|
train_variants() {
|
||||||
mkdir -p "${CHECKPOINT_ROOT}" "${RESULT_ROOT}"
|
mkdir -p "${CHECKPOINT_ROOT}" "${RESULT_ROOT}"
|
||||||
for seed in "${seeds[@]}"; do
|
for seed in "${seeds[@]}"; do
|
||||||
for variant in "${variants[@]}"; do
|
for variant in "${variants[@]}"; do
|
||||||
checkpoint="${CHECKPOINT_ROOT}/${variant}_seed${seed}.pth"
|
checkpoint="${CHECKPOINT_ROOT}/${variant}_seed${seed}.pth"
|
||||||
result="${RESULT_ROOT}/${variant}_seed${seed}"
|
if [[ -e "${checkpoint}" ]]; then
|
||||||
local_feature_root="${ORACLE_ROOT}/${variant}"
|
echo "Refusing to overwrite ${checkpoint}" >&2
|
||||||
if [[ "${variant}" == "audio" ]]; then
|
|
||||||
local_feature_root="${LOCAL_ROOT}"
|
|
||||||
fi
|
|
||||||
if [[ -e "${checkpoint}" || -e "${result}" ]]; then
|
|
||||||
echo "Refusing to overwrite ${checkpoint} or ${result}" >&2
|
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
local_args=()
|
local_args=()
|
||||||
@@ -111,7 +106,28 @@ train_and_screen() {
|
|||||||
--expected-mode "${EXPECTED_MODE}" \
|
--expected-mode "${EXPECTED_MODE}" \
|
||||||
--seed "${seed}" \
|
--seed "${seed}" \
|
||||||
--device cuda:0
|
--device cuda:0
|
||||||
|
done
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
screen_variants() {
|
||||||
|
mkdir -p "${RESULT_ROOT}"
|
||||||
|
for seed in "${seeds[@]}"; do
|
||||||
|
for variant in "${variants[@]}"; do
|
||||||
|
checkpoint="${CHECKPOINT_ROOT}/${variant}_seed${seed}.pth"
|
||||||
|
result="${RESULT_ROOT}/${variant}_seed${seed}"
|
||||||
|
local_feature_root="${ORACLE_ROOT}/${variant}"
|
||||||
|
if [[ "${variant}" == "audio" ]]; then
|
||||||
|
local_feature_root="${LOCAL_ROOT}"
|
||||||
|
fi
|
||||||
|
if [[ ! -f "${checkpoint}" ]]; then
|
||||||
|
echo "Missing checkpoint: ${checkpoint}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [[ -e "${result}" ]]; then
|
||||||
|
echo "Refusing to overwrite ${result}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
echo "$(date): screening ${variant}, seed=${seed} on ${screen_sessions[*]}"
|
echo "$(date): screening ${variant}, seed=${seed} on ${screen_sessions[*]}"
|
||||||
"${PYTHON_BIN}" infer_turn_head.py \
|
"${PYTHON_BIN}" infer_turn_head.py \
|
||||||
--feature-root "${FEATURE_ROOT}" \
|
--feature-root "${FEATURE_ROOT}" \
|
||||||
@@ -130,6 +146,30 @@ train_and_screen() {
|
|||||||
--protocol causal_valid_only
|
--protocol causal_valid_only
|
||||||
}
|
}
|
||||||
|
|
||||||
|
train_all_fixed_and_test() {
|
||||||
|
if [[ "${EPOCHS}" != "4" ]]; then
|
||||||
|
echo "The final protocol requires EPOCHS=4, got ${EPOCHS}" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
normalized_train=()
|
||||||
|
for session in "${train_sessions[@]}"; do
|
||||||
|
normalized_train+=("${session#Session_}")
|
||||||
|
done
|
||||||
|
if [[ "${normalized_train[*]}" != "4 5 6 7 8 9 10 11 12" ]]; then
|
||||||
|
echo "The final protocol requires exactly EasyCom Sessions 4--12 for training" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# No development checkpoint selection: the final epoch is fixed in advance.
|
||||||
|
# Keep these artifacts separate from the historical Session-12 screen.
|
||||||
|
dev_sessions=()
|
||||||
|
CHECKPOINT_ROOT="${FINAL_CHECKPOINT_ROOT:-${CHECKPOINT_ROOT}_train4_12_fixed4}"
|
||||||
|
RESULT_ROOT="${FINAL_RESULT_ROOT:-${RESULT_ROOT}_train4_12_fixed4}"
|
||||||
|
train_variants
|
||||||
|
test_selected
|
||||||
|
}
|
||||||
|
|
||||||
test_selected() {
|
test_selected() {
|
||||||
mkdir -p "${RESULT_ROOT}_test"
|
mkdir -p "${RESULT_ROOT}_test"
|
||||||
for seed in "${seeds[@]}"; do
|
for seed in "${seeds[@]}"; do
|
||||||
@@ -170,11 +210,17 @@ case "${stage}" in
|
|||||||
prepare_oracles
|
prepare_oracles
|
||||||
;;
|
;;
|
||||||
screen)
|
screen)
|
||||||
train_and_screen
|
train_variants
|
||||||
|
screen_variants
|
||||||
;;
|
;;
|
||||||
all)
|
all)
|
||||||
prepare_oracles
|
prepare_oracles
|
||||||
train_and_screen
|
train_variants
|
||||||
|
screen_variants
|
||||||
|
;;
|
||||||
|
final)
|
||||||
|
prepare_oracles
|
||||||
|
train_all_fixed_and_test
|
||||||
;;
|
;;
|
||||||
test)
|
test)
|
||||||
test_selected
|
test_selected
|
||||||
@@ -186,7 +232,7 @@ case "${stage}" in
|
|||||||
--protocol causal_valid_only
|
--protocol causal_valid_only
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
echo "Usage: $0 {prepare|screen|all|test|summarize} [seed ...]" >&2
|
echo "Usage: $0 {prepare|screen|all|final|test|summarize} [seed ...]" >&2
|
||||||
exit 2
|
exit 2
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|||||||
Reference in New Issue
Block a user