Add causal person fusion and MuVAP stage-3 baselines

local_turn_features.py

@@ -15,6 +15,7 @@ import torch
 
 LOCAL_FORMAT = "qwen-local-pooled-v1"
 ORACLE_FORMAT = "easycom-oracle-person-history-v1"
+HIGHRATE_PERSON_AV_FORMAT = "causal-highrate-person-av-v2"
 MODALITIES = {"visual": ("visual",), "audio": ("audio",), "av": ("visual", "audio")}
 
 
@@ -279,12 +280,29 @@ def _load_validated_payload(
     modalities: str,
     expected_contract: dict[str, Any] | None,
     oracle_analysis: bool = False,
+    highrate_person_av: bool = False,
+    allow_highrate_audio: bool = False,
 ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any], dict[str, Any], dict[str, Any]]:
     if modalities not in MODALITIES:
         raise ValueError(f"Unknown local modalities: {modalities}")
     times, indices = reference_alignment(reference)
     payload = torch.load(path, map_location="cpu", weights_only=False)
-    expected_format = ORACLE_FORMAT if oracle_analysis else LOCAL_FORMAT
+    if (
+        allow_highrate_audio
+        and modalities == "audio"
+        and isinstance(payload, dict)
+        and payload.get("format") == HIGHRATE_PERSON_AV_FORMAT
+    ):
+        highrate_person_av = True
+    if oracle_analysis and highrate_person_av:
+        raise ValueError("Oracle and learned high-rate local formats are distinct")
+    expected_format = (
+        ORACLE_FORMAT
+        if oracle_analysis
+        else HIGHRATE_PERSON_AV_FORMAT
+        if highrate_person_av
+        else LOCAL_FORMAT
+    )
     if not isinstance(payload, dict) or payload.get("format") != expected_format:
         raise ValueError(f"Invalid local feature format: {path}")
     if payload.get("sample_id") != sample_id or payload.get("dataset") != dataset:
@@ -308,6 +326,24 @@ def _load_validated_payload(
             raise ValueError(
                 f"Oracle features lack the explicit causal/oracle contract: {path}"
             )
+    elif highrate_person_av:
+        if (
+            config.get("causal_policy") != "right-edge-causal-highrate-asd"
+            or config.get("uses_speech_labels") is not False
+            or config.get("model_fps") != 25.0
+            or config.get("output_hz") != 5.0
+            or config.get("person_feature_kind")
+            != "muvap-asd-causal-visual-plus-activity-logit"
+            or config.get("audio_feature_kind")
+            != "muvap-cpc-causal-audio-state"
+            or config.get("audio_source") != "single-shared-traditional-cpc-stream"
+            or config.get("person_visual_dim") != config.get("visual_dim") - 1
+            or config.get("person_activity_logit_dim") != 1
+            or config.get("person_tokens_duplicate_audio") is not False
+        ):
+            raise ValueError(
+                f"High-rate person AV features lack the causal ASD contract: {path}"
+            )
     elif (
         config.get("causal_policy") != "half-open-trailing-window"
         or config.get("uses_speech_labels") is not False
@@ -336,6 +372,7 @@ def load_local_features(
         dataset=dataset,
         modalities=modalities,
         expected_contract=expected_contract,
+        allow_highrate_audio=True,
     )
     chunks = []
     for name in MODALITIES[modalities]:
@@ -365,6 +402,7 @@ def load_frame_set_features(
     expected_contract: dict[str, Any] | None = None,
     track_max_gap_seconds: float | None = None,
     oracle_analysis: bool = False,
+    highrate_person_av: bool = False,
 ) -> tuple[torch.Tensor, FramePersonFeatures, dict[str, Any]]:
     """Load direct audio and the latest causal per-frame person set per 5 Hz row."""
 
@@ -376,6 +414,7 @@ def load_frame_set_features(
         modalities="av",
         expected_contract=expected_contract,
         oracle_analysis=oracle_analysis,
+        highrate_person_av=highrate_person_av,
     )
     audio = payload.get("audio")
     audio_valid = payload.get("audio_valid")
@@ -441,18 +480,22 @@ def load_frame_set_features(
     if len(frame_times) and float(frame_times[-1]) > float(times[-1]) + 1e-8:
         raise ValueError(f"Person frame timeline extends beyond the reference: {path}")
 
-    # Normal visual observations must be strictly earlier than the cutoff.
+    # Normal visual observations are instantaneous samples and must be strictly
+    # earlier than the cutoff.  High-rate ASD states instead summarize complete
+    # 25 Hz intervals ending at the stored right edge, so equality with a 5 Hz
+    # cutoff is causal and intentional.  Oracle current state has the same
+    # lookup geometry but remains a separate, analysis-only format.
     # Oracle speech state is defined and sampled at the cutoff itself, so its
     # explicit analysis-only format uses an inclusive current-state lookup.
     row_to_frame = torch.searchsorted(
         frame_times,
         times,
-        right=oracle_analysis,
+        right=oracle_analysis or highrate_person_av,
     ) - 1
     chosen = torch.full(times.shape, float("nan"), dtype=torch.float64)
     has_frame = row_to_frame >= 0
     chosen[has_frame] = frame_times[row_to_frame[has_frame]]
-    if oracle_analysis:
+    if oracle_analysis or highrate_person_av:
         if bool((chosen[has_frame] > times[has_frame]).any()):
             raise ValueError(f"Oracle person mapping selected future evidence: {path}")
     elif bool((chosen[has_frame] >= times[has_frame]).any()):
@@ -482,6 +525,36 @@ def load_frame_set_features(
     return direct_audio, frame_sets, contract
 
 
+def load_highrate_person_av_features(
+    path: Path,
+    reference: dict[str, Any],
+    *,
+    sample_id: str,
+    dataset: str,
+    expected_contract: dict[str, Any] | None = None,
+) -> tuple[torch.Tensor, FramePersonFeatures, dict[str, Any]]:
+    """Load frozen 25 Hz causal ASD states sampled at 5 Hz right edges.
+
+    The returned dense tensor contains the single shared causal CPC audio state
+    and its validity bit. The visual dimension in the contract belongs to the
+    ragged, variable-cardinality set of visual-track state plus ASD score; it is
+    not a pooled image observation consumed by the head.
+    """
+
+    direct_audio, people, contract = load_frame_set_features(
+        path,
+        reference,
+        sample_id=sample_id,
+        dataset=dataset,
+        expected_contract=expected_contract,
+        highrate_person_av=True,
+    )
+    config = contract["config"]
+    if config.get("audio_source") != "single-shared-traditional-cpc-stream":
+        raise ValueError(f"Unexpected high-rate sidecar audio source: {path}")
+    return direct_audio, people, contract
+
+
 def _build_previous_track_links(
     records: list[Any],
     frame_times: torch.Tensor,