revise manuscript method and add architecture figure

manuscript/main/Makefile

@@ -1,10 +1,16 @@
 LATEXMK ?= latexmk
 
-.PHONY: all clean
+.PHONY: all clean figure
 
-all:
+all: figures/method_architecture.pdf
 	mkdir -p build
 	$(LATEXMK) -pdf -interaction=nonstopmode -halt-on-error -outdir=build main.tex
 
+figure: figures/method_architecture.pdf
+
+figures/method_architecture.pdf: figures/method_architecture.tex
+	$(LATEXMK) -pdf -interaction=nonstopmode -halt-on-error -outdir=figures $<
+
 clean:
 	$(LATEXMK) -C -outdir=build main.tex
+	$(LATEXMK) -c -outdir=figures figures/method_architecture.tex

manuscript/main/README.md

@@ -26,13 +26,18 @@ Current source layout:
 main/
   main.tex
   references.bib
+  figures/
+    method_architecture.tex  # reproducible TikZ source
+    method_architecture.pdf  # manuscript-ready vector figure
   latexmkrc
   Makefile
   README.md
   build/            # generated, ignored
 ```
 
-The manuscript is intentionally a content skeleton with visible TODOs. Do not
+Running `make` regenerates the method figure when its TikZ source changes
+before compiling the manuscript. The manuscript is intentionally a content
+skeleton with visible TODOs. Do not
 turn planned implementations, controls, or results into completed claims.
 
 The frozen paper story is documented in

manuscript/main/figures/.gitignore

@@ -0,0 +1,4 @@
+method_architecture.aux
+method_architecture.fdb_latexmk
+method_architecture.fls
+method_architecture.log

manuscript/main/figures/method_architecture.tex

@@ -0,0 +1,97 @@
+\documentclass[tikz,border=2pt]{standalone}
+\usepackage{amsmath}
+\usetikzlibrary{arrows.meta,calc,positioning}
+
+\definecolor{navy}{HTML}{17324D}
+\definecolor{bluefill}{HTML}{EAF2F8}
+\definecolor{teal}{HTML}{137C8B}
+\definecolor{tealfill}{HTML}{E8F6F5}
+\definecolor{orange}{HTML}{D97706}
+\definecolor{orangefill}{HTML}{FFF3E0}
+\definecolor{grayline}{HTML}{73808C}
+
+\begin{document}
+\begin{tikzpicture}[
+  font=\sffamily\scriptsize,
+  >=Latex,
+  flow/.style={-{Latex[length=2.0mm]}, line width=0.65pt, draw=navy},
+  aux/.style={-{Latex[length=1.8mm]}, line width=0.6pt, draw=teal},
+  box/.style={draw=navy, rounded corners=1.5pt, line width=0.65pt,
+              fill=bluefill, minimum height=7mm, align=center, inner sep=2.3pt},
+  localbox/.style={box, draw=teal, fill=tealfill},
+  residualbox/.style={box, draw=orange, fill=orangefill},
+  labelbox/.style={align=center, text=navy},
+  note/.style={font=\sffamily\tiny, text=grayline, align=center}
+]
+
+\node[labelbox] (av) at (0,2.65) {causal full-scene AV\\prefix $\mathcal X_{\leq t_n}$};
+\node[box, right=7mm of av, minimum width=24mm] (qwen) {frozen\\Qwen2.5-Omni};
+\node[box, right=8mm of qwen, minimum width=12mm] (g) {global\\$G_{t_n}$};
+
+\node[labelbox] (audio) at (0,1.45) {mixed audio\\$\leq t_n$};
+\node[localbox, right=7mm of audio, minimum width=24mm] (cpc) {frozen causal CPC\\25-Hz processing};
+\node[localbox, right=8mm of cpc, minimum width=12mm] (u) {shared\\$U_{t_n}$};
+
+\node[labelbox] (faces) at (0,0.15) {visible face crops\\$\leq t_n$};
+\node[localbox, right=7mm of faces, minimum width=24mm] (asd) {frozen causal AV/ASD\\25-Hz processing};
+\node[localbox, right=8mm of asd, minimum width=20mm] (persons) {anonymous set\\$\mathcal P_{t_n}=\{[V_i;s_i]\}$};
+
+\node[box, minimum width=21mm] (base) at (8.25,2.05) {$G+U$ base head\\$Z^{\rm base}_{t_n}$};
+\node[localbox, minimum width=22mm] (attn) at (8.25,0.70) {masked person MHA\\query: $G_{t_n}+U_{t_n}$\\\scriptsize memory: $\mathcal P_{t_n}+$ NULL};
+\node[residualbox, minimum width=19mm] (res) at (10.65,0.70) {zero-init residual\\$\Delta Z_{t_n}$};
+\node[circle, draw=navy, fill=white, line width=0.7pt,
+      minimum size=5.5mm, inner sep=0pt] (sum) at (11.55,2.05) {$+$};
+\node[box, minimum width=21mm] (out) at (13.15,2.05) {$10\!\times\!3$ future activity\\$0.2$--$2.0$ s};
+
+\draw[flow] (av) -- (qwen);
+\draw[flow] (qwen) -- (g);
+\draw[aux] (audio) -- (cpc);
+\draw[aux] (cpc) -- (u);
+\draw[aux] (faces) -- (asd);
+\draw[aux] (asd) -- (persons);
+\draw[flow] (g.east) -| ($(base.west)+(-0.25,0.12)$) -- (base.west);
+\draw[aux] (u.east) -| ($(base.west)+(-0.25,-0.12)$) -- (base.west);
+\draw[flow] (g.east) -- ++(0.3,0) |- (attn.north west);
+\draw[aux] (u.east) -- ++(0.5,0) |- (attn.west);
+\draw[aux] (persons.east) -- (attn.west);
+\draw[flow] (base) -- (sum);
+\draw[aux] (attn) -- (res);
+\draw[flow, draw=orange] (res.north) |- (sum.south);
+\draw[flow] (sum) -- (out);
+
+\node[note, below=1.5mm of base] {trained first, then frozen};
+\node[note, below=1.2mm of res] {trainable adaptation};
+
+\draw[grayline, line width=0.45pt] (-0.15,-0.62) -- (14.35,-0.62);
+\node[anchor=west, font=\sffamily\scriptsize\bfseries, text=navy]
+  at (-0.05,-0.92) {Causal streaming at cutoff $t_n$};
+
+\node[anchor=east, text=navy] at (2.25,-1.45) {Omni input};
+\draw[fill=bluefill, draw=navy, line width=0.55pt] (2.45,-1.67) rectangle (4.35,-1.23);
+\draw[fill=bluefill, draw=navy, line width=0.55pt] (4.35,-1.67) rectangle (6.25,-1.23);
+\node[text=navy] at (3.40,-1.45) {cached 2-s blocks};
+\node[text=navy] at (5.30,-1.45) {$\cdots$};
+\draw[fill=orangefill, draw=orange, line width=0.65pt] (6.25,-1.67) rectangle (9.15,-1.23);
+\node[text=orange] at (7.70,-1.45) {current partial block};
+\draw[dashed, draw=orange, line width=0.8pt] (9.15,-2.90) -- (9.15,-1.02);
+\node[anchor=south, text=orange, font=\sffamily\tiny\bfseries] at (9.15,-1.02) {$t_n$};
+\node[note, anchor=west] at (9.40,-1.45) {latest observed input included\\up to 8-s rolling budget};
+
+\node[anchor=east, text=teal] at (2.25,-2.15) {local encoder};
+\foreach \x in {2.55,2.85,...,8.85} {
+  \draw[draw=teal, line width=0.55pt] (\x,-2.33) -- (\x,-1.97);
+}
+\draw[draw=teal, line width=0.55pt] (2.55,-2.15) -- (9.15,-2.15);
+\node[note, anchor=west, text=teal] at (9.40,-2.15) {25 Hz internal states};
+
+\node[anchor=east, text=navy] at (2.25,-2.72) {forecast grid};
+\draw[draw=navy, line width=0.65pt] (2.55,-2.72) -- (9.15,-2.72);
+\foreach \x/\lab in {3.15/0.2,4.65/0.4,6.15/0.6,7.65/0.8,9.15/$t_n$} {
+  \draw[draw=navy, line width=0.75pt] (\x,-2.92) -- (\x,-2.52);
+  \node[below=0.5mm, font=\sffamily\tiny, text=navy] at (\x,-2.92) {\lab};
+  \draw[-{Latex[length=1.4mm]}, draw=teal, line width=0.55pt] (\x,-2.32) -- (\x,-2.50);
+}
+\node[note, anchor=west, text=navy] at (9.40,-2.72) {5 Hz fusion and prediction\\every fifth local right edge};
+
+\end{tikzpicture}
+\end{document}

manuscript/main/main.tex

@@ -107,117 +107,125 @@ corresponding TODO gates pass:
   positives and explicitly annotated hard-negative opportunity regions.
 \end{itemize}
 
-\begin{figure}[t]
+\begin{figure*}[t]
 \centering
-\fbox{\parbox[c][1.00in][c]{0.94\columnwidth}{\centering
+\includegraphics[width=0.98\textwidth]{figures/method_architecture.pdf}
-\todo{Draw only the selected method: causal AV to the long-context Omni path;
+\caption{Our strictly causal global--local architecture (top) and streaming
-the short-window mixed-audio and anonymous person-set path; residual fusion;
+alignment (bottom). At every 5-Hz cutoff $t_n$, the global state includes the
-the common $10\times3$ head; and the separate wearer-entry/INT-yield evaluation.
+latest partial 2-s Omni block, while every fifth causal 25-Hz local state
-Mark physical cutoffs, trainable modules, person masks, and NULL/OFFSCREEN.
+provides shared audio and an anonymous visible-person set. Masked person
-Do not draw oracle labels, rejected track variants, or multiple backbones.}}}
+attention contributes only through a zero-initialized residual to the matched
-\caption{Long-context Omni understanding and short-window local floor evidence
+$G+U$ baseline.}
-for wearer-relative future-activity prediction.}
 \label{fig:method}
-\end{figure}
+\end{figure*}
 
 \section{Method}
 \label{sec:method}
 
-\subsection{Causal wearer-relative projection}
+\subsection{Model Architecture}
+\label{sec:model_arch}
 
-At decision time $t$, the model observes only the stream
+At decision time $t$, the model observes only the audiovisual stream
-$\mathcal X_{\leq t}$ and predicts $K=10$ future bins at $\Delta=0.2$~s. For
+$\mathcal X_{\leq t}$ and predicts $K=10$ future bins spaced by
-bin $k$, the multilabel target
+$\Delta=0.2$~s. For horizon $k$, the multilabel target
 $\vect y_{t,k}=[y^{\mathrm{bg}},y^{\mathrm{w}},y^{\mathrm{o}}]$ denotes
-background, camera-wearer activity, and activity by one or more other people.
+background, camera-wearer activity, and activity by one or more other people;
-Wearer--other overlap is allowed. The model outputs
+wearer--other overlap is allowed. The output is
 \begin{equation}
  \hat{\vect Y}_t=f_\theta(\mathcal X_{\leq t})
  \in[0,1]^{K\times3}.
  \label{eq:task}
 \end{equation}
-The wearer-activity forecast also provides causal behavioral scores at every
-horizon. At decision time $t$, if the wearer is silent,
-$\hat y^{\mathrm w}_{t,k}$ acts as a take-turn score for future bin $k$; if
-the wearer is active, $1-\hat y^{\mathrm w}_{t,k}$ acts as a yield score.
-These scores are trained only from future activity targets and are not event
-hazards or committed controller actions.
-Adjudicated wearer-entry and INT-yield events used for behavioral evaluation are
-defined separately from participant-resolved annotations in
-Sec.~\ref{sec:experiments}.
 
-\subsection{Global Omni baseline}
+Our architecture complements globally integrated Omni context with explicitly
-
+accessible local floor evidence (Fig.~\ref{fig:method}). Let
-Let $\vect g_t\in\mathbb R^{d_g}$ be the causal global readout of
+$\vect g_t\in\mathbb R^{d_g}$ denote the causal Qwen2.5-Omni Thinker readout
-Qwen2.5-Omni at the physical cutoff $t$ \cite{xu2025qwenomni}. The frozen
+at $t$ \cite{xu2025qwenomni}. In parallel, the frozen MuVAP ASD frontend
-baseline trains only a dimension-matched prediction head. A matched PEFT/LoRA
+\cite{qi2026muvap} produces one shared mixed-audio CPC state $\vect u_t$ and a
-baseline updates a pre-registered subset of the Omni model under the same
+variable-size set of visible-person states
-examples, context, optimizer budget, head, and evaluator. Its purpose is to
-test whether generic parameter adaptation is sufficient; one negative recipe
-is not interpreted as a general limitation of LoRA or all Omni models.
-
-\todo{Freeze the exact Qwen2.5-Omni-3B revision, processor, input rates,
-streaming/cache semantics, readout layer, context, dtype, LoRA targets/rank,
-trainable parameter budget, and cutoff audit. Add Qwen2.5-Omni-7B only as a
-capacity-scaling control. MiniCPM-o 4.5 is restricted to the diagnostic
-streaming/cache ablation below and does not enter the core method or support a
-cross-family superiority claim.}
-
-\subsection{Causal local floor adapter}
-
-The candidate local path operates on the most recent $L_\ell$ seconds. At local instant
-$\tau\leq t$, a causal audio encoder produces $\vect a_\tau$ from mixed audio.
-For the anonymous visible-person set
-$\mathcal P_\tau=\{(\vect x_{\tau,i},\vect b_{\tau,i})\}_{i=1}^{N_\tau}$,
-a shared visual encoder and masked set operator produce
 \begin{equation}
- \vect v_\tau=\mathrm{SetPool}
+ \mathcal P_t=\{\vect p_{t,i}\}_{i=1}^{N_t},\qquad
- \left(\{E_v(\vect x_{\tau,i},\vect b_{\tau,i})\}_{i=1}^{N_\tau}
+ \vect p_{t,i}=[\vect v_{t,i};s_{t,i}],
- \cup\{\vect v_\varnothing\}\right),
+ \label{eq:person_state}
- \label{eq:set}
 \end{equation}
-where $\vect v_\varnothing$ handles an empty or offscreen visible set. The
+where $\vect v_{t,i}$ is a causal visual state and $s_{t,i}$ is one
-operator is permutation invariant and uses no participant-ID embedding or
+audio--visual speaking logit. The shared audio state is not duplicated in
-cross-time identity. A causal short-window encoder summarizes local dynamics,
+each person token. The prediction model receives neither participant IDs nor
+cross-time track identity; provided causal face boxes are used only to isolate
+the representation and fusion question.
+
+A matched global--audio head first maps $[\vect g_t;\vect u_t]$ to baseline
+logits $\vect Z_t^{\mathrm{base}}\in\mathbb R^{K\times3}$ and an intermediate
+state $\vect h_t$. The same inputs query the current person set:
 \begin{equation}
- \vect \ell_t=F_\ell
+ \begin{aligned}
- \left(\{[\vect a_\tau;\vect v_\tau]\}_{\tau\in(t-L_\ell,t]}\right).
+  \vect q_t &= W_g\vect g_t+W_a\vect u_t,\\
- \label{eq:local}
+  \mathcal M_t &= \{W_p\vect p_{t,i}\}_{i=1}^{N_t}
+                  \cup\{\vect p_\varnothing\},\\
+  \vect c_t &= \mathrm{MHA}(\vect q_t,\mathcal M_t,\mathcal M_t),
+ \end{aligned}
+ \label{eq:person_attention}
 \end{equation}
-The adapter preserves the global initialization through a zero-initialized
+where masking supports variable cardinality and the learned null token
-residual output,
+$\vect p_\varnothing$ makes the attention well defined for an empty set. If
+$m_t$ indicates that at least one valid person observation exists, the person
+branch supplies only a residual logit correction,
 \begin{equation}
- \vect z_t=\vect g_t+W_o\,
+ \vect Z_t=\vect Z_t^{\mathrm{base}}
- \sigma\!\left(W_g\vect g_t+W_\ell\vect\ell_t\right),
+ {}+m_t\,\mathrm{reshape}\!\left(W_r[\vect h_t;\vect c_t;m_t]\right),
- \qquad W_o\vert_{\mathrm{init}}=0,
+ \qquad W_r\vert_{\mathrm{init}}=0,
- \label{eq:fusion}
+ \label{eq:residual}
 \end{equation}
-and the common head maps $\vect z_t$ to Eq.~\eqref{eq:task}. This implements an
+and $\hat{\vect Y}_t=\mathrm{softmax}_c(\vect Z_t)$. Thus, after copying and
-evidence-access hypothesis rather than adding an unconstrained second model:
+freezing the trained $G+U$ head, the complete model initially reproduces that
-the global path retains broad context, while the local path preserves timely
+baseline exactly; only the person projection, query, masked attention, and
-floor evidence.
+residual readout are trained in the adaptation stage.
 
-\todo{Freeze $L_\ell$, local audio/video rates, crop source and resolution,
+We use the EgoSpeak/LSTR multi-positive cross-entropy over causal-valid rows,
-set pooling, causal temporal encoder, masks, reset rules, parameter count, and
-latency. If the local path runs faster than the 5-Hz global/output grid, compare
-it with a same-parameter 5-Hz variant before using the term ``dual-rate.'' Use
-provided causal boxes only as a clearly labeled method-isolation condition;
-online detection/tracking is not a paper contribution unless actually run.}
-
-\subsection{Training}
-
-The primary objective is class-balanced binary cross entropy over valid
-horizons and activity channels,
 \begin{equation}
- \mathcal L_{\mathrm{act}}=
+ \begin{aligned}
- \sum_{t,k,c}m_{t,k}w_c\,
+  \mathcal L&=-\frac{1}{M}\sum_{t,k,c}m_{t,k}y^c_{t,k}
- \mathrm{BCE}(\hat y^c_{t,k},y^c_{t,k}).
+  \log\mathrm{softmax}_c(\vect Z_{t,k}),\\
+  M&=\sum_{t,k}m_{t,k}.
+ \end{aligned}
  \label{eq:loss}
 \end{equation}
-No transcript, target-of-speech label, close microphone, future frame, or
+For an overlap target, both active channels contribute a log-probability term.
-per-person speech label enters the deployable model. \todo{Add an active-speaker
+No transcript, target-of-speech annotation, close-microphone signal, speech
-or event auxiliary loss only if it improves the frozen primary protocol and
+label, or future observation is supplied to the forecasting model. The
-its label/input boundary is fully reported.}
+wearer forecast is interpreted only at evaluation time: it is an entry proxy
+when the wearer is silent and its complement is a yield proxy when the wearer
+is active, not a trained event hazard or committed controller action.
+
+\subsection{Causal Streaming Architecture}
+\label{sec:causal_streaming}
+
+We distinguish encoder processing granularity from the common forecasting
+rate. Predictions are emitted at $t_n=n\Delta$, $n\geq1$, while every
+representation used at $t_n$ is restricted to media available by that physical
+cutoff.
+
+The global path uses an 8-s rolling causal presentation budget with
+source-aligned 2-s audiovisual blocks. Completed blocks are cached and
+replayed, while the currently incomplete block is a temporary tail. At every
+200-ms cutoff, the tail is rebuilt with all newly observed audio and video and
+included in computing $\vect g_{t_n}$; when it reaches 2~s, it is committed to
+the reusable history. Consequently, the Omni state is not held constant for
+2~s: it includes the latest available content at every decision cutoff without
+observing media after $t_n$. We retain the final four Thinker query-layer
+states and fuse them in the matched head with a concatenation MLP.
+
+The local frontend operates causally at 25~Hz. State $j$ consumes the
+half-open interval $[j/25,(j+1)/25)$ and is timestamped at its right edge.
+Mixed audio is encoded once as a continuous shared CPC stream, while each
+available face crop is processed by the causal visual and audio--visual
+speaking modules. Ten-second extraction chunks may prepend up to 2~s of past
+context, never future context. At a 5-Hz forecasting cutoff, we select the
+local state whose right edge equals $t_n$; hence every fifth 25-Hz state aligns
+with one Omni prediction row. The two paths therefore describe the same
+causal prefix: the Omni path preserves integrated conversational context,
+whereas the local path exposes fine-grained audio and visible-person floor
+evidence. Fusion and prediction occur only on the common 5-Hz grid.
 
 
 \section{Experiments}
@@ -301,8 +309,8 @@ matched PEFT/LoRA adaptation, global features with recent local audio, a
 parameter- and history-matched temporal local branch, and the proposed causal
 local audio--visual adapter. Qwen2.5-Omni-7B is used as a capacity-scaling
 control. All learned variants use identical causal examples, prediction
-horizons, prediction heads, and the class-balanced binary cross-entropy
+horizons, prediction heads, and the multi-positive cross-entropy objective in
-objective in Eq.~\eqref{eq:loss}.
+Eq.~\eqref{eq:loss}.
 
 The diagnostic streaming ablation additionally uses MiniCPM-o 4.5
 \cite{cui2026minicpmo45}. Its native cached stream is evaluated at its native