wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Restructure local-evidence manuscript and record analysis controls

Browse Source
This commit is contained in:
Xin Wang
2026-09-17 02:14:56 +08:00
parent 5c29457cc6
commit a183fe5944
11 changed files with 1009 additions and 742 deletions
Download Patch File Download Diff File Expand all files Collapse all files
manuscript/main/figures/method_architecture.pdf
BIN
View File
Binary file not shown.
manuscript/main/figures/method_architecture.tex
+71 -80
View File
@@ -1,7 +1,6 @@
\documentclass[tikz,border=2pt]{standalone}
\usepackage{amsmath}
\usetikzlibrary{arrows.meta,calc,positioning}
\definecolor{navy}{HTML}{17324D}
\definecolor{bluefill}{HTML}{EAF2F8}
\definecolor{teal}{HTML}{137C8B}
@@ -9,89 +8,81 @@
\definecolor{orange}{HTML}{D97706}
\definecolor{orangefill}{HTML}{FFF3E0}
\definecolor{grayline}{HTML}{73808C}
\begin{document}
\begin{tikzpicture}[
font=\sffamily\scriptsize,
>=Latex,
flow/.style={-{Latex[length=2.0mm]}, line width=0.65pt, draw=navy},
aux/.style={-{Latex[length=1.8mm]}, line width=0.6pt, draw=teal},
box/.style={draw=navy, rounded corners=1.5pt, line width=0.65pt,
fill=bluefill, minimum height=7mm, align=center, inner sep=2.3pt},
localbox/.style={box, draw=teal, fill=tealfill},
residualbox/.style={box, draw=orange, fill=orangefill},
labelbox/.style={align=center, text=navy},
note/.style={font=\sffamily\tiny, text=grayline, align=center}
font=\sffamily\scriptsize, >=Latex,
flow/.style={-{Latex[length=1.8mm]},line width=0.65pt,draw=navy},
localflow/.style={flow,draw=teal},
box/.style={draw=navy,rounded corners=1.5pt,line width=0.65pt,
fill=bluefill,minimum height=8mm,align=center,inner sep=2.2pt},
localbox/.style={box,draw=teal,fill=tealfill},
note/.style={font=\sffamily\tiny,text=grayline,align=center}
]
\node[labelbox] (av) at (0,2.65) {causal full-scene AV\\prefix $\mathcal X_{\leq t_n}$};
\node[box, right=7mm of av, minimum width=24mm] (qwen) {frozen\\Qwen2.5-Omni};
\node[box, right=8mm of qwen, minimum width=12mm] (g) {global\\$G_{t_n}$};
\node[labelbox] (audio) at (0,1.45) {mixed audio\\$\leq t_n$};
\node[localbox, right=7mm of audio, minimum width=24mm] (cpc) {frozen causal CPC\\25-Hz processing};
\node[localbox, right=8mm of cpc, minimum width=12mm] (u) {shared\\$U_{t_n}$};
\node[labelbox] (faces) at (0,0.15) {visible face crops\\$\leq t_n$};
\node[localbox, right=7mm of faces, minimum width=24mm] (asd) {frozen causal AV/ASD\\25-Hz processing};
\node[localbox, right=8mm of asd, minimum width=20mm] (persons) {anonymous set\\$\mathcal P_{t_n}=\{[V_i;s_i]\}$};
\node[box, minimum width=21mm] (base) at (8.25,2.05) {$G+U$ base head\\$Z^{\rm base}_{t_n}$};
\node[localbox, minimum width=22mm] (attn) at (8.25,0.70) {masked person MHA\\query: $G_{t_n}+U_{t_n}$\\\scriptsize memory: $\mathcal P_{t_n}+$ NULL};
\node[residualbox, minimum width=19mm] (res) at (10.65,0.70) {zero-init residual\\$\Delta Z_{t_n}$};
\node[circle, draw=navy, fill=white, line width=0.7pt,
minimum size=5.5mm, inner sep=0pt] (sum) at (11.55,2.05) {$+$};
\node[box, minimum width=21mm] (out) at (13.15,2.05) {$10\!\times\!3$ future activity\\$0.2$--$2.0$ s};
\draw[flow] (av) -- (qwen);
\draw[flow] (qwen) -- (g);
\draw[aux] (audio) -- (cpc);
\draw[aux] (cpc) -- (u);
\draw[aux] (faces) -- (asd);
\draw[aux] (asd) -- (persons);
\draw[flow] (g.east) -| ($(base.west)+(-0.25,0.12)$) -- (base.west);
\draw[aux] (u.east) -| ($(base.west)+(-0.25,-0.12)$) -- (base.west);
\draw[flow] (g.east) -- ++(0.3,0) |- (attn.north west);
\draw[aux] (u.east) -- ++(0.5,0) |- (attn.west);
\draw[aux] (persons.east) -- (attn.west);
\draw[flow] (base) -- (sum);
\draw[aux] (attn) -- (res);
\draw[flow, draw=orange] (res.north) |- (sum.south);
\draw[flow] (sum) -- (out);
\node[note, below=1.5mm of base] {trained first, then frozen};
\node[note, below=1.2mm of res] {trainable adaptation};
\draw[grayline, line width=0.45pt] (-0.15,-0.62) -- (14.35,-0.62);
\node[anchor=west, font=\sffamily\scriptsize\bfseries, text=navy]
at (-0.05,-0.92) {Causal streaming at cutoff $t_n$};
\node[anchor=east, text=navy] at (2.25,-1.45) {Omni input};
\draw[fill=bluefill, draw=navy, line width=0.55pt] (2.45,-1.67) rectangle (4.35,-1.23);
\draw[fill=bluefill, draw=navy, line width=0.55pt] (4.35,-1.67) rectangle (6.25,-1.23);
\node[text=navy] at (3.40,-1.45) {cached 2-s blocks};
\node[text=navy] at (5.30,-1.45) {$\cdots$};
\draw[fill=orangefill, draw=orange, line width=0.65pt] (6.25,-1.67) rectangle (9.15,-1.23);
\node[text=orange] at (7.70,-1.45) {current partial block};
\draw[dashed, draw=orange, line width=0.8pt] (9.15,-2.90) -- (9.15,-1.02);
\node[anchor=south, text=orange, font=\sffamily\tiny\bfseries] at (9.15,-1.02) {$t_n$};
\node[note, anchor=west] at (9.40,-1.45) {latest observed input included\\up to 8-s rolling budget};
\node[anchor=east, text=teal] at (2.25,-2.15) {local encoder};
\foreach \x in {2.55,2.85,...,8.85} {
\draw[draw=teal, line width=0.55pt] (\x,-2.33) -- (\x,-1.97);
\node[anchor=west,font=\sffamily\scriptsize\bfseries,text=navy]
at (-0.15,3.15) {Model: global context + recent evidence};
\node[align=center,text=navy] (av) at (0.65,2.45)
{full-scene AV\\observed prefix};
\node[box] (omni) at (3.05,2.45) {frozen Qwen2.5-Omni\\8-s maximum context};
\node[box] (global) at (5.65,2.45) {last-four-layer\\projection $G_n$};
\node[align=center,text=teal] (audio) at (0.65,1.28)
{recent 1-s\\mixed audio};
\node[localbox] (ae) at (3.05,1.28) {frozen Qwen audio\\valid-token mean};
\node[localbox] (ap) at (5.65,1.28) {local projection\\$\widetilde A_n$ + validity};
\node[align=center,text=teal] (video) at (0.65,0.08)
{recent 1-s\\person crops};
\node[localbox,dashed] (ve) at (3.05,0.08)
{frozen Qwen visual\\recent crop pooling};
\node[localbox,dashed] (vp) at (5.65,0.08)
{optional projection\\$\widetilde V_n$ + validity};
\node[box,minimum width=20mm] (fusion) at (8.35,1.28)
{feature concatenation\\trainable MLP};
\node[box,minimum width=23mm] (out) at (11.15,1.28)
{$10\times3$ future activity\\0.2--2.0 s};
\draw[flow] (av) -- (omni);
\draw[flow] (omni) -- (global);
\draw[localflow] (audio) -- (ae);
\draw[localflow] (ae) -- (ap);
\draw[localflow,dashed] (video) -- (ve);
\draw[localflow,dashed] (ve) -- (vp);
\draw[flow] (global.east) -- ++(0.35,0) |- (fusion.north);
\draw[localflow] (ap) -- (fusion);
\draw[localflow,dashed] (vp.east) -- ++(0.35,0) |- (fusion.south);
\draw[flow] (fusion) -- (out);
\node[note] at (10.95,0.22)
{background / wearer / aggregate others\\no semantic event or identity output};
\draw[grayline,line width=0.45pt] (-0.15,-0.65) -- (12.5,-0.65);
\node[anchor=west,font=\sffamily\scriptsize\bfseries,text=navy]
at (-0.15,-0.98) {Stream: committed history + temporary current tail};
\node[anchor=east,text=navy] at (2.05,-1.65) {Omni media};
\foreach \a/\b/\lab in {2.3/3.9/2 s,3.9/5.5/2 s,5.5/7.1/2 s} {
\draw[fill=bluefill,draw=navy,line width=0.55pt]
(\a,-1.87) rectangle (\b,-1.43);
\node[text=navy] at ({(\a+\b)/2},-1.65) {\lab};
}
\draw[draw=teal, line width=0.55pt] (2.55,-2.15) -- (9.15,-2.15);
\node[note, anchor=west, text=teal] at (9.40,-2.15) {25 Hz internal states};
\node[anchor=east, text=navy] at (2.25,-2.72) {forecast grid};
\draw[draw=navy, line width=0.65pt] (2.55,-2.72) -- (9.15,-2.72);
\foreach \x/\lab in {3.15/0.2,4.65/0.4,6.15/0.6,7.65/0.8,9.15/$t_n$} {
\draw[draw=navy, line width=0.75pt] (\x,-2.92) -- (\x,-2.52);
\node[below=0.5mm, font=\sffamily\tiny, text=navy] at (\x,-2.92) {\lab};
\draw[-{Latex[length=1.4mm]}, draw=teal, line width=0.55pt] (\x,-2.32) -- (\x,-2.50);
\draw[fill=orangefill,draw=orange,line width=0.65pt]
(7.1,-1.87) rectangle (8.5,-1.43);
\node[text=orange,font=\sffamily\tiny] at (7.8,-1.65) {partial tail};
\draw[fill=white,draw=grayline,dashed,line width=0.55pt]
(8.5,-1.87) rectangle (8.7,-1.43);
\node[note,anchor=west] at (9.0,-1.65)
{three completed blocks + observed tail\\8-s maximum; future portion excluded};
\node[note,text=navy] at (4.7,-2.13) {reuse completed media payloads};
\node[note,text=orange] at (7.8,-2.13) {recompute at each cutoff};
\node[anchor=east,text=teal] at (2.05,-2.62) {local evidence};
\draw[draw=teal,line width=1.8pt] (7.7,-2.62) -- (8.5,-2.62);
\node[note,anchor=west,text=teal] at (9.0,-2.62)
{recent 1-s audio / optional vision\\refreshed on the common 5-Hz grid};
\node[anchor=east,text=navy] at (2.05,-3.18) {decision grid};
\draw[draw=navy,line width=0.6pt] (2.3,-3.18) -- (8.5,-3.18);
\foreach \x in {2.42,2.58,...,8.34} {
\draw[draw=navy,line width=0.5pt] (\x,-3.28) -- (\x,-3.08);
}
\node[note, anchor=west, text=navy] at (9.40,-2.72) {5 Hz fusion and prediction\\every fifth local right edge};
\draw[draw=navy,line width=0.9pt] (8.5,-3.3) -- (8.5,-3.06);
\draw[dashed,draw=orange,line width=0.75pt]
(8.5,-3.35) -- (8.5,-1.15);
\node[anchor=south,text=orange,font=\sffamily\tiny\bfseries]
at (8.5,-1.15) {cutoff $t_n$};
\node[note,anchor=west,text=navy] at (9.0,-3.18)
{one prediction every 200 ms\\same cutoff and target index};
\end{tikzpicture}
\end{document}