wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update targeted ICASSP 2027 manuscript and results

Browse Source
This commit is contained in:
Xin Wang
2026-09-24 11:07:48 +08:00
parent 6de4873b45
commit 36b63bc283
11 changed files with 536 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
manuscript/targeted/figures/system_overview.pdf
BIN
View File
Binary file not shown.
manuscript/targeted/figures/system_overview.tex
+56 -77
View File
@@ -1,92 +1,71 @@
\documentclass[tikz,border=2pt]{standalone}
\usepackage{amsmath}
\usetikzlibrary{arrows.meta,calc,positioning}
\usetikzlibrary{arrows.meta,positioning,calc}
\definecolor{navy}{HTML}{19324A}
\definecolor{bluefill}{HTML}{EAF1F7}
\definecolor{teal}{HTML}{0B7A75}
\definecolor{tealfill}{HTML}{E5F4F2}
\definecolor{orange}{HTML}{C96A12}
\definecolor{teal}{HTML}{087F78}
\definecolor{tealfill}{HTML}{E6F4F1}
\definecolor{orange}{HTML}{B85D0F}
\definecolor{orangefill}{HTML}{FFF0DF}
\definecolor{purple}{HTML}{6E4B8B}
\definecolor{purplefill}{HTML}{F1EBF6}
\definecolor{grayline}{HTML}{697782}
\definecolor{grayline}{HTML}{687782}
\begin{document}
\begin{tikzpicture}[
font=\sffamily\scriptsize, >=Latex,
flow/.style={-{Latex[length=1.7mm]},line width=0.65pt,draw=navy},
font=\sffamily\scriptsize,>=Latex,
flow/.style={-{Latex[length=1.6mm]},line width=.65pt,draw=navy},
localflow/.style={flow,draw=teal},
auditflow/.style={flow,draw=purple,dashed},
box/.style={draw=navy,rounded corners=1.5pt,line width=0.65pt,
fill=bluefill,minimum height=8mm,align=center,inner sep=2.2pt},
localbox/.style={box,draw=teal,fill=tealfill},
auditbox/.style={box,draw=purple,fill=purplefill,dashed},
note/.style={font=\sffamily\tiny,text=grayline,align=center}
]
block/.style={draw=navy,fill=bluefill,rounded corners=1.6pt,
line width=.6pt,minimum height=8.5mm,align=center,inner sep=2pt},
local/.style={block,draw=teal,fill=tealfill},
hint/.style={font=\sffamily\tiny,text=grayline,align=center}]
\node[anchor=west,font=\sffamily\scriptsize\bfseries,text=navy]
at (-0.1,3.2) {Prediction path: integrated history plus explicit recent evidence};
\node[align=center,text=navy] (av) at (0.7,2.45)
{observed full-scene\\audio--video prefix};
\node[box] (omni) at (3.25,2.45) {frozen Qwen2.5-Omni\\8-s maximum context};
\node[box] (global) at (5.95,2.45) {last-four-layer\\global state $G_n$};
\node[align=center,text=teal] (audio) at (0.7,1.23)
{most recent 1-s\\mixed audio};
\node[localbox] (ae) at (3.25,1.23) {frozen audio encoder\\valid-token mean};
\node[localbox] (ap) at (5.95,1.23) {local projection\\$\widetilde A_n$ + validity};
\node[align=center,text=teal] (video) at (0.7,0.02)
{recent causal\\person crops};
\node[localbox,dashed] (ve) at (3.25,0.02)
{frozen visual encoder\\crop / person pooling};
\node[localbox,dashed] (vp) at (5.95,0.02)
{optional projection\\$\widetilde V_n$ + validity};
\node[box,minimum width=21mm] (fusion) at (8.65,1.23)
{cutoff-aligned\\fusion head};
\node[box,minimum width=24mm] (out) at (11.55,1.23)
{$10\!\times\!3$ activity forecast\\0.2--2.0 s};
\draw[flow] (av) -- (omni);
\draw[flow] (omni) -- (global);
at (0,3.12) {Deployable prediction path at the same causal cutoff $t_n$};
\node[block,minimum width=25mm] (history) at (1.35,2.26)
{observed AV history\\8-s maximum budget};
\node[block,minimum width=25mm] (omni) at (4.15,2.26)
{frozen Omni Thinker\\last-four-layer state};
\node[local,minimum width=25mm] (audio) at (1.35,1.13)
{last 1-s mixed audio\\cropped at $t_n$};
\node[local,minimum width=25mm] (ae) at (4.15,1.13)
{frozen audio encoder\\valid-token mean};
\node[local,dashed,minimum width=25mm] (video) at (1.35,0)
{optional recent frames\\provided person boxes};
\node[local,dashed,minimum width=25mm] (ve) at (4.15,0)
{frozen visual encoder\\pooled person tokens};
\node[block,minimum width=20mm] (head) at (7.15,1.13)
{normalized features\\+ validity $\to$ MLP};
\node[block,minimum width=26mm] (out) at (10.4,1.13)
{$10\times3$ forecast\\0.2--2.0-s horizons};
\draw[flow] (history) -- (omni);
\draw[localflow] (audio) -- (ae);
\draw[localflow] (ae) -- (ap);
\draw[localflow,dashed] (video) -- (ve);
\draw[localflow,dashed] (ve) -- (vp);
\draw[flow] (global.east) -- ++(0.30,0) |- (fusion.north);
\draw[localflow] (ap) -- (fusion);
\draw[localflow,dashed] (vp.east) -- ++(0.30,0) |- (fusion.south);
\draw[flow] (fusion) -- (out);
\node[note] at (11.45,0.18)
{background / invisible wearer / aggregate others};
\draw[grayline,line width=0.45pt] (-0.1,-0.62) -- (12.9,-0.62);
\draw[flow] (omni.east) -- ++(.38,0) |- (head.north);
\draw[localflow] (ae) -- (head);
\draw[localflow,dashed] (ve.east) -- ++(.38,0) |- (head.south);
\draw[flow] (head) -- (out);
\node[hint,text=navy] at (10.4,.17)
{background / wearer / aggregate others};
\draw[draw=grayline,line width=.45pt] (0,-.62)--(11.85,-.62);
\node[anchor=west,font=\sffamily\scriptsize\bfseries,text=navy]
at (-0.1,-0.94) {Strictly causal stream at cutoff $t_n$};
\node[anchor=east,text=navy] at (2.05,-1.62) {Omni media};
\foreach \a/\b in {2.3/3.85,3.85/5.4,5.4/6.95} {
\draw[fill=bluefill,draw=navy,line width=0.55pt]
(\a,-1.84) rectangle (\b,-1.40);
\node[text=navy,font=\sffamily\tiny] at ({(\a+\b)/2},-1.62) {completed 2 s};
at (0,-.94) {Bounded streaming state: no media after $t_n$};
\node[anchor=east,text=navy] at (1.9,-1.62) {Omni blocks};
\foreach \a/\b in {2.12/3.55,3.55/4.98,4.98/6.41} {
\draw[draw=navy,fill=bluefill,line width=.55pt]
(\a,-1.84) rectangle (\b,-1.39);
\node[font=\sffamily\tiny,text=navy] at ({(\a+\b)/2},-1.615) {completed};
}
\draw[fill=orangefill,draw=orange,line width=0.65pt]
(6.95,-1.84) rectangle (8.45,-1.40);
\node[text=orange,font=\sffamily\tiny] at (7.70,-1.62) {observed tail};
\draw[dashed,draw=orange,line width=0.75pt]
(8.45,-2.95) -- (8.45,-1.10);
\node[anchor=south,text=orange,font=\sffamily\tiny\bfseries]
at (8.45,-1.10) {cutoff $t_n$};
\node[note,anchor=west] at (8.85,-1.62)
{replay completed payloads; recompute only\\the temporary tail; exclude future media};
\node[anchor=east,text=teal] at (2.05,-2.31) {local evidence};
\draw[draw=teal,line width=1.8pt] (7.65,-2.31) -- (8.45,-2.31);
\node[note,anchor=west,text=teal] at (8.85,-2.31)
{one-second causal window, refreshed every 200 ms};
\node[anchor=east,text=navy] at (2.05,-2.86) {decision grid};
\draw[draw=navy,line width=0.6pt] (2.3,-2.86) -- (8.45,-2.86);
\foreach \x in {2.42,2.58,...,8.34} {
\draw[draw=navy,line width=0.5pt] (\x,-2.95) -- (\x,-2.77);
}
\node[auditbox,minimum width=42mm] (oracle) at (11.02,-2.78)
{analysis only: past visible-person activity\\aggregate / per-person / delayed / shuffled};
\draw[auditflow] (oracle.north) -- ++(0,0.32) -| (fusion.south);
\node[note,text=purple] at (11.02,-3.35)
{privileged information audit; never a deployable model input};
\draw[draw=orange,fill=orangefill,line width=.65pt]
(6.41,-1.84) rectangle (7.85,-1.39);
\node[font=\sffamily\tiny,text=orange] at (7.13,-1.615) {observed tail};
\draw[draw=orange,dashed,line width=.7pt] (7.85,-2.43)--(7.85,-1.12);
\node[anchor=south,font=\sffamily\tiny\bfseries,text=orange]
at (7.85,-1.12) {$t_n$};
\node[hint,anchor=west] at (8.16,-1.62)
{reuse completed payloads;\\recompute temporary tail only};
\node[anchor=east,text=teal] at (1.9,-2.33) {local 1 s};
\draw[draw=teal,line width=2pt] (7.14,-2.33)--(7.85,-2.33);
\node[hint,anchor=west,text=teal] at (8.16,-2.33)
{recent evidence refreshed every 200 ms};
\node[hint,anchor=west] at (2.12,-2.74)
{Privileged past-activity labels are evaluated separately as an information audit; they never enter this prediction path.};
\end{tikzpicture}
\end{document}
manuscript/targeted/figures/system_overview_onecolumn.pdf
BIN
View File
Binary file not shown.
manuscript/targeted/figures/system_overview_onecolumn.tex
+50
View File
@@ -0,0 +1,50 @@
\documentclass[tikz,border=2pt]{standalone}
\usetikzlibrary{arrows.meta,positioning}
\definecolor{navy}{HTML}{19324A}
\definecolor{bluefill}{HTML}{EAF1F7}
\definecolor{teal}{HTML}{087F78}
\definecolor{tealfill}{HTML}{E6F4F1}
\definecolor{orange}{HTML}{B85D0F}
\definecolor{orangefill}{HTML}{FFF0DF}
\begin{document}
\begin{tikzpicture}[
font=\sffamily\small,
>=Latex,
box/.style={draw=navy,fill=bluefill,rounded corners=1.5pt,
align=center,minimum height=7mm,inner sep=2pt},
local/.style={box,draw=teal,fill=tealfill},
flow/.style={-{Latex[length=1.6mm]},draw=navy,line width=.6pt}]
\node[box,text width=20mm] (global) at (1.12,3.9)
{Observed AV\\8-s maximum};
\node[box,text width=22mm] (thinker) at (4.13,3.9)
{Frozen Thinker\\readout $G$};
\node[local,text width=20mm] (audio) at (1.12,2.9)
{Recent 1-s\\mixed audio};
\node[local,text width=22mm] (aenc) at (4.13,2.9)
{Frozen encoder\\local $A$};
\node[local,dashed,text width=20mm] (video) at (1.12,1.9)
{Past person\\crops (optional)};
\node[local,dashed,text width=22mm] (venc) at (4.13,1.9)
{Frozen encoder\\pooled $V$};
\node[box,text width=18mm] (head) at (7.05,2.9)
{Validities +\\trained head};
\draw[flow] (global) -- (thinker);
\draw[flow,draw=teal] (audio) -- (aenc);
\draw[flow,draw=teal,dashed] (video) -- (venc);
\draw[flow] (thinker.east) -- ++(.18,0) |- (head.north);
\draw[flow,draw=teal] (aenc) -- (head);
\draw[flow,draw=teal,dashed] (venc.east) -- ++(.18,0) |- (head.south);
\node[box,text width=32mm] (out) at (5.7,.88)
{$10\times3$ future activity\\0.2--2.0-s horizons};
\draw[flow] (head.south) -- (out.north);
\draw[draw=navy,line width=.5pt] (0,.12) -- (8.25,.12);
\node[anchor=west,text=navy] at (0,-.28) {Completed blocks};
\foreach \x in {0,1.04,2.08} {
\draw[draw=navy,fill=bluefill] (\x,-.93) rectangle ++(.96,.35);
}
\draw[draw=orange,fill=orangefill] (3.12,-.93) rectangle ++(.96,.35);
\node[anchor=west,text=orange] at (4.25,-.76) {Observed tail};
\node[anchor=west,text=navy] at (0,-1.33)
{Reuse completed payloads; query a temporary tail at $t_n$.};
\end{tikzpicture}
\end{document}
manuscript/targeted/figures/task_overview.pdf
BIN
View File
Binary file not shown.
manuscript/targeted/figures/task_overview.tex
+39
View File
@@ -0,0 +1,39 @@
\documentclass[tikz,border=2pt]{standalone}
\usepackage{amsmath}
\usetikzlibrary{arrows.meta,calc}
\definecolor{navy}{HTML}{19324A}
\definecolor{teal}{HTML}{087F78}
\definecolor{orange}{HTML}{B85D0F}
\definecolor{light}{HTML}{EAF1F7}
\definecolor{warm}{HTML}{FFF0DF}
\begin{document}
\begin{tikzpicture}[font=\sffamily\scriptsize,>=Latex]
\node[anchor=west,font=\sffamily\scriptsize\bfseries,text=navy] at (0,2.48)
{Predict the invisible wearer's future activity};
\draw[draw=navy,fill=light,rounded corners=2pt] (0,1.25) rectangle (2.55,1.95);
\node[align=center,text=navy] at (1.275,1.60)
{camera + mixed audio\\visible people, unseen wearer};
\draw[-{Latex[length=1.7mm]},draw=navy,line width=.7pt]
(2.65,1.60)--(3.25,1.60);
\draw[draw=navy,line width=.7pt] (3.32,1.28)--(3.32,1.94);
\node[anchor=south,font=\sffamily\tiny\bfseries,text=navy] at (3.32,1.97)
{decision $t_n$};
\draw[draw=orange,fill=warm,rounded corners=2pt] (4.02,1.25) rectangle (7.6,1.95);
\node[align=center,text=orange] at (5.81,1.60)
{10 future states, every 0.2 s\\background / wearer / others};
\draw[draw=navy,line width=.55pt] (.35,.66)--(7.55,.66);
\foreach \x in {.35,1.08,1.81,2.54,3.27,4.00,4.73,5.46,6.19,6.92,7.55}
{\draw[draw=navy,line width=.5pt] (\x,.57)--(\x,.75);}
\draw[draw=teal,line width=2pt] (.35,.94)--(3.27,.94);
\node[anchor=east,font=\sffamily\tiny,text=teal] at (.25,.94) {input};
\draw[draw=orange,line width=2pt] (3.27,.94)--(7.55,.94);
\node[anchor=west,font=\sffamily\tiny,text=orange] at (7.65,.94) {target};
\node[font=\sffamily\tiny,text=navy] at (3.27,.36) {$t_n$};
\node[font=\sffamily\tiny,text=navy] at (4.0,.36) {$+.2$ s};
\node[font=\sffamily\tiny,text=navy] at (7.55,.36) {$+2.0$ s};
\node[anchor=west,font=\sffamily\tiny,text=navy] at (.35,.02)
{ENTRY-like: wearer silent $\to$ active};
\node[anchor=west,font=\sffamily\tiny,text=navy] at (4.0,.02)
{YIELD-like: wearer active $\to$ silent};
\end{tikzpicture}
\end{document}