icassp2027-experiment/manuscript/main/main.tex

% ICASSP 2027 working draft: results and mechanism claims are not yet frozen.
% Four technical pages maximum; the optional fifth page contains references only.
% Use the final author kit and resolve every TODO before submission.
\documentclass{article}
\usepackage{spconf}
\usepackage{amsmath,amssymb,graphicx,hyperref,xcolor}
\hypersetup{hidelinks}
\newcommand{\todo}[1]{\textcolor{red}{\textbf{[TODO: #1]}}}
\newcommand{\vect}[1]{\boldsymbol{#1}}
\setlength{\emergencystretch}{2em}
\title{Causal Local Evidence Adaptation of Omni Models for
Egocentric Speech Anticipation}
\name{Internal Manuscript Draft}
\address{}
\begin{document}
\ninept
\maketitle

\begin{abstract}
Egocentric conversational agents need timely predictions of an unseen
camera wearer's speech participation. We study causal local evidence
adaptation of a frozen Omni model, combining its integrated audiovisual
context with explicitly accessible recent audio and optional local visual
observations. The model predicts future activity every 200~ms at ten horizons
from 0.2 to 2.0~s. A bounded 8-s streaming context reuses completed audiovisual
blocks while recomputing the current partial tail at each cutoff; this exposes
newly observed evidence without waiting for the next complete block.
We evaluate dense activity forecasting and wearer onset/offset anticipation
separately, because improvements in continuous activity scores need not imply
equivalent improvements near state changes.
\todo{Insert matched multi-seed results and ablation-supported conclusions.}
\end{abstract}

\begin{keywords}
egocentric speech anticipation, multimodal adaptation, conversational agents,
causal streaming, voice activity projection
\end{keywords}

\section{Introduction}
\label{sec:intro}

A conversational agent must predict not only what to say, but when to
participate. In egocentric multiparty conversation, the camera wearer is
usually invisible, and the visible participants provide indirect context for
the wearer's next action. We use observed wearer speech as an imitation target,
forecasting participation rather than generating a response or committing
a turn-taking action.

Voice Activity Projection (VAP) learns continuous future speech activity
\cite{ekstedt2022vap}. EgoSpeak extends activity forecasting to untrimmed
first-person audiovisual streams and studies modalities and context length
\cite{kim2025egospeak}. Visual predictive turn-taking
\cite{russell2025mmvap} and MuVAP's causal face-grounded multiparty projections
\cite{qi2026muvap} further motivate explicit local evidence. Our output,
however, is wearer-relative: visible people provide context, not candidate
speaker identities. We do not infer addressees or semantic floor ownership
from aggregate activity labels.

General-purpose Omni representations integrate audio, video, and broader
context \cite{xu2025qwenomni}, but their usefulness for dense anticipation
depends on how the prediction head accesses recent observations.
Two interfaces matter. First, a short-window modality encoder can expose
recent evidence separately from the integrated Thinker state. Second, a
blockwise stream must include its currently incomplete block if predictions
are required before the next block closes. These are representation-access
and alignment questions, not evidence that all Omni models lack turn-taking
ability.

We organize both interfaces under \emph{causal local evidence adaptation}.
A frozen Omni path summarizes conversational context, while a recent-audio
path supplies explicitly accessible local evidence. Visual features are an
optional extension whose benefit must be measured rather than assumed.
The stream incorporates the observed partial tail at every decision time.
We retain EgoSpeak-compatible outputs and distinguish dense forecasting
from onset/offset anticipation.

The paper is organized around three contributions:
\begin{itemize}
  \item A global--local adaptation framework for invisible-wearer speech
  anticipation, combining frozen Omni context with recent modality evidence
  through a lightweight forecasting head.
  \item A cutoff-aligned streaming realization with bounded context and
  temporary partial-tail recomputation, separating completed-history reuse
  from access to the latest observed input.
  \item A controlled evaluation of dense forecasting and wearer onset/offset
  anticipation, isolating local-evidence and streaming effects rather than
  equating dense gains with improved semantic turn-taking.
\end{itemize}
% Contributions describe implementation and study design. Mechanism claims
% require matched ablations; ordinary caching is not claimed as a new algorithm.

\begin{figure*}[t]
\centering
\includegraphics[width=0.96\textwidth]{figures/method_architecture.pdf}
\caption{Causal local evidence adaptation. Top: integrated Omni context and
recent audio feed a common ten-horizon head; local vision is optional.
Bottom: completed 2-s blocks are reused within an 8-s context budget, while
the observed partial tail is recomputed at each 200-ms cutoff.
No future media or temporary readout tokens enter committed history.}
\label{fig:method}
\end{figure*}

\section{Method}
\label{sec:method}

\subsection{Model Architecture}
\label{sec:model_arch}

\textbf{Task definition.}
At cutoff $t_n=0.2n$~s, $n\geq1$, the model observes only media available by
$t_n$ and predicts activity at ten horizons, $0.2,0.4,\ldots,2.0$~s.
Each horizon has three channels in fixed order: background, camera wearer,
and aggregate others. Background indicates no speech; wearer and others may
both be active during overlap. The target is future activity, not a semantic
ENTRY/YIELD label. The tensor contains ten three-channel outputs per cutoff.

\textbf{Global and local features.}
The global path uses frozen Qwen2.5-Omni-3B \cite{xu2025qwenomni}.
We retain the final four Thinker readout-layer states, concatenate them,
and apply a layer-normalized projection with GELU to obtain a
2048-dimensional context vector $\vect g_n$. It can contain audio and visual
information; global context is not audio-free.

The local audio vector $\vect a_n$ is extracted independently from the most
recent 1~s of mixed monaural audio, clipped to the available prefix at startup.
The frozen Qwen audio encoder processes this cropped waveform, and valid
output tokens are averaged into a 2048-dimensional vector. Cropping precedes
preprocessing, preventing access to future source samples.
This exposes recent evidence without another Thinker pass; it does not
separate participants or explicitly estimate prosody.

An optional visual vector $\vect v_n$ uses causal person crops sampled at
2~FPS. Spatial tokens are averaged within each crop, visible people receive
equal weight within a frame, and valid frames are averaged over the recent
1~s. The output is another 2048-dimensional vector. Provided boxes isolate
the representation question, not an online tracking system. Pooling does not
preserve identity or establish active-speaker or addressee grounding.

\textbf{Feature fusion and learning.}
Each enabled local vector is layer-normalized, projected to 256 dimensions
with GELU, and multiplied by its validity indicator. The indicator is also
concatenated into the head, distinguishing missing observations from valid
zero-valued evidence. The common forecast is
\begin{equation}
 \hat{\vect Y}_n =
 \mathrm{softmax}_{c}\!
 \left(F_\theta\!\left[
 \vect g_n;\widetilde{\vect a}_n;m_n^{a};
 \widetilde{\vect v}_n;m_n^{v}\right]\right),
 \label{eq:fusion}
\end{equation}
with visual terms omitted in the audio-core variant.
$F_\theta$ is a layer-normalized MLP with GELU, dropout, and a 30-logit output.
The same head family supports global-only, local-only, and fused controls;
parameter counts are reported because input dimensions differ.

The encoders and Thinker are frozen; only the layer projection, local
projections, and head are trained. This is \emph{frozen-backbone adaptation},
not end-to-end Omni fine-tuning. We use EgoSpeak/LSTR multi-positive
cross-entropy over causal-valid source--horizon pairs. Both active channels
contribute during overlap; no semantic-event objective is added.
Transcripts, participant speech labels, target-of-speech metadata, and LLM
annotations are never model inputs.

\subsection{Streaming Inference Architecture}
\label{sec:causal_streaming}

\textbf{Bounded history.}
The Omni path processes source-aligned 2-s audiovisual blocks within a maximum
8-s presentation budget. After warm-up, it retains three completed blocks
plus the current block, whose observed duration ranges from 0.2 to 2.0~s.
The available span therefore varies with phase: 8~s is a maximum, not an
exactly sliding crop. Older blocks are dropped at block transitions.
Retained completed payloads are replayed to rebuild the bounded language-model
cache; completed modality payloads are reused rather than re-encoded.
All caches reset at session or source discontinuities.

\textbf{Current-tail handling.}
At each 200-ms cutoff, the current block is cropped at the physical cutoff
and only observed audio and video are encoded. This incomplete block is the
\emph{tail}. It is appended to a temporary copy of the completed-history cache,
followed by a readout query to obtain the Thinker state.
The temporary tail and query are not committed. When a block completes,
its media payload is committed once; the next cutoff starts a new tail.
The state is thus refreshed at 5~Hz instead of being held until a 2-s block
closes.

Completed history and the tail are represented in one integrated Omni state,
not a separate tail-content or learned block-phase embedding.
% The auxiliary preserved-EgoSpeak residual has a learned phase embedding;
% its on/off ablation is separate from the core stream described here.

\textbf{Alignment and availability.}
Recent audio and optional vision are refreshed on the common 5-Hz grid and
joined by cutoff and target index, never by nearest future observation.
Only video frames available before the cutoff and truncated audio enter
preprocessing. No future completed block is backfilled into earlier rows.
Tail-aware refresh ensures access to observed input, not a measured
real-time throughput or predictive gain. Correctness, refresh effects,
and latency are assessed separately in Section~\ref{sec:analysis}.

\section{Experimental Setup}
\label{sec:experiments}

\subsection{Data and Training Protocol}

We use EasyCom \cite{donley2021easycom} with the EgoSpeak official split:
Sessions~4--12 for training and Sessions~1--3 for testing.
Targets follow the common 5-Hz grid and class order.
Training-side folds determine recipe changes; a session seen by a parent
checkpoint is not held-out validation for its continuation.
The final recipe is fixed before training on all nine training sessions.
Already inspected test runs are exploratory and are not used to choose
learning rates, windows, checkpoints, or ablations.

The primary comparison includes the released EgoSpeak MiniROADA A+V reference,
Qwen audio encoder-only, global Omni $G$, audio-core $G+A$, and optional
$G+A+V$. Preserved-EgoSpeak-plus-Omni residual fusion is an auxiliary
architecture control, not the default method. Zero initialization guarantees
initial baseline recovery, not non-degradation after training.
Omni LoRA is deferred; the core uses frozen-backbone adaptation.
Qwen Omni 7B and MiniCPM-o 4.5 are planned transfer controls, not the core.

\todo{Insert the locked recipe, provenance, trainable counts, and common
valid-pair counts. Report three seeds and per-session results.}

\subsection{Evaluation}

\textbf{Dense forecasting.}
For each horizon, valid frame scores are pooled across evaluated sessions
to compute AP for background, wearer, and others. Their arithmetic mean is
horizon mAP; averaging the ten horizon mAPs yields mean mAP.
Wearer AP is also reported separately. Comparisons use the intersection of
valid source--horizon pairs.

\textbf{Raw onset/offset anticipation.}
Onset evaluation uses source rows where the wearer is silent. A speaking
future endpoint is positive, scored by predicted wearer probability.
Offset evaluation uses source rows where the wearer is active. A silent
future endpoint is positive, scored by one minus predicted wearer probability.
Current ground-truth activity defines evaluation subsets only, not inputs.
We report AP per horizon and its ten-horizon mean for each direction.

These are \emph{endpoint state-change anticipation} metrics, not one-event-
one-vote timestamp detection: multiple source rows can correspond to one
boundary, and reversals before the endpoint are not captured.
Dense and conditional AP have different prevalence and are not directly
comparable in absolute value. Positive counts and denominators are reported.
AP is threshold-free and requires no calibration session.

\textbf{Scope.}
Semantic ENTRY/YIELD opportunity evaluation is deferred during annotation
auditing. Provisional results and annotation workload are retained outside
the submission body. Activity transitions do not establish intent, takeover,
or floor ownership; we make no semantic control, false-trigger, or event
detection latency claim.

\section{Results and Analysis}
\label{sec:analysis}

\subsection{Main Comparison}

Table~\ref{tab:main} retains exploratory references and reserves the locked
comparison. Historical runs have unmatched training/selection protocols;
the three-seed F1 screen establishes stability, not matched superiority.
\todo{Fill the locked rows with three-seed mean and sample standard deviation.}

\begin{table*}[t]
\caption{EasyCom Sessions~1--3 dense future-activity and raw-boundary
screening. All values are percentages. ``Avg.'' averages dense mAP over the ten
horizons; On/Off are mean raw wearer onset/offset AP and are timing diagnostics,
not semantic ENTRY/INT scores. $\dagger$ denotes historical seed-0 runs with an
unmatched training/selection protocol; $\ddagger$ denotes a three-seed screen.
Missing horizon curves and the final matched comparison will be filled after
the matched dense/boundary reruns.}
\label{tab:main}
\centering
\scriptsize
\setlength{\tabcolsep}{2.6pt}
\resizebox{\textwidth}{!}{%
\begin{tabular}{@{}lccccccccccccc@{}}
\hline
& \multicolumn{10}{c}{Dense mAP at prediction horizon (s) $\uparrow$}
& & \multicolumn{2}{c}{Raw boundary AP $\uparrow$}\\
\cline{2-11}\cline{13-14}
Method & .2 & .4 & .6 & .8 & 1.0 & 1.2 & 1.4 & 1.6 & 1.8 & 2.0
& Avg. & On & Off\\
\hline
EgoSpeak MiniROADA A+V
  & 74.20 & 68.35 & 64.31 & 61.13 & 59.77 & 58.81 & 57.50 & 57.09 & 55.95 & 54.82
  & 61.19 & 24.49 & 38.02\\
Omni $G{+}A^{\dagger}$
  & 81.72 & -- & -- & -- & -- & -- & -- & -- & -- & --
  & 65.11 & 25.96 & 45.25\\
Omni $G{+}A{+}$pooled-$V^{\dagger}$
  & 82.03 & -- & -- & -- & -- & -- & -- & -- & -- & --
  & 65.40 & 26.02 & 45.37\\
Current-frame F1$^{\ddagger}$
  & $81.78{\pm}0.16$ & -- & -- & -- & -- & -- & -- & -- & -- & --
  & $64.95{\pm}0.09$ & -- & --\\
Locked local $A$
  & -- & -- & -- & -- & -- & -- & -- & -- & -- & --
  & -- & -- & --\\
Locked Omni $G$
  & -- & -- & -- & -- & -- & -- & -- & -- & -- & --
  & -- & -- & --\\
Locked $G+A$
  & -- & -- & -- & -- & -- & -- & -- & -- & -- & --
  & -- & -- & --\\
Locked $G+A+V$ (optional)
  & -- & -- & -- & -- & -- & -- & -- & -- & -- & --
  & -- & -- & --\\
\hline
\end{tabular}}
\end{table*}

Alongside these models, a planned causal current-activity classifier repeats
its predicted probabilities across horizons as a persistence reference.
It uses no ground-truth current activity as input.
\todo{Report its dense and endpoint APs; distinguish state recognition from
anticipation.}

\subsection{Local Evidence Analysis}

The $A/G/G+A/G+A+V$ comparisons separate recent audio, integrated context,
and optional vision. We will audit parameter counts and train a wider
global-only head as a capacity control. These branches expose different
history ranges, so the comparison is not a pure Thinker-versus-encoder test.
Past-delayed local inputs provide separate fixed-checkpoint sensitivity
tests, not substitutes for retrained modality-removal controls.
\todo{Insert audio and visual marginal deltas in wearer, onset, and offset
AP; retain negative findings even if overall mAP increases.}

\textbf{Backbone transfer.}
We reserve paired $G/G+A$ controls for Qwen2.5-Omni 3B/7B and MiniCPM-o 4.5,
fixing the same recent-audio features. Within-backbone deltas test transfer;
absolute scores also reflect pretraining and interface differences.
Readout, refresh, and actual history must be audited. Native 1-Hz hold needs
a matched Qwen 1-Hz control; MiniCPM 5-Hz tail rows require GPU verification.

\subsection{Streaming and Context Analysis}

Table~\ref{tab:ablation} reserves a tail-refresh $\times$ local-audio
comparison. Completed-only states update at 2-s closures and are causally
held between them; tail-aware states use the observed partial block at every
cutoff. Enabled local audio remains 5~Hz. Common startup rows and valid pairs
are used. This tests whether the two paths are complementary or redundant.

\begin{table}[t]
\caption{Planned controls. M/W: mean mAP/wearer AP; On/Off: mean endpoint
AP (percent). Transfer reports $G+A-G$ deltas (points) after interface audits.
Dashes are unmeasured, not zero.}
\label{tab:ablation}
\centering
\scriptsize
\setlength{\tabcolsep}{3pt}
\begin{tabular}{@{}lcccc@{}}
\hline
Condition & M & W & On & Off\\
\hline
\multicolumn{5}{l}{\emph{Tail refresh $\times$ local audio; 8-s cap}}\\
Completed-only $G$ & -- & -- & -- & --\\
Completed-only $G+A$ & -- & -- & -- & --\\
Tail-aware $G$ & -- & -- & -- & --\\
Tail-aware $G+A$ & -- & -- & -- & --\\
\hline
\multicolumn{5}{l}{\emph{Context cap; tail-aware $G+A$}}\\
4~s & -- & -- & -- & --\\
8~s & -- & -- & -- & --\\
16~s & -- & -- & -- & --\\
\hline
\multicolumn{5}{l}{\emph{Backbone transfer; paired $G+A-G$ deltas}}\\
Qwen2.5-Omni-3B & -- & -- & -- & --\\
Qwen2.5-Omni-7B & -- & -- & -- & --\\
MiniCPM-o 4.5 & -- & -- & -- & --\\
\hline
\end{tabular}
\end{table}

The planned 4/8/16-s context-cap comparison keeps local evidence at 1~s
and reports actual history spans and block phases. Each cap requires its
own extracted states and matched training; a fixed-checkpoint context
intervention is a separate distribution-shift diagnostic.
Cache reuse is checked against the same block decomposition, with
future-media interventions and reset checks.
\todo{Insert tail/audio interaction, window and phase results, and full
pipeline P50/P95 latency, throughput, and peak memory.}

\subsection{Error and Qualitative Analysis}

We will stratify activity errors around other-speaker pauses, other-to-other
transitions, and overlap, reporting positive/negative counts at fixed
horizons. Pure-negative slices receive score distributions, not AP;
any FPR uses training-side calibration. A wearer backchannel can be a raw
onset positive despite being a semantic ENTRY negative. Semantic
interpretations remain deferred pending annotation audit and protocol v2.
\todo{Insert scene-level counts and score/AP differences, plus one shared
failure; do not infer floor ownership from activity errors.}

Figure~\ref{fig:qualitative_floor_events} reserves cases chosen by
pre-declared rules, aligning source cutoffs and corresponding future target
times at the same horizon for EgoSpeak and the selected ablations.

\begin{figure*}[t]
\centering
\fbox{\parbox[c][0.065\textheight][c]{0.965\textwidth}{
\centering
\textbf{Error and qualitative comparison placeholder}\\[0.5em]
Wearer onset \qquad Wearer offset \qquad Unchanged state / shared failure\\
Activity timeline + source cutoff + same-horizon wearer forecasts
}}
\caption{Planned onset, offset, and hard-negative/shared-failure timelines.
Show raw activity and same-horizon forecasts, not an envelope of ten horizons.
This placeholder contains no result; semantic panels await audited labels.}
\label{fig:qualitative_floor_events}
\end{figure*}

\subsection{Limitations}

EasyCom has only three official test sessions. The backbone remains frozen,
and local audio averages tokens rather than explicitly modeling prosody.
Optional pooling does not establish identity or addressee understanding.
Tail recomputation may increase cost despite completed-block reuse.
Visual and tail-mechanism conclusions require the matched controls above.

\section{Conclusion}

We study causal local evidence adaptation for egocentric speech anticipation,
combining integrated Omni context with recent modality evidence and a
cutoff-aligned partial-tail stream. Dense forecasting and onset/offset
anticipation are evaluated separately.
\todo{Insert supported complementarity, onset/offset trade-offs, tail effect,
and limitations without semantic ENTRY/YIELD claims.}

% Submission check: disclose AI-assisted substantive drafting under applicable
% IEEE policy; finalize author details, kit, and all TODOs.
\bibliographystyle{IEEEbib}
\bibliography{references}
\end{document}