icassp2027-experiment/manuscript/main/main.tex

% ICASSP 2027 content skeleton. The final paper must use the official 2027
% author kit and contain no visible \todo{} markers. Planned experiments are
% intentionally marked rather than presented as completed work.
\documentclass{article}
\usepackage{spconf}
\usepackage{amsmath,amssymb,graphicx,hyperref,xcolor}

\newcommand{\todo}[1]{\textcolor{red}{\textbf{[TODO: #1]}}}
\newcommand{\vect}[1]{\boldsymbol{#1}}
\setlength{\emergencystretch}{2em}

\title{Is It My Turn? Causal Local Evidence Adaptation of Omni Models for
Egocentric Floor Forecasting}

% TODO: Insert the final author list and affiliations after the ICASSP 2027
% review policy and author order are finalized.
\name{Internal Manuscript Draft}
\address{}

\begin{document}
\ninept
\maketitle

\begin{abstract}
Real-time audiovisual agents must decide not only what to say, but whether an
emerging floor opportunity belongs to them. This is difficult in multiparty
egocentric interaction: the target wearer is behind the camera, while the
visible people provide only indirect evidence about whether the surrounding
floor remains occupied. We study strictly causal forecasting of the wearer's
future speech activity every 200~ms. Although a general-purpose Omni model
provides rich long-context audio--visual representations, its global readout is
not explicitly trained to expose sub-second, wearer-relative floor evidence.
We therefore study a candidate causal local floor adapter that combines the global
Omni state with short-window mixed audio and an anonymous set of visible-person
observations, without assuming persistent speaker identity. On EasyCom,
\todo{report the final three-seed dense and competing-risk-censored event
results: false-entry reduction at matched wearer-onset recall, onset latency,
yield behavior, confidence intervals, and oracle-gap recovery}. Controlled
\todo{time-shift/remove-vision/same-rate or capacity-matched} interventions
show that \todo{state the supported mechanism, without claiming addressee or
speaker identity}. These results demonstrate \todo{the narrow conclusion
actually supported by the final experiments}.
\end{abstract}

\begin{keywords}
conversational agents, turn-taking, egocentric video, multimodal adaptation,
voice activity projection
\end{keywords}

\section{Introduction}
\label{sec:intro}

Natural turn-taking requires a conversational system to anticipate not only
when speech will stop, but whether a floor opportunity actually belongs to it.
In a dyadic exchange, a pause after one interlocutor finishes often invites the
other to respond. In a group conversation, the same pause may instead mark a
handover between two other participants. An egocentric agent that reacts only
to silence or aggregate speech may therefore interrupt a conversation that
should continue without it. We study \emph{egocentric floor forecasting}:
continuously predicting the near-future speech participation of the
first-person wearer from strictly causal audio--visual observations.

Voice Activity Projection (VAP) introduced continuous prediction of future
voice activity for turn-taking \cite{ekstedt2022vap}. EgoSpeak brought this
formulation to causal, untrimmed first-person streams, using future camera-
wearer activity as an imitation target \cite{kim2025egospeak}. Complementary
work adds visual gaze, head pose, and facial behavior to predictive turn-taking
\cite{russell2025mmvap}, while MuVAP combines global acoustic dynamics with
visible face tracks to forecast visible floor holders in multiparty scenes
\cite{qi2026muvap}. Our target has a different role structure: the participant
whose future action is predicted is the one participant absent from the
visible set. The visible people are contextual evidence for the unseen
wearer's decision, not candidate output identities.

This role asymmetry exposes an adaptation problem for general-purpose Omni
models. Omni pretraining favors globally integrated representations useful for
multimodal understanding and generation. Egocentric floor control instead
depends on timely prosodic and visible-person activity at sub-second scale,
including whether other participants continue to occupy the floor. Increasing
model capacity or applying a generic parameter-efficient update does not
explicitly preserve or route this evidence to a dense wearer-relative
prediction head. The open question is therefore not merely whether an Omni
backbone can be applied to EgoSpeak, but what task structure is still required
to turn general audiovisual understanding into reliable floor control.

Our candidate decomposition separates long-context understanding from
short-window floor evidence. A
causal global Omni state represents the broader scene and conversation. In
parallel, a lightweight local path encodes recent mixed audio and a changing,
anonymous set of visible-person observations. A residual adapter makes this
local evidence accessible to the common ten-horizon activity head. The primary
variant deliberately avoids cross-time identity: the current development
oracle supports timely aggregate visible-other activity but not person
continuity. Track-aware structure is considered only if it defeats no-ID,
identity-corrupted, and capacity-matched controls.

The final paper will claim the following three contributions only if the
corresponding TODO gates pass:
\begin{itemize}
  \item We identify and test a temporal--role mismatch between a global Omni
  readout and causal egocentric floor forecasting for an invisible wearer.
  \item We evaluate a wearer-relative causal local floor adapter that combines
  long-context Omni information with short-window audio and anonymous
  visible-person evidence.
  \item We evaluate whether the adapter reduces false wearer entries during
  multi-other exchanges at matched valid-onset recall, using causal alignment
  interventions and competing-risk-censored entry/yield analysis.
\end{itemize}

\begin{figure}[t]
\centering
\fbox{\parbox[c][1.00in][c]{0.94\columnwidth}{\centering
\todo{Draw only the selected method: causal AV to the long-context Omni path;
the short-window mixed-audio and anonymous person-set path; residual fusion;
the common $10\times3$ head; and the wearer-silent/active event interpretation.
Mark physical cutoffs, trainable modules, person masks, and NULL/OFFSCREEN.
Do not draw oracle labels, rejected track variants, or multiple backbones.}}}
\caption{Long-context Omni understanding and short-window local floor evidence
for wearer-relative future-activity prediction.}
\label{fig:method}
\end{figure}

\section{Method}
\label{sec:method}

\subsection{Causal wearer-relative projection}

At decision time $t$, the model observes only the stream
$\mathcal X_{\leq t}$ and predicts $K=10$ future bins at $\Delta=0.2$~s. For
bin $k$, the multilabel target
$\vect y_{t,k}=[y^{\mathrm{bg}},y^{\mathrm{w}},y^{\mathrm{o}}]$ denotes
background, camera-wearer activity, and activity by one or more other people.
Wearer--other overlap is allowed. The model outputs
\begin{equation}
 \hat{\vect Y}_t=f_\theta(\mathcal X_{\leq t})
 \in[0,1]^{K\times3}.
 \label{eq:task}
\end{equation}
The same wearer-activity forecast has two controller interpretations. In a
wearer-silent opportunity, $\hat y^{\mathrm w}_{t,1}$ is an entry score. While
the wearer is active, $1-\hat y^{\mathrm w}_{t,1}$ is a near-term yield score.
The current self-state is used to define these observable event strata, not as
an unreported oracle model input. \todo{If the final adapter explicitly uses
self-state, define its strictly causal and deployment-available source and add
a matched no-state ablation; otherwise keep self-state evaluation-only.}

\subsection{Global Omni baseline}

Let $\vect g_t\in\mathbb R^{d_g}$ be the causal global readout of
Qwen2.5-Omni at the physical cutoff $t$ \cite{xu2025qwenomni}. The frozen
baseline trains only a dimension-matched prediction head. A matched PEFT/LoRA
baseline updates a pre-registered subset of the Omni model under the same
examples, context, optimizer budget, head, and evaluator. Its purpose is to
test whether generic parameter adaptation is sufficient; one negative recipe
is not interpreted as a general limitation of LoRA or all Omni models.

\todo{Freeze the exact Qwen2.5-Omni-3B revision, processor, input rates,
streaming/cache semantics, readout layer, context, dtype, LoRA targets/rank,
trainable parameter budget, and cutoff audit. Add Qwen2.5-Omni-7B only as a
capacity-scaling control. Do not add another backbone family unless it is
needed to support and experimentally earns a cross-family claim.}

\subsection{Causal local floor adapter}

The candidate local path operates on the most recent $L_\ell$ seconds. At local instant
$\tau\leq t$, a causal audio encoder produces $\vect a_\tau$ from mixed audio.
For the anonymous visible-person set
$\mathcal P_\tau=\{(\vect x_{\tau,i},\vect b_{\tau,i})\}_{i=1}^{N_\tau}$,
a shared visual encoder and masked set operator produce
\begin{equation}
 \vect v_\tau=\mathrm{SetPool}
 \left(\{E_v(\vect x_{\tau,i},\vect b_{\tau,i})\}_{i=1}^{N_\tau}
 \cup\{\vect v_\varnothing\}\right),
 \label{eq:set}
\end{equation}
where $\vect v_\varnothing$ handles an empty or offscreen visible set. The
operator is permutation invariant and uses no participant-ID embedding or
cross-time identity. A causal short-window encoder summarizes local dynamics,
\begin{equation}
 \vect \ell_t=F_\ell
 \left(\{[\vect a_\tau;\vect v_\tau]\}_{\tau\in(t-L_\ell,t]}\right).
 \label{eq:local}
\end{equation}
The adapter preserves the global initialization through a zero-initialized
residual output,
\begin{equation}
 \vect z_t=\vect g_t+W_o\,
 \sigma\!\left(W_g\vect g_t+W_\ell\vect\ell_t\right),
 \qquad W_o\vert_{\mathrm{init}}=0,
 \label{eq:fusion}
\end{equation}
and the common head maps $\vect z_t$ to Eq.~\eqref{eq:task}. This implements an
evidence-access hypothesis rather than adding an unconstrained second model:
the global path retains broad context, while the local path preserves timely
floor evidence.

\todo{Freeze $L_\ell$, local audio/video rates, crop source and resolution,
set pooling, causal temporal encoder, masks, reset rules, parameter count, and
latency. If the local path runs faster than the 5-Hz global/output grid, compare
it with a same-parameter 5-Hz variant before using the term ``dual-rate.'' Use
provided causal boxes only as a clearly labeled method-isolation condition;
online detection/tracking is not a paper contribution unless actually run.}

\subsection{Training}

The primary objective is class-balanced binary cross entropy over valid
horizons and activity channels,
\begin{equation}
 \mathcal L_{\mathrm{act}}=
 \sum_{t,k,c}m_{t,k}w_c\,
 \mathrm{BCE}(\hat y^c_{t,k},y^c_{t,k}).
 \label{eq:loss}
\end{equation}
No transcript, target-of-speech label, close microphone, future frame, or
per-person speech label enters the deployable model. \todo{Add an active-speaker
or event auxiliary loss only if it improves the frozen primary protocol and
its label/input boundary is fully reported.}

\section{Experiments} 
\label{sec:experiments} 

\subsection{Dataset and Data Preparation}

We evaluate on EasyCom following the EgoSpeak split, using Sessions 4--12 for training and development and Sessions 1--3 for final evaluation \cite{donley2021easycom}. All hyperparameters, event windows, operating thresholds, and checkpoints are selected without access to the test sessions. The input stream is converted to strictly causal examples at 5~Hz. Each decision point is paired with ten future 0.2-s activity bins for background, camera wearer, and aggregate non-wearer speech. Provided person boxes are used only to construct the visible-person inputs; participant identity, close-mic audio, transcripts, and future observations are not available to the model. \todo{Report dataset duration, number of training/test examples and events, and the frozen development/calibration split.}

\subsection{Model Configuration and Training} 

Our main backbone is Qwen2.5-Omni-3B. We compare a frozen global Omni readout, matched PEFT/LoRA adaptation, global features with recent local audio, a parameter- and history-matched temporal local branch, and the proposed causal local audio--visual adapter. Qwen2.5-Omni-7B is used only as a capacity-scaling control. All learned variants use the same causal context, prediction head, training examples, and class-balanced binary cross-entropy objective in Eq.~\eqref{eq:loss}. Unless otherwise stated, model selection is performed on the development data and final numbers are averaged over three random seeds. \todo{Freeze context length, local window/rates, optimizer, epochs, LoRA configuration, trainable parameter counts, and inference latency.}

\subsection{Evaluation Metrics}

We evaluate both dense future-activity prediction and event-level floor
control. For comparability with EgoSpeak, we report per-frame AP at
$0.2$--$2.0$~s horizons and their mean mAP.

For event-level evaluation, a \emph{wearer entry} is a transition from wearer
silence to speech, while a \emph{wearer yield} is a transition from speech to
silence. An entry prediction is considered correct if it is committed within
the predefined response window of a ground-truth wearer onset. We report entry
recall, false entries per minute of eligible wearer-silent time, and signed
onset latency. Yield events are evaluated analogously using recall and latency.

Multiparty interaction introduces a competing-risk condition. After another
speaker releases the floor, the current wearer-entry opportunity is terminated
if another non-wearer participant starts speaking before the wearer does.
Predictions after this competing onset are therefore not matched to the
previous opportunity. This prevents an other-to-other handover from being
treated as an open floor for the wearer.

Operating thresholds are selected on the development split and frozen for test
evaluation. Our primary behavioral metric is false wearer entries per minute at
a matched wearer-entry recall of $0.8$; dense mAP and wearer-yield performance
are reported as complementary measures.


\section{Results and Analysis}
\label{sec:results}

\subsection{Baselines}
The confirmatory comparison contains: an EgoSpeak reproduction; the frozen
Qwen2.5-Omni-3B global state $G$; matched LoRA/PEFT; $G$ plus recent local
audio; a parameter- and history-matched temporal local branch; and the proposed
audio--visible-person adapter. Qwen2.5-Omni-7B is a capacity-scaling control,
not a second paper storyline. \todo{Freeze all baseline implementations and
ensure the EgoSpeak comparison uses identical targets, split, causal validity,
and evaluation.}


\begin{table}[t]
\caption{Confirmatory EasyCom results. Lower multi-other false entries are
better; all event rows use a training-side selected operating point.}
\label{tab:main}
\centering
\small
\resizebox{\columnwidth}{!}{%
\begin{tabular}{lccccc}
\hline
Method & 0.2-s mAP & Mean mAP & FP/min@$R=.8$ & Onset lat. & Yield R\\
\hline
EgoSpeak reproduction       & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\
Omni-3B frozen $G$          & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\
Omni-3B matched PEFT        & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\
$G+$local audio             & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\
$G+$matched temporal local  & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\
$G+$causal local AV (ours)  & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\
Omni-7B scaling control     & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\
\hline
\end{tabular}}
\end{table}

\textbf{Information opportunity.}
On development Session~12, an analysis-only seed-0 oracle supplied perfectly
timed visible-other speech activity through an explicit non-deployable path.
Aggregate visible activity improved 0.2-s mAP by 6.97 points and ten-horizon
mean mAP by 2.91 points over matched global-plus-local-audio. Visibility-only
and one-second-delayed activity remained near audio, while per-person histories
did not outperform identity-shuffled histories. This motivates timely local
floor evidence, not participant identity. \todo{After seeds 1--2, report the
interval or remove the exact oracle numbers from the four-page paper.}

\textbf{Main result.}
Table~\ref{tab:main} should establish one result: \todo{the selected local
adapter reduces multi-other false entry at matched valid-onset recall, retains
onset latency/yield behavior, improves dense forecasting, and recovers a
meaningful fraction of the oracle opportunity}. The 7B row should determine
whether \todo{capacity alone closes the gap or the adaptation structure remains
useful at larger scale}. No conclusion is drawn across all Omni families from
Qwen-only evidence.

\begin{table}[t]
\caption{Mechanism controls for the selected Omni-3B local adapter.}
\label{tab:ablation}
\centering
\small
\begin{tabular}{lccc}
\hline
Local condition & Mean mAP & FP/min@$R=.8$ & Yield R\\
\hline
Correct causal local AV       & \todo{--} & \todo{--} & \todo{--}\\
Remove local vision           & \todo{--} & \todo{--} & \todo{--}\\
Time-shift local vision       & \todo{--} & \todo{--} & \todo{--}\\
Global/full-frame visual pool & \todo{--} & \todo{--} & \todo{--}\\
Same-rate/capacity control    & \todo{--} & \todo{--} & \todo{--}\\
\hline
\end{tabular}
\end{table}

\textbf{Mechanism and limitations.}
Table~\ref{tab:ablation} should test whether gains require correctly timed local
visual evidence rather than extra parameters, regularization, or generic scene
context. \todo{Report paired intervals and the supported interpretation. If
correct and shifted/removed vision do not separate, drop the audiovisual
evidence-routing claim. If dense mAP improves but event behavior does not, do
not claim improved floor control. If legitimate onset/yield degrades, stop the
adapter rather than describing lower false entry as success.}

\todo{Add one compact, pre-selected qualitative timeline only if space permits:
one multi-other false-entry improvement paired with one preserved real wearer
onset or one failure. Examples illustrate aggregate rates and never establish
the claim by themselves.}

\section{Conclusion}
\label{sec:conclusion}

This study asks whether a general-purpose Omni representation can be adapted
for causal floor forecasting of an invisible egocentric wearer. The candidate
method separates long-context Omni understanding from short-window local floor
evidence without assuming persistent speaker identity. \todo{State the final
learned-method improvement, matched-recall false-entry result, onset/yield
non-degradation, causal-control evidence, and main limitation. If no learned
variant passes these tests, reframe the submission as an information audit or
do not make the method claim.}

\bibliographystyle{IEEEbib}
\bibliography{references}

\end{document}