icassp2027-experiment/manuscript/main/main.tex

% Draft ICASSP 2027 manuscript using the official ICASSP 2026 template
% currently bundled in this repository. Replace the template when the ICASSP
% 2027 author kit is released.
% Compile from this directory with: make
\documentclass{article}
\usepackage{spconf}
\usepackage{amsmath,graphicx,hyperref}

\title{NOT EVERY TURN IS MINE: ADDRESSEE-GROUNDED FLOOR-ENTRY PROJECTION IN EGOCENTRIC MULTIPARTY CONVERSATION}

% TODO: Insert the final author list and affiliations after the ICASSP 2027
% review policy and author order are finalized.
\name{Internal Manuscript Draft}
\address{}

\begin{document}
\ninept
\maketitle

\begin{abstract}
Egocentric conversational agents must decide not only when a speaking
opportunity arises, but also whether that opportunity belongs to them.
Existing speech-initiation forecasting collapses all non-wearer speakers, so
strong frame-level accuracy can coexist with intrusive entries while two
bystanders converse. We study addressee-grounded floor-entry projection in
causal multiparty audio-video streams. Our formulation retains dense forecasts
of wearer and non-wearer activity at 200-ms resolution over a two-second
horizon, while grounding the wearer-entry decision in person-aligned visual
context. Using EasyCom target-of-speech annotations, we introduce an event-level
protocol that separates addressed-to-wearer responses from third-party
exchanges and measures false entries at matched response recall, together with
decision latency. Wrong-person and time-shifted visual controls test whether
any improvement follows correct social grounding rather than additional model
capacity. Experiments on EasyCom, with Ego4D as an activity-forecasting
generalization set, will evaluate this hypothesis under a strictly causal
protocol.
\end{abstract}

\begin{keywords}
turn-taking, speech initiation, addressee detection, egocentric video,
audio-visual learning
\end{keywords}

\section{Introduction}
\label{sec:intro}

Conversational agents must continuously decide whether to speak or keep
listening. In a dyadic exchange, a pause or an apparent end of turn is
often treated as an invitation to respond. This assumption fails in
multiparty interaction: one participant may finish an utterance addressed to
another participant, and an egocentric agent that reacts only to acoustic
availability will enter a floor that was never offered to it. The resulting
error is socially more disruptive than a small framewise prediction error.
Accordingly, a useful model must distinguish \emph{when the floor may change}
from \emph{whether the change is directed to the camera wearer}.

Voice Activity Projection (VAP) models future joint speech activity and derives
turn-taking events without manually labeling every event
\cite{ekstedt2022vap}. EgoSpeak extends future activity prediction to causal,
untrimmed egocentric video, forecasting wearer and non-wearer activity over the
next two seconds \cite{kim2025egospeak}. Its role-relative output is attractive
for a wearable agent, but aggregating all visible participants into a single
``others'' channel removes the distinction between an utterance addressed to
the wearer and an exchange among bystanders. Consequently, conventional
per-frame average precision can be dominated by ordinary speech activity and
can hide precisely the false-entry behavior that visual social cues should
prevent. Recent event-oriented evaluation likewise emphasizes the joint
trade-off among recall, false positives, and latency rather than a single
frame-level score \cite{jiang2026turnbench}.

Related work provides two complementary pieces of the problem but does not
unify them. Egocentric talking-to-me detection predicts whether a completed
utterance addresses the camera wearer and benefits from long-term social
context \cite{kong2024sicnet}. Multiparty VAP grounds future activity in
visible face tracks to predict floor shifts and next speakers
\cite{qi2026muvap}. The former is generally evaluated as utterance-level
addressee classification rather than continuous anticipation, whereas the
latter selects among visible participants and does not ask whether an
invisible camera wearer should enter. We target the intersection: causal
floor-entry forecasting for an egocentric agent whose decision depends on the
speaker--addressee relation among the people in view.

We formulate this problem as \emph{addressee-conditioned egocentric activity
projection}. A global streaming audio-video representation predicts future
wearer and non-wearer activity, while a person-aligned visual pathway preserves
participant identity and recent social context. The final wearer-entry score
can therefore represent both floor availability and evidence that the current
utterance is directed to the wearer. EasyCom is particularly suitable for this
analysis because it provides synchronized egocentric audio-video, per-person
tracks, voice activity, and target-of-speech annotations
\cite{donley2021easycom}. Target annotations are used for training-side
auxiliary supervision and evaluation strata, never as model input at test
time.

Our contributions are threefold. First, we cast speech initiation as
recipient-aware floor-entry projection, connecting dense egocentric activity
forecasting with multiparty addressee grounding. Second, we propose an
event-level evaluation that contrasts addressed-to-wearer response events with
third-party exchanges and reports false entries at matched response recall,
alongside latency and the original EgoSpeak metrics. Third, we use
wrong-person, temporally shifted, and visibility-conditioned visual controls to
test whether improvements arise from the correct social evidence instead of
extra feature capacity or globally conservative predictions.

% TODO: Add Method, Experimental Setup, Results, and Conclusion after the
% event protocol and person-aligned model are frozen.

\bibliographystyle{IEEEbib}
\bibliography{references}

\end{document}