diff --git a/manuscript/main/main.tex b/manuscript/main/main.tex index b63cdfa..d2cbbf3 100644 --- a/manuscript/main/main.tex +++ b/manuscript/main/main.tex @@ -10,7 +10,7 @@ \newcommand{\vect}[1]{\boldsymbol{#1}} \setlength{\emergencystretch}{2em} -\title{Adapting Omni Models for Egocentric Multiparty Floor-Entry Forecasting} +\title{Adapting Omni Models for Causal Egocentric Multiparty Floor-Entry Forecasting} % TODO: Insert the final author list and affiliations after the ICASSP 2027 % review policy and author order are finalized. @@ -22,24 +22,26 @@ \maketitle \begin{abstract} -An egocentric conversational agent operating among several people must decide -whether to speak or keep listening before generating a response. Following -EgoSpeak, we use causal first-person audio--video as observation and the future -speaking behavior of the human camera wearer as an imitation target for this -agent decision. We ask whether a jointly pretrained Omni model exposes the -fine-grained floor cues required every 200~ms. Our global--local formulation -uses a causal global Omni readout for broad scene and conversational context, -while a causal local branch preserves recent mixed audio and a variable set of -track-organized person observations. Anonymous tracks are an organizational -interface whose incremental value must be established, not an assumption that -the model knows speaker identity. On EasyCom development data, a single-seed -oracle analysis finds that perfectly aligned aggregate activity among visible -non-wearers improves 0.2-s mAP by 6.97 points and ten-horizon mean mAP by 2.91 -points over global Omni plus recent audio. Person-resolved histories do not -outperform an identity-shuffled control. This result motivates timely local -floor evidence, while explicitly withholding a person-continuity claim. The -full study compares frozen, parameter-efficient, and learned local variants -and evaluates false agent entry at matched wearer-onset recall. +In multiparty conversation, a pause need not invite an egocentric participant +to speak: it may instead mark a handover between two other people. We study +causal egocentric multiparty floor-entry forecasting, where the target is the +future speech participation of the first-person camera wearer---the one +participant who is necessarily absent from the visible speaker set. This +wearer-relative target distinguishes our setting from recent multimodal VAP +models that forecast visible floor holders or observe human--human turn +dynamics. Following EgoSpeak, we use causal first-person audio--video and the +human wearer's future speaking behavior as an imitation target, and ask how a +jointly pretrained audio--visual Omni model should be adapted for predictions +every 200~ms. Our global--local formulation combines a causal Omni readout with +a lightweight pathway for recent mixed audio and visible-person observations. +On EasyCom development data, a single-seed oracle analysis finds that perfectly +aligned aggregate activity among visible non-wearers improves 0.2-s mAP by +6.97 points and ten-horizon mean mAP by 2.91 points over global Omni plus recent +audio, whereas person-resolved histories do not outperform an +identity-shuffled control. This motivates timely local floor evidence without +assuming that person continuity is useful. The full study compares frozen, +parameter-efficient, and learned local adaptation and evaluates false wearer +entry at matched wearer-onset recall together with yield behavior. \todo{After the train-side study is locked, replace the final two sentences with the selected learned local variant and insert: its oracle-gap recovery, multi-other false-entry reduction at 80\% onset recall, onset/yield @@ -55,107 +57,94 @@ omni models \section{Introduction} \label{sec:intro} -We consider a conversational agent equipped with an egocentric camera and -microphone and deployed in a scene where several people may speak. Before -response generation and speech synthesis, its causal floor controller must -decide every 200~ms whether to continue listening, begin speaking, or yield. A -pause in dyadic dialogue may invite a response; in multiparty interaction it -may instead mark a handover between two other people. An agent that reacts only -to acoustic availability can therefore interrupt a conversation that it -should continue observing. +Natural turn-taking requires a conversational system to anticipate not only +when speech will stop, but whether an emerging floor opportunity actually +belongs to it. This distinction is particularly important in multiparty +interaction. In a dyadic conversation, a pause after one interlocutor finishes +often provides an opportunity for the other to respond; in a group +conversation, the same pause may simply mark a handover between two other +participants. An egocentric conversational agent that reacts only to silence +or aggregate speech activity may therefore enter the floor while the +surrounding conversation should continue without it. We study this problem as +\emph{egocentric multiparty floor-entry forecasting}: continuously predicting +the near-future speech participation of the first-person participant from +causal audio--visual observations. -Existing datasets do not contain autonomous agents making these decisions at -scale. Following EgoSpeak \cite{kim2025egospeak}, we use human egocentric -recordings to construct supervision: the camera wearer defines the first-person -sensor origin, and the wearer's observed speaking behavior defines the desired -floor-action target. This is an imitation-learning proxy for a future agent -under comparable observations. Throughout the paper, \emph{wearer} names this -dataset role and target channel; it is neither the deployed agent nor a -semantic model input. Response content and closed-loop effects of deploying an -agent remain outside our scope. +Voice Activity Projection (VAP) provides a predictive formulation of +turn-taking by continuously projecting the joint future voice activity of +conversational participants, from which turn shifts, holds, and backchannels +can be inferred \cite{ekstedt2022vap}. EgoSpeak extends predictive speech +initiation to the first-person setting, using causal, untrimmed egocentric +audio--visual observations and the future speaking behavior of the camera +wearer as supervision \cite{kim2025egospeak}. This behavior is a useful +imitation target for the perceptual decision faced by an egocentric agent: +given everything observed so far, is the first-person participant likely to +enter the floor next? EgoSpeak establishes that this prediction can be learned +from real-world egocentric streams, but leaves open how recent jointly +pretrained audio--visual Omni models should be used for such fine-grained +causal forecasting. -EgoSpeak already establishes real-time speech-initiation prediction in causal, -untrimmed egocentric video. It combines separately pretrained ResNet-50 and -wav2vec2 feature extractors with Transformer/LSTR, single-layer GRU, and Mamba -temporal models, forecasting the camera wearer and aggregate other-person -activity over the next two seconds \cite{kim2025egospeak}. Thus our novelty is -not the task definition or the use of first-person data. The unresolved model -question is whether a jointly pretrained audio--visual Omni backbone, -operating on causal raw media, exposes an adequate state for this dense -floor-action controller. To our knowledge, such omni representations have not been -systematically evaluated or adapted for the EgoSpeak task. +In parallel, recent work has expanded predictive turn-taking along multimodal +and multiparty dimensions. MM-VAP incorporates facial expression, head pose, +and gaze into predictive turn-taking for two-party visual interaction +\cite{russell2025mmvap}. A recent social-robot MM-VAP framework adapts +voice-activity-related audio--visual encoders for anticipating human--human +turn dynamics in observer or mediation settings \cite{cano2026mmvap}. MuVAP +further extends VAP to multiparty interaction using a monaural mixed-audio +stream and a single camera, grounding acoustic dynamics in visible face tracks +and predicting current and future visible floor holders \cite{qi2026muvap}. +These formulations are complementary to our egocentric setting. In MuVAP, the +candidate future speakers belong to the set visible to the camera; in +mediation-oriented MM-VAP, the system observes turn dynamics between human +participants. Here, by contrast, the participant whose future floor entry must +be predicted is located \emph{behind the camera} and is therefore absent from +the visible participant set. -Our working hypothesis is a representational decomposition. A global causal -Omni state may retain broad scene, speech, and semantic context while making -recent prosody, activity boundaries, and localized person motion difficult for -a shallow 5-Hz controller to recover. A causal local branch therefore -preserves recent mixed audio and a variable-size set of person observations -before fusing them back into the global state. The local set can be pooled -within a frame or organized into anonymous tracks across time. Whether that -continuity adds useful information beyond aggregate activity is itself a -falsifiable question. Frozen, parameter-efficiently tuned, and locally -augmented variants must consequently be compared within each backbone. +This leaves an underexplored intersection: continuous causal prediction of an +egocentric participant's own future floor participation in a changing +multiparty scene using a general-purpose audio--visual Omni model. The defining +difference is its viewpoint and prediction target: the model must project the +activity of the unseen first-person participant from the causal stream used to +observe everyone else. We investigate how pretrained Omni representations can +be adapted under the EgoSpeak wearer-relative forecasting objective. Current +Qwen2.5-Omni results motivate this study, but do not establish a limitation +shared by Omni models in general. -An initial causal oracle screen sharpens this hypothesis. On held-out EasyCom -Session~12, perfectly timed aggregate activity from currently visible -non-wearers improves a matched global-plus-audio model by 6.97 points at 0.2~s -and 2.91 points in mean mAP. Visibility alone and activity delayed by one -second remain near the audio baseline. However, person-resolved histories are -weaker than aggregate histories and indistinguishable from histories whose -identity continuity is shuffled. Thus the current evidence supports the value -of timely local floor information, not a claim that person identity or track -continuity has already been exploited. +Our working hypothesis separates broad context from timely floor evidence. A +causal global Omni state can represent the scene and conversational history, +while a lightweight local pathway preserves recent mixed audio and a changing +set of visible-person observations. The latter may be aggregated per frame or +organized into anonymous tracks; track continuity becomes a contribution only +if it outperforms aggregate, no-ID, shuffled, and capacity-matched controls. An +analysis-only oracle screen on held-out EasyCom Session~12 supports the weaker +hypothesis: perfectly timed aggregate visible-other activity improves a +matched global-plus-audio model by 6.97 points at 0.2~s and 2.91 points in +ten-horizon mean mAP, whereas person-resolved histories do not outperform an +identity-shuffled control. Thus the current evidence motivates timely local +floor information, not speaker identity or addressee understanding. -Voice Activity Projection (VAP) provides the broader future-activity view of -turn taking \cite{ekstedt2022vap}. EgoSpeak's role-relative output preserves -compatibility with that view, but its aggregate ``others'' channel discards -whether the same or different visible people produced recent activity. -Our oracle result shows that this discarded distinction is not automatically -useful under the dense target: identity continuity must beat aggregate and -shuffled controls before it enters the method claim. -Moreover, conventional per-frame average precision can hide the behavioral -failure of entering during an exchange between others. We therefore retain the -dense EgoSpeak target while adding event evaluation of agent entry and yield; -event-oriented work likewise emphasizes recall, false alarms, and latency -rather than a single frame-level score \cite{jiang2026turnbench}. +We retain EgoSpeak-compatible dense future-activity prediction for +comparability and add a behavior-aligned event analysis. In particular, +per-frame average precision can hide the failure that motivates this work: +entering during an exchange among other participants. We therefore measure +false wearer entries during multi-other opportunities at matched wearer-onset +recall, together with response latency and wearer-yield behavior. These events +are observable timing-derived interaction proxies, not semantic EOT, INT, or +addressee labels; the final protocol uses competing-risk censoring so an event +cannot claim a later action across an incompatible floor change. This follows +the broader event-oriented emphasis on recall, false alarms, and latency +\cite{jiang2026turnbench}. -Related work supplies complementary parts of the problem. Egocentric -talking-to-me detection predicts whether a completed utterance addresses the -camera wearer and benefits from long-term social context -\cite{kong2024sicnet}. Multiparty VAP grounds future activity in visible face -tracks to predict floor shifts and next visible speakers \cite{qi2026muvap}, -while AV-CONV reconstructs current person-level speaking/listening relations -from egocentric audio--video \cite{jia2024avconv}. The former talking-to-me -task is generally utterance-level rather than continuous anticipation; -MuVAP predicts visible participants from a static third-person view; and -AV-CONV estimates a current interaction graph rather than strictly causal -future wearer action. Our gap is therefore narrower than ``audio--visual -multiparty turn taking'': we study whether omni representations support -direct, continuous agent floor-action prediction and whether causal local -evidence helps when the visible participant set changes. - -We call this task \emph{egocentric floor-action projection}. The study asks -four questions: (RQ1) how accurately a frozen global Omni state predicts the -wearer-derived agent target and other-person activity; (RQ2) whether timely -causal local audio and visible-person evidence complement that state; (RQ3) -whether aggregate person-set evidence is sufficient or anonymous track -continuity adds measurable value; and (RQ4) whether any dense gain reduces -false agent entries without making the controller globally passive. -Parameter-efficient tuning is a matched adaptation control rather than an -assumed remedy. EasyCom is suitable because -it provides synchronized egocentric audio--video, per-person tracks, and voice -activity \cite{donley2021easycom}. Target-of-speech annotations, when used, are -analysis strata only and are never model input or required output. - -The intended contributions are therefore: a controlled study of global Omni -representations for EgoSpeak-style floor action; an oracle information audit -that separates timely aggregate floor evidence from person continuity; a -global--local adaptation selected from progressively structured candidates; -and a behavior-aligned event evaluation that maps wearer demonstrations to -agent entry and yield decisions. Current Qwen2.5-Omni observations are -motivation, not a claim that all Omni models intrinsically fail. Likewise, -track-consistent cross-attention becomes the final method only if it beats -aggregate, shuffled/no-ID, and capacity-matched controls. +Our contributions are: (1) a controlled study of frozen and +parameter-efficient Omni adaptation for EgoSpeak-style causal forecasting; +(2) an oracle information audit that distinguishes timely aggregate local +floor evidence from person continuity; (3) a global--local adaptation selected +through causal, alignment, and capacity-matched controls; and (4) an event +protocol that tests whether dense gains reduce false first-person floor entry +without suppressing valid entry or yield. EasyCom supplies synchronized +egocentric audio--video, per-person tracks, and voice activity +\cite{donley2021easycom}; target-of-speech annotations, when used, serve only +as analysis strata and never as model input or required output. \begin{figure}[t] \centering diff --git a/manuscript/main/references.bib b/manuscript/main/references.bib index c08b3c5..820b6a0 100644 --- a/manuscript/main/references.bib +++ b/manuscript/main/references.bib @@ -1,16 +1,45 @@ @inproceedings{ekstedt2022vap, author = {Erik Ekstedt and Gabriel Skantze}, - title = {Voice Activity Projection: Self-Supervised Learning of Turn-Taking Events}, - booktitle = {Proc. Interspeech}, - year = {2022} + title = {{Voice Activity Projection: Self-Supervised Learning of Turn-Taking Events}}, + booktitle = {Proc. Interspeech 2022}, + pages = {5190--5194}, + year = {2022}, + doi = {10.21437/Interspeech.2022-10955} } @inproceedings{kim2025egospeak, author = {Junhyeok Kim and Min Soo Kim and Jiwan Chung and Jungbin Cho and Jisoo Kim and Sungwoong Kim and Gyeongbo Sim and Youngjae Yu}, title = {{EgoSpeak}: Learning When to Speak for Egocentric Conversational Agents in the Wild}, - booktitle = {Findings of the Association for Computational Linguistics: NAACL}, + booktitle = {Findings of the Association for Computational Linguistics: NAACL 2025}, + publisher = {Association for Computational Linguistics}, + address = {Albuquerque, New Mexico}, pages = {2990--3005}, - year = {2025} + year = {2025}, + doi = {10.18653/v1/2025.findings-naacl.163}, + url = {https://aclanthology.org/2025.findings-naacl.163/} +} + +@inproceedings{russell2025mmvap, + author = {Sam O'Connor Russell and Naomi Harte}, + title = {Visual Cues Enhance Predictive Turn-Taking for Two-Party Human Interaction}, + booktitle = {Findings of the Association for Computational Linguistics: ACL 2025}, + publisher = {Association for Computational Linguistics}, + address = {Vienna, Austria}, + pages = {209--221}, + year = {2025}, + doi = {10.18653/v1/2025.findings-acl.12}, + url = {https://aclanthology.org/2025.findings-acl.12/} +} + +@misc{cano2026mmvap, + author = {Antonio Cano and Guillermo P{\'e}rez and Luis Merino and Randy Gomez}, + title = {Multimodal Voice Activity Projection for Turn-Taking in Social Robots with Voice-Activity-Related Pretrained Encoders}, + year = {2026}, + eprint = {2607.07294}, + archivePrefix = {arXiv}, + primaryClass = {cs.RO}, + note = {Accepted at IEEE RO-MAN 2026}, + url = {https://arxiv.org/abs/2607.07294} } @article{jiang2026turnbench, @@ -27,11 +56,13 @@ year = {2024} } -@article{qi2026muvap, - author = {Haotian Qi and Gabriel Skantze}, - title = {{MuVAP}: Multimodal Multiparty Voice Activity Projection for Turn-Taking Prediction in the Wild}, - journal = {arXiv preprint arXiv:2606.16731}, - year = {2026} +@inproceedings{qi2026muvap, + author = {Haotian Qi and Gabriel Skantze}, + title = {{MuVAP}: Multimodal Multiparty Voice Activity Projection for Turn-Taking Prediction in the Wild}, + booktitle = {Proc. Interspeech 2026}, + year = {2026}, + note = {arXiv:2606.16731}, + url = {https://arxiv.org/abs/2606.16731} } @article{donley2021easycom,