EgoFusion: Unified Semantic and Scale-Aware Prompt Fusion for Egocentric Action Recognition

EgoFusion: Unified Semantic and Scale-Aware Prompt Fusion for Egocentric Action Recognition | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article EgoFusion: Unified Semantic and Scale-Aware Prompt Fusion for Egocentric Action Recognition Hechenrui Fan, Huaihai Lyu, Chaofan Chen This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-8283592/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 10 Mar, 2026 Read the published version in Multimedia Systems → Version 1 posted 10 You are reading this latest preprint version Abstract Egocentric video understanding has gained escalating attention for its unique capacity to capture rich first-person sensory signals and interaction dynamics. As a core research direction within this field, action recognition has become a focal point due to its critical role in decoding behavioral intentions and interaction processes in first-person scenarios.Despite the progress of existing methods, two core challenges remain: (1) they often overlook the inherent semantic dependencies between verbs and nouns, treating them as independent tasks, which results in semantically inconsistent or implausible action predictions; and (2) they struggle to effectively fuse information from objects at different scales, leading to incomplete capture of both fine-grained interaction details and global contextual cues.To address these issues, we propose EgoFusion , a prompt learning framework specifically designed for egocentric action recognition. This framework resolves the aforementioned problems through two key modules: the Component Semantic Interaction module leverages the cross-attention mechanism of verb-noun prompts to enhance their semantic alignment and co-occurrence capabilities; the Hierarchical Feature Aggregator module enriches the semantic expression of hand-object interaction information through multi-scale feature fusion. Experiments on datasets such as Ego4D and Epic-Kitchens demonstrate that EgoFusion significantly improves recognition accuracy and generalization performance in within-dataset, cross-dataset, and base-to-novel settings, validating its effectiveness for the unique challenges of egocentric action recognition. Egocentric action recognition Domain generalization Prompt tuning Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Published Journal Publication published 10 Mar, 2026 Read the published version in Multimedia Systems → Version 1 posted Editorial decision: Revision requested 30 Dec, 2025 Reviews received at journal 30 Dec, 2025 Reviews received at journal 25 Dec, 2025 Reviewers agreed at journal 12 Dec, 2025 Reviewers agreed at journal 10 Dec, 2025 Reviewers agreed at journal 09 Dec, 2025 Reviewers invited by journal 09 Dec, 2025 Editor assigned by journal 09 Dec, 2025 Submission checks completed at journal 08 Dec, 2025 First submitted to journal 04 Dec, 2025 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-8283592","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":559199092,"identity":"e9c9073f-220d-4d71-8d60-40420da56e93","order_by":0,"name":"Hechenrui Fan","email":"","orcid":"","institution":"University of Chinese Academy of Sciences","correspondingAuthor":false,"prefix":"","firstName":"Hechenrui","middleName":"","lastName":"Fan","suffix":""},{"id":559199093,"identity":"ef025656-57be-4172-91e2-e90068d2f2d1","order_by":1,"name":"Huaihai Lyu","email":"","orcid":"","institution":"University of Chinese Academy of Sciences","correspondingAuthor":false,"prefix":"","firstName":"Huaihai","middleName":"","lastName":"Lyu","suffix":""},{"id":559199098,"identity":"9a9a8642-200a-428e-9c3e-ff494fbe7b62","order_by":2,"name":"Chaofan Chen","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA2UlEQVRIiWNgGAWjYDACCRBhcwBIMIMICRkitaQBVbOxJYC4PKRo4TEAMQlr4Z/dfOzBj4Q7idvlez6/ulFjwcPAfvjoBryW3DmWbtiT8CxxZxvvNuucY0CH8aSl3cCnxUAix0ya8cfhxA3HeLcZ57ABtUjwmBHQkv9NmiEBpIXnmXHOP6K05LDBtDA/zm0jQovEjTQzSaBfjHe2pZkx5/ZJ8LAR8gv/jORnEsAQk93OfPjx55xvdXL87IeP4dWCcCEwZsBxxEaUcqgW5g9Eqx4Fo2AUjIIRBQAwlkiocjk7dAAAAABJRU5ErkJggg==","orcid":"","institution":"University of Chinese Academy of Sciences","correspondingAuthor":true,"prefix":"","firstName":"Chaofan","middleName":"","lastName":"Chen","suffix":""}],"badges":[],"createdAt":"2025-12-05 02:53:19","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-8283592/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-8283592/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1007/s00530-026-02282-1","type":"published","date":"2026-03-10T15:57:56+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":98433040,"identity":"0b33a882-acc0-4af4-94ed-bee8d11d3353","added_by":"auto","created_at":"2025-12-17 16:50:12","extension":"json","order_by":0,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":6144,"visible":true,"origin":"","legend":"","description":"","filename":"98888db3e4cb4fe98b30e000151eecb8.json","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/d64fc360bd9a22d011d9c924.json"},{"id":98431979,"identity":"a8e13017-9dd4-42f1-b57d-bd4792bcc164","added_by":"auto","created_at":"2025-12-17 16:48:43","extension":"xml","order_by":1,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":88848,"visible":true,"origin":"","legend":"","description":"","filename":"98888db3e4cb4fe98b30e000151eecb81enriched.xml","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/4b9c25a360fc9b99897f18b8.xml"},{"id":98432559,"identity":"6f7eed3f-308a-4b6b-9454-0116b18bba94","added_by":"auto","created_at":"2025-12-17 16:49:41","extension":"png","order_by":2,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":578050,"visible":true,"origin":"","legend":"","description":"","filename":"all.png","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/f5469cc810e0eb2acb741d51.png"},{"id":98431580,"identity":"d041b34b-606a-49fd-a662-6b66f4f59ee4","added_by":"auto","created_at":"2025-12-17 16:47:59","extension":"png","order_by":3,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":476090,"visible":true,"origin":"","legend":"","description":"","filename":"compare.png","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/83a693e5c66f28e31c9b0232.png"},{"id":98199746,"identity":"db3b5035-069d-4768-aeec-cdfc43435c24","added_by":"auto","created_at":"2025-12-15 07:35:13","extension":"pdf","order_by":4,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":1257181,"visible":true,"origin":"","legend":"","description":"","filename":"egofusion.pdf","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/80c749c476ab23aa06c84bcf.pdf"},{"id":98199735,"identity":"c814e99c-788d-497c-905a-6eb01608a067","added_by":"auto","created_at":"2025-12-15 07:35:13","extension":"eps","order_by":5,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":2890,"visible":true,"origin":"","legend":"","description":"","filename":"empty.eps","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/12823b1a28ebf09b3912ec65.eps"},{"id":98199739,"identity":"9558b914-5218-4804-81ef-d86f2b6c99dc","added_by":"auto","created_at":"2025-12-15 07:35:13","extension":"eps","order_by":6,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":91593,"visible":true,"origin":"","legend":"","description":"","filename":"fig.eps","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/7d1cc65d454a6d24d6286a69.eps"},{"id":98433141,"identity":"ee730bac-bc3b-4c6e-8d86-4c9b5d07c283","added_by":"auto","created_at":"2025-12-17 16:50:20","extension":"bst","order_by":7,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":146013,"visible":true,"origin":"","legend":"","description":"","filename":"snapacite.bst","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/cdf0d7813dbb62707014613c.bst"},{"id":98432014,"identity":"b0a583e1-7358-4922-b7e4-c1e4dee3bcc1","added_by":"auto","created_at":"2025-12-17 16:48:50","extension":"bst","order_by":8,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":29828,"visible":true,"origin":"","legend":"","description":"","filename":"snaps.bst","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/5cd605a01546dc167cf90b6f.bst"},{"id":98199744,"identity":"28e07e33-f2dc-4db3-b15f-845eae5aaecf","added_by":"auto","created_at":"2025-12-15 07:35:13","extension":"pdf","order_by":9,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":421391,"visible":true,"origin":"","legend":"","description":"","filename":"snarticle.pdf","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/760480b4c29b76e97593558b.pdf"},{"id":98199736,"identity":"b3a3ef9b-f9b5-4ec5-af9d-8021bf5e0093","added_by":"auto","created_at":"2025-12-15 07:35:13","extension":"bst","order_by":10,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":35515,"visible":true,"origin":"","legend":"","description":"","filename":"snbasic.bst","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/3e64d3395198337e1a69b9c4.bst"},{"id":98199742,"identity":"3c971f3d-e3d1-4574-a5b3-3da7f295e689","added_by":"auto","created_at":"2025-12-15 07:35:13","extension":"bst","order_by":11,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":33968,"visible":true,"origin":"","legend":"","description":"","filename":"snchicago.bst","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/2b4147cf7fa6661542bfe6a2.bst"},{"id":98199747,"identity":"477c58ac-3e7f-4113-b924-63a02a503d82","added_by":"auto","created_at":"2025-12-15 07:35:13","extension":"cls","order_by":12,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":55857,"visible":true,"origin":"","legend":"","description":"","filename":"snjnl.cls","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/f3f8abcc21d60f763eabf2ad.cls"},{"id":98430868,"identity":"14976695-3fb0-4eb8-a065-863c06f9440f","added_by":"auto","created_at":"2025-12-17 16:46:22","extension":"bst","order_by":13,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":64023,"visible":true,"origin":"","legend":"","description":"","filename":"snmathphysay.bst","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/b2acd5617f4f77500101bea9.bst"},{"id":98199754,"identity":"e9855b6c-bd96-4271-a2ec-b49d84e27566","added_by":"auto","created_at":"2025-12-15 07:35:13","extension":"bst","order_by":14,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":64166,"visible":true,"origin":"","legend":"","description":"","filename":"snmathphysnum.bst","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/ed7899af993499e0b3d55bd6.bst"},{"id":98432331,"identity":"cda35ed2-d0c1-48e0-965d-88fe4fe44b21","added_by":"auto","created_at":"2025-12-17 16:49:24","extension":"bst","order_by":15,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":37333,"visible":true,"origin":"","legend":"","description":"","filename":"snnature.bst","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/6106320bf23466256f0435f7.bst"},{"id":98199750,"identity":"ea839422-532c-462a-8ed5-a7eccac1e36d","added_by":"auto","created_at":"2025-12-15 07:35:13","extension":"bst","order_by":16,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":39951,"visible":true,"origin":"","legend":"","description":"","filename":"snvancouveray.bst","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/a67e463d31384eba795c2cf7.bst"},{"id":98199751,"identity":"9353997e-d636-4f53-abf3-32e00dd52137","added_by":"auto","created_at":"2025-12-15 07:35:13","extension":"bst","order_by":17,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":40758,"visible":true,"origin":"","legend":"","description":"","filename":"snvancouvernum.bst","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/268e512c2dfc68a15f7366f7.bst"},{"id":98199755,"identity":"a8d6c89b-8697-410c-b149-6e8b56f561a0","added_by":"auto","created_at":"2025-12-15 07:35:14","extension":"pdf","order_by":18,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":418495,"visible":true,"origin":"","legend":"","description":"","filename":"usermanual.pdf","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/427f163093b4c30ed5a4d45d.pdf"},{"id":98430526,"identity":"a4e896fa-9bbe-41f1-8abb-36802f150e66","added_by":"auto","created_at":"2025-12-17 16:45:37","extension":"xml","order_by":21,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":92640,"visible":true,"origin":"","legend":"","description":"","filename":"98888db3e4cb4fe98b30e000151eecb81structuring.xml","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/244cba0c62f38612fbe6e7f8.xml"},{"id":98199752,"identity":"e9bbdba7-8be9-4227-a48a-b8a84df3aff0","added_by":"auto","created_at":"2025-12-15 07:35:13","extension":"html","order_by":22,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":103781,"visible":true,"origin":"","legend":"","description":"","filename":"earlyproof.html","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1/5d4e0087369fb4f97e1c2f21.html"},{"id":104739927,"identity":"7f63cb8e-3b39-4756-b7c0-9f1cdbfc8059","added_by":"auto","created_at":"2026-03-16 16:13:35","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":1118224,"visible":true,"origin":"","legend":"","description":"","filename":"egofusion.pdf","url":"https://assets-eu.researchsquare.com/files/rs-8283592/v1_covered_cc32a167-2a12-40f9-9542-2802ddf64fc8.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"EgoFusion: Unified Semantic and Scale-Aware Prompt Fusion for Egocentric Action Recognition","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"multimedia-systems","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"mmsj","sideBox":"Learn more about [Multimedia Systems](http://link.springer.com/journal/530)","snPcode":"530","submissionUrl":"https://submission.nature.com/new-submission/530/3","title":"Multimedia Systems","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Egocentric action recognition, Domain generalization, Prompt tuning","lastPublishedDoi":"10.21203/rs.3.rs-8283592/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-8283592/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eEgocentric video understanding has gained escalating attention for its unique capacity to capture rich first-person sensory signals and interaction dynamics. As a core research direction within this field, action recognition has become a focal point due to its critical role in decoding behavioral intentions and interaction processes in first-person scenarios.Despite the progress of existing methods, two core challenges remain: (1) they often overlook the inherent semantic dependencies between verbs and nouns, treating them as independent tasks, which results in semantically inconsistent or implausible action predictions; and (2) they struggle to effectively fuse information from objects at different scales, leading to incomplete capture of both fine-grained interaction details and global contextual cues.To address these issues, we propose \u003cem\u003e\u003cb\u003eEgoFusion\u003c/b\u003e\u003c/em\u003e, a prompt learning framework specifically designed for egocentric action recognition. This framework resolves the aforementioned problems through two key modules: the \u003cb\u003eComponent Semantic Interaction\u003c/b\u003e module leverages the cross-attention mechanism of verb-noun prompts to enhance their semantic alignment and co-occurrence capabilities; the \u003cb\u003eHierarchical Feature Aggregator\u003c/b\u003e module enriches the semantic expression of hand-object interaction information through multi-scale feature fusion. Experiments on datasets such as Ego4D and Epic-Kitchens demonstrate that EgoFusion significantly improves recognition accuracy and generalization performance in within-dataset, cross-dataset, and base-to-novel settings, validating its effectiveness for the unique challenges of egocentric action recognition.\u003c/p\u003e","manuscriptTitle":"EgoFusion: Unified Semantic and Scale-Aware Prompt Fusion for Egocentric Action Recognition","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-12-15 07:35:08","doi":"10.21203/rs.3.rs-8283592/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2025-12-31T02:09:45+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-12-30T10:48:18+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-12-25T14:19:06+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"162323616967571511122970129379572604950","date":"2025-12-12T06:12:29+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"76890952240102550915995038940469992692","date":"2025-12-10T08:40:00+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"221715168876272809729384003870663755454","date":"2025-12-10T02:45:25+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2025-12-10T02:16:12+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2025-12-09T09:33:18+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2025-12-09T04:02:55+00:00","index":"","fulltext":""},{"type":"submitted","content":"Multimedia Systems","date":"2025-12-05T02:51:17+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"multimedia-systems","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"mmsj","sideBox":"Learn more about [Multimedia Systems](http://link.springer.com/journal/530)","snPcode":"530","submissionUrl":"https://submission.nature.com/new-submission/530/3","title":"Multimedia Systems","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"0107379b-5db2-4fa3-871b-5392d5f83ee8","owner":[],"postedDate":"December 15th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[],"tags":[],"updatedAt":"2026-03-16T16:10:16+00:00","versionOfRecord":{"articleIdentity":"rs-8283592","link":"https://doi.org/10.1007/s00530-026-02282-1","journal":{"identity":"multimedia-systems","isVorOnly":false,"title":"Multimedia Systems"},"publishedOn":"2026-03-10 15:57:56","publishedOnDateReadable":"March 10th, 2026"},"versionCreatedAt":"2025-12-15 07:35:08","video":"","vorDoi":"10.1007/s00530-026-02282-1","vorDoiUrl":"https://doi.org/10.1007/s00530-026-02282-1","workflowStages":[]},"version":"v1","identity":"rs-8283592","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-8283592","identity":"rs-8283592","version":["v1"]},"buildId":"XKTyCvWXoU3ODBz1xrDgd","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}