A Multimodal Framework for Continuous Music Emotion Recognition

A Multimodal Framework for Continuous Music Emotion Recognition | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Article A Multimodal Framework for Continuous Music Emotion Recognition Wei Shen, Xingang Mou, Lvlong Chen This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-9472655/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 7 You are reading this latest preprint version Abstract Continuous music emotion recognition (MER) remains challenging in practice because performance often drops across genres and because current models still struggle to capture affective features that remain stable across styles and application scenarios. We propose Cockpit-EmoNet, a multimodal MER model that combines an Acoustic Context Encoder with a Semantic Narrative Encoder, and fuses both streams through bidirectional cross-modal attention and adaptive gating for continuous valence-arousal regression. On the unified PMEmo-DEAM benchmark, Cockpit-EmoNet achieves valence PCC/CCC of 0.69/0.67 and arousal PCC/CCC of 0.81/0.79, outperforming unimodal and fusion baselines. In downstream offline validation, the predicted affective trajectories produce smoother lighting transitions, more conservative brightness changes, and preserved expressiveness under constraints. These results indicate that foundation-model-based multimodal fusion improves MER generalization and provides preliminary offline evidence for constrained cockpit-lighting mapping. Physical sciences/Engineering Physical sciences/Mathematics and computing Biological sciences/Neuroscience Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Under Review Version 1 posted Reviews received at journal 12 May, 2026 Reviewers agreed at journal 12 May, 2026 Reviewers agreed at journal 10 May, 2026 Reviewers invited by journal 05 May, 2026 Editor assigned by journal 24 Apr, 2026 Submission checks completed at journal 24 Apr, 2026 First submitted to journal 20 Apr, 2026 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-9472655","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Article","associatedPublications":[],"authors":[{"id":638388541,"identity":"1b518e85-bc67-4f36-8e51-d15e4af08969","order_by":0,"name":"Wei Shen","email":"","orcid":"","institution":"Key Laboratory of Advanced Manufacturing Technology for High Performance Parts, Ministry of Education","correspondingAuthor":false,"prefix":"","firstName":"Wei","middleName":"","lastName":"Shen","suffix":""},{"id":638388542,"identity":"e43eeb80-85bb-47f0-9bdf-8e35076363fb","order_by":1,"name":"Xingang Mou","email":"","orcid":"","institution":"Key Laboratory of Advanced Manufacturing Technology for High Performance Parts, Ministry of Education","correspondingAuthor":false,"prefix":"","firstName":"Xingang","middleName":"","lastName":"Mou","suffix":""},{"id":638388543,"identity":"fb865456-1636-42a8-afb9-7be76b8b851a","order_by":2,"name":"Lvlong Chen","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAxElEQVRIiWNgGAWjYBACfvmHDQcSDGx47NsbiNQi2ZDc+OBBQZqMAc8BIrUYHEhvNnzw4bCNgUQC0bYcbJNIMGDmMZd8vPEGQ41NNEEt/IyNIC1sPJaz04otGI6l5TYQtKWZEaSFh4fhdo6ZBGPDYcJaDI6BtUjwMNw8Q6yWM4zNBgkGBjwGN3iI1CI5g7HxQYJBAo9kD9AvCcT4hV+C/cHBH3/+2/OzH95440ONDWEtKI4kOmqQtJCqYxSMglEwCkYGAAB+zD+bDCfrDwAAAABJRU5ErkJggg==","orcid":"","institution":"Key Laboratory of Advanced Manufacturing Technology for High Performance Parts, Ministry of Education","correspondingAuthor":true,"prefix":"","firstName":"Lvlong","middleName":"","lastName":"Chen","suffix":""}],"badges":[],"createdAt":"2026-04-20 13:08:54","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-9472655/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-9472655/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":109079974,"identity":"fc5228b3-3f30-48ab-a884-4f357c2ca476","added_by":"auto","created_at":"2026-05-12 11:23:07","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":3136806,"visible":true,"origin":"","legend":"","description":"","filename":"manuscriptforsubmissionstoScientificReports.pdf","url":"https://assets-eu.researchsquare.com/files/rs-9472655/v1_covered_713fb673-a290-4c41-b3e1-15208ff8a543.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"A Multimodal Framework for Continuous Music Emotion Recognition","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"scientific-reports","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"scirep","sideBox":"Learn more about [Scientific Reports](http://www.nature.com/srep/)","snPcode":"","submissionUrl":"","title":"Scientific Reports","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Scientific Reports","inReviewEnabled":true,"inReviewRevisionsEnabled":true},"keywords":"","lastPublishedDoi":"10.21203/rs.3.rs-9472655/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-9472655/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Continuous music emotion recognition (MER) remains challenging in practice because performance often drops across genres and because current models still struggle to capture affective features that remain stable across styles and application scenarios. We propose Cockpit-EmoNet, a multimodal MER model that combines an Acoustic Context Encoder with a Semantic Narrative Encoder, and fuses both streams through bidirectional cross-modal attention and adaptive gating for continuous valence-arousal regression. On the unified PMEmo-DEAM benchmark, Cockpit-EmoNet achieves valence PCC/CCC of 0.69/0.67 and arousal PCC/CCC of 0.81/0.79, outperforming unimodal and fusion baselines. In downstream offline validation, the predicted affective trajectories produce smoother lighting transitions, more conservative brightness changes, and preserved expressiveness under constraints. These results indicate that foundation-model-based multimodal fusion improves MER generalization and provides preliminary offline evidence for constrained cockpit-lighting mapping.","manuscriptTitle":"A Multimodal Framework for Continuous Music Emotion Recognition","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2026-05-12 11:14:31","doi":"10.21203/rs.3.rs-9472655/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"editorInvitedReview","content":"","date":"2026-05-12T11:57:11+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"50368635513402299594225093683600049106","date":"2026-05-12T11:54:04+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"142013985965516181777039254375231016764","date":"2026-05-10T10:30:32+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2026-05-05T07:50:35+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2026-04-24T09:09:21+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2026-04-24T09:09:18+00:00","index":"","fulltext":""},{"type":"submitted","content":"Scientific Reports","date":"2026-04-20T13:04:52+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"scientific-reports","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"scirep","sideBox":"Learn more about [Scientific Reports](http://www.nature.com/srep/)","snPcode":"","submissionUrl":"","title":"Scientific Reports","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Scientific Reports","inReviewEnabled":true,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"cdc32df2-0e24-4f4e-9b95-cabbb4b6bd00","owner":[],"postedDate":"May 12th, 2026","published":true,"recentEditorialEvents":[{"type":"editorInvitedReview","content":"","date":"2026-05-12T11:57:11+00:00","index":34,"fulltext":""},{"type":"reviewerAgreed","content":"50368635513402299594225093683600049106","date":"2026-05-12T11:54:04+00:00","index":33,"fulltext":""},{"type":"reviewerAgreed","content":"142013985965516181777039254375231016764","date":"2026-05-10T10:30:32+00:00","index":23,"fulltext":""},{"type":"reviewersInvited","content":"18","date":"2026-05-05T07:50:35+00:00","index":"","fulltext":""}],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[{"id":67944715,"name":"Physical sciences/Engineering"},{"id":67944716,"name":"Physical sciences/Mathematics and computing"},{"id":67944717,"name":"Biological sciences/Neuroscience"}],"tags":[],"updatedAt":"2026-05-12T11:14:31+00:00","versionOfRecord":[],"versionCreatedAt":"2026-05-12 11:14:31","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-9472655","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-9472655","identity":"rs-9472655","version":["v1"]},"buildId":"XKTyCvWXoU3ODBz1xrDgd","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}