CRRGM:A Contextualized  Real-time RGAT and GraphTransformer Method for multimodal emotion recognition in reinforcement learning

CRRGM:A Contextualized Real-time RGAT and GraphTransformer Method for multimodal emotion recognition in reinforcement learning | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article CRRGM:A Contextualized Real-time RGAT and GraphTransformer Method for multimodal emotion recognition in reinforcement learning Guoshun Chen, Xiaopeng Cao, Shuai Liu This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-4335876/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 4 You are reading this latest preprint version Abstract The task of emotion recognition in dialogues is crucial for constructing empathetic machines. Current research primarily focuses on learning emotion-related common features in multimodal data. However, it does not adequately address various dependency information of emotional features in dialogues. This oversight may lead to lower accuracy in multimodal emotion recognition and inability to recognize emotion in real time. To address this problem, we propose a contextualized approach using enhanced Relational Graph Attention Network and GraphTransformer for multimodal emotion recognition. This model employs Transformer to capture the global information between modalities. It then constructs a heterogeneous graph using the extracted global features and employs enhanced RGAT and GraphTransformer to model the complex dependencies in a conversation. Finally, a reinforcement learning algorithm is used to implement a real-time emotion recognition model. Extensive experiments on two benchmark datasets indicate that CRRGM achieves state-of-the-art performance. Multimodal Emotion Recognition Attention Mechanism Feature Extraction Full Text Supplementary Files snarticle.log snarticle.out Cite Share Download PDF Status: Under Review Version 1 posted Reviewers agreed at journal 17 Aug, 2024 Reviewers invited by journal 17 Aug, 2024 Editor assigned by journal 17 Aug, 2024 First submitted to journal 07 May, 2024 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-4335876","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":341545199,"identity":"d9b88924-8c3f-40bd-99d2-89c6938b9833","order_by":0,"name":"Guoshun Chen","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAz0lEQVRIiWNgGAWjYHACxgdIHGaitDAbMCRA9DYQq4VNgjQt8hHJxyp//jjMYHAj9/gDhgrrxAb2swfwajG8kZZ2myfhMIPkjLzEBoYz6YkNPHkJ+LXMyDG7zQDUwi+RY9jA2HY4sUGCx4CglsIfQC1sYC3/iNAiL5FjxsADt6WBCC0GPM+SpXnS0nkke94lzkg4lm7cxpNDwJb25IMff9hYyxkczz3w4UONtWw/+xkCthyA0DxglABkseFVD7KlAc7kIaR2FIyCUTAKRioAAO2vQMAzsN4+AAAAAElFTkSuQmCC","orcid":"https://orcid.org/0009-0008-7271-7947","institution":"Xi'an University of Posts and Telecommunications","correspondingAuthor":true,"prefix":"","firstName":"Guoshun","middleName":"","lastName":"Chen","suffix":""},{"id":341545200,"identity":"f3aede15-5a06-4c2b-ae66-29390efe4a15","order_by":1,"name":"Xiaopeng Cao","email":"","orcid":"","institution":"Xi'an University of Posts and Telecommunications","correspondingAuthor":false,"prefix":"","firstName":"Xiaopeng","middleName":"","lastName":"Cao","suffix":""},{"id":341545201,"identity":"2063309c-0492-4713-b635-cce3e414e917","order_by":2,"name":"Shuai Liu","email":"","orcid":"","institution":"Xi'an University of Posts and Telecommunications","correspondingAuthor":false,"prefix":"","firstName":"Shuai","middleName":"","lastName":"Liu","suffix":""}],"badges":[],"createdAt":"2024-04-28 02:50:23","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-4335876/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-4335876/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":64475250,"identity":"d798c1d4-e869-47f4-9155-1a64a0f521d4","added_by":"auto","created_at":"2024-09-13 15:19:17","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":686217,"visible":true,"origin":"","legend":"","description":"","filename":"SOCOD2402414.pdf","url":"https://assets-eu.researchsquare.com/files/rs-4335876/v1_covered_46e9a3c0-1ebe-4fa4-bb2e-e19996598843.pdf"},{"id":64474546,"identity":"e3949023-786a-4410-851b-b348b0115501","added_by":"auto","created_at":"2024-09-13 15:11:15","extension":"log","order_by":12,"title":"","display":"","copyAsset":false,"role":"supplement","size":75113,"visible":true,"origin":"","legend":"","description":"","filename":"snarticle.log","url":"https://assets-eu.researchsquare.com/files/rs-4335876/v1/e2b5928e956cf4a6eff76fe9.log"},{"id":64474547,"identity":"cdce2c34-4ba9-4a4d-952d-ac18f50f0000","added_by":"auto","created_at":"2024-09-13 15:11:15","extension":"out","order_by":13,"title":"","display":"","copyAsset":false,"role":"supplement","size":2326,"visible":true,"origin":"","legend":"","description":"","filename":"snarticle.out","url":"https://assets-eu.researchsquare.com/files/rs-4335876/v1/0266dce3093895eaad0e6885.out"}],"financialInterests":"","formattedTitle":"CRRGM:A Contextualized Real-time RGAT and GraphTransformer Method for multimodal emotion recognition in reinforcement learning","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"soft-computing","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"soco","sideBox":"Learn more about [Soft Computing](https://www.springer.com/journal/500)","snPcode":"500","submissionUrl":"https://submission.nature.com/new-submission/500/3","title":"Soft Computing","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Multimodal, Emotion Recognition, Attention Mechanism, Feature Extraction","lastPublishedDoi":"10.21203/rs.3.rs-4335876/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-4335876/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eThe task of emotion recognition in dialogues is crucial for constructing empathetic machines. Current research primarily focuses on learning emotion-related common features in multimodal data. However, it does not adequately address various dependency information of emotional features in dialogues. This oversight may lead to lower accuracy in multimodal emotion recognition and inability to recognize emotion in real time. To address this problem, we propose a contextualized approach using enhanced Relational Graph Attention Network and GraphTransformer for multimodal emotion recognition. This model employs Transformer to capture the global information between modalities. It then constructs a heterogeneous graph using the extracted global features and employs enhanced RGAT and GraphTransformer to model the complex dependencies in a conversation. Finally, a reinforcement learning algorithm is used to implement a real-time emotion recognition model. Extensive experiments on two benchmark datasets indicate that CRRGM achieves state-of-the-art performance.\u003c/p\u003e","manuscriptTitle":"CRRGM:A Contextualized Real-time RGAT and GraphTransformer Method for multimodal emotion recognition in reinforcement learning","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-09-13 15:11:10","doi":"10.21203/rs.3.rs-4335876/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"reviewerAgreed","content":"","date":"2024-08-17T19:01:19+00:00","index":0,"fulltext":""},{"type":"reviewersInvited","content":"","date":"2024-08-17T18:44:12+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2024-08-17T16:03:16+00:00","index":"","fulltext":""},{"type":"submitted","content":"Soft Computing","date":"2024-05-08T00:38:34+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"soft-computing","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"soco","sideBox":"Learn more about [Soft Computing](https://www.springer.com/journal/500)","snPcode":"500","submissionUrl":"https://submission.nature.com/new-submission/500/3","title":"Soft Computing","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"139c25fa-3853-4c08-89c6-371ac66ef7c0","owner":[],"postedDate":"September 13th, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[],"tags":[],"updatedAt":"2024-09-13T15:11:10+00:00","versionOfRecord":[],"versionCreatedAt":"2024-09-13 15:11:10","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-4335876","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-4335876","identity":"rs-4335876","version":["v1"]},"buildId":"qtupq5eGEP_6zYnWcrvyt","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}