A Web-Enabled Real-Time Multimodal Emotion Detection System Integrating BERT Text Embeddings and CNN-Based Speech and Facial Models

A Web-Enabled Real-Time Multimodal Emotion Detection System Integrating BERT Text Embeddings and CNN-Based Speech and Facial Models | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article A Web-Enabled Real-Time Multimodal Emotion Detection System Integrating BERT Text Embeddings and CNN-Based Speech and Facial Models Sreeja Kamera, Dhanusha Challa, Hannah Bwalya, D.Ramesh Reddy, and 1 more This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-9449877/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 7 You are reading this latest preprint version Abstract The emotional well-being and mental health of human beings are important but neglected aspects of human health. The small signs of emotional unease are often dismissed and unnoticed. As the use of artificial intelligence grows, the basis for using it to create systems that can monitor and understand an individual's emotional state in real time is also increasing. Therefore, this work proposes a multimodal emotion recognition system that analyzes text, speech, and facial expressions to evaluate a person’s emotional state. A BERT-based Transformer model was implemented for text emotion recognition, for facial emotion recognition, a deep 2D-Convolutional Neural Network (CNN) was used. Speech emotion recognition was classified by using a 1D-CNN model. The results from these independent unimodal models were integrated using a weighted decision-level late fusion strategy. Each model was trained independently, and they each obtained high performance: 98.48% (Facial), 90.14% (Speech), 84.77% (Text BERT) validation accuracy. Emotion detection Multimodal learning Mental health Deep learning Artificial intelligence (AI) BERT Convolutional Neural Network (CNN) Speech emotion recognition Facial expression analysis Text sentiment analysis Mel-frequency cepstral coefficients (MFCCs) Chroma features Mel spectrograms Real-time monitoring. Full Text Additional Declarations No competing interests reported. Supplementary Files SupplementaryFile1.pdf Cite Share Download PDF Status: Under Review Version 1 posted Reviews received at journal 27 Apr, 2026 Reviewers agreed at journal 27 Apr, 2026 Reviewers agreed at journal 24 Apr, 2026 Reviewers invited by journal 21 Apr, 2026 Editor assigned by journal 19 Apr, 2026 Submission checks completed at journal 19 Apr, 2026 First submitted to journal 17 Apr, 2026 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-9449877","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":630007983,"identity":"9b40c974-8e03-4dc0-a571-93bd3b67f4fc","order_by":0,"name":"Sreeja Kamera","email":"","orcid":"","institution":"NIT Warangal","correspondingAuthor":false,"prefix":"","firstName":"Sreeja","middleName":"","lastName":"Kamera","suffix":""},{"id":630007984,"identity":"9d66c0c2-2bd4-435d-bb7b-e8c86dc4b262","order_by":1,"name":"Dhanusha Challa","email":"","orcid":"","institution":"NIT Warangal","correspondingAuthor":false,"prefix":"","firstName":"Dhanusha","middleName":"","lastName":"Challa","suffix":""},{"id":630007987,"identity":"038afde6-6c3b-4bf5-a04b-e7229dfdecb7","order_by":2,"name":"Hannah Bwalya","email":"","orcid":"","institution":"NIT Warangal","correspondingAuthor":false,"prefix":"","firstName":"Hannah","middleName":"","lastName":"Bwalya","suffix":""},{"id":630007989,"identity":"437bc68a-1b0b-4b77-b622-5f4525eede3d","order_by":3,"name":"D.Ramesh Reddy","email":"","orcid":"","institution":"VNR VJIET","correspondingAuthor":false,"prefix":"","firstName":"D.Ramesh","middleName":"","lastName":"Reddy","suffix":""},{"id":630007990,"identity":"bd4a6ad8-0984-4849-badf-4436dad1b4f1","order_by":4,"name":"Prof. Prakash Kodali","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA2ElEQVRIiWNgGAWjYBACCSBmbGBgSOAHUyDATKwWyQbGxgbStBgcgFtDAEjO7n34cUZNXZ7x7eb2xzwMdvIM7LwH8GqRljluLLnh2OFiszsHG5t5GJING5j5EvBqkZNIY5B8wHYgcduNxMbmHAbmBAZmHgNCWph/PvhXl7h5BlhLPWEt0hJpbJIb25gTN0iAtRwmrEVyzjE2y5l9hxNnAB02+4/BccM2Qlokbrcx3+z5VpfYPyP9wccZFdXy/Pxn8GsBRwwCABWz4VePoWUUjIJRMApGARYAAP7lQuvhoc+IAAAAAElFTkSuQmCC","orcid":"","institution":"NIT Warangal","correspondingAuthor":true,"prefix":"","firstName":"Prof.","middleName":"Prakash","lastName":"Kodali","suffix":""}],"badges":[],"createdAt":"2026-04-17 13:41:48","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-9449877/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-9449877/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":108491032,"identity":"c9afd825-d227-4c56-a3ac-c15158de6541","added_by":"auto","created_at":"2026-05-05 09:51:24","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":1667360,"visible":true,"origin":"","legend":"","description":"","filename":"SIVPRealtimeMultimodalEmotionRecognitionSystem.pdf","url":"https://assets-eu.researchsquare.com/files/rs-9449877/v1_covered_a6aee4a7-5db7-4340-bc50-4b9aefd3159c.pdf"},{"id":108220299,"identity":"d4087487-1957-4792-b065-f466e42a2d7a","added_by":"auto","created_at":"2026-04-30 15:19:09","extension":"pdf","order_by":0,"title":"","display":"","copyAsset":false,"role":"supplement","size":2900236,"visible":true,"origin":"","legend":"","description":"","filename":"SupplementaryFile1.pdf","url":"https://assets-eu.researchsquare.com/files/rs-9449877/v1/833e9afe25af172b83f8cbae.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"\u003cp\u003eA Web-Enabled Real-Time Multimodal Emotion Detection System Integrating BERT Text Embeddings and CNN-Based Speech and Facial Models\u003c/p\u003e","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"signal-image-and-video-processing","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"sivp","sideBox":"Learn more about [Signal, Image and Video Processing](http://link.springer.com/journal/11760)","snPcode":"11760","submissionUrl":"https://submission.nature.com/new-submission/11760/3","title":"Signal, Image and Video Processing","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Emotion detection, Multimodal learning, Mental health, Deep learning, Artificial intelligence (AI), BERT, Convolutional Neural Network (CNN), Speech emotion recognition, Facial expression analysis, Text sentiment analysis, Mel-frequency cepstral coefficients (MFCCs), Chroma features, Mel spectrograms, Real-time monitoring.","lastPublishedDoi":"10.21203/rs.3.rs-9449877/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-9449877/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eThe emotional well-being and mental health of human beings are important but neglected aspects of human health. The small signs of emotional unease are often dismissed and unnoticed. As the use of artificial intelligence grows, the basis for using it to create systems that can monitor and understand an individual's emotional state in real time is also increasing. Therefore, this work proposes a multimodal emotion recognition system that analyzes text, speech, and facial expressions to evaluate a person\u0026rsquo;s emotional state. A BERT-based Transformer model was implemented for text emotion recognition, for facial emotion recognition, a deep 2D-Convolutional Neural Network (CNN) was used. Speech emotion recognition was classified by using a 1D-CNN model. The results from these independent unimodal models were integrated using a weighted decision-level late fusion strategy. Each model was trained independently, and they each obtained high performance: 98.48% (Facial), 90.14% (Speech), 84.77% (Text BERT) validation accuracy.\u003c/p\u003e","manuscriptTitle":"A Web-Enabled Real-Time Multimodal Emotion Detection System Integrating BERT Text Embeddings and CNN-Based Speech and Facial Models","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2026-04-30 15:19:05","doi":"10.21203/rs.3.rs-9449877/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"editorInvitedReview","content":"","date":"2026-04-27T05:18:26+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"268413291134875829902007436801521349176","date":"2026-04-27T05:10:02+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"65712604159738751993953040495367313265","date":"2026-04-24T10:06:45+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2026-04-21T22:44:19+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2026-04-20T01:18:59+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2026-04-20T01:18:34+00:00","index":"","fulltext":""},{"type":"submitted","content":"Signal, Image and Video Processing","date":"2026-04-17T13:23:43+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"signal-image-and-video-processing","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"sivp","sideBox":"Learn more about [Signal, Image and Video Processing](http://link.springer.com/journal/11760)","snPcode":"11760","submissionUrl":"https://submission.nature.com/new-submission/11760/3","title":"Signal, Image and Video Processing","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"514f19f2-890f-47cb-8212-d9cafc9cad02","owner":[],"postedDate":"April 30th, 2026","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[],"tags":[],"updatedAt":"2026-04-30T15:19:05+00:00","versionOfRecord":[],"versionCreatedAt":"2026-04-30 15:19:05","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-9449877","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-9449877","identity":"rs-9449877","version":["v1"]},"buildId":"XKTyCvWXoU3ODBz1xrDgd","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}