MMA-Net: A Multimodal Multitask Network Utilizing Dual Attention Mechanisms for Enhanced Modality Fusion and Task Exchange in Cognitive Load Assessment

MMA-Net: A Multimodal Multitask Network Utilizing Dual Attention Mechanisms for Enhanced Modality Fusion and Task Exchange in Cognitive Load Assessment | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article MMA-Net: A Multimodal Multitask Network Utilizing Dual Attention Mechanisms for Enhanced Modality Fusion and Task Exchange in Cognitive Load Assessment Long Nguyen-Phuoc, Renald Gaboriau, Dimitri Delacroix, Laurent Navarro This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-4539841/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 05 Jun, 2025 Read the published version in Signal, Image and Video Processing → Version 1 posted 5 You are reading this latest preprint version Abstract In this paper, we introduce the Multimodal Multitask Dual Attention Network (MMA-Net), a novel neural architecture tailored for simultaneous multimodal and multitask learning. MMA-Net effectively processes audio and visual inputs through specialized encoders—AudioNet and VideoNet—to capture the distinct and complementary information present across different sensory modalities. A key element of our model is the implementation of Bidirectional Cross-Modality Attention, which refines the multimodal features by leveraging interdependencies between the modalities, enhancing the model's ability to handle complex datasets. This is complemented by the MultiTask Exchange Block, a sophisticated mechanism that dynamically integrates and refines task-specific features, promoting effective information exchange and synergy across multiple tasks. Experimental results demonstrate that MMA-Net achieves significant improvements over existing methods and sets a new benchmark in Cognitive Load Assessment. multimodal multitask cognitive load assessment attention mechanism Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Published Journal Publication published 05 Jun, 2025 Read the published version in Signal, Image and Video Processing → Version 1 posted Editorial decision: Revision requested 14 Aug, 2024 Reviewers invited by journal 11 Jul, 2024 Editor assigned by journal 07 Jun, 2024 Submission checks completed at journal 07 Jun, 2024 First submitted to journal 06 Jun, 2024 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-4539841","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":314821985,"identity":"39447d58-e869-4790-8568-bc2189f33577","order_by":0,"name":"Long Nguyen-Phuoc","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABDUlEQVRIie3OMUvEMBTA8RcC7ZKz6zvU049QCPQE/TDNUpcutzmcJVNcPPd+i7rdWAhel3Cud3RQEMTB4VzkRBBLkcNDUh0d8h9CEvIjD8Dl+oftbJ2IbBZftvvARryfhJXtvi//TjD+hfgT/USm56KoJiUdTfUgyB9urtdngEOL8dg8OSKmEoWZxzQ3mmOdJDUzgHulhWAahUTNRLFIh+89pUWxm0Y1UZChbbANuXsOaUv6t6/Ltw/ADsLviRo3v7Avgsxb9GQHYSYCoUqemzSkuTrlaJKoZjO0ksC/5KsXle1fVSakI3U8CC7043I9PrGSdrYYNByWAPTbbQdoHq4AMjiQ28Tlcrlcmz4B4pBauSPgpwsAAAAASUVORK5CYII=","orcid":"","institution":"Mines Saint-Étienne","correspondingAuthor":true,"prefix":"","firstName":"Long","middleName":"","lastName":"Nguyen-Phuoc","suffix":""},{"id":314821986,"identity":"1f7ac48b-7655-4693-ac53-4697e88ee131","order_by":1,"name":"Renald Gaboriau","email":"","orcid":"","institution":"MJ INNOV","correspondingAuthor":false,"prefix":"","firstName":"Renald","middleName":"","lastName":"Gaboriau","suffix":""},{"id":314821987,"identity":"c0e0ccda-b3cb-4763-aa4c-2042d84e5758","order_by":2,"name":"Dimitri Delacroix","email":"","orcid":"","institution":"MJ INNOV","correspondingAuthor":false,"prefix":"","firstName":"Dimitri","middleName":"","lastName":"Delacroix","suffix":""},{"id":314821988,"identity":"d74233aa-d413-4684-9283-99356dee66f0","order_by":3,"name":"Laurent Navarro","email":"","orcid":"","institution":"Mines Saint-Étienne","correspondingAuthor":false,"prefix":"","firstName":"Laurent","middleName":"","lastName":"Navarro","suffix":""}],"badges":[],"createdAt":"2024-06-06 11:00:12","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-4539841/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-4539841/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1007/s11760-025-04096-6","type":"published","date":"2025-06-05T15:56:54+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":84242343,"identity":"de0ba800-c0c6-4de8-9143-7b75b19a5db4","added_by":"auto","created_at":"2025-06-09 16:03:03","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":594694,"visible":true,"origin":"","legend":"","description":"","filename":"MMAshort.pdf","url":"https://assets-eu.researchsquare.com/files/rs-4539841/v1_covered_7ba052a9-de25-4cbe-80c2-c185605cbc75.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"MMA-Net: A Multimodal Multitask Network Utilizing Dual Attention Mechanisms for Enhanced Modality Fusion and Task Exchange in Cognitive Load Assessment","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"signal-image-and-video-processing","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"sivp","sideBox":"Learn more about [Signal, Image and Video Processing](http://link.springer.com/journal/11760)","snPcode":"11760","submissionUrl":"https://submission.nature.com/new-submission/11760/3","title":"Signal, Image and Video Processing","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"multimodal, multitask, cognitive load assessment, attention mechanism","lastPublishedDoi":"10.21203/rs.3.rs-4539841/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-4539841/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"In this paper, we introduce the Multimodal Multitask Dual Attention Network (MMA-Net), a novel neural architecture tailored for simultaneous multimodal and multitask learning. MMA-Net effectively processes audio and visual inputs through specialized encoders—AudioNet and VideoNet—to capture the distinct and complementary information present across different sensory modalities. A key element of our model is the implementation of Bidirectional Cross-Modality Attention, which refines the multimodal features by leveraging interdependencies between the modalities, enhancing the model's ability to handle complex datasets. This is complemented by the MultiTask Exchange Block, a sophisticated mechanism that dynamically integrates and refines task-specific features, promoting effective information exchange and synergy across multiple tasks. Experimental results demonstrate that MMA-Net achieves significant improvements over existing methods and sets a new benchmark in Cognitive Load Assessment.","manuscriptTitle":"MMA-Net: A Multimodal Multitask Network Utilizing Dual Attention Mechanisms for Enhanced Modality Fusion and Task Exchange in Cognitive Load Assessment","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-06-21 05:59:11","doi":"10.21203/rs.3.rs-4539841/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2024-08-14T18:45:59+00:00","index":"","fulltext":""},{"type":"reviewersInvited","content":"","date":"2024-07-11T06:08:41+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2024-06-07T13:59:07+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2024-06-07T13:58:20+00:00","index":"","fulltext":""},{"type":"submitted","content":"Signal, Image and Video Processing","date":"2024-06-06T10:58:48+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"signal-image-and-video-processing","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"sivp","sideBox":"Learn more about [Signal, Image and Video Processing](http://link.springer.com/journal/11760)","snPcode":"11760","submissionUrl":"https://submission.nature.com/new-submission/11760/3","title":"Signal, Image and Video Processing","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"85352b28-1281-4c15-b26b-5399c25f0a72","owner":[],"postedDate":"June 21st, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[],"tags":[],"updatedAt":"2025-06-09T15:58:17+00:00","versionOfRecord":{"articleIdentity":"rs-4539841","link":"https://doi.org/10.1007/s11760-025-04096-6","journal":{"identity":"signal-image-and-video-processing","isVorOnly":false,"title":"Signal, Image and Video Processing"},"publishedOn":"2025-06-05 15:56:54","publishedOnDateReadable":"June 5th, 2025"},"versionCreatedAt":"2024-06-21 05:59:11","video":"","vorDoi":"10.1007/s11760-025-04096-6","vorDoiUrl":"https://doi.org/10.1007/s11760-025-04096-6","workflowStages":[]},"version":"v1","identity":"rs-4539841","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-4539841","identity":"rs-4539841","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}