Towards Multimodal Retrieval-Augmented Generation for Medical Visual Question Answering

Towards Multimodal Retrieval-Augmented Generation for Medical Visual Question Answering | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Article Towards Multimodal Retrieval-Augmented Generation for Medical Visual Question Answering Mai A. Shaaban, Mohammad Reza Zarei, Adnan Khan, Abbas Akkasi, and 2 more This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-7752202/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 15 You are reading this latest preprint version Abstract Medical visual question answering (MedVQA) is a critical AI healthcare task that combines medical image analysis with natural language understanding to assist clinicians in decision-making. While medical vision-language models have shown promise in this domain, they struggle with factual inaccuracies and hallucinations. Retrieval-augmented generation (RAG) improves the factual accuracy by grounding responses in external knowledge, yet text-only or image-only retrieval systems struggle with the inherently multimodal nature of medical data, leading to information loss. This paper proposes a novel multimodal RAG framework tailored for MedVQA, which leverages multimodal data, including medical images, reports, and generated captions, to provide more accurate clinical answers. We introduce a training paradigm that uses captions as auxiliary supervision, enhancing cross-modal alignment via contrastive learning. Comprehensive evaluations on MedVQA benchmarks demonstrate the framework’s effectiveness, achieving a 7% average accuracy improvement over unimodal RAG baselines. Our study has the potential to better support clinicians in delivering accurate, timely, and trustworthy patient care by improving the reliability of MedVQA systems. The code is publicly available at https://github.com/AiMl-hub/MM-RAG-MedVQA . Scientific community and society/Business and industry Biological sciences/Computational biology and bioinformatics Health sciences/Health care Physical sciences/Mathematics and computing Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Under Review Version 1 posted Editorial decision: Revision requested 04 Dec, 2025 Reviews received at journal 04 Dec, 2025 Reviews received at journal 23 Nov, 2025 Reviewers agreed at journal 21 Nov, 2025 Reviewers agreed at journal 18 Nov, 2025 Reviews received at journal 16 Nov, 2025 Reviewers agreed at journal 16 Nov, 2025 Reviews received at journal 05 Nov, 2025 Reviewers agreed at journal 20 Oct, 2025 Reviewers agreed at journal 20 Oct, 2025 Reviewers invited by journal 15 Oct, 2025 Editor invited by journal 15 Oct, 2025 Editor assigned by journal 06 Oct, 2025 Submission checks completed at journal 04 Oct, 2025 First submitted to journal 30 Sep, 2025 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-7752202","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Article","associatedPublications":[],"authors":[{"id":534766373,"identity":"be05ca3f-bee3-459f-a15e-12e3f4cacc6c","order_by":0,"name":"Mai A. Shaaban","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABDElEQVRIiWNgGAWjYFACxgYQYjAAsh4wFEDEJIjVwmwAIonQAtEFUswmQZQW3fbDDYw/d9jJm7OfPVbxw8AmccMB5oO3eRhs7BpwaDE7k9jAzHsm2XBnT17azR6DNGODA2zJ1jwMack4tRwAamFsY2bccCDH7AaPwWE5gwM8ZtI8DIeTcTnM7PxDoMPa6u03nH9jVvjH4D+PwQH+b0At/3FruZHYwMDbdjhxw40cM2agepAtbEAtB+xwa3nYcJi37XjyzhlvjKVlDJKNJQ+zGVvOMUhOwO2w9IcPf7ZV227nzzH8+KbCLrHvePPDG0CGPS4tIHAAlcsMIgwYgA4mFeC1ZRSMglEwCkYUAAAx11fwVbJUSwAAAABJRU5ErkJggg==","orcid":"","institution":"Alexandria University","correspondingAuthor":true,"prefix":"","firstName":"Mai","middleName":"A.","lastName":"Shaaban","suffix":""},{"id":534766374,"identity":"750918d9-148b-4859-a151-10aec880587f","order_by":1,"name":"Mohammad Reza Zarei","email":"","orcid":"","institution":"Carleton University","correspondingAuthor":false,"prefix":"","firstName":"Mohammad","middleName":"Reza","lastName":"Zarei","suffix":""},{"id":534766375,"identity":"0e1f8149-ee85-4762-abf5-c96122e0296d","order_by":2,"name":"Adnan Khan","email":"","orcid":"","institution":"Carleton University","correspondingAuthor":false,"prefix":"","firstName":"Adnan","middleName":"","lastName":"Khan","suffix":""},{"id":534766376,"identity":"b2724e46-4df0-4efc-b1af-069545b18627","order_by":3,"name":"Abbas Akkasi","email":"","orcid":"","institution":"Carleton University","correspondingAuthor":false,"prefix":"","firstName":"Abbas","middleName":"","lastName":"Akkasi","suffix":""},{"id":534766377,"identity":"2a76e902-93e4-4e30-9a53-62e97d20ff83","order_by":4,"name":"Mohammad Yaqub","email":"","orcid":"","institution":"Mohamed bin Zayed University of Artificial Intelligence","correspondingAuthor":false,"prefix":"","firstName":"Mohammad","middleName":"","lastName":"Yaqub","suffix":""},{"id":534766378,"identity":"783ab1a0-b62a-4e0b-8bbe-b4afabf2ef42","order_by":5,"name":"Majid Komeili","email":"","orcid":"","institution":"Carleton University","correspondingAuthor":false,"prefix":"","firstName":"Majid","middleName":"","lastName":"Komeili","suffix":""}],"badges":[],"createdAt":"2025-09-30 13:53:25","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-7752202/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-7752202/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":94584339,"identity":"3f9b0de0-939d-4e35-ad91-3ec1d168aef2","added_by":"auto","created_at":"2025-10-28 18:15:09","extension":"json","order_by":0,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":7151,"visible":true,"origin":"","legend":"","description":"","filename":"3a86251ed0ab4edd82d44308ebaf820d.json","url":"https://assets-eu.researchsquare.com/files/rs-7752202/v1/3f9348b85ff799dbd82128d5.json"},{"id":94595900,"identity":"da86bee4-5b2e-4ee9-ad97-15a2872a45b0","added_by":"auto","created_at":"2025-10-28 18:37:11","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":1164725,"visible":true,"origin":"","legend":"","description":"","filename":"nbhrmdqcqsvrhjzpnbyrgcddmcbdgmpf.pdf","url":"https://assets-eu.researchsquare.com/files/rs-7752202/v1_covered_2adb2175-a0b2-46cb-a165-0ac3f42cc1f8.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Towards Multimodal Retrieval-Augmented Generation for Medical Visual Question Answering","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"scientific-reports","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"scirep","sideBox":"Learn more about [Scientific Reports](http://www.nature.com/srep/)","snPcode":"","submissionUrl":"","title":"Scientific Reports","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Scientific Reports","inReviewEnabled":true,"inReviewRevisionsEnabled":true},"keywords":"","lastPublishedDoi":"10.21203/rs.3.rs-7752202/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-7752202/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Medical visual question answering (MedVQA) is a critical AI healthcare task that combines medical image analysis with natural language understanding to assist clinicians in decision-making. While medical vision-language models have shown promise in this domain, they struggle with factual inaccuracies and hallucinations. Retrieval-augmented generation (RAG) improves the factual accuracy by grounding responses in external knowledge, yet text-only or image-only retrieval systems struggle with the inherently multimodal nature of medical data, leading to information loss. This paper proposes a novel multimodal RAG framework tailored for MedVQA, which leverages multimodal data, including medical images, reports, and generated captions, to provide more accurate clinical answers. We introduce a training paradigm that uses captions as auxiliary supervision, enhancing cross-modal alignment via contrastive learning. Comprehensive evaluations on MedVQA benchmarks demonstrate the framework’s effectiveness, achieving a 7% average accuracy improvement over unimodal RAG baselines. Our study has the potential to better support clinicians in delivering accurate, timely, and trustworthy patient care by improving the reliability of MedVQA systems. The code is publicly available at https://github.com/AiMl-hub/MM-RAG-MedVQA.","manuscriptTitle":"Towards Multimodal Retrieval-Augmented Generation for Medical Visual Question Answering","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-10-28 16:31:49","doi":"10.21203/rs.3.rs-7752202/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2025-12-04T11:28:40+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-12-04T09:50:58+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-11-24T02:24:04+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"61842547999823834340793372859184796340","date":"2025-11-22T03:50:34+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"280666096529627292557918258339645820342","date":"2025-11-19T01:17:24+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-11-17T03:19:05+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"184972274266597815973960340721348141826","date":"2025-11-17T02:41:26+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-11-05T18:20:11+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"177863128778648903239139913566734355472","date":"2025-10-20T12:28:07+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"263946577568236532990442087572923524376","date":"2025-10-20T11:19:24+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2025-10-15T07:23:04+00:00","index":"","fulltext":""},{"type":"editorInvited","content":"","date":"2025-10-15T05:51:33+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2025-10-06T10:22:33+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2025-10-04T07:35:09+00:00","index":"","fulltext":""},{"type":"submitted","content":"Scientific Reports","date":"2025-09-30T13:42:07+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"scientific-reports","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"scirep","sideBox":"Learn more about [Scientific Reports](http://www.nature.com/srep/)","snPcode":"","submissionUrl":"","title":"Scientific Reports","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Scientific Reports","inReviewEnabled":true,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"7b5a9541-30c1-44b9-9968-8dd8d13076bf","owner":[],"postedDate":"October 28th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[{"id":56857127,"name":"Scientific community and society/Business and industry"},{"id":56857128,"name":"Biological sciences/Computational biology and bioinformatics"},{"id":56857129,"name":"Health sciences/Health care"},{"id":56857130,"name":"Physical sciences/Mathematics and computing"}],"tags":[],"updatedAt":"2026-04-09T15:25:01+00:00","versionOfRecord":[],"versionCreatedAt":"2025-10-28 16:31:49","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-7752202","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-7752202","identity":"rs-7752202","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}