SMILES Challenge 2025: Multitask Learning with Contrastive and Natural Language Generation for Enhanced Medical Image Classification

SMILES Challenge 2025: Multitask Learning with Contrastive and Natural Language Generation for Enhanced Medical Image Classification | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article SMILES Challenge 2025: Multitask Learning with Contrastive and Natural Language Generation for Enhanced Medical Image Classification Raja Vavekanand, Teerath Kumar This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-7782188/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 10 Mar, 2026 Read the published version in Signal, Image and Video Processing → Version 1 posted 9 You are reading this latest preprint version Abstract This article proposes a novel multitask learning framework that integrates contrastive learning and natural language generation (NLG) to enhance medical image classification and report generation. The goal is to improve disease classification accuracy and interpretability in medical diagnostics. The model architecture consists of a Vision Transformer (ViT) as a visual encoder, a transformer-based text encoder, and a multimodal decoder. The visual encoder processes medical images, while the text encoder handles disease-related text prompts. These components are trained jointly using image-text contrastive loss and language generation loss. Evaluations on the MIMICCXR and Chexpert datasets show that the model with NLG (Plain + NLG) outperforms the baseline contrastive learning model (Plain) in disease classification. For example, in the MIMICCXR dataset, the accuracy for Atelectasis increased from 17.44%(Plain) to 41.5% (Plain + NLG), and for Cardiomegaly, it improved from 19.25% to 47.4%. In Chexpert, the accuracy for Atelectasis increased from 12.5% to 58.5%, and for Pleural Effusion, from 61.10% to 64.0%. The model also demonstrated improvements in F1 scores, particularly for complex diseases like Cardiomegaly and Consolidation. The proposed multitask framework effectively combines contrastive learning with NLG, leading to improved disease classification and medical report generation. This approach has potential clinical applications by enhancing AI's interpretability and accuracy in medical decision-making. Image Classification Medical Imaging Multimodal Learning Contrastive Learning Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Published Journal Publication published 10 Mar, 2026 Read the published version in Signal, Image and Video Processing → Version 1 posted Editorial decision: Revision requested 26 Oct, 2025 Reviews received at journal 26 Oct, 2025 Reviewers agreed at journal 26 Oct, 2025 Reviews received at journal 26 Oct, 2025 Reviewers agreed at journal 14 Oct, 2025 Reviewers invited by journal 12 Oct, 2025 Editor assigned by journal 06 Oct, 2025 Submission checks completed at journal 06 Oct, 2025 First submitted to journal 04 Oct, 2025 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-7782188","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":534173283,"identity":"76892368-78d3-4e1d-9bf7-f819c38e6679","order_by":0,"name":"Raja Vavekanand","email":"","orcid":"","institution":"Shaheed Benazir Bhutto University","correspondingAuthor":false,"prefix":"","firstName":"Raja","middleName":"","lastName":"Vavekanand","suffix":""},{"id":534173284,"identity":"7908b19b-4956-4dd8-920a-eb2d7e8f9dac","order_by":1,"name":"Teerath Kumar","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABMUlEQVRIiWNgGAWjYPCCA2CSGUTwszcwMCQUMDA2wMVRAQ+YTEDSItkD5CQYkKLF4EYCiIRowQbs2ZuPffj54w6Dbnvv4deFbXb5DDdfJ354YMAg2z8jgfHAByy28BxLntmT8IzB7My5NOuZbcmWjbNzN0sAHWY8A2jdwRlYtEjkGDPwJBxmMLuRY2bMu43ZgFk6dwNISyKQZDjMg0WL/PvPjH9AWu6/AWmpN2CTPLv5B1zLH2y28DAzQ2zhMX7Mu+2wAY8E7zaELdhC7EyaMbNM2mEeszM5Zsy8/44bSPDkbrNIMJAwnnHmYcPBHkwt7O2HHzO+sTksZ3b8jPFnnjPVBvbHz26++aPCRra/Pfnwhx84QpoBEj9sEkgCIDbOqIEDZiyRMApGwSgYBaOAgQEAG51rub2qWGoAAAAASUVORK5CYII=","orcid":"","institution":"Dublin City University","correspondingAuthor":true,"prefix":"","firstName":"Teerath","middleName":"","lastName":"Kumar","suffix":""}],"badges":[],"createdAt":"2025-10-05 00:53:06","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-7782188/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-7782188/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1007/s11760-026-05214-8","type":"published","date":"2026-03-10T15:59:49+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":94445649,"identity":"2e5d2135-f0b3-456a-9c26-72c22b8fdd6e","added_by":"auto","created_at":"2025-10-27 14:33:23","extension":"json","order_by":0,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":4262,"visible":true,"origin":"","legend":"","description":"","filename":"bd7a874a511e4bafbab5cd09ff935a12.json","url":"https://assets-eu.researchsquare.com/files/rs-7782188/v1/fae15c0ce02e4c9ecd7f9986.json"},{"id":104740001,"identity":"710dbd16-c2a1-4ca6-b744-d8456398ba40","added_by":"auto","created_at":"2026-03-16 16:14:30","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":604231,"visible":true,"origin":"","legend":"","description":"","filename":"SMILES2025MultitaskLearningwithContrastiveandNaturalLanguageGenerationforEnhancedMedicalImageClassification.pdf","url":"https://assets-eu.researchsquare.com/files/rs-7782188/v1_covered_650e0d94-5a51-4247-836b-e238e2a7ef50.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"SMILES Challenge 2025: Multitask Learning with Contrastive and Natural Language Generation for Enhanced Medical Image Classification","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"signal-image-and-video-processing","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"sivp","sideBox":"Learn more about [Signal, Image and Video Processing](http://link.springer.com/journal/11760)","snPcode":"11760","submissionUrl":"https://submission.nature.com/new-submission/11760/3","title":"Signal, Image and Video Processing","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Image Classification, Medical Imaging, Multimodal Learning, Contrastive Learning","lastPublishedDoi":"10.21203/rs.3.rs-7782188/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-7782188/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eThis article proposes a novel multitask learning framework that integrates contrastive learning and natural language generation (NLG) to enhance medical image classification and report generation. The goal is to improve disease classification accuracy and interpretability in medical diagnostics. The model architecture consists of a Vision Transformer (ViT) as a visual encoder, a transformer-based text encoder, and a multimodal decoder. The visual encoder processes medical images, while the text encoder handles disease-related text prompts. These components are trained jointly using image-text contrastive loss and language generation loss. Evaluations on the MIMICCXR and Chexpert datasets show that the model with NLG (Plain + NLG) outperforms the baseline contrastive learning model (Plain) in disease classification. For example, in the MIMICCXR dataset, the accuracy for Atelectasis increased from 17.44%(Plain) to 41.5% (Plain + NLG), and for Cardiomegaly, it improved from 19.25% to 47.4%. In Chexpert, the accuracy for Atelectasis increased from 12.5% to 58.5%, and for Pleural Effusion, from 61.10% to 64.0%. The model also demonstrated improvements in F1 scores, particularly for complex diseases like Cardiomegaly and Consolidation. The proposed multitask framework effectively combines contrastive learning with NLG, leading to improved disease classification and medical report generation. This approach has potential clinical applications by enhancing AI's interpretability and accuracy in medical decision-making.\u003c/p\u003e","manuscriptTitle":"SMILES Challenge 2025: Multitask Learning with Contrastive and Natural Language Generation for Enhanced Medical Image Classification","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-10-27 11:11:46","doi":"10.21203/rs.3.rs-7782188/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2025-10-26T13:28:19+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-10-26T10:43:53+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"89053716940197391495078017936630724493","date":"2025-10-26T10:10:28+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-10-26T04:41:47+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"177193670483752636076302962239387180906","date":"2025-10-15T02:08:53+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2025-10-13T01:40:15+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2025-10-06T12:32:04+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2025-10-06T12:31:34+00:00","index":"","fulltext":""},{"type":"submitted","content":"Signal, Image and Video Processing","date":"2025-10-05T00:49:11+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"signal-image-and-video-processing","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"sivp","sideBox":"Learn more about [Signal, Image and Video Processing](http://link.springer.com/journal/11760)","snPcode":"11760","submissionUrl":"https://submission.nature.com/new-submission/11760/3","title":"Signal, Image and Video Processing","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"60e8c672-f3e5-40d5-bec3-48135a069f96","owner":[],"postedDate":"October 27th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[],"tags":[],"updatedAt":"2026-03-16T16:09:55+00:00","versionOfRecord":{"articleIdentity":"rs-7782188","link":"https://doi.org/10.1007/s11760-026-05214-8","journal":{"identity":"signal-image-and-video-processing","isVorOnly":false,"title":"Signal, Image and Video Processing"},"publishedOn":"2026-03-10 15:59:49","publishedOnDateReadable":"March 10th, 2026"},"versionCreatedAt":"2025-10-27 11:11:46","video":"","vorDoi":"10.1007/s11760-026-05214-8","vorDoiUrl":"https://doi.org/10.1007/s11760-026-05214-8","workflowStages":[]},"version":"v1","identity":"rs-7782188","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-7782188","identity":"rs-7782188","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}