A Transformer Driven Hybrid Feature Fusion Framework for Multi-Modal Medical Image Analysis

A Transformer Driven Hybrid Feature Fusion Framework for Multi-Modal Medical Image Analysis | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Article A Transformer Driven Hybrid Feature Fusion Framework for Multi-Modal Medical Image Analysis S. Vidhya, R. Nithya This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-7712468/v1 This work is licensed under a CC BY 4.0 License Status: Posted Version 1 posted You are reading this latest preprint version Abstract Early disease diagnosis greatly depends on strong medical image classification models. In this paper, a hybrid method is proposed to combine handcrafted descriptors (HOG, BoVW) and deep features (VGG19) to form an integrative feature fusion representation. The combined features are then fed into an optimized Vision Transformer (FFXViT), which allows stronger global context modelling while maintaining key local information. Two reference modalities, histopathology images with three classes - adenocarcinoma, squamous cell carcinoma, benign and chest X-ray images with four classes - COVID-19, lung opacity, normal, viral pneumonia, were experimented on. The proposed approach FFXViT attained 99.50% on histopathology and 97.41% on chest X-rays accuracy, a remarkable improvement over state-of-the-art CNNs, transformer and hybrid baselines. The experiment showcases the scalability, robustness, and interpretability of the framework and empirically verify FFXViT as a viable solution for robust cross-modality medical image analysis and clinical decision support. Biological sciences/Cancer Biological sciences/Computational biology and bioinformatics Health sciences/Diseases Physical sciences/Mathematics and computing Health sciences/Medical research Health sciences/Oncology Customized Vision Transformer Deep Features Feature Fusion Handcrafted Features Image Classification Vision Transformer Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Posted Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-7712468","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Article","associatedPublications":[],"authors":[{"id":531896983,"identity":"a674a094-f318-43c8-8b29-20ca5262d2ca","order_by":0,"name":"S. Vidhya","email":"","orcid":"","institution":"SASTRA Deemed to be University","correspondingAuthor":false,"prefix":"","firstName":"S.","middleName":"","lastName":"Vidhya","suffix":""},{"id":531896985,"identity":"0b19e7db-65af-45b2-962c-a76d1dad7bcf","order_by":1,"name":"R. Nithya","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA/klEQVRIiWNgGAWjYDACZgY2MM2PKsyGTwszRFayASrAQ1ALA1SLwQFitZiz8x97zFPDkG987fDDDwx/7OztpQ8/YPhQdhinFstmZnZjnmMMlttupxlLMLYlJ/bwpRkwzjiHW4vBYWY2aR42BgOz2zkMEowNzAk8PAwGzLxthLT8YzAwnp3D/IPhT709Dw/7B+a/hLTwtjEYGEjnsEkwsB1m7OHhMWBmxK/FTHJun4SBxO00M4vEtuOJPWd4Cg72nEvHreX8wWcSb77ZGPDPTn5848Ofanv2HvaND36UWePUAgJMPAwSEFYCVOQAXvVAwPiDkIpRMApGwSgY2QAAKpFHC89wJDkAAAAASUVORK5CYII=","orcid":"","institution":"SASTRA Deemed to be University","correspondingAuthor":true,"prefix":"","firstName":"R.","middleName":"","lastName":"Nithya","suffix":""}],"badges":[],"createdAt":"2025-09-25 11:38:19","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-7712468/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-7712468/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":94041256,"identity":"15ea68ae-7f6b-480e-8c2a-d9d8f8b1e60a","added_by":"auto","created_at":"2025-10-21 18:39:17","extension":"docx","order_by":0,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":4607703,"visible":true,"origin":"","legend":"","description":"","filename":"TransformerbasedFusionforCXRImageAnalysis.docx","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/bfa8db02725ce02877634c5c.docx"},{"id":94041246,"identity":"81812fca-c11a-415a-bc72-e3b222769330","added_by":"auto","created_at":"2025-10-21 18:39:17","extension":"json","order_by":1,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":4069,"visible":true,"origin":"","legend":"","description":"","filename":"d61068a88ae34aa3940b609475c26c1a.json","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/bca8f92ad857a26c1183f5f1.json"},{"id":94042222,"identity":"362cd33e-2255-4361-b8ef-351754bbaa22","added_by":"auto","created_at":"2025-10-21 18:55:17","extension":"xml","order_by":2,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":133394,"visible":true,"origin":"","legend":"","description":"","filename":"d61068a88ae34aa3940b609475c26c1a1enriched.xml","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/504b7693bb5e6ce016fa9194.xml"},{"id":94040912,"identity":"78aa6cb0-612e-4e7c-8030-79d1e57032f2","added_by":"auto","created_at":"2025-10-21 18:31:17","extension":"png","order_by":3,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":211213,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage1.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/0ab2f31c16a66073ccf1309f.png"},{"id":94040910,"identity":"09f749bb-19e6-43d9-868c-43ddd49b8558","added_by":"auto","created_at":"2025-10-21 18:31:17","extension":"png","order_by":4,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":39392,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage10.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/9524863a46fbd5fb8db2926d.png"},{"id":94041247,"identity":"5fd87ad3-27e0-4e79-bbae-a11461f47b9c","added_by":"auto","created_at":"2025-10-21 18:39:17","extension":"png","order_by":5,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":39398,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage11.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/f32b817bad6d47f75e32f29f.png"},{"id":94040907,"identity":"d200f291-fc3d-477c-9caf-7b5863fa2124","added_by":"auto","created_at":"2025-10-21 18:31:17","extension":"png","order_by":6,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":28000,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage12.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/94c5b1d7cf3f889aba7f82da.png"},{"id":94040905,"identity":"d83a6b73-6057-45b4-b451-57bc71bab90d","added_by":"auto","created_at":"2025-10-21 18:31:17","extension":"png","order_by":7,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":43970,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage13.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/ea45d3749631f610097edb80.png"},{"id":94042223,"identity":"20ccc57a-04d1-4e3a-b3ce-4abd926a475d","added_by":"auto","created_at":"2025-10-21 18:55:17","extension":"png","order_by":8,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":43782,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage14.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/597b484be6072bcce4217c41.png"},{"id":94041249,"identity":"cdd10fcc-8eeb-4912-9238-eec15ec7606f","added_by":"auto","created_at":"2025-10-21 18:39:17","extension":"png","order_by":9,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":41941,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage15.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/bd37ff299cb293fa53a18bec.png"},{"id":94041737,"identity":"dadf9f25-ed94-4a9a-964a-4c7a01ac7b66","added_by":"auto","created_at":"2025-10-21 18:47:20","extension":"png","order_by":10,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":43392,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage16.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/ec67ad8f161ba24aa5f2c5a6.png"},{"id":94040916,"identity":"5e0c8dc1-ba4d-4df2-ba4e-fefdec79c73c","added_by":"auto","created_at":"2025-10-21 18:31:17","extension":"png","order_by":11,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":42356,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage17.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/63e8d3e5fae2354c01e67c73.png"},{"id":94041738,"identity":"c57e5327-77a3-4bac-8da9-5c18da747654","added_by":"auto","created_at":"2025-10-21 18:47:21","extension":"png","order_by":12,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":43060,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage18.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/2e75fc58ab5a292b3191b720.png"},{"id":94041251,"identity":"1e9690a4-3580-412e-b3bb-5e6951645ff4","added_by":"auto","created_at":"2025-10-21 18:39:17","extension":"png","order_by":13,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":40404,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage19.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/97fac65552a109ed7dae265a.png"},{"id":94040913,"identity":"793b0ec1-dd1d-47f0-80ca-71fc9bed1639","added_by":"auto","created_at":"2025-10-21 18:31:17","extension":"png","order_by":14,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":126531,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage2.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/01e3616258e259b2ac83b57e.png"},{"id":94041254,"identity":"3f5a02ed-80a9-41fc-9ec6-720b0261a5bf","added_by":"auto","created_at":"2025-10-21 18:39:17","extension":"png","order_by":15,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":40603,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage20.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/44a976015f57361569b9ece8.png"},{"id":94041253,"identity":"d1185ca8-1ff1-4d8e-aed3-f797d42f4232","added_by":"auto","created_at":"2025-10-21 18:39:17","extension":"png","order_by":16,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":39070,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage21.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/a164d66f0b1d211c1b69ace5.png"},{"id":94041742,"identity":"b1e9da22-fa04-42d2-b719-2477244ef6a6","added_by":"auto","created_at":"2025-10-21 18:47:30","extension":"jpeg","order_by":17,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":1078285,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage22.jpeg","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/10dde0cd1c7233037bc48a78.jpeg"},{"id":94042225,"identity":"d68bd49e-b993-49d5-b254-b3909114de48","added_by":"auto","created_at":"2025-10-21 18:55:17","extension":"jpeg","order_by":18,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":313789,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage23.jpeg","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/66e443dca9d6c93987f223f7.jpeg"},{"id":94041740,"identity":"80f50238-360e-4473-a95b-328fce9ceed7","added_by":"auto","created_at":"2025-10-21 18:47:27","extension":"png","order_by":19,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":34285,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage24.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/46cbe432071b6af81773490b.png"},{"id":94040918,"identity":"2e187a84-f8b1-480c-9ad1-d2e382b085e7","added_by":"auto","created_at":"2025-10-21 18:31:17","extension":"png","order_by":20,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":388762,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage3.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/26b413ac3fe4b1dfd948938a.png"},{"id":94040921,"identity":"b6a33853-5ae2-4d42-b930-bec5e446fec6","added_by":"auto","created_at":"2025-10-21 18:31:17","extension":"png","order_by":21,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":579701,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage4.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/dff651518f71ddafc9b20a75.png"},{"id":94041728,"identity":"9686247a-304a-4ad4-bd4a-add4f840a7aa","added_by":"auto","created_at":"2025-10-21 18:47:17","extension":"png","order_by":22,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":234735,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage5.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/e5e50fff743127fa253fc58b.png"},{"id":94040946,"identity":"928689f9-1f96-4a77-a40e-38077bfaaae4","added_by":"auto","created_at":"2025-10-21 18:31:18","extension":"png","order_by":23,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":1473239,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage6.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/45bc0d74065c7f9cfdfa47ad.png"},{"id":94041731,"identity":"51d99b37-abd7-482d-b003-106b4490bef6","added_by":"auto","created_at":"2025-10-21 18:47:18","extension":"png","order_by":24,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":40203,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage7.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/89aaae9363db4d4b154fc34f.png"},{"id":94041258,"identity":"cb988e6a-4322-4527-a3e6-eb6b66ccfee2","added_by":"auto","created_at":"2025-10-21 18:39:17","extension":"png","order_by":25,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":42073,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage8.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/c1ccdfada0912d568bafc795.png"},{"id":94040930,"identity":"1d771ee8-b3d8-4135-8a18-b81440baf6e6","added_by":"auto","created_at":"2025-10-21 18:31:17","extension":"jpeg","order_by":26,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":209745,"visible":true,"origin":"","legend":"","description":"","filename":"floatimage9.jpeg","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/5562fa1ea9869959a4d6c565.jpeg"},{"id":94041263,"identity":"b0924874-cb14-4d2d-995e-9f6afd793049","added_by":"auto","created_at":"2025-10-21 18:39:17","extension":"png","order_by":27,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":42057,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage1.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/c071225e871d74c39180a328.png"},{"id":94040924,"identity":"e20adc76-c67c-42b8-8d5d-c8530c4546c7","added_by":"auto","created_at":"2025-10-21 18:31:17","extension":"png","order_by":28,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":11919,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage10.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/eaeef0a008a522deca7059b0.png"},{"id":94041729,"identity":"e9b305f0-b7bc-42f7-a51c-475f39309619","added_by":"auto","created_at":"2025-10-21 18:47:17","extension":"png","order_by":29,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":11923,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage11.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/d20d135378239e1324e6f8b4.png"},{"id":94040926,"identity":"6265386f-b650-45bc-b145-5c0bcc6a1680","added_by":"auto","created_at":"2025-10-21 18:31:17","extension":"png","order_by":30,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":9009,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage12.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/983952aeea702412994d51c4.png"},{"id":94040933,"identity":"b5ed3815-7701-4533-a4fe-7132b1b2821c","added_by":"auto","created_at":"2025-10-21 18:31:17","extension":"png","order_by":31,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":13480,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage13.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/7266b56f5d384b27e9fd6e83.png"},{"id":94041267,"identity":"a7cd334a-a6ad-4af9-8343-32ed11e17530","added_by":"auto","created_at":"2025-10-21 18:39:18","extension":"png","order_by":32,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":12933,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage14.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/754138105b34ac83835f7ed6.png"},{"id":94041730,"identity":"d2f88b41-4aab-4adb-b46f-be4242c94836","added_by":"auto","created_at":"2025-10-21 18:47:18","extension":"png","order_by":33,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":12712,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage15.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/5a10afffc14835ac5ed019d1.png"},{"id":94042226,"identity":"c5cbae3b-2582-408c-a810-593ab890ed0d","added_by":"auto","created_at":"2025-10-21 18:55:17","extension":"png","order_by":34,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":11515,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage16.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/0a1649bde2aa96b2386572a9.png"},{"id":94040944,"identity":"933c95f8-2179-4433-97b5-f9ae975431ed","added_by":"auto","created_at":"2025-10-21 18:31:18","extension":"png","order_by":35,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":11235,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage17.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/680ce81119385402ce3fed8e.png"},{"id":94040942,"identity":"ddf0ed18-4032-4572-b89b-c439441a1b42","added_by":"auto","created_at":"2025-10-21 18:31:18","extension":"png","order_by":36,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":11487,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage18.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/d4197df7f05509048ba7968e.png"},{"id":94040939,"identity":"45cc97bd-b4a8-4667-a31a-9b9502073b0a","added_by":"auto","created_at":"2025-10-21 18:31:18","extension":"png","order_by":37,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":12132,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage19.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/5011b7709c3cf76ad5d21dd2.png"},{"id":94040940,"identity":"122aa05f-70f1-40c2-bf96-550e48f592ba","added_by":"auto","created_at":"2025-10-21 18:31:18","extension":"png","order_by":38,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":25928,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage2.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/256bf4008c88b4437c1db2ec.png"},{"id":94041268,"identity":"601e3953-3ade-4f6d-bdda-601fa1c6c287","added_by":"auto","created_at":"2025-10-21 18:39:18","extension":"png","order_by":39,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":12341,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage20.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/b2bd72563a9c6a40ea530e24.png"},{"id":94041269,"identity":"f62ab746-bac1-46a3-bf08-33ccff7a4b4d","added_by":"auto","created_at":"2025-10-21 18:39:18","extension":"png","order_by":40,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":11659,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage21.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/6ccb91243861d84edf543e36.png"},{"id":94041733,"identity":"2406f409-897a-4655-879a-451e2b87a91f","added_by":"auto","created_at":"2025-10-21 18:47:18","extension":"png","order_by":41,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":190148,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage22.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/ef4625bc4b9bfdaa546eaa80.png"},{"id":94040945,"identity":"5c6e4100-8141-4048-8f28-0c2d30969370","added_by":"auto","created_at":"2025-10-21 18:31:18","extension":"png","order_by":42,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":56798,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage23.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/7ddfdfb64c5cc867ef7f33fb.png"},{"id":94040949,"identity":"7383dc0c-3ab6-4d62-9d92-149383c0bfe6","added_by":"auto","created_at":"2025-10-21 18:31:18","extension":"png","order_by":43,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":11279,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage24.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/85644f96130813fdffee38b0.png"},{"id":94041271,"identity":"2dfe005a-d22c-4aac-87c5-97290d218b8d","added_by":"auto","created_at":"2025-10-21 18:39:18","extension":"png","order_by":44,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":112172,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage3.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/29dc01e75cf97247db1f730a.png"},{"id":94040941,"identity":"e3cc65d6-c08a-4cce-b0fb-9b210cd60335","added_by":"auto","created_at":"2025-10-21 18:31:18","extension":"png","order_by":45,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":104681,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage4.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/44005b52ebc39254e91d763d.png"},{"id":94042227,"identity":"958be99b-4f4d-4f25-b4c0-469023f808d8","added_by":"auto","created_at":"2025-10-21 18:55:18","extension":"png","order_by":46,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":37377,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage5.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/c3ca43a1bfa38a482e58bbf1.png"},{"id":94040947,"identity":"1c64a2f6-ace4-4226-8c60-e5af59a5290e","added_by":"auto","created_at":"2025-10-21 18:31:18","extension":"png","order_by":47,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":905279,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage6.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/741b59d1141daa4b16bfbf60.png"},{"id":94040943,"identity":"85e0315a-58fb-4318-8c88-4024e628efe2","added_by":"auto","created_at":"2025-10-21 18:31:18","extension":"png","order_by":48,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":10861,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage7.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/d92c816a4975cfa6417ce698.png"},{"id":94041274,"identity":"c2f6c4fa-f8fe-4ad5-8d23-49d820a5402c","added_by":"auto","created_at":"2025-10-21 18:39:18","extension":"png","order_by":49,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":11265,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage8.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/fda7d9fba2d380bd3f07fb27.png"},{"id":94040952,"identity":"7622dcee-e922-4bb7-9787-e36c75cef88e","added_by":"auto","created_at":"2025-10-21 18:31:18","extension":"png","order_by":50,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":52575,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinefloatimage9.png","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/c46c12b4610be8cd6961a50e.png"},{"id":94041741,"identity":"72a68c0d-a967-4891-8080-6cdc76ed6510","added_by":"auto","created_at":"2025-10-21 18:47:30","extension":"xml","order_by":51,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":133138,"visible":true,"origin":"","legend":"","description":"","filename":"d61068a88ae34aa3940b609475c26c1a1structuring.xml","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/abc07d487b2e7c97dba57da4.xml"},{"id":94040958,"identity":"1c41067e-cbb1-47e5-9ea8-4e0fef6d8335","added_by":"auto","created_at":"2025-10-21 18:31:18","extension":"html","order_by":52,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":146048,"visible":true,"origin":"","legend":"","description":"","filename":"earlyproof.html","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1/c375bbb0926108227eb15dbf.html"},{"id":98535522,"identity":"0c6d2261-0428-4798-a05c-5b960454efd4","added_by":"auto","created_at":"2025-12-18 16:42:00","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":1629674,"visible":true,"origin":"","legend":"","description":"","filename":"TransformerbasedFusionforCXRImageAnalysis.pdf","url":"https://assets-eu.researchsquare.com/files/rs-7712468/v1_covered_1c8c79a1-36b8-48f7-8781-eb29baca4767.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"A Transformer Driven Hybrid Feature Fusion Framework for Multi-Modal Medical Image Analysis","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":true,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"Customized Vision Transformer, Deep Features, Feature Fusion, Handcrafted Features, Image Classification, Vision Transformer","lastPublishedDoi":"10.21203/rs.3.rs-7712468/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-7712468/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eEarly disease diagnosis greatly depends on strong medical image classification models. In this paper, a hybrid method is proposed to combine handcrafted descriptors (HOG, BoVW) and deep features (VGG19) to form an integrative feature fusion representation. The combined features are then fed into an optimized Vision Transformer (FFXViT), which allows stronger global context modelling while maintaining key local information. Two reference modalities, histopathology images with three classes - adenocarcinoma, squamous cell carcinoma, benign and chest X-ray images with four classes - COVID-19, lung opacity, normal, viral pneumonia, were experimented on. The proposed approach FFXViT attained 99.50% on histopathology and 97.41% on chest X-rays accuracy, a remarkable improvement over state-of-the-art CNNs, transformer and hybrid baselines. The experiment showcases the scalability, robustness, and interpretability of the framework and empirically verify FFXViT as a viable solution for robust cross-modality medical image analysis and clinical decision support.\u003c/p\u003e","manuscriptTitle":"A Transformer Driven Hybrid Feature Fusion Framework for Multi-Modal Medical Image Analysis","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-10-21 18:31:12","doi":"10.21203/rs.3.rs-7712468/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"8ee6c32a-5b3d-4b3d-b676-b752fe5ec8cd","owner":[],"postedDate":"October 21st, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"posted","subjectAreas":[{"id":56544844,"name":"Biological sciences/Cancer"},{"id":56544845,"name":"Biological sciences/Computational biology and bioinformatics"},{"id":56544846,"name":"Health sciences/Diseases"},{"id":56544847,"name":"Physical sciences/Mathematics and computing"},{"id":56544848,"name":"Health sciences/Medical research"},{"id":56544849,"name":"Health sciences/Oncology"}],"tags":[],"updatedAt":"2025-12-18T16:41:11+00:00","versionOfRecord":[],"versionCreatedAt":"2025-10-21 18:31:12","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-7712468","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-7712468","identity":"rs-7712468","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}