Reinforced Visual Interaction Fusion Radiology Report Generation

Reinforced Visual Interaction Fusion Radiology Report Generation | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Reinforced Visual Interaction Fusion Radiology Report Generation Liya Wang, Haipeng Chen, Yu Liu, Yingda Lyu, Feng Qiu This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-4576817/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 29 Sep, 2024 Read the published version in Multimedia Systems → Version 1 posted 12 You are reading this latest preprint version Abstract The explosion in the number of more complex types of chest X-rays and CT scans in recent years has placed a significant workload on physicians, particularly in radiology departments, to interpret and produce radiology reports. There is therefore a need for more efficient generation of medical reports. In this paper, we propose the Reinforced Visual Interaction Fusion (RVIF) radiology report generation model, which adopts a novel and effective visual interaction fusion module, which is more conducive to extracting fused visual features of radiology images with clinical diagnostic significance and performing subsequent correlation. Sexual analysis and processing. In addition, a reinforcement learning step from image captioning to this task is introduced to further enhance the aligned diagnosis effect brought by the visual interactive fusion module to generate accurate and highly credible radiology reports. Quantitative experiments and visualization results prove that our model performs well on two public medical report generation datasets, IU X-Ray, and MIMIC-CXR, surpassing some SOTA methods. Compared with the SOTA model COMG+RL in 2024, the BLEU@1, 2, and 3 of the NLG metrics increased by 3.9%, 2.8%, and 0.5% respectively, METEOR increased by 2.2%, the precision P of the CE index increased by 0.4%, and the recall rate R increased by 1.5%, F1-score increased by 1.8%. Source code in https://github.com/200084/RVIF-Radiology-Report-Generation . Radiology Report Generation Multimodal Fusion Reinforcement Learning Transformer Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Published Journal Publication published 29 Sep, 2024 Read the published version in Multimedia Systems → Version 1 posted Editorial decision: Revision requested 02 Aug, 2024 Reviews received at journal 01 Aug, 2024 Reviews received at journal 26 Jul, 2024 Reviewers agreed at journal 12 Jul, 2024 Reviewers agreed at journal 12 Jul, 2024 Reviewers agreed at journal 10 Jul, 2024 Reviewers agreed at journal 10 Jul, 2024 Reviewers agreed at journal 10 Jul, 2024 Reviewers invited by journal 10 Jul, 2024 Editor assigned by journal 07 Jul, 2024 Submission checks completed at journal 17 Jun, 2024 First submitted to journal 13 Jun, 2024 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-4576817","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":332810308,"identity":"2de0e536-94ef-44db-8de1-03e37cb2794d","order_by":0,"name":"Liya Wang","email":"","orcid":"","institution":"Jilin University","correspondingAuthor":false,"prefix":"","firstName":"Liya","middleName":"","lastName":"Wang","suffix":""},{"id":332810309,"identity":"14d91d71-f1d8-495d-9da8-9e0e4df3cfab","order_by":1,"name":"Haipeng Chen","email":"","orcid":"","institution":"Jilin University","correspondingAuthor":false,"prefix":"","firstName":"Haipeng","middleName":"","lastName":"Chen","suffix":""},{"id":332810310,"identity":"2b0e418e-887f-4543-ad70-6f2c45d15d92","order_by":2,"name":"Yu Liu","email":"","orcid":"","institution":"Jilin University","correspondingAuthor":false,"prefix":"","firstName":"Yu","middleName":"","lastName":"Liu","suffix":""},{"id":332810311,"identity":"b2e28adf-7246-45b6-8da4-af7de252aa5d","order_by":3,"name":"Yingda Lyu","email":"","orcid":"","institution":"Jilin University","correspondingAuthor":false,"prefix":"","firstName":"Yingda","middleName":"","lastName":"Lyu","suffix":""},{"id":332810312,"identity":"a59cc972-8945-49b1-9b11-56c7cf905bc2","order_by":4,"name":"Feng Qiu","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA40lEQVRIiWNgGAWjYBACNv7mA4f/VNjYsbE3H3yQUFFDWAufxLHEBzxn0pL5eY4lGzw4c4ywFjmGHGUD3rbDjDNn+JhJPmxhJsJhDGfYJCTY0pgNbjCYVSQ2sDHwt3cn4NfC3HtMwoDHhs/gdkPajcQdMgwSZ85uIGDLuTSJBAmgLXcOHLuReIaNwUAil5CWHDOJAwaHGTfcSGwrSGxjJkqLsWFDAsj7yWwMxGkBBvJjhgPgQGaWSDhzjIegX+T7gVHJ+A8Ulf0fP/6oqJHjb+/FrwUD8JCmfBSMglEwCkYBVgAAjdVMcxJPmbQAAAAASUVORK5CYII=","orcid":"","institution":"Publicity Department of Jilin Provincial Committee of CPC","correspondingAuthor":true,"prefix":"","firstName":"Feng","middleName":"","lastName":"Qiu","suffix":""}],"badges":[],"createdAt":"2024-06-13 14:22:52","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-4576817/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-4576817/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1007/s00530-024-01504-8","type":"published","date":"2024-09-29T15:57:30+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":65628014,"identity":"de994c96-e4a6-4c1d-be83-137845191a3a","added_by":"auto","created_at":"2024-09-30 16:17:10","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":970364,"visible":true,"origin":"","legend":"","description":"","filename":"ReinforcedVisualInteractionFusionRadiologyReportGeneration.pdf","url":"https://assets-eu.researchsquare.com/files/rs-4576817/v1_covered_62f5c241-cb12-4e69-8bce-a6ba14e577eb.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Reinforced Visual Interaction Fusion Radiology Report Generation","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"multimedia-systems","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"mmsj","sideBox":"Learn more about [Multimedia Systems](http://link.springer.com/journal/530)","snPcode":"530","submissionUrl":"https://submission.nature.com/new-submission/530/3","title":"Multimedia Systems","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Radiology Report Generation, Multimodal Fusion, Reinforcement Learning, Transformer","lastPublishedDoi":"10.21203/rs.3.rs-4576817/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-4576817/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eThe explosion in the number of more complex types of chest X-rays and CT scans in recent years has placed a significant workload on physicians, particularly in radiology departments, to interpret and produce radiology reports. There is therefore a need for more efficient generation of medical reports. In this paper, we propose the Reinforced Visual Interaction Fusion (RVIF) radiology report generation model, which adopts a novel and effective visual interaction fusion module, which is more conducive to extracting fused visual features of radiology images with clinical diagnostic significance and performing subsequent correlation. Sexual analysis and processing. In addition, a reinforcement learning step from image captioning to this task is introduced to further enhance the aligned diagnosis effect brought by the visual interactive fusion module to generate accurate and highly credible radiology reports. Quantitative experiments and visualization results prove that our model performs well on two public medical report generation datasets, IU X-Ray, and MIMIC-CXR, surpassing some SOTA methods. Compared with the SOTA model COMG+RL in 2024, the BLEU@1, 2, and 3 of the NLG metrics increased by 3.9%, 2.8%, and 0.5% respectively, METEOR increased by 2.2%, the precision P of the CE index increased by 0.4%, and the recall rate R increased by 1.5%, F1-score increased by 1.8%. Source code in \u003cspan class=\"ExternalRef\"\u003e\u003cspan class=\"RefSource\"\u003ehttps://github.com/200084/RVIF-Radiology-Report-Generation\u003c/span\u003e\u003cspan address=\"https://github.com/200084/RVIF-Radiology-Report-Generation\" targettype=\"URL\" class=\"RefTarget\"\u003e\u003c/span\u003e\u003c/span\u003e.\u003c/p\u003e","manuscriptTitle":"Reinforced Visual Interaction Fusion Radiology Report Generation","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-07-31 21:59:46","doi":"10.21203/rs.3.rs-4576817/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2024-08-02T13:58:59+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2024-08-01T09:22:10+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2024-07-26T08:26:39+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"212717428949551883937941324408796021710","date":"2024-07-12T08:20:05+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"50528223912987606865271059604681188546","date":"2024-07-12T08:04:09+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"13435108711509455469446327691641832535","date":"2024-07-10T12:59:24+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"140705144993282776421066886394476837968","date":"2024-07-10T10:24:54+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"151482632110212669923226233621761346114","date":"2024-07-10T08:19:56+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2024-07-10T07:20:12+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2024-07-08T02:35:06+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2024-06-17T15:07:58+00:00","index":"","fulltext":""},{"type":"submitted","content":"Multimedia Systems","date":"2024-06-13T14:21:34+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"multimedia-systems","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"mmsj","sideBox":"Learn more about [Multimedia Systems](http://link.springer.com/journal/530)","snPcode":"530","submissionUrl":"https://submission.nature.com/new-submission/530/3","title":"Multimedia Systems","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"2e55a7d3-c1d7-4d59-8df3-38f93d85a570","owner":[],"postedDate":"July 31st, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[],"tags":[],"updatedAt":"2024-09-30T16:10:06+00:00","versionOfRecord":{"articleIdentity":"rs-4576817","link":"https://doi.org/10.1007/s00530-024-01504-8","journal":{"identity":"multimedia-systems","isVorOnly":false,"title":"Multimedia Systems"},"publishedOn":"2024-09-29 15:57:30","publishedOnDateReadable":"September 29th, 2024"},"versionCreatedAt":"2024-07-31 21:59:46","video":"","vorDoi":"10.1007/s00530-024-01504-8","vorDoiUrl":"https://doi.org/10.1007/s00530-024-01504-8","workflowStages":[]},"version":"v1","identity":"rs-4576817","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-4576817","identity":"rs-4576817","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}