Enhancing Clinical Reasoning in Medical Vision-Language Model through Structured Prompts

Enhancing Clinical Reasoning in Medical Vision-Language Model through Structured Prompts | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Enhancing Clinical Reasoning in Medical Vision-Language Model through Structured Prompts View ORCID Profile Kavya Dasaramoole Prakash , View ORCID Profile Kiseong Kim , View ORCID Profile Youngmahn Han doi: https://doi.org/10.1101/2025.07.31.25331901 Kavya Dasaramoole Prakash 1 Bio-AI R&D center , AIdanBio Inc, Daejeon, Republic of Korea Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Kavya Dasaramoole Prakash Kiseong Kim 3 R&D center, BioBrain Inc , Daejeon, Republic of Korea Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Kiseong Kim Youngmahn Han 1 Bio-AI R&D center , AIdanBio Inc, Daejeon, Republic of Korea 2 Center for Biomedical Computing Div. of National Supercomputing, Korea Institute of Science and Technology Information , Daejeon, Republic of Korea Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Youngmahn Han For correspondence: hans{at}kisti.re.kr Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Medical Vision-Language Models (MVLMs) are emerging as powerful tools for tasks such as Visual Question Answering (VQA); however, they often struggle with hallucination and limited reasoning transparency, particularly in complex diagnostic scenarios. In this work, we enhance the MedVLM-R1 framework by fine-tuning it using clinically informed prompt structures tailored specifically for radiology-based reasoning. Without altering the original model architecture or training strategy, we redesign the system prompts and question templates to guide the model through structured, modality-aware, and step-by-step diagnostic reasoning. Fine-tuning is performed using MRI-based question-answer (QA) pairs, and evaluations are conducted across three diagnostic imaging: MRI, CT, and X-ray to assess both in-domain and out-of-domain generalization. Our approach improves reasoning transparency and accuracy, achieving 96.00% on MRI, 72.67% on CT, and 75.2% on X-ray. Compared to the original MedVLM-R1, our method closes the gap in MRI accuracy while significantly enhancing generalization performance on CT and X-ray modalities. These results demonstrate that clinically grounded prompting effectively improves both reasoning fidelity and robustness across imaging modalities. The code is available at our GitHub repository: https://github.com/aidanbio/AIdanMed 1 Introduction In recent years, language modeling has made remarkable progress. Large Language Models (LLMs) have shown the ability to handle an extensive number of tasks, driving their growing popularity. Originally designed for text-only inputs, these models are now being extended to process visual information as well. Integrating vision with language opens the door to numerous applications that are central to the ongoing AI-driven technological transformation [ 1 ]. Vision-Language Models (VLMs) mark a significant breakthrough in artificial intelligence, facilitating multimodal understanding and reasoning by bridging textual and visual data. MVLMs mark a significant breakthrough in AI-powered healthcare, with the potential to improve the analysis of medical images, automate report generation, and support clinical decision-making [ 2 ]. Recent initiatives aim to enhance the capabilities of VLMs by incorporating explicit reasoning methods that prioritize interpretability and step-by-step explanation. Chain-of-Thought (CoT) prompting is a popular strategy for guiding models to generate intermediate reasoning steps prior to obtaining a final solution [ 3 ]. Although CoT prompting increases reasoning transparency in VLMs, training procedures are also important in aligning model behavior with clinical reasoning. Conventional fine-tuning techniques, such as Supervised Fine-Tuning (SFT) [ 4 , 5 ], frequently rely on static labeled datasets, which may not adequately capture the complex reasoning or diverse interpretations needed in clinical situations. To address this limitation, Reinforcement Learning (RL)-based techniques [ 6 ] have gained popularity, allowing models to iteratively adjust their outputs based on feedback signals that emphasize correctness, coherence, and clinical relevance. Group Relative Policy Optimization (GRPO) [ 7 ] is a promising RL approach for increasing reasoning accuracy and answer quality in multimodal challenges. Unlike conventional RL or SFT, GRPO employs a group-based comparison reward structure, allowing the model to optimize not just for the single best response but also for the relative quality of sampled candidate solutions. This helps the model generalize more effectively across clinical circumstances and varying levels of reasoning depth. One of the most recent developments is MedVLM-R1 [ 8 ], a powerful vision-language model designed for medical use. MedVLM-R1, built on a strong multimodal foundation, combines instruction-tuned language models with visual encoders and uses reinforcement learning strategies like GRPO to improve reasoning. It has shown remarkable performance on clinical tasks such as generating radiology reports and VQA, especially when using group-based reward optimization techniques with CoT prompting. Nevertheless, MedVLM-R1 has several drawbacks despite its advantages, most notably its dependence on generic system prompts that are neither task-specific nor clinically grounded. Especially in complex imaging scenarios involving multiple modalities, this often results in outputs that are semantically fluent but clinically imprecise or underexplained. Previous research has also documented cases of unsupported conclusions or hallucinated reasoning [ 9 ], in which the model produces explanations that seem convincing but lack clinical validity. Better alignment of model inputs and training signals with real-world diagnostic expectations remains critical, as medical decision-making necessitates precise, fine-grained, and context-aware reasoning. 1.1 Previous Work General Vision-Language Models : The foundation for multimodal comprehension was established by early developments in VLMs like BLIP [ 10 ], Flamingo [ 11 ], and LLaVA [ 12 ], which aligned visual and textual modalities through extensive pretraining on interleaved image-text datasets. These models frequently achieved competitive performance in zero-shot and few-shot conditions, exhibiting strong generalization performance across multiple downstream tasks, including visual question answering, captioning, and instruction-based tasks. However, although general-purpose VLMs perform well on open-domain tasks, they perform poorly in domain-specific contexts such as healthcare, where medical expertise and deeper reasoning are crucial. The complex visual patterns and textual semantics present in healthcare settings are frequently missed due to their generic training material and prompts. Medical Vision-Language Models : To address the domain gap, several medical-specific VLMs have been developed. Models such as BioViL [ 13 ] and MedCLIP [ 14 ] used contrastive learning to match radiological images with corresponding reports, enhancing visual representations in clinical settings. With more varied training signals, BioMedCLIP [ 15 ] expanded this method to larger biomedical corpora, improving cross-modal comprehension. Additional developments include instruction-tuned models that use CoT reasoning to provide clinically appropriate outputs, including LLaVA-Med [ 16 ], Med-PaLM 2 [ 17 ], and Med-R1 [ 18 ]. These models demonstrate the value of combining structured prompting with medical knowledge by achieving strong results on benchmarks such as VQA-RAD and PathVQA [ 19 ] [ 20 ]. Despite these strengths, the majority of medical VLMs continue to use general system prompts and static evaluation, which may restrict their capacity to adapt to more complex, ambiguous clinical situations. Although medically credible, their results frequently lack contextual harmony with real-world decision-making processes and deeper reasoning. Reasoning with Chain-of-Thought and Reinforcement Learning : To overcome these constraints, structured reasoning techniques like CoT prompting [ 3 ] have been developed. They encourage models to generate intermediate reasoning stages, thereby enhancing performance on challeng-ing inference tasks. Approaches based on Reinforcement Learning (RL) have been proposed to overcome static tuning and fixed datasets. Techniques such as Reinforcement Learning with Human Feedback (RLHF) [ 21 ] and Direct Preference Optimization (DPO) [ 22 ] refine behavior in a task-aligned approach by optimizing models based on human feedback. GRPO introduces group-based reward comparison, enabling the model to learn from relative quality rather than absolute correctness. This approach has shown improvements in reasoning depth, output diversity, and clinical relevance. However, even with GRPO, performance depends heavily on the quality and specificity of prompts, especially in multimodal clinical scenarios. Generic instructions often fail to adequately direct the model toward reasoning paths that are task-specific or clinically appropriate. Prompt Design and Instruction Tuning in Clinical Tasks : To mitigate these issues, recent work has emphasized the role of prompt engineering in guiding medical VLMs [ 23 ] [ 24 ]. Studies have shown that well-designed, clinically informed prompts improve interpretability, accuracy, and conformity to domain-specific expectations. Instruction tuning with structured prompts allows models to better simulate expert reasoning, enhancing their performance in diagnostic tasks, report generation, and VQA. In the following section, we build on these insights by enhancing MedVLM-R1 through domain-specific fine-tuning and radiology-grounded prompt design. We further evaluate the model’s reasoning consistency and generalization across modalities, demonstrating how targeted prompting can help overcome the limitations of generic instruction formats in producing accurate and interpretable outputs across diverse clinical scenarios. 1.2 Problem Statement Previous research has also documented cases of unsupported conclusions or hallucinated reasoning [ 9 ], in which the model produces explanations that seem convincing but lack clinical validity. Improved alignment of model inputs and training signals with real-world diagnostic expectations remains critical, as medical decision-making necessitates precise, fine-grained, and context-aware reasoning. To address these gaps, this work presents the following contributions: Fine-tuning of MedVLM-R1 using revised, clinically grounded prompts on existing domain-specific medical imaging datasets, enhancing diagnostic reasoning and improving performance across imaging modalities. Clinically informed prompt design that incorporates structured reasoning templates and modality-aware system instructions, designed to reduce hallucinated reasoning and improve alignment with expert clinical expectations. Evaluation across X-ray, CT, and MRI modalities , highlighting the effectiveness and generalizability of the proposed prompt modifications in producing accurate and interpretable outputs across diverse clinical scenarios. 2 Methodology We build upon MedVLM-R1, a vision-language model developed for medical visual question answering, which integrates a Qwen2-VL-2B backbone [ 25 ]. To encourage robust and interpretable responses, we adopt GRPO, originally proposed in [ 7 ] and later employed in the medical domain by MedVLM-R1 [ 8 ]. GRPO [ 7 ] is an extension of Proximal Policy Optimization (PPO) [ 6 ] that replaces the traditional value function with a group-relative advantage computed from sampled outputs. This approach helps align model behavior with task-specific preferences while avoiding the instability of learned critics. G candidate answers at every training stage is sampled from the previous policy and evaluated using a corresponding reward function r i . The group-normalized advantage A i is computed as: This encourages the model to prefer responses that are relatively better than others in the same batch, regardless of absolute reward scale. The model parameters are then updated by maximizing the GRPO objective: where is the importance sampling ratio and D KL ( π θ ∥ π ref ) is a KL-divergence penalty[ 26 ] that regularizes deviation from a reference policy π ref . The clipping threshold and KL regularization strength are controlled by the hyperparameters ϵ and β , respectively. This objective ensures preference alignment, robustness, and training stability across different modalities. The reward function [ 27 ] used during training consists of two components: Format Reward : This reward drives the model to adhere to the expected output format. Specifically, the model is required to enclose the reasoning in … tags and the final answer in … tags. If each of the four tags appears precisely once and no additional text appears beyond these tags, a reward of 1 is given. Any deviation from this, such as missing or duplicated tags or the presence of text beyond the tags, results in a reward of 0. Accuracy Reward : Once the format is validated, the final answer within the tag is checked against the ground-truth label. If the model correctly selects the letter corresponding to the ground-truth choice, it receives a full accuracy reward of 1. If the letter is accurate but the response includes extra information, it is considered a partial match with a reward of 0.5. Incorrect answers, missing answers, or format violations result in an accuracy reward of 0. The total reward r i ∈ [0, 2] is computed as the sum of the format and accuracy rewards. This two-step reward structure ensures that the model first learns to produce interpretable, structured outputs before optimizing for clinical correctness. 2.1 Prompt Design 2.1.1 MedVLM-R1 Prompt MedVLM-R1 uses a general-purpose prompt format aimed at guiding the model through structured reasoning. The original prompt is as follows: System Prompt “A conversation between User and Assistant. The user asks a question, and the Assistant solves it. The Assistant first thinks about the reasoning process in the mind and then provides the user with the answer. The reasoning process and answer are enclosed within … and … tags, respectively.” Question Template “ {Question} Your task: Think through the question step by step, and enclose your reasoning process in … tags. Then provide the correct single-letter choice (A, B, C, D) inside … tags. No extra information or text outside of these tags.” While effective in general settings, this format lacks clinical context and radiology-specific guidance. To address this limitation, we propose a revised prompt design that introduces structured medical reasoning, modality awareness, and anatomical precision. Our updated prompt is as follows: 2.1.2 Proposed prompt System Prompt A conversation between a Radiology VQA Assistant and a User. The User provides a medical image-related question based on modalities like MRI, CT, or X-ray. The Assistant is a clinically informed expert model trained to analyze radiological images, identify visual features, reason step-by-step, and generate structured diagnostic responses. The Assistant always first performs detailed clinical reasoning, including image interpretation, anatomical localization, and potential differentials. This reasoning is enclosed in … tags. Then, the Assistant provides a final answer (A, B, C, D) enclosed in … tags. Do not generate any other text outside these tags. Question Template {Question} Your task: Carefully analyze the radiological image and extract key visual features relevant to the question. Perform clinical reasoning step-by-step, considering anatomy, pathology, and modality-specific characteristics. Enclose your reasoning process inside … tags. Then, provide the most appropriate single-letter answer (A, B, C, D) based on your reasoning, enclosed in … tags. Do not include any extra explanation or information outside these tags. This revised prompt structure introduces radiology-specific context, encourages step-by-step clinical reasoning, and constrains output formatting for improved evaluation and interpretability, as shown in Figure 1 . Download figure Open in new tab Figure 1. Block diagram of prompting and reasoning Figure 2 demonstrates the qualitative impact of our proposed clinically informed prompt design compared to the original MedVLM-R1 prompt. While both prompts yield the correct answer (B – Chest region), the explanation generated under the proposed prompt is noticeably more detailed and clinically grounded. The MedVLM-R1 prompt produces a generic anatomical reference, whereas our revised prompt encourages modality-specific interpretation (e.g., “chest CT scan”) and inclusion of relevant radiological features such as “lung nodules”. This example illustrates how structured, domain-specific instructions enhance the model’s diagnostic reasoning and reduce generic or underspecified outputs. Download figure Open in new tab Figure 2. Example of improved reasoning with clinically informed prompt 3 Experiment 3.1 Dataset Preparation We utilize the HuatuoGPT-Vision dataset [ 28 ], a unified benchmark combining over 17,000 multiple-choice VQA samples from sources such as PMC-VQA [ 29 ], SLAKE [ 30 ], VQA-RAD [ 19 ], OmniMedVQA [ 31 ], and PathVQA [ 20 ]. These samples span diverse clinical tasks and imaging modalities, each paired with diagnostic images and two to six answer choices to reflect real-world medical decision-making. In this study, we focus on three key modalities: MRI, CT, and X-ray. Training is conducted exclusively on 600 MRI image-question pairs, leveraging MRI’s structural complexity and rich diagnostic detail. To ensure fair comparison, our data split follows the same configuration as MedVLM-R1 in both size and task format. For evaluation, we use 300 MRI samples as in-domain tests and 300 samples each from CT and X-ray for out-of-domain (OOD) evaluation, as shown in Table 1 . This consistent setup enables a controlled analysis of the model’s reasoning ability and its generalization across unseen modalities. View this table: View inline View popup Download powerpoint Table 1. Dataset composition for training and evaluation 3.2 Implementation We fine-tuned MedVLM-R1, a vision-language model based on the Qwen2-VL-2B backbone [ 25 ], which was originally pretrained on a combination of curated web data, open datasets, and synthetic image-text pairs. While we preserve the original model architecture, reward structure, and training algorithm, we introduce a new radiology-specific system prompt and question template to better guide clinical reasoning and output structure. This modification enables us to evaluate the impact of domain-aware prompting while keeping all other training conditions identical. Fine-tuning was conducted on two NVIDIA A100 80GB PCIe GPUs for 300 steps with a per-device batch size of 1. We used 6 candidate generations per input ( G = 6), a maximum prompt length of 512 tokens, and a maximum response length of 512 tokens. All other training parameters, including learning rate, GRPO configuration, and reward functions, were kept identical to those used in the original MedVLM-R1 setup to ensure a controlled comparison. 4 Results and Discussion We evaluate our prompt-augmented version of MedVLM-R1 on the HuatuoGPT-Vision dataset [ 28 ] across three different imaging modalities: MRI (in-domain), and CT and X-ray (out-of-domain). Our model was trained only on MRI samples, allowing us to assess its generalization capability to unseen modalities. Table 2 shows that our prompt-tuned model achieves an accuracy of 96.00% on MRI, outperforming the MedVLM-R1 baseline by a small but consistent margin. More notably, we observe larger improvements on CT (+2.67%) and X-ray (+6.2%) modalities, despite no exposure to these during training. These results suggest that structured clinical reasoning prompts enhance the model’s generalization performance. View this table: View inline View popup Download powerpoint Table 2. Accuracy comparison between baseline MedVLM-R1 and our prompt-tuned model across imaging modalities. Qualitative analysis further reveals that the revised system prompt encourages more complete and medically grounded reasoning traces, reducing instances of hallucinated or underspecified explanations. The … segments in our model’s outputs consistently include modality-specific anatomical references, diagnostic pathways, and relevant differential considerations. For instance, in Figure 4 , panel (a) illustrates the model identifying a meniscal abnormality in a knee MRI, with a detailed anatomical interpretation of the patella and surrounding tissues. In panel (b), the model correctly associates a CT image of the abdomen with liver disease and supports it with contextual knowledge about cross-sectional imaging and liver pathology. Similarly, panels (c) and (d) show that the model can accurately determine imaging modality (CT) and anatomical region (chest X-ray), while reasoning in a clinically consistent manner. In contrast, the baseline MedVLM-R1, while fluent, often defaults to generic or template-like responses that fail to reflect the complexity of radiological interpretation, especially in OOD settings. This confirms that prompt design plays a crucial role in aligning model behavior with expert reasoning patterns, even without altering the model architecture or training dataset. Figure 3 illustrates the training loss curve of the model over 300 optimization steps. The curve shows an overall decreasing trend, indicating progressive learning and convergence. While the majority of training steps yield low and stable loss values, several prominent spikes are observed, most notably around steps 85, 110, and 190. These transient increases in loss likely result from the inherent variance in reinforcement learning with group-based sampling, where challenging or diverse candidate responses can momentarily destabilize optimization. Despite these fluctuations, the model consistently recovers, with the final loss settling at a low value. These observations suggest that the fine-tuning process was effective and that the model successfully adapted to the training objective with improved alignment and output quality. Download figure Open in new tab Figure 3. Training loss curve Download figure Open in new tab Figure 4. Medical VQA outputs of the proposed model across different modalities. 4.1 Limitations and Future Work Although this study demonstrates the impact of prompt engineering in strengthening the diagnostic reasoning abilities of medical Vision-Language Models, several limitations remain. These reflect the scope of our current experiments and suggest key areas for future work. The model was fine-tuned only on MRI-based visual question answering samples. While this allowed a focused study on prompt design and enabled generalization tests on out-of-domain modalities like CT and X-ray, the lack of modality diversity during training limits the model’s ability to adapt to other imaging types. Including multi-modal training data could improve robustness and reduce bias toward MRI-specific features. The prompts used in this work were hand-crafted and static. Despite clear gains in interpretability and reasoning quality, such fixed templates may limit flexibility across varied medical tasks or user needs. Future work could explore adaptive prompt strategies, such as instruction tuning, soft prompt learning, or reinforcement learning-based prompt generation, to support real-time adaptability. Our evaluation focused on multiple-choice VQA accuracy, which, while useful, does not fully capture clinical reasoning quality. Broader evaluation metrics, including hallucination detection, reasoning plausibility, and expert-rated interpretability, would provide a more complete performance picture. Lastly, the underlying model was pretrained on general-domain data. Further pretraining or adaptation using biomedical and radiology-specific corpora could improve alignment with clinical language and visual features, reducing errors from irrelevant associations. In summary, this work highlights the promise of prompt-based alignment for improving structure and domain specificity in medical VLMs. Future efforts should address multi-modal training, adaptive prompting, comprehensive evaluation, and clinical pretraining to build transparent, reliable AI systems for healthcare. In future work, we will consider these limitations to further improve the model and develop more effective AI systems for medical applications. 5 Conclusion In this study, we show that clinically informed prompts can significantly improve the reasoning capability and diagnostic accuracy of medical Vision-Language Models. Building upon the MedVLM-R1 framework, we introduced a radiology-specific prompt structure designed to encourage structured, step-by-step clinical reasoning. Without modifying the model architecture or optimization process, this prompt-tuned approach achieved improved performance across in-domain and out-of-domain tasks, notably surpassing the baseline model in both accuracy and interpretability. Our findings reinforce the importance of input design in medical AI systems, particularly in high-stakes settings where hallucinations, under-explained answers, and lack of clinical grounding can limit model reliability. By embedding domain expertise directly into the prompt structure, we effectively aligned the model’s output format with expert expectations while preserving transparency and consistency in reasoning traces. yellow While the results are promising, this work also highlights areas for future exploration, including multi-modality training, dynamic prompt generation, and richer interpretability frameworks. As medical VLMs continue to evolve, such prompt-aware strategies will be essential for integrating general language models into robust and secure clinical decision support frameworks. Data Availability All data used in this study are publicly available. The medical imaging data (MRI, CT, and X-ray) were obtained from the HuatuoGPT-Vision dataset, available at: https://huggingface.co/FreedomIntelligence/HuatuoGPT-Vision-7B No new data were generated in this study. https://huggingface.co/FreedomIntelligence/HuatuoGPT-Vision-7B 6 Acknowledgements This work was supported by the research program of Korea Institute of Science and Technology Information (KISTI), the Korea Bio Data Station (K-BDS) with computing resources including technical support and the Institute of Information & Communications Technology Planning & Evaluation (IITP, under grant number RS-2024-00397359). Footnotes kavya{at}aidanbio.com , ks{at}biobrain.kr References [1]. ↵ Florian Bordes , Richard Yuanzhe Pang , Anurag Ajay , Alexander C. Li , Adrien Bardes , Suzanne Petryk , Oscar Mañas , Zhiqiu Lin , Anas Mahmoud , Bargav Jayaraman , Mark Ibrahim , and Melissa Hall . An introduction to vision-language modeling , 2024 . URL https://arxiv.org/abs/2405.17247 . [2]. ↵ Beria Chingnabe Kalpelbe , Angel Gabriel Adaambiik , and Wei Peng . Vision language models in medicine , 2025 . URL https://arxiv.org/abs/2503.01863 . [3]. ↵ Jason Wei , Xuezhi Wang , Dale Schuurmans , Maarten Bosma , Brian Ichter , Fei Xia , Ed Chi , Quoc Le , and Denny Zhou . Chain-of-thought prompting elicits reasoning in large language models , 2023 . URL https://arxiv.org/abs/2201.11903 . [4]. ↵ OpenAI , Josh Achiam , Steven Adler , Sandhini Agarwal , Lama Ahmad , Ilge Akkaya , Florencia Leoni Aleman , Diogo Almeida , Janko Altenschmidt , Sam Altman , Shyamal Anadkat , Red Avila , Igor Babuschkin , Suchir Balaji , Valerie Balcom , Paul Baltescu , Haiming Bao , Mohammad Bavarian , Jeff Belgum , Irwan Bello , Jake Berdine , Gabriel Bernadett-Shapiro , and Christopher Berner . Gpt-4 technical report , 2024 . URL https://arxiv.org/abs/2303.08774 . [5]. ↵ Qwen Team . Qwq: Reflect deeply on the boundaries of the unknown . https://qwenlm.github.io/blog/ qwq-32b-preview/, November 2024 . [6]. ↵ John Schulman , Filip Wolski , Prafulla Dhariwal , Alec Radford , and Oleg Klimov . Proximal policy optimization algorithms , 2017 . URL https://arxiv.org/abs/1707.06347 . [7]. ↵ Zhihong Shao , Peiyi Wang , Qihao Zhu , Runxin Xu , Junxiao Song , Xiao Bi , Haowei Zhang , Mingchuan Zhang , Y. K. Li , Y. Wu , and Daya Guo . Deepseekmath: Pushing the limits of mathematical reasoning in open language models , 2024 . URL https://arxiv.org/abs/2402.03300 . [8]. ↵ Jiazhen Pan , Che Liu , Junde Wu , Fenglin Liu , Jiayuan Zhu , Hongwei Bran Li , Chen Chen , Cheng Ouyang , and Daniel Rueckert . Medvlm-r1: Incentivizing medical reasoning capability of vision-language models (vlms) via reinforcement learning , 2025 . URL https://arxiv.org/abs/2502.19634 . [9]. ↵ Ziwei Xu , Sanjay Jain , and Mohan Kankanhalli . Hallucination is inevitable: An innate limitation of large language models , 2025 . URL https://arxiv.org/abs/2401.11817 . [10]. ↵ Junnan Li , Dongxu Li , Caiming Xiong , and Steven Hoi . Blip: Bootstrapping language-image pre-training for unified vision-language understanding and generation , 2022 . URL https://arxiv.org/abs/2201.12086 . [11]. ↵ Jean-Baptiste Alayrac , Jeff Donahue , Pauline Luc , Antoine Miech , Iain Barr , Yana Hasson , Karel Lenc , Arthur Mensch , Katie Millican , Malcolm Reynolds , Roman Ring , Eliza Rutherford , Serkan Cabi , Tengda Han , Zhitao Gong , Sina Samangooei , Marianne Monteiro , Jacob Menick , Sebastian Borgeaud , Andrew Brock , Aida Nematzadeh , Sahand Sharifzadeh , Mikolaj Binkowski , Ricardo Barreira , Oriol Vinyals , Andrew Zisserman , and Karen Simonyan . Flamingo: a visual language model for few-shot learning , 2022 . URL https://arxiv.org/abs/2204.14198 . [12]. ↵ Haotian Liu , Chunyuan Li , Qingyang Wu , and Yong Jae Lee . Visual instruction tuning , 2023 . URL https://arxiv.org/abs/2304.08485 . [13]. ↵ Benedikt Boecking , Naoto Usuyama , Shruthi Bannur , Daniel C. Castro , Anton Schwaighofer , Stephanie Hyland , Maria Wetscherek , Tristan Naumann , Aditya Nori , Javier Alvarez-Valle , Hoifung Poon , and Ozan Oktay . Making the Most of Text Semantics to Improve Biomedical Vision–Language Processing , page 1 – 21 . Springer Nature Switzerland, 2022 . ISBN 9783031200595 . doi: 10.1007/978-3-031-20059-5_1 . URL http://dx.doi.org/10.1007/978-3-031-20059-5_1. OpenUrl CrossRef [14]. ↵ Zifeng Wang , Zhenbang Wu , Dinesh Agarwal , and Jimeng Sun . Medclip: Contrastive learning from unpaired medical images and text , 2022 . URL https://arxiv.org/abs/2210.10163 . [15]. ↵ Sheng Zhang , Yanbo Xu , Naoto Usuyama , Hanwen Xu , Jaspreet Bagga , Robert Tinn , Sam Preston , Rajesh Rao , Mu Wei , Naveen Valluri , Cliff Wong , Andrea Tupini , and Yu Wang . Biomedclip: a multimodal biomedical foundation model pretrained from fifteen million scientific image-text pairs , 2025 . URL https://arxiv.org/abs/2303.00915 . [16]. ↵ Chunyuan Li , Cliff Wong , Sheng Zhang , Naoto Usuyama , Haotian Liu , Jianwei Yang , Tristan Naumann , Hoifung Poon , and Jianfeng Gao . Llava-med: Training a large language-and-vision assistant for biomedicine in one day , 2023 . URL https://arxiv.org/abs/2306.00890 . [17]. ↵ Karan Singhal , Tao Tu , Juraj Gottweis , Rory Sayres , Ellery Wulczyn , L. Hou , Kevin Clark , Stephen Pfohl , Heather Cole-Lewis , Darlene Neal , and Mike Schaekermann . Towards expert-level medical question answering with large language models , 2023 . URL https://arxiv.org/abs/2305.09617 . [18]. ↵ Yuxiang Lai , Jike Zhong , Ming Li , Shitian Zhao , and Xiaofeng Yang . Med-r1: Reinforcement learning for generalizable medical reasoning in vision-language models , 2025 . URL https://arxiv.org/abs/2503.13939 . [19]. ↵ Jason J. Lau , Soumya Gayen , Asma Ben Abacha , and Dina Demner-Fushman . A dataset of clinically generated visual questions and answers about radiology images . Scientific Data , 5 ( 1 ): 180251 , 2018 . doi: 10.1038/sdata.2018.251 . URL https://doi.org/10.1038/sdata.2018.251 . OpenUrl CrossRef PubMed [20]. ↵ Xuehai He , Yichen Zhang , Luntian Mou , Eric Xing , and Pengtao Xie . Pathvqa: 30000+ questions for medical visual question answering , 2020 . URL https://arxiv.org/abs/2003.10286 . [21]. ↵ Long Ouyang , Jeff Wu , Xu Jiang , Diogo Almeida , Carroll L. Wainwright , Pamela Mishkin , Chong Zhang , Sandhini Agarwal , Katarina Slama , Alex Ray , John Schulman , Jacob Hilton , Fraser Kelton , Luke Miller , Maddie Simens , Amanda Askell , Peter Welinder , Paul Christiano , Jan Leike , and Ryan Lowe . Training language models to follow instructions with human feedback , 2022 . URL https://arxiv.org/abs/2203.02155 . [22]. ↵ Rafael Rafailov , Archit Sharma , Eric Mitchell , Stefano Ermon , Christopher D. Manning , and Chelsea Finn . Direct preference optimization: Your language model is secretly a reward model , 2024 . URL https://arxiv.org/abs/2305.18290 . [23]. ↵ Danfeng Guo and Demetri Terzopoulos . Prompting medical large vision-language models to diagnose pathologies by visual question answering . Machine Learning for Biomedical Imaging , 3 ( March 2024 ): 59 – 71 , March 2025. ISSN 2766-905X . doi: 10.59275/j.melba.2025-1a8b . URL http://dx.doi.org/10.59275/j.melba.2025-1a8b . OpenUrl CrossRef [24]. ↵ Fuxiao Liu , Kevin Lin , Linjie Li , Jianfeng Wang , Yaser Yacoob , and Lijuan Wang . Mitigating hallucination in large multi-modal models via robust instruction tuning , 2024 . URL https://arxiv.org/abs/2306.14565 . [25]. ↵ Peng Wang , Shuai Bai , Sinan Tan , Shijie Wang , Zhihao Fan , Jinze Bai , Keqin Chen , Xuejing Liu , Jialin Wang , Wenbin Ge , Yang Fan , Kai Dang , Mengfei Du , Xuancheng Ren , Rui Men , Dayiheng Liu , Chang Zhou , Jingren Zhou , and Junyang Lin . Qwen2-vl: Enhancing vision-language model’s perception of the world at any resolution , 2024 . URL https://arxiv.org/abs/2409.12191 . [26]. ↵ Long Ouyang , Jeff Wu , Xu Jiang , Diogo Almeida , Carroll L. Wainwright , Pamela Mishkin , Chong Zhang , Sandhini Agarwal , Katarina Slama , Alex Ray , John Schulman , Jacob Hilton , Fraser Kelton , Luke Miller , Maddie Simens , Amanda Askell , Peter Welinder , Paul Christiano , Jan Leike , and Ryan Lowe . Training language models to follow instructions with human feedback , 2022 . URL https://arxiv.org/abs/2203.02155 . [27]. ↵ DeepSeek-AI , Daya Guo , Dejian Yang , Haowei Zhang , Junxiao Song , Ruoyu Zhang , Runxin Xu , Qihao Zhu , Shirong Ma , Peiyi Wang , Xiao Bi , Xiaokang Zhang , Xingkai Yu , Yu Wu , Z. F. Wu , Zhibin Gou , Zhihong Shao , and Zhuoshu Li . Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning , 2025 . URL https://arxiv.org/abs/2501.12948 . [28]. ↵ Junying Chen , Chi Gui , Ruyi Ouyang , Anningzhe Gao , Shunian Chen , Guiming Hardy Chen , Xidong Wang , Ruifei Zhang , Zhenyang Cai , Ke Ji , Guangjun Yu , Xiang Wan , and Benyou Wang . Huatuogpt-vision, towards injecting medical visual knowledge into multimodal llms at scale , 2024 . URL https://arxiv.org/abs/2406.19280 . [29]. ↵ Xiaoman Zhang , Chaoyi Wu , Ziheng Zhao , Weixiong Lin , Ya Zhang , Yanfeng Wang , and Weidi Xie . Pmc-vqa: Visual instruction tuning for medical visual question answering , 2024 . URL https://arxiv.org/abs/2305.10415 . [30]. ↵ Bo Liu , Li-Ming Zhan , Li Xu , Lin Ma , Yan Yang , and Xiao-Ming Wu . Slake: A semantically-labeled knowledge-enhanced dataset for medical visual question answering , 2021 . URL https://arxiv.org/abs/2102.09542 . [31]. ↵ Yutao Hu , Tianbin Li , Quanfeng Lu , Wenqi Shao , Junjun He , Yu Qiao , and Ping Luo . Omnimedvqa: A new large-scale comprehensive evaluation benchmark for medical lvlm , 2024 . URL https://arxiv.org/abs/2402.09181 . View the discussion thread. Back to top Previous Next Posted August 01, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Enhancing Clinical Reasoning in Medical Vision-Language Model through Structured Prompts Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Enhancing Clinical Reasoning in Medical Vision-Language Model through Structured Prompts Kavya Dasaramoole Prakash , Kiseong Kim , Youngmahn Han medRxiv 2025.07.31.25331901; doi: https://doi.org/10.1101/2025.07.31.25331901 Share This Article: Copy Citation Tools Enhancing Clinical Reasoning in Medical Vision-Language Model through Structured Prompts Kavya Dasaramoole Prakash , Kiseong Kim , Youngmahn Han medRxiv 2025.07.31.25331901; doi: https://doi.org/10.1101/2025.07.31.25331901 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Radiology and Imaging Subject Areas All Articles Addiction Medicine (566) Allergy and Immunology (863) Anesthesia (297) Cardiovascular Medicine (4411) Dentistry and Oral Medicine (443) Dermatology (380) Emergency Medicine (606) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1505) Epidemiology (15205) Forensic Medicine (30) Gastroenterology (1119) Genetic and Genomic Medicine (6575) Geriatric Medicine (666) Health Economics (994) Health Informatics (4511) Health Policy (1365) Health Systems and Quality Improvement (1608) Hematology (537) HIV/AIDS (1263) Infectious Diseases (except HIV/AIDS) (15903) Intensive Care and Critical Care Medicine (1103) Medical Education (620) Medical Ethics (144) Nephrology (666) Neurology (6573) Nursing (345) Nutrition (998) Obstetrics and Gynecology (1139) Occupational and Environmental Health (954) Oncology (3319) Ophthalmology (968) Orthopedics (369) Otolaryngology (420) Pain Medicine (435) Palliative Medicine (129) Pathology (662) Pediatrics (1689) Pharmacology and Therapeutics (691) Primary Care Research (710) Psychiatry and Clinical Psychology (5423) Public and Global Health (9205) Radiology and Imaging (2191) Rehabilitation Medicine and Physical Therapy (1367) Respiratory Medicine (1191) Rheumatology (593) Sexual and Reproductive Health (709) Sports Medicine (529) Surgery (709) Toxicology (99) Transplantation (288) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9fec0d5b2828df88',t:'MTc3OTI4NzU2MQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();