Evaluation of Large Language Models in the Clinical Management of Patients With Upper Gastrointestinal Bleeding : Insights From Real-World Patient Data

Evaluation of Large Language Models in the Clinical Management of Patients With Upper Gastrointestinal Bleeding : Insights From Real-World Patient Data | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Evaluation of Large Language Models in the Clinical Management of Patients With Upper Gastrointestinal Bleeding : Insights From Real-World Patient Data Mohsen Rajabnia , Farbod Davoodi , View ORCID Profile Mahsa Hajisafarali , Shadi Asadinia , Mahsa Shiri , Fatemeh Shirzad , Peyman Saeedi , Mahsa Mohammadi doi: https://doi.org/10.1101/2025.11.10.25339858 Mohsen Rajabnia 1 Alborz University of Medical Sciences , Karaj, Iran Find this author on Google Scholar Find this author on PubMed Search for this author on this site Farbod Davoodi 2 Department of Computer Science, Missouri University of Science and Technology , Rolla, Missouri, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Mahsa Hajisafarali 1 Alborz University of Medical Sciences , Karaj, Iran Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Mahsa Hajisafarali For correspondence: mahsa.safarali{at}gmail.com Shadi Asadinia 1 Alborz University of Medical Sciences , Karaj, Iran Find this author on Google Scholar Find this author on PubMed Search for this author on this site Mahsa Shiri 1 Alborz University of Medical Sciences , Karaj, Iran Find this author on Google Scholar Find this author on PubMed Search for this author on this site Fatemeh Shirzad 1 Alborz University of Medical Sciences , Karaj, Iran Find this author on Google Scholar Find this author on PubMed Search for this author on this site Peyman Saeedi 1 Alborz University of Medical Sciences , Karaj, Iran Find this author on Google Scholar Find this author on PubMed Search for this author on this site Mahsa Mohammadi 1 Alborz University of Medical Sciences , Karaj, Iran Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Data/Code Preview PDF ABSTRACT Objective Upper gastrointestinal bleeding (UGIB) is a life-threatening emergency requiring rapid risk assessment. Current scoring tools have limited accuracy. Large language models (LLMs) may support clinical decision-making, but their role in UGIB management is unclear. This study evaluated LLMs for patient risk classification, prediction of endoscopic findings, and alignment with routine clinical decision-making. Methods In this retrospective study, we analyzed electronic health records(EHRs) of 384 UGIB patients presented to two referral centers in Karaj, Iran, between March and December 2024. Included cases underwent upper gastrointestinal endoscopy; incomplete records were excluded. Five LLMs including GPT-5, Llama 4, Gemini-2.5-Flash, DeepSeek R1, and Grok were assessed using in-context learning for (i) risk classification, (ii) prediction of probable endoscopic findings, and (iii) clinical justification generation. Performance metrics included accuracy, precision, recall, and F1-score, compared with conventional machine learning models. Two gastroenterologists independently assessed justifications across seven domains: relevance, clarity, originality, completeness, specificity, correctness, and consistency. Results All LLMs outperformed conventional models (highest baseline accuracy 0.54). GPT-5 achieved the highest risk classification accuracy (0.66), followed by Llama 4 (0.64). Grok performed best in predicting endoscopic findings (0.32). gastroenterologists noted variability in reasoning: GPT-5 and Grok provided the most complete justifications, though GPT-5 occasionally over-classified urgent cases. Llama-4 and Gemini-2.5-Flash were less specific, while DeepSeek R1 offered detailed patient summaries but lacked predictive outputs. Conclusions LLMs improved UGIB risk prediction and generate interpretive reasoning, but accuracy limitations, inconsistent reasoning, and occasional risk misclassification highlight the need for clinician oversight and prospective validation before clinical use. What is already known on this topic UGIB is a medical emergency requiring rapid risk stratification and timely management. LLMs are promising tools for clinical decision support, but their role in UGIB management remains unclear. What this study adds LLMs can improve risk prediction and interpretive reasoning in UGIB, but limitations in accuracy, inconsistent reasoning, and occasional misclassification highlight the need for clinician oversight and prospective validation. How this study might affect research, practice, or policy LLMs provide structured, human-readable explanations that could support clinical decision-making, potentially reduce unnecessary emergency endoscopies, improve care efficiency, and alleviate physician workload. INTRODUCTION Gastrointestinal bleeding(GIB) is a common medical emergency that requires rapid risk stratification and timely management. UGIB accounts for over half of all GIB cases and is associated with mortality rates ranging from 2% to 10.( 1 ). Urgent endoscopy is often performed for suspected GIB during nights or holidays; however, in many cases, bleeding has already stopped spontaneously by the time of the procedure. Current guidelines emphasize the importance of classifying patients into high-risk groups requiring prompt intervention such as transfusions or hemostatic procedures (endoscopic, surgical, or interventional radiology) and low-risk groups who do not need emergent intervention and can be safely discharged from the emergency department for outpatient management ( 1 , 2 ). Several scoring systems have been developed and validated to predict both risk and mortality. These include the Rockall score, Glasgow-Blatchford Score (GBS), modified GBS, and AIMS65 score( 3 ). Effective risk stratification can reduce unnecessary emergency endoscopies, thereby improving the efficiency of medical care and alleviating the workload on physicians( 1 ). but the predictive performance of existing tools remains suboptimal and their application in routine clinical practice remains limited( 2 ). The rapid advancement of LLMs has driven their widespread use and demonstrated their ability to comprehend and generate complex medical information ( 4 - 6 ). Physicians are increasingly integrating these tools into clinical practice. Evidence suggests that, relative to human specialists, LLMs possess a broad understanding across multiple medical domains, underscoring their potential as versatile diagnostic aids ( 4 ). Conventional risk scores often fail to capture the complex interplay of clinical, laboratory, and endoscopic factors influencing outcomes. LLMs are simple to implement, integrate diverse data, and provide context-specific predictions with accuracy comparable to complex rule-based approaches. These features make them promising tools for free-text classification in clinical decision support( 7 , 8 ). Recent studies highlight that LLMs can play a valuable role in gastroenterology, particularly in detecting and characterizing GI bleeding. An LLM-based pipeline trained on over 17,000 nursing notes accurately identified overt and recurrent bleeding, improving coding accuracy and achieving an AUC of 0.986 for recurrence( 7 ). Another study compared LLMs with physicians in 67 complex GI cases found that Claude 3.5 Sonnet achieved the highest diagnostic coverage (76.1%), significantly outperforming gastroenterologists and those using traditional resources ( 4 ). ChatGPT was also assessed for accuracy in gastroenterology using 16 dialogue-based and 13 definition-based queries reviewed by three gastroenterologists in six quality domains. 44% and 69% of responses, respectively, were appropriate. All outputs were written at a college level but limited readability.( 5 ). Another study assessed the reliability of ChatGPT in answering patient questions about common endoscopic procedures. It gave mostly accurate guidance for routine procedures (like esophagogastroduodenoscopy and colonoscopy) but produced multiple errors for complex interventions(like endoscopic retrograde cholangiopancreatography (ERCP), and endoscopic ultrasound (EUS)) procedures ( 9 ). In an effort a domain-specific LLM, GutGPT, was trained on over 190,000 GI disease question–answer datasets derived from real-world physician–patient conversations, medical guidelines, knowledge graphs, and Chinese medical licensing examinations. Compared with 16 existing models, GutGPT demonstrated superior performance, achieving a 9.6% improvement in diagnostic accuracy in expert assessments and a 22.5% on public datasets, while also providing empathetic responses, highlighting its potential as a decision-support and patient engagement tool( 10 ). Together, these findings underscore the potential of LLMs as effective tools for both clinical decision support and the enhancement of administrative processes in gastroenterology. However, their role in the management of UGIB has not yet been systematically evaluated. Considering important limitations in accuracy, reliability, and accessibility that must be addressed before widespread clinical integration. The aim of this study was to evaluate the potential of LLMs to classify patients presenting with UGIB as high-risk requiring urgent intervention or low-risk suitable for outpatient management, and to predict probable endoscopic findings. Additionally, the study assessed the concordance of LLM-based predictions with routine clinical decision-making, as evaluated by two gastroenterologists, to determine their potential as decision-support tools in the acute care setting. METHODS Study Design This retrospective observational study was conducted at two referral centers affiliated with Alborz University of Medical Sciences in Karaj, Iran, between March 2024 and December 2024. Both centers provide 24-hour endoscopic and critical care services. The study was approved by the Institutional Ethics Committee (ethical code: IR.ABZUMS.REC.1403.202), and written informed consent was obtained from all participants. The study population included patients presenting to the emergency department with suspected UGIB defined as hematemesis, coffee-ground emesis, melena, or hematochezia of presumed upper gastrointestinal origin. Patients were eligible if they underwent upper gastrointestinal endoscopy during hospitalization. Those without endoscopy or with incomplete clinical records were excluded. From each EHR, a standardized dataset was collected, including demographics, presenting symptoms, comorbidities, lifestyle and medication history, vital signs and physical examination at admission, relevant laboratory investigations, and endoscopic findings. Data were collected independently by four researchers, and discrepancies were resolved by consensus. A total of 384 cases met the eligibility criteria and were included in the final dataset. All data were entered into a structured Microsoft Excel spreadsheet and checked for accuracy, completeness, and consistency prior to analysis. The collected variables for each patient are summarized in Table 1 . View this table: View inline View popup Table 1. Variables collected from EHRs of patients with UGIB Model Evaluation We first evaluated conventional ML models to establish baseline performance for prediction tasks. For evaluation of LLMs, a structured prompt incorporating relevant clinical and laboratory data for each case was generated. We applied in-context-learning ( 11 ) techniques, including chain-of-thought ( 12 ) and tree of thought( 13 ) to guide the models’ stepwise evaluation of clinical parameters and decision pathways. Prompts first presented the patient’s symptoms and then requested an assessment of the risk of requiring endoscopic intervention. To ensure systematic inference, model outputs were structured according to a Tree-of-Thought framework. Each prompt elicited three prespecified outputs: Endoscopy timing: classification of patients as high-risk requiring urgent intervention or low-risk eligible for outpatient management. Predicted endoscopic findings: including peptic ulcers (Forrest I–III), erosive gastropathy, esophageal varices, Mallory–Weiss tears, tumors, and other lesions. Rationale: explanation of the reasoning behind the model’s predictions. We initially evaluated several LLMs including MedGemma( 14 ), MedAlpaca( 15 ), Meditron( 16 ), MedicalBERT( 17 ), ClinicalBERT( 18 ), and BioBERT( 18 ) before selecting GPT-5( 19 ), Gemini-2.5-Flash( 20 ), Llama 4( 21 ), DeepSeek R1( 22 , 23 ), and Grok( 24 ). We excluded the former set for several reasons. First, models such as MedAlpaca, Meditron, and BERT-based variants have limited context windows, which is insufficient to process our full clinical prompts. Second, because these models generally have fewer parameters and narrower domain fine-tuning, they often fail to achieve adequate performance on metrics such as accuracy, precision, recall, and F1-score when applied to complex medical reasoning tasks. Third, some specialized medical models require computational resources that exceed the practical limits of our infrastructure. The selected models supported larger input contexts, produced higher-quality outputs, and were compatible with our computational resources. Each prompt was submitted independently to the five selected LLMs and the generated responses were recorded, annotated, and evaluated using predefined criteria for accuracy, precision, recall, and F1-score. This approach ensured reproducibility and standardized assessment of model performance across varied clinical scenarios. RESULTS Application of Conventional Machine Learning Models We initially assessed conventional ML models to establish baseline performance for the prediction tasks. Table 2 presents accuracy, F1-score, and computation time for each model. Overall, conventional models showed moderate to low predictive performance. LabelSpreading achieved the highest F1-score (0.52), while BaggingClassifier demonstrated balanced accuracy with minimal computation time. View this table: View inline View popup Table 2. Comparative Performance Metrics of Conventional ML Models LLM Performance in Risk Stratification Table 3 summarizes the performance of evaluated LLMs in binary classifying patients into high-risk and low-risk groups. GPT-5 and Llama 4 achieved the highest overall accuracy (0.66 and 0.64, respectively), indicating robust and balanced performance. Grok and DeepSeek R1 demonstrated moderate performance, while Gemini-2.5-Flash showed the lowest overall accuracy (0.40) but high recall for high-risk cases (0.89), indicating sensitivity at the expense of specificity. View this table: View inline View popup Table 3. Performance of LLMs in classifying Patients into High- and Low-Risk Groups and Multi-label prediction of endoscopic findings Multi-label prediction of endoscopic findings revealed that Grok achieved the highest exact and partial match accuracy (0.32), although F1-scores remained low across all models, reflecting the complexity of multi-label classification. These findings suggest that GPT-5 and Llama 4 are generally well-balanced, while Grok may be preferable for multi-label prediction tasks. Overall performance across accuracy, precision, recall, F1-score, exact match accuracy, and area under the curve (AUC) is presented in Figure 1 , highlighting the relative strengths and limitations of each model. Download figure Open in new tab Figure 1. Performance of evaluated large language models across key metrics, including accuracy, precision, recall, F1-score, exact match accuracy, and AUC. Model performance in predicting high- and low-risk patients is shown in Table 4 . GPT-5 and Llama 4 demonstrated the most balanced performance, with GPT-5 achieving the highest precision for high-risk patients (0.37) and Gemini-2.5-Flash the highest for low-risk patients (0.83). Gemini-2.5-Flash had high recall for high-risk patients (0.89) but low recall for low-risk patients (0.21), indicating a tendency to over-predict high-risk cases. Grok and DeepSeek R1 showed moderate, balanced performance. Specificity was generally higher for low-risk predictions, while F1-scores highlight the challenge of achieving strong performance across both classes. Overall, low-risk patients were consistently identified with higher precision, recall, and F1-scores than high-risk patients, indicating that the models were generally more effective at detecting low-risk cases. View this table: View inline View popup Download powerpoint Table 4. Binary Classification Performance of ML Models in Predicting High-Risk and Low-Risk Patients Human Justification Evaluation Results by two expert gastroenterologists assessed a randomly selected sample of 100 datas to evaluate The quality and Clinical reliability of justifications produced by Llama-4, GPT-5, Gemini-2.5-Flash, Grok, and DeepSeek R1. Justifications were evaluated across seven metrics: Relevance – alignment of the justification with the patient’s health record. Clarity – readability and comprehensibility of the text. Originality – evidence of synthesis rather than verbatim reproduction. Completeness – coverage of all relevant aspects of the patient’s record. Specificity – precision in addressing the prediction. Correctness – factual accuracy and consistency with clinical information. Consistency – logical coherence and alignment with documented reasoning. Strong concordance between human and automated assessments supported the reliability of the evaluation framework. Model-specific observations included: GPT-5: produced complete but occasionally repetitive justifications, relying on key criteria and over-classifying urgent cases when GIB co-occurred with tachycardia. Llama-4: generated short, vague, and low-specificity justifications; occasionally correct but generally weak in reasoning. Grok: provided more complete explanations with sometimes appropriate decisions, though reasoning was occasionally inconsistent or incoherent. Gemini-2.5-Flash: integrated multiple clinical features in structured justifications but frequently misclassified cases due to incorrect prioritization or overly general reasoning. DeepSeek R1: provided detailed, patient-specific summaries describing clinical features, but did not generate predictive results, and showed occasional tendencies toward overtreatment. Overall, the Human Justification Evaluation of models demonstrated variation between justification depth, reasoning consistency, and predictive accuracy. DISCUSSION In this study, we evaluated different LLMs for predicting endoscopic findings and classifying patients with UGIB into high-risk requiring urgent intervention or low-risk eligible for outpatient management using in-context-learning, chain-of-thought, and tree-of-thought techniques for prompting. We selected five LLMs—GPT-5, Llama 4, Grok, DeepSeek R1, and Gemini-2.5-Flash—based on performance and feasibility. Among the selected models, GPT-5 and Llama 4 demonstrated moderate accuracy with balanced precision, recall, and F1-scores, while Grok performed best for multi-label prediction of endoscopic findings. Conventional ML models showed limited predictive ability, highlighting the advantage of LLMs in integrating complex clinical, laboratory, and procedural data. These selected LLMs generated structured, human-readable explanations, advancing explainable AI in gastroenterology but Models outputs varied notably. GPT-5 produced the most complete justifications but was sometimes repetitive and over-classified urgent cases, while Grok offered similarly detailed explanations with less consistent reasoning. Llama-4 generated brief, vague outputs, and Gemini-2.5-Flash integrated multiple clinical features but often misclassified cases. DeepSeek R1 excelled in patient-specific detail but did not provide predictive outputs and occasionally suggested overtreatment. Overall, GPT-5 and Grok showed stronger reasoning, Llama-4 and Gemini-2.5-Flash were limited in specificity and accuracy, and DeepSeek R1 was descriptive but non-predictive. Our findings align with previous studies demonstrating the potential of LLMs to support complex medical reasoning in gastroenterology( 4 , 5 , 7 , 9 , 10 ). By synthesizing diverse clinical features, LLMs may overcome limitations of traditional scoring systems, which are often rigid and underperform in heterogeneous populations. Nonetheless, modest overall predictive accuracy and multi-label performance reveal ongoing challenges, including handling rare presentations, heterogeneous datasets, and avoiding mis-triage of high-risk patients. Limitations of this study include its retrospective design and reliance on medical records, which may be incomplete or inconsistently documented, limiting causal inference. Only patients undergoing endoscopy were included, potentially excluding milder cases and introducing selection bias. The study was conducted at two referral centers in a single region over a limited period, restricting generalizability. Automated LLM-based evaluation processes may inflate performance metrics or reveal inconsistencies between models, reflecting the trade-off between scalability and nuanced clinical assessment. Models like MedAlpaca, Meditron, and BERT variants were excluded due to limited context, narrower domain tuning, and high computational demands. Ethical considerations are essential when applying LLMs in clinical practice. Models trained on large datasets may contain inherent biases—legal, technical, or linguistic—which can influence generated explanations and decision-making, potentially compromising fairness and accuracy. LLMs may also produce unreal or factually incorrect outputs, potentially leading to misinterpretation of patient records. Automated predictions should therefore complement, not replace, human expertise, with clinicians remaining central to verifying and interpreting AI-generated insights. Future research should focus on prospective validation, integration with electronic health records, domain-specific fine-tuning, and evaluation of effects on patient outcomes, workflow efficiency, and healthcare resource utilization. In conclusion, LLMs demonstrate promising potential for risk stratification and endoscopic prediction in patients with UGIB, offering structured, interpretable explanations that surpass traditional scoring systems in handling complex clinical data. All models, however, exhibited limitations in accuracy, reasoning consistency, and risk misclassification. While these findings support the complementary use of LLMs in gastroenterology, human expertise remains essential, and further prospective validation and domain-specific optimization are needed to ensure safe and effective clinical implementation. Data Availability All data produced in the present study are available upon reasonable request to the authors AI use declaration ChatGPT (OpenAI, GPT-5) was used solely for language editing, including improving grammar, clarity, and style. The authors reviewed and approved all AI-generated suggestions, and the final content reflects their own interpretation and conclusions. ACKNOWLEDGEMENTS The authors gratefully acknowledge the support of the Cardiovascular Research Center (CRC) at Alborz University of Medical Sciences and Imam Ali Hospital, Karaj, Iran. We also thank the study participants for providing their data. The authors thank Dr Bahar Seifi for her valuable assistance in the preparation of this manuscript. Footnotes Email: dr.rajabnia{at}outlook.com Email: farboddavoodi{at}mst.edu Email: shadiasadinia8{at}gmail.com Email: mahsashiri20{at}gmail.com Email: fatemehshirzad01{at}gmail.com Email: peyman.saeedi.md{at}gmail.com Email: dr.mahsa.mohammadi1989{at}gmail.com Declarations: Funding: None Ethics Approval: This study was approved by the Institutional Ethics Committee of Alborz University of Medical Sciences (Ethical code: IR.ABZUMS.REC.1403.202). All patient data used in this study were fully anonymized. No identifiable personal information was included, and the study was conducted in accordance with the ethical standards of Alborz University of Medical Sciences. Patient Consent: Written informed consent was obtained from all participants. Competing Interests: The authors declare no competing interests. REFERENCES 1. ↵ Sasaki Y , Abe T , Kawamura N , Keitoku T , Shibata I , Ohno S , et al. Prediction of the need for emergency endoscopic treatment for upper gastrointestinal bleeding and new score model: a retrospective study . BMC Gastroenterology . 2022 ; 22 ( 1 ): 337 . OpenUrl PubMed 2. ↵ Shung DL , Au B , Taylor RA , Tay JK , Laursen SB , Stanley AJ , et al. Validation of a Machine Learning Model That Outperforms Clinical Risk Scoring Systems for Upper Gastrointestinal Bleeding . Gastroenterology . 2020 ; 158 ( 1 ): 160 – 7 . OpenUrl CrossRef PubMed 3. ↵ Deshmukh F , Merchant SS . Explainable machine learning model for predicting GI bleed mortality in the intensive care unit . Official journal of the American College of Gastroenterology| ACG . 2020 ; 115 ( 10 ): 1657 – 68 . OpenUrl 4. ↵ Yang X , Li T , Wang H , Zhang R , Ni Z , Liu N , et al. Multiple large language models versus experienced physicians in diagnosing challenging cases with gastrointestinal symptoms . npj Digital Medicine . 2025 ; 8 ( 1 ): 85 . OpenUrl PubMed 5. ↵ Toiv A , Saleh Z , Ishak A , Alsheik E , Venkat D , Nandi N , et al. Digesting Digital Health: A Study of Appropriateness and Readability of ChatGPT-Generated Gastroenterological Information . Clinical and Translational Gastroenterology . 2024 ; 15 ( 11 ): e00765 . OpenUrl 6. ↵ Cankurtaran RE , Polat YH , Aydemir NG , Umay E , Yurekli OT . Reliability and usefulness of ChatGPT for inflammatory bowel diseases: an analysis for patients and healthcare professionals . Cureus . 2023 ; 15 ( 10 ). 7. ↵ Zheng NS , Keloth VK , You K , Kats D , Li DK , Deshpande O , et al. Detection of Gastrointestinal Bleeding With Large Language Models to Aid Quality Improvement and Appropriate Reimbursement . Gastroenterology . 2025 ; 168 ( 1 ): 111 - 20.e4 . OpenUrl PubMed 8. ↵ Johnson B , Bath T , Huang X , Lamm M , Earles A , Eddington H , et al. Large language models for extracting histopathologic diagnoses of colorectal cancer and dysplasia from electronic health records . BMJ Open Gastroenterology . 2025 ; 12 ( 1 ): e001896 . OpenUrl Abstract / FREE Full Text 9. ↵ Lim DYZ , Sng GGR , Tung JYM , Tan DMY , Tan C-K. ChatGPT for advice on common GI endoscopic procedures: the promise and the peril . iGIE . 2023 ; 2 ( 4 ): 547 - 53.e26 . OpenUrl 10. ↵ Zhang R-Y , Qiang P-P , Hao Y-X , Tan H-Y , Zhao K , Cai L-J , et al. GutGPT: A multidimensional knowledge-enhanced large language model for gastrointestinal medicine . Journal of Biomedical Informatics . 2025 ; 169 : 104885 . OpenUrl PubMed 11. ↵ Lee SA , Wu A , Chiang JN . Clinical modernbert: An efficient and long context encoder for biomedical text . arXiv preprint arxiv: 250403964 . 2025 . 12. ↵ Wei J , Wang X , Schuurmans D , Bosma M , Xia F , Chi E , et al. Chain-of-thought prompting elicits reasoning in large language models . Advances in neural information processing systems . 2022 ; 35 : 24824 – 37 . OpenUrl 13. ↵ Yao S , Yu D , Zhao J , Shafran I , Griffiths TL , Cao Y , et al. Tree of thoughts: Deliberate problem solving with large language models, 2023 . URL https://arxivorg/abs/230510601 . 2023 ; 3 : 1 . OpenUrl 14. ↵ Sellergren A , Kazemzadeh S , Jaroensri T , Kiraly A , Traverse M , Kohlberger T , et al. Medgemma technical report . arXiv preprint arxiv: 250705201 . 2025 . 15. ↵ Han T , Adams LC , Papaioannou J-M , Grundmann P , Oberhauser T , Löser A , et al. MedAlpaca--an open-source collection of medical conversational AI models and training data . arXiv preprint arxiv: 230408247 . 2023 . 16. ↵ Chen Z , Cano AH , Romanou A , Bonnet A , Matoba K , Salvi F , et al. Meditron-70b: Scaling medical pretraining for large language models . arXiv preprint arxiv: 231116079 . 2023 . 17. ↵ Rasmy L , Xiang Y , Xie Z , Tao C , Zhi D. Med-BERT: pretrained contextualized embeddings on large-scale structured electronic health records for disease prediction . NPJ digital medicine . 2021 ; 4 ( 1 ): 86 . OpenUrl CrossRef PubMed 18. ↵ Lee J , Yoon W , Kim S , Kim D , Kim S , So CH , et al. BioBERT: a pre-trained biomedical language representation model for biomedical text mining . Bioinformatics . 2020 ; 36 ( 4 ): 1234 – 40 . OpenUrl CrossRef PubMed 19. ↵ Wang S , Hu M , Li Q , Safari M , Yang X. Capabilities of gpt-5 on multimodal medical reasoning . arXiv preprint arxiv: 250808224 . 2025 . 20. ↵ Team G , Anil R , Borgeaud S , Alayrac J-B , Yu J , Soricut R , et al. Gemini: a family of highly capable multimodal models . arXiv preprint arxiv: 231211805 . 2023 . 21. ↵ Touvron H , Lavril T , Izacard G , Martinet X , Lachaux M-A , Lacroix T , et al. Llama: Open and efficient foundation language models . arXiv preprint arxiv: 230213971 . 2023 . 22. ↵ Liu A , Feng B , Xue B , Wang B , Wu B , Lu C , et al. Deepseek-v3 technical report . arXiv preprint arxiv: 241219437 . 2024 . 23. ↵ Guo D , Yang D , Zhang H , Song J , Zhang R , Xu R , et al. Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning . arXiv preprint arxiv: 250112948 . 2025 . 24. ↵ Lv A , Xie R , Sun X , Kang Z , Yan R. Language models” grok” to copy . arXiv preprint arxiv: 240909281 . 2024 . View the discussion thread. Back to top Previous Next Posted November 13, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Evaluation of Large Language Models in the Clinical Management of Patients With Upper Gastrointestinal Bleeding : Insights From Real-World Patient Data Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Evaluation of Large Language Models in the Clinical Management of Patients With Upper Gastrointestinal Bleeding : Insights From Real-World Patient Data Mohsen Rajabnia , Farbod Davoodi , Mahsa Hajisafarali , Shadi Asadinia , Mahsa Shiri , Fatemeh Shirzad , Peyman Saeedi , Mahsa Mohammadi medRxiv 2025.11.10.25339858; doi: https://doi.org/10.1101/2025.11.10.25339858 Share This Article: Copy Citation Tools Evaluation of Large Language Models in the Clinical Management of Patients With Upper Gastrointestinal Bleeding : Insights From Real-World Patient Data Mohsen Rajabnia , Farbod Davoodi , Mahsa Hajisafarali , Shadi Asadinia , Mahsa Shiri , Fatemeh Shirzad , Peyman Saeedi , Mahsa Mohammadi medRxiv 2025.11.10.25339858; doi: https://doi.org/10.1101/2025.11.10.25339858 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Gastroenterology Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15229) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6600) Geriatric Medicine (668) Health Economics (997) Health Informatics (4536) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (541) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3332) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9232) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a00bf9bd6860df88',t:'MTc3OTYyMjMwMg=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();