Predicting 30-Day Mortality and Readmission Using Hospital Discharge Summaries: A Comparative Analysis of Machine Learning Models, Large Language Models, and Physicians

Predicting 30-Day Mortality and Readmission Using Hospital Discharge Summaries: A Comparative Analysis of Machine Learning Models, Large Language Models, and Physicians | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Predicting 30-Day Mortality and Readmission Using Hospital Discharge Summaries: A Comparative Analysis of Machine Learning Models, Large Language Models, and Physicians Alexander Scarlat , Francis X. Campion doi: https://doi.org/10.1101/2025.03.26.25324714 Alexander Scarlat 1 Health AI Group - Open Health Solutions, The MITRE Corporation MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: drscarlat{at}gmail.com Francis X. Campion 1 Health AI Group - Open Health Solutions, The MITRE Corporation MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract This study evaluates the comparative performance of trained machine learning models, commercial off-the-shelf (COTS) large language models (LLMs), and physicians in predicting 30-day mortality and readmission using discharge summaries from the MIMIC-IV dataset. While machine learning models demonstrated superior performance compared to both LLMs and human evaluators, none of the approaches achieved clinical-grade reliability, defined by us as a Matthews Correlation Coefficient > 0.8. Interestingly, all methods performed better at predicting mortality (3% incidence) than readmission (21.1% incidence), suggesting that the signal-to-noise ratio may be higher when predicting mortality. The universal difficulty both machines and humans encountered in these prediction tasks indicates fundamental challenges in forecasting post-discharge outcomes from discharge summaries alone. These findings highlight both the potential and current limitations of artificial intelligence in clinical prediction tasks, while emphasizing the inherent complexity of such predictions regardless of the approach used. Introduction Accurate prediction of 30-day mortality and readmission risk is crucial for clinical assessment and resource optimization [ 1 , 2 ]. Discharge summaries, which contain detailed information of patient hospital stays and care plans, represent a rich but challenging source for predictive modeling. While traditional statistical methods have shown promise, the emergence of machine learning and natural language processing has opened new possibilities for analyzing these complex clinical narratives. Historically, artificial intelligence (AI) methods for predicting readmission have used machine learning models trained on discrete data such as health insurance claims data or clinical data such as diagnosis codes, laboratory values, and vital signs from the electronic health record [ 3 , 4 ]; however, implementing such solutions is challenging and usually limited by data constraints of individual hospitals. In 2020, Rumshisky and colleagues incorporated natural language processing on electronic health record (EHR) text, combined with machine learning of discrete data to outperform human raters in predicting psychiatric readmissions, revealing the nuanced capabilities of AI in processing narrative summaries [ 5 ]. Recent studies have demonstrated the potential of more advanced machine learning, and new AI applications using Large Language Models (LLMs) to evaluate EHR text documents. Innovative open-source projects such as Meditron-70B from researchers at Yale University are promising. They have demonstrated the use of extended pre-training using a medical document corpus and fine tuning to improve performance of Llama-2 for medical question and answering tasks but have not yet published on prediction tasks [ 6 ]. However, the clinical applicability of these technologies requires thorough validation, particularly in comparison to human expert performance. In particular, clinicians may be attracted to use general purpose LLMs because of their ease-of-use, but risks related to bias, privacy, and hallucination have the potential to cause patient harm [ 7 ]. The use of general purpose LLMs for consequential classification and prediction tasks such as hospital readmission and mortality remain to be critically evaluated. In this study, we demonstrate the use of a standardized test data set of EHR discharge summaries to compare the tasks of prediction of 30-day readmission and 30-day mortality after hospitalization for adult patients. We compare performance of machine learning, neural networks, and general purpose LLMs with physician performance. Methods Data Source and Cohort Selection We used data from the Medical Information Mart for Intensive Care IV (MIMIC-IV) database, which contains 331,794 discharge summaries corresponding to 145,915 unique individuals [ 8 ]. Downstream embedding models accepted a limited maximum number of tokens, requiring the exclusion of all discharge summaries longer than 8,190 tokens. This requirement removed approximately 0.18% of available samples. We also excluded individuals who died during the index hospital admission (2% of the dataset). This yielded 323,076 discharge summaries from 141,987 unique individuals, of which the average discharge summary contained 1,672 words (99% of summaries remain under 3,987 words). In this overall cohort, mortality within 30 days of discharge was 2.94% and the 30-day readmission rate was 20.93%. From this pool of 323,076 eligible discharge summaries, we randomly selected 20,000 summaries for further analysis. These 20,000 summaries corresponded to 17,572 unique individuals. Within this 20,000-sample subset, the 30-day mortality rate was 3.0%, and the 30-day readmission rate was 21.1%. Train/Test Split by Individuals To avoid data leakage, we split the dataset at the level of the patient rather than the discharge ( Figure 1 ). Splitting by discharges would risk placing different samples from the same patient into both training and testing sets. We therefore assigned 80% of individuals to training (n = 14,057 individuals; 16,013 discharges) and 20% of individuals to testing (n = 3,515 individuals; 3,987 discharges). An individual may appear in the train or in the test subset, but not in both. Download figure Open in new tab Figure 1. Data Selection, Preprocessing, and Modeling Pipeline. Evaluation Metric: Matthews Correlation Coefficient (MCC) We selected the Matthews Correlation Coefficient (MCC) as our primary evaluation metric, utilizing all four components of the confusion matrix—true positives, false positives, true negatives, and false negatives (TP, FP, TN, FN). The MCC ranges from −1 to +1: +1 indicates perfect agreement between predictions and ground truth, 0 indicates random (or chance-level) performance, and −1 indicates complete disagreement (i.e., inverse prediction). This makes MCC especially suitable for our imbalanced dataset, as it does not overly reward correct predictions on the majority class or penalize misclassifications of the minority class. In contrast, common metrics such as the F1 score (which ignores TN) and the area under the ROC curve (which ignores both TN and FN) can be misleading when class distributions are heavily skewed. By considering each type of prediction error and success, MCC provides a more comprehensive and balanced assessment of model performance [ 9 ]. Baselines We established three simple baseline methods to compare the performance of each model: Human benchmark: Five physicians each independently evaluated 20 randomly selected discharge summaries (100 in total) from the test subset, providing two binary predictions for each case: (1) whether the patient died within 30 days of discharge and (2) whether the patient was readmitted during that interval. Majority Guessing: For each of the two binary outcomes (30-day mortality and 30-day readmission), we predicted the majority class. For instance, if 97% of patients in the test set survived past 30 days, we predicted survival for all patients; similarly, if 80% of patients were not readmitted, we predicted no readmissions for all patients. Ground-Truth Shuffling: Using the same 3,987 test subset, we shuffled the outcome labels (while preserving overall event rates) to produce another naive prediction baseline. This approach maintains the original proportion of positive and negative events but eliminates any relationship between the input features and the actual outcomes. Feature Extraction (Embedding) To transform the free-text discharge summaries into a numerical representation suitable for downstream modeling, we generated 1,536-dimensional embedding vectors using the ADA v2 model (“ada002”) from OpenAI, with no external exposure of data. This embedding step converts each discharge summary into a fixed-length numeric vector reflecting its semantic content. After embedding, we obtained 16,013 vectors for the training set and 3,987 vectors for the testing set, each vector comprising a list of 1,536 numbers. These numbers are the features extracted from the text summarizing its semantic meaning. Downstream Predictive Modeling Using the resulting 1,536-dimensional vector embeddings, we trained three supervised learning models on the training set: a Keras-based neural network, the autoML from H2O, and AWS AutoGluon frameworks. All models received the 16,013 vectors (each representing one discharge summary) as inputs. We then evaluated model performance on the reserved test set (3,987 vectorized discharge summaries). LLM Zero Shot Evaluation Additionally, we evaluated zero-shot performance of large language models (LLMs): Llama3.1 8B and 70B, Mixtral 8×7B and 8×22B directly on the text of the same 3,987-test-sample set, without performing any embedding or fine-tuning. The prompts used for each LLM were as follows: Mortality prompt “Read the discharge summary and then predict whether the patient will die within 30 days of discharge as 1, and if the patient lives as 0 . ” Readmission prompt “Read the discharge summary and then predict whether the patient is readmitted within 30 days of discharge as 1, and if the patient is not readmitted as 0 . ” Results View this table: View inline View popup Download powerpoint View this table: View inline View popup Download powerpoint Discussion This study highlights the complex challenges associated with utilizing discharge summaries to predict 30-day mortality and readmission rates. A particularly notable finding is the universal difficulty encountered by both machine learning models and human experts in achieving highly reliable predictions. Neither advanced AI systems nor experienced physicians met our threshold for reliability, defined as an MCC greater than 0.8. This underscores the inherent complexity of predicting post-discharge events using narrative summaries alone and raises questions about the adequacy of such summaries as standalone predictive tools. Supervised machine learning models consistently outperformed COTS LLMs and human evaluators, demonstrating the value of task-specific training on domain-relevant datasets for clinical prediction tasks. However, the inability of any model to reach the desired reliability threshold indicates that current ML technologies remain insufficient for autonomous application in these specific clinical decision-making contexts. An unexpected yet significant observation was the consistently superior performance in predicting mortality (3% incidence) compared to readmission (21.1% incidence) across all models and human evaluators. This counterintuitive finding, despite the lower incidence of mortality, may reflect a higher signal-to-noise ratio in the text related to mortality outcomes and warrants further exploration. It may be that the AI models and humans were able to associate particularly severe medical problems (e.g. end stage cancer) with likely mortality. However, readmission likelihood may vary with social factors such as availability of a caregiver at home, transportation factors, or other social determinants of health, which are less likely to be found in a discharge summary [ 10 ]. Limitations Several limitations of this study warrant discussion. This study was limited to open-source LLMs without fine tuning and excluded larger commercial models from providers like OpenAI, Google, and Anthropic, which may have achieved better performance. Second, the dataset used, while extensive, is limited to a single institution, and might be insufficient to fully capture the variability in discharge summaries. A larger and more diverse dataset could potentially improve model robustness and generalizability. Lastly, the physician evaluation involved only 100 discharge summaries, a significantly smaller sample than the 3,987 cases evaluated by ML models. A larger and more representative sample size for human evaluators would provide a stronger benchmark for comparison. We encourage future studies focused on the evaluation of AI applications in healthcare. In particular, there is substantial experimentation in the fine-tuning of LLMs, which could potentially enhance their predictive performance. For example, researchers at New York University have developed NYUTron, a language model trained specifically using clinical care documents and have demonstrated promising performance in specific prediction tasks [ 11 ]. Future research should also evaluate commercially available LLMs, such as those from OpenAI, Anthropic, and Google, alongside other open-source LLMs, to determine their utility for these prediction tasks as well as combination software solutions extracting discrete data from other parts of the electronic health record in combination with LLM assessment of discharge summaries. Conclusion Predicting 30-day mortality and readmission from discharge summaries remains a challenging endeavor. Despite leveraging a range of approaches, including machine learning models, COTS large language models (LLMs), and physician evaluations, none met our predefined clinical-grade reliability threshold of a Matthews Correlation Coefficient (MCC)>0.8. This limitation may be attributed to the low signal-to-noise ratio within discharge summaries for these predictive tasks or our current inability to extract this signal effectively. It is likely that additional data, not consistently found in the discharge summary, may be needed to generate reliable predictions for mortality and readmission. This study underscores the inherent complexity of predicting post-discharge outcomes. It highlights the importance of measuring AI performance with a robust metric and experimentation with diverse AI models. Data Availability All data produced in the present work are contained in the manuscript https://physionet.org/content/mimiciv/3.1/ Approved for Public Release; Distribution Unlimited. Public Release Case Number 25-0765. ©2025 The MITRE Corporation. ALL RIGHTS RESERVED References 1. ↵ Centers for Medicare & Medicaid Services . Hospital Readmissions Reduction Program (HRRP) . 2020 . 2. ↵ Jencks SF , Williams MV , Coleman EA . Rehospitalizations among patients in the Medicare fee-for-service program . N Engl J Med . 2009 ; 360 ( 14 ): 1418 – 28 . OpenUrl CrossRef PubMed Web of Science 3. ↵ Mortazavi BJ , Downing NS , Bucholz EM , et al. Analysis of machine learning techniques for heart failure readmissions . Circ Cardiovasc Qual Outcomes . 2016 ; 9 ( 6 ): 629 – 40 . OpenUrl Abstract / FREE Full Text 4. ↵ Shadmi E , Flaks-Manov N , Hoshen M , Goldman O , Bitterman H , Balicer RD . Predicting 30-day readmissions with preadmission electronic health record data . Med Care . 2015 ; 53 ( 3 ): 283 – 9 . OpenUrl CrossRef PubMed 5. ↵ Rumshisky A , Ghassemi M , Naumann T , et al. Hard for humans, hard for machines: predicting readmission after psychiatric hospitalization using narrative notes . Transl Psychiatry . 2020 ; 10 ( 1 ): 1 – 10 . OpenUrl PubMed 6. ↵ Chen Z , Hernández Cano A , Romanou A , et al. MEDITRON-70B: Scaling Medical Pretraining for Large Language Models . 2023 Nov 27. Available from: https://arxiv.org/abs/2311.16079 7. ↵ Liu J , Wang C , Liu S. Utility of ChatGPT in Clinical Practice . J Med Internet Res . 2023 ; 25 : e48568 . OpenUrl CrossRef PubMed 8. ↵ Physionet . MIMIC-IV [Internet] . Available from: https://physionet.org/content/mimiciv/3.1/ 9. ↵ Shmueli B. Matthews Correlation Coefficient is The Best Classification Metric You’ve Never Heard Of . Towards Data Science [Internet] . Available from: https://towardsdatascience.com/the-best-classification-metric-youve-never-heard-of-the-matthews-correlation-coefficient-3bf50a2f3e9a 10. ↵ Obuobi S , Chua RFM , Besser SA , Tabit CE . Social determinants of health and hospital readmissions: can the HOSPITAL risk score be improved by the inclusion of social factors? BMC Health Serv Res . 2021 ; 21 ( 1 ): 5 . doi: 10.1186/s12913-020-05989-7 . PMID: 33397379 ; PMCID: PMC7780407 . OpenUrl CrossRef PubMed 11. ↵ Beltagy I , Lo K , Cohan A. Health system-scale language models are all-purpose prediction engines . Nature . 2023 ; 621 ( 7980 ): 1247 – 54 . OpenUrl View the discussion thread. Back to top Previous Next Posted March 27, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Predicting 30-Day Mortality and Readmission Using Hospital Discharge Summaries: A Comparative Analysis of Machine Learning Models, Large Language Models, and Physicians Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Predicting 30-Day Mortality and Readmission Using Hospital Discharge Summaries: A Comparative Analysis of Machine Learning Models, Large Language Models, and Physicians Alexander Scarlat , Francis X. Campion medRxiv 2025.03.26.25324714; doi: https://doi.org/10.1101/2025.03.26.25324714 Share This Article: Copy Citation Tools Predicting 30-Day Mortality and Readmission Using Hospital Discharge Summaries: A Comparative Analysis of Machine Learning Models, Large Language Models, and Physicians Alexander Scarlat , Francis X. Campion medRxiv 2025.03.26.25324714; doi: https://doi.org/10.1101/2025.03.26.25324714 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (297) Cardiovascular Medicine (4420) Dentistry and Oral Medicine (443) Dermatology (381) Emergency Medicine (606) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1507) Epidemiology (15210) Forensic Medicine (30) Gastroenterology (1120) Genetic and Genomic Medicine (6579) Geriatric Medicine (667) Health Economics (995) Health Informatics (4516) Health Policy (1366) Health Systems and Quality Improvement (1611) Hematology (538) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15906) Intensive Care and Critical Care Medicine (1103) Medical Education (620) Medical Ethics (144) Nephrology (667) Neurology (6578) Nursing (345) Nutrition (998) Obstetrics and Gynecology (1139) Occupational and Environmental Health (956) Oncology (3322) Ophthalmology (970) Orthopedics (369) Otolaryngology (420) Pain Medicine (435) Palliative Medicine (129) Pathology (663) Pediatrics (1689) Pharmacology and Therapeutics (691) Primary Care Research (710) Psychiatry and Clinical Psychology (5428) Public and Global Health (9211) Radiology and Imaging (2192) Rehabilitation Medicine and Physical Therapy (1368) Respiratory Medicine (1194) Rheumatology (593) Sexual and Reproductive Health (709) Sports Medicine (529) Surgery (709) Toxicology (99) Transplantation (288) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9fef561e1f7bdf88',t:'MTc3OTMyMTk5OQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();