Evaluating Large Language Models for Automatic Detection of In-Hospital Cardiac Arrest: Multi-Site Analysis of Clinical Notes

Evaluating Large Language Models for Automatic Detection of In-Hospital Cardiac Arrest: Multi-Site Analysis of Clinical Notes | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Evaluating Large Language Models for Automatic Detection of In-Hospital Cardiac Arrest: Multi-Site Analysis of Clinical Notes View ORCID Profile Uğurcan Vurgun , View ORCID Profile Aarthi Kaviyarasu , View ORCID Profile Sy Hwang , View ORCID Profile Ashley Batugo , Sunil Thomas , View ORCID Profile Brandon Tang , View ORCID Profile Ana Acevedo , View ORCID Profile Oscar Mitchell , View ORCID Profile Danielle L. Mowery doi: https://doi.org/10.1101/2025.08.04.25331524 Uğurcan Vurgun 1 Department of Biostatistics, Epidemiology, & Informatics; Department of Emergency Medicine, Division of Pulmonary, Allergy, and Critical Care, Center for Resuscitation Science University of Pennsylvania Philadephia , PA, 19104, United States Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Uğurcan Vurgun Aarthi Kaviyarasu 1 Department of Biostatistics, Epidemiology, & Informatics; Department of Emergency Medicine, Division of Pulmonary, Allergy, and Critical Care, Center for Resuscitation Science University of Pennsylvania Philadephia , PA, 19104, United States Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Aarthi Kaviyarasu Sy Hwang 1 Department of Biostatistics, Epidemiology, & Informatics; Department of Emergency Medicine, Division of Pulmonary, Allergy, and Critical Care, Center for Resuscitation Science University of Pennsylvania Philadephia , PA, 19104, United States Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Sy Hwang Ashley Batugo 1 Department of Biostatistics, Epidemiology, & Informatics; Department of Emergency Medicine, Division of Pulmonary, Allergy, and Critical Care, Center for Resuscitation Science University of Pennsylvania Philadephia , PA, 19104, United States Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Ashley Batugo Sunil Thomas 1 Department of Biostatistics, Epidemiology, & Informatics; Department of Emergency Medicine, Division of Pulmonary, Allergy, and Critical Care, Center for Resuscitation Science University of Pennsylvania Philadephia , PA, 19104, United States Find this author on Google Scholar Find this author on PubMed Search for this author on this site Brandon Tang 1 Department of Biostatistics, Epidemiology, & Informatics; Department of Emergency Medicine, Division of Pulmonary, Allergy, and Critical Care, Center for Resuscitation Science University of Pennsylvania Philadephia , PA, 19104, United States Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Brandon Tang Ana Acevedo 1 Department of Biostatistics, Epidemiology, & Informatics; Department of Emergency Medicine, Division of Pulmonary, Allergy, and Critical Care, Center for Resuscitation Science University of Pennsylvania Philadephia , PA, 19104, United States Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Ana Acevedo Oscar Mitchell 1 Department of Biostatistics, Epidemiology, & Informatics; Department of Emergency Medicine, Division of Pulmonary, Allergy, and Critical Care, Center for Resuscitation Science University of Pennsylvania Philadephia , PA, 19104, United States Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Oscar Mitchell Danielle L. Mowery 1 Department of Biostatistics, Epidemiology, & Informatics; Department of Emergency Medicine, Division of Pulmonary, Allergy, and Critical Care, Center for Resuscitation Science University of Pennsylvania Philadephia , PA, 19104, United States Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Danielle L. Mowery For correspondence: dlmowery{at}pennmedicine.upenn.edu Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract In-hospital cardiac arrest (IHCA) affects over 200,000 patients annually in the United States, yet its detection through manual chart review remains resource-intensive and often delayed. We evaluated the performance of four open-source large language models (LLMs) and GPT-4o in identifying IHCA cases from 2,674 clinical notes across five hospitals. While GPT-4o achieved the highest performance (F1-score: 0.90, recall: 0.97), several open-source models demonstrated comparable capabilities, suggesting their viability for clinical applications. Our systematic analysis of model outputs revealed that performance was strongly influenced by site-specific documentation practices, with inter-site agreement rates varying by over 20%. Through detailed error analysis, we identified key challenges including medical terminology hallucinations and structural inconsistencies in model reasoning. These findings establish a framework for implementing LLM-based IHCA detection systems while highlighting critical considerations for their clinical deployment. 1. Introduction Accurate and timely reporting of IHCA events is crucial to facilitate quality improvement (QI) initiatives, such as reviewing cardiopulmonary resuscitation (CPR) quality, optimizing team performance, and benchmarking IHCA outcomes. Although care teams can readily recognize IHCA during the event itself, subsequent reporting of these events after the event often lacks consistency and poses a major barrier to QI and research efforts. Reliance on administrative billing codes or human-driven processes lacks precision, can be costly, and can delay or miss critical data collection. 1 , 2 This issue is particularly acute during the night and weekends, when nursing managers and other key champions may not be available, and during periods of staff shortages and turnover. LLMs provide capabilities for interpreting unstructured clinical text, reducing human variability, and streamlining data extraction. 3 Yet, their validity in critical care depends on rigorous testing of accuracy, robustness, and instruction adherence. 2. Related Work Traditional IHCA identification methods rely on billing codes, such as the International Classification of Diseases (ICD), or manually-created QI and research registries, which often lack timeliness and accuracy. 1 , 2 Previous works found that using ICD-9 codes to identify cardiac arrest patients lacked both sensitivity and specificity, with one large registry study reporting that over 36% of IHCA events were not identified through billing and procedure code analyses. 1 , 2 Consistent with these works, our group found the use of ICD-10 codes to have low sensitivity for the identification of IHCA events from a QI database, with 32% of events being missed, 4 demonstrating a need for novel approaches to accurately identify IHCA and facilitate research and QI efforts. Recent advancements highlight the potential of AI in medical text analysis, including the use of LLMs to interpret complex notes. 5 , 6 Luo et al. 7 introduced BioGPT, a generative model pre-trained on biomedical corpora, and had shown promise in extracting nuanced clinical information and summarizing patient records. Singhal and colleagues 8 demonstrated that foundational LLMs could effectively disambiguate medical terminology without the need for domain-specific fine-tuning. Rapid advancements have enabled practical applications across various medical domains, including identifying within clinical narratives cognitive decline in the elderly, 9 headache frequency in neurology consultations, 10 and sleep patterns in Alzheimer’s disease patients. 11 In this pilot study, we conducted a summative evaluation to assess how well LLMs could discern positive IHCA cases from negative IHCA cases and a formative evaluation to understand conformance with IHCA criterion. 3. Methodology 3.1. Data Collection and Preprocessing We collected 2,674 clinical notes from 684 patients from 5 hospitals from southeastern Pennsylvania, emphasizing discharge summaries, progress notes, plan-of-care notes, and assessments. To protect patient privacy, notes were de-identified using PHIlter. 12 Domain experts annotated each note- and patient-level labels to identify features relevant to IHCA, including patient demographics, comorbidities, vital signs, and events preceding the arrest. Double annotation with adjudication by a senior clinician ensured labeling consistency. Patient-level labels were assigned as IHCA-positive when there was documented evidence of a cardiac arrest occurring during an inpatient hospital stay, requiring chest compressions or advanced cardiac life support interventions. Cases were labeled as IHCA-negative when there was no documented cardiac arrest or when cardiac arrests occurred in outpatient settings such as clinics or emergency departments prior to admission. For model evaluation, we provided structured instructions requiring verification of three hierarchical criteria: (1) explicit documentation of cardiac arrest, (2) confirmation that the event occurred in an inpatient setting after admission, and (3) documentation of resuscitative interventions including chest compressions or advanced cardiovascular life support (ACLS) protocols. Note-level labels were derived from patient-level annotations, with notes documenting the IHCA event or its immediate aftermath labeled as positive, enabling temporal analysis of documentation patterns. 3.2. Model Selection and Evaluation We evaluated five LLMs: GPT-4o and four open-source alternatives (Mistral-Nemo-Instruct-2407, Meta-Llama-3.1-8B-Instruct, princeton-nlp/gemma-2-9b-it-SimPO, and THUDM/glm-4-9b-chat). All models were applied with their base configuration without additional fine-tuning, employing a zero-shot learning approach with chain-of-thought prompting. Models were evaluated using an identical prompt and generation parameters (temperature=0.2) to ensure fair comparison: Given the patient’s notes summary, assess whether the patient had an in-hospital cardiac arrest (IHCA) event. IHCA refers to a cardiac arrest that occurs in an inpatient setting where the patient is admitted and receiving care. Cardiac arrests occurring in outpatient settings, such as the hospital’s cafeteria or during a visit, do not count as IHCA. Cardiac arrest refers to the sudden loss of cardiac function leading to a lack of a pulse or signs requiring that the patient receives chest compressions, or ACLS. Steps to determine IHCA: Identify if there is a mention of cardiac arrest. Determine if the cardiac arrest occurred in an inpatient setting. Confirm if the patient received chest compressions. Respond with ‘positive’ or ‘negative’ and give your rationale. 3.3. Summative Evaluation Metrics To determine how well LLMs discerned IHCA positive from IHCA negative cases, we evaluated model performance using standard binary classification metrics at the patient-level: F1-score, balanced accuracy, precision, and recall. To assess statistical robustness, we computed 95% confidence intervals using non-parametric bootstrapping with 100,000 resamples. Additionally, we measured instruction prompt following performance through a binary assessment of each model response: responses were considered compliant if they (1) began with an explicit ‘positive’ or ‘negative’ classification and (2) provided a rationale referencing the required IHCA criteria. We aggregated these assessments into an instruction following score, calculated as the proportion of responses meeting both criteria across all cases. 3.4. Formative Evaluation Metrics To better understand the rationale of IHCA classification and conformance with IHCA criterion, we randomly sampled and manual reviewed up to 5 patients and their subsequent 4 notes from each of the concordant (true positives and negatives) and dis-concordant (false positives and negatives) cells of the contingency matrix from each of the 5 hospitals. Three reviewers with clinical expertise (AK, BT, AA) evaluated the reliability of the responses from the LLM at a note-level using criterion of coverage, coherence, relevance , and hallucinations defined in Table 1 . Coverage and coherence were measured by 5-point Likert scales; relevance were measured by a 3-point scale. Review and evaluation was conducted through REDCap, an encrypted, HIPAA-compliant, and hospital-approved data management software. 13 View this table: View inline View popup Download powerpoint Table 1. Evaluation criteria and scales used by annotators. 4. Results We evaluated five LLMs on 2,674 clinical notes from 684 unique patients. Each patient contributed up to four notes, primarily discharge summaries, progress notes, plan-of-care notes, and assessments, chosen for their frequent documentation of IHCA events. 4.1. Summative Evaluation: LLM Classification at Patient-level Figure 1 summarizes each model’s performance metrics. GPT-4o obtained the highest F1-score (0.90 ± 0.01), along with a balanced accuracy of 0.90 ± 0.01 and a 0.97 recall for IHCA cases. Several open-source LLMs exhibited competitive performance: Mistral-Nemo-Instruct-2407 (F1: 0.84 ± 0.05), princeton-nlp/gemma-2-9b-it-SimPO (F1: 0.82 ± 0.07), THUDM/glm-4-9b-chat (F1: 0.79 ± 0.00), and Meta-Llama-3.1-8B-Instruct (F1: 0.79 ± 0.05). Confidence intervals were derived through non-parametric bootstrapping with 100,000 resamples to assess robustness. Download figure Open in new tab Fig. 1. Performance comparison of five LLMs on IHCA detection, including F1-score, precision, and recall with confidence intervals. 4.2. Formative Evaluation: Classification and Rationale Subanalysis on Note-level To evaluate not just binary predictions but also rationale quality, focusing on coverage, coherence, relevance, and hallucination detection, we selected a subset of 256 clinical notes from 64 patients across five hospitals from the Llama-3.1-8B-Instructmodel for annotation. We selected this model because of its high F1-score and high instruction following capabilities which leads itself nicely to a human-in-the-loop review model. Each clinical note was evaluated independent of any classifications at the patient-level, as well as the other three notes from the same patient record. Table 1 shows the criteria and the evaluation scales adapted from Sudeshna et al.’s study. 14 The analyzed notes predominantly comprised of progress notes (67.2%, n=172) and discharge summaries (26.6%, n=68). Figure 2 illustrates the distribution of IHCA indicators in these clinical notes. Explicit cardiac arrest mentions were most frequent, followed by inpatient setting documentation. Chest compression documentation appeared less commonly, while a small fraction of notes contained no clear IHCA indicators. Download figure Open in new tab Fig. 2. Distribution of IHCA indicators in clinical notes from the annotated dataset. Figure 3 shows the agreement rates between human observations and model inferences for key IHCA indicators. The capacity of LLMs to accurately interpret IHCA indicators varied, with high detection rates for explicit cardiac arrest mentions ( > 90%), but lower agreement for inpatient status (55%) and chest compressions (38%). Overall agreement rates also varied substantially across hospital sites: Sites 2 and 3 achieved 71.4% agreement, Site 4 70.0%, Site 5 53.8%, and Site 1 only 50.0% (not shown). Download figure Open in new tab Fig. 3. Agreement rates between human observations and model inferences for key IHCA indicators. Figure 4 displays the distribution of coverage and coherence ratings across reviewed model outputs. Here, coverage indicates the model’s ability to capture critical IHCA-related information, and coherence reflects the readability and structural quality of the outputs. Most cases scored 5 indicating high coverage of criteria (83.6%) and readability (85.9%). Download figure Open in new tab Fig. 4. Distribution of coverage and coherence ratings across reviewed model outputs. Figure 5 presents the hallucination rates across the three most common clinical note types. Hallucination rates varied by note type, with plan of care showing the highest rate (37.5%), discharge summaries the second highest (19.1%), and progress notes the lowest (12.8%). Download figure Open in new tab Fig. 5. Analysis of hallucination rates across the three most common clinical note types. 5. Discussion 5.1. Summative Evaluation: LLM Classification at Patient-level When selecting an LLM for operations, its important to not only consider classification accuracy, but also costs associated with deployment i.e., GPT-4o costs are calculated by tokens which can be computationally expensive compared to open-weight models. Although GPT-4o produced the highest performance, open-weight LLMs could produce comparable or even better recall or precision. 5.2. Formative Evaluation: Classification and Rationale Subanalysis on Note-level Rationales are critical to explainability and trust in LLM predictions. The feedback notes written by our expert annotators further reveal several error types in the model’s outputs. These errors are summarized in Table 2 . View this table: View inline View popup Download powerpoint Table 2. Summary of error types from Meta-Llama-3.1-8B-Instruct. Our error analysis revealed important implementation challenges that must be addressed. The significant variation in inter-site agreement rates (>20% difference) highlights how site-specific documentation practices influence model performance. We identified critical error patterns, including medical terminology hallucinations, unsubstantiated clinical details, and misinterpretation of negative statements, with varying rates across different note types. The identified error patterns have significant implications for clinical implementation of LLM-based IHCA detection systems. The models’ tendency to hallucinate medical terminology and mis-interpret negative statements could lead to both false positives (e.g., incorrectly identifying chest compressions) and false negatives (e.g., missing actual IHCA events due to misinterpreted documentation). Moreover, the fabrication of clinical details like ICU transfers and medication dosages suggests that these models require robust verification mechanisms before deployment. Particularly concerning is the models’ difficulty in distinguishing between different types of resuscitation events and their context, which could impact the accuracy of quality improvement initiatives. These findings emphasize the necessity for a multi-layered verification system: (1) implementation of medical terminology-aware prompt engineering to reduce hallucinations, (2) development of specialized verification steps for high-stakes clinical assertions like CPR status and ICU transfers, and (3) maintenance of human oversight for final validation of IHCA classifications. 15 To ensure that the LLM/AI system follows the AI Bill of Rights and provides safe and effective clinical decision support, 15 we advocate for implementation using a “human-in-the-loop” paradigm. A QI chart reviewer expert would be presented with the LLM predictions and rationales and original narratives to then either accept or reject the classification. In addition, we would provide periodic fine-tuning and assessment of the LLM performance across hospital sites and populations. From our formative error analysis, we learned that some note types are prone to phenomenon like hallucinations. To mitigate the impact of these types of errors, we will explore providing more information about common terminology and treatments associated with IHCA and facilitating use of few-shot learning to reduce these errors. 6. Limitations and Future Work We identified potential areas of improvement. Specifically, we noted 235 structural artifacts, including abrupt narrative endings and repetitive PHI redactions. The 21.4% difference between the highest and lowest agreement rates confirm significant variability in model performance across sites, highlighting the need for standardized documentation and site-specific optimization. 7. Conclusion Our systematic evaluation demonstrates the potential of LLMs to transform IHCA detection in clinical settings. Although GPT-4o achieved the highest performance (F1: 0.90, recall: 0.97), several open-source alternatives showed comparable capabilities, suggesting the viability of cost-effective solutions for clinical deployment. However, significant challenges in validity must be addressed before LLM-based detection systems could enable more timely and accurate IHCA identification, supporting quality improvement initiatives and enhanced post-arrest care delivery. Data Availability Only coded data in the present study are available upon reasonable request to the authors Disclosure of Interests The authors declare no competing interests relevant to this article. Acknowledgment This work was funded by the Zoll Foundation and approved by the University of Pennsylvania Institute Review Board. Preprint of an article submitted for consideration in Pacific Symposium on Biocomputing © 2026 World Scientific Publishing Co., Singapore, http://psb.stanford.edu/ Footnotes Emails: firstname.lastname{at}pennmedicine.upenn.edu ↵ * joint co-last authorship References 1. ↵ R. Khera , J. A. Spertus , M. A. Starks and et al. , Administrative Codes for Capturing In-Hospital Cardiac Arrest , JAMA Cardiology 2 , 1275 ( 2017 ). OpenUrl PubMed 2. ↵ C. DeZorzi , B. Boyle , A. Qazi , K. Luthra , R. Khera , P. S. Chan and S. Girotra , Administrative billing codes for identifying patients with cardiac arrest , Journal of the American College of Cardiology 73 , 1598 ( 2019 ). OpenUrl CrossRef PubMed 3. ↵ A. J. Thirunavukarasu , D. S. J. Ting , K. Elangovan , L. Gutierrez , T. F. Tan and D. S. W. Ting , Large language models in medicine , Nature Medicine 29 , 1930 ( 2023 ). OpenUrl CrossRef PubMed 4. ↵ A. Kaviyarasu , O. J. Mitchell , S. Thomas , A. Batugo , D. L. Mowery and B. S. Abella , Abstract 328: Identification of in-hospital cardiac arrest using administrative billing codes , Circulation 148 , A328 ( 2023 ). OpenUrl 5. ↵ Y. Li , S. Rao , J. R. A. Solares , A. Hassaine , R. Ramakrishnan , D. Canoy , Y. Zhu , K. Rahimi and G. Salimi-Khorshidi , BEHRT: Transformer for electronic health records , Scientific Reports 10 , 1 ( 2020 ). OpenUrl PubMed 6. ↵ L. Rasmy , Y. Xiang , Z. Xie , C. Tao and D. Zhi , Med-BERT: Pretrained contextualized embeddings on large-scale structured electronic health records for disease prediction , NPJ Digital Medicine 4 , 1 ( 2021 ). OpenUrl CrossRef PubMed 7. ↵ R. Luo , L. Sun , Y. Xia , T. Qin , S. Zhang , H. Poon and T.-Y. Liu , BioGPT: generative pre-trained transformer for biomedical text generation and mining , Briefings in Bioinformatics 23 , p. bbac409 ( 2022 ). OpenUrl CrossRef PubMed 8. ↵ K. Singhal , S. Azizi , T. Tu , S. S. Mahdavi , J. Wei , H. W. Chung , N. Scales , A. Tanwani , H. Cole-Lewis , S. Pfohl et al. , Large language models encode clinical knowledge , Nature 620 , 172 ( 2023 ). OpenUrl CrossRef PubMed 9. ↵ X. Du , J. Novoa-Laurentiev , J. M. Plasek , Y.-W. Chuang , L. Wang , G. A. Marshall , S. K. Mueller , F. Chang , S. Datta , H. Paek et al. , Enhancing early detection of cognitive decline in the elderly: a comparative study utilizing large language models in clinical notes , EBioMedicine 109 ( 2024 ). 10. ↵ C.-C. Chiang , M. Luo , G. Dumkrieger , S. Trivedi , Y.-C. Chen , C.-J. Chao , T. J. Schwedt , A. Sarker and I. Banerjee , A large language model–based generative natural language processing framework fine-tuned on clinical notes accurately extracts headache frequency from electronic health records , Headache: The Journal of Head and Face Pain 64 , 400 ( 2024 ). OpenUrl 11. ↵ S. Sivarajkumar , T. Y. C. Tam , H. A. Mohammad , S. Viggiano , D. Oniani , S. Visweswaran and Y. Wang , Extraction of sleep information from clinical notes of Alzheimer’s disease patients using natural language processing , Journal of the American Medical Informatics Association 31 , 2217 ( 2024 ). OpenUrl PubMed 12. ↵ B. Norgeot , K. Muenzen , T. A. Peterson , X. Fan , B. S. Glicksberg , G. Schenk , E. Rutenberg , Oskotsky , M. Sirota , J. Yazdany et al. , Protected Health Information filter (Philter): Accurately and securely de-identifying free-text clinical notes , NPJ Digital Medicine 3 , p. 57 ( 2020 ). OpenUrl PubMed 13. ↵ S. Wiberg , M. J. Holmberg , M. W. Donnino , J. Kjaergaard , C. Hassager , L. Witten , K. M. Berg , A. Moskowitz , L. W. Andersen , A. Grossestreuer , D. Edelson , J. Ornato , M. A. Peberdy , M. Churpek , M. Kurz , M. Anderson Starks , P. Chan , S. Girotra , S. Perman and Z. Goldberger , Age-dependent trends in survival after adult in-hospital cardiac arrest , Resuscitation 151 , 189 ( 2020 ). OpenUrl PubMed 14. ↵ S. Das , Y. Ge , Y. Guo , S. Rajwal , J. Hairston , J. Powell , D. Walker , S. Peddireddy , S. Lakamana , S. Bozkurt , M. Reyna , R. Sameni , Y. Xiao , S. Kim , R. Chandler , N. Hernandez , D. Mowery , R. Wightman , J. Love , A. Spadaro , J. Perrone and A. Sarker , Two-layer Retrieval-Augmented Generation framework for low-resource medical question answering using Reddit data: Proof-of-Concept study , J Med Internet Res 27 , p. e66220 ( Jan 2025 ). OpenUrl PubMed 15. ↵ Office of Science, Technology, Policy - The White House, Blueprint for an AI Bill of Rights: Making Automated Systems Work for the American People . ( October 2022 ). View the discussion thread. Back to top Previous Next Posted August 06, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Evaluating Large Language Models for Automatic Detection of In-Hospital Cardiac Arrest: Multi-Site Analysis of Clinical Notes Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Evaluating Large Language Models for Automatic Detection of In-Hospital Cardiac Arrest: Multi-Site Analysis of Clinical Notes Uğurcan Vurgun , Aarthi Kaviyarasu , Sy Hwang , Ashley Batugo , Sunil Thomas , Brandon Tang , Ana Acevedo , Oscar Mitchell , Danielle L. Mowery medRxiv 2025.08.04.25331524; doi: https://doi.org/10.1101/2025.08.04.25331524 Share This Article: Copy Citation Tools Evaluating Large Language Models for Automatic Detection of In-Hospital Cardiac Arrest: Multi-Site Analysis of Clinical Notes Uğurcan Vurgun , Aarthi Kaviyarasu , Sy Hwang , Ashley Batugo , Sunil Thomas , Brandon Tang , Ana Acevedo , Oscar Mitchell , Danielle L. Mowery medRxiv 2025.08.04.25331524; doi: https://doi.org/10.1101/2025.08.04.25331524 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (572) Allergy and Immunology (864) Anesthesia (302) Cardiovascular Medicine (4448) Dentistry and Oral Medicine (444) Dermatology (383) Emergency Medicine (609) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1515) Epidemiology (15240) Forensic Medicine (30) Gastroenterology (1130) Genetic and Genomic Medicine (6612) Geriatric Medicine (669) Health Economics (1000) Health Informatics (4550) Health Policy (1371) Health Systems and Quality Improvement (1613) Hematology (543) HIV/AIDS (1267) Infectious Diseases (except HIV/AIDS) (15926) Intensive Care and Critical Care Medicine (1105) Medical Education (624) Medical Ethics (147) Nephrology (668) Neurology (6619) Nursing (346) Nutrition (999) Obstetrics and Gynecology (1147) Occupational and Environmental Health (957) Oncology (3341) Ophthalmology (977) Orthopedics (369) Otolaryngology (421) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1695) Pharmacology and Therapeutics (693) Primary Care Research (714) Psychiatry and Clinical Psychology (5459) Public and Global Health (9247) Radiology and Imaging (2205) Rehabilitation Medicine and Physical Therapy (1371) Respiratory Medicine (1197) Rheumatology (597) Sexual and Reproductive Health (715) Sports Medicine (530) Surgery (714) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a0275ad69e60de16',t:'MTc3OTkwOTM5NA=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();