Zero-Shot Large Language Models for Long Clinical Text Summarization with Temporal Reasoning

Zero-Shot Large Language Models for Long Clinical Text Summarization with Temporal Reasoning | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Zero-Shot Large Language Models for Long Clinical Text Summarization with Temporal Reasoning Maya Kruse , Shiyue Hu , Nicholas Derby , Yifu Wu , Samantha Stonbraker , Bingsheng Yao , Dakuo Wang , Elizabeth Goldberg , View ORCID Profile Yanjun Gao doi: https://doi.org/10.1101/2025.07.21.25331947 Maya Kruse 1 University of Colorado Anschutz Medical Campus Find this author on Google Scholar Find this author on PubMed Search for this author on this site Shiyue Hu 1 University of Colorado Anschutz Medical Campus 2 University of Colorado Boulder Find this author on Google Scholar Find this author on PubMed Search for this author on this site Nicholas Derby 1 University of Colorado Anschutz Medical Campus 2 University of Colorado Boulder Find this author on Google Scholar Find this author on PubMed Search for this author on this site Yifu Wu 1 University of Colorado Anschutz Medical Campus Find this author on Google Scholar Find this author on PubMed Search for this author on this site Samantha Stonbraker 1 University of Colorado Anschutz Medical Campus Find this author on Google Scholar Find this author on PubMed Search for this author on this site Bingsheng Yao 3 Northeastern University Find this author on Google Scholar Find this author on PubMed Search for this author on this site Dakuo Wang 3 Northeastern University Find this author on Google Scholar Find this author on PubMed Search for this author on this site Elizabeth Goldberg 1 University of Colorado Anschutz Medical Campus Find this author on Google Scholar Find this author on PubMed Search for this author on this site Yanjun Gao 1 University of Colorado Anschutz Medical Campus Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Yanjun Gao For correspondence: yanjun.gao{at}cuanschutz.edu Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Recent advances in large language models (LLMs) have shown potential in clinical text summarization, but their ability to handle long patient trajectories with multi-modal data spread across time remains underexplored. This study systematically evaluates several state-of-the-art open-source LLMs, their Retrieval Augmented Generation (RAG) variants and chain-of-thought (CoT) prompting on long-context clinical summarization and prediction. We examine their ability to synthesize structured and unstructured Electronic Health Records (EHR) data while reasoning over temporal coherence, by re-engineering existing tasks, including discharge summarization and diagnosis prediction from two publicly available EHR datasets. Our results indicate that long context windows improve input integration but do not consistently enhance clinical reasoning, and LLMs are still struggling with temporal progression and rare disease prediction. While RAG shows improvements in hallucination in some cases, it does not fully address these limitations. Our work fills the gap in long clinical text summarization, establishing a foundation for evaluating LLMs with multi-modal data and temporal reasoning. 1 Introduction Electronic Health Records (EHRs) encapsulate a wide range of multi-modal data, such as vital signs, laboratory results, radiology findings, and free text clinical notes ( Mohsen et al., 2022 ; Li et al., 2022 ; Belden et al., 2017 ). Organized across various timestamps, they reflect the dynamic nature of patient care. In clinical settings, particularly for older patients in intensive care units (ICUs) with multiple encounters, chronic conditions, and complex treatment plans, EHRs become especially lengthy and intricate. Clinicians reviewing these long patient histories face significant cognitive burden, often leading to inaccurate decisions such as diagnostic errors ( Dymek et al., 2021 ; Singh et al., 2017 ). Automated summarization can help improve care continuity and decision-making ( Dymek et al., 2021 ; Adams et al., 2021 ; Gao et al., 2023a ; Laxmisan et al., 2012 ; Pivovarov and Elhadad, 2015 ; Liang et al., 2019 ), but the volume and complexity of EHRs pose challenges, requiring models that handle diverse and evolving clinical data. As shown in Figure 1 , summarizing longitudinal patient trajectories requires extracting temporally ordered events from multi-modal data, with temporal reasoning critical for preserving clinical causality. Download figure Open in new tab Figure 1: An illustration of the longitudinal patient trajectory summarization process from multi-modal EHRs. Key causal relationships among medical observations and interventions are highlighted, leading to the final physician diagnosis. Our work evaluates large language models (LLMs) for summarizing complex, longitudinal EHRs, capturing full patient trajectories rather than isolated clinical snapshots. While LLMs have shown promise in clinical NLP tasks ( Silcox et al., 2024 ; Wachter and Brynjolfsson, 2024 ; Garcia et al., 2024 ; Adams et al., 2024 ; Gao et al., 2022 ), existing evaluations primarily focus on short-context settings and task-specific fine-tuning. Clinical summarization benchmarks, such as discharge summarization (“Discharge Me”( Xu et al., 2024a )) and diagnosis generation from progress notes (“ProbSum”( Gao et al., 2023a )), evaluate LLMs on isolated segments of patient records rather than full hospital trajectories. While Retrieval-Augmented Generation (RAG) has been applied to clinical tasks like diagnosis prediction and discharge summarization ( Xu et al .; Gao et al., 2023b ; Lewis et al., 2020 ; Myers et al., 2024 ; Lyu et al., 2024 ), its effectiveness for long, temporally rich clinical narratives remains unclear. This work addresses two key gaps: (1) Current evaluations rarely test long-context LLMs in zero-shot settings, leaving their inherent capabilities and limitations in complex medical reasoning unknown; and (2) most clinical benchmarks focus on single time-point summaries, overlooking the demands of longitudinal patient care . We systematically evaluate LLMs on full patient trajectories to assess their ability to process temporally evolving, multi-modal clinical data. In this paper, we focus on two public EHR datasets: Medical Information Mart for Intensive Care (MIMIC-III) ( Johnson et al., 2020 ) and EHRShot ( Wornow et al., 2023 ). We evaluate five state-of-the-art LLMs and their RAG variants: Mistral-7B-Instruct-v0.1 ( Jiang et al., 2023 ), Llama3-8B-Instruct ( AI@Meta, 2024 ), Qwen2.5-7B ( Yang et al., 2024 ), DeepSeek-R1-Distill-Qwen-32B ( DeepSeek-AI et al., 2025 ) and Llama2-13B-chat-hf ( Touvron et al., 2023 ). Our work advances clinical LLM summarization through the following contributions: We reformulate discharge summarization, progress note summarization, and diagnosis classification into new long-context tasks requiring temporal reasoning, aligned with real clinical workflows. Our tasks integrate structured and unstructured data across multiple timestamps, enabling analysis of modality and temporal context effects. We compare direct generation, retrieval-augmented generation and chain-of-thought (CoT) prompting approaches for handling long clinical documents. In the absence of benchmarks combining multi-modal inputs, temporal structure , and clinical work-flow alignment , our study provides a first step toward evaluating LLMs in real-world longitudinal summarization. The results highlight key limitations and suggest directions for more temporally grounded and clinically usable models. 2 Related Work Clinical text summarization Existing work includes discharge summarization ( Xu et al., 2024a ; Lyu et al., 2024 ), diagnosis summarization ( Gao et al., 2023a , 2022 ; Liang et al., 2019 ), hospital course summarization ( Adams et al., 2021 , 2024 ). While these tasks inherently involve multi-document summarization by human physicians, NLP formulations typically treat them as single-document summarization or multi-document summarization with fixed timestamps (e.g. at the time when patient is discharged). LLMs for long clinical document Directly processing the long document input can lead to LLMs’ “lost-in-the-middle” problem ( Liu et al., 2024 ). As a result, RAG has been the default when handling long clinical documents, evidenced by its superior performance in diagnosis prediction, discharge summarization and information extraction ( Myers et al., 2024 ; Lyu et al., 2024 ; Lopez et al., 2025 ). Due to the task setup, these works have only investigated single modality EHRs with long input length. Our work addresses this problem by extending RAG and CoT-based approaches to multi-modal EHR summarization and clinical prediction, incorporating both structured (tabular) and unstructured (clinical notes) data to enhance the completeness and accuracy of patient trajectory summaries. We evaluate how well LLMs and their RAG set ups can capture and preserve temporal and causal relationships across patient’s hospital stay, bridging the gap between isolated document processing and comprehensive longitudinal understanding. 3 Dataset and Tasks We use two complementary datasets, MIMIC-III and EHRShot, to evaluate LLMs’ clinical reasoning. They differ in three key aspects: 1) Modalities: MIMIC-III includes both structured data and clinical notes, while EHRShot contains only structured data. 2) Output type: MIMIC-III tasks focus on generating short summaries of patient progress or discharge status, whereas EHRShot involves classification-based diagnosis prediction. 3) Decision time span: MIMIC-III targets immediate ICU-related summaries, while EHRShot predicts diagnoses within a year of discharge, emphasizing long-term forecasting based on prior visits. Despite these differences, both datasets address longitudinal patient information, requiring models to reason over time and synthesize complex clinical histories. 3.1 MIMIC-III MIMIC-III is a publicly available dataset comprising de-identified health records from over 40,000 ICU patients. For this study, we focus on a subset of patients with hospital stays exceeding 72 hours to ensure sufficient context for evaluating long-document summarization and temporal reasoning. Unlike prior MIMIC-III studies, our selected cohort emphasizes complex, multi-day ICU stays where accurate summaries are most impactful. This setup allows us to assess LLMs’ ability to handle prolonged and evolving clinical narratives. We use both structured and unstructured data. Specifically, we extract chart events (vital signs, ventilator settings), lab events (e.g., white blood cell counts), input events (e.g., IV infusions, feedings), and medications. These features reflect key aspects of clinical decision-making and support rich, temporally grounded summarization tasks. Discharge Summarization Discharge summaries are a crucial part of a patient’s hospital care process, providing a comprehensive overview of their hospital stay and key clinical events. It is usually written by the physician after the patient is discharged, and contains three main sections: DIAGNOSIS: a list of diagnoses requiring an understanding of the patient’s clinical progression up to discharge; BRIEF HOSPITAL COURSE: clinical summary of treatments, interventions, and significant events during hospitalization; DISCHARGE INSTRUCTIONS: post-discharge guidance, including medication plans, dietary recommendations and follow-up care. Discharge Summarization has been explored in several studies, including ( Xu et al., 2024b ) and ( Ando et al., 2022 ), often focusing on specific sections like the Brief Hospital Course or discharge instructions. In this paper, we generate all three sections by extracting and chronologically ordering structured and unstructured data from a hospital admission. We limit input to the last 24 hours to prevent overloading the LLM while prioritizing the most relevant information for discharge, considering the generally long hospital stays ( ≥ 3 days). We also test a 48-hour window to capture a broader clinical context. We design four input settings to evaluate LLMs on multi-modal and temporal reasoning: NOTE ONLY where input only contains clinical notes, TABULAR ONLY where input only contains tabular data, SHUFFLED TABULAR where we shuffled the tabular data by their timestamps, COMBINED where we combined both notes and tabular data, sorted by their timestamps. Assessment and Plan (A&P) Generation Daily progress notes are the documents where physicians record diagnoses, treatment, and clinical status, providing key insights into a patient’s condition throughout their hospital stay. A progress note typically consists of four sections: Subjective, Objective, Assessment and Plan ( Weed et al., 1968 ; Wright et al., 2014 ). The Subjective and Objective sections serve as the “evidence” and observation of the patient on that day, with Subjective comprising unstructured free text describing symptoms, status, and treatment, while Objective consists of structured tabular data like lab results and charted values. In contrast, the Assessment and Plan sec-tions capture the physician’s reasoning and clinical hypothesis based on this evidence, with diagnoses and treatment plans listed. To assess LLM abilities in summarizing longitudinal data, the Assessment and Plan sections provide a great testbed, requiring integration of information from multiple days, capturing the evolution of the patient’s condition, and synthesizing key clinical evidence into a concise yet comprehensive summary. The setup of the task is illustrated in Figure 2 , which explains the three input methods: the NO PRIOR CONTEXT (BASELINE) method uses only the current day’s structured EHR data, the SINGLE-DAY CONTEXT method adds the previous day’s progress note for limited historical context, and the MULTI-DAY CONTEXT method incorporates all prior progress notes for a more comprehensive patient history. For day i , the input includes progress notes up to i − 1 (depending on the method), but the note for i is excluded and compared to the generated version. This setup reflects the real clinical workflow, where physicians reference prior notes when writing new ones. Download figure Open in new tab Figure 2: Illustration of the A&P generation workflow (top) and a comparison of input data formulations (bottom). How long is the input in MIMIC? Figure 3 shows input lengths across tasks and settings. For discharge summarization, inputs average 500 tokens (24-hour window) and 1,940 tokens (48-hour window). In contrast, the Assessment and Plan (A&P) generation task involves much longer inputs due to the inclusion of prior progress notes and structured data. The No Context baseline starts with a relatively high token count (3,125 tokens on Day 1) but remains the shortest overall. Single-Day Context, which adds the previous day’s note, increases input length to 5,375 tokens. Multi-Day Context, which accumulates all prior notes, shows the steepest token growth—reaching nearly 10,000 tokens by Day 4 and averaging 6,875 tokens. Download figure Open in new tab Figure 3: The token length of input data used for A&P generation across days and methods (left), as well as that of input data used for Discharge Summarization across time windows and modalities(right). 3.2 Diagnosis Prediction from EHRShot EHRShot ( Wornow et al., 2023 ) is a longitudinal dataset comprising fully structured EHR data from 6,739 patients at Stanford Medicine, with over 41 million clinical events. In contrast to MIMIC-III, which contains a mix of structured and unstructured data, EHRShot offers a clean, structured-only setting, allowing for controlled evaluation of temporal reasoning in longitudinal patient records. We include EHRShot to complement the MIMIC-based summarization tasks in two important ways. First, EHRShot focuses on future prediction—each task requires predicting whether a patient will receive one of six diagnoses (e.g., HYPERTENSION, PANCREATIC CANCER, ACUTE MI) within one year after discharge. This differs from our MIMIC tasks, which emphasize retrospective summarization of observed hospital stays. Second, its purely structured nature allows us to isolate models’ abilities to handle temporal patterns in tabular data without the added complexity of clinical language. Together, these contrasts offer a broader assessment of LLM capabilities in modeling temporal EHR information across different modalities and predictive targets. The six diagnosis tasks vary in prevalence from common conditions like HYPERLIPIDEMIA (31.85%) to rarer ones like CELIAC (3.46%). Inputs have a mean token length of 1,989 ( σ : 1,270), with a median of 1,851, and an average of 74 unique measurements per patient. More statistics are provided in Appendix A.2. 3.3 Representing multi-modal, longitudinal patient data We convert structured EHR data into natural language to enable LLM-based summarization, following prior table-to-text approaches ( Gao et al., 2024 ; Yu et al., 2023 ). As in Table 1 , each measurement is verbalized using simple templates (e.g., [MEASUREMENT] IS [VALUE][UNIT]), grouped by timestamp, and temporally ordered using relative time references. Medications and input events follow a similar format (e.g., “is administered”). View this table: View inline View popup Download powerpoint Table 1: An example of a patient’s compiled data, including relative timestamp (minutes and hours since previous recorded data), structured data converted into narrative format as well as unstructured note data. To reduce redundancy from repeated or copy-pasted entries, we deduplicate records across modalities and retain only the most recent entry when multiple identical values appear within a short time window. Clinical notes are filtered similarly. For EHRShot, structured inputs are divided into six hour chunks and repetitive phrases are removed. Appendix A.1. 4 Methods We evaluate five LLMs with varying context capacities. LLaMA2-13B and Mistral-7B support up to 4K tokens, while LLaMA3-8B allows for 8K. To test long-context capabilities, we include Qwen2.5-7B and DeepSeek-R1 (32B), both of which support input lengths up to 128K tokens. While we did explore biomedical LLMs (PMC-LLaMa, BioGPT, BioMistral), their performance was extremely poor, so we did not include them in this paper. 4.1 Approaches We compare a baseline direct generation, structured RAG and CoT event extraction approach. The key difference between these approaches lies in how they handle long temporal contexts. Direct generation processes the entire input at once but may suffer from the lost-in-the-middle problem, where relevant information buried in long documents is overlooked. In contrast, RAG segments temporal information into retrievable chunks, which helps manage long sequences but may disrupt temporal dependencies between events. For CoT event extraction, the long context is first compressed into a series of clinical events, over which the model then summarizes. 1). Direct Generation The patient’s chronological data, formatted according to the specific task requirements, is provided to the model as a direct input without any additional structural modifications. This serves as a baseline to assess the capability of current state-of-the-art language models in processing and generating clinical summaries from raw sequential data, without the aid of task-specific adaptations or architectural enhancements. 2). RAG For this setup, we choose the same selection as models used for direct generation, with the exception of DeepSeek, as it is considerably larger than the other LLMs (32B parameters vs 7-13B) yet fails to outperform these smaller models. This combination of high computational cost and suboptimal performance led us to exclude DeepSeek for RAG. Myers et al. (2024) studied the quality of embedding in medical RAG and found BGE ( Xiao et al., 2023 ) yielded the highest performance. Thus, we adopt BGE embedding and apply it to all LLM RAG set ups in our work. Additionally, the queries presented in the paper are slightly modified and used to perform query optimization. Hyperparameter optimization is carried out on the standard RAG hyperparameters chunk size, chunk overlap and top-k retrieved documents. 3). Event Extraction via Chain-of-Thought (CoT) As an alternative to retrieval or direct generation, we introduce a chain-of-thought (CoT) ( Wei et al., 2022 ) prompting approach that incorporates a clinical reasoning step before summarization. LLMs are prompted to first identify temporally ordered clinical events from the input data, which are appended to the original input as structured context for generation. This intermediate step emphasizes salient clinical signals and reduces input noise, helping the model focus on meaningful patterns. We develop our prompt with guidance from recent studies showing that LLMs perform better on clinical tasks when given clear, structured instructions tailored to the specific goal. For example, Wang et al. (2023) finds that prompts targeting specific event types like symptoms, lab results, treatments, and medical decisions help models focus on clinically meaningful information and improve event detection. Yuan et al. (2023) also shows that when complex tasks are broken into smaller reasoning steps, especially for understanding how events are ordered over time, models tend to produce more consistent and accurate outputs. Figure 4 illustrates the CoT prompt design for A&P generation task. Download figure Open in new tab Figure 4: Chain-of-Thought prompting template and example output used for ICU daily event extraction from structured EHR data for A&P generation. We experiment with seven temporal input configurations for event extraction: using only the current day’s data, combining it with previous days (FORWARD), or with subsequent days (BACKWARD), each spanning 1, 3, or all available days. We apply these settings to a development batch of 20 patients and found that, within each respective group of experiments, the FORWARD setting using all previous days and the BACKWARD setting using one following day produced the best results. Based on these findings, we report performance under three representative temporal contexts: +0 (Current Day Only, baseline), +N (Forward Context with all prior days), and –1 (Backward Context with one following day). Experiment settings For all LLMs, we run their 8-bit quantized version. We set the output token length as 1,000, but almost all tasks output is significantly shorter than this limit. For RAG setup, we used Langchain ( Chase, 2022 ) with Faiss ( Douze et al., 2024 ) for semantic retrieval in a vector database. We perform hyperparameter tuning with chunk size between [250, 750], top k between [10, 50], chunk overlap between [50,200]. The detailed results of hyperparameter searching are in Appendix A.4. All experiments are run on 2 H100 94GB GPUs. 4.2 Evaluation On MIMIC, we report standard summarizatoin evaluation metrics that capture string overlap and semantics similarity. ROUGE-L ( Lin, 2004 ) measures string overlap, while BERTScore ( Zhang et al., 2019 ) assesses maximum token pairwise similarity. We use SapBERT as the backend for BERTScore due to its superior performance in biomedical entity representation ( Liu et al., 2020 ). On EHRShot, given that the diagnosis prediction tasks are essentially binary classification, we report macro-average accuracy and F-scores. 5 Results The results are organized by task: Discharge Summarization and Assessment and Plan Generation performed on MIMIC-III and Diagnosis Prediction on EHRSHOT. Additionally, prompt optimization was carried out on all tasks, using a small sample set (Appendix A.5). Discharge Summarization Table 2 presents results from comparison between the Direct Generation and RAG approaches. Qwen achieves the highest ROUGE-L score of 17.91 using RAG. In general, most LLMs record a performance increase in their RAG variants, the biggest of which is Llama3’s, with an increase of 4.08 on ROUGE-L. Mistral is the exception, with a minor performance decrease of −1.24. DeepSeek, as the largest LLM, reports mediocre performance Table 3 shows section-wise analysis for Mistral comparing direct generation and CoT methods, where CoT event extraction performs slightly worse than direct generation across most sections. Other LLMs exhibit similar trends, so we report Mistral only for brevity. In Appendix 7, we provide more section-wise analysis on the Discharge Summ task. Results further confirm the challenge of LLMs reasoning over patient trajectories. View this table: View inline View popup Download powerpoint Table 2: Results on discharge summarization, comparing the direct generation and RAG approaches across all models. View this table: View inline View popup Download powerpoint Table 3: Mistral performance on the three sections of discharge summarization (Dx: Diagnosis, HC: Hospital course, DI: Discharge Instruction), comparing direct generation approach with the event extraction CoT. Figure 5 shows the effects of the four modalities (notes + tabular data, notes only, tabular only and shuffled tabular) and two context window sizes (24 and 48 hours) on performance. For the 24 hour window, the pure note modality dominates, performing best on Mistral, Qwen, DeepSeek and Llama2. Llama3 is the outlier and reports the best performance on the combined and shuffled tabular modalities. In the 48h window, trends are less clear, but most models still favor notes. Download figure Open in new tab Figure 5: Average f1 results across modalities and time windows (on direct generation) Temporal order appears unhelpful, as shuffled tabular data performs slightly better than chronological tabular data, likely because the data is near the discharge state, where patients’ clinical conditions stabilize, leading to fewer changes over time. A& P Generation . Table 4 shows that adding prior context (Single-or Multi-Day) improves performance across all models, particularly for ROUGE scores. Llama3 performs best on direct generation, especially in Single- and Multi-Day Contexts, but is outperformed by Qwen and Mistral on RAG. However, Multi-Day Context does not always yield performance gains over Single-Day, suggesting models struggle with longer patient histories. RAG generally improves performance, particularly for Mistral and Qwen, but its impact varies across models: Llama3 performs better without retrieval. View this table: View inline View popup Download powerpoint Table 4: Performance of Direct Generation and RAG methods on A&P generation across models and input settings. RL = ROUGE-L score; BS = BERTScore computed using SapBERT embeddings. This task specifically requires clinical reasoning over past captured data (evidence) to summarize patient progression and plan for treatment, yet the relatively low ROUGE and BERTScore values indicate that models still struggle with temporal reasoning and integrating historical context effectively. Table 5 presents results of the event extraction CoT method. CoT did not yield improvements over direct generation or RAG approaches across ROUGE-L or BERTScore. This suggests that, while the event extraction step offers interpretability and control, it may not directly enhance generation quality when applied in a pipeline without further model adaptation. We view this as a useful diagnostic approach that could be further refined or integrated with instruction tuning in future work. View this table: View inline View popup Download powerpoint Table 5: Event extraction CoT results on A&P generation task across settings. View this table: View inline View popup Download powerpoint Table 6: Results on the binary diagnosis prediction task using EHRShot data (macro-average). View this table: View inline View popup Download powerpoint Table 7: Discharge Summarization: Llama2 and Mistral Performance Across Sections: Diagnosis (Dx), Hospital Course (HC), and Discharge Instructions (DI). View this table: View inline View popup Download powerpoint Table 8: Performance comparison on A&P generation, aggregated over the patient’s entire hospital stay. For readers interested in a more detailed breakdown, we refer them to the Fig 10 , where we provide per-day performance results for a subset of patients with a length of stay of at least 5 days. View this table: View inline View popup Download powerpoint Table 9: Mistral performance on the three sections of discharge summarization (Dx: Diagnosis, HC: Hospital course, DI: Discharge Instruction), comparing direct generation approach with the event extraction CoT. Including CUI f-score. Diagnosis Prediction We evaluate Mistral and Qwen for this task based on their prior performance. Both models prioritize majority class predictions, leading to inflated accuracy but severely low F1-scores. RAG improves accuracy but does not meaningfully enhance recall or F1-score. The task remains highly challenging, especially for rare diseases, where models struggle to predict positive cases. Qwen achieves high accuracy but an F-score of zero for celiac disease and lupus, reflecting severe class imbalance. It mostly predicts negatives, occasionally misclassifying positives but failing to identify true cases. Given that 96.54% of celiac and 95.56% of lupus cases are true negatives, its accuracy (96.05% and 95.06%) is barely lower than predicting all negatives. Overall Performance Figure 6 illustrate how input length impacts model performance across different LLMs, considering their varying context windows. Qwen (128K context), shows a slight positive correlation, suggesting that longer inputs may improve its performance. In contrast, Mistral and Llama2 (both 4K context), decline as input length increases, likely due to exceeding their optimal processing capacity. Llama3 (8K context), remains stable with minimum impact from the length. Models with shorter contexts struggle with longer inputs, while larger-context models handle them better, though benefits remain inconsistent. Download figure Open in new tab Figure 6: Correlation of token length and average performance across all tasks and metrics 6 Discussion Another critical aspect to consider is running time and memory usage. All 7B LLMs require approximately 15GB of GPU RAM for direct generation. RAG demands significantly more memory, peaking at 32–33GB, nearly twice that of direct generation. For Qwen, it takes 18-20 minutes to run RAG on one progress note with prior context input, while direct generation takes 10 minutes on average. On EHRShot where input is shorter, diagnosis prediction takes just 3–4 seconds. For other LLMs with smaller context window, they run quicker. Expert error analysis A senior board-certified Emergency Department physician was consulted to provide more in-depth analysis of our generated outputs. This assessment was not intended to yield quantitative results but rather to gain insights into how physicians perceive the generated summaries beyond automated metrics and to identify areas for improvement. We focus specifically on discharge summarization and sample 10 pairs of RAG and Direct Generation output from Qwen, as it reached relatively high performance on automated metrics. The physician evaluated the output based on six criteria that have previously been deployed for clinical text evaluation: OVERALL ACCURACY (factual correctness), HALLUCINATION,OMISSION, READABILITY, CLINICAL RELEVANCE and SPECIFICITY ( Singhal et al., 2023 ; Xu et al., 2024a ; Ben Abacha et al., 2023 ; Aljamaan et al., 2024 ; Croxford et al., 2024 ; Williams et al., 2024 ). The results of this analysis are given in Figure 7 , and the annotation guidelines can be found in Table 19 . Overall, RAG performs slightly better, with fewer hallucinations and more clinically relevant summaries. However, both methods still struggle—often prioritizing less relevant diagnoses, oversimplifying summaries, and failing to discard outdated or disproven diagnoses. These issues highlight ongoing challenges with temporal reasoning. Download figure Open in new tab Figure 7: Expert review scores for Qwen direct generation and RAG on Discharge Summarization 7 Conclusion We evaluated LLMs on long-context clinical summarization and prediction using two public EHR datasets. Current models struggle with accurate summarization and temporal reasoning, making it hard to interpret medical event sequences. While RAG offers some improvement, overall performance remains inadequate, highlighting the need for further progress. Limitations This study focused on current LLMs’ capabilities on the task of clinical summarization for long context documents including temporal information. We evaluated several different models, including Qwen, Llama 3 and DeepSeek, but we acknowledge that this selection is by no means comprehensive and could have limited our analysis. It also does not contain any closed-source LLMs, as the use of these models with MIMIC and EHRSHOT data is prohibited by Data Use Agreements. Due to the dearth of publicly available EHR datasets, we were only able to include data from two sources: MIMIC-III and EHRSHOT. Our human evaluation was constrained by real-world clinical scheduling limitations, allowing us to consult only one emergency department (ED) physician. Furthermore, we did not formally validate our proposed survey instrument, though the survey questions were aggregated from established prior work (see Appendix regarding the literatures these questions came from). Our goal is to leverage domain experts to better understand LLM limitations, and we plan to expand human evaluation efforts in future studies to provide a more comprehensive assessment. Ethical Statement This study utilizes de-identified patient data from publicly available datasets (MIMIC-III and EHRSHOT), ensuring that no identifiable patient information is used. As a result, our work does not involve human subjects research and poses no risk to patient privacy or confidentiality. Additionally, our study is purely retrospective and computational, focusing on evaluating LLMs for clinical summarization. The models analyzed do not interact with real-world clinical workflows and are not used for actual medical decision-making. Therefore, there is no potential harm to patients or healthcare providers as a result of this research. Our goal is to assess and improve LLM capabilities for handling complex clinical data, with the long-term aim of developing safe, explainable, and effective AI tools to support healthcare professionals in the future. However, we acknowledge that applying existing LLMs in clinical workflows carries real risks, including privacy leakage, algorithmic biases, and the potential for incorrect decisions. Addressing these challenges is critical for ensuring the safe and ethical deployment of AI in healthcare. Data Availability All data produced in the present study are available upon reasonable request to the authors. A Appendix A.1 More Preprocessing Details In addition to the preprocessing steps outlined in section 3.3 , we attempt, as far as possible, to only include laboratory measurements and vital sign values that fall outside the normal range. These values are more likely to indicate a problem the patient is facing and are thus more salient. For lab measurements, we only keep those values flagged as ‘abnormal’. For chart or vital sign values, there exists a warning flag, which is either 0 or 1. However, some values have an NaN value instead. We exclude values with the flag set to 0, but keep those with 1 or NaN, since it is unknown whether the value corresponding to the NaN type is abnormal or not without a deeper medical analysis. A.2 EHRSHOT Statistics Figure 8 reports detailed statistics of EHRShot cohort, regarding the input token length and number of distinct clinical measurements. The prompt token lengths exhibit a wide range, with a mean of 1,989 tokens and a standard deviation of 1,270 tokens. The distribution is right-skewed, with a minimum of 178 tokens and a maximum of 5,919 tokens, while the median (50th percentile) is 1,851 tokens. Download figure Open in new tab Figure 8: Additional statistics on the EHRSHOT dataset. Number of distinct measurements given above, with the token length distribution below. Download figure Open in new tab Figure 9: Chain-of-Thought prompting template and example output used for discharge summary event extraction based on the final 48 hours of hospitalization. For distinct measurement types per patient, the data is also highly variable, with a mean of 74 and a standard deviation of 48. The number of distinct measurements ranges from 1 to 351, with a median of 66 and an interquartile range from 39 (25th percentile) to 100 (75th percentile). Both distributions highlight significant variation in input complexity, reinforcing the need for models to handle long-context dependencies and multi-modal data effectively. A.3 More results on direct generation, RAG and CoT performance comparison A.4 Hyperparameter searching results In this section, we report our hyperparameter tuning on the RAG setup for all LLMs. Table 10 covers the hyperparameter selection. All tuning are done on Discharge Summarization and A&P generation tasks using a small held-out set (n=5). The best set of parameters are used for official experiments reported in the main text. View this table: View inline View popup Download powerpoint Table 10: Hyperparameter configurations for each experiment. View this table: View inline View popup Download powerpoint Table 11: Hyperparameter Tuning for Discharge Summaries Results (averaged across sections) View this table: View inline View popup Download powerpoint Table 12: Hyperparameter Tuning for A&P Generation A.5 Prompt and query optimization A.6 Event extraction COT prompt for discharge summarization A.7 RAG and direct generation prompt optimization We optimized the prompts and queries used for RAG systems on all tasks. Both prompts and queries are similar in terms of its instruction, the only difference is that queries for RAG has the “Retrieve” component. Table 13 , 14 , 16 , 17 , 15 presents all prompts and queries for MIMIC tasks. View this table: View inline View popup Download powerpoint Table 13: Queries used for retrieving the primary diagnosis. Highest performing on models bolded with respective model in parentheses. View this table: View inline View popup Download powerpoint Table 14: Queries used for retrieving the hospital course. Highest performing on models bolded with respective model in parentheses. View this table: View inline View popup Download powerpoint Table 15: Queries used for retrieving the discharge instructions. Highest performing on models bolded with respective model in parentheses. View this table: View inline View popup Download powerpoint Table 16: Queries used for retrieving the Assessment section of a progress note. Highest performing on models bolded with respective model in parentheses. View this table: View inline View popup Download powerpoint Table 17: Queries used for retrieving the Plan section of a progress note. Highest performing on models bolded with respective model in parentheses. A.8 Impact of context length on consecutive patient data Figure 10 illustrates the impact of incorporating prior-day context on F1 scores for patient note generation. The x-axis represents consecutive relative days within a patient’s hospital stay, where Day 0 corresponds to the first day of admission and serves as the ground truth (i.e., no evaluation is performed on this day). The y-axis shows the average F1 score across a subset of 10 patients with a length of stay of at least 5 days. Download figure Open in new tab Figure 10: Impact of prior-day context on F1 score across consecutive days Since Day 0 (Day 1 in the figure) is not evaluated, its F1 score is set to 1.0 for all methods to ensure a clear visual comparison with subsequent days. The results demonstrate that incorporating prior-day context improves performance over time. The Single-Day Context and Multi-Day Context methods achieve substantially higher F1 scores than the No Prior Context method, particularly on Day 2 and Day 3, suggesting that leveraging past information helps generate more accurate and coherent patient notes. However, after Day 3, the performance of the Multi-Day Context method begins to decline, indicating that while longer historical context can be beneficial, it may introduce additional noise or redundant information. Overall, these findings highlight that considering prior context enhances the accuracy of generated patient notes, with Single-Day Context yielding the highest performance in later days, while Multi-Day Context shows initial improvements but exhibits diminishing returns over time. A.9 More results for using ground-truth progress notes vs. generated progress notes on A&P Generation This section presents further results comparing ground-truth progress notes and model-generated progress notes as input for A&P Generation. Our analysis evaluates how using prior ground-truth notes versus LLM-generated notes impacts the overall quality, coherence, and accuracy of the generated A&P sections. Table 18 presents results on all LLMs direct generation. Overall, there are performance decreases when moving from using generated progress notes as context input. The results highlight key differences in information retention and propagation, where models using ground-truth progress notes tend to maintain better clinical consistency, while those using generated notes may accumulate errors over multiple days, leading to drift and hallucination in longitudinal patient summaries. These findings emphasize the need for robust calibration and error correction mechanisms when relying on LLM-generated progress notes for iterative summarization. View this table: View inline View popup Download powerpoint Table 18: Comparison of ground-truth (GT, same results we report in main text) and generated (GEN) settings on assessment and plan generation (results of direct generation approach shown) View this table: View inline View popup Download powerpoint Table 19: Physician evaluation survey Additionally, we include Figure 11 to show the impact of incorporating prior-day context on the composition of generated patient notes. The figure illustrates how different methods—ranging from no prior-day context (Method −1) to single-day (Method 1) and multi-day context (Method 2)—affect the alignment between generated elements (GEN), ground truth elements (GT), and common elements across multiple admissions. As the number of prior days included increases, we observe changes in the proportion of correctly retained ground truth elements and newly generated content, highlighting the role of historical context in improving consistency and completeness in patient note generation. Additionally, the inclusion of prior context reduces the number of newly generated elements, indicating a shift away from generating potentially extraneous or less relevant content. Download figure Open in new tab Figure 11: Comparison of generated patient notes across different methods of incorporating prior-day context. Each subplot represents a different patient admission, with bars indicating the composition of generated notes across Days 2, 3, and 4. The stacked bars show the proportion of common elements (overlapping between ground truth and generated notes), GT elements (present in the ground truth but missing in generated notes), and GEN elements (newly generated content not found in the ground truth). Method −1 (no prior-day context), Method 1 (single prior-day context), and Method 2 (multi-day context) demonstrate how historical context influences the balance between accurate retention and newly introduced information. However, the degree of improvement varies across different admissions, suggesting that some cases benefit more from historical context than others, potentially due to differences in case complexity or the structure of prior notes. Furthermore, the differences between generated and ground truth elements become more pronounced as days progress, highlighting the challenge of maintaining consistency in patient notes over time. Overall, the findings suggest that incorporating multi-day context enhances the accuracy and stability of generated patient notes, reducing hallucinated content while preserving clinically relevant information. Break-down results behind Figure 5 Recall that Figure 5 aggregates all metrics for LLM direct generation on discharge summarization. To provide comprehensive results analysis, we include Figure 12 for 24h window and 48h window. The trends of performance changing across different modalities are consistent with the Figure 5 . Download figure Open in new tab Figure 12: Metric breakdown across modalities for the 24 hour time window (on direct generation) Details about human evaluation Last but not least, we present the full survey questions aggregated from existing work in Table 19 . We would like to emphasize that this survey has not been validated, rather, it selects the criteria after consulting with a physician regarding what they value in LLM generated text and findings from prior work, as cited in the table. These questions allow us to evaluate the accuracy, faithfulness, readability, and clinical relevance of LLM-generated text, providing a structured framework for assessing their strengths and limitations in a clinical setting. References ↵ Griffin Adams , Emily Alsentzer , Mert Ketenci , Jason Zucker , and Noémie Elhadad . 2021 . What’s in a summary? laying the groundwork for advances in hospital-course summarization . In Proceedings of the conference. Association for Computational Linguistics. North American Chapter. Meeting, volume 2021, page 4794 . NIH Public Access . ↵ Griffin Adams , Jason Zucker , and Noémie Elhadad . 2024 . Speer: Sentence-level planning of long clinical summaries via embedded entity retrieval . arXiv preprint arxiv: 2401.02369 . ↵ AI@Meta . 2024 . Llama 3 model card . ↵ Fadi Aljamaan , Mohamad-Hani Temsah , Ibraheem Altamimi , Ayman Al-Eyadhy , Amr Jamal , Khalid Alhasan , Tamer A Mesallam , Mohamed Farahat , Khalid H Malki , et al. 2024 . Reference hallucination score for medical artificial intelligence chatbots: development and usability study . JMIR Medical Informatics , 12 ( 1 ): e54345 . OpenUrl ↵ Kenichiro Ando , Takashi Okumura , Mamoru Komachi , Hiromasa Horiguchi , and Yuji Matsumoto . 2022 . Is artificial intelligence capable of generating hospital discharge summaries from inpatient records? PLOS Digital Health , 1 ( 12 ): e0000158 . OpenUrl ↵ Jeffery L Belden , Richelle J Koopman , Sonal J Patil , Nathan J Lowrance , Gregory F Petroski , and Jamie B Smith . 2017 . Dynamic electronic health record note prototype: seeing more by showing less . The Journal of the American Board of Family Medicine , 30 ( 6 ): 691 – 700 . OpenUrl PubMed ↵ Asma Ben Abacha , Wen-wai Yim , George Michalopoulos , and Thomas Lin . 2023 . An investigation of evaluation methods in automatic medical note generation . In Findings of the Association for Computational Linguistics: ACL 2023 , pages 2575 – 2588 , Toronto, Canada . Association for Computational Linguistics . ↵ Harrison Chase . 2022 . Langchain . Accessed: 2025-02-14 . ↵ E. Croxford , Y. Gao , B. Patterson , D. To , S. Tesch , D. Dligach , A. Mayampurath , M. M. Churpek , and M. Afshar . 2024 . Development of a human evaluation framework and correlation with automated metrics for natural language generation of medical diagnoses . In American Medical Informatics Association Annual Symposium, page 2024.03.20.24304620 . ↵ DeepSeek-AI , Daya Guo , Dejian Yang , Haowei Zhang , Junxiao Song , Ruoyu Zhang , Runxin Xu , Qihao Zhu , Shirong Ma , Peiyi Wang , Xiao Bi , Xiaokang Zhang , Xingkai Yu , Yu Wu , Z. F. Wu , Zhibin Gou , Zhihong Shao , Zhuoshu Li , Ziyi Gao , Aixin Liu , Bing Xue , Bingxuan Wang , Bochao Wu , Bei Feng , Chengda Lu , Chenggang Zhao , Chengqi Deng , Chenyu Zhang , Chong Ruan , Damai Dai , Deli Chen , Dongjie Ji , Erhang Li , Fangyun Lin , Fucong Dai , Fuli Luo , Guangbo Hao , Guanting Chen , Guowei Li , H. Zhang , Han Bao , Hanwei Xu , Haocheng Wang , Honghui Ding , Huajian Xin , Huazuo Gao , Hui Qu , Hui Li , Jianzhong Guo , Jiashi Li , Jiawei Wang , Jingchang Chen , Jingyang Yuan , Junjie Qiu , Junlong Li , J. L. Cai , Jiaqi Ni , Jian Liang , Jin Chen , Kai Dong , Kai Hu , Kaige Gao , Kang Guan , Kexin Huang , Kuai Yu , Lean Wang , Lecong Zhang , Liang Zhao , Litong Wang , Liyue Zhang , Lei Xu , Leyi Xia , Mingchuan Zhang , Minghua Zhang , Minghui Tang , Meng Li , Miaojun Wang , Mingming Li , Ning Tian , Panpan Huang , Peng Zhang , Qiancheng Wang , Qinyu Chen , Qiushi Du , Ruiqi Ge , Ruisong Zhang , Ruizhe Pan , Runji Wang , R. J. Chen , R. L. Jin , Ruyi Chen , Shanghao Lu , Shangyan Zhou , Shanhuang Chen , Shengfeng Ye , Shiyu Wang , Shuiping Yu , Shunfeng Zhou , Shuting Pan , S. S. Li , Shuang Zhou , Shaoqing Wu , Shengfeng Ye , Tao Yun , Tian Pei , Tianyu Sun , T. Wang , Wangding Zeng , Wanjia Zhao , Wen Liu , Wenfeng Liang , Wenjun Gao , Wenqin Yu , Wentao Zhang , W. L. Xiao , Wei An , Xiaodong Liu , Xiaohan Wang , Xiaokang Chen , Xiaotao Nie , Xin Cheng , Xin Liu , Xin Xie , Xingchao Liu , Xinyu Yang , Xinyuan Li , Xuecheng Su , Xuheng Lin , X. Q. Li , Xiangyue Jin , Xiaojin Shen , Xiaosha Chen , Xiaowen Sun , Xiaoxiang Wang , Xinnan Song , Xinyi Zhou , Xianzu Wang , Xinxia Shan , Y. K. Li , Y. Q. Wang , Y. X. Wei , Yang Zhang , Yanhong Xu , Yao Li , Yao Zhao , Yaofeng Sun , Yaohui Wang , Yi Yu , Yichao Zhang , Yifan Shi , Yiliang Xiong , Ying He , Yishi Piao , Yisong Wang , Yixuan Tan , Yiyang Ma , Yiyuan Liu , Yongqiang Guo , Yuan Ou , Yuduan Wang , Yue Gong , Yuheng Zou , Yujia He , Yunfan Xiong , Yuxiang Luo , Yuxiang You , Yuxuan Liu , Yuyang Zhou , Y. X. Zhu , Yanhong Xu , Yanping Huang , Yaohui Li , Yi Zheng , Yuchen Zhu , Yunxian Ma , Ying Tang , Yukun Zha , Yuting Yan , Z. Z. Ren , Zehui Ren , Zhangli Sha , Zhe Fu , Zhean Xu , Zhenda Xie , Zhengyan Zhang , Zhewen Hao , Zhicheng Ma , Zhigang Yan , Zhiyu Wu , Zihui Gu , Zijia Zhu , Zijun Liu , Zilin Li , Ziwei Xie , Ziyang Song , Zizheng Pan , Zhen Huang , Zhipeng Xu , Zhongyu Zhang , and Zhen Zhang . 2025 . Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning . Preprint , arxiv: 2501.12948 . ↵ Matthijs Douze , Alexandr Guzhva , Chengqi Deng , Jeff Johnson , Gergely Szilvasy , Pierre-Emmanuel Mazaré , Maria Lomeli , Lucas Hosseini , and Hervé Jégou . 2024 . The faiss library . arXiv preprint arxiv: 2401.08281 . ↵ Christine Dymek , Bryan Kim , Genevieve B Melton , Thomas H Payne , Hardeep Singh , and Chun-Ju Hsiao . 2021 . Building the evidence-base to reduce electronic health record–related clinician burden . Journal of the American Medical Informatics Association , 28 ( 5 ): 1057 – 1061 . OpenUrl CrossRef PubMed ↵ Yanjun Gao , Dmitriy Dligach , Timothy Miller , Matthew M Churpek , and Majid Afshar . 2023a . Overview of the problem list summarization (probsum) 2023 shared task on summarizing patients’ active diagnoses and problems from electronic health record progress notes . In Proceedings of the conference. Association for Computational Linguistics. Meeting , volume 2023 , page 461 . NIH Public Access . OpenUrl ↵ Yanjun Gao , Timothy Miller , Dongfang Xu , Dmitriy Dligach , Matthew M Churpek , and Majid Afshar . 2022 . Summarizing patients’ problems from hospital progress notes using pre-trained sequence-tosequence models . In Proceedings of COLING. International Conference on Computational Linguistics , volume 2022 , page 2979 . NIH Public Access . OpenUrl ↵ Yanjun Gao , Skatje Myers , Shan Chen , Dmitriy Dligach , Timothy A Miller , Danielle Bitterman , Matthew Churpek , and Majid Afshar . 2024 . When raw data prevails: Are large language model embeddings effective in numerical data representation for medical machine learning applications? arXiv preprint arxiv: 2408.11854 . ↵ Yunfan Gao , Yun Xiong , Xinyu Gao , Kangxiang Jia , Jinliu Pan , Yuxi Bi , Yi Dai , Jiawei Sun , and Haofen Wang . 2023b . Retrieval-augmented generation for large language models: A survey . arXiv preprint arxiv: 2312.10997 . ↵ Patricia Garcia , Stephen P Ma , Shreya Shah , Margaret Smith , Yejin Jeong , Anna Devon-Sand , Ming TaiSeale , Kevin Takazawa , Danyelle Clutter , Kyle Vogt , et al. 2024 . Artificial intelligence–generated draft replies to patient inbox messages . JAMA Network Open , 7 ( 3 ): e243201 – e243201 . OpenUrl CrossRef ↵ Albert Q Jiang , Alexandre Sablayrolles , Arthur Mensch , Chris Bamford , Devendra Singh Chaplot , Diego de las Casas , Florian Bressand , Gianna Lengyel , Guillaume Lample , Lucile Saulnier , et al. 2023 . Mistral 7b . arXiv preprint arxiv: 2310.06825 . ↵ Alistair Johnson , Lucas Bulgarelli , Tom Pollard , Steven Horng , Leo Anthony Celi , and Roger Mark . 2020 . Mimic-iv . PhysioNet . Available online at: https://physionet.org/content/mimiciv/1.0/ x(accessed August 23, 2021 ), pages 49 – 55 . Shreya Johri , Jaehwan Jeong , Benjamin A Tran , Daniel I Schlessinger , Shannon Wongvibulsin , Leandra A Barnes , Hong-Yu Zhou , Zhuo Ran Cai , Eliezer M Van Allen , David Kim , et al. 2025 . An evaluation framework for clinical use of large language models in patient interaction tasks . Nature Medicine , pages 1 – 10 . ↵ Archana Laxmisan , Allison B McCoy , Adam Wright , and Dean F Sittig . 2012 . Clinical summarization capabilities of commercially-available and internally-developed electronic health records . Applied clinical informatics , 3 ( 01 ): 80 – 93 . OpenUrl PubMed ↵ Patrick Lewis , Ethan Perez , Aleksandra Piktus , Fabio Petroni , Vladimir Karpukhin , Naman Goyal , Heinrich Küttler , Mike Lewis , Wen-tau Yih , Tim Rocktäschel , et al. 2020 . Retrieval-augmented generation for knowledge-intensive nlp tasks . Advances in Neural Information Processing Systems , 33 : 9459 – 9474 . OpenUrl ↵ Rui Li , Fenglong Ma , and Jing Gao . 2022 . Integrating multimodal electronic health records for diagnosis prediction . In AMIA Annual Symposium Proceedings, volume 2021, page 726 . ↵ Jennifer Liang , Ching-Huei Tsou , and Ananya Poddar . 2019 . A novel system for extractive clinical note summarization using EHR data . In Proceedings of the 2nd Clinical Natural Language Processing Workshop , pages 46 – 54 , Minneapolis, Minnesota, USA . Association for Computational Linguistics . ↵ Chin-Yew Lin . 2004 . Rouge: A package for automatic evaluation of summaries . In Text summarization branches out , pages 74 – 81 . ↵ Fangyu Liu , Ehsan Shareghi , Zaiqiao Meng , Marco Basaldella , and Nigel Collier . 2020 . Self-alignment pretraining for biomedical entity representations . arXiv preprint arxiv: 2010.11784 . ↵ Nelson F Liu , Kevin Lin , John Hewitt , Ashwin Paranjape , Michele Bevilacqua , Fabio Petroni , and Percy Liang . 2024 . Lost in the middle: How language models use long contexts . Transactions of the Association for Computational Linguistics , 12 : 157 – 173 . OpenUrl CrossRef ↵ Ivan Lopez , Akshay Swaminathan , Karthik Vedula , Sanjana Narayanan , Fateme Nateghi Haredasht , Stephen P Ma , April S Liang , Steven Tate , Manoj Maddali , Robert Joseph Gallo , et al. 2025 . Clinical entity augmented retrieval for clinical information extraction . npj Digital Medicine , 8 ( 1 ): 45 . OpenUrl PubMed ↵ Mengxian Lyu , Cheng Peng , Daniel Paredes , Ziyi Chen , Aokun Chen , Jiang Bian , and Yonghui Wu . 2024 . Uf-hobi at “discharge me!”: A hybrid solution for discharge summary generation through prompt-based tuning of gatortrongpt models . In Proceedings of the 23rd Workshop on Biomedical Natural Language Processing , pages 685 – 695 . ↵ Farida Mohsen , Hazrat Ali , Nady El Hajj , and Zubair Shah . 2022 . Artificial intelligence-based methods for fusion of electronic health records and imaging data . Scientific Reports , 12 ( 1 ): 17981 . OpenUrl PubMed ↵ Skatje Myers , Timothy A Miller , Yanjun Gao , Matthew M Churpek , Anoop Mayampurath , Dmitriy Dligach , and Majid Afshar . 2024 . Lessons learned on information retrieval in electronic health records: a comparison of embedding models and pooling strategies . Journal of the American Medical Informatics Association , page ocae308. ↵ Rimma Pivovarov and Noémie Elhadad . 2015 . Automated methods for the summarization of electronic health records . Journal of the American Medical Informatics Association , 22 ( 5 ): 938 – 947 . OpenUrl CrossRef PubMed ↵ Christina Silcox , Eyal Zimlichmann , Katie Huber , Neil Rowen , Robert Saunders , Mark McClellan , Charles N Kahn III, Claudia A Salzberg , and David W Bates . 2024 . The potential for artificial intelligence to transform healthcare: perspectives from international health leaders . NPJ Digital Medicine , 7 ( 1 ): 88 . OpenUrl PubMed ↵ Hardeep Singh , Gordon D Schiff , Mark L Graber , Igho Onakpoya , and Matthew J Thompson . 2017 . The global burden of diagnostic errors in primary care . BMJ quality & safety , 26 ( 6 ): 484 – 494 . OpenUrl Abstract / FREE Full Text ↵ Karan Singhal , Shekoofeh Azizi , Tao Tu , S Sara Mahdavi , Jason Wei , Hyung Won Chung , Nathan Scales , Ajay Tanwani , Heather Cole-Lewis , Stephen Pfohl , et al. 2023 . Large language models encode clinical knowledge . Nature , 620 ( 7972 ): 172 – 180 . OpenUrl CrossRef PubMed ↵ Hugo Touvron , Louis Martin , Kevin Stone , Peter Albert , Amjad Almahairi , Yasmine Babaei , Nikolay Bashlykov , Soumya Batra , Prajjwal Bhargava , Shruti Bhosale , et al. 2023 . Llama 2: Open foundation and fine-tuned chat models . arXiv preprint arxiv: 2307.09288 . ↵ Robert M Wachter and Erik Brynjolfsson . 2024 . Will generative artificial intelligence deliver on its promise in health care? Jama , 331 ( 1 ): 65 – 69 . OpenUrl CrossRef PubMed ↵ Sijia Wang , Mo Yu , and Lifu Huang . 2023 . The art of prompting: Event detection based on type specific prompts . In The 61st Annual Meeting Of The Association For Computational Linguistics . ↵ Lawrence L Weed et al. 1968 . Medical records that guide and teach . N Engl J Med , 278 ( 11 ): 593 – 600 . OpenUrl CrossRef PubMed Web of Science ↵ Jason Wei , Xuezhi Wang , Dale Schuurmans , Maarten Bosma , Fei Xia , Ed Chi , Quoc V Le , Denny Zhou , et al. 2022 . Chain-of-thought prompting elicits reasoning in large language models . Advances in neural information processing systems , 35 : 24824 – 24837 . OpenUrl ↵ Christopher YK Williams , Brenda Y Miao , Aaron E Kornblith , and Atul J Butte . 2024 . Evaluating the use of large language models to provide clinical recommendations in the emergency department . Nature Communications , 15 ( 1 ): 8236 . OpenUrl PubMed ↵ Michael Wornow , Rahul Thapa , Ethan Steinberg , Jason Fries , and Nigam Shah . 2023 . Ehrshot: An ehr benchmark for few-shot evaluation of foundation models . ↵ Adam Wright , Dean F Sittig , Julie McGowan , Joan S Ash , and Lawrence L Weed . 2014 . Bringing science to medicine: an interview with larry weed, inventor of the problem-oriented medical record . Journal of the American Medical Informatics Association , 21 ( 6 ): 964 – 968 . OpenUrl CrossRef PubMed ↵ Shitao Xiao , Zheng Liu , Peitian Zhang , and Niklas Muennighoff . 2023 . C-pack: Packaged resources to advance general chinese embedding . Preprint , arxiv: 2309.07597 . ↵ Justin Xu , Zhihong Chen , Andrew Johnston , Louis Blankemeier , Maya Varma , Jason Hom , William J. Collins , Ankit Modi , Robert Lloyd , Benjamin Hopkins , Curtis Langlotz , and Jean-Benoit Delbrouck . 2024a . Overview of the first shared task on clinical text generation: RRG24 and “discharge me!” . In Proceedings of the 23rd Workshop on Biomedical Natural Language Processing , pages 85 – 98 , Bangkok, Thailand . Association for Computational Linguistics . ↵ Justin Xu , Zhihong Chen , Andrew Johnston , Louis Blankemeier , Maya Varma , Jason Hom , William J Collins , Ankit Modi , Robert Lloyd , Benjamin Hopkins , et al. 2024b . Overview of the first shared task on clinical text generation: Rrg24 and “discharge me!” . In Proceedings of the 23rd Workshop on Biomedical Natural Language Processing , pages 85 – 98 . Peng Xu , Wei Ping , Xianchao Wu , Lawrence McAfee , Chen Zhu , Zihan Liu , Sandeep Subramanian , Evelina Bakhturina , Mohammad Shoeybi , and Bryan Catanzaro . Retrieval meets long context large language models . In The Twelfth International Conference on Learning Representations . ↵ An Yang , Baosong Yang , Beichen Zhang , Binyuan Hui , Bo Zheng , Bowen Yu , Chengyuan Li , Dayiheng Liu , Fei Huang , Haoran Wei , et al. 2024 . Qwen2. 5 technical report . arXiv preprint arxiv: 2412.15115 . ↵ Bowen Yu , Cheng Fu , Haiyang Yu , Fei Huang , and Yongbin Li . 2023 . Unified language representation for question answering over text, tables, and images . In Findings of the Association for Computational Linguistics: ACL 2023 , pages 4756 – 4765 . ↵ Chenhan Yuan , Qianqian Xie , and Sophia Ananiadou . 2023 . Zero-shot temporal relation extraction with ChatGPT . In The 22nd Workshop on Biomedical Natural Language Processing and BioNLP Shared Tasks , pages 92 – 102 , Toronto, Canada . Association for Computational Linguistics . ↵ Tianyi Zhang , Varsha Kishore , Felix Wu , Kilian Q Weinberger , and Yoav Artzi . 2019 . Bertscore: Evaluating text generation with bert . arXiv preprint arxiv: 1904.09675 . View the discussion thread. Back to top Previous Next Posted July 23, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Zero-Shot Large Language Models for Long Clinical Text Summarization with Temporal Reasoning Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Zero-Shot Large Language Models for Long Clinical Text Summarization with Temporal Reasoning Maya Kruse , Shiyue Hu , Nicholas Derby , Yifu Wu , Samantha Stonbraker , Bingsheng Yao , Dakuo Wang , Elizabeth Goldberg , Yanjun Gao medRxiv 2025.07.21.25331947; doi: https://doi.org/10.1101/2025.07.21.25331947 Share This Article: Copy Citation Tools Zero-Shot Large Language Models for Long Clinical Text Summarization with Temporal Reasoning Maya Kruse , Shiyue Hu , Nicholas Derby , Yifu Wu , Samantha Stonbraker , Bingsheng Yao , Dakuo Wang , Elizabeth Goldberg , Yanjun Gao medRxiv 2025.07.21.25331947; doi: https://doi.org/10.1101/2025.07.21.25331947 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15228) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6599) Geriatric Medicine (668) Health Economics (997) Health Informatics (4536) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3332) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9231) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a00719a29993c13d',t:'MTc3OTU3MTE3OQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();