Development of an LLM Pipeline Surpassing Physicians in Cardiovascular Risk Score Calculation

Development of an LLM Pipeline Surpassing Physicians in Cardiovascular Risk Score Calculation | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Development of an LLM Pipeline Surpassing Physicians in Cardiovascular Risk Score Calculation View ORCID Profile Tobias Roeschl , View ORCID Profile Marie Hoffmann , View ORCID Profile Axel Unbehaun , View ORCID Profile Henryk Dreger , View ORCID Profile Gerhard Hindricks , View ORCID Profile Volkmar Falk , View ORCID Profile Ran Balicer , View ORCID Profile Radu Tanacli , View ORCID Profile Felix Hohendanner , View ORCID Profile Alexander Meyer doi: https://doi.org/10.1101/2025.11.11.25340002 Tobias Roeschl 1 Deutsches Herzzentrum der Charité, Department of Cardiology, Angiology and Intensive Care Medicine , Charitéplatz 1, 10117 Berlin, Germany 2 Charité – Universitätsmedizin Berlin, corporate member of Freie Universität Berlin and Humboldt Universität zu Berlin , Charitéplatz 1, 10117 Berlin, Germany 3 Charité – Universitätsmedizin Berlin, Institute for Artificial Intelligence in Medicine , Invalidenstraße 120, 10115 Berlin, Germany 4 Berlin Institute of Health at Charité – Universitätsmedizin Berlin , Charitéplatz 1, 10117 Berlin, Germany 5 DZHK (German Centre for Cardiovascular Research) , partner site Berlin, Berlin, Germany 6 Deutsches Herzzentrum der Charité, Department of Cardiothoracic and Vascular Surgery , Augustenburger Platz 1, 13353 Berlin, Germany MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Tobias Roeschl Marie Hoffmann 2 Charité – Universitätsmedizin Berlin, corporate member of Freie Universität Berlin and Humboldt Universität zu Berlin , Charitéplatz 1, 10117 Berlin, Germany 3 Charité – Universitätsmedizin Berlin, Institute for Artificial Intelligence in Medicine , Invalidenstraße 120, 10115 Berlin, Germany 5 DZHK (German Centre for Cardiovascular Research) , partner site Berlin, Berlin, Germany 6 Deutsches Herzzentrum der Charité, Department of Cardiothoracic and Vascular Surgery , Augustenburger Platz 1, 13353 Berlin, Germany PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Marie Hoffmann Axel Unbehaun 2 Charité – Universitätsmedizin Berlin, corporate member of Freie Universität Berlin and Humboldt Universität zu Berlin , Charitéplatz 1, 10117 Berlin, Germany 6 Deutsches Herzzentrum der Charité, Department of Cardiothoracic and Vascular Surgery , Augustenburger Platz 1, 13353 Berlin, Germany MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Axel Unbehaun Henryk Dreger 2 Charité – Universitätsmedizin Berlin, corporate member of Freie Universität Berlin and Humboldt Universität zu Berlin , Charitéplatz 1, 10117 Berlin, Germany 7 Deutsches Herzzentrum der Charité, Department of Cardiology, Angiology and Intensive Care Medicine , Augustenburger Platz 1, 13353 Berlin, Germany MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Henryk Dreger Gerhard Hindricks 1 Deutsches Herzzentrum der Charité, Department of Cardiology, Angiology and Intensive Care Medicine , Charitéplatz 1, 10117 Berlin, Germany 2 Charité – Universitätsmedizin Berlin, corporate member of Freie Universität Berlin and Humboldt Universität zu Berlin , Charitéplatz 1, 10117 Berlin, Germany MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Gerhard Hindricks Volkmar Falk 2 Charité – Universitätsmedizin Berlin, corporate member of Freie Universität Berlin and Humboldt Universität zu Berlin , Charitéplatz 1, 10117 Berlin, Germany 5 DZHK (German Centre for Cardiovascular Research) , partner site Berlin, Berlin, Germany 6 Deutsches Herzzentrum der Charité, Department of Cardiothoracic and Vascular Surgery , Augustenburger Platz 1, 13353 Berlin, Germany 8 Translational Cardiovascular Technologies, Institute of Translational Medicine, Department of Health Sciences and Technology, Swiss Federal Institute of Technology , 8092 Zürich, Switzerland MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Volkmar Falk Ran Balicer 2 Charité – Universitätsmedizin Berlin, corporate member of Freie Universität Berlin and Humboldt Universität zu Berlin , Charitéplatz 1, 10117 Berlin, Germany 9 Innovation Division, Clalit Health Services, Clalit Research Institute , Tel Aviv, Israel Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Ran Balicer Radu Tanacli 2 Charité – Universitätsmedizin Berlin, corporate member of Freie Universität Berlin and Humboldt Universität zu Berlin , Charitéplatz 1, 10117 Berlin, Germany 7 Deutsches Herzzentrum der Charité, Department of Cardiology, Angiology and Intensive Care Medicine , Augustenburger Platz 1, 13353 Berlin, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Radu Tanacli Felix Hohendanner 1 Deutsches Herzzentrum der Charité, Department of Cardiology, Angiology and Intensive Care Medicine , Charitéplatz 1, 10117 Berlin, Germany 2 Charité – Universitätsmedizin Berlin, corporate member of Freie Universität Berlin and Humboldt Universität zu Berlin , Charitéplatz 1, 10117 Berlin, Germany 7 Deutsches Herzzentrum der Charité, Department of Cardiology, Angiology and Intensive Care Medicine , Augustenburger Platz 1, 13353 Berlin, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Felix Hohendanner Alexander Meyer 2 Charité – Universitätsmedizin Berlin, corporate member of Freie Universität Berlin and Humboldt Universität zu Berlin , Charitéplatz 1, 10117 Berlin, Germany 3 Charité – Universitätsmedizin Berlin, Institute for Artificial Intelligence in Medicine , Invalidenstraße 120, 10115 Berlin, Germany 4 Berlin Institute of Health at Charité – Universitätsmedizin Berlin , Charitéplatz 1, 10117 Berlin, Germany 5 DZHK (German Centre for Cardiovascular Research) , partner site Berlin, Berlin, Germany 10 Berlin Institute for the Foundations of Learning and Data – TU Berlin , Berlin, Germany MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Alexander Meyer For correspondence: alexander.meyer{at}dhzc-charite.de Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Background Risk scores are essential to evidence-based cardiovascular care, but manual calculation is labor-intensive and error-prone. Large language models (LLMs) could automate this process, yet LLMs are limited by their propensity for calculation errors and factual hallucinations. Pipelines that separate LLM-based data extraction from deterministic score computation may improve reliability and transparency. Methods We conducted a retrospective diagnostic study at a quaternary heart center in Germany (January 2020 – July 2023). Patients with atrial fibrillation (n=179) from an ablation registry and patients with severe aortic stenosis (n=76) evaluated by a heart team were included. Five LLMs (DeepSeek-R1, Qwen3, GPT-4 Turbo, Llama 3.1, and PaLM 2) were tested in standalone and pipeline configurations to compute HAS-BLED, CHA₂DS₂-VASc, and EuroSCORE II scores from routine clinical reports. Accuracy was assessed by comparing predictions to expert-adjudicated ground truth, using root mean squared error (RMSE), Krippendorff’s alpha for categorical agreement, and calibration analysis. Results Pipeline-generated scores showed substantially higher agreement with expert adjudication than standalone LLMs and treating clinicians (mean Krippendorff’s alpha: 0.79 vs 0.32 vs 0.31) and demonstrated superior calibration. The Qwen3-based pipeline, achieved the highest accuracy with lower RMSEs than clinicians for HAS-BLED (0.20 vs 0.87), CHA₂DS₂-VASc (0.53 vs 1.08), and EuroSCORE II (1.99 vs 2.05). Conclusion LLM-based pipelines enable accurate, well-calibrated, and scalable cardiovascular risk score computation from unstructured real-world clinical data, outperforming clinicians and standalone LLMs with the potential to reduce clinician workload and support evidence-based care. Introduction Clinical risk scores are foundational instruments in medicine, translating complex patient data into actionable risk estimates that guide therapeutic decisions and support guideline-adherent care. As medicine becomes increasingly data-driven 1 , accurate calculation of these scores is vital. Typically, physicians must manually extract variables from unstructured electronic health records – a workflow that is both laborious and susceptible to error. Large language models (LLMs) could automate such text-heavy clinical tasks, yet their application to quantitative risk scoring is fundamentally undermined by well-documented failure modes. These models are prone to factual and contextual hallucinations 2 – generating confident but incorrect outputs – and often exhibit poor arithmetic reasoning 3 – 5 , limitations that are particularly hazardous where patient safety and guideline adherence are at stake. While recent work 6 has shown that integrating LLMs with external calculators can improve numerical accuracy, a critical translational gap persists. Previous studies have relied on synthetic vignettes or concise case reports, which do not reflect the complexity and noise of authentic clinical documents such as multi-page discharge letters or imaging reports 3 , 6 – 8 . Performance of these systems in a real-world clinical setting therefore remains unknown. To date, no study has validated an LLM-based system for risk score calculation on authentic, unstructured patient reports, or rigorously benchmarked against physicians. In our study, we aimed to address these gaps by evaluating how accurately standalone LLMs and LLM-based pipelines compute cardiovascular risk scores from routine clinical text reports compared to treating physicians. The LLM-based pipeline we developed integrates the language-understanding capabilities of LLMs with the reliability of deterministic computation. It comprises three stages: (1) expert-curated, guideline-concordant score definitions, (2) LLM-driven extraction of relevant data from original clinical reports, and (3) a separate, formula-based engine for the final score computation. We conducted this comparison using three widely adopted cardiovascular risk scores – CHA₂DS₂-VASc⁹, HAS-BLED¹⁰, and EuroSCORE II¹¹– across two patient cohorts. Our findings show that pipeline models achieve superior accuracy and calibration, outperforming both standalone LLMs and treating physicians. This demonstrates its potential as a robust and scalable framework for automated clinical risk assessment. Methods Study Design and Patient Cohorts Our retrospective study included two distinct patient cohorts from a single quaternary care heart center. The first cohort included 179 of 182 consecutively screened adult patients from an in-house registry who underwent catheter ablation for atrial fibrillation (AF) between January 2020 and July 2023, each with a digitized discharge letter. The second cohort included 76 of 90 adults with severe aortic stenosis (AS) evaluated by one of our institutional heart teams during 2022, each with comprehensive digitized documentation. For the AS cohort, sufficient documentation was defined as the availability of at least one physician letter, echocardiography report, cardiac catheterization report, computed tomography (CT) report, and a heart team protocol. Patient characteristics for both cohorts are detailed in Supplementary Table 1 and Supplementary Table 2 . The study protocol was approved by the research ethics committee of Charité – Universitätsmedizin Berlin (EA2/183/23). Data Acquisition and Preparation All clinical documents were extracted as Portable Document Format (PDF) files from the hospital information system and converted into plain text using optical character recognition. For each patient in the AF cohort, the physician letter following the ablation procedure was used as the input document. For the AS cohort, a comprehensive single document was created for each patient by concatenating their two most recent discharge letters, the invasive coronary angiography report, echocardiography report, and CT scan report. This was done to simulate a complete chart review. All reports were manually anonymized by replacing identifiable information with synthetic data, while preserving the original structure and clinical complexity. To maintain temporal relationships between events, all dates for a given patient were shifted by a randomly sampled offset. We manually extracted risk scores documented by treating clinicians in discharge letters and heart team protocols to establish a reference standard for benchmarking. These values were then removed from the documents provided to the LLMs to prevent data leakage. Model Architectures and Prompting Strategies Figure 1 provides a comprehensive overview of the two approaches evaluated. In the standalone LLM-approach, we instructed LLMs to calculate scores directly from clinical reports with reference only to the original score publications. In a single, comprehensive prompt, the full clinical document was provided alongside the instruction to calculate the complete risk score according to the respective original publication and output both the final score and all contributing score components in a structured JSON format. Download figure Open in new tab Figure 1: Experimental design Three experimental arms evaluate risk score derivation in two patient cohorts based on original patient documents. Left: Risk scores treating physicians had manually calculated during clinical practice. Center: In the standalone LLM approach, a single prompt is used to directly predict the risk score end-to-end from patient documents. Right: In LLM pipeline, a three-stage process is applied: Stage I, Medical knowledge is incorporated into prompts for granular data extraction; Stage II, the LLM outputs structured values for each variable; and Stage III, a deterministic scoring engine derives aggregated component values and applies validated formulas to compute the final risk score. In the pipeline approach, risk score computation was organized into three stages: (1) provision of score component definitions, including surrogate parameters, enhanced by domain-specific knowledge incorporated into all prompt templates to integrate clinical expertise (e.g., symptom interpretation criteria) and specifications from the original score publications; (2) LLM-based extraction of structured data from clinical reports; and (3) deterministic calculation of the final score. The inclusion of surrogate parameters beyond the classical score components enabled a more comprehensive assessment of patient risk. For example, the models were prompted to use the most recent HbA1c value to infer the presence of diabetes mellitus and assign a corresponding point in the CHA₂DS₂-VASc score, even when the diagnosis was not explicitly mentioned in the discharge letter. To improve extraction accuracy, we issued a separate query for each individual score component. A core prompt template was used for each score component across all models. Prompt templates are provided in the Supplementary material, and representative examples are shown in Figure 2 . As in the standalone LLM approach, we instructed the LLMs to output structured output in JSON format. We developed a deterministic scoring engine, coded in Python, that used a rule-based approach to further process the extracted data and apply logical and arithmetic operations to compute the final score. Download figure Open in new tab Figure 2: Prompt templates Representative examples of the prompt templates used in the CHA 2 DS 2 -VASc scoring pipeline. In both approaches we tested five different LLMs - two optimized for reasoning (DeepSeek-R1, Qwen3-235B-A22B) and three leading models (GPT-4 Turbo, Llama 3.1, PaLM 2). All prompts were executed in a zero-shot configuration with the temperature parameter set to zero to minimize output variation and enhance reproducibility. Ground Truth Adjudication Process To establish a reference standard for all risk scores, two senior resident cardiologists independently reviewed all patient documents for both cohorts, blinded to the scores documented by the treating physicians. The cardiologists used a standardized data collection form detailing the definitions for every component of the HAS-BLED, CHA 2 DS 2 -VASc, and EuroSCORE II scores, as specified in their original publications ( Supplementary Table 3 ). Raters were instructed to use the most recent value for any duplicate entries and to assume normal reference ranges for missing laboratory values. A standardized mapping was used for qualitative descriptions of left-ventricular ejection fraction (e.g., ‘moderately impaired’ was mapped to 35%). Any disagreements between the two primary raters were resolved by a third, independent senior cardiologist to reach a final consensus for each score component. Ground truth patient ages were programmatically calculated based on the date of birth and either the date of discharge or the date of the heart team meeting. Ground truth scores were calculated from these components using the score points and regression coefficients reported in the original publications, respectively. To evaluate agreement between clinical raters during the establishment of ground truth values for score components, we employed Krippendorff’s alpha, using the nominal or ordinal variant as appropriate based on the measurement level of each variable. Due to the instability or inapplicability of these metrics for instances with sparse data, we additionally reported raw agreement as the proportion of identical ratings between raters. Once consensus ground truth scores were established, we evaluated the performance of standalone LLMs, pipeline models and physician-derived scores against these reference standards. Performance metrics and Statistical Analysis Patient characteristics were reported as counts and percentages for binary variables, as means with 95% confidence intervals for normally distributed continuous variables, and as medians with interquartile ranges for non-normally distributed variables. Normality was assessed using the Shapiro–Wilk test. We evaluated agreement between risk scores derived by the standalone LLMs, the pipeline models, and treating physicians, and the ground truth scores across clinically relevant score categories. Specifically, we used the following threshold groupings: <2 vs ≥2 for CHA 2 DS 2 -VASc 12 , <3 vs ≥3 for HAS-BLED 10 , and 8% for EuroSCORE II 13 . Agreement across these categories was assessed using Krippendorff’s alpha, excluding instances with missing data. Prediction accuracy was quantified using root mean squared error (RMSE) between each derived score and the ground truth. Calibration of physician-derived scores and pipeline model-derived scores was evaluated using kernel density estimation (KDE) plots, with ground truth scores on the x-axis and predicted scores on the y-axis. The diagonal dashed line indicates perfect calibration, with density contours closer to this line reflecting better calibration. Per-component accuracies, defined as the proportion of correctly assigned values for each score component, were visualized using radar plots. For the age variable, predictions within ±1 year were counted as correct for this metric. Frequency distributions of LLM-derived scores were visualized using histograms to detect output patterns suggestive of repetitive responses. All statistical analyses were conducted using Python version 3.9, scikit-learn 1.6, numpy 1.26, and krippendorff 0.8. Results Standalone LLMs do not ensure valid risk assessments for complex risk scores Applying these evaluation metrics, the standalone LLMs tasked with end-to-end score calculation produced inconsistent and often erroneous results. While some LLMs matched the accuracy of treating physicians for the simpler additive HAS-BLED and CHA 2 DS 2 -VASc scores, all models failed to correctly calculate the more complex EuroSCORE II, which is derived from a logistic regression model applying the logistic function to a weighted linear combination of clinical variables. All LLMs tested markedly overestimated this score. This failure was evident in the extremely high RMSE values and poor agreement for EuroSCORE II scores derived by standalone LLMs ( Figure 3 and Supplementary Table 4 ). Notably, the apparent low RMSE for PaLM 2 was the result of a constant-output hallucination 14 , where the model produced a limited set of fixed values irrespective of the patient data provided ( Supplementary Figure 1 ). These findings confirm that standalone LLMs are currently not reliable for complex clinical calculations from real-world data. Download figure Open in new tab Figure 3: Agreement, error, and calibration Bar plots represent Krippendorff’s alphas and root mean squared errors (RMSE) for scores derived by treating physicians, standalone LLMs, and LLM pipelines, respectively. Agreement was assessed based on clinically relevant risk score categories 15 , 16 i.e., <2 vs ≥2 for CHA 2 DS 2 -VASc 12 , <3 vs ≥3 for HAS-BLED 17 , and 8% for EuroSCORE II 13 . Calibration is visualized using kernel density estimation plots, with the ground truth score on the x-axis and the predicted score on the y-axis. The diagonal dashed line indicates perfect calibration. Densities closer to this line reflect better calibration between predicted and true scores. Knowledge conflicts and algebraic limitations undermine LLM accuracy for complex risk scores Analysis of per-component accuracies of standalone LLMs ( Figure 4 ) revealed two primary failure modes. First, LLMs often relied on their internal, generalized definitions of medical terms instead of the specific criteria used in the original score publications, a phenomenon known as knowledge conflict 15 or contextual hallucination 2 . For instance, models frequently assigned a point for ‘Hypertension’ in the HAS-BLED score even when the patient’s blood pressure was well-controlled and did not meet the score’s specific criterion of systolic blood pressure >160 mmHg ( Supplementary Table 1 ). As shown in the radar plots ( Figure 4 ), this led to poor per-component accuracies for several variables. Download figure Open in new tab Figure 4: Per-component accuracy Radar plots represent mean accuracies for individual score components. Mean per-component accuracies across all LLMs are 0.83 (HAS-BLED), 0.85 (CHA 2 DS 2 -VASc), and 0.81 (EuroSCORE II) for the standalone LLM, and 0.99 (HAS-BLED), 0.96 (CHA 2 DS 2 -VASc), and 0.91 (EuroSCORE II) for the LLM pipeline approach. Score component abbreviations are shown in Supplementary Table 3 . Second, the LLMs demonstrated significant algebraic limitations. Models struggled to correctly classify numerical values, such as categorizing a patient’s age or renal function into the appropriate score strata. In an exploratory analysis, PaLM 2 correctly extracted the patient’s age in 98.3% of cases but failed to assign the correct age-related points for the CHA 2 DS 2 -VASc score in 37.4% of those instances. We also frequently observed arithmetic inconsistencies where the sum of component points did not match the final total score reported by the model. LLM-based Pipelines Overcome LLM Shortcomings in Algebraic Reasoning Our pipeline model, which decouples data extraction from score calculation, successfully mitigated these failure modes. Instructing the LLMs to extract only granular data points (e.g., laboratory values, dates, medication data) and delegating all arithmetic and logical operations to a deterministic downstream algorithm substantially improved performance. For most of the LLMs tested, the pipeline approach clearly outperformed both the standalone LLMs and treating physicians across all risk scores and evaluation metrics ( Figure 3 , Figure 4 , Supplementary Table 4 ). Agreement with the ground truth in clinically relevant categories was markedly higher for the pipeline models compared to scores derived by standalone LLMs and scores documented by treating physicians (mean Krippendorff’s alpha across all scores: 0.79 vs 0.32 vs 0.31). The pipeline models generally achieved lower RMSEs across all scores and demonstrated superior calibration compared to treating physicians. Within the LLM group, models designed for reasoning generally outperformed non-reasoning focused models in both the standalone and pipeline configurations. Notably, treating physicians tended to underestimate patient risk compared to the expert-adjudicated ground truth ( Figure 3 , Supplementary Table 5 ). Current LLMs are reliable data extractors but may be poor medical data interpreters Despite the pipeline’s high performance, per-component accuracies remained low for EuroSCORE II variables that require nuanced clinical interpretation, such as NYHA class and urgency. This suggests that while LLMs are reliable at structured data extraction, they are less adept at interpreting ambiguous clinical context. However, these same score components also exhibited the lowest interrater agreement among the expert cardiologists who established the ground truth ( Supplementary Table 6 ). These finding highlights that the challenge in evaluating such variables is rooted in the interpretive nature of their clinical definitions, affecting both humans and artificial intelligence. LLM-based pipelines show promise in treatment optimization In an exploratory analysis, we assessed whether the pipeline could identify discrepancies between guideline-recommended and actual treatment. The DeepSeek-R1-powered pipeline model correctly identified 11 patients in the AF cohort who had a clear indication for anticoagulation (i.e., CHA 2 DS 2 -VASc score ≥2) as well as two patients who might not have had an indication for anticoagulation therapy (i.e., CHA 2 DS 2 -VASc score <2), contrary to the physician’s assessment, respectively ( Figure 5 ). Furthermore, the model identified four patients with documented diabetes mellitus who lacked a prescription for antidiabetic therapy, highlighting the pipeline’s potential to serve as an automated tool for auditing care quality and optimizing treatment. Download figure Open in new tab Figure 5: Theoretical clinical impact of re-calculated CHA 2 DS 2 -VASc scores (LLM pipeline using DeepSeek-R1) on anticoagulation management in 142 AF patients: 11 patients (dark green) would have been newly and correctly recommended to initiate anticoagulation therapy, while two patients (light green) would have appropriately had therapy discontinued or at least re-evaluated. Two patients who should have received a recommendation for anticoagulation therapy were missed by both the pipeline model and the treating physicians (red). Discussion In summary, we provide the first evidence that an LLM pipelines can calculate clinical risk scores from unstructured, real-world patient data with an accuracy and calibration superior to that of scores documented by treating physicians. Previous studies have highlighted the potential of LLM-based tools for clinical calculations, but their reliance on synthetic vignettes or simplified case reports limited their clinical applicability. Our work directly addresses the calls from those studies to evaluate performance on authentic clinical documents and against the benchmark of real-world human practice, confirming that a thoughtfully architected pipeline can overcome the known limitations of LLMs in this domain. A striking finding of our study was the substantial discrepancy between physician-documented scores and the expert-adjudicated ground truth, with treating physicians frequently underestimating patient risk ( Supplementary Table 5 ). While the precise causes are multifactorial, several explanations are plausible. First, clinicians may have an incomplete recall of the specific, nuanced definitions of score components, a known source of miscalculation 16 , 18 . Second, in fast-paced clinical environments, time constraints may lead to the reuse of outdated scores from previous encounters, a practice that, while pragmatic, contravenes guideline recommendations for regular risk reassessment 12 . Similarly, time constraints in routine clinical practice likely explain the observation that, despite explicit guideline recommendations, the HAS-BLED score was calculated in only 20.1% of patients within the AF cohort. While it could be argued that physicians may have had access to additional clinical information not captured in the written records, this is unlikely to account for the observed discrepancy. Such undocumented risk factors would typically lead to higher calculated scores. In contrast, we observed that the scores documented by treating physicians were, on average, lower than the ground truth scores ( Supplementary Table 1 , Supplementary Table 2 , Supplementary Table 5 ). The successful performance of our pipeline models suggests that such systems could be integrated into clinical decision support tools to automate and standardize risk assessment. By decoupling language understanding from mathematical computation, our approach provides a blueprint for leveraging the strengths of LLMs while mitigating their weaknesses. This may remove a key historical barrier in clinical predictive modelling: the reliance on simple, manually calculable risk scores 19 . The ability of LLMs to perform automated, granular data extraction from free-text could unlock the use of more sophisticated and potentially more accurate predictive models that incorporate a far richer set of variables, including narrative-based data that is currently inaccessible to traditional algorithms. Beyond risk scoring, the pipeline methodology shows promise for broader applications aimed at improving clinical workflow efficiency and patient safety. As our exploratory analysis demonstrated, such a system could be used to audit care and flag potential treatment gaps, such as patients with an indication for anticoagulation or antidiabetic therapy who were not receiving appropriate treatment. This capability could be invaluable for tasks ranging from identifying clinical trial candidates to compiling data for multidisciplinary heart team or tumor board reviews. Limitations Our study has several limitations. The retrospective, single-center design limits generalizability and requires validation in external cohorts. The pipeline depends on expert-crafted prompts, a manual process that can hinder scalability and risks overfitting to institutional documentation. Although emerging frameworks like DSPy 20 and retrieval-augmented generation (RAG) 21 may automate prompt engineering and knowledge integration, these remain nascent and often require substantial manual training data and document curation. Finally, although ground truth was established by a rigorous three-expert consensus, it is a proxy, and inherent subjectivity – especially for variables like NYHA class – is a limitation for both human and AI assessment. Conclusion This study provides a validated framework for using an LLM-based pipeline to perform clinical risk scoring from real-world, unstructured data. We show that this approach is not only feasible but can exceed the accuracy of scores documented during routine clinical care. This work lays the foundation for developing robust AI tools and agent-based applications capable of improving the consistency and reliability of clinical risk stratification, ultimately facilitating safer and more guideline-adherent medical practice. Code Availability The analysis pipeline is provided as an MCP server application under github.com/charite-iaim/mcp-risk-server. Data availability statement The (anonymized) data underlying this article will be shared on reasonable request to the corresponding author. Supplementary Material Supplementary Figures Download figure Open in new tab Supplementary Figure 1: Frequency distribution of EuroSCORE II predictions generated by PaLM 2. The standalone LLM (PaLM 2) produced a limited range of repeated values across heterogeneous patient profiles – suggesting a constant-output hallucination pattern. Supplementary Tables View this table: View inline View popup Download powerpoint Supplementary Table 1: Baseline characteristics of patients with symptomatic atrial fibrillation (n=179) Patient characteristics were reported as counts and percentages for binary variables, as means with 95% confidence intervals (square brackets) for normally distributed continuous variables, and as medians with interquartile ranges (parentheses) for non-normally distributed variables. View this table: View inline View popup Download powerpoint Supplementary Table 2: Baseline characteristics of patients with severe aortic stenosis (n=76) Patient characteristics were reported as counts and percentages for binary variables, as means with 95% confidence intervals (square brackets) for normally distributed continuous variables, and as medians with interquartile ranges (parentheses) for non-normally distributed variables. View this table: View inline View popup Supplementary Table 3: Score definitions Score components, their definitions and the respective points/logistic regression coefficients used in our study are shown for the CHA2DS2-VASc 12 , HAS-BLED 13 and EuroSCORE II 14 next to the abbreviations used in Figure 4 . View this table: View inline View popup Download powerpoint Supplementary Table 4: Performance metrics Krippendorff’s Alpha and root mean squared error (RMSE) for Physician, standalone LLM and LLM pipeline experiments with 95% confidence intervals. View this table: View inline View popup Download powerpoint Supplementary Table 5: Mean error between ground truth and physician-derived scores with 95% confidence intervals N denotes the number of samples with non-missing scores (i.e., available data) used in the analysis. View this table: View inline View popup Download powerpoint Supplementary Table 6: Interrater Agreement Metrics Among Clinical Raters Krippendorff’s alpha is reported for each variable where sufficient variability across categories allowed its calculation. For variables with sparse data—defined as any category represented by a single observation—only raw agreement is presented due to the instability or inapplicability of standard reliability coefficients. Abbreviations as in Supplementary Table 3 . Footnotes ↵ ‡ These authors jointly supervised this work. References 1. ↵ Nateghi R , Aven T . Risk analysis in the age of big data: The promises and pitfalls . Risk Anal . 2021 ; 41 ( 10 ): 1751 – 1758 . OpenUrl PubMed 2. ↵ Chuang YS , Qiu L , Hsieh CY , Krishna R , Kim Y , Glass J . Lookback lens: Detecting and mitigating contextual hallucinations in large language models using only attention maps . ArXiv Prepr ArXi v240707071 . Published online 2024 . 3. ↵ Khandekar N , Jin Q , Xiong G , et al. Medcalc-bench: Evaluating large language models for medical calculations . Adv Neural Inf Process Syst . 2024 ; 37 : 84730 – 84745 . OpenUrl 4. Boye J , Moell B. Large language models and mathematical reasoning failures . ArXiv Prepr ArXiv250211574 . Published online 2025 . 5. ↵ Mirzadeh I , Alizadeh K , Shahrokhi H , Tuzel O , Bengio S , Farajtabar M. Gsm-symbolic: Understanding the limitations of mathematical reasoning in large language models . ArXiv Prepr ArXiv241005229 . Published online 2024 . 6. ↵ Goodell AJ , Chu SN , Rouholiman D , Chu LF . Large language model agents can use tools to perform clinical calculations . Npj Digit Med . 2025 ; 8 ( 1 ): 163 . OpenUrl PubMed 7. Wan N , Jin Q , Chan J , et al. Humans continue to outperform large language models in complex clinical decision-making: a study with medical calculators . ArXiv Prepr ArXiv241105897 . Published online 2024 . 8. ↵ Zhu Y , Wei S , Wang X , Xue K , Zhang X , Zhang S. MeNTi: Bridging medical calculator and LLM agent with nested tool calling . ArXiv Prepr ArXiv241013610 . Published online 2024 . 9. Lip GY , Nieuwlaat R , Pisters R , Lane DA , Crijns HJ . Refining clinical risk stratification for predicting stroke and thromboembolism in atrial fibrillation using a novel risk factor-based approach: the euro heart survey on atrial fibrillation . Chest . 2010 ; 137 ( 2 ): 263 – 272 . OpenUrl CrossRef PubMed Web of Science 10. ↵ Pisters R , Lane DA , Nieuwlaat R , De Vos CB , Crijns HJGM , Lip GYH . A Novel User-Friendly Score (HAS-BLED) To Assess 1-Year Risk of Major Bleeding in Patients With Atrial Fibrillation . Chest . 2010 ; 138 ( 5 ): 1093 – 1100 . doi: 10.1378/chest.10-0134 OpenUrl CrossRef PubMed Web of Science 11. Nashef SAM , Roques F , Sharples LD , et al. EuroSCORE II . Eur J Cardiothorac Surg . 2012 ; 41 ( 4 ): 734 - 745 . doi: 10.1093/ejcts/ezs043 OpenUrl CrossRef PubMed 12. ↵ Hindricks G , Potpara T , Dagres N , et al. 2020 ESC guidelines for the diagnosis and management of atrial fibrillation developed in collaboration with the european association for cardio-thoracic surgery (EACTS) the task force for the diagnosis and management of atrial fibrillation of the european society of cardiology (ESC) developed with the special contribution of the european heart rhythm association (EHRA) of the ESC . Eur Heart J . 2021 ; 42 ( 5 ): 373 - 498 . OpenUrl CrossRef PubMed 13. ↵ Vahanian A , Beyersdorf F , Praz F , et al. 2021 ESC/EACTS Guidelines for the management of valvular heart disease: Developed by the Task Force for the management of valvular heart disease of the European Society of Cardiology (ESC) and the European Association for Cardio-Thoracic Surgery (EACTS) . Eur Heart J . 2022 ; 43 ( 7 ): 561 - 632 . doi: 10.1093/eurheartj/ehab395 OpenUrl CrossRef PubMed 14. ↵ Greco CM , La Cava L , Tagarelli A. Talking the talk does not entail walking the walk: On the limits of large language models in lexical entailment recognition . ArXiv Prepr ArXiv240614894 . Published online 2024 . 15. ↵ Longpre S , Perisetla K , Chen A , Ramesh N , DuBois C , Singh S. Entity-based knowledge conflicts in question answering . ArXiv Prepr ArXiv210905052 . Published online 2021 . 16. ↵ Zhang J , Lenarczyk R , Marin F , et al. The interpretation of CHA2DS2-vasc score components in clinical practice: a joint survey by the european heart rhythm association (EHRA) scientific initiatives committee, the EHRA young electrophysiologists, the association of cardiovascular nursing and allied professionals, and the european society of cardiology council on stroke . EP Eur . 2020 ; 23 ( 2 ): 314 – 322 . doi: 10.1093/europace/euaa358 OpenUrl CrossRef 17. ↵ Pisters R , Lane DA , Nieuwlaat R , De Vos CB , Crijns HJ , Lip GY . A novel user-friendly score (HAS-BLED) to assess 1-year risk of major bleeding in patients with atrial fibrillation: the Euro Heart Survey . Chest . 2010 ; 138 ( 5 ): 1093 – 1100 . OpenUrl CrossRef PubMed Web of Science 18. ↵ Zhang C , Shen L , Pan MM , Zheng YL , Gu ZC , Lin HW . Perceptions and knowledge gaps on CHA2DS2-VASc score components: a joint survey of Chinese clinicians and clinical pharmacists . Postgrad Med . 2022 ; 134 ( 1 ): 64 – 77 . doi: 10.1080/00325481.2021.1996815 OpenUrl CrossRef 19. ↵ Sachs MC , Sjölander A , Gabriel EE . Aim for clinical utility, not just predictive accuracy . Epidemiol Camb Mass . 2020 ; 31 ( 3 ): 359 – 364 . OpenUrl 20. ↵ Khattab O , Singhvi A , Maheshwari P , et al. Dspy: Compiling declarative language model calls into self-improving pipelines . ArXiv Prepr ArXiv231003714 . Published online 2023 . 21. ↵ Lewis P , Perez E , Piktus A , et al. Retrieval-augmented generation for knowledge-intensive nlp tasks . Adv Neural Inf Process Syst . 2020 ; 33 : 9459 – 9474 . OpenUrl View the discussion thread. Back to top Previous Next Posted November 13, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Development of an LLM Pipeline Surpassing Physicians in Cardiovascular Risk Score Calculation Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Development of an LLM Pipeline Surpassing Physicians in Cardiovascular Risk Score Calculation Tobias Roeschl , Marie Hoffmann , Axel Unbehaun , Henryk Dreger , Gerhard Hindricks , Volkmar Falk , Ran Balicer , Radu Tanacli , Felix Hohendanner , Alexander Meyer medRxiv 2025.11.11.25340002; doi: https://doi.org/10.1101/2025.11.11.25340002 Share This Article: Copy Citation Tools Development of an LLM Pipeline Surpassing Physicians in Cardiovascular Risk Score Calculation Tobias Roeschl , Marie Hoffmann , Axel Unbehaun , Henryk Dreger , Gerhard Hindricks , Volkmar Falk , Ran Balicer , Radu Tanacli , Felix Hohendanner , Alexander Meyer medRxiv 2025.11.11.25340002; doi: https://doi.org/10.1101/2025.11.11.25340002 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (297) Cardiovascular Medicine (4417) Dentistry and Oral Medicine (443) Dermatology (381) Emergency Medicine (606) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1507) Epidemiology (15209) Forensic Medicine (30) Gastroenterology (1120) Genetic and Genomic Medicine (6578) Geriatric Medicine (666) Health Economics (995) Health Informatics (4516) Health Policy (1366) Health Systems and Quality Improvement (1610) Hematology (538) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15906) Intensive Care and Critical Care Medicine (1103) Medical Education (620) Medical Ethics (144) Nephrology (667) Neurology (6578) Nursing (345) Nutrition (998) Obstetrics and Gynecology (1139) Occupational and Environmental Health (956) Oncology (3321) Ophthalmology (970) Orthopedics (369) Otolaryngology (420) Pain Medicine (435) Palliative Medicine (129) Pathology (663) Pediatrics (1689) Pharmacology and Therapeutics (691) Primary Care Research (710) Psychiatry and Clinical Psychology (5425) Public and Global Health (9211) Radiology and Imaging (2192) Rehabilitation Medicine and Physical Therapy (1368) Respiratory Medicine (1194) Rheumatology (593) Sexual and Reproductive Health (709) Sports Medicine (529) Surgery (709) Toxicology (99) Transplantation (288) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9fef1fe4ee1809d6',t:'MTc3OTMxOTc3OA=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();