Evaluating Accuracy and Reasoning Capabilities of Large Language Models for Acute Ischemic Stroke Management

Evaluating Accuracy and Reasoning Capabilities of Large Language Models for Acute Ischemic Stroke Management | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Evaluating Accuracy and Reasoning Capabilities of Large Language Models for Acute Ischemic Stroke Management View ORCID Profile Aymen Meddeb , Navid Bakhtiari , View ORCID Profile Ida Rangus , Leonard Fetscher , Bastien Leguellec , View ORCID Profile Felix Busch , Alexandre Doucet , View ORCID Profile Vi-Tuan Hua , Fuong Verot-Nguyen , Laurentiu Valentin Paiusan , View ORCID Profile Pierre Francois Manceau , View ORCID Profile Paolo Pagano , Mike P. Wattjes , View ORCID Profile Solene Moulin , View ORCID Profile Laurent Pierot , View ORCID Profile Sebastien Soize doi: https://doi.org/10.1101/2025.09.03.25335060 Aymen Meddeb 1 Department of Neuroradiology, Hôpital Maison-Blanche, CHU Reims, Université Reims- Champagne-Ardenne , Reims, France 2 Department of Neuroradiology, Charité – Universitätsmedizin Berlin , Berlin, Germany 3 Berlin Institute of Health at Charité – Universitätsmedizin Berlin Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Aymen Meddeb For correspondence: aymen.meddeb{at}charite.de Navid Bakhtiari 1 Department of Neuroradiology, Hôpital Maison-Blanche, CHU Reims, Université Reims- Champagne-Ardenne , Reims, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ida Rangus 5 Center for Stroke Research Berlin, Charité – Universitätsmedizin Berlin , Berlin, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Ida Rangus Leonard Fetscher 2 Department of Neuroradiology, Charité – Universitätsmedizin Berlin , Berlin, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site Bastien Leguellec 6 Department of Neuroradiology, Lille University Hospital, Salengro Hospital , Lille, France 7 2U1172 LilNCog, Lille Neuroscience & Cognition, Université Lille , Lille, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site Felix Busch 8 Institute for Diagnostic and Interventional Radiology, School of Medicine and Health, TUM Klinikum, Technical University of Munich (TUM) , Munich, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Felix Busch Alexandre Doucet 1 Department of Neuroradiology, Hôpital Maison-Blanche, CHU Reims, Université Reims- Champagne-Ardenne , Reims, France 4 Department of Neurology, Hôpital Maison-Blanche, CHU Reims, Université Reims- Champagne-Ardenne , Reims, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site Vi-Tuan Hua 4 Department of Neurology, Hôpital Maison-Blanche, CHU Reims, Université Reims- Champagne-Ardenne , Reims, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Vi-Tuan Hua Fuong Verot-Nguyen 1 Department of Neuroradiology, Hôpital Maison-Blanche, CHU Reims, Université Reims- Champagne-Ardenne , Reims, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site Laurentiu Valentin Paiusan 1 Department of Neuroradiology, Hôpital Maison-Blanche, CHU Reims, Université Reims- Champagne-Ardenne , Reims, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site Pierre Francois Manceau 1 Department of Neuroradiology, Hôpital Maison-Blanche, CHU Reims, Université Reims- Champagne-Ardenne , Reims, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Pierre Francois Manceau Paolo Pagano 1 Department of Neuroradiology, Hôpital Maison-Blanche, CHU Reims, Université Reims- Champagne-Ardenne , Reims, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Paolo Pagano Mike P. Wattjes 2 Department of Neuroradiology, Charité – Universitätsmedizin Berlin , Berlin, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site Solene Moulin 4 Department of Neurology, Hôpital Maison-Blanche, CHU Reims, Université Reims- Champagne-Ardenne , Reims, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Solene Moulin Laurent Pierot 1 Department of Neuroradiology, Hôpital Maison-Blanche, CHU Reims, Université Reims- Champagne-Ardenne , Reims, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Laurent Pierot Sebastien Soize 1 Department of Neuroradiology, Hôpital Maison-Blanche, CHU Reims, Université Reims- Champagne-Ardenne , Reims, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Sebastien Soize Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Background Acute ischemic stroke (AIS) management has evolved substantially over the past two decades, with mechanical thrombectomy adding complexity that requires specialized centers. As many patients initially present to primary care facilities, rapihtmld and accurate triage is critical. Large language models (LLMs) may help bridge expertise gaps, especially where stroke specialists are not immediately available. This study evaluates the diagnostic accuracy and reasoning quality of four LLMs in determining eligibility for intravenous thrombolysis (IVT) and mechanical thrombectomy (MT), compared with experienced clinicians and real-world treatment decisions. Methods We retrospectively collected 80 acute ischemic stroke cases from two stroke centers. Cases were presented to LLMs as well to clinicians as clinical vignettes containing demographic, clinical, and imaging data. Four LLMs (DeepSeek R1, OpenAI o3 mini, Gemini 2.0, LLaMA 3.3) and six stroke experts (two neurologists, four neuroradiologists) independently reviewed the cases and recommended one or more treatment strategies including IVT and MT. The ground truth was defined as the institutional treatment decision. Accuracy for MT and IVT recommendations was calculated for both LLMs and clinicians. Additionally, a qualitative error analysis evaluated the reasoning ability of LLMs. Results Open-source reasoning model DeepSeek R1 outperformed all other LLMs and clinicians for MT (87% accuracy) and achieved 78% accuracy for IVT. Across models, accuracy was generally higher for MT than for IVT. Neurologists reached 81% (MT) and 80% (IVT), while neuroradiologists achieved 84% (MT) and 76% (IVT). Reasoning analysis for MT recommendations showed that most errors were clinically reasonable but differed from real-world decision, whereas IVT errors were primarily due to guideline non-adherence. Conclusions LLMs can match or even exceed expert clinician performance in MT and IVT eligibility decisions, while providing transparent reasoning. These findings support prospective evaluation of LLM-based decision support in acute stroke care, especially in settings without immediate specialist expertise. Introduction Since the landmark trials of 2015, mechanical thrombectomy (MT) has become the standard of care for selected patients with acute ischemic stroke due to large vessel occlusion (LVO) in the anterior circulation [ 1 , 2 ]. This therapeutic breakthrough has led to significant reductions in morbidity and mortality and has catalyzed national and international efforts to improve care for stroke patients [ 3 ]. Comprehensive stroke centers and tertiary hospitals have been at the forefront of these efforts, developing multidisciplinary stroke teams and training programs to ensure that stroke neurologists and interventional neuroradiologists are equipped with the expertise needed to deliver timely and effective reperfusion therapies [ 4 ]. Simultaneously, infrastructural investments have sought to optimize imaging workflows, streamline triage algorithms, and facilitate rapid patient transfers where needed [ 5 ]. However, a critical gap remains in the delivery of stroke care: patients presenting to non-tertiary or primary care centers are frequently assessed by physicians lacking expertise in stroke neurology or interventional neuroradiology [ 6 ]. In such settings, the decision whether or not to refer a patient for thrombectomy should be based on an interplay of clinical, radiological, and logistical factors, which may overwhelm non-specialists, particularly under time pressure [ 7 , 8 ]. This expertise gap leads to delays in reperfusion, inappropriate triage, or missed treatment opportunities, ultimately impacting patient outcomes [ 9 ]. Bridging this gap with scalable, accessible, and expert-level decision support is a pressing challenge in stroke systems of care. Artificial intelligence, particularly through the emergence of large language models (LLMs), offers a promising pathway to addressing this challenge[ 10 ]. Beyond their initial applications in general language tasks, LLMs have rapidly found utility in medicine [ 11 ]. In neurology, LLMs have demonstrated impressive performance: passing board certification-style examinations [ 12 ], solving complex diagnostic cases, interpreting patient records [ 13 ], and even simulating empathetic physician-patient interactions to facilitate informed consent discussions [ 14 ]. In radiology, LLMs have been applied to generate structured report summaries [ 15 ], translate reports into multiple languages [ 16 ], extract key findings from unstructured clinical texts [ 17 ], and support diagnostic workflows [ 18 ]. Despite their growing relevance, the potential of LLMs as real-time clinical decision support tools in time-critical real-world scenarios such as acute stroke remains underexplored. Unlike diagnostic classification tasks or post hoc summarization of patient data, stroke management requires nuanced clinical reasoning, integration of multimodal data (e.g., clinical findings, imaging results, timing of symptom onset), and adherence to dynamic treatment guidelines [ 19 ]. In this study, we aim to evaluate the capabilities of state-of-the-art large language models in the triage and management of acute ischemic stroke patients. Specifically, we assess their performance in determining the eligibility for intravenous thrombolysis and mechanical thrombectomy across a cohort of real-world cases. We benchmark their decisions against two standards: (1) the real-world treatment decisions made by stroke teams in tertiary centers, and (2) decisions by experienced stroke neurologists and interventional neuroradiologists. Furthermore, we examine not only the accuracy of LLM recommendations but also the reasoning pathways and potential for clinical harm from incorrect suggestions. By systematically analyzing the strengths and limitations of LLMs in this domain, we aim to evaluate their readiness for deployment in clinical decision-making, especially in settings where expert knowledge is limited or absent. Methods Patient population This study includes data from two independent datasets collected at two university hospitals located in two European countries, Charité – Medical University Berlin, Germany and Centre Hospitalier Universitaire de Reims, France, both certified as comprehensive stroke centers. The first dataset comprises reports from patients who underwent mechanical thrombectomy for acute ischemic stroke between September 2022 and June 2023 at the first institution. The second dataset includes consecutive patient reports from the second university hospital, covering the period from September 2023 to March 2024. All data collection procedures complied with the ethical principles of the Declaration of Helsinki. Beyond the patient’s age and sex, no identifiable features about the patient were transmitted to the LLM models; especially, no patient-identifying information was provided to the AI. This retrospective study received approval from the French National Research Ethics Committee for Medical Imaging (Reference: CRM-2501-444). The requirement for informed consent was waived due to the retrospective nature of the study. LLM Inference and Prompt Structure Two commercial LLMs (OpenAI o3 mini and Gemini 2.0) and two open-source LLMs (LLaMA 3.3 and DeepSeek R1) were used in this study. The prompt was implemented using structured case vignettes derived from anonymized clinical reports. Each clinical vignette was directly incorporated into this template, followed by a clear and unambiguous question: “Patient: [Age, Sex, Medical history, Medications, NIHSS score, Time since symptom onset, Imaging modality, ASPECTS score, Vessel occlusion, Collateral status]. Based on current acute ischemic stroke guidelines, should this patient receive Intravenous Thrombolysis (IVT), Mechanical Thrombectomy (MT), or a combination of both?” LLMs were instructed to provide both a recommendation and a brief justification. Inference was performed using the latest available checkpoints of o3 mini (OpenAI), LLaMA 3.3 (Meta), and Gemini 2.0 (Google), accessed through official APIs, and DeepSeek-R1 through Together AI platform. To reduce randomness and ensure deterministic responses were possible, all LLM sessions were conducted with default model settings, with a temperature parameter set to 0. The very first response generated by each LLM for each vignette was recorded as its final therapeutic decision. No follow-up questions, clarifications, or feedback were provided to the models between cases. Human Rating Six human experts participated in the rating task: two neurologists (I.R., A.D.) and four neuroradiologists (L.V.P., F.V.N, P.F.M, P.P), all with at least five years of experience in stroke care. Each expert independently reviewed the 80 anonymized vignettes, presented in identical order, and selected one or more appropriate treatments (IVT and/or MT). Responses were collected via Google Forms and subsequently encoded into binary variables for each treatment decision. Diagnostic Accuracy Evaluation Final institutional decisions recorded in the clinical charts were defined as the ground truth (GT). For each rater (human or LLM), we computed diagnostic accuracy by comparing the binary predictions for each treatment category (IVT and MT) to the ground truth. Accuracy was calculated per individual and then averaged per group. Inter-rater agreement among human raters was assessed using Fleiss’ Kappa for each treatment and group. To complement accuracy metrics, we calculated the Jaccard index as a measure of agreement between predicted and ground truth treatment decisions. Unlike accuracy, which only captures correct versus incorrect classifications, the Jaccard index accounts for partial or complete overlap for multiple treatment categories. LLM Reasoning Evaluation To assess model reasoning capabilities and identify failure modes, we conducted a structured qualitative error analysis of all false positive and false negative outputs from the four LLMs. Each incorrect LLM decision was independently reviewed by two authors (N.B. and A.M.) and categorized into one of four predefined error types: Omission of critical information: cases where the LLM failed to incorporate key details explicitly mentioned in the input. Guideline non-adherence: cases where the LLM’s decision contradicted published stroke treatment guidelines. Borderline case misclassification: cases involving nuanced scenarios with clinical ambiguity or gray zones, such as IVT eligibility beyond 4.5 hours or borderline ASPECTS scores. Correct recommendation, discrepant ground truth: cases where the LLM’s answer aligned with guidelines and input data but differed from the real-world decision, usually due to information that was unavailable to the LLM but known to clinicians. Disagreements in categorization were resolved by consensus. We then quantified the distribution of these errors across models and treatment categories. Representative examples from each category were extracted to illustrate common reasoning patterns and limitations in model-generated recommendations. Results Patient population The characteristics of the study cohort are shown in Table 1 . The average age of study participants was 68.7 ± 14.7 years, with a near 1:1 sex ratio (51% of female patients, n=41). The patient population exhibited a high prevalence of established vascular risk factors consistent with typical stroke cohorts including hypertension (70%), diabetes mellitus (11%), and atrial fibrillation (10%). Approximately a quarter of the patients were on antiplatelet therapy (22%), and a lower proportion (17%) were treated with anticoagulants. The median NIHSS score on admission was 11 (Interquartile Range [IQR]: 5–16), indicating a patient population presenting with moderate to severe neurological deficits. MRI was performed as the initial neuroimaging modality in 50% of the cases (n = 40), using a standardized protocol that included diffusion-weighted imaging (DWI), fluid-attenuated inversion recovery (FLAIR), time-of-flight (TOF) angiography, and susceptibility-weighted imaging (SWI). 41% of the cases (n=33) received non-contrast CT plus CTA. Both imaging modalities were performed in six patients (7%). The mean ASPECTS score was 8.2 ± 2.1, suggesting limited early ischemic changes on baseline imaging for most patients. View this table: View inline View popup Table 1. Characteristics of the Study Cohort (n=80) Human Rating evaluation Neurologists (n=2) achieved an accuracy of 81% for MT, and 80% for IVT. Neuroradiologists (n=4) achieved an accuracy of 84% for MT, and 76% for IVT. For MT eligibility, the interrater agreement was moderate across neurologists (κ = 0.618) and neuroradiologists (κ = 0.595). For IVT decisions, there was a substantial agreement among neurologists (κ = 0.725) but only fair agreement among neuroradiologists (κ = 0.374). The high Jaccard scores among humans (0.644 for clinicians) indicate a high overlap with ground truth decisions across the full range of treatment options. LLM Diagnostic Accuracy evaluation Accuracy and Jaccard index values for MT and IVT decisions are summarized in Table 2 . Among LLMs, DeepSeek R1 demonstrated the best MT accuracy at 0.87 (95% CI, 0.80– 0.94) and the highest Jaccard index (0.512), followed by OpenAI o3 mini (MT accuracy 0.80; Jaccard 0.473). Gemini 2.0 and LLaMA 3.3 showed lower IVT accuracy (0.65 and 0.63, respectively) and comparatively lower Jaccard scores (0.347 and 0.285). Across both human and LLM groups, MT accuracies were generally higher than IVT accuracies, reflecting the increased complexity and variability of IVT decision-making. Figure 1 displays the accuracy of LLMs and clinicians. View this table: View inline View popup Download powerpoint Table 2. Accuracies for MT, IVT and Jaccard Index. Download figure Open in new tab Figure 1. Accuracy of Large Language Models and Stroke Experts in Acute Ischemic Stroke Treatment Decisions. Bar plot showing the accuracy (mean ± 95% confidence interval) of four large language models (DeepSeek R1, OpenAI o3 mini, Gemini 2.0, LLaMA 3.3) and stroke experts (neurologists, neuroradiologists, and pooled physicians) in recommending intravenous thrombolysis (IVT, purple) and mechanical thrombectomy (MT, yellow) for 80 acute ischemic stroke cases. LLM Reasoning Evaluation Error composition analysis revealed distinct patterns across large language models (LLMs) and treatment decision categories ( Figure 2 ). In the Mechanical Thrombectomy (MT) group, most cases were free from reasoning errors, ranging from 79% for LLaMA 3.3 to 89% for DeepSeek R1. The most frequent error category in MT was Discrepant Ground Truth , accounting for 8–15% of cases depending on the model, followed by Borderline Case Misclassification (1–5%). Guideline non-adherence was rare (<5%), and no omissions of critical information were observed in any MT decision. Detailed error rates for MT across categories and LLMs are presented in Table 3 . View this table: View inline View popup Download powerpoint Table 3. Errors in Mechanical Thrombectomy group (n=80) Download figure Open in new tab Figure 2. Error composition of LLMs in treatment decision-making for acute ischemic stroke. Stacked bar plots showing the proportion of cases with different reasoning error types across four large language models for (A) Mechanical Thrombectomy (MT) and (B) Intravenous Thrombolysis (IVT). In contrast, the Intravenous Thrombolysis (IVT) group demonstrated slightly higher proportions of reasoning errors, with “error-free” rates between 64% (LLaMA 3.3) and 79% (DeepSeek R1). Discrepant Ground Truth errors occurred in 11–18% of IVT cases, while Guideline non-adherence was notably more frequent, reaching 20% for GEMINI 2.0 and LLaMA 3.3. Omission of Critical Information was rare (<2%), and Edge Case Misclassification remained ≤3%. Detailed error rates for IVT across categories and LLMs are presented in Table 4 . View this table: View inline View popup Download powerpoint Table 4. Errors in Intravenous Thrombolysis group (n=80) Notably, two patients were excluded from the error analysis: one declined treatment, and the other was enrolled in a clinical study. The supplementary file provides illustrative examples of LLM-generated reasoning across different error categories. Discussion Acute ischemic stroke (AIS) management is a complex, multidisciplinary process requiring coordinated input from neurologists, neuroradiologists, anesthesiologists, and well-equipped stroke units [ 6 ]. Despite advances in regional stroke networks, approximately one third of stroke patients initially presents to community hospitals and must be transferred to comprehensive stroke centers for definitive treatment [ 7 ]. In such settings, the lack of immediate specialist expertise may delay critical decisions, potentially jeopardizing patient outcomes [ 20 ]. Our findings suggest that large language models (LLMs) with advanced reasoning capabilities could serve as effective decision support tools in this context. Their performance in determining eligibility for mechanical thrombectomy (MT) and intravenous thrombolysis (IVT) matched or even exceeded that of experienced physicians. Notably, the top-performing model, DeepSeek-R1, achieved an MT decision accuracy of 87%, outperforming neurologists (81%) and neuroradiologists (84%). These results highlight the potential of generative AI to support high-stakes, time-critical decisions in stroke care, particularly in scenarios where immediate specialist consultation is not available. In our retrospective, multi-center study, 80 real-world cases from two large stroke centers in different European countries were evaluated. Fourteen clinical variables were provided to both human raters and LLMs, including demographic, clinical, and imaging data. Neurologists (n=2) achieved accuracies of 0.81 for MT and 0.80 for IVT, while neuroradiologists (n=4) achieved 0.84 for MT and 0.76 for IVT. For MT decisions, interrater agreement was moderate among neurologists (κ = 0.618) and neuroradiologists (κ = 0.595), indicating reasonable consistency. IVT decisions showed substantial agreement among neurologists (κ = 0.725) but only fair agreement among neuroradiologists (κ = 0.374), reflecting differences in clinical roles; neurologists typically lead IVT decision-making, whereas neuroradiologists focus primarily on MT eligibility. The LLMs demonstrated robust performance in integrating heterogeneous clinical inputs and applying relevant treatment guidelines. DeepSeek-R1, an open-source model, achieved 87% accuracy for MT and 78% for IVT. By contrast, LLaMA 3.3 ranked lowest, with 76% for MT and 63% for IVT. These results indicate that both commercial and open-source LLMs can synthesize demographic, clinical, and imaging information to generate patient-specific treatment recommendations, with some open-source models performing comparably to proprietary systems, an important consideration for privacy-preserving clinical deployment. The consistently higher accuracies observed for MT compared to IVT across both human raters and LLMs likely reflect differences in decision complexity. MT eligibility is primarily determined by well-defined imaging and clinical criteria, such as large vessel occlusion and time from symptom onset, which may be more straightforward to assess. In contrast, IVT decisions require consideration of a broader range of clinical factors, including contraindications (e.g., recent surgery, anticoagulation status, bleeding risk) and time-sensitive nuances, which introduce variability and uncertainty. This complexity may explain why IVT accuracy was more variable and generally lower, particularly among LLMs, and why guideline non-adherence errors were more prevalent in the IVT reasoning evaluation. Our results are in line with Kottlors et al., who investigated GPT-3 for thrombectomy triage using radiology reports and limited clinical data in a single-center German cohort [ 21 ]. In their study, the LLMs achieved an overall accuracy of 88%, with a specificity of 96% and a sensitivity of 80%. The present study extends these findings by incorporating a more heterogeneous, multi-center dataset and benchmarking performance against both expert consensus and real-world treatment decisions. Moreover, in addition to the binary “yes/no” outputs in Kottlors et al., our LLMs were required to provide explanatory rationales for their recommendations. This feature not only improves transparency and clinician trust but also offers educational value by making the decision-making process explicit. Additionally, we assessed both commercial and open-source models, demonstrating that open-source systems can achieve non-inferior performance without the data privacy concerns associated with proprietary cloud-based tools. The ability of LLMs to produce reasoning is particularly important in clinical AI applications. Even accurate recommendations may be dismissed by clinicians if not accompanied by an explanation. Our error composition analysis revealed clear differences between treatment decision types. For MT, most LLM outputs were free from reasoning errors (79–89%), with Discrepant Ground Truth errors being the most common (8–15%), followed by infrequent Borderline Case Misclassifications (1–5%) and rare guideline deviations (<5%). No omissions of critical information occurred. In contrast, IVT decisions showed lower rates of error-free reasoning (64–79%) and a higher incidence of Guideline non-adherence, particularly in Gemini 2.0 and LLaMA 3.3 (20%). Discrepant Ground Truth Errors were observed in 11–18% of cases, while omissions of critical information (<2%) and borderline case misclassifications (≤3%) remained uncommon. These findings indicate that, while all models maintained a high proportion of error-free reasoning in MT decisions, IVT recommendations were more susceptible to errors, reflecting the greater complexity of IVT decision-making, which involves multiple eligibility criteria, time-dependent factors, and considering numerous contraindications. This study has several limitations. Although based on real-world cases from two centers, the ground truth remains subject to retrospective bias and variability in interdisciplinary decision-making. Some influencing factors, such as family discussions or pre-defined treatment limitations, were not documented and therefore not available to neither LLMs nor human raters. Additionally, while we evaluated several state-of-the-art LLMs, performance was not uniform, and generalizability to other populations and health systems remains uncertain. Finally, while we assessed reasoning quality qualitatively, more standardized and quantitative metrics for evaluating factual accuracy, logical coherence, and guideline adherence in AI-generated rationales are needed. In conclusion, our study demonstrates that LLMs can match or even surpass expert clinician performance in AIS treatment eligibility decisions, while also providing explainable rationales that may enhance clinician trust and facilitate learning. Compared with prior work, our approach leverages a more complex, real-world dataset, incorporates reasoning-based evaluation, and highlights the viability of open-source models for privacy-preserving deployment. Future research should focus on prospective, real-time validation in clinical workflows, multilingual adaptability, and integration with stroke imaging and workflow data to create robust, trustworthy decision support systems for acute stroke care. Data Availability The datasets analyzed during the current study are not publicly available due to patient privacy and institutional data protection regulations. Upon acceptance of this manuscript in a peer-reviewed journal, the code used for data analysis and model evaluation will be made available in a publicly accessible GitHub repository. References 1. ↵ Nogueira , R.G. ; Jadhav , A.P. ; Haussen , D.C. ; Bonafe , A. ; Budzik , R.F. ; Bhuva , P. ; Yavagal , D.R. ; Ribo , M. ; Cognard , C. ; Hanel , R.A. ; et al. Thrombectomy 6 to 24 Hours after Stroke with a Mismatch between Deficit and Infarct . N. Engl. J. Med . 2017 , 378 , 11 – 21 , doi: 10.1056/nejmoa1706442 . OpenUrl CrossRef 2. ↵ Berkhemer , O.A. ; Fransen , P.S.S. ; Beumer , D. ; Berg , L.A. van den ; Lingsma , H.F. ; Yoo , A.J. ; Schonewille , W.J. ; Vos , J.A. ; Nederkoorn , P.J. ; Wermer , M.J.H. ; et al. A Randomized Trial of Intraarterial Treatment for Acute Ischemic Stroke . N. Engl. J. Med . 2015 , 372 , 11 – 20 , doi: 10.1056/nejmoa1411587 . OpenUrl CrossRef PubMed Web of Science 3. ↵ Schwamm , L.H. ; Fonarow , G.C. ; Reeves , M.J. ; Pan , W. ; Frankel , M.R. ; Smith , E.E. ; Ellrodt , G. ; Cannon , C.P. ; Liang , L. ; Peterson , E. ; et al. Get With the Guidelines–Stroke Is Associated With Sustained Improvement in Care for Patients Hospitalized With Acute Stroke or Transient Ischemic Attack . Circulation 2009 , 119 , 107 – 115 , doi: 10.1161/circulationaha.108.783688 . OpenUrl Abstract / FREE Full Text 4. ↵ Chaudhry , F.S. ; Schneck , M.J. ; Warady , J. ; Platakis , J. ; Morales-Vidal , S.G. ; Biller , J. ; Flaster , M. Primary Stroke Center Concept: Strengths and Limitations . Front. Neurol . 2012 , 3 , 108 , doi: 10.3389/fneur.2012.00108 . OpenUrl CrossRef PubMed 5. ↵ Alberts , M.J. ; Latchaw , R.E. ; Selman , W.R. ; Shephard , T. ; Hadley , M.N. ; Brass , L.M. ; Koroshetz , W. ; Marler , J.R. ; Booss , J. ; Zorowitz , R.D. ; et al. Recommendations for Comprehensive Stroke Centers . Stroke 2005 , 36 , 1597 – 1616 , doi: 10.1161/01.str.0000170622.07210.b4 . OpenUrl Abstract / FREE Full Text 6. ↵ Goyal , M. ; Menon , B.K. ; Wilson , A.T. ; Almekhlafi , M.A. ; McTaggart , R. ; Jayaraman , M. ; Demchuk , A.M. ; Hill , M.D. Primary to Comprehensive Stroke Center Transfers: Appropriateness, Not Futility . Int. J. Stroke 2017 , 13 , 550 – 553 , doi: 10.1177/1747493018764072 . OpenUrl CrossRef 7. ↵ Ali , S.F. ; Fonarow , G. ; Liang , L. ; Xian , Y. ; Smith , E.E. ; Bhatt , D.L. ; Schwamm , L. Rates, Characteristics, and Outcomes of Patients Transferred to Specialized Stroke Centers for Advanced Care . Circ.: Cardiovasc. Qual. Outcomes 2018 , 11 , e003359 , doi: 10.1161/circoutcomes.116.003359 . OpenUrl CrossRef PubMed 8. ↵ Goyal , M. ; Ospel , J.M. Stroke Systems of Care . Stroke 2020 , 51 , 1928 – 1931 , doi: 10.1161/strokeaha.120.030288 . OpenUrl CrossRef PubMed 9. ↵ Ifejika , N.L. ; Wiegand , J. ; Harbold , H. ; Botello , A.A. ; Babalola , B.A. ; Venkatachalam , A.M. ; Novakovic , R. ; Cannell , M.B. The “Network Effect” on Interfacility Transfers Among Regional Stroke Certified Hospitals . J. Stroke Cerebrovasc. Dis . 2021 , 30 , 106056 , doi: 10.1016/j.jstrokecerebrovasdis.2021.106056 . OpenUrl CrossRef 10. ↵ Omiye , J.A. ; Gui , H. ; Rezaei , S.J. ; Zou , J. ; Daneshjou , R. Large Language Models in Medicine: The Potentials and Pitfalls: A Narrative Review . Ann. Intern. Med . 2024 , 177 , 210 – 220 , doi: 10.7326/m23-2772 . OpenUrl CrossRef PubMed 11. ↵ Cascella , M. ; Montomoli , J. ; Bellini , V. ; Bignami , E. Evaluating the Feasibility of ChatGPT in Healthcare: An Analysis of Multiple Clinical and Research Scenarios . J. Méd. Syst . 2023 , 47 , 33 , doi: 10.1007/s10916-023-01925-4 . OpenUrl CrossRef PubMed 12. ↵ Schubert , M.C. ; Wick , W. ; Venkataramani , V. Performance of Large Language Models on a Neurology Board–Style Examination . JAMA Netw. Open 2023 , 6 , e2346721 , doi: 10.1001/jamanetworkopen.2023.46721 . OpenUrl CrossRef PubMed 13. ↵ Goh , E. ; Bunning , B. ; Khoong , E. ; Gallo , R. ; Milstein , A. ; Centola , D. ; Chen , J.H. ChatGPT Influence on Medical Decision-Making, Bias, and Equity: A Randomized Study of Clinicians Evaluating Clinical Vignettes . medRxiv 2023 , 2023.11.24.23298844 , doi: 10.1101/2023.11.24.23298844 . OpenUrl Abstract / FREE Full Text 14. ↵ Shi , Q. ; Luzuriaga , K. ; Allison , J.J. ; Oztekin , A. ; Faro , J.M. ; Lee , J.L. ; Hafer , N. ; McManus , M. ; Zai , A.H. Transforming Informed Consent Generation Using Large Language Models: Mixed Methods Study . JMIR Méd. Inform . 2025 , 13 , e68139 , doi: 10.2196/68139 . OpenUrl CrossRef 15. ↵ Veen , D.V. ; Uden , C.V. ; Blankemeier , L. ; Delbrouck , J.-B. ; Aali , A. ; Bluethgen , C. ; Pareek , A. ; Polacin , M. ; Reis , E.P. ; Seehofnerova , A. ; et al. Clinical Text Summarization: Adapting Large Language Models Can Outperform Human Experts . Res. Sq . 2023 , rs.3.rs-3483777 , doi: 10.21203/rs.3.rs-3483777/v1 . OpenUrl CrossRef 16. ↵ Meddeb , A. ; Lüken , S. ; Busch , F. ; Adams , L. ; Ugga , L. ; Koltsakis , E. ; Tzortzakakis , A. ; Jelassi , S. ; Dkhil , I. ; Klontzas , M.E. ; et al. Large Language Model Ability to Translate CT and MRI Free-Text Radiology Reports Into Multiple Languages . Radiology 2024 , 313 , e241736 , doi: 10.1148/radiol.241736 . OpenUrl CrossRef 17. ↵ Meddeb , A. ; Ebert , P. ; Bressem , K.K. ; Desser , D. ; Dell’Orco , A. ; Bohner , G. ; Kleine , J.F. ; Siebert , E. ; Grauhan , N. ; Brockmann , M.A. ; et al. Evaluating Local Open-Source Large Language Models for Data Extraction from Unstructured Reports on Mechanical Thrombectomy in Patients with Ischemic Stroke . J. NeuroInterventional Surg . 2024 , jnis-2024-022078 , doi: 10.1136/jnis-2024-022078 . OpenUrl Abstract / FREE Full Text 18. ↵ Reiner , L.N. ; Chelbi , M. ; Fetscher , L. ; Stöckel , J.C. ; Csapó-Schmidt , C. ; Guseynova , S. ; Mohamad , F.A. ; Bressem , K.K. ; Nawabi , J. ; Siebert , E. ; et al. Automated MRI Protocoling in Neuroradiology in the Era of Large Language Models . Radiol. Med . 2025 , 1 – 11 , doi: 10.1007/s11547-025-02040-9 . OpenUrl CrossRef 19. ↵ Powers , W.J. ; Rabinstein , A.A. ; Ackerson , T. ; Adeoye , O.M. ; Bambakidis , N.C. ; Becker , K. ; Biller , J. ; Brown , M. ; Demaerschalk , B.M. ; Hoh , B. ; et al. 2018 Guidelines for the Early Management of Patients With Acute Ischemic Stroke: A Guideline for Healthcare Professionals From the American Heart Association/American Stroke Association . Stroke 2018 , 49 , e46 – e99 , doi: 10.1161/str.0000000000000158 . OpenUrl Abstract / FREE Full Text 20. ↵ Venema , E. ; Burke , J.F. ; Roozenbeek , B. ; Nelson , J. ; Lingsma , H.F. ; Dippel , D.W.J. ; Kent , D.M. Prehospital Triage Strategies for the Transportation of Suspected Stroke Patients in the United States . Stroke 2020 , 51 , 3310 – 3319 , doi: 10.1161/strokeaha.120.031144 . OpenUrl CrossRef PubMed 21. ↵ Kottlors , J. ; Hahnfeldt , R. ; Görtz , L. ; Iuga , A.-I. ; Fervers , P. ; Bremm , J. ; Zopfs , D. ; Laukamp , K.R. ; Onur , O.A. ; Lennartz , S. ; et al. Large Language Models–Supported Thrombectomy Decision-Making in Acute Ischemic Stroke Based on Radiology Reports: Feasibility Qualitative Study . J. Méd. Internet Res . 2025 , 27 , e48328 , doi: 10.2196/48328 . OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted September 05, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Evaluating Accuracy and Reasoning Capabilities of Large Language Models for Acute Ischemic Stroke Management Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Evaluating Accuracy and Reasoning Capabilities of Large Language Models for Acute Ischemic Stroke Management Aymen Meddeb , Navid Bakhtiari , Ida Rangus , Leonard Fetscher , Bastien Leguellec , Felix Busch , Alexandre Doucet , Vi-Tuan Hua , Fuong Verot-Nguyen , Laurentiu Valentin Paiusan , Pierre Francois Manceau , Paolo Pagano , Mike P. Wattjes , Solene Moulin , Laurent Pierot , Sebastien Soize medRxiv 2025.09.03.25335060; doi: https://doi.org/10.1101/2025.09.03.25335060 Share This Article: Copy Citation Tools Evaluating Accuracy and Reasoning Capabilities of Large Language Models for Acute Ischemic Stroke Management Aymen Meddeb , Navid Bakhtiari , Ida Rangus , Leonard Fetscher , Bastien Leguellec , Felix Busch , Alexandre Doucet , Vi-Tuan Hua , Fuong Verot-Nguyen , Laurentiu Valentin Paiusan , Pierre Francois Manceau , Paolo Pagano , Mike P. Wattjes , Solene Moulin , Laurent Pierot , Sebastien Soize medRxiv 2025.09.03.25335060; doi: https://doi.org/10.1101/2025.09.03.25335060 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Neurology Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15229) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6600) Geriatric Medicine (668) Health Economics (997) Health Informatics (4538) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (541) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3333) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9232) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a00dea64fc850de1',t:'MTc3OTY0MjY0NQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();