Reliability of Artificial Intelligence-enhanced Electrocardiography

Reliability of Artificial Intelligence-enhanced Electrocardiography | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Reliability of Artificial Intelligence-enhanced Electrocardiography View ORCID Profile Lovedeep S Dhingra , Philip M Croon , Bruno Batinica , View ORCID Profile Arya Aminorroaya , Aline F Pedroso , View ORCID Profile Evangelos K Oikonomou , View ORCID Profile Rohan Khera doi: https://doi.org/10.1101/2025.11.04.25339526 Lovedeep S Dhingra 1 Cardiovascular Data Science (CarDS) Lab, Yale School of Medicine , New Haven, CT, USA 2 Section of Cardiovascular Medicine, Department of Internal Medicine, Yale School of Medicine , New Haven, CT, USA MBBS, MHS Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Lovedeep S Dhingra Philip M Croon 1 Cardiovascular Data Science (CarDS) Lab, Yale School of Medicine , New Haven, CT, USA 2 Section of Cardiovascular Medicine, Department of Internal Medicine, Yale School of Medicine , New Haven, CT, USA 3 Department of Cardiology, Amsterdam Cardiovascular Sciences, Amsterdam University Medical Centre, University of Amsterdam , Amsterdam, The Netherlands MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Bruno Batinica 1 Cardiovascular Data Science (CarDS) Lab, Yale School of Medicine , New Haven, CT, USA 2 Section of Cardiovascular Medicine, Department of Internal Medicine, Yale School of Medicine , New Haven, CT, USA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Arya Aminorroaya 1 Cardiovascular Data Science (CarDS) Lab, Yale School of Medicine , New Haven, CT, USA 2 Section of Cardiovascular Medicine, Department of Internal Medicine, Yale School of Medicine , New Haven, CT, USA MD MPH Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Arya Aminorroaya Aline F Pedroso 1 Cardiovascular Data Science (CarDS) Lab, Yale School of Medicine , New Haven, CT, USA 2 Section of Cardiovascular Medicine, Department of Internal Medicine, Yale School of Medicine , New Haven, CT, USA PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Evangelos K Oikonomou 1 Cardiovascular Data Science (CarDS) Lab, Yale School of Medicine , New Haven, CT, USA 2 Section of Cardiovascular Medicine, Department of Internal Medicine, Yale School of Medicine , New Haven, CT, USA MD, DPhil Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Evangelos K Oikonomou Rohan Khera 1 Cardiovascular Data Science (CarDS) Lab, Yale School of Medicine , New Haven, CT, USA 2 Section of Cardiovascular Medicine, Department of Internal Medicine, Yale School of Medicine , New Haven, CT, USA 3 Department of Cardiology, Amsterdam Cardiovascular Sciences, Amsterdam University Medical Centre, University of Amsterdam , Amsterdam, The Netherlands 4 Center for Outcomes Research and Evaluation (CORE), Yale New Haven Hospital , New Haven, CT, USA 5 Section of Biomedical Informatics and Data Science, Yale School of Medicine , New Haven, CT, USA MD, MS Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Rohan Khera For correspondence: rohan.khera{at}yale.edu Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF ABSTRACT Background The scientific literature on artificial intelligence-enabled electrocardiography (AI-ECG) has defined a robust performance of AI models in detecting and predicting several structural heart disorders (SHDs) using ECGs. However, as a diagnostic test, the real-world clinical utility of AI-ECG reliability requires the consistency of its results when repeated under similar conditions. Aim To evaluate the reliability of AI-ECG models for different ECGs for the same person, across different diagnostic labels, and using varied modeling approaches. Methods We used ECG images (2000-2024) from 5 hospitals and an outpatient network within a large, integrated US health system. For each individual, we identified multiple ECGs recorded within a 30-day period. We evaluated 7 models: 6 convolutional neural networks (CNNs) trained to detect individual SHDs, including LV systolic dysfunction, left valve diseases and severe LVH; an ensemble XGBoost integrating individual CNNs as a composite screen for multiple SHDs. We used concordance correlation coefficient (CCC), Spearman correlation, Cohen’s kappa, and percent agreement in binary screen status to test model reliability. We evaluated factors associated with different AI-ECG outputs (Δ probability> 0.5) and assessed stability across ECG layouts (digital, printed, photo). Results Across sites, we identified 1,118,263 ECG pairs, with a median 1 (1-3) days between ECGs. The ensemble XGBoost had the higher test-retest correlation (CCC: 0.89-0.92) and agreement (kappa: 0.75-0.82) between pairs compared with CNNs (CCC: 0.78-0.88; kappa: 0.57-0.72). After adjusting for demographics, ECG pairs that included one or both inpatient ECG were significantly more likely to yield unstable predictions (ORs: 1.60 [1.50-1.70] and 1.91 [1.78-2.05], respectively) compared with pairs with both ECGs obtained in outpatient settings. Among outpatient pairs across sites, the XGBoost model had a CCC of 0.89-0.94, a Spearman correlation of 0.90-0.94, and a kappa of 0.78-0.84, with concordance rates of 89-92%. Notably, ensemble model predictions were also stable across different ECG layouts. Conclusion An ensemble AI-ECG model integrating multiple CNN predictions had higher reliability compared with models for individual disorders. Discordance was more common in inpatient ECGs, suggesting instability in high-acuity settings. Reliable ensemble AI-ECG model outputs support readiness for clinical implementation for SHD screening. Study Design Abbreviations: AR, aortic regurgitation; AS, aortic stenosis; CNN, convolutional neural network; ECG, electrocardiogram; FC, fully-connected layers; LVSD, left ventricular systolic dysfunction; MR, mitral regurgitation; SHD, structural heart diseases; sLVH, severe left ventricular hypertrophy, XGBoost, extreme gradient boosting. Download figure Open in new tab BACKGROUND Artificial intelligence-enabled electrocardiography (AI-ECG) has demonstrated robust performance in identifying structural heart disorders (SHDs) across a variety of clinical and community-based settings. 1 , 2 In particular, AI-ECG tools using images of ECGs offer scalability and accessibility for population-level screening, addressing the critical challenges associated with limited access to echocardiography and trained personnel. 1 , 3 – 6 However, translating this promise of AI-ECG into clinical adoption requires more than high disease discrimination alone. 7 – 9 As diagnostic tests that can alter the course of care of patients, it is essential to evaluate whether AI-ECG demonstrates the reliability of its performance when deployed on repeat measures for the same patient in a similar setting. 8 Following recent U.S. Food and Drug Administration approvals for AI-ECG tools that detect hidden cardiovascular disorders, such as LVSD and hypertrophic cardiomyopathy, and the reimbursement of AI-ECG for LVSD detection, it is especially urgent to evaluate this retest reliability in real-world settings. 10 AI-ECG models have traditionally been validated at a population level using a single ECG per individual, both to ensure generalizability and to avoid over-representing individuals with more frequent testing. 6 , 11 – 13 However, the absence of repeated testing in most studies might mask individual-level reliability. Despite the significant advantage AI-based biomarkers have in providing an objective marker of risk by providing quantifiable outputs from a single input, variations inherent in real-world scenarios can degrade model performance. 14 , 15 In the case of AI-ECG, modest changes in acquisition methods, such as lead placement, noise, or clinical context, could potentially affect model outputs, especially in real-world environments where standardization is limited. Therefore, there is an unmet need to evaluate the stability of AI-ECG model outputs when subjected to minor but realistic variations in ECG acquisitions across models trained for different target conditions and using different development strategies. 1 In this study, we aimed to comprehensively assess the test-retest reliability of seven AI-ECG models, across more than one million ECG pairs drawn from five hospitals and outpatient networks. We then evaluated whether models that were specifically developed to detect certain SHDs were more reliable than others, and whether an ensemble model meant for a composite of SHDs was more reliable. We specifically quantified model-level agreement and examined factors associated with prediction discordance. Finally, we assessed whether predictions remained stable across ECG image formats and acquisition strategies, reflecting the heterogeneity of real-world use. METHODS The Yale Institutional Review Board approved the study protocol and waived the need for informed consent, as this analysis involved the secondary use of previously acquired, de-identified data. Data Sources and Study Population We used ECG data from the Yale New Haven Health System (YNHHS) spanning 2000 to 2024. YNHHS is the largest referral health system in southern New England, comprising 5 hospitals: Yale New Haven Hospital (YNHH), Bridgeport Hospital, Greenwich Hospital, Lawrence + Memorial Hospital, Westerly Hospital, and the outpatient network, Northeast Medical Group (NEMG). The data were derived from the Epic electronic health records (EHR) system via the Clarity database. YNHH, where models were developed, is a 1,541-bed tertiary care academic medical center. External validation involved Bridgeport Hospital, a 501-bed general medical center in Bridgeport; Greenwich Hospital, a 206-bed regional medical facility serving Fairfield County, Connecticut, and Westchester County, New York; and Lawrence + Memorial and Westerly Hospitals, providing care to communities in southeastern Connecticut and Rhode Island. NEMG is an extensive ambulatory network that provides primary and specialty care across Connecticut. For each adult patient, we identified pairs of ECGs recorded within 1 to 30 days. ECGs were categorized based on whether they were acquired during inpatient hospital stays or outpatient visits. 12-lead clinical ECGs were acquired as 10-second recordings at a sampling frequency of 250 Hz or 500 Hz, primarily using Philips PageWriter and GE MAC series machines. AI-ECG Models and Deployment Methodology In this study, we deployed previously validated AI-ECG models that demonstrated excellent performance in detecting and predicting SHDs across clinical and community-based settings. These included 6 convolutional neural networks (CNNs) and an ensemble extreme gradient boosting (XGBoost) model, PRESENT-SHD (Practical scREening using ENsemble machine learning sTrategy for SHD detection) developed using 261,228 ECGs from 93,693 patients at YNHH between 2015 and 2023. The individual SHD models were those that have been developed by many groups independently, including (i) left ventricular systolic dysfunction (LVSD, ejection fraction 15 mm with concomitant diastolic dysfunction). These abnormalities were noted on an echocardiogram within 30 days of the ECG. The ensemble model considered the presence of any of these conditions to be a marker of SHD. Each CNN was pretrained using self-supervised contrastive learning, which improved label efficiency for downstream supervised learning and enabled each model to have maximal performance for each of the labels. To enhance robustness to novel ECG image layouts and formats, each ECG in the development set was plotted using one of multiple formats that varied in lead layouts, font size and type, thickness and color of background grid and ECG trace, and rhythm strip placement ( Supplementary Figure 1 ). For the ensemble model, outputs from each CNN were integrated into a classifier along with the individual’s age and sex, using an XGBoost framework. This ensemble, PRESENT-SHD, was trained to detect the presence of any SHD. The individual CNNs achieved AUROCs ranging from 0.805 to 0.914 for their target conditions ( Supplementary Figure 2 ), while PRESENT-SHD had AUROCs of 0.853-0.900 across validation sites. The ensemble model demonstrated superior discrimination and generalizability and is publicly available for research use at https://www.cards-lab.org/present-shd . ECG Image Generation For evaluation, each included ECG was plotted in standard clinical layout from signal waveform data, with a voltage calibration of 10 mm/mV, with the limbs and precordial leads arranged in four columns of 2.5-second each, representing leads I, II, and III; aVR, aVL, and aVF; V1, V2, and V3; and V4, V5, and V6 ( Supplementary Figure 3 ). A 10-second recording of the lead I signal was included as a rhythm strip. Further, in a subset of 10,000 ECGs from YNHH, we deployed the model on four novel image formats that were not encountered during training ( Supplementary Figure 4 ). These included formats with altered background grid color, modified ECG trace color, a new ECG lead layout, and a composite format incorporating all three changes. Model Evaluation on Screenshots and Smartphone Photographs of ECGs To assess real-world applicability, we selected a random subset of 100 ECGs from YNHH. For these ECGs, we extracted the clinical images directly from the EHR as PDFs and captured screenshots. Additionally, we photographed these ECGs displayed on a laptop screen and printed on A4-sized sheets, using three different smartphones with their default camera settings ( Supplementary Figure 5 ). We evaluated the model predictions from these real-world images, comparing them against standard-format ECG images to assess consistency. Statistical Analysis To evaluate AI-ECG reliability, we calculated concordance correlation coefficients (CCC) and Spearman correlation between the continuous numeric model predictions for the paired ECGs. CCC captures both precision and accuracy of agreement, while Spearman assesses rank-based monotonic correlation, making them complementary measures of stability. For each model, binary classification thresholds for screen-positive status were derived based on achieving 90% sensitivity in the internal validation set. For binary screening results, we assessed Cohen’s kappa and the percentage of agreement between the ECG pairs. We used a logistic regression model to identify factors associated with unstable predictions. Here, a difference greater than 0.5 between paired AI-ECG predictions for the ensemble model was used as the dependent variable with demographic features and the clinical setting of ECG pair acquisition included as independent variables. Bland-Altman plots were generated to visualize agreement between novel ECG formats and real-world image capture methods with the standard ECG images, examining systematic biases and limits of agreement. 16 All statistical analyses were two-sided, and a significance threshold was set at 0.05. The 95% confidence intervals (CIs) for agreement metrics were derived through bootstrapping (1,000 iterations). Analyses were executed using Python version 3.11.2. RESULTS Study Population We identified a total of 1,118,263 ECG pairs across study sites. At YNHH, 950,448 ECG pairs of individual patients?? were included, of which 385,698 (40.6%) pairs had both ECGs performed in an outpatient setting. The median age of patients with these ECGs was 65 years (IQR, 53-76), and 436,087 (45.9%) were women. Across race/ethnicity groups, 638,802 (67.2%) patients were White, 174,890 (18.4%) were Black, 96,887 (10.2%) were Hispanic, and 12,150 (1.3%) were Asian ( Table 1 ). View this table: View inline View popup Download powerpoint Table 1. Demographic Characteristics for Electrocardiogram Pairs across Study Sites. Abbreviations: ECG, electrocardiogram. Across the four external hospital sites, we included 49,246 ECG pairs from Bridgeport, 14,007 from Greenwich, 71,059 from Lawrence + Memorial, and 15,672 from Westerly Hospitals. The median age across hospital sites ranged from 68-72 years, with 44.9% to 49.3% of ECGs were from women, and 17.9% to 50.5% of ECG pairs obtained in outpatient settings. At NEMG, we identified 17,831 ECG pairs, with a median age of 72 years (IQR, 62-80), and 7,481 (42.0%) ECG pairs from women. Of these, 15,263 (85.6%) were from White patients, 916 (5.1%) from Black patients, 934 (5.2%) from Hispanic patients, and 181 (1.0%) from Asian patients ( Table 1 ). Test-Retest Correlation Across All ECG Pairs and AI-ECG Models Across all ECG pairs at the Yale New Haven Hospital, the ensemble model for SHD detection had a CCC of 0.90 (95% CI, 0.90-0.90), followed by the CNN models for detecting any left sided valve disease (0.86 [95% CI, 0.86-0.86]), severe LVH (0.84 [95% CI, 0.84-0.84]), mitral regurgitation (0.84 [95% CI, 0.84-0.84]), LV systolic dysfunction (0.83 [95% CI, 0.83-0.83]), aortic stenosis (0.83 [95% CI, 0.83-0.83]), and aortic regurgitation (0.80 [95% CI, 0.80-0.80]; Figure 1 ). Similarly, across external hospitals, the CCC for PRESENT-SHD ranged from 0.89-0.92, while the CNN models had CCCs ranged from 0.79 to 0.88. In NEMG, the ensemble model’s CCC was 0.89 (95% CI, 0.88-0.89), compared with 0.79-0.84 for disease-specific CNN models ( Figure 1 ). Download figure Open in new tab Figure 1. Test-Retest Agreement Artificial Intelligence-enhanced Electrocardiography Models and Study Sites Using Concordance Correlation Coefficient (All Electrocardiogram Pairs). Abbreviations: CNN, convolutional neural network; L&M, Lawrence and Memorial Hospital; NEMG, Northeast Medical Group; YNHH, Yale New Haven Hospital. Similarly, Spearman’s rank correlation coefficients for PRESENT-SHD predictions between paired ECGs ranged from 0.89 to 0.93 across sites, compared with 0.79 to 0.88 for CNN models ( Supplementary Table 1 ). For agreement in screen status for all ECG pairs, Cohen’s kappa for the ensemble model was 0.75-0.82 across sites. For CNN models, the Cohen’s kappa for binarized screen status ranged from 0.57-0.72 across sites ( Supplementary Table 1 ). Factors Associated With Instability in Model Predictions After accounting for other patient demographics and the clinical setting of the paired ECGs, patients aged below 65 years were over 150% more likely to have large differences between paired ensemble model predictions (OR, 2.52 [95% CI, 2.38-2.67]), compared with those above 65 years. Men were more likely to have unstable AI-ECG scores (OR, 1.10 [95% CI, 1.04-1.16]) compared with women. Compared with White patients, odds of large score differences between paired ECGs were higher for Black patients (OR, 1.14 [95% CI, 1.07-1.22]) and those of other racial groups (OR, 1.17 [95% CI, 1.01-1.35]), and lower for Hispanic patients (OR, 0.91 [95% CI, 0.83-0.99]). Odds were similar for Asian vs White patients (OR, 1.09 [95% CI, 0.87-1.36]).r Notably, adjusted for the patients’ demographic features, the clinical setting in which the ECGs were acquired was significantly associated with prediction variability. Compared with pairs where both ECGs were captured in outpatient settings, odds of unstable predictions were higher when one ECG was inpatient (OR, 1.60 [95% CI, 1.50-1.70]) or both ECGs were inpatient (OR, 1.91 [95% CI, 1.78-2.05]; Figure 2 ). Download figure Open in new tab Figure 2. Demographic and Clinical Factors Associated With Prediction Instability. Site-Level Performance for Outpatient ECG pairs For outpatient ECG pairs at YNHH, the ensemble XGBoost model demonstrated robust reliability, with a CCC of 0.92 (95% CI, 0.92-0.92; Supplementary Figure 6 ). Notably, the Spearman correlation coefficient was 0.92 (95% CI, 0.92-0.92), and for binary screen status indicated substantial agreement, with Cohen’s kappa of 0.79 (95% CI, 0.79-0.80; Figures 3 and 4 , Supplementary Table 2 ). In comparison, Spearman correlations for the CNN models ranged from 0.84 to 0.89, and Cohen’s kappa values ranged from 0.65 to 0.72. Download figure Open in new tab Figure 3. Agreement Metrics Across Artificial Intelligence-enhanced Electrocardiography Models in the Yale New Haven Hospital (Outpatient Electrocardiogram Pairs): Spearman Correlation, Cohen’s Kappa, and Concordant Screen Status. Abbreviations: CNN, convolutional neural network; LVH, left ventricular hypertrophy; XGBoost, extreme gradient boosting model. Download figure Open in new tab Figure 4. Identity Scatter Plots Comparing Paired Outpatient Electrocardiogram Predictions for the Ensemble Structural Heart Disease Model across Health System Sites. Abbreviations: L&M, Lawrence + Memorial Hospital; NEMG, Northeast Medical Group; YNHH, Yale New Haven Hospital. Across external hospital outpatient ECG pairs, the ensemble model achieved CCC values from 0.92 to 0.94, with Spearman correlations consistently between 0.92 to 0.94. Cohen’s kappa values across these hospitals varied from 0.78 to 0.84, with near-perfect agreement in binary classification observed at Greenwich, Lawrence + Memorial, and Westerly Hospitals ( Figure 3 , Supplementary Table 2 ). In the NEMG outpatient network, the ensemble model had a Spearman correlation of 0.90 (95% CI, 0.89-0.90), and Cohen’s kappa of 0.76 (95% CI, 0.75-0.77). Across all external sites, CNN model performance for outpatient ECG pairs showed Spearman correlations ranging from 0.83 to 0.89 and Cohen’s kappa statistics from 0.56 to 0.71 ( Supplementary Table 2 ). Ensemble Model Stability Across Novel ECG Image Formats The ensemble PRESENT-SHD model exhibited strong stability across novel ECG image formats not encountered during model training. For ECG images plotted with novel background grid colors, the Spearman correlation between the standard and novel formats was 0.994 (95% CI, 0.993-0.994), with a Cohen’s kappa of 0.943 (95% CI, 0.936-0.949). Similarly, when ECG trace colors were altered, Spearman correlation was 0.998 (95% CI, 0.997-0.998), and kappa was 0.970 (95% CI, 0.966-0.975). The image format introducing a new ECG lead layout had a Spearman correlation of 0.971 (95% CI, 0.970-0.973) and a kappa of 0.862 (95% CI, 0.853-0.871). Evaluated on ECG images with novel grid, trace colors, and lead layout changes, the correlation with standard format predictions remained high at 0.969 (95% CI, 0.967-0.970) and kappa at 0.856 (95% CI, 0.848-0.865; Table 2 , Figure 5 ). Download figure Open in new tab Figure 5. Bland-Altman plots comparing the Ensemble Artificial Intelligence Model predictions from plotted electrocardiograms vs. novel image formats. View this table: View inline View popup Download powerpoint Table 2. Agreement Metrics for the Ensemble Artificial Intelligence-enhanced Electrocardiography Model Across Novel Image Formats and Real-world Images. Abbreviations: CI, confidence intervals; ECG, electrocardiograms. Model Stability While Testing on Real-World ECG Images The ensemble model outputs on 100 ECG screenshots captured from the electronic health record showed high concordance with those from standard plotted images of the same ECG (Spearman correlation, 0.962 [95% CI, 0.942–0.975]; Cohen’s kappa, 0.800 [95% CI, 0.686–0.899]). Similar agreement was observed for smartphone photographs of ECGs displayed on computer screens, with Spearman correlation of 0.957 (95% CI, 0.933-0.976) and Cohen’s kappa of 0.845 (95% CI, 0.741-0.940). Notably, photographs of ECG printouts also demonstrated strong concordance (Spearman correlation, 0.942 [95% CI, 0.920-0.964]; kappa, 0.786 [95% CI, 0.659-0.900]; Table 2 ). Further, Bland-Altman analyses indicated minimal systematic bias and narrow limits of agreement across formats ( Figure 6 ). Download figure Open in new tab Figure 6. Performance of the Ensemble AI-ECG Model on Real-World ECG Image Capture Strategies. DISCUSSION In this multicenter evaluation of the test-retest reliability of AI-ECG models, we evaluated outputs from seven AI-ECG models when applied to pairs of ECGs acquired for the same individuals within a short time window. While individual CNNs exhibited moderate consistency, an ensemble XGBoost model integrating outputs and detecting multiple SHDs concurrently yielded substantially more reliable results across diverse populations and care settings. Prediction discordance was more likely when either one or both ECGs in a pair were acquired in inpatient settings, indicating the influence of acute illness or transient physiological states. Conversely, ECGs recorded in outpatient settings yielded more stable predictions, across all sites. Notably, in addition to the higher overall agreement between paired ECGs, the ensemble model also showed greater resilience to contextual and technical variations, including those introduced by ECG layout and acquisition modality. Despite the widespread enthusiasm for AI-ECG-based disease detection, assessments of reliability remain limited. 17 , 18 Prior studies have focused primarily on discrimination performance in cross-sectional analyses, typically using one ECG per individual. 5 , 6 , 11 , 12 , 19 – 22 While this approach avoids analytic bias from repeated sampling and is appropriate for model validation, it does not inform how these tools perform in the real world clinical practice, with an outstanding question on whether a patient’s transient clinical state or acquisition parameters would make the models unreliable in care. Moreover, with the increasing regulatory approval and reimbursement of AI-ECG tools, rigorous evaluations of their implementation characteristics are needed to ensure reliability. 23 , 24 These evaluations have been missing even from the models that have been FDA-approved. Our current work addresses this gap by evaluating model stability at scale, incorporating over a million ECG pairs across multiple hospitals and care settings. Importantly, our findings underscore that deep learning models, particularly CNNs trained for individual disease endpoints, may be susceptible to variations in input conditions, even when those variations are modest and clinically irrelevant. Strategies including robust data augmentation during model development may mitigate these effects to some extent, but residual instability persists. 1 , 25 In contrast, ensemble models can offer greater reliability by synthesizing multiple weak learners and smoothing over individual fluctuations. 1 , 2 , 25 Our findings have several important implications for the clinical integration of AI-ECG tools. First, we demonstrate the strength of ensemble AI-ECG models as biomarkers, not only for their diagnostic accuracy, but also for their output consistency observed across a wide range of testing scenarios. This consistency was evident not only in overall agreement metrics but also in their performance across different care settings and ECG acquisition formats. Importantly, this approach adds significant value by increasing the yield of actionable findings through a composite screen across multiple SHDs, offering a practical advantage in settings where confirmatory testing resources are limited. 26 Second, our analysis of discordance by care setting revealed that AI-ECG predictions were less stable in inpatient ECGs than in outpatient ECGs. Hospitalizations may represent dynamic cardiac states, experiencing volume shifts, arrhythmias, or other acute perturbations, that contribute to fluctuating ECG features, even in the absence of true structural cardiac changes. 27 , 28 These fluctuations may disproportionately affect deep learning-based model predictions. Outpatient ECGs, in contrast, are more likely to reflect chronic baseline physiology, making them more appropriate inputs for screening and risk stratification tools. 29 These findings advice against the use of AI-ECG detection tools for SHDs in acute care environments, where model predictions may not be reliable enough to drive diagnostic or therapeutic decision-making. We also observed that prediction stability was generally lower in younger individuals. Older individuals with manifest disease may have more reproducible ECG signatures. These patterns highlight the value of evaluating sources of discordance, not only to identify model limitations but also to refine clinical context and population targets for model use. 29 Another critical implication of this study lies in the assessment of stability across ECG acquisition formats. Unlike most prior validation studies that have focused exclusively on model inputs derived from digitally stored, uniformly processed waveforms, we tested each model’s reliability across a broad spectrum of real-world ECG image types. 26 , 30 , 31 These included EHR screenshots and smartphone photographs of computer screens and ECG printouts. This analysis simulates actual deployment environments, particularly in low-resource settings where ECGs are not stored digitally or where remote screening relies on photographs captured on personal devices. 1 , 8 Notably, the ensemble model for SHD detection demonstrated robustness to variation in ECG image modalities. These findings are particularly important given that many institutions lack the technical infrastructure for ECG waveform integration, and therefore rely on images for both clinical review and AI-based inference. 26 , 31 Together, our results provide a blueprint for the responsible implementation of AI-ECG tools. Models designed for detection or risk stratification must be evaluated for their reliability, particularly when they are intended for longitudinal use or decision support. 15 , 23 Our findings suggest that ensemble models applied in outpatient settings, and validated across heterogeneous input formats, represent the most implementation-ready tools across the suite of available AI-ECG models. As AI-ECG tools continue to gain regulatory traction, test-retest stability should be a core requirement for deployment, on par with traditional performance metrics like discrimination or calibration. 15 , 23 Our study has potential limitations that merit consideration. First, our analysis was conducted in a selected population of individuals who underwent multiple ECGs within a short interval as part of routine clinical care. While this may introduce bias toward patients with greater healthcare utilization, we mitigated this concern by including a large outpatient network in our analysis and observing consistent prediction stability across sites and subgroups. Second, although some ECGs may have been repeated due to technical issues or artifact, we required a minimum interval of one day between ECGs, reducing the likelihood that ECG pairs reflected immediate repetition for quality assurance. Third, our analysis focused exclusively on image-based AI-ECG models. However, prior studies have shown that image-based models perform comparably to those using ECG waveform signals. 1 , 32 Moreover, the use of image inputs may represent a more stringent test of model reliability, as each ECG image reflects only 2.5-second lead segments, compared with the full 10-second data typically used in signal-based models. Finally, although we evaluated the model on variations in ECG layouts and acquisition strategies, additional prospective work is warranted to evaluate AI-ECG tools in deployment scenarios, particularly in low-resource settings. Future studies using serial ECGs in clinically stable individuals and standardized acquisition protocols may offer additional insights into the optimal use of AI-ECG biomarkers in care delivery. CONCLUSION AI-ECG models developed to detect individual SHDs have modest test-retest reliability, with ensemble models designed to detect a composite of SHDs demonstrated significantly higher reliability across clinical settings and ECG formats. These findings support its readiness for outpatient screening and highlight test-retest stability as a critical benchmark for the real-world implementation of AI-enabled cardiovascular biomarkers. Data Availability Individual-level data for the Yale New Haven Health System cannot be made available due to HIPAA regulations enforced by the Yale IRB. The model is publicly accessible for research use on our website and programming code will be made publicly available on GitHub. Author Contributions L.S.D and R.K. conceived and designed the study. L.S.D. conducted statistical analyses, interpreted the results, wrote the manuscript, and prepared the figures. P.M.C., B.B., A.A., A.F.P., E.K.O., and R.K. interpreted the results and critically revised the manuscript. R.K. acquired the funding. All authors approved the final version for submission. Funding Dr. Khera was supported by the National Institutes of Health (under awards R01AG089981, R01HL167858, and K23HL153775) and the Doris Duke Charitable Foundation (under award 2022060). Dr. Oikonomou was supported by the National Heart, Lung, and Blood Institute of the National Institutes of Health (under award F32HL170592). The funders had no role in the design and conduct of the study; collection, management, analysis, and interpretation of the data; preparation, review, or approval of the manuscript. Conflict of Interest Disclosures Dr. Khera is an Associate Editor of JAMA. Dr. Khera is a coinventor of U.S. Provisional Patent Application No. 63/346,610, “Articles and methods for format-independent detection of hidden cardiovascular disease from printed electrocardiographic images using deep learning” and a co-founder of Ensight-AI. Dr. Khera receives support from the National Institutes of Health (under awards R01AG089981, R01HL167858, and K23HL153775) and the Doris Duke Charitable Foundation (under award 2022060). He receives support from the Blavatnik Foundation through the Blavatnik Fund for Innovation at Yale. He also receives research support, through Yale, from Bristol-Myers Squibb, BridgeBio, and Novo Nordisk. In addition to 63/346,610, Dr. Khera is a coinventor of U.S. Pending Patent Applications WO2023230345A1, US20220336048A1, 63/484,426, 63/508,315, 63/580,137, 63/606,203, 63/619,241, and 63/562,335. Dr. Khera and Dr. Oikonomou are co-founders of Evidence2Health, a precision health platform to improve evidence-based cardiovascular care. Dr. Oikonomou has been a consultant for Caristo Diagnostics Ltd and Ensight-AI Inc, and has received royalty fees from technology licensed through the University of Oxford, outside the submitted work. All other authors declare no competing interests. Data Sharing Statement Individual-level data for the Yale New Haven Health System cannot be made available due to HIPAA regulations enforced by the Yale IRB. The model is publicly accessible for research use on our website and programming code will be made publicly available on GitHub. ABBREVIATIONS LIST AR aortic regurgitation AS aortic stenosis AUROC area under the receiver operating characteristic curve CCC concordance correlation coefficient CI confidence intervals CNN convolutional neural network ECG electrocardiogram LVH left ventricular hypertrophy LVSD left ventricular systolic dysfunction MR mitral regurgitation MS mitral stenosis PRESENT-SHD Practical scREening using ENsemble machine learning sTrategy for Structural Heart Disease SHD structural heart disease TTE transthoracic echocardiogram XGBoost extreme gradient boosting YNHH Yale New Haven Hospital YNHHS Yale New Haven Health System REFERENCES 1. ↵ Dhingra LS , Aminorroaya A , Sangha V , Pedroso AF , Shankar SV , Coppi A , Foppa M , Brant LC , Barreto SM , Ribeiro ALP , Krumholz HM , Oikonomou EK , Khera R. Ensemble deep learning algorithm for structural heart disease screening using electrocardiographic images: PRESENT SHD . J Am Coll Cardiol [Internet] . 2025 [cited 2025 Mar 24];Available from: https://www.jacc.org/doi/10.1016/j.jacc.2025.01.030 2. ↵ Ulloa-Cerna AE , Jing L , Pfeifer JM , Raghunath S , Ruhl JA , Rocha DB , Leader JB , Zimmerman N , Lee G , Steinhubl SR , Good CW , Haggerty CM , Fornwalt BK , Chen R . rECHOmmend: An ECG-Based Machine Learning Approach for Identifying Patients at Increased Risk of Undiagnosed Structural Heart Disease Detectable by Echocardiography . Circulation . 2022 ; 146 : 36 – 47 . OpenUrl CrossRef PubMed 3. ↵ Picano E . Economic and biological costs of cardiac imaging . Cardiovasc Ultrasound [Internet] . 2005 ; 3 . Available from : doi: 10.1186/1476-7120-3-13 OpenUrl CrossRef PubMed 4. Vitola JV , Shaw LJ , Allam AH , Orellana P , Peix A , Ellmann A , Allman KC , Lee BN , Siritara C , Keng FYJ , Sambuceti G , Kiess MC , Giubbini R , Bouyoucef SE , He Z-X , Thomas GS , Mut F , Dondi M . Assessing the need for nuclear cardiology and other advanced cardiac imaging modalities in the developing world . J Nucl Cardiol . 2009 ; 16 : 956 – 961 . OpenUrl CrossRef PubMed 5. ↵ Dhingra LS , Aminorroaya A , Sangha V , Pedroso AF , Asselbergs FW , Brant LCC , Barreto SM , Ribeiro ALP , Krumholz HM , Oikonomou EK , Khera R . Heart failure risk stratification using artificial intelligence applied to electrocardiogram images: a multinational study . Eur Heart J [Internet] . 2025 ;Available from : doi: 10.1093/eurheartj/ehae914 OpenUrl CrossRef 6. ↵ Sangha V , Nargesi AA , Dhingra LS , Khunte A , Mortazavi BJ , Ribeiro AH , Banina E , Adeola O , Garg N , Brandt CA , Miller EJ , Ribeiro ALJ , Velazquez EJ , Giatti L , Barreto SM , Foppa M , Yuan N , Ouyang D , Krumholz HM , Khera R . Detection of Left Ventricular Systolic Dysfunction From Electrocardiographic Images . Circulation [Internet] . 2023 ;Available from : doi: 10.1161/CIRCULATIONAHA.122.062646 OpenUrl CrossRef PubMed 7. ↵ Antoniades C , Chan K . Using artificial intelligence to spot heart failure from ECGs: is it prime time? Eur Heart J . 2025 ; 46 : 1054 – 1056 . OpenUrl PubMed 8. ↵ Yoon M , You SC . Implementing ECG-AI in Clinical Care: From Pixel to Practice . J Am Coll Cardiol . 2025 ; 85 : 1314 – 1316 . OpenUrl PubMed 9. ↵ Leppink J , Pérez-Fuster P . We need more replication research – A case for test-retest reliability . Perspect Med Educ . 2017 ; 6 : 158 – 164 . OpenUrl CrossRef PubMed 10. ↵ Anumana ECG-AI LEF [Internet] . [cited 2025 Feb 11];Available from: https://anumana.ai/ecg-ai-lef 11. ↵ Bjerkén LV , Rønborg SN , Jensen MT , Ørting SN , Nielsen OW . Artificial intelligence enabled ECG screening for left ventricular systolic dysfunction: a systematic review . Heart Fail Rev . 2023 ; 28 : 419 – 430 . OpenUrl PubMed 12. ↵ Attia ZI , Kapa S , Lopez-Jimenez F , McKie PM , Ladewig DJ , Satam G , Pellikka PA , Enriquez-Sarano M , Noseworthy PA , Munger TM , Asirvatham SJ , Scott CG , Carter RE , Friedman PA . Screening for cardiac contractile dysfunction using an artificial intelligence-enabled electrocardiogram . Nat Med . 2019 ; 25 : 70 – 74 . OpenUrl CrossRef PubMed 13. ↵ Cohen-Shelly M , Attia ZI , Friedman PA , Ito S , Essayagh BA , Ko W-Y , Murphree DH , Michelena HI , Enriquez-Sarano M , Carter RE , Johnson PW , Noseworthy PA , Lopez-Jimenez F , Oh JK . Electrocardiogram screening for aortic valve stenosis using artificial intelligence . Eur Heart J . 2021 ; 42 : 2885 – 2896 . OpenUrl PubMed 14. ↵ Khera R , Oikonomou EK , Nadkarni GN , Morley JR , Wiens J , Butte AJ , Topol EJ . Transforming cardiovascular care with artificial intelligence: From discovery to practice . J Am Coll Cardiol . 2024 ; 84 : 97 – 114 . OpenUrl PubMed 15. ↵ Oikonomou EK , Khera R . Artificial intelligence-enhanced patient evaluation: bridging art and science . Eur Heart J . 2024 ; 45 : 3204 – 3218 . OpenUrl PubMed 16. ↵ Giavarina D . Understanding Bland Altman analysis . Biochem Med (Zagreb) . 2015 ; 25 : 141 – 151 . OpenUrl PubMed 17. ↵ Siontis KC , Noseworthy PA , Attia ZI , Friedman PA . Artificial intelligence-enhanced electrocardiography in cardiovascular disease management . Nat Rev Cardiol . 2021 ; 18 : 465 – 478 . OpenUrl PubMed 18. ↵ Attia ZI , Harmon DM , Behr ER , Friedman PA . Application of artificial intelligence to the electrocardiogram . Eur Heart J . 2021 ; 42 : 4717 – 4730 . OpenUrl CrossRef PubMed 19. ↵ Sudarshan VK , Acharya UR , Oh SL , Adam M , Tan JH , Chua CK , Chua KP , Tan RS . Automated diagnosis of congestive heart failure using dual tree complex wavelet transform and statistical features extracted from 2s of ECG signals . Comput Biol Med . 2017 ; 83 : 48 – 58 . OpenUrl PubMed 20. Aminorroaya A , Dhingra LS , Sangha V , Oikonomou EK , Khunte A , Vasisht Shankar S , Pedroso Camargos A , Haynes NA , Hofer I , Ouyang D , Nadkarni GN , Khera R . Deep learning-enabled detection of aortic stenosis from noisy single lead electrocardiograms [Internet] . medRxiv . 2023 ; Available from : doi: 10.1101/2023.09.29.23296310 OpenUrl Abstract / FREE Full Text 21. Sangha V , Dhingra LS , Aminorroaya A , Croon PM , Sikand NV , Sen S , Martinez MW , Maron MS , Krumholz HM , Asselbergs FW , Oikonomou EK , Khera R . Identification of hypertrophic cardiomyopathy on electrocardiographic images with deep learning . Nat Cardiovasc Res . 2025 ; 4 : 991 – 1000 . OpenUrl PubMed 22. ↵ Dhingra LS , Sangha V , Aminorroaya A , Bryde R , Gaballa A , Ali AH , Mehra N , Krumholz HM , Sen S , Kramer CM , Martinez MW , Desai MY , Oikonomou EK , Khera R . A multicenter evaluation of the impact of therapies on deep learning-based electrocardiographic hypertrophic cardiomyopathy markers . Am J Cardiol . 2024 ; 237 : 35 – 40 . OpenUrl PubMed 23. ↵ Zhu S , Gilbert M , Chetty I , Siddiqui F. The 2021 landscape of FDA-approved artificial intelligence/machine learning-enabled medical devices: An analysis of the characteristics and intended use . Int J Med Inform . 2022 ; 165 : 104828 . OpenUrl CrossRef PubMed 24. ↵ Ose B , Sattar Z , Gupta A , Toquica C , Harvey C , Noheria A . Artificial intelligence interpretation of the electrocardiogram: A state-of-the-art review . Curr Cardiol Rep . 2024 ; 26 : 561 – 580 . OpenUrl CrossRef PubMed 25. ↵ Aminorroaya A , Dhingra LS , Pedroso AF , Shankar SV , Coppi A , Khunte A , Foppa M , Brant LCC , Barreto SM , Ribeiro ALP , Krumholz HM , Oikonomou EK , Khera R . Development and multinational validation of an ensemble deep learning algorithm for detecting and predicting structural heart disease using noisy single-lead electrocardiograms . Eur Heart J Digit Health [Internet] . 2025 ;Available from : doi: 10.1093/ehjdh/ztaf034 OpenUrl CrossRef 26. ↵ Mocumbi AO . Cardiovascular health care in low- and middle-income countries . Circulation . 2024 ; 149 : 557 – 559 . OpenUrl PubMed 27. ↵ Larson TS , Brady WJ . Electrocardiographic monitoring in the hospitalized patient: a diagnostic intervention of uncertain clinical impact . Am J Emerg Med . 2008 ; 26 : 1047 – 1055 . OpenUrl CrossRef PubMed Web of Science 28. ↵ Ansermot N , Bochatay M , Schläpfer J , Gholam M , Gonthier A , Conus P , Eap CB . Prevalence of ECG abnormalities and risk factors for QTc interval prolongation in hospitalized psychiatric patients . Ther Adv Psychopharmacol [Internet] . 2019 ; 9 . Available from : doi: 10.1177/2045125319891386 OpenUrl CrossRef 29. ↵ Reddy KP , Eberly LA , Nathan AS . Inequities in access to structural heart disease interventions . JAMA Cardiol . 2023 ; 8 : 5 – 6 . OpenUrl PubMed 30. ↵ Steinberg DH , Staubach S , Franke J , Sievert H . Defining structural heart disease in the adult patient: current scope, inherent challenges and future directions . Eur Heart J Suppl . 2010 ; 12 : E2 – E9 . OpenUrl CrossRef 31. ↵ Diamantino AC , Nascimento BR , Nunes MCP , Sable CA , Oliveira KKB , Rabelo LC , Franco J , Diamantino LC , Barbosa MM , Reese AT , Olivieri L , de Lima EM , Martins LNA , Colosimo EA , Beaton AZ , Ribeiro ALP , PROVAR+ (Programa de RastreamentO da VAlvopatia Reumática e outras Doenças Cardiovasculares) investigators. Impact of incorporating echocardiographic screening into a clinical prediction model to optimise utilisation of echocardiography in primary care . Int J Clin Pract . 2021 ; 75 : e13686 . OpenUrl PubMed 32. ↵ Sau A , Zeidaabadi B , Patlatzoglou K , Pastika L , Ribeiro AH , Sabino E , Peters NS , Ribeiro ALP , Kramer DB , Waks JW , Ng FS . A comparison of artificial intelligence-enhanced electrocardiography approaches for prediction of time-to-mortality using electrocardiogram images . Eur Heart J Digit Health [Internet] . 2024 ;Available from : doi: 10.1093/ehjdh/ztae090 OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted November 06, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Reliability of Artificial Intelligence-enhanced Electrocardiography Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Reliability of Artificial Intelligence-enhanced Electrocardiography Lovedeep S Dhingra , Philip M Croon , Bruno Batinica , Arya Aminorroaya , Aline F Pedroso , Evangelos K Oikonomou , Rohan Khera medRxiv 2025.11.04.25339526; doi: https://doi.org/10.1101/2025.11.04.25339526 Share This Article: Copy Citation Tools Reliability of Artificial Intelligence-enhanced Electrocardiography Lovedeep S Dhingra , Philip M Croon , Bruno Batinica , Arya Aminorroaya , Aline F Pedroso , Evangelos K Oikonomou , Rohan Khera medRxiv 2025.11.04.25339526; doi: https://doi.org/10.1101/2025.11.04.25339526 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Cardiovascular Medicine Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4436) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15229) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6600) Geriatric Medicine (668) Health Economics (997) Health Informatics (4538) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (542) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3333) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9232) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a00e583889c38650',t:'MTc3OTY0NzE0Mw=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();