Using Deep learning to Predict Cardiovascular Magnetic Resonance Findings from Echocardiography Videos

Using Deep learning to Predict Cardiovascular Magnetic Resonance Findings from Echocardiography Videos | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Using Deep learning to Predict Cardiovascular Magnetic Resonance Findings from Echocardiography Videos Yuki Sahashi , Milos Vukadinovic , Grant Duffy , View ORCID Profile Debiao Li , View ORCID Profile Susan Cheng , View ORCID Profile Daniel S. Berman , View ORCID Profile David Ouyang , View ORCID Profile Alan C. Kwan doi: https://doi.org/10.1101/2024.04.16.24305936 Yuki Sahashi 1 Department of Cardiology, Smidt Heart Institute, Cedars-Sinai Medical Center , Los Angeles, CA MD, MSc Find this author on Google Scholar Find this author on PubMed Search for this author on this site Milos Vukadinovic 1 Department of Cardiology, Smidt Heart Institute, Cedars-Sinai Medical Center , Los Angeles, CA 3 Biomedical Imaging Research Institute, Cedars-Sinai Medical Center , Los Angeles, CA BS Find this author on Google Scholar Find this author on PubMed Search for this author on this site Grant Duffy 1 Department of Cardiology, Smidt Heart Institute, Cedars-Sinai Medical Center , Los Angeles, CA BS Find this author on Google Scholar Find this author on PubMed Search for this author on this site Debiao Li 3 Biomedical Imaging Research Institute, Cedars-Sinai Medical Center , Los Angeles, CA PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Debiao Li Susan Cheng 1 Department of Cardiology, Smidt Heart Institute, Cedars-Sinai Medical Center , Los Angeles, CA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Susan Cheng Daniel S. Berman 4 Department of Imaging and Medicine, Cedars-Sinai Medical Center , Los Angeles, CA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Daniel S. Berman David Ouyang 1 Department of Cardiology, Smidt Heart Institute, Cedars-Sinai Medical Center , Los Angeles, CA 2 Division of Artificial Intelligence in Medicine, Cedars-Sinai Medical Center , Los Angeles, CA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for David Ouyang For correspondence: alan.kwan{at}cshs.org David.ouyang{at}cshs.org Alan C. Kwan 1 Department of Cardiology, Smidt Heart Institute, Cedars-Sinai Medical Center , Los Angeles, CA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Alan C. Kwan For correspondence: alan.kwan{at}cshs.org David.ouyang{at}cshs.org Abstract Full Text Info/History Metrics Data/Code Preview PDF ABSTRACT Background Echocardiography is the most common modality for assessing cardiac structure and function. While cardiac magnetic resonance (CMR) imaging is less accessible, CMR can provide unique tissue characterization including late gadolinium enhancement (LGE), T1 and T2 mapping, and extracellular volume (ECV) which are associated with tissue fibrosis, infiltration, and inflammation. While deep learning has been shown to uncover findings not recognized by clinicians, it is unknown whether CMR-based tissue characteristics can be derived from echocardiography videos using deep learning. We hypothesized that deep learning applied to echocardiography could predict CMR-based measurements. Methods In a retrospective single-center study, adult patients with CMRs and echocardiography studies within 30 days were included. A video-based convolutional neural network was trained on echocardiography videos to predict CMR-derived labels including wall motion abnormality (WMA) presence, LGE presence, and abnormal T1, T2 or ECV across echocardiography views. The model performance was evaluated in a held-out test dataset not used for training. Results The study population included 1,453 adult patients (mean age 56±18 years, 42% female) with 2,556 paired echocardiography studies occurring on average 2 days after CMR (interquartile range 2 days prior to 6 days after). The model had high predictive capability for presence of WMA (AUC 0.873 [95%CI 0.816-0.922]), however, the model was unable to reliably detect the presence of LGE (AUC 0.699 [0.613-0.780]), native T1 (AUC 0.614 [0.500-0.715]), T2 0.553 [0.420-0.692], or ECV 0.564 [0.455-0.691]). Conclusions Deep learning applied to echocardiography accurately identified CMR-based WMA, but was unable to predict tissue characteristics, suggesting that signal for these tissue characteristics may not be present within ultrasound videos, and that the use of CMR for tissue characterization remains essential within cardiology. Clinical Perspective Tissue characterization of the heart muscle is useful for clinical diagnosis and prognosis by identifying myocardial fibrosis, inflammation, and infiltration, and can be measured using cardiac MRI. While echocardiography is highly accessible and provides excellent functional information, its ability to provide tissue characterization information is limited at this time. Our study using a deep learning approach to predict cardiac MRI-based tissue characteristics from echocardiography showed limited ability to do so, suggesting that alternative approaches, including non-deep learning methods should be considered in future research. Download figure Open in new tab Graphical Abstract Overview of the study pipeline and results. A large echocardiography dataset involving 2,566 studies from 1,453 patients paired with CMR and echocardiography within 30 days from Cedars-Sinai Medical Center was identified. A convolutional neural network with residual connections and spatiotemporal convolutions was trained to predict each CMR finding and detect abnormal findings from echocardiography. Results showed strong prediction of functional abnormalities, but poor prediction of CMR-specific tissue characterization. Introduction Echocardiography plays a central role within cardiovascular care, providing essential information on cardiac structure and function in a highly-accessible format 1 2 . Cardiac MRI (CMR) is also critical to cardiovascular care but has reduced accessibility due to the availability of scanners and qualified physicians to interpret the images. In addition to providing information on cardiac structure and function, CMR is able to provide unique tissue characterization that is helpful for assessing for etiology of disease 3 . This includes myocardial composition including scarring through late gadolinium enhancement (LGE) and infiltration, edema, and diffuse fibrosis through relaxometry techniques such as T1 and T2 mapping, and extracellular volume (ECV) fraction. Despite the utility of CMR 4 , logistical barriers continue to limit its broader uptake, including the availability of CMR resources, the examination cost, the use of gadolinium contrast, and logistical complexity. Deep learning applied to medical imaging provides the opportunity to obtain more information than currently recognized by clinicians in standard clinical care 5 – 9 . For example, deep learning applied to echocardiography has been shown to identify hidden features invisible to the human eye such as age, gender, serological biomarkers, prognosis, and tissue characteristics of cardiac amyloidosis and hypertrophic cardiomyopathy 10 , 11 . We hypothesized that deep learning applied to echocardiography could predict CMR-based measurements. In this study, we train a video-based convolutional neural network to predict CMR features from patients with paired CMR and echocardiography studies. Predicted CMR characteristics including wall motion abnormalities, myocardial scar, and markers of tissue infiltration, edema, and diffuse fibrosis were assessed for deep learning evaluation. Methods Data and study population We identified all adults aged over 18 years at a large cardiac quaternary care center who received clinical CMR, with at least one clinical transthoracic echocardiogram within 30 days of the CMR between May 2011 and June 2022. All echocardiography were performed using Philips EPIQ 7 or iE33 ultrasound machines. Echocardiography views including apical four-chamber (A4c), apical two-chamber (A2c), and parasternal long axis (PLAX), were automatically extracted using an automated view classifier. Videos underwent automated image preprocessing including removing identifying information, electrocardiogram and respirometer tracings, and cropping and downsampling images using cubic interpolation to a standard size and resolution of 112 × 112-pixels 12 . This study was approved by the Institutional Review Board at Cedars-Sinai Medical Center and informed consent was waived due to the retrospective analysis. Echocardiography studies were paired with the nearest CMR study within 30 days. Deep learning models were trained on echocardiogram videos with labels derived from the clincal report of the temporally closest CMR if a single echocardiography study had multiple CMRs within 30 days. Labels included presence or absence of wall motion abnormalities within the given echocardiography view (e.g., within AHA segments included within an apical 4-chamber view versus 2-chamber versus parasternal long axis) or globally (within any segment), presence or absence of LGE within a given view or globally, and both continuous and dichotomized measures of T1, T2, and ECV. Dichotomization was based on abnormal values of native T1 times over 1060ms, T2 times over 58ms, and ECV values above 33%, appropriate to scanner vendor and strength and based on local practice and published values 13 14 . Overall study pipeline is demonstrated in the Graphical Abstract . CMR protocol and assessment All CMR examinations were clinically-ordered studies performed using a 1.5T Avanto scanner (Siemens Healthineers, Erlangen, Germany). While clinical protocols varied over time, cine SSFP images were graded at the time of acquisition for the presence or absence of regional wall motion abnormalities based on an American Heart Association (AHA) 17 segmentation model 15 . If available, T1 mapping was performed using a standard 5(3)3 modified inversion look-locker (MOLLI) sequence, with measurement of the T1 value within the mid-sepum within the mid slice. If overt tissue abnormalities were present in this region, measurement representative of the diffuse tissue composition would be performed in a secondary region, most commonly basal mid-septal slice, consistent with guidelines. If available, T2 mapping was performed using a T2-prepped SSFP sequence, with similar measurement approach as used for T1 values. Post-contrast images included T1-mapping using a short-T1 optimized MOLLI approach. ECV values were calculated from the pre- and post-contrast T1 maps in combination with point-of-care hematocrit masurement. LGE was measured 12-20 min after gadolinium contrast injection (Gadbutrol), using turbo FLASH or magnitude weighted and phase-sensitive inversion recovery gradient echo high-resolution short axis stacks, correlated with long axis LGE images, to grade presence, severity, and location of scarring using the AHA 17-segment model. All clinical examinations were reviewed by two clinicians including an advanced cardiac imaging fellow, and advanced cardiac imaging attending with Level-3 equivalent training. Overview of AI model and training For model training and testing, we used a convolutional neural network with residual connections and spatiotemporal convolutions 12 to predict CMR findings, including the presence of wall motion abnormalities, the presence of myocardial scar, native T1 value, T2 value, and ECV fraction. For binary classification tasks for predicting dichotomized CMR findings, we used binary cross-entropy loss and trained to maximize the area under the receiver operating characteristics using an AdamW optimizer with an initial learning rate of 0.001. For classification tasks including wall motion abnormalities, scarring, abnormal T1, T2, and ECV, predictions were organized by presence or absence of abnormality within the AHA segments corresponding to the specific echocardiography view. To assess for the global presence or absence of any wall motion abnormalities or myocardial scar, we combined the predictions for A4c, A2c, and PLAX views through logistic regression for a final prediction of these measurements. For a regression tasks applied for prediction of continuous labels (native T1 value, T2 value, and ECV fraction), the model was similarly trained in A4c, A2c, and PLAX views and combined to provide a global result. The model was specified to minimize the mean average error using squared loss. In both classification and regression tasks, early stopping with 10 epochs was applied, and the batch size and the number of epochs were set to 10 and 50, respectively. The dataset was randomly split at an 8:1:1 ratio for model training, validation, and held-out testing. The weights from the epoch with the best metrics were used on the held-out test dataset. All model training and evaluation were conducted using Python 3.8 and the publicly available PyTorch library. Statistical analysis All performance analyses were performed using a held-out test dataset not involved in model training. For dichotomous outcomes, the model’s ability was assessed by calculating the area under the receiver operating characteristic (AUROC) curve. For continuous values including T1 value, T2 value, and ECV, mean absolute error (MAE) and coefficient of determination (R 2 ) were calculated. Bland-Altman plots where the average of two measurements was plotted against the difference were used to check the agreement between the actual and predicted values from echocardiography. 95% confidence intervals were calculated with 10,000 bootstrapping samples. All data were analyzed using Python and R. RESULTS Patient characteristics We trained and tested a model using a dataset that included 2,556 echocardiography studies paired with CMR findings from a total of 1,453 patients (mean age: 56.0±17.9 years, 41.8% female). The patient population had a range of cardiovascular comorbidities including hypertension (37.7%), hyperlipidemia (28.5%), and diabetes (16.5%) ( Table 1 ). The mean left ventricular ejection fraction (LVEF) reported by echocardiography and CMR was 48.0±18.5% and 49.2±17.4%, respectively. In the CMR assessment, 48.3% had wall motion abnormalities and 49.0% had scar findings in one or more of the AHA 17 segments. Mean native T1 value was 1020±72.1 ms with 26.0% of the patients having elevated values, mean T2 value was 48.7±6.1ms with 8.2% having elevated values, and mean ECV was 28.4±5.7% with 21.8% having elevated values. The median time interval between the echocardiography and CMR was 2 days (interquartile range, −2 to +6 days). View this table: View inline View popup Table 1: Baseline patient characteristics, cardiac magnetic resonance and echocardiogram findings Model Performance Prediction of wall motion abnormalities was robust, with the AUROC for prediction within the A4c segments of 0.817 (95% CI: 0.791-0.843), A2c of 0.756 (0.707-0.802) and PLAX of 0.812 (0.777-0.847). The combination of these view prediction for global wall motion abnormalities showed strong prediction, with AUROC of 0.873 (0.816-0.922) ( Figure 1 ). On the other hand, prediction of tissue composition performed poorly overall. LGE prediction was low, at AUROC of 0.657 (0.620-0.693) in the A4c views and poor at AUROC of 0.591 (0.522-0.650) in the A2c view and 0.541 (0.483 - 0.594) in the PLAX views. The global prediction of 0.699 (0.613-0.780) was the highest ( Figure 1 ). AUROC for prediction of T1, T2, and ECV was similarly limited, with global prediction of abnormal T1 time of 0.614 (0.500-0.715), T2 time of 0.553 (0.420-0.692), and ECV of 0.564 (0.455-0.691). These limited capabilities were consistent across A4c, A2c, and PLAX view videos ( Table 2 ). Prediction of continuous measures globally showed that the models were minimally predictive, with R 2 of 0.04 and MAE of 49.8 ms (47.5-52.2) for T1, R 2 of 0.002 and MAE of 5.39 ms (5.07-5.71) for T2 and R 2 of 0.07 and MAE of 4.38% (4.17-4.58) for ECV ( Supplemental Figure 1 ). Download figure Open in new tab Supplementary Figure 1: Performance of deep-learning model on prediction of abnormalities and estimation of CMR-specific tissue composition. (A) Receiver-operating-characteristic (ROC) curve for predicting abnormal CMR findings, including native T1 value, T2 value, and extracellular volume fraction. Green curve denotes the prediction of native T1 finding ≥1060ms. Purple denotes the prediction of T2 ≥ 58ms. The orange curve denotes the prediction of ECV ≥33%. Scatterplot for predicted versus measured (B) native T1 value, (C) T2 value, and (D) ECV. Download figure Open in new tab Figure 1. Performance of deep-learning on a held-out test dataset. Receiver-operating-characteristic (ROC) curve for predicting myocardial wall motion abnormalities and myocardial scar finding detected by CMR. A: prediction of wall motion abnormalities, B: prediction of myocardial scar. Black curves denote the performance characteristics of a deep learning model for presence of global abnormal findings. Red, blue, and green curves demonstrated the prediction of abnormal findings within A4c, A2c, and PLAX views respectively. View this table: View inline View popup Table 2: Diagnostic accuracy of deep learning for predicting wall motion abnormalities, myocardial scar, and abnormal tissue composition by echocardiographic view Discussion In this study, we investigated the accuracy of a video-based deep learning model for predicting CMR findings from echocardiography, with the express goal to bridge the gap between the accessibility of echocardiography and the diagnostic information of CMR. While the model was able to predict wall motion abnormalities, it was unable to reliably predict fundamental CMR tissue characteristics including LGE, T1, T2, and ECV. Of these, the LGE prediction was the highest at 0.699 (0.613-0.780), which is unlikely to be accurate enough for clinical utility. Overall, we would consider the attempt to predict meaningful CMR tissue characteristics from echocardiography unsuccessful. There has been limited previous cross-modality research specifically linking echocardiography to CMR findings, with our literature review revealing no deep-learning-based publications for LGE, let alone T1, T2, and ECV. A smaller study leveraging a radiomics-based approach in patients admitted for heart failure, was able to identify the presence or absence of LGE in the anteroseptal and posterior wall myocardial segments, within the specific regions where regions of interest were placed for feature extraction. This was performed as a subset of a larger study, focusing on 89 patients for training and 40 patients for testing, all with echocardiography within 48 hours of clinical CMR, resulting in an AUROC of 0.84 16 . Other approaches such as echocardiography speckle-tracking strain has also been used to predict LGE with variable success – One publication demonstrated a significant association in 155 patients specifically diagnosed with carbon monoxide poisoning 17 , whereas a more general and smaller 50-patient population was unable to find an association with echo-based strain (AUROC of 0.58), but stronger associations with CMR-based strain (AUROC 0.67-0.78 including longitudinal and circumferential strain) 18 . Electrocardiogram (ECG) deep learning approaches have been applied to predict LGE and appear to have potentially stronger associations than were found within our study, though these are typically limited to specific populations. For example, prediction of LGE in patients with mitral valve prolapse using a CNN-based approach was able to achieve an AUROC of 0.75 in approximately 600 patients 19 , and 0.76 in a hypertrophic cardiomyopathy population of 1,930 patients though the AUROC decreased to 0.68 in external validation 20 . A smaller study in 114 patients achieved AUROC up to 0.81 for ECG prediction; however, this was from a 6-fold cross-validation without a hold-out dataset, so there is likely a significant contribution of overfitting to the model 21 . In reconciling our results with the established literature, we note that our population was typically both larger, and more general than previous works. Inclusion of diverse clinical conditions may have reduced the ability to predict CMR tissue characteristics, as the histological correlates of LGE, T1, T2, and ECV can vary between disease processes. However, our overarching motivation for echocardiographic tissue characterization was broad accessibility independent of specific disease processes, and therefore we felt that that this was the most appropriate approach. Additionally, given that clinical echocardiography is already commonly used to accurately identify WMA, we present the strong deep learning prediction of WMA not as a proposed clinical application, but to provide quality assurance for the workflow in mapping CMR labels and echo images, and our ability to train deep learning models for echo. We also recognize that of all of the tissue characterization markers, LGE was the highest and showed small signal for prediction (AUROC of 0.699); however, without a much stronger signal, we felt this was most likely due to prediction of confounding factors such as reduced ejection fraction or thinned myocardium. These factors may be easily visible on echocardiography and are associated with, but not equivalent to LGE; and thus not clinically useful to predict with deep learning. Overall, the results of our study suggest that at least within the current population and deep learning architecture, CMR-based tissue characteristics are unable to be derived from standard clinical echocardiography at this time. Additional experiments testing prediction of deep learning across various sample size and model architectures had only modest differences in performance (not shown). At the inception of this study, we recognized that the ability to derive magnetic resonance-specific findings from an ultrasound-based modality may have limited biological plausibility, as the image acquisition and reconstruction process between the two modalities are extremely technically distinct. Historically however, echocardiography tissue characteristics such as granular sparkling has been seen as suggestive of cardiac amyloidosis 22 and integrated backscatter has been proposed for use within both inflammatory and fibrotic conditions 23 , 24 . Detection of histological characteristics by echocardiography thus may be best directly quantified, as while CMR-based tissue characterization is well-accepted, it still represents a surrogate of the true tissue composition. Novel echocardiography techniques such as shear-wave elastography can provide signal for fibrosis not available through standard clinical images and may be able to expand the applicability of echocardiography independent of deep-learning based techniques 25 , 26 . Thus, while our results support the ongoing utility for CMR independent of echocardiography, we are optimistic for the future of echocardiography to provide highly accessible tissue characterization. Limitations There are several limitations in the present study. First, as a retrospective single center study, our results were the result of a limited dataset. In particular, CMR practices may vary significantly between locations, and it remains possible stronger relationships can be found with larger datasets. The 30-day interval was selected as being a relatively short time frame but with a high number of eligible studies, and we recognize that incident clinical events or resolution of acute findings may have occurred between the two studies, though the short median time interval gives some degree of assurance. CMR referral was clinical and included a wide range of diseases conditions and severities. This heterogeneity may have increased the challenge of finding significant associations. Conclusion In conclusion, we found that a video-based deep learning architecture using echocardiography was able to identify CMR-based WMA, but was unable to accurately identify CMR-based tissue characteristics including LGE, T1 time, T2 time, and ECV. Further testing using alternative populations and approaches should be considered. At present, our study supports the ongoing use of CMR for tissue characterization in appropriate patients, despite challenges to patient access. Disclosures ACK reports support from the American Heart Association (AHA; 23CDA1053659) and National Institutes of Health (NIH; UL1TR001881), and consulting fees from InVision Medical Technology DO reports support from the National Institute of Health (NIH; NHLBI R00HL157421) and Alexion, and consulting or honoraria for lectures from EchoIQ, Ultromics, Pfizer, InVision, the Korean Society of Echo, and the Japanese Society of Echo. Sources of Funding ACK: AHA 23CDA1053659, NIH UL1TR001881, 75N92020D00021 DO: NHLBI R00HL157421, 75N92020D00021 YS: Japanese Society for the Promotion of Science, Grants-in-Aid for Scientific Research (JSPS-KAKENHI) Author contributions Concept and design: YS, DO, AK Acquisition, analysis, or interpretation of data: YS, DO, AK, MV, GD, DB, SC Drafting and Critical revision of the manuscript: All authors Statistical analysis: YS, DO, AK Obtained funding: DO, YS, AK Supervision: AK, DO, SC, DB, DL Data and code availability All of the code for the present study is available at https://github.com/echonet/ . Data Availability Data available on reasonable request Footnotes ↵ * Co-Senior Author Abbreviations AHA American Heart Association AUROC Area under receiver operating characteristic A4C Apical 4 chamber A2C Apical 2 chamber CMR Cardiac magnetic resonance ECV Extracellular volume LVEF Left ventricular ejection fraction LGE late gadolinium enhancement PLAX Parasternal long axis MOLLI modified inversion look-locker Reference 1. ↵ Lancellotti , P. et al. The use of echocardiography in acute cardiovascular care: recommendations of the European Association of Cardiovascular Imaging and the Acute Cardiovascular Care Association . Eur Heart J Acute Cardiovasc Care 4 , 3 – 5 ( 2015 ). OpenUrl CrossRef PubMed 2. ↵ Omar , A. M. S. , Bansal , M. & Sengupta , P. P . Advances in Echocardiographic Imaging in Heart Failure With Reduced and Preserved Ejection Fraction . Circ. Res . 119 , 357 – 374 ( 2016 ). OpenUrl Abstract / FREE Full Text 3. ↵ Marwick , T. H. , Neubauer , S. & Petersen , S. E . Use of cardiac magnetic resonance and echocardiography in population-based studies: why, where, and when? Circ. Cardiovasc. Imaging 6 , 590 – 596 ( 2013 ). OpenUrl FREE Full Text 4. ↵ Goldfarb , J. W. & Weber , J . Trends in Cardiovascular MRI and CT in the U.S. Medicare Population from 2012 to 2017 . Radiol Cardiothorac Imaging 3 , e200112 ( 2021 ). OpenUrl 5. ↵ Poplin , R. et al. Prediction of cardiovascular risk factors from retinal fundus photographs via deep learning . Nat Biomed Eng 2 , 158 – 164 ( 2018 ). OpenUrl 6. Holmstrom , L. et al. Deep learning-based electrocardiographic screening for chronic kidney disease . Commun. Med . 3 , 73 ( 2023 ). 7. Ouyang , D. et al. Electrocardiographic deep learning for predicting post-procedural mortality: a model development and validation study . The Lancet Digital Health ( 2023 ) doi: 10.1016/S2589-7500(23)00220-0 . OpenUrl CrossRef 8. Attia , Z. I. et al. Age and Sex Estimation Using Artificial Intelligence From Standard 12-Lead ECGs . Circ. Arrhythm. Electrophysiol . 12 , e007284 ( 2019 ). OpenUrl CrossRef 9. ↵ Banerjee , I. , et al. Reading Race: AI Recognises Patient’s Racial Identity In Medical Images . arXiv [cs.CV] ( 2021 ). 10. ↵ Ghorbani , A. et al. Deep learning interpretation of echocardiograms . NPJ Digit Med 3 , 10 ( 2020 ). 11. ↵ Hughes , J. W. et al. Deep learning evaluation of biomarkers from echocardiogram videos . EBioMedicine 73 , 103613 ( 2021 ). 12. ↵ Ouyang , D. et al. Video-based AI for beat-to-beat assessment of cardiac function . Nature 580 , 252 – 256 ( 2020 ). OpenUrl 13. ↵ Schulz-Menger , J. et al. Standardized image interpretation and post-processing in cardiovascular magnetic resonance - 2020 update: Society for Cardiovascular Magnetic Resonance (SCMR): Board of Trustees Task Force on Standardized Post-Processing . J. Cardiovasc. Magn. Reson . 22 , 19 ( 2020 ). 14. ↵ Kawel-Boehm , N. et al. Reference ranges (“normal values”) for cardiovascular magnetic resonance (CMR) in adults and children: 2020 update . J. Cardiovasc. Magn. Reson . 22 , 87 ( 2020 ). 15. ↵ Cerqueira , M. D. et al. Standardized myocardial segmentation and nomenclature for tomographic imaging of the heart. A statement for healthcare professionals from the Cardiac Imaging Committee of the Council on Clinical Cardiology of the American Heart Association . Circulation 105 , 539 – 542 ( 2002 ). OpenUrl FREE Full Text 16. ↵ Kagiyama , N. et al. A low-cost texture-based pipeline for predicting myocardial tissue remodeling and fibrosis using cardiac ultrasound . EBioMedicine 54 , 102726 ( 2020 ). 17. ↵ Cho , D.-H. et al. Clinical and Echocardiographic Predictors for the Presence of Late Gadolinium Enhancement on Cardiac Magnetic Resonance Imaging in Patients with Carbon Monoxide Poisoning . Diagnostics (Basel ) 14 , ( 2023 ). 18. ↵ Erley , J. et al. Echocardiography and cardiovascular magnetic resonance based evaluation of myocardial strain and relationship with late gadolinium enhancement . J. Cardiovasc. Magn. Reson . 21 , 46 ( 2019 ). 19. ↵ Tison , G. H. , et al. Identifying Mitral Valve Prolapse at Risk for Arrhythmias and Fibrosis From Electrocardiograms Using Deep Learning . JACC Adv 2 , ( 2023 ). 20. ↵ Carrick , R. T. et al. Identification of High-Risk Imaging Features in Hypertrophic Cardiomyopathy using Electrocardiography: A Deep-Learning Approach . Heart Rhythm ( 2024 ) doi: 10.1016/j.hrthm.2024.01.031 . OpenUrl CrossRef 21. ↵ Gumpfer , N. , Grün , D. , Hannig , J. , Keller , T. & Guckert , M . Detecting myocardial scar using electrocardiogram data and deep neural networks . Biol. Chem . 402 , 911 – 923 ( 2021 ). OpenUrl 22. ↵ Cacciapuoti , F . The role of echocardiography in the non-invasive diagnosis of cardiac amyloidosis . J. Echocardiogr . 13 , 84 – 89 ( 2015 ). OpenUrl 23. ↵ Hoyt , R. H. , Collins , S. M. , Skorton , D. J. , Ericksen , E. E. & Conyers , D . Assessment of fibrosis in infarcted human hearts by analysis of ultrasonic backscatter . Circulation 71 , 740 – 744 ( 1985 ). OpenUrl Abstract / FREE Full Text 24. ↵ Omi , W. et al. Ultrasonic tissue characterization in acute myocarditis: a case report . Circ. J . 66 , 416 – 418 ( 2002 ). OpenUrl CrossRef PubMed Web of Science 25. ↵ Caenen Annette et al. Ultrasound Shear Wave Elastography in Cardiology . JACC Cardiovasc. Imaging 17 , 314 – 329 ( 2024 ). OpenUrl 26. ↵ Wouters , L. et al. Septal Scar Detection in Patients With Left Bundle Branch Block Using Echocardiographic Shear Wave Elastography . JACC Cardiovasc. Imaging 16 , 713 – 715 ( 2023 ). OpenUrl View the discussion thread. Back to top Previous Next Posted April 22, 2024. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Using Deep learning to Predict Cardiovascular Magnetic Resonance Findings from Echocardiography Videos Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Using Deep learning to Predict Cardiovascular Magnetic Resonance Findings from Echocardiography Videos Yuki Sahashi , Milos Vukadinovic , Grant Duffy , Debiao Li , Susan Cheng , Daniel S. Berman , David Ouyang , Alan C. Kwan medRxiv 2024.04.16.24305936; doi: https://doi.org/10.1101/2024.04.16.24305936 Share This Article: Copy Citation Tools Using Deep learning to Predict Cardiovascular Magnetic Resonance Findings from Echocardiography Videos Yuki Sahashi , Milos Vukadinovic , Grant Duffy , Debiao Li , Susan Cheng , Daniel S. Berman , David Ouyang , Alan C. Kwan medRxiv 2024.04.16.24305936; doi: https://doi.org/10.1101/2024.04.16.24305936 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Cardiovascular Medicine Subject Areas All Articles Addiction Medicine (573) Allergy and Immunology (865) Anesthesia (303) Cardiovascular Medicine (4457) Dentistry and Oral Medicine (445) Dermatology (383) Emergency Medicine (610) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1517) Epidemiology (15244) Forensic Medicine (30) Gastroenterology (1132) Genetic and Genomic Medicine (6620) Geriatric Medicine (669) Health Economics (1002) Health Informatics (4557) Health Policy (1372) Health Systems and Quality Improvement (1615) Hematology (543) HIV/AIDS (1272) Infectious Diseases (except HIV/AIDS) (15936) Intensive Care and Critical Care Medicine (1106) Medical Education (624) Medical Ethics (147) Nephrology (670) Neurology (6634) Nursing (346) Nutrition (999) Obstetrics and Gynecology (1148) Occupational and Environmental Health (957) Oncology (3348) Ophthalmology (980) Orthopedics (369) Otolaryngology (421) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1696) Pharmacology and Therapeutics (693) Primary Care Research (714) Psychiatry and Clinical Psychology (5463) Public and Global Health (9257) Radiology and Imaging (2210) Rehabilitation Medicine and Physical Therapy (1371) Respiratory Medicine (1198) Rheumatology (598) Sexual and Reproductive Health (716) Sports Medicine (532) Surgery (714) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a033fe433e10df94',t:'MTc4MDA0MTkxNw=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();