Cardiac Measurement Calculation on Point-of-Care Ultrasonography with Artificial Intelligence

Cardiac Measurement Calculation on Point-of-Care Ultrasonography with Artificial Intelligence | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Cardiac Measurement Calculation on Point-of-Care Ultrasonography with Artificial Intelligence Sarah F Mercaldo , Bernardo C Bizzo , Tsion Sadore , Madeleine A Halle , Ashley L MacDonald , Isabella Newbury-Chaet , Eric L’Italien , Alex S Schultz , Victor Tam , Sheila M Hegde , Judy R Mangion , Praveen Mehrotra , Qiong Zhao , Justina Wu , View ORCID Profile James M Hillis doi: https://doi.org/10.1101/2025.06.27.25330438 Sarah F Mercaldo 1 Digital Clinical Research Organization , Mass General Brigham AI, Boston, MA, USA 2 Department of Radiology, Massachusetts General Hospital , Boston, MA, USA 3 Harvard Medical School , Boston, MA, USA PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Bernardo C Bizzo 1 Digital Clinical Research Organization , Mass General Brigham AI, Boston, MA, USA 2 Department of Radiology, Massachusetts General Hospital , Boston, MA, USA 3 Harvard Medical School , Boston, MA, USA MD PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Tsion Sadore 1 Digital Clinical Research Organization , Mass General Brigham AI, Boston, MA, USA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Madeleine A Halle 1 Digital Clinical Research Organization , Mass General Brigham AI, Boston, MA, USA BSc Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ashley L MacDonald 1 Digital Clinical Research Organization , Mass General Brigham AI, Boston, MA, USA BSc Find this author on Google Scholar Find this author on PubMed Search for this author on this site Isabella Newbury-Chaet 1 Digital Clinical Research Organization , Mass General Brigham AI, Boston, MA, USA BSc Find this author on Google Scholar Find this author on PubMed Search for this author on this site Eric L’Italien 1 Digital Clinical Research Organization , Mass General Brigham AI, Boston, MA, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Alex S Schultz 1 Digital Clinical Research Organization , Mass General Brigham AI, Boston, MA, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Victor Tam 1 Digital Clinical Research Organization , Mass General Brigham AI, Boston, MA, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Sheila M Hegde 3 Harvard Medical School , Boston, MA, USA 4 Division of Cardiovascular Medicine, Department of Medicine, Brigham and Women’s Hospital , Boston, MA, USA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Judy R Mangion 3 Harvard Medical School , Boston, MA, USA 4 Division of Cardiovascular Medicine, Department of Medicine, Brigham and Women’s Hospital , Boston, MA, USA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Praveen Mehrotra 5 Division of Cardiology, Sidney Kimmel Medical College, Thomas Jefferson University , PA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Qiong Zhao 6 Inova Schar Heart and Vascular , Inova Health System, VA MD PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Justina Wu 3 Harvard Medical School , Boston, MA, USA 4 Division of Cardiovascular Medicine, Department of Medicine, Brigham and Women’s Hospital , Boston, MA, USA MD PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site James M Hillis 1 Digital Clinical Research Organization , Mass General Brigham AI, Boston, MA, USA 3 Harvard Medical School , Boston, MA, USA 7 Department of Neurology, Massachusetts General Hospital , Boston, MA, USA MBBS DPhil Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for James M Hillis For correspondence: james.hillis{at}mgh.harvard.edu Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Introduction Point-of-care ultrasonography (POCUS) enables clinicians to obtain critical diagnostic information at the bedside especially in resource limited settings. This information may include 2D cardiac quantitative data, although measuring the data manually can be time-consuming and subject to user experience. Artificial intelligence (AI) can potentially automate this quantification. This study assessed the interpretation of key cardiac measurements on POCUS images by an AI-enabled device (AISAP Cardio V1.0). Methods This retrospective diagnostic accuracy study included 200 POCUS cases from four hospitals (two in Israel and two in the United States). Each case was independently interpreted by three cardiologists and the device for seven measurements (left ventricular (LV) ejection fraction, inferior vena cava (IVC) maximal diameter, left atrial (LA) area, right atrial (RA) area, LV end diastolic diameter, right ventricular (RV) fractional area change and aortic root diameter). The endpoints were the root mean square error (RMSE) of the device compared to the average cardiologist measurement (LV ejection fraction and IVC maximal diameter were primary endpoints; the other measurements were secondary endpoints). Predefined passing criteria were based on the upper bounds of the RMSE 95% confidence intervals (CIs). The inter-cardiologist RMSE was also calculated for reference. Results The device achieved the passing criteria for six of the seven measurements. While not achieving the passing criterion for RV fractional area change, it still achieved a better RMSE than the inter-cardiologist RMSE. The RMSE was 6.20% (95% CI: 5.57 to 6.83; inter-cardiologist RMSE of 8.23%) for LV ejection fraction, 0.25cm (95% CI: 0.20 to 0.29; 0.36cm) for IVC maximal diameter, 2.39cm 2 (95% CI: 1.96 to 2.82; 4.39cm 2 ) for LA area, 2.11cm 2 (95% CI: 1.75 to 2.47; 3.49cm 2 ) for RA area, 5.06mm (95% CI: 4.58 to 5.55; 4.67mm) for LV end diastolic diameter, 10.17% (95% CI: 9.01 to 11.33; 14.12%) for RV fractional area change and 0.19cm (95% CI: 0.16 to 0.21; 0.24cm) for aortic root diameter. Discussion The device accurately calculated these cardiac measurements especially when benchmarked against inter-cardiologist variability. Its use could assist clinicians who utilize POCUS and better enable their clinical decision-making. Introduction Point-of-care ultrasonography (POCUS) has revolutionized many aspects of acute clinical care over recent decades. 1 – 3 It enables clinicians to obtain more diagnostic information at the bedside, and has been adopted by multiple specialties especially emergency medicine and hospital internal medicine. 4 – 6 With some handheld probes connecting directly to a smartphone or tablet, it can be particularly beneficial in resource limited settings. 7 – 10 Its use includes the quantification of key cardiac measurements such as left ventricular (LV) ejection fraction and inferior vena cava (IVC) size. 11 Artificial intelligence (AI) has similarly started to revolutionize clinical care with many of its current applications relating to medical image interpretation. 12 , 13 One of its core uses is the automatic quantification of measurements, especially those that otherwise require time-consuming manual tracings. It has previously been used to quantify cardiac measurements using conventional echocardiograms. 14 – 16 It has also been shown to improve interobserver variability and workflow efficiency for echocardiogram interpretation. 17 , 18 Echocardiography, however, typically provides more comprehensive evaluations with a higher image resolution than POCUS. 19 This study evaluated the ability of an AI-enabled device to calculate seven key cardiac measurements based on POCUS images including LV ejection fraction, IVC maximal diameter, left atrial (LA) area, right atrial (RA) area, LV end diastolic diameter, right ventricular (RV) fractional area change and aortic root diameter. Methods Study design This diagnostic study was conducted using POCUS cases acquired prospectively from four hospitals in Israel and the United States. The acquisition of cases was approved by local Institutional Review Boards at each of Baruch Padeh Medical Center, Inova Health, Sheba Medical Center and Thomas Jefferson University Hospital. All patients provided informed consent. The subsequent diagnostic study involved transfer of deidentified cases to Mass General Brigham and was approved by the Mass General Brigham Institutional Review Board. The study was conducted in accordance with relevant guidelines and regulations. This manuscript followed the Standards for Reporting of Diagnostic Accuracy (STARD) reporting guideline. Inclusion criteria Eligible participants were identified in a consecutive manner in both the inpatient and outpatient settings. They were patients with suspected or established cardiac pathology, and with an indication for cardiac ultrasound. They were initially identified by treating physicians including cardiologists, internal medicine physicians and emergency department physicians, and were consented by local research personnel. The inclusion criteria included male or female individuals aged 21 or older, the provision of a signed and dated informed consent form, a stated willingness to comply with all study procedures and availability for the study duration. The exclusion criteria included unstable clinical condition as determined by the principal investigator (where ultrasonography was considered detrimental to the participant), clinical conditions that prevented an ultrasonography study according to the investigators or treating physicians, presence of a prosthetic cardiac valve of any type or position, pregnancy or lactation, contact isolation due to infectious disease, presence of a left ventricular assist device, known congenital heart disease, body mass index (BMI) of 40 or greater, history of lung resection, known very poor cardiac ultrasonography image quality based on prior comprehensive echocardiography examinations, or mechanical ventilation or other situations where the required ultrasonography views could not be adequately obtained. POCUS The POCUS cases were acquired by physicians and sonographers at the bedside using mostly handheld devices. They contained only four cardiac views according to the American Society of Echocardiography guidelines. 21 These views included parasternal long axis with and without color Doppler, apical four chamber with and without color Doppler, parasternal short axis at the aortic valve level, and subcostal IVC view. Cohort selection Once cases were acquired at individual sites, they underwent de-identification prior to transfer to the sponsor. These cases were maintained separately from the cases used for device development. Only two of the four sites had contributed cases for device development, which had all been acquired prior to the cases used for the validation study. The deidentified cases were subsequently transferred to Mass General Brigham for the diagnostic study. The final cohort required only a subset of cases based on powering calculations, which was selected in a randomized manner. Ground truth process The three ground truth cardiologists were US board-certified with a minimum of 5 years of experience in echocardiography. They received training for this project and performed their annotations using a validated custom interface in Siemens syngo.via (version VA20). They reviewed the available loops of the relevant view(s) for each measurement ( Supplementary Table 1 ) and chose the most adequate loop to perform the measurement. They did not receive additional clinical information about the cases. For LV ejection fraction, they visually estimated the measurement; other measurements were calculated based on the American Society of Echocardiography guidelines. 21 RV fractional area change was calculated from manually traced end-diastolic and end-systolic RV areas. The component measurements for RV fractional area change were measured 3 times on different beats and the average fractional area change was calculated; each cardiologist obtained the other measurements once per case. View this table: View inline View popup Download powerpoint Supplementary Table 1: Views used to obtain each measurement. Abbreviations: A4, apical four chamber; PLAX, parasternal long axis. Separately they indicated when a measurement could not be obtained and should therefore be excluded; if at least two of the cardiologists indicated that the measurement could not be obtained then the measurement was excluded for that case; if only one of them indicated that the measurement could be obtained then that cardiologist was asked to review the case and make an actual best determination of the measurement. Device development and inference The device evaluated in this study was trained using labeled echocardiography data acquired at Baruch Padeh Medical Center and Sheba Medical Center between 2009 and 2020. The development set included 145,217 adult patient examinations, which were split into a training set (95,167 cases, 65.5%), validation set (29,090 cases, 20.0%), and a hold-out test set (20,960 cases, 14.4%; which was separate to the cases evaluated in the current study). This development set represented a mix of inpatient and outpatient studies on a range of echocardiography machines. Each examination comprised 20-80 ultrasound images and loops, capturing various cardiac views according to echocardiography protocols. In brief, the device architecture consists of 2 types of models. The first is a model which has a combination of convolutional neural networks (CNNs) and transformers. Feature vectors are extracted using designated trained EfficientNetV2 22 like architecture and fed into transformer blocks with connected CNNs to highlight relevant features across time. The final output vectors then underwent max-pooling and were processed by a regressor head. The second type of model is an instance segmentation CNN model trained for segmentation of relevant anatomical parts within an ultrasound image. The device analyzed parasternal, apical and subcostal views; for measurements requiring multiple views, frames from both views were concatenated. The presence of multiple loops of a view during inference resulted in averaged predictions in an effort to increase the robustness of measurements. The device performed inference independently of the ground truth cardiologists and did not receive additional clinical information about the cases. The device provided a single value for each of the 7 measurements. It could also provide an output of “NULL” that reflected it could not interpret that measurement for the case. The reasons included insufficient quality of images (as the device sees them) or inability to calculate the measurement due to the absence of key points in the image used by the device. Statistical Analysis The statistical analysis was conducted using software R version 4.2.0 (R Foundation for Statistical Computing, Vienna, Austria, 2019). The predefined primary and secondary endpoints were the diagnostic accuracy of the device as determined by the root mean square error (RMSE) when compared with the average of the three ground truth cardiologists. LV ejection fraction and IVC maximal diameter were considered primary endpoints given these metrics often impact patient care more directly; the other measurements were considered secondary endpoints. The RMSE was calculated as the square root of the mean of the squared differences between the device measurements and the average cardiologist measurements, providing a measure of overall deviation. 23 The inter-cardiologist RMSE was also calculated for reference, which was the average of the pairwise RMSEs between each combination of cardiologists. The passing criteria for the primary endpoints were RMSEs of less than 7.0% for LV ejection fraction and 0.3cm for IVC maximal diameter, which were to be met by the upper bounds of the 95% confidence intervals (CIs). The passing criteria for the secondary endpoints were RMSEs less than 20% of the average of the absolute measurement from the three cardiologists, which similarly was to be met by the upper bounds of the 95% CIs. To further evaluate the interchangeability and accuracy of the device measurements compared to cardiologist measurements, additional predefined secondary assessments were calculated including the individual equivalence coefficient (IEC; to compare variability of the relative differences between the device and the cardiologists with the variability of the relative differences between cardiologists), 24 mean absolute error (MAE; to assess the average magnitude of errors), 23 limits of agreement (LOA) using Bland-Altman analysis (to evaluate the presence of systematic errors between the device and cardiologists), 25 and intraclass correlation coefficient (ICC; to measure the consistency and reproducibility of the measurements). 26 MAE was computed as the mean of the absolute differences between the device and cardiologist measurements, providing a metric of average error magnitude without being influenced by the direction of errors. The Bland-Altman analysis involved plotting the mean of each device-cardiologist measurement pair against their difference, allowing for visual assessment of bias and agreement. The 95% LOAs were defined as the mean difference ± 1.96 × standard deviation of the differences, indicating the range within which most differences are expected to lie. For the IEC and ICC calculations, the measurements from each of the cardiologists were considered. For MAE and LOA, the device was compared with the average measurement from the three cardiologists. Subgroup analyses were calculated for demographic (sex, age, race and ethnicity, BMI), disease (LV ejection fraction) and image (ultrasound probe manufacturer, image quality score, role of person acquiring scan) parameters. The subgroups were predefined except for race and role of person acquiring scan (which were added as post-hoc analyses), and BMI and LV ejection fraction (for which the subgroups were updated to have more subgroups following regulatory feedback). Each ultrasound probe manufacturer only had a single model so both the manufacturers and models are specified in the results. All metrics were accompanied by bootstrapped 95% CIs with 2,000 resamples. An emphasis was placed on the interpretation of CIs to infer the reliability of calculations. No adjustments for multiple comparisons were made. Sample size calculations were performed a priori for the primary endpoints and powered based on preliminary device results. Results Participant characteristics The overall cohort included cases from 200 patients ( Figure 1 ). The mean age (± standard deviation) was 65.4 (± 16.0) years and 74 (37.0%) were female patients ( Table 1 ). A case was excluded from the analysis for a given measurement if the cardiologists or the device could not obtain the measurement ( Supplementary Table 2 ). View this table: View inline View popup Download powerpoint Supplementary Table 2: Excluded cases. The cases were excluded when the cardiologists could not obtain the measurement or the device outputted “NULL” (see Methods for explanation). If a row exceeds a total of 200 cases, it indicates that a case was both excluded by the cardiologists and received a “NULL” device output. Abbreviations: IVC, inferior vena cava; LA, left atrial; LV, left ventricular; RA, right atrial; RV, right ventricular. Download figure Open in new tab Figure 1: Cohort selection. Abbreviations: IVC, inferior vena cava; LA, left atrial; LV, left ventricular; RA, right atrial; RV, right ventricular. View this table: View inline View popup Table 1: Baseline demographic and technical information. Abbreviations: BMI, body mass index; LV, left ventricular; SD, standard deviation. Primary and secondary endpoint analysis The RMSE was calculated between the device and the three cardiologists ( Table 2 ). Both primary endpoint passing criteria were met, with the RMSEs for LV ejection fraction and IVC maximal diameter having a 95% CI upper bound less than 7.0% and 0.3cm respectively. Four out of the five secondary endpoint passing criteria were met, with the RMSEs achieving a 95% CI upper bound less than 20% of the mean cardiologist measurement. RV fractional area change was the one secondary endpoint that did not achieve its passing criterion, although the RMSE was still better between the device and cardiologists than between the cardiologists. The other measurements, except for LV end diastolic diameter, also had better RMSEs between the device and the cardiologists than between the cardiologists. View this table: View inline View popup Download powerpoint Table 2: Primary and secondary endpoint results. The RMSE between the device and the average of the three cardiologists for the seven measurements (LV ejection fraction and IVC maximal diameter were primary endpoints; the others were secondary endpoints). The inter-cardiologist RMSE is provided for reference. This table also incorporates the number of cases and mean measurements for each measurement. Abbreviations: CI, confidence interval; IVC, inferior vena cava; LA, left atrial; LV, left ventricular; N, number of cases; RA, right atrial; RMSE, root mean square error; RV, right ventricular; SD, standard deviation. Secondary assessment analysis The IEC, MAE, LOA and ICC were calculated between the device and the three cardiologists ( Table 3 ). The IECs were mostly less than 0, showing better variability of the relative differences between the device and cardiologists compared with the variability of the relative differences between the cardiologists. The MAEs, which provided an additional statistical method to RMSEs for measuring the error of the device, were less than the RMSEs. The LOAs, which can show the presence of systematic bias in a device, had mean differences that were unlikely to be clinically meaningful. The ICCs were mostly above 0.75, reflecting the strength of the correlation between the measurements from the device and the measurements from the cardiologists. View this table: View inline View popup Download powerpoint Table 3: Secondary assessment results. The IEC, MAE, LOA and ICC between the device and the three cardiologists for the seven measurements. The LOA is expressed as the mean difference ± 1.96 × standard deviation of the differences. This table also incorporates the number of cases for each measurement. Abbreviations: CI, confidence interval; ICC, intraclass coefficient; IEC, individual equivalence coefficient; IVC, inferior vena cava; LA, left atrial; LOA, limits of agreement; LV, left ventricular; MAE, mean absolute error; N, number of cases; RA, right atrial; RV, right ventricular. Subgroup analyses The subgroup analyses evaluated the RMSEs across demographic, disease and image parameter subgroups ( Table 4 ). The vast majority of RMSEs were similar across subgroups as suggested by overlapping ranges of 95% CIs: there were overlapping ranges for all measurements in the subgroups for age, sex, race and ethnicity, image quality score and role of person acquiring scan. The measurements with non-overlapping ranges included RA area for BMI subgroups (the <25 subgroup had lower RMSE than the [30,35) subgroup), LV ejection fraction for LV ejection fraction subgroups (the <30% subgroup had lower RMSE than the ≥30% to <53% and ≥53% subgroups), LV end diastolic diameter for LV ejection fraction subgroups (the <30% subgroup had lower RMSE than the ≥53% subgroup), and LV ejection fraction for manufacturer subgroups (the GE Healthcare subgroup had lower RMSE than the Philips or Wisonic subgroups). In addition, the BMI unknown subgroup had only 2 cases and often had non-overlapping 95% CIs. View this table: View inline View popup Table 4: Subgroup analysis results. The RMSE between the device and the average of the three cardiologists for the seven measurements for demographic, disease and image parameter subgroups. Abbreviations: BMI, body mass index; CI, confidence interval; IVC, inferior vena cava; LA, left atrial; LV, left ventricular; N, number of cases; RA, right atrial; RMSE, root mean square error; RV, right ventricular. Discussion This study assessed the performance of an AI-enabled device at interpreting seven key cardiac measurements on images acquired using POCUS devices. It achieved both of its primary endpoint passing criteria and all but one of its secondary endpoint passing criteria, which were based on the RMSE between the device and the average of three cardiologists for the measurements. It also demonstrated better performance compared to the inter-cardiologist RMSE for all but one of the measurements. Importantly, this study was based mostly on images acquired from inexpensive, predominantly handheld devices that could perform a short POCUS scan at the bedside unlike many other studies that have used images from comprehensive echocardiography examinations obtained in echocardiography labs. 14 – 16 Overall, this performance suggests that the device provides accurate and reliable measurements. The improved RMSEs compared with cardiologists may reflect the ability of the device to average across multiple beats and loops, with the cardiologists having been asked to quantify most measurements only once. This situation, however, mirrors the real-world in that cardiologists and other clinicians are often constrained by the need to manually obtain measurements while automated measurements can leverage a greater ability to average. A similar improvement in interobserver variability has previously been shown for conventional echocardiograms. 17 The ability to average may also reflect the ability of the device to learn from the measurement styles of multiple cardiologists; in the context of electroencephalogram interpretation, this use of AI to average has been described as simulating a committee of independent experts. 27 The one measurement where the device did not achieve the passing criteria was for RV fractional area change. For this measurement, however, the RMSE between the device and cardiologists was still better than between the cardiologists, reflecting the large variability that can be seen with this measurement and reinforcing the benefit that the device can provide through averaging. LV end diastolic diameter was the one measurement where the RMSE was greater between the device and cardiologists compared to between the cardiologists, although the RMSE between the device and cardiologists still achieved the secondary endpoint passing criterion. POCUS provides clinicians with the ability to obtain critical diagnostic information at the bedside. For example, the knowledge of a patient’s LV ejection fraction or IVC maximal diameter can guide next management steps in a critical ill patient including whether to administer fluids. 28 , 29 The Lumify (Philips) ultrasound probe, which was one of four assessed here, can connect directly to smart phones and tablets, 30 while the Kosmos (EchoNous) ultrasound probe can connect directly to tablets. 31 The use of the device with these ultrasound probes may be particularly beneficial in resource limited settings. A key challenge with POCUS acquisition is that the image acquisition is often not as standardized as a conventional echocardiogram performed in an echocardiography laboratory. This study assessed how the device handles this challenge in multiple ways. Firstly, the device has quality control checks to ensure sufficient image quality and the presence of key points within images; the device did not provide a quantification for between 0.5% (for LV ejection fraction) and 13.5% (for LV end diastolic diameter) of cases for each measurement. Secondly, an image quality score was provided for each case; the RMSEs for image quality scores of ≤4 and >4 were similar for all measurements. Thirdly, the RMSEs were also similar for all measurements when the person acquiring the scan was a physician or a sonographer. Limitations A key limitation of this study is that it was performed outside the clinical environment and therefore did not assess the impact of the device use on patient care and outcomes. Further to this point, the study did not assess the ability to generate the measurements in real-time with POCUS acquisition, where the device may provide the greatest clinical utility. Nevertheless, the images in this study were prospectively acquired at the bedside by mostly handheld devices and not as part of formal echocardiography studies obtained in an echocardiography lab; they therefore represented the real-world scenario where this device would be used. This study was also limited to four sites and the device will inevitably encounter new scenarios with clinical use, including further ultrasound probe manufacturers and models. For these reasons, the ongoing assessment of the device will be critical as it is deployed in the clinical environment. Conclusion This AI-enabled device accurately calculated seven cardiac measurements on POCUS especially when benchmarked against inter-cardiologist variability. It could facilitate better clinical decision-making including in the acute care environment. Data Availability All data produced in the present study are available upon reasonable request to the authors. Funding / Role of Funder SFM, BCB, TS, MAH, ALM, INC, EL, ASS, VT, SMH, JRM, JW and JH were employees of Mass General Brigham and/or affiliated hospitals at the time of the study, which had received institutional funding from the sponsor (AISAP LTD) to conduct this project. PM and QZ were employees of Thomas Jefferson University and Inova Health System respectively, which had received institutional funding from the sponsor to conduct this project. The sponsor was involved in the design and conduct of the study; preparation, review, or approval of the manuscript; and decision to submit the manuscript for publication. The sponsor was involved in the collection of POCUS cases, which were transferred to Mass General Brigham; the sponsor was otherwise not involved in the collection, management, analysis, and interpretation of the data. Conflicts of Interest SMH received fees paid to her institution for core lab services from Cytokinetics and BMS; she is on the advisory board for Cytokinetics. Data Accountability SFM had full access to all the data in the study and takes responsibility for the integrity of the data and the accuracy of the data analysis. Acknowledgements The authors thank the broader team members at the hospitals that acquired cases (Baruch Padeh Medical Center, Inova Health, Sheba Medical Center and Thomas Jefferson University Hospital), at Mass General Brigham AI and at AISAP LTD. Footnotes This revision corrects an error in describing the training dataset. References 1. ↵ Bernstein E , Wang TY . Point-of-Care Ultrasonography: Visually Satisfying Medicine or Evidence-Based Medicine? JAMA Intern Med . Dec 1 2021 ; 181 ( 12 ): 1558 – 1559 . doi: 10.1001/jamainternmed.2021.5831 OpenUrl CrossRef 2. ↵ Moore CL , Copel JA . Point-of-care ultrasonography . N Engl J Med . Feb 24 2011 ;364 ( 8 ): 749 – 57 . doi: 10.1056/NEJMra0909487 OpenUrl CrossRef PubMed Web of Science 3. ↵ Narula J , Chandrashekhar Y , Braunwald E . Time to Add a Fifth Pillar to Bedside Physical Examination: Inspection, Palpation, Percussion , Auscultation, and Insonation. JAMA Cardiol . Apr 1 2018 ; 3 ( 4 ): 346 – 350 . doi: 10.1001/jamacardio.2018.0001 OpenUrl CrossRef PubMed 4. ↵ Soni NJ , Schnobrich D , Mathews BK , et al. Point-of-Care Ultrasound for Hospitalists: A Position Statement of the Society of Hospital Medicine . J Hosp Med . Jan 2 2019 ; 14 : E1 – E6 . doi: 10.12788/jhm.3079 OpenUrl CrossRef 5. Durston W , Carl ML , Guerra W . Patient satisfaction and diagnostic accuracy with ultrasound by emergency physicians . Am J Emerg Med . Nov 1999 ; 17 ( 7 ): 642 – 6 . doi: 10.1016/s0735-6757(99)90150-x OpenUrl CrossRef PubMed Web of Science 6. ↵ Ultrasound Guidelines: Emergency, Point-of-Care , and Clinical Ultrasound Guidelines in Medicine . Ann Emerg Med . Sep 2023 ; 82 ( 3 ): e115 – e155 . doi: 10.1016/j.annemergmed.2023.06.005 OpenUrl CrossRef PubMed 7. ↵ Le MT , Voigt L , Nathanson R , et al. Comparison of four handheld point-of-care ultrasound devices by expert users . Ultrasound J . Jul 7 2022 ; 14 ( 1 ): 27 . doi: 10.1186/s13089-022-00274-6 OpenUrl CrossRef PubMed 8. Rykkje A , Carlsen JF , Nielsen MB . Hand-Held Ultrasound Devices Compared with High-End Ultrasound Systems: A Systematic Review . Diagnostics (Basel) . Jun 15 2019 ; 9 (2) doi: 10.3390/diagnostics9020061 OpenUrl CrossRef 9. Stringer JSA , Pokaprakarn T , Prieto JC , et al. Diagnostic Accuracy of an Integrated AI Tool to Estimate Gestational Age From Blind Ultrasound Sweeps . JAMA . Aug 27 2024 ; 332 ( 8 ): 649 – 657 . doi: 10.1001/jama.2024.10770 OpenUrl CrossRef PubMed 10. ↵ Thavanathan RS , Woo MY , Hall G . The future is in your hands - Handheld ultrasound in the emergency department . CJEM . Nov 2020 ; 22 ( 6 ): 742 – 744 . doi: 10.1017/cem.2020.449 OpenUrl CrossRef PubMed 11. ↵ Kirkpatrick JN , Grimm R , Johri AM , et al. Recommendations for Echocardiography Laboratories Participating in Cardiac Point of Care Cardiac Ultrasound (POCUS) and Critical Care Echocardiography Training: Report from the American Society of Echocardiography . J Am Soc Echocardiogr . Apr 2020 ; 33 ( 4 ): 409 – 422 e4. doi: 10.1016/j.echo.2020.01.008 OpenUrl CrossRef PubMed 12. ↵ Rajpurkar P , Lungren MP . The Current and Future State of AI Interpretation of Medical Images . N Engl J Med. May 25 2023 ; 388 ( 21 ): 1981 – 1990 . doi: 10.1056/NEJMra2301725 OpenUrl CrossRef PubMed 13. ↵ Thrall JH , Li X , Li Q , et al. Artificial Intelligence and Machine Learning in Radiology: Opportunities, Challenges, Pitfalls, and Criteria for Success . J Am Coll Radiol . Mar 2018 ; 15 ( 3 Pt B ): 504 - 508 . doi: 10.1016/j.jacr.2017.12.026 OpenUrl CrossRef PubMed 14. ↵ Tromp J , Bauer D , Claggett BL , et al. A formal validation of a deep learning-based automated workflow for the interpretation of the echocardiogram . Nat Commun . Nov 9 2022 ; 13 ( 1 ): 6776 . doi: 10.1038/s41467-022-34245-1 OpenUrl CrossRef 15. Tromp J , Seekings PJ , Hung CL , et al. Automated interpretation of systolic and diastolic function on the echocardiogram: a multicohort study . Lancet Digit Health . Jan 2022 ; 4 ( 1 ): e46 – e54 . doi: 10.1016/S2589-7500(21)00235-1 OpenUrl CrossRef 16. ↵ Zhang J , Gajjala S , Agrawal P , et al. Fully Automated Echocardiogram Interpretation in Clinical Practice . Circulation. Oct 16 2018 ; 138 ( 16 ): 1623 – 1635 . doi: 10.1161/CIRCULATIONAHA.118.034338 OpenUrl CrossRef PubMed 17. ↵ Mor-Avi V , Blitz A , Schreckenberg M , et al. Deep learning assisted measurement of echocardiographic left heart parameters: improvement in interobserver variability and workflow efficiency . Int J Cardiovasc Imaging . Dec 2023 ; 39 ( 12 ): 2507 – 2516 . doi: 10.1007/s10554-023-02960-5 OpenUrl CrossRef PubMed 18. ↵ Hirata Y , Nomura Y , Saijo Y , Sata M , Kusunose K . Reducing echocardiographic examination time through routine use of fully automated software: a comparative study of measurement and report creation time . J Echocardiogr . Sep 2024 ; 22 ( 3 ): 162 – 170 . doi: 10.1007/s12574-023-00636-6 OpenUrl CrossRef PubMed 19. ↵ Lee L , DeCara JM . Point-of-Care Ultrasound . Curr Cardiol Rep . Sep 17 2020 ; 22 ( 11 ): 149 . doi: 10.1007/s11886-020-01394-y OpenUrl CrossRef PubMed 20. Bossuyt PM , Reitsma JB , Bruns DE , et al. STARD 2015: An Updated List of Essential Items for Reporting Diagnostic Accuracy Studies . Radiology . Dec 2015 ; 277 ( 3 ): 826 – 32 . doi: 10.1148/radiol.2015151516 OpenUrl CrossRef PubMed 21. ↵ Mitchell C , Rahko PS , Blauwet LA , et al. Guidelines for Performing a Comprehensive Transthoracic Echocardiographic Examination in Adults: Recommendations from the American Society of Echocardiography . J Am Soc Echocardiogr . Jan 2019 ; 32 ( 1 ): 1 – 64 . doi: 10.1016/j.echo.2018.06.004 OpenUrl CrossRef PubMed 22. ↵ Tan M , Le Q . Efficientnetv2: Smaller models and faster training . PMLR ; 2021 : 10096 – 10106 . 23. ↵ Chai T , Draxler RR . Root mean square error (RMSE) or mean absolute error (MAE)? – Arguments against avoiding RMSE in the literature. Geosci Model Dev . 2014 ; 7 ( 3 ): 1247 – 1250 . doi: 10.5194/gmd-7-1247-2014 OpenUrl CrossRef 24. ↵ Barnhart HX , Kosinski AS , Haber MJ . Assessing individual agreement . J Biopharm Stat . 2007 ; 17 ( 4 ): 697 – 719 . doi: 10.1080/10543400701329489 OpenUrl CrossRef PubMed Web of Science 25. ↵ Bland JM , Altman DG . Statistical methods for assessing agreement between two methods of clinical measurement . Lancet . Feb 8 1986 ; 1 ( 8476 ): 307 - 10 . OpenUrl CrossRef PubMed Web of Science 26. ↵ Bartko JJ . The Intraclass Correlation Coefficient as a Measure of Reliability . Psychological Reports . 1966 ; 19 ( 1 ): 3 – 11 . doi: 10.2466/pr0.1966.19.1.3 OpenUrl CrossRef PubMed Web of Science 27. ↵ Jing J , Sun H , Kim JA , et al. Development of Expert-Level Automated Detection of Epileptiform Discharges During Electroencephalogram Interpretation . JAMA Neurol . Jan 1 2020 ; 77 ( 1 ): 103 – 108 . doi: 10.1001/jamaneurol.2019.3485 OpenUrl CrossRef PubMed 28. ↵ Furtado S , Reis L . Inferior vena cava evaluation in fluid therapy decision making in intensive care: practical implications . Rev Bras Ter Intensiva . Jun 27 2019 ; 31 ( 2 ): 240 – 247 . Avaliacao da veia cava inferior na decisao de fluidoterapia em cuidados intensivos: implicacoes praticas. doi: 10.5935/0103-507X.20190039 OpenUrl CrossRef 29. ↵ Miller A , Mandeville J . Predicting and measuring fluid responsiveness with echocardiography . Echo Res Pract . Jun 2016 ; 3 ( 2 ): G1 – G12 . doi: 10.1530/ERP-16-0008 OpenUrl Abstract / FREE Full Text 30. ↵ Philips - Lumify . Accessed February 3, 2025 . https://www.usa.philips.com/healthcare/product/HC795005/lumify 31. ↵ Kosmos AI-driven POCUS device . Accessed February 3, 2025 . https://echonous.com/product/kosmos-ultraportable-ultrasound/ View the discussion thread. Back to top Previous Next Posted October 02, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Cardiac Measurement Calculation on Point-of-Care Ultrasonography with Artificial Intelligence Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Cardiac Measurement Calculation on Point-of-Care Ultrasonography with Artificial Intelligence Sarah F Mercaldo , Bernardo C Bizzo , Tsion Sadore , Madeleine A Halle , Ashley L MacDonald , Isabella Newbury-Chaet , Eric L’Italien , Alex S Schultz , Victor Tam , Sheila M Hegde , Judy R Mangion , Praveen Mehrotra , Qiong Zhao , Justina Wu , James M Hillis medRxiv 2025.06.27.25330438; doi: https://doi.org/10.1101/2025.06.27.25330438 Share This Article: Copy Citation Tools Cardiac Measurement Calculation on Point-of-Care Ultrasonography with Artificial Intelligence Sarah F Mercaldo , Bernardo C Bizzo , Tsion Sadore , Madeleine A Halle , Ashley L MacDonald , Isabella Newbury-Chaet , Eric L’Italien , Alex S Schultz , Victor Tam , Sheila M Hegde , Judy R Mangion , Praveen Mehrotra , Qiong Zhao , Justina Wu , James M Hillis medRxiv 2025.06.27.25330438; doi: https://doi.org/10.1101/2025.06.27.25330438 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Cardiovascular Medicine Subject Areas All Articles Addiction Medicine (572) Allergy and Immunology (864) Anesthesia (302) Cardiovascular Medicine (4448) Dentistry and Oral Medicine (444) Dermatology (383) Emergency Medicine (609) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1515) Epidemiology (15240) Forensic Medicine (30) Gastroenterology (1130) Genetic and Genomic Medicine (6613) Geriatric Medicine (669) Health Economics (1000) Health Informatics (4550) Health Policy (1372) Health Systems and Quality Improvement (1613) Hematology (543) HIV/AIDS (1267) Infectious Diseases (except HIV/AIDS) (15927) Intensive Care and Critical Care Medicine (1105) Medical Education (624) Medical Ethics (147) Nephrology (668) Neurology (6620) Nursing (346) Nutrition (999) Obstetrics and Gynecology (1147) Occupational and Environmental Health (957) Oncology (3341) Ophthalmology (977) Orthopedics (369) Otolaryngology (421) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1695) Pharmacology and Therapeutics (693) Primary Care Research (714) Psychiatry and Clinical Psychology (5459) Public and Global Health (9247) Radiology and Imaging (2205) Rehabilitation Medicine and Physical Therapy (1371) Respiratory Medicine (1197) Rheumatology (597) Sexual and Reproductive Health (715) Sports Medicine (530) Surgery (714) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a027cf85c90e4193',t:'MTc3OTkxNDE3NA=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();