Evaluation of the Impact of a 90-day Mortality Prediction Model Integrated in the EHR for Advanced Care Planning*

Evaluation of the Impact of a 90-day Mortality Prediction Model Integrated in the EHR for Advanced Care Planning* | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Evaluation of the Impact of a 90-day Mortality Prediction Model Integrated in the EHR for Advanced Care Planning * View ORCID Profile Lorenzo A. Rossi , Laura Roberts , Marinel Olivares , Amis Christian , Stefanie Mooney , Denise Morse , Finly Zachariah doi: https://doi.org/10.1101/2024.10.08.24315123 Lorenzo A. Rossi 1 City of Hope National Medical Center , Duarte, CA, USA PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Lorenzo A. Rossi For correspondence: lorenzo.rossi{at}gmail.com Laura Roberts 1 City of Hope National Medical Center , Duarte, CA, USA MS Find this author on Google Scholar Find this author on PubMed Search for this author on this site Marinel Olivares 1 City of Hope National Medical Center , Duarte, CA, USA MSW Find this author on Google Scholar Find this author on PubMed Search for this author on this site Amis Christian 1 City of Hope National Medical Center , Duarte, CA, USA MS Find this author on Google Scholar Find this author on PubMed Search for this author on this site Stefanie Mooney 1 City of Hope National Medical Center , Duarte, CA, USA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Denise Morse 1 City of Hope National Medical Center , Duarte, CA, USA MBA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Finly Zachariah 1 City of Hope National Medical Center , Duarte, CA, USA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Medical centers have begun to integrate AI models for mortality prediction in their electronic health records (EHR) systems to support advance care planning. 1 There is evidence that advance care planning can reduce suffering and unwanted, costly treatment near the end of life. 2 To better target patients at a risk of death with advance directive (AD) capture and goals of care (GOC) conversations at our comprehensive cancer center, we developed and integrated a 90-day mortality prediction model in the EHR. The model has been live since February, 2021. Performance in production has matched very closely pre-production pilots, achieving an 85% AUROC (AI Showcase Stage I). We implemented 4 clinical decision support (CDS) tools driven by the model to promote goal concordant care in inpatient and outpatient encounters (AI Showcase Stage II). 3 In this document, we analyze the impact of one of the aforementioned CDS applications, implemented as an inter-ruptive alert for the clinical social workers (CSWs) to facilitate advance directive (AD) completion in the inpatient setting. We also present preliminary results on volumes of GOC conversations prompted by one of the model-driven alerts. Our analysis is based on 12 months of data, collected between March 1, 2021 and February 28, 2022. The challenges for our assessment included inconsistent EHR documentation practices, coexistence with (traditional) non-AI driven processes tasked for similar goals and many external factors that can affect target outcomes. We believe such challenges to be rather common across healthcare organizations integrating AI in their workflows. Methods Our analysis focused patients admitted without an AD. The AD interruptive alert went live on 2.16.2021, targeting adult inpatients without an AD in place (roughly 2/3 of the admissions). CSWs have historically relied upon a mix of criteria to prioritize patients to approach for AD completion, ranging from seriousness of the condition to lack of clarity in healthcare agents, because of insufficient capacity to approach every admitted patient. Such criteria remained in place even after alert go-live. The process of AD completion initiated in the hospital is often completed after discharge e . g . to allow patients and families more time to process the documents. On a filed AD there is no indication if a CSW facilitated the process. The aforementioned factors lead to uncertainties on the number of AD completions that (1) can be associated with the hospital stay and, (2) can be credited to the model driven alert. Based on feedback from domain experts, we assumed all the ADs completed between admission and up to 14 days after patient discharge to be associated with the hospital stay (either because of the model or other processes). The assumption was further validated after inspecting the distribution of the counts of ADs completed with post-discharge tolerance windows ranging from 0 to 28 days. As implied, some of the ADs filed after the firing of the alert may be actually due to pre-existing (non-AI based) processes. To increase our confidence on the actual use of the alert, we compared rates of the AD completions across acknowledgement reasons selected by the CSWs when prompted with the alert. Since an alert may fire more than once for a patient encounter, we focused on the last acknowledgment reason given ( Fig. 1a ). Then for simplicity, we credited every AD filed after an alert to the model. Download figure Open in new tab We computed the monthly counts of completed ADs and ratios of 90-day mortality rates for patients who completed AD to the 90-day mortality rates for all the patients without AD (both evaluated at discharge) ( Fig. 1b ). The latter is to measure performance of the AD process in targeting patients at high risk of death: the metric would be approximately equal to one if the patients solicited for AD completions were chose randomly. Results We observed the highest rate of AD completion per alert when the last acknowledgment reason was “I will follow up with patient/family” (27%, p = .008) and a relatively lower AD completion rate when the last acknowledgement reason was “I’m not the assigned CSW” (15%, p = .048). We computed counts of patients with completed ADs, for which we evaluated metrics including length of stay (LOS), in-hospital mortality rates for patients with AD completed and the aforementioned ratio of 90-day mortality rate at discharge ( Tab. 1 ). View this table: View inline View popup Download powerpoint Table 1. Advance Directives completed after firing of the alert vs. ones completed without alert involved We also examined a GOC interruptive alert. The alert was used to order 169 supportive care consults (with an associated 73% Precision) for GOC in its first year and resulted in at least 67 documented GOC discussions. Discussion The results in Tab. 1 and Fig. 1b and additional analyses not included here suggest that up to half of the AD completions attributed to the alert could have been facilitated via non-AI driven approaches. Our study showed that the model has been used by the clinical social workers as an additional tool to facilitate advance directive completion for inpatients at higher risk of death, facilitating up to the 25.8% of the ADs in a year, The tool showed a significant superior ability to target patients nearing end-of-life thanks to the accuracy of the underlying model. We have been making adjustments (i.e. increasing decision threshold of the AD alert) to increase use and impact of the alert. A model-driven GOC alert also resulted in documentation of GOC for a subset of patients that may otherwise not have been prioritized for such discussions. Data Availability The data associated with the current study is not publicly available as the data are not legally certified as being de-identified. Footnotes * Podium Abstract presented at the 2022 AMIA AI Evaluation Showcase, Stage III, Annual Symposium, Washington DC, November 2023. Format adjusted per medRxiv requirements. References 1. ↵ Erwin Wang , Vincent J Major , Nicole Adler , Kevin Hauck , Jonathan Austrian , Yindalon Aphinyanaphongs , and Leora I Horwitz . Supporting acute advance care planning with precise, timely mortality risk predictions . NEJM Catalyst , 2 ( 3 ), 2021 . 2. ↵ Vipul Bhatia , Roberta Geidner , Kamna Mirchandani , Yue Huang , and Haider J Warraich . Systemwide advance care planning during the COVID-19 pandemic: the impact on patient outcomes and cost . NEJM Catalyst , 2 ( 9 ), 2021 . 3. ↵ Laura Roberts , Lorenzo A. Rossi , and Finly Zachariah . Design, EHR integration and evaluation of clinical decision support workflows driven by a mortality prediction model to promote goal concordant care . In Clinical Informatics Conf. AMIA , 2022 . View the discussion thread. Back to top Previous Next Posted October 10, 2024. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Evaluation of the Impact of a 90-day Mortality Prediction Model Integrated in the EHR for Advanced Care Planning* Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Evaluation of the Impact of a 90-day Mortality Prediction Model Integrated in the EHR for Advanced Care Planning * Lorenzo A. Rossi , Laura Roberts , Marinel Olivares , Amis Christian , Stefanie Mooney , Denise Morse , Finly Zachariah medRxiv 2024.10.08.24315123; doi: https://doi.org/10.1101/2024.10.08.24315123 Share This Article: Copy Citation Tools Evaluation of the Impact of a 90-day Mortality Prediction Model Integrated in the EHR for Advanced Care Planning * Lorenzo A. Rossi , Laura Roberts , Marinel Olivares , Amis Christian , Stefanie Mooney , Denise Morse , Finly Zachariah medRxiv 2024.10.08.24315123; doi: https://doi.org/10.1101/2024.10.08.24315123 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (573) Allergy and Immunology (865) Anesthesia (304) Cardiovascular Medicine (4457) Dentistry and Oral Medicine (445) Dermatology (383) Emergency Medicine (610) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1517) Epidemiology (15244) Forensic Medicine (30) Gastroenterology (1132) Genetic and Genomic Medicine (6620) Geriatric Medicine (669) Health Economics (1002) Health Informatics (4557) Health Policy (1372) Health Systems and Quality Improvement (1615) Hematology (543) HIV/AIDS (1272) Infectious Diseases (except HIV/AIDS) (15936) Intensive Care and Critical Care Medicine (1106) Medical Education (624) Medical Ethics (147) Nephrology (670) Neurology (6634) Nursing (346) Nutrition (999) Obstetrics and Gynecology (1148) Occupational and Environmental Health (957) Oncology (3348) Ophthalmology (980) Orthopedics (369) Otolaryngology (421) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1696) Pharmacology and Therapeutics (693) Primary Care Research (714) Psychiatry and Clinical Psychology (5463) Public and Global Health (9257) Radiology and Imaging (2210) Rehabilitation Medicine and Physical Therapy (1371) Respiratory Medicine (1198) Rheumatology (598) Sexual and Reproductive Health (716) Sports Medicine (532) Surgery (714) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a0365547888df047',t:'MTc4MDA2NjQ1Mw=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();