Suicide Death Predictive Models using Electronic Health Record Data

Suicide Death Predictive Models using Electronic Health Record Data | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Suicide Death Predictive Models using Electronic Health Record Data Shweta Srikanth , Lina M. Montoya , Matthew M. Turnure , Brian W. Pence , Naoko Fulcher , Bradley N. Gaynes , David B. Goldston , Timothy S. Carey , View ORCID Profile Shabbar I. Ranapurwala doi: https://doi.org/10.1101/2024.09.26.24314402 Shweta Srikanth 1 Department of Biostatistics, Gillings School of Global Public Health, University of North Carolina , Chapel Hill, NC Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: shwetasrikanth6{at}gmail.com Lina M. Montoya 1 Department of Biostatistics, Gillings School of Global Public Health, University of North Carolina , Chapel Hill, NC Find this author on Google Scholar Find this author on PubMed Search for this author on this site Matthew M. Turnure 2 Department of Epidemiology, Gillings School of Global Public Health, University of North Carolina , Chapel Hill, NC 3 Injury Prevention Research Center, University of North Carolina , Chapel Hill, NC Find this author on Google Scholar Find this author on PubMed Search for this author on this site Brian W. Pence 2 Department of Epidemiology, Gillings School of Global Public Health, University of North Carolina , Chapel Hill, NC 3 Injury Prevention Research Center, University of North Carolina , Chapel Hill, NC Find this author on Google Scholar Find this author on PubMed Search for this author on this site Naoko Fulcher 3 Injury Prevention Research Center, University of North Carolina , Chapel Hill, NC Find this author on Google Scholar Find this author on PubMed Search for this author on this site Bradley N. Gaynes 2 Department of Epidemiology, Gillings School of Global Public Health, University of North Carolina , Chapel Hill, NC 4 Department of Psychiatry, University of North Carolina at Chapel Hill School of Medicine , Chapel Hill, NC Find this author on Google Scholar Find this author on PubMed Search for this author on this site David B. Goldston 5 Department of Psychiatry and Behavioral Sciences, Duke University School of Medicine , Durham, NC Find this author on Google Scholar Find this author on PubMed Search for this author on this site Timothy S. Carey 6 Department of Internal Medicine, University of North Carolina at Chapel Hill , Chapel Hill, NC Find this author on Google Scholar Find this author on PubMed Search for this author on this site Shabbar I. Ranapurwala 2 Department of Epidemiology, Gillings School of Global Public Health, University of North Carolina , Chapel Hill, NC 3 Injury Prevention Research Center, University of North Carolina , Chapel Hill, NC Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Shabbar I. Ranapurwala Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract In the realm of medical research, particularly in the study of suicide risk assessment, the integration of machine learning techniques with traditional statistics methods has become increasingly prevalent. This paper used data from the UNC EHR system from 2006 to 2020 to build models to predict suicide-related death. The dataset, with 1021 cases and 10185 controls consisted of demographic variables and short-term informa-tion, on the subject’s prior diagnosis and healthcare utilization. We examined the efficacy of the super learner ensemble method in predicting suicide-related death lever-aging its capability to combine multiple predictive algorithms without the necessity of pre-selecting a single model. The study compared the performance of the super learner against five base models, demonstrating its superiority in terms of cross-validated neg-ative log-likelihood scores. The super learner improved upon the best algorithm by 60% and the worst algorithm by 97.5%. We also compared the cross-validated AUC’s of the models optimized to have the best AUC to highlight the importance of the choice of risk function. The results highlight the potential of the super learner in complex predictive tasks in medical research, although considerations of computational expense and model complexity must be carefully managed. Introduction Suicide is a serious public health problem that can have lasting harmful effects on indi-viduals, families, and communities. In 2020, suicide was the second leading cause of death among individuals between the ages of 10-14 and 25-34, the third leading cause of death among individuals between the ages of 15-24, and the fifth leading cause of death among individuals between the ages of 35 and 44. The total age-adjusted suicide rate in the United States increased 35.2% from 10.4 per 100,000 in 2000 to 14.2 per 100,000 in 2018, before de-clining to 13.9 per 100,000 in 2019 and to 13.5 per 100,000 in 2020 and once again increasing to 14.0 per 100,000 in 2021. 1 The suicide trends in North Carolina (NC) are similar to those of the nation overall, with increases from about 13 suicide deaths to 16 suicide deaths per 100,000 from 2012 to 2018. In NC, suicide is the third leading cause of death among those aged 5–44 years. Suicide mortality rates vary by gender, race, and age. 2 , 3 Risk factors associated with suicide are multifaceted, involving a mix of genetic, biolog-ical, clinical, physiological, environmental, and economic factors, among others. The main risk factors for suicide mortality identified in the meta-analyses conducted by Brent et al. 4 include a previous suicide attempt, suicidal ideation, psychiatric disorders (such as mood and psychotic disorders), physical illnesses (such as cancer and epilepsy), socio-demographic fac-tors (including unemployment and low education), contact with the criminal justice system, state care in childhood, access to firearms, and parental death by suicide. These risk factors were found to be associated with an elevated risk of suicide mortality, with varying degrees of association. The study also highlighted the multi-factorial nature of suicide mortality and emphasized the need for prevention strategies that account for these identified risk factors and their relative strengths. Considerable research has investigated the relationship between individual predictors and the risk of suicide mortality. Mental health and substance use disorders are recognized as primary predictors of suicide and suicidal tendencies. For example, studies on suicide deaths show that individuals with borderline personality disorder have a standardized mortality ratio (SMR) of 45.1 (95% Confidence Interval [CI] 29.0 to 61.3), while those with opioid use disorder have an SMR of 13.5 (95% CI 10.5 to 17.2). 5 – 7 A review by Stack et al. 8 provides a comprehensive analysis of the environmental factors associated with suicide, highlighting the impact of political, social, cultural, and economic influences. It emphasizes the protec-tive role of social institutions such as marriage, parenting, and religiousness, as well as the influence of economic factors such as unemployment and low income on suicide risk. Fur-thermore, research involving patients from US health maintenance organizations, notably Kaiser Permanente, 9 highlighted traumatic brain injury as a significant physical health risk factor for suicide, with an adjusted odds ratio (OR) of 8.80 (95% CI 7.37 to 10.50). While many studies have identified different risk factors linked to self-harm and suicide, focusing on these factors alone does not offer much guidance for clinicians on how to identify those most at risk for suicide. Importantly, it was found that 54% of individuals who died by suicide had not been diagnosed with any mental health condition, indicating that even with notable SMRs linked to mental health issues, these alone do not account for the majority of suicide cases. This suggests a gap in the identification of mental health disorders, highlighting the complex nature of suicide prevention efforts 10 – 13 Several existing studies indicate that healthcare utilization tends to rise just before an individual dies by suicide, suggesting that the healthcare system could play a crucial role in suicide prevention efforts. 14 – 18 In a national cohort study in Sweden Crump et al. 19 found that 30.4% and 52.3% of individuals with drug use disorders (DUD) who died by suicide had visited healthcare facilities within 2 weeks and 3 months before their death, respectively, in contrast to 5.9% and 24.3% among controls. The research underscores the critical need for recognizing and addressing suicidal tendencies in DUD patients, particularly in primary and specialty outpatient settings, where many of these patients had their last clinical visits shortly before their suicide. The study by Qin et al. 20 identifies critical suicide risk peaks immediately after psychiatric admission and discharge, noting higher risks for those with shorter hospital stays and significant impacts of affective disorders on suicide rates. The study by Ho et al. 21 reveals a significantly elevated suicide risk among psychiatric patients in Hong Kong in the year following hospital discharge, particularly within the first 28 days, with the highest risks observed in young adults and females, regardless of specific diagnoses. However, the populations studied were subject to healthcare systems vastly different from that of the United States. Hence large-scale, longitudinal studies integrating multi-ple databases and perspectives are essential for identifying predictors of suicide, building prediction models and informing targeted interventions. There have been several existing studies that have built models for predicting suicide-related outcomes. Building an effective prediction model for suicide-related outcomes re-quires a nuanced understanding of both epidemiological and clinical aspects of suicide pre-vention. 22 In parametric regression, the bias-trade variance trade-off is pronounced. A logistic regression may overfit the data if there are more parameters than observations. But if a model is too simple, the results may not accurately capture the predictor-outcome rela-tionship. In this context, flexible machine learning procedures offer a promising alternative, allowing researchers to uncover intricate relationships among suicide risk factors, with safe-guards against an overfit. With a growing abundance of data, there is an increasing interest in utilizing machine learning for developing suicide prediction models across various popu-lations. 23 – 29 But the choice of the most appropriate model for the data remains a challenge. Ensemble machine learning techniques provide researchers with the flexibility to explore multiple algorithms simultaneously. These methods, combine predictions from various base learners to create a more robust and accurate model. By incorporating diverse algorithms into the ensemble, researchers can leverage the strengths of each individual method while mitigating their weaknesses. This approach allows for a comprehensive exploration of differ-ent modeling approaches, including both traditional statistical methods and more complex machine learning algorithms. By employing ensemble learning techniques, these models of-fer valuable insights into the intricate interplay of suicide risk factors. Consequently, while statistical advancements show promise for enhancing suicide risk assessment, a nuanced un-derstanding of model capabilities and limitations is essential for informed clinical decision-making and the development of effective suicide prevention strategies. The paper follows this structure: Section 2 provides details on the dataset and methods, Section 3 presents the results, and Section 4 discusses the findings. Methods 0.1 Participants The current study 30 links mortality information from 2006-2020 from two comprehensive statewide systems, the NC death certificate data and The North Carolina Violent Death Reporting System (NCVDRS), to four different target subpopulations: electronic health records of a large healthcare system serving residents of all 100 coun-ties of the state claims records of a large single private health insurance carrier in the state claims records of the state Medicaid program justice individuals released from the NC prisons following incarceration. For the scope of this paper, our focus will be on the data associated with UNC EHR ex-clusively. Death certificates from 2006 to 2022, publicly accessible records managed by the NC Department of Health and Human Services (DHHS), are utilized. NCVDRS, a state-based surveillance system, collates information from various sources including law en-forcement, coroners, medical examiners, vital statistics, and crime laboratories to compile detailed records of violent deaths within each state. The study incorporates NCVDRS data spanning from 2006 to 2022. The UNC electronic health records (UNC EHR) encompass data from the UNC Health System, which delivers medical care to over 1 million patients annually, constituting approximately 10% of the North Carolina population, across all 100 counties. The study employed a nested case-control design, where the total population is those within the UNC Healthcare system. The cases and controls are sampled from within a risk set. A person enters the risk set when they accrue 3 months of having been in care at the UNC Healthcare System, they leave the risk set if they go ≥ 3 months without any encounter in the UNC Healthcare System. Case events are all suicide deaths as defined by ICD codes, from within the risk set. Control Events are randomly selected per Case Event. Controls are matched to Case Events based on inclusion in the respective monthly Risk Set, as well as factors such as sex, and age group. Random selection is without replacement per Case Event and with replacement across Control Events. Selecting controls based on incidence density-matched controls addresses significant selec-tion bias concerns often encountered with convenience or high-risk control groups commonly found in suicide literature. 31 These cases and controls will then be utilized to formulate a series of predictive algorithms aimed at identifying individuals at risk of short-term suicide death. 0.2 Covariates and Outcome The outcome of suicide death is identified using ICD 10 codes (U03, X60–X70, X71–X83 and Y87.0) via linkage of NC death records and NCVDRS with UNC EHR data. The primary covariates of interest include demographics like age, sex, and race, healthcare utilization and prior diagnosis. We are specifically interested in short-term information, which we define as data captured within one month leading up to the event. Diagnosis variables are categorized into four groups: acute, sub-acute, chronic, and per-manent diseases. The acute conditions such as acute pain contain one-month prior informa-tion. For sub-acute conditions, we add 6 additional months of history, for chronic conditions we add 12 additional months of history and for permanent health conditions, we use all avail-able history. Each diagnosis has 3 variables with information about whether the person had any encounters, number of encounters and number of unique encounters in the healthcare system related to that diagnosis in the defined time period. Healthcare utilization variables include flag variables to indicate if a person has had any inpatient, outpatient or emergency room visits, number of inpatient, outpatient, or emergency room visits, and total number of days spent in the hospital as an inpatient in the past 1 month. This gives us a total of 48 covariates. 0.3 Model for Predicting Suicide Deaths The primary aim is to predict the risk score of suicide death. Our statistical task is to estimate the probability of suicide death, given the covariates measured in the study. The method used in this paper is a super learner. Superleaner is an ensembling machine learning approach that combines multiple algorithms into a single algorithm and returns a prediction function with the lowest cross-validated risk of choice such as MSE or expected Negative Log Likelihood. 32 This approach hence frees investigators from the need to commit to a single regression technique from the beginning; instead, it allows them to simultaneously employ multiple methods. By leveraging several relevant algorithms, researchers are able to determine the most effective function tailored to their specific data set. Thus with Super Learning, several algorithms can“work together” to create a combination of algorithms that can outperform any single candidate algorithm in the library (including simple, parametric regressions or more data-adaptive algorithms such as random forests or neural networks). Figure 1 describes the super learner algorithm. Download figure Open in new tab Figure 1: Superlearner algorithm flowchart We define the full data structure as X = ( W, Y ) ∼ P X, 0 with j covariates W 1 , …, W j and binary outcome Y , indicating suicide death. The full data structure refers to all subjects in the risk set. We introduce a random variable O to represent the observed data, O = ( Y, Δ, Δ X ) = ( Y, Δ, Δ( W, Y )) ∼ P 0 where Δ is an indicator of inclusion in nested case-control sample from the risk set. The target statistical parameter of interest as a function of the of the full data distribution of X is given by Q̅ = E X, 0 ( Y |W ) and the full-data statistical model denoted by M F (full data) is non-parametric. E 0 L ( O , Q̅ ) is the expected loss, which evaluates the candidate Q̅ . We choose negative log likelihood as the loss function, represented by L F ( X , Q̅ ). If our sample had consisted of n i.i.d. observations X i , we would estimate Q̅ = E X, 0 ( Y |W ) simply using the loss function L F ( X , Q̅ ). However, given that we observe O, we estimate Q̅ with super learning and weights for observation i = 1 , …n , which now gives us an inverse probability weighted loss function with input X : Thus our target parameter of interest is defined as Q̅ = arg min Q̅ E 0 L ( O , Q̅ )) where Q̅ is a possible function that is compatible with our current prediction problem. Q̅ is minimized at the optimal choice of Q̅ 0 . The super learner algorithm is weighted due to the nested case control study design. The probability of being sampled in the nested case-control study, given that a person experienced suicide death= 1 because all eligible cases in the risk set are sampled. The probability of being sampled in the nested case-control study, given that a person did not experience suicide death is calculated as the Total number sampled as controls/ Total number at risk of being sampled as controls, which was equal to 0.000131. We implement super learning in the R programming language 34 with the SuperLearner package 35 to estimate Q̅ 0 . In the construction of our predictive model, we used a diverse library of candidate learn-ers to ensure robustness and accuracy. We only use candidate learners that allow for the incorporation of weights. This library included: the mean algorithm, generalized linear mod-els, elastic net penalized linear regression models, generalized additive models, and recursive partitioning and regression trees. Each learner was chosen for its unique statistical properties and strengths in handling different types of data structures and relationships, facilitating a comprehensive approach to prediction. We restricted to these 5 models due to computational limitations. Results The summary of the dataset can be found in Tables 1 and 2 . Of the 11,206 observations, 1021 are cases with a suicide death and 10,185 are controls without an event. 72.4% of the observations are male, 68.7% are white, and the mean age is 51.58 with a standard deviation of 17.91. The tables also contains information about the person’s prior diagnosis and healthcare utilization information. There are a total of 48 variables, and we present a select few variables in the tables. The most prevalent diagnoses are cardiovascular diseases, mental health and chronic pain. Around 15% of the subjects had an outpatient visit, 0.3% had an emergency room visit and 1.1% had an inpatient visit in the month before the event. View this table: View inline View popup Table 1: Descriptive Statistics of Categorical Variables View this table: View inline View popup Table 2: Descriptive Statistics of Continuous Variables Before running the model we removed continuous variables with 0 variablilty and remove categorical variables with less than 1% representation in the dataset. Observation weights within the super learner were assigned based on the inverse proba-bility of missingness, . Thus cases were given observation weights equal to 1 and controls were given observation weights of 1/0.000131 = 7633.588. This number was calculated based on the monthly risk set information and the number of cases and controls in the dataset. Table 3 presents candidate learners and libraries used in the model. The negative log-likelihood for each algorithm, along with the calculated weight associated with each algorithm in the ensemble SL model is presented in Table 4 . The highest weight of 0.753 is associated with recursive partitioning and regression trees, followed by a weight of 0.119 for generalized additive model. View this table: View inline View popup Download powerpoint Table 3: Candidate Learners View this table: View inline View popup Download powerpoint Table 4: CV NLL and Coefficient of each candidate Figure 2 represents the minimum, average, and maximum negative log-likelihood of each model. Download figure Open in new tab Figure 2: 10 fold CV Risk Estimate of models Table 5 presents the Relative Efficiency with respect to negative log-likelihood for each al-gorithm. Relative efficiency for each of the k algorithms was defined as RE = CV NNL(k)/CV NNL(super learner). Superlearner improved upon the best algorithm generalized additive models by around 60%. The super learner improved upon the worst algorithm, main terms logistic regression by 97.5%. We also fit another model, with the same 5 base learners, but this time used AUC as the risk function so that they are optimized to have the best AUC. In Table 6 we present the cross-validated AUC of the different base learners and the ensemble superlearner. In this case, ensemble SL has the highest AUC of 0.74, with the other base learners having AUC’s ranging from 0.5 to 0.72. View this table: View inline View popup Download powerpoint Table 5: Relative Efficiency With Respect to Cross-Validated NNL View this table: View inline View popup Download powerpoint Table 6: CV AUC for model with AUC as Risk fucntion Discussion Research in medicine and epidemiology has focused significantly on developing func-tions for predicting risk scores, aiming to generate risk assessments for various diseases. In the context of suicide, several studies have developed models for predicting suicide-related outcomes. Apart from traditional statistics methods, there has been a growing interest in in-tegrating machine learning techniques, owing to the increasing abundance of data. However, determining the superior algorithm for a given dataset remains a challenge prior to analysis. The primary advantage of using an ensemble algorithm such as the super learner is that it eliminates the need to choose a single algorithm a priori. This method allows for the simultaneous application of multiple candidate learners to identify an enhanced, optimal estimator. It also facilitates the selection of the most appropriate loss function for optimizing the algorithm, and employs cross-validation to prevent overfitting. In this study, we chose the Negative Log Likelihood because our main interest lies in accurately predicting probabilities rather than merely distinguishing between events, for which the AUC (Area Under the Curve) would be a more suitable choice. The ensemble super learner had the best performance compared to all 5 models. The model significantly improved upon all the base learners with respect to CV Expected Non-Negative Log Likelihood. Even in the case when the improvement from the base learners is small, superlearner is still advantageous because it can provide researchers with the ability to choose the best model, without needing to commit to any single model a prior. The choice of the risk function is critical and must be based on the scientific question of interest. In our case, the focus is on risk score prediction, which relies on accurately predicting the probabilities rather than merely distinguishing between the events. In this case, Negative Log Likelihood is a better metric than AUC. The major drawback with implementing super learner is the computational time and intensity when compared to running a single model. This becomes further more a constraint with increasing number of base learners. The success of super learner is contingent upon the base learners that are provided. Ideally, one would use all relevant algorithms as base learners to find the most optimal model. However, due to above mentioned constraints, there is often a limitation on the number of base learners one specifies. Hence the decision regarding which algorithms to use must be systematic and well informed. For future work, focusing on assessing the importance of variables within the super learner framework could be valuable. Understanding which features contribute most significantly to the predictive performance of the model would prove to be very valuable insights to clinicians. Prior research has consistently indicated that suicide attempts and suicide ideations are robust predictors of suicide-related death 36 Furthermore, certain medications have been asso-ciated with an increased risk of suicide attempts and fatalities. 37 To enhance predictive mod-els for suicide risk, it is crucial to incorporate these elements into the analytical framework. By integrating variables such as history of suicide attempts, presence of suicide ideation, and use of high-risk medications, the model can offer a more comprehensive approach to assessing suicide risk. Relying solely on EHR data also introduces several limitations. The accuracy and re-liability of EHR data can vary significantly depending on the quality and meticulousness of data entry practices. 38 Additionally, EHRs may not capture non-clinical factors that significantly contribute to suicide risk, such as socioeconomic status or life stressors, which are often not documented systematically. Therefore, while the integration of EHR data en-hances predictive modeling for suicide risk, it is essential to acknowledge and address these limitations to develop more comprehensive and effective risk assessment tools in the field of suicide prevention. Conclusions In this study, we applied the super learner ensemble algorithm to predict suicide-related outcomes. Our analysis demonstrates the effectiveness of ensemble learning in integrating diverse algorithms, notably recursive partitioning and regression trees, to enhance predictive accuracy. Despite computational challenges, the super learner offers a systematic approach to model selection and optimization, resulting in significant improvements over individual base learners. Looking ahead, the super learner algorithm presents a promising avenue for enhancing predictive modeling in healthcare research. Future efforts should concentrate on refining this algorithm to overcome computational limitations and unlock its full potential. It will also be important to explore the development of clinically relevant tools, such as identifying which variables are most predictive of outcomes. Overall, our study highlights the significant value of ensemble techniques, especially the super learner, in improving risk assessments and aiding clinical decision-making processes. Data Availability The data are not available upon request to the authors. References 1. ↵ Suicide . https://www.nimh.nih.gov/health/statistics/suicide . Accessed: 2024-4-14 . 2. ↵ Explore suicide in North Carolina . https://www.americashealthrankings.org/explore/measures/suicide Accessed: 2024-4-14 . 3. ↵ Dph NC. N.C. dph: IVP branch: Violent death data . https://injuryfreenc.dph.ncdhhs.gov/DataSurveillance/ViolentDeathData.htm . Ac-cessed: 2024-4-14 . 4. ↵ Turecki G , Brent DA , Gunnell D , et al. Suicide and suicide risk . Nature Reviews Disease Primers . 2019 ; 5 ( 1 ): 74 . OpenUrl 5. ↵ Hawton K , Heeringen K . Suicide . Lancet . 2009 ; 373 ( 9672 ): 1372 – 1381 . OpenUrl CrossRef PubMed Web of Science 6. Bachmann S . Epidemiology of suicide and the psychiatric perspective . Int. J. Environ. Res. Public Health . 2018 ; 15 ( 7 ): 1425 . OpenUrl CrossRef PubMed 7. ↵ Nock MK , Borges G , Bromet EJ , Cha CB , Kessler RC , Lee S. Suicide and suicidal behavior . Epidemiol. Rev . 2008 ; 30 ( 1 ): 133 – 154 . OpenUrl CrossRef PubMed Web of Science 8. ↵ Stack S . Contributing factors to suicide: Political, social, cultural and economic . Prev. Med . 2021 ; 152 ( 106498 ): 106498 . OpenUrl 9. ↵ Ahmedani BK , Peterson EL , Hu Y , et al. Major physical health conditions and risk of suicide . Am. J. Prev. Med . 2017 ; 53 ( 3 ): 308 – 315 . OpenUrl PubMed 10. ↵ Sivaraman JJC , Greene SB , Naumann RB , Proescholdbell S , Ranapurwala SI , Marshall SW . Association between medical diagnoses and suicide in a Medicaid beneficiary pop-ulation, North Carolina 2014-2017 . Epidemiology . 2022 ; 33 ( 2 ): 237 – 245 . OpenUrl PubMed 11. Olfson M , Wall M , Wang S , et al. Suicide after deliberate self-harm in adolescents and young adults . Pediatrics . 2018 ; 141 ( 4 ): e20173517 . OpenUrl CrossRef PubMed 12. Olfson M , Wall M , Wang S , Crystal S , Gerhard T , Blanco C . Suicide following deliberate self-harm . Am. J. Psychiatry . 2017 ; 174 ( 8 ): 765 – 774 . OpenUrl CrossRef PubMed 13. ↵ Devin J , Lyons S , Murphy L , O’Sullivan M , Lynn E . Factors associated with suicide in people who use drugs: a scoping review . BMC Psychiatry . 2023 ; 23 ( 1 ): 655 . OpenUrl 14. ↵ Braciszewski JM , Lanier A , Yeh HH , et al. Health diagnoses and service utilization in the year before youth and young adult suicide . Psychiatr. Serv . 2023 ; 74 ( 6 ): 566 – 573 . OpenUrl 15. Kohlbeck S , Armanious M , Pickett M . Assessing the healthcare utilization of youth who died by suicide: A case-control study . Arch. Suicide Res . 2023 ; 27 ( 1 ): 80 – 88 . OpenUrl CrossRef 16. Meehan J , Kapur N , Hunt IM , et al. Suicide in mental health in-patients and within 3 months of discharge. National clinical survey . Br. J. Psychiatry . 2006 ; 188 : 129 – 134 . OpenUrl Abstract / FREE Full Text 17. Schou Pedersen H , Fenger-Grøn M , Bech BH , Erlangsen A , Vestergaard M . Frequency of health care utilization in the year prior to completed suicide: A Danish nationwide matched comparative study . PLoS One . 2019 ; 14 ( 3 ): e0214605 . OpenUrl PubMed 18. ↵ Hunt IM , Kapur N , Robinson J , et al. Suicide within 12 months of mental health service contact in different age and diagnostic groups: National clinical survey . Br. J. Psychiatry . 2006 ; 188 : 135 – 142 . OpenUrl Abstract / FREE Full Text 19. ↵ Crump C , Kendler KS , Sundquist J , Edwards AC , Sundquist K . Health care utilization prior to suicide in adults with drug use disorders . J. Psychiatr. Res . 2021 ; 135 : 230 – 236 . OpenUrl 20. ↵ Qin P , Nordentoft M . Suicide risk in relation to psychiatric hospitalization: evidence based on longitudinal registers . Arch. Gen. Psychiatry . 2005 ; 62 ( 4 ): 427 – 432 . OpenUrl CrossRef PubMed Web of Science 21. ↵ Ho TP . The suicide risk of discharged psychiatric patients . J. Clin. Psychiatry . 2003 ; 64 ( 6 ): 702 – 707 . OpenUrl CrossRef PubMed Web of Science 22. ↵ Large MM . The role of prediction in suicide prevention . Dialogues Clin. Neurosci . 2018 ; 20 ( 3 ): 197 – 205 . OpenUrl PubMed 23. ↵ Belsher BE , Smolenski DJ , Pruitt LD , et al. Prediction models for suicide attempts and deaths: A systematic review and simulation . JAMA Psychiatry . 2019 ; 76 ( 6 ): 642 – 651 . OpenUrl 24. Huang S , Lewis MO , Bao Y , et al. Predictive modeling for suicide-related outcomes and risk factors among patients with pain conditions: A systematic review . J. Clin. Med . 2022 ; 11 ( 16 ): 4813 . OpenUrl 25. Kessler RC , Bossarte RM , Luedtke A , Zaslavsky AM , Zubizarreta JR . Suicide prediction models: a critical review of recent research with recommendations for the way forward . Mol. Psychiatry . 2020 ; 25 ( 1 ): 168 – 179 . OpenUrl CrossRef PubMed 26. Lee J , Pak TY . Machine learning prediction of suicidal ideation, planning, and attempt among Korean adults: A population-based study . SSM Popul. Health . 2022 ; 19 ( 101231 ): 101231 . OpenUrl 27. Seyedsalehi A , Fazel S . Suicide risk assessment tools and prediction models: new evidence, methodological innovations, outdated criticisms . BMJ Ment Health . 2024 ; 27 ( 1 ): e300990 . OpenUrl Abstract / FREE Full Text 28. Su C , Aseltine R , Doshi R , Chen K , Rogers SC , Wang F . Machine learning for suicide risk prediction in children and adolescents with electronic health records . Transl. Psychiatry . 2020 ; 10 ( 1 ): 413 . OpenUrl 29. ↵ Nordin N , Zainol Z , Mohd Noor MH , Chan LF . An explainable predictive model for suicide attempt risk using an ensemble learning and Shapley Additive Explanations (SHAP) approach . Asian J. Psychiatr . 2023 ; 79 ( 103316 ): 103316 . OpenUrl 30. ↵ Ranapurwala SI , Miller VE , Carey TS , et al. Innovations in suicide prevention re-search (INSPIRE): a protocol for a population-based case-control study . Inj. Prev . 2022 ; 28 ( 5 ): 483 – 490 . OpenUrl Abstract / FREE Full Text 31. ↵ Cole JA , Taylor JS , Hangartner TN , Weinreb NJ , Mistry PK , Khan A . Reducing se-lection bias in case-control studies from rare disease registries . Orphanet J. Rare Dis . 2011 ; 6 ( 1 ): 61 . OpenUrl CrossRef PubMed 32. ↵ Phillips RV , Laan MJ , Lee H , Gruber S . Practical considerations for specifying a super learner . Int. J. Epidemiol . 2023 ; 52 ( 4 ): 1276 – 1285 . OpenUrl 33. Rose S , Fireman B , Laan MJ . Nested case-control risk score prediction . in Targeted LearningSpringer series in statistics : 239 – 245 New York, NY : Springer New York 2011 . 34. ↵ R Core Team . R: A Language and Environment for Statistical Computing . R Foundation for Statistical ComputingVienna, Austria 2021 . 35. ↵ Polley EC , van der Laan MJ ., SuperLearner: Super Learner Prediction, Package Ver-sion 2.0–4, 2011Vienna, AustriaR Foundation for Statistical Computing http://cran.r-project.org/web/packages/SuperLearner/ . Accessed: 2024-4-14 . 36. ↵ Bostwick JM , Pabbati C , Geske JR , McKean AJ . Suicide attempt as a risk factor for com-pleted suicide: Even more lethal than we knew . Am. J. Psychiatry . 2016 ; 173 ( 11 ): 1094 – 1100 . OpenUrl CrossRef PubMed 37. ↵ Gibbons R , Hur K , Lavigne J , Wang J , Mann JJ . Medications and suicide: High dimen-sional empirical Bayes screening (iDEAS) . Harvard Data Science Review . 2019 . 38. ↵ Greiver M , Barnsley J , Glazier RH , Harvey BJ , Moineddin R . Measuring data reli-ability for preventive services in electronic medical records . BMC Health Serv. Res . 2012 ; 12 ( 1 ): 116 . OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted September 27, 2024. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Suicide Death Predictive Models using Electronic Health Record Data Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Suicide Death Predictive Models using Electronic Health Record Data Shweta Srikanth , Lina M. Montoya , Matthew M. Turnure , Brian W. Pence , Naoko Fulcher , Bradley N. Gaynes , David B. Goldston , Timothy S. Carey , Shabbar I. Ranapurwala medRxiv 2024.09.26.24314402; doi: https://doi.org/10.1101/2024.09.26.24314402 Share This Article: Copy Citation Tools Suicide Death Predictive Models using Electronic Health Record Data Shweta Srikanth , Lina M. Montoya , Matthew M. Turnure , Brian W. Pence , Naoko Fulcher , Bradley N. Gaynes , David B. Goldston , Timothy S. Carey , Shabbar I. Ranapurwala medRxiv 2024.09.26.24314402; doi: https://doi.org/10.1101/2024.09.26.24314402 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Epidemiology Subject Areas All Articles Addiction Medicine (573) Allergy and Immunology (865) Anesthesia (302) Cardiovascular Medicine (4453) Dentistry and Oral Medicine (444) Dermatology (383) Emergency Medicine (609) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1515) Epidemiology (15242) Forensic Medicine (30) Gastroenterology (1131) Genetic and Genomic Medicine (6615) Geriatric Medicine (669) Health Economics (1001) Health Informatics (4552) Health Policy (1372) Health Systems and Quality Improvement (1614) Hematology (543) HIV/AIDS (1270) Infectious Diseases (except HIV/AIDS) (15929) Intensive Care and Critical Care Medicine (1106) Medical Education (624) Medical Ethics (147) Nephrology (670) Neurology (6625) Nursing (346) Nutrition (999) Obstetrics and Gynecology (1148) Occupational and Environmental Health (957) Oncology (3344) Ophthalmology (979) Orthopedics (369) Otolaryngology (421) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1696) Pharmacology and Therapeutics (693) Primary Care Research (714) Psychiatry and Clinical Psychology (5461) Public and Global Health (9252) Radiology and Imaging (2207) Rehabilitation Medicine and Physical Therapy (1371) Respiratory Medicine (1197) Rheumatology (597) Sexual and Reproductive Health (715) Sports Medicine (530) Surgery (714) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a02d3fbe8b54df88',t:'MTc3OTk3MTE5OQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();