GLUCOSE: A Distributional Reinforcement Learning Model for Optimal Glucose Control After Cardiac Surgery

GLUCOSE: A Distributional Reinforcement Learning Model for Optimal Glucose Control After Cardiac Surgery | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search GLUCOSE: A Distributional Reinforcement Learning Model for Optimal Glucose Control After Cardiac Surgery Jacob M. Desman , Zhang-Wei Hong , Moein Sabounchi , Ashwin S. Sawant , Jaskirat Gill , Ana C Costa , Gagan Kumar , Rajeev Sharma , Arpeta Gupta , Paul McCarthy , Veena Nandwani , Doug Powell , Alexandra Carideo , Donnie Goodwin , Sanam Ahmed , Umesh Gidwani , Matthew Levin , Robin Varghese , Farzan Filsoufi , Robert Freeman , Avniel Shetreat-Klein , Alexander W Charney , Ira Hofer , Lili Chan , David Reich , Patricia Kovatch , Roopa Kohli-Seth , Monica Kraft , Pulkit Agrawal , John A. Kellum , Girish N. Nadkarni , Ankit Sakhuja doi: https://doi.org/10.1101/2025.01.01.25319851 Jacob M. Desman 1 The Charles Bronfman Institute for Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA 2 Division of Data-Driven and Digital Medicine, Department of Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Zhang-Wei Hong 3 Improbable AI Lab, Massachusetts Institute of Technology , Cambridge, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Moein Sabounchi 1 The Charles Bronfman Institute for Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA 2 Division of Data-Driven and Digital Medicine, Department of Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ashwin S. Sawant 1 The Charles Bronfman Institute for Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA 2 Division of Data-Driven and Digital Medicine, Department of Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA 4 Division of Hospital Medicine, Department of Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Jaskirat Gill 5 Institute for Critical Care Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ana C Costa 6 Department of Cardiothoracic Surgery, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Gagan Kumar 7 Department of Pulmonary and Critical Care Medicine, Northeast Georgia Medical Center , Gainesville, Georgia , USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Rajeev Sharma 8 Division of Endocrinology, Hackensack University Medical Center , Hackensack, New Jersey, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Arpeta Gupta 9 Division of Endocrinology, Millenium Physician Group , Jacksonville, FL, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Paul McCarthy 10 Section of Cardiovascular Critical Care, Departm ent of Cardiovascular and Thoracic Surgery, West Virginia University , Morgantown, West Virginia, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Veena Nandwani 10 Section of Cardiovascular Critical Care, Departm ent of Cardiovascular and Thoracic Surgery, West Virginia University , Morgantown, West Virginia, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Doug Powell 10 Section of Cardiovascular Critical Care, Departm ent of Cardiovascular and Thoracic Surgery, West Virginia University , Morgantown, West Virginia, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Alexandra Carideo 5 Institute for Critical Care Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Donnie Goodwin 10 Section of Cardiovascular Critical Care, Departm ent of Cardiovascular and Thoracic Surgery, West Virginia University , Morgantown, West Virginia, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Sanam Ahmed 5 Institute for Critical Care Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Umesh Gidwani 5 Institute for Critical Care Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Matthew Levin 11 Department of Anesthesiology, Perioperative, and Pain Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Robin Varghese 5 Institute for Critical Care Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA 6 Department of Cardiothoracic Surgery, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Farzan Filsoufi 6 Department of Cardiothoracic Surgery, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Robert Freeman 1 The Charles Bronfman Institute for Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Avniel Shetreat-Klein 12 Department of Rehabilitation and Physical Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Alexander W Charney 1 The Charles Bronfman Institute for Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ira Hofer 1 The Charles Bronfman Institute for Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA 2 Division of Data-Driven and Digital Medicine, Department of Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA 11 Department of Anesthesiology, Perioperative, and Pain Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Lili Chan 1 The Charles Bronfman Institute for Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA 2 Division of Data-Driven and Digital Medicine, Department of Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA 13 Division of Nephrology, Department of Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site David Reich 11 Department of Anesthesiology, Perioperative, and Pain Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Patricia Kovatch 14 Scientific Computing, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Roopa Kohli-Seth 5 Institute for Critical Care Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Monica Kraft 15 Samuel Bronfman Department of Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Pulkit Agrawal 3 Improbable AI Lab, Massachusetts Institute of Technology , Cambridge, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site John A. Kellum 16 Department of Critical Care Medicine, University of Pittsburgh School of Medicine , Pittsburgh, PA, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Girish N. Nadkarni 1 The Charles Bronfman Institute for Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA 2 Division of Data-Driven and Digital Medicine, Department of Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA 13 Division of Nephrology, Department of Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ankit Sakhuja 1 The Charles Bronfman Institute for Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA 2 Division of Data-Driven and Digital Medicine, Department of Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA 5 Institute for Critical Care Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: ankit.sakhuja{at}mssm.edu Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF ABSTRACT This study introduces Glucose Level Understanding and Control Optimized for Safety and Efficacy (GLUCOSE), a distributional offline reinforcement learning algorithm for optimizing insulin dosing after cardiac surgery. Trained on 5,228 patients, tested on 920, and externally validated on 649, GLUCOSE achieved a mean estimated reward of 0.0 [-0.07, 0.06] in internal testing and -0.63 [-0.74, -0.52] in external validation, outperforming clinician returns of -1.29 [-1.37, -1.20] and -1.02 [-1.16, -0.89]. In multi-phase human validation, GLUCOSE first showed a significantly lower mean absolute error (MAE) in insulin dosing, with 0.9 units MAE versus clinicians’ 1.97 units (p < 0.001) in internal testing and 1.90 versus 2.24 units (p = 0.003) in external validation. The second and third phases found GLUCOSE‘s performance as comparable to or exceeding that of senior clinicians in MAE, safety, effectiveness, and acceptability. These findings suggest GLUCOSE as a robust tool for improving postoperative glucose management. INTRODUCTION Cardiac surgery elicits a substantial metabolic stress response resulting in postoperative hyperglycemia regardless of diabetic status ( 1 ). Post-operative hyperglycemia after cardiac surgery is common, occurring in 60-80% patients with diabetes ( 2 ), and over 50% non-diabetic patients ( 3 ). It is associated with higher rates of post-operative infections ( 4 , 5 ), acute kidney injury ( 3 , 6 – 8 ), cardiac arrhythmias ( 3 ), longer length of stay ( 3 ), and higher mortality ( 6 – 9 ). Due to its significance, the Society of Thoracic Surgeons (STS) recommends maintaining blood glucose levels below 180mg/dL after cardiac surgery ( 10 ). Achieving adequate glucose control post-operatively is challenging. A study found that only 15% of patients had appropriate glucose control, defined as glucose level between 70mg/dL to 180mg/dL, within the first day after cardiac surgery ( 11 ). This early post-operative period, when patients are critically ill and require care in intensive care unit (ICU) settings, is highly dynamic with rapidly changing clinical characteristics of patients. Currently, post-operative glucose management involves titration of regular insulin based on hospital specific protocols and the experience of treating clinicians. However, due to the highly dynamic nature of this early post-operative period, some treatment regimens may be more suitable for certain patients or only effective for a limited time as their condition evolves. This leads to high rates of hyperglycemic and hypoglycemic episodes ( 11 , 12 ), both associated with worse outcomes, as these protocolized regimens often fail to account for individual patient variability in real-world settings ( 13 , 14 ). Therefore, personalized and dynamic insulin titration is crucial for improving glucose control in patients following cardiac surgery. Reinforcement Learning (RL), a type of machine learning where an agent learns to make decisions by performing actions in an environment to maximize cumulative rewards, offers a promising solution to this challenge ( 15 ). RL algorithms receive feedback in the form of rewards or penalties based on the actions taken, allowing the agent to improve its policy over time. This adaptability makes RL particularly well-suited for tasks that involve complex decision-making and require real-time adjustments, such as insulin titration in the dynamic postoperative environment. Implementing an RL-based system for insulin titration can address the limitations of current glucose management protocols. By continuously learning from individual patient data, RL can provide personalized treatment plans that account for specific patient variability and maintain glucose in optimal range. Additionally, RL’s capability to adapt to rapidly changing clinical characteristics ensures that insulin dosing remains optimal as patient conditions evolve. Traditionally, offline RL, where the agent learns from a fixed dataset without further interaction with the environment, has been limited by its focus on expected rewards, often overlooking the uncertainty in patient responses ( 16 ). This limitation can lead to suboptimal treatment plans, as it fails to account for the full spectrum of possible outcomes. As a result, offline RL systems may not adequately address the diverse risk profiles associated with different patient actions, potentially compromising the safety and effectiveness of interventions ( 17 ). Our approach addresses this limitation by integrating distributional RL, which characterizes the entire distribution of potential outcomes rather than just the expected reward ( 17 ). This methodology provides a more comprehensive understanding of the risks and benefits associated with various actions, allowing for more nuanced decision-making under uncertainty ( 17 – 19 ). By considering the full range of potential patient responses, distributional RL can enhance the personalization and safety of insulin titration protocols, ensuring optimal dosing as patient conditions change. Our proposed model, Glucose Level Understanding and Control Optimized for Safety and Efficacy (GLUCOSE), aims to improve glucose management on the first day after cardiac surgery, potentially leading to better patient outcomes and more effective clinical decision-making. We have developed GLUCOSE using data of patients undergoing cardiac surgery in the Medical Information Mart for Intensive Care-IV (MIMIC-IV) database ( 20 ). We then validated the model externally with cardiac surgery patients from the eICU Collaborative Research Database (eICU-CRD), a diverse, multicenter database of critically ill patients ( 21 ). RESULTS Study Population GLUCOSE was trained and validated on two separate ICU datasets: the MIMIC-IV database ( 20 ) and the eICU-CRD database ( 21 ). MIMIC-IV was used as the development cohort and split into training and internal testing sets. eICU-CRD was used as the external validation dataset. Based on the inclusion and exclusion criteria, our study included 6,148 patients in development dataset and 649 patients in external validation dataset. The mean age of patients in the development dataset was 67.8 ± 11.6 years with 71.1% males, and in external validation dataset was 67.0 ± 11.3 years with 67.2% males. At least one hypoglycemic event (180 mg/dL) occurred in 47.8% of patients in development dataset and 47.3% of patients in external validation dataset. The baseline characteristics of the patients are shown in Table 1 and Supplementary Table 1. The overall structure of our study is illustrated in Fig. 1 . View this table: View inline View popup Download powerpoint Table 1. Patient characteristics Download figure Open in new tab Figure 1. Study overview. a) Schema for model development, testing, and selection. b) Overview of clinician validation study. Performance of GLUCOSE To mitigate the sampling and stochastic biases inherent in offline RL ( 22 ) we trained, in line with previous literature, multiple models until we observed no significant improvements in the RL policies ( 23 , 24 ) ( 25 ). Consequently, we trained 200 independent models and selected the model with the maximal lower bound of the 95% CI of mean estimated performance returns within the internal testing set as the GLUCOSE model. We compared the estimated performance returns of GLUCOSE at the lower bound of its 95% CI with the upper bound of clinicians’ 95% CI ( Fig. 1 ) using fitted Q estimation (FQE) for off policy evaluation (OPE) ( 26 , 27 ), illustrating the differences in average estimated performance after evaluating 200 policies. The dotted blue and dotted orange lines reflect the 95% confidence intervals of the mean performance for the observed clinicians behavior in internal testing and external validation, respectively, while their non-dotted counterparts reflect the estimated performance of GLUCOSE through OPE ( Fig. 2a ). The best model, GLUCOSE, resulted in a mean estimated performance return of 0.0 [-0.07, 0.06] in the internal testing set and -0.63 [-0.74, -0.52] in the external validation dataset, showing significant improvements over the clinician returns of -1.29 [-1.37, -1.20] in the internal testing set and -1.02 [-1.16, -0.89] in the external validation dataset. Download figure Open in new tab Figure 2. GLUCOSE performance. a) OPE counterfactual estimated performance of the model computed by FQE (solid lines) compared to the returns by the treating clinicians (dotted lines) with 95% CI. b) Comparison of TIR and average glucose relative to insulin dosing differences with 95% CI. c) Average insulin doses across several glucose ranges with 95% CI. GLUCOSE Policy Analysis We further assessed the model by evaluating the time in range (TIR) of 70-180 mg/dL for glucose level when the actual clinician administered insulin dose was similar or different from the dose recommended by GLUCOSE. In the internal testing set, 27.3% of the time patients received insulin doses from clinicians identical to those recommended by GLUCOSE, while in the external validation dataset, this occurred 20.3% of the time. As shown in Fig. 2b , patients who received insulin doses like those suggested by GLUCOSE had the highest average TIR in both the internal testing set and external validation dataset. The TIR decreased as the difference of model recommended doses minus clinician administered doses increased, indicating that the model identifies areas for improvement in insulin management. For example, at more negative cumulative differences, where the average glucose is also lower, GLUCOSE suggests less insulin to mitigate the risk of hypoglycemia ( Fig. 2b,c ). Conversely, at more positive cumulative differences, where average blood glucose is higher and average TIR is worse, the model suggests higher insulin doses to avoid hyperglycemia. The action distribution of Fig. 2c further illustrates these trends, showing that insulin doses are lower on average at lower glucose levels and higher at higher glucose levels. To gain insight into model representations and ensure its clinical interpretability, we derived feature importances for GLUCOSE using SHapley Additive exPlanations (SHAP) (Supplementary Fig. 1) ( 28 ). This analysis revealed that the most heavily weighted features align well with clinical intuition. The current glucose level emerged as the most critical feature, with recent glucose values from the previous 4 hours and weight also being among the highest contributors. Additionally, factors indicating patient acuity, such as the use and duration of mechanical ventilation, were weighted heavily. These findings suggest that GLUCOSE uses clinically relevant information in its decision making. Human Evaluations of GLUCOSE For clinical applicability and robustness, we conducted a multi-phased human evaluation. In the first phase, two senior endocrinologists, each with over 10 years of clinical experience, provided their recommendations for hourly insulin dosing for the first day after cardiac surgery for 10 patients in both internal testing and external validation datasets. To allow the endocrinologists to provide the most accurate dosing schemes to use as a reference, we provided them with the entire time series of patient data, including the insulin doses actually administered by the treating clinicians, and resultant glucose levels. We compared the hourly insulin doses recommended by GLUCOSE, which unlike the endocrinologists had access only to the current state, to those actually administered by clinicians using the average endocrinologist doses as the reference. Across both datasets, GLUCOSE achieved significantly lower mean absolute error (MAE) in hourly insulin dosing, indicating that its dosing scheme more closely aligned with the recommendations of the endocrinologists. In the internal testing set, GLUCOSE had an MAE of 0.9 units compared to the treating clinician’s 1.97 units MAE (p 5 years’ experience), two junior cardiac intensivists (<5 years’ experience), and two cardiac intensive care unit nurse practitioners provided their recommendations for hourly insulin doses for the same patients. These clinicians were also provided with the entire time series of data, actual insulin administration record, and glucose levels to allow them to generate their most retrospectively optimal possible human policies. We then compared the GLUCOSE recommended doses, which again only had access to a single state of information at the current timestep, to those recommended by these 6 clinicians, with the endocrinologist recommendations as the reference ( Fig. 3a ). In internal testing, GLUCOSE achieved an MAE of 0.90 units compared to that of senior intensivists’ 0.82 unit MAE (p=0.57), junior intensivists’ 1.15 unit MAE (p=0.25), and nurse practitioners’ 1.23 unit MAE (p=0.21). In external validation, GLUCOSE achieved an MAE of 1.90 units compared to that of senior intensivists’ 1.58 units (p=0.32), junior intensivists 2.15 units (p=0.53), and nurse practitioners 2.28 units (p=0.38). Although the differences in MAE did not reach statistical significance, GLUCOSE demonstrates a trend toward lower MAEs than that of junior intensivists and nurse practitioners when compared against endocrinologists as the reference. Download figure Open in new tab Figure 3. Human validation study results. a) MAE of clinician groupings and GLUCOSE relative to an endocrinologist baseline with standard error of the mean. b) Blinded ratings across safety, effectiveness, and acceptability of all clinicians by a blinded senior intensivist panel with standard error of the mean. In the final phase, we conducted a blinded evaluation of GLUCOSE and all 8 clinician dosing recommendations using an expert panel of 2 separate senior intensivists to assess practical safety, effectiveness, and acceptability of the model’s recommended insulin doses. The two additional senior intensivists used a 5-point Likert scale to assess the safety (to reduce hypoglycemia), effectiveness (if the regimen would bring glucose into an acceptable range), and acceptability (if the regimen would be acceptable in a clinical scenario) of each recommended insulin regimen for the same group of 20 patients. In both internal testing and external validation datasets, GLUCOSE’s rated safety, effectiveness, and acceptability demonstrated either comparable performance or statistically significant improvements over all human policies ( Fig. 3b ). Notably, GLUCOSE performed at or above the level of senior cardiac surgery intensivists across all domains in both the internal testing and external validation datasets. This demonstrates GLUCOSE’s reliability and consistent high-level performance across diverse clinical scenarios. Evaluation of GLUCOSE’s Recommendations Among Excluded Patients Finally, we evaluated GLUCOSE‘s recommendations in subsets of the external validation dataset that were excluded from the primary analysis due to the presence of ambiguous administration of insulin, vasopressors or inotropes. These patients had documented insulin, vasopressor, or inotrope administration but with insufficient information to determine the exact timing or dose - an issue commonly reported in the multicenter eICU-CRD database ( 29 ). Based on the affected medication, we performed this evaluation separately in patients with only ambiguous insulin data (3,001 patients) and in those with ambiguous data for both insulin and vasopressors/inotropes (1,804 patients). As there were only 33 patients with non-ambiguous insulin but ambiguous vasopressor/inotrope data, we excluded them from this analysis. The external validation cohort, the ambiguous insulin subset, and the ambiguous medication subset were similar in terms of age, gender, and weight. However, these additional subsets included a higher proportion of white patients (66.6% vs 81.1% vs 89.7%, p < 0.001) and fewer patients with type 2 diabetes (9.1% vs. 3.1% vs. 1.2%, p < 0.001). While there were statistically significant differences in the average glucose levels (134.5 mg/dL vs. 132.2 mg/dL vs. 130.6 mg/dL, p < 0.001) these small differences are not clinically meaningful. Full demographic analysis can be found in Supplementary Table 2. Due to the lack of accurately recorded insulin in these subgroups, we were unable to perform OPE or direct comparisons which depend on accurate insulin administration records. However, the overall distribution of model recommended actions was comparable across datasets (Supplementary Fig. 2). Although there were statistically significant differences, all differences in average insulin across all glucose ranges were less than half a unit and therefore not clinically significant (Supplementary Fig. 2). DISCUSSION In this study we have developed GLUCOSE, a distributional offline RL based algorithm, that dynamically suggests personalized regular insulin dosing for patients in the first day after cardiac surgery. The algorithm was validated on an independent multicenter dataset and further demonstrated its robustness and safety through rigorous human evaluations. Hyperglycemia early after cardiac surgery is associated with higher rates of post-operative infections ( 4 , 5 ), acute kidney injury ( 3 , 6 – 8 ), cardiac arrhythmias ( 3 ), longer length of stay ( 3 ), and higher mortality ( 6 – 9 ). This underscores the importance of glucose control in the early post-operative period. Moreover, research indicates that the harmful effects of hyperglycemia are dose-dependent, with longer exposure and higher glucose levels leading to worse outcomes ( 30 ). Therefore, it is essential to manage both the severity and duration of hyperglycemia. The STS recommends maintaining blood glucose levels below 180mg/dL after cardiac surgery ( 10 ). To achieve this target, most cardiac surgery centers employ institutional protocols for managing hyperglycemia ( 31 – 34 ). However, a significant challenge in early postoperative glucose management is that insulin, the primary treatment for hyperglycemia, has a narrow therapeutic window ( 35 ). Since these protocolized regimens often fail to account for individual patient variability in real-world settings ( 13 , 36 ), hypoglycemia becomes a significant risk, particularly with intensive insulin dosing schemes ( 37 , 38 ). Hypoglycemia, defined as a blood glucose level <70mg/dL, can trigger increased sympathetic activity leading to increased heart rate or arrhythmias ( 39 ), impairment of autonomic cardiac reflexes ( 40 ), poor neurological outcomes ( 41 ), and death ( 12 ). This hypoglycemia is seen in 5-21% patients after cardiac surgery ( 11 , 12 , 37 ) prompting a more conservative insulin dosing which, in turn, can result in persistent hyperglycemia. Thus, not surprisingly, these protocols frequently fall short, with only 15% of patients reaching the recommended glucose levels without hypoglycemia on the first day after surgery ( 11 ), which is the most critical and dynamic period after cardiac surgery. Clinicians review over 1,300 data points per patient each day, making it difficult to effectively use all this information for clinical decision-making ( 42 ). An algorithm that can systematically process and interpret these data points can significantly enhance clinician workflow while improving patient outcomes. The GLUCOSE model addresses this by evaluating over 70 features, such as vasopressor doses, SOFA score, mechanical ventilation needs, past glucose values, and BMI, every hour. It recommends personalized insulin doses that account for the patient’s evolving clinical status. Importantly, the algorithm prioritizes features that are clinically relevant, as reflected in the feature importance analysis. Consistently, the GLUCOSE dosing scheme outperforms traditional clinician-driven dosing strategies in terms of estimated average performance. The TIR for glucose was highest when the administered insulin dose closely matched the model’s recommendations. As the discrepancy between the administered doses and GLUCOSE’s suggested doses increased, the time in range decreased. Notably, when the difference was negative, meaning GLUCOSE recommended less insulin than what was administered, the average glucose level was lower. Conversely, with positive differences—where GLUCOSE suggested more insulin than what was given—the average glucose level was higher. This indicates that aligning insulin doses more closely with GLUCOSE’s recommendations could potentially increase time in range and reduce glucose variability, which is associated with worse clinical outcomes ( 13 , 14 ). Although there have been algorithms developed to assist clinicians in insulin doses, they are mostly limited to simulated settings without any human evaluations, include exclusively patients with diabetes, and none specifically target post-cardiac surgery patients ( 43 – 45 ). To ensure clinical applicability and acceptability of our study we performed a comprehensive 3-phase human evaluation inspired by prior work ( 43 ), which is a significant strength of our study. The results of our multi-phased human evaluation underscore the clinical robustness and reliability of the GLUCOSE algorithm in guiding insulin dosing for post-cardiac surgery patients. The significant reduction in MAE achieved by GLUCOSE compared to observed clinician dosing across both internal and external datasets highlights the algorithm’s agreement with rigorous clinical evaluation. Particularly noteworthy is GLUCOSE’s performance in the final phase of the evaluation, where it was assessed by senior intensivists on safety, effectiveness, and acceptability. The algorithm was either comparable to or exceeded the standards set by experienced clinicians, including senior cardiac surgery intensivists. It is important to note, that unlike GLUCOSE, which only had access to patient data till each current time-step, the clinicians that performed human evaluations had access to the entire patient time series of data. This made their approach nearly optimal, against which GLUCOSE’s performance was measured. In reality, clinicians also only have access to data up to the current time step, making GLUCOSE ‘s performance particularly notable in this context. This suggests that GLUCOSE can provide a valuable tool in the management of hyperglycemia in this critically ill patient population, offering a level of reliability and clinical applicability that is on par with traditional human-driven dosing strategies. The ability of GLUCOSE to maintain high performance across diverse clinical scenarios further supports its potential integration into clinical practice, where it could enhance patient outcomes by reducing variability in insulin dosing and minimizing the risks associated with both hyperglycemia and hypoglycemia. Incorporation of distributional RL is another significant strength of this study. Even among patients with seemingly similar clinical profiles, there can be considerable variation in physiological responses. Distributional RL is particularly well-suited to address this challenge, as it quantifies the intrinsic uncertainty within a Markov Decision Process (MDP), which is characteristic of stochastic environments ( 18 ). By learning to approximate the distribution of potential outcomes, this approach strengthens the model by preparing it to handle the inherent uncertainties of real-world clinical settings. While GLUCOSE demonstrates significant potential, several limitations should be considered when interpreting these results. First, although our retrospective study strongly supports the use of GLUCOSE as a clinical decision support tool, these findings require validation through prospective studies and clinical trials involving large and diverse patient cohorts. Second, the GLUCOSE model has been trained, tested, and externally validated only for the first day following cardiac surgery. Though this is the most dynamic time-period for patients after cardiac surgery, expanding this work to evaluate insulin regimens over longer postoperative periods would further enhance GLUCOSE’s clinical utility. Third, our current algorithm does not incorporate dietary data. In practice, precise dietary data is seldom collected, and cardiac surgery patients often have restricted dietary intake during the first postoperative day, reducing the relevance of this variable. Prior work has also shown that RL algorithms can achieve successful blood glucose control without explicit dietary information ( 44 ). GLUCOSE learns underlying patterns associated with meals rather than relying on direct meal data, making its implementation more practical by reducing the need for collecting non-routine, precise dietary information. Finally, to ensure accurate training and validation of the model, we restricted ourselves to patients that had accurately documented doses of medications such as insulin, given that it was the action, and vasopressors/inotropes, which indicate the risk of disease severity and thus may portend a higher risk of hyperglycemia, in both development and external validation cohorts. With this, we did not need to exclude any patients in MIMIC-IV, but had to exclude 4,838 patients in eICU-CRD dataset. This missingness in eICU-CRD is a known issue with the dataset ( 29 ), but with patients from over 200 hospitals, it is a highly heterogenous dataset and thus remains impactful for external validation. To ensure that our model’s recommendations still generalize appropriately in the excluded subset of the external validation dataset we assessed the distribution of recommendations of the model in the subset with just ambigious insulin data, and in the subset with ambiguous data about both insulin and vasopressor/inotropes. We found that the distribution of recommendations was very similar in the 3 groups, with no clinically meaningful differences in actions, which suggests good generalizability of the model. In summary, we have developed and externally validated GLUCOSE, a distributional RL based model to dynamically optimize glucose management in cardiac surgery patients. The comprehensive three-phase human evaluations support GLUCOSE’s clinical robustness and safety, demonstrating its effectiveness in real-world settings and its performance on par with or surpassing that of experienced clinicians. Future studies should be focused on randomized controlled trials to further evaluate the effectiveness and safety of GLUCOSE in diverse clinical settings. METHODS Study Design and Databases For this retrospective study, we used the MIMIC-IV database ( 20 ) to develop the GLUCOSE algorithm (Development dataset). MIMIC-IV is a single-center database constructed from deidentified ICU admissions at the Beth Israel Deaconess Medical Center from 2008-2019. We externally validated the derived policy using the heterogeneous eICU Collaborative Research Database (eICU-CRD) ( 21 ) (External Validation Dataset). eICU-CRD is constructed from over 200,000 de-identified admissions to 208 United States hospitals between 2014 and 2015. Study Population We included all adult patients (age ≥ 18 years) who were admitted to ICU after cardiac surgery. We used ICD-9-PCS and ICD-10-PCS codes to identify patients who underwent cardiac surgery in MIMIC-IV database (Supplementary Table 3). The eICU-CRD database does not include ICD-9-PCS or ICD-10-PCS procedure codes. As per prior literature ( 21 ), we have identified patients admitted to the ICU after cardiac surgery using the “admissiondx” table that provides the primary diagnosis for ICU admissions (Supplementary Table 4). We excluded patients who died within first 24 hours of ICU admission, had ambiguous medication administration information such that it did not allow us to calculate the exact dose of medication administered, or did not have available glucose levels within first three hours of documented ICU admission time after surgery. As our focus was to develop a policy to personalize the administration of regular insulin, we excluded patients who received other short acting insulins (aspart, lispro, NPH, insulin 70/30) (Supplementary Fig. 3). Feature Extraction and Preprocessing We extracted information about patient demographics (age, sex, race), comorbidities (history of diabetes, hypertension, end stage renal disease, chronic obstructive pulmonary disease, asthma, prior myocardial infarction, congestive heart failure, Elixhauser comorbidity score), laboratory values (complete blood count, comprehensive metabolic panel, coagulation studies, and blood gases), vital signs (systolic blood pressure, diastolic blood pressure, mean arterial pressure, heart rate, respiratory rate, temperature, and oxygen saturation), vasopressor and inotrope doses, mechanical ventilation status, and SOFA scores. We extracted the data as multidimensional discrete time series in 1-h time intervals, with features summed or averaged as clinically appropriate. We excluded features with over 30% missingness. In line with standard approach to handling missingness in these data, we used forward fill imputation for all features with k -nearest neighbor ( k -NN) imputation ( k =5) to impute any remaining missing data ( 23 , 46 ). Only the first 24 hours of data for each patient was utilized. All features were checked for outliers using a frequency histogram and descriptive statistics. Errors were corrected as appropriate, such as conversion of temperature to Fahrenheit to Celsius. The full feature list can be found in the Supplement Table 1. All features across all datasets were normalized into range [0, 1] based on the training set to improve training stability. Our outcome was appropriate glucose control, defined as an hourly glucose level between 70-180mg/dL ( 10 , 47 ), in the first day after cardiac surgery. Consequently, we began recording timesteps from the availability of the first glucose level measurement after admission to ICU. Computational Modeling We used conservative Q learning (CQL), a state-of-the-art offline RL algorithm that allows model to suggest clinical actions while regularizing the learned policy to mitigate overestimation in low-coverage or out-of-distribution state-action pairs ( 48 ). To further enhance the model’s understanding of uncertainty and risk, we integrated CQL with distributional RL, an approach that captures the entire distribution of returns rather than just the expected return. All models use a multi-layer perceptron (MLP) network with 3 512-dimension hidden layers. This integration is crucial for making safe and effective decisions especially with clinical actions that have a narrow therapeutic index, such as insulin dosing. To achieve this, we incorporated Implicit Quantile Networks (IQN) into CQL, leveraging the strengths of distributional RL to better model the variability in patient responses, thereby improving the robustness of GLUCOSE ( 19 ). Unlike other distributional methods that require explicitly defining the number of quantiles, IQN implicitly models the entire return distribution, providing a more comprehensive representation of potential outcomes while improving upon its non distributional counterparts ( 17 – 19 ). To the best of our knowledge, this is the first application of integration of CQL with distributional Q functions in healthcare. Finally, we implemented a batch training sampling strategy for offline RL, which avoids overregularization by low-return actions, allowing the learned policy to reflect more high-return trajectories ( 49 ). State Space RL typically considers problems as MDPs. An MDP can be represented as a tuple of ( s t , a, r, s t+1 ) for each time step t . Here, s t represents a vector observation of features at that hour index t , and s t+1 represents the the state at the next hour index after taking action a. The reward, r , is given for taking action a at state s t . We used the features derived from demographics, comorbidities, laboratory values, vital signs, medications, mechanical ventilation, and SOFA scores binned into hourly time-steps to develop the state space. Based on previous literature, we incorporated the prior four hours of glucose values, when available, into the RL model ( 50 ). To provide additional context, we included information on glucose level changes during this period and calculated the ratio of glucose change to insulin dose for each hour, with the minimum insulin dose set at 0.1 for this calculation. Action Space In our offline RL model, actions are defined as the amount of regular insulin administered each hour, utilizing a continuous action space. For ease of interpretability, we have rounded the recommended insulin doses to the nearest integer. This practice aligns the model’s recommendations with practical clinical standards and facilitates the clinical implementation of suggested doses. Additionally, we capped the insulin doses recommended by GLUCOSE at a maximum of 10 units per hour. This threshold is more than two standard deviations above the mean administered dose of 2.2 units in MIMIC-IV. We selected this cap due to both the low coverage of higher doses in the training data and to mitigate potential safety risks associated with recommending excessively high doses. Reward We designed our reward to maximize optimal glycemic control while strongly discouraging behavior that would result in both hypoglycemia (glucose 180 mg/dL). We provided a maximum reward of +0.2 within the 140-180 mg/dL range and penalties become increasingly negative, down to to -1, outside that range. We chose a reward with relatively low magnitude to improve training stability, and a relatively negative reward to disincentive any out of range values ( 51 ). The reward is outlined in Equation 1 : To ensure the safety of insulin dosing recommendations, considering insulin’s narrow therapeutic window, we implemented an exponentially increasing penalty that serves to discourage large overcorrections and promotes more cautious dosing adjustments ( 52 ): GLUCOSE Model Training and External Validation To train the policies, we first split the development dataset into 85% training and 15% internal testing sets. Since RL is particularly subject to high stochastic training variation ( 22 ), we sought to mitigate sampling and stochastic biases by training multiple models on subsampled 80% splits of the training set. Each training run and sampling split utilized a unique seed to ensure different training sets and distinct sampling order while maintaining reproducibility. We continued training models until substantial and significant improvements over the clinician policies were observed in the 15% internal testing set. The final model (GLUCOSE) was selected as the one that had the highest lower bound of the 95% CI for estimated performance returns in the internal testing set ( 23 ). We then evaluate GLUCOSE on the external validation set ( Fig. 1 ). Training was conducted in batches of 256, with actor and critic learning rates of 1e -4 and 3e -4 , respectively. The discount factor γ was set at 0.67, corresponding to a 3 hour effective horizon (calculated as 1/(1-γ)). Discount factors are problem specific, and the choice of a lower discount factor is critical in the context of this problem. Higher discount factors, such as those exceeding 0.95, extend the effective horizon beyond the episode length, resulting in future rewards being weighted nearly as heavily as immediate rewards. Glucose levels can change rapidly, which could lead to suboptimal policy development as the model may either overly prioritize distant future rewards unaffected by the current state or become insensitive to immediate low reward states. Using a lower discount factor aligns the temporal focus of the model to ensure it remains responsive to rapidly changing glucose levels. A dropout rate of p =0.1 was applied during training to improve policy generalization. All models were implemented in Python 3.8.2 using d3rlpy ( 53 ). Off-Policy Evaluation We used fitted-Q-evaluation (FQE) for offline policy estimation (OPE) to estimate the performance of the learned policies ( 26 , 27 ). FQE is effective in handling large policy deviations from observed behaviors as well as stochasticity, and it has shown consistency and calibration in various healthcare-specific benchmarks ( 54 , 55 ). Bootstrapping was applied across all episodes in the datasets to generate 95% confidence intervals by sampling with replacement 10,000 times. We then estimated the performance using FQE on both internal testing set and external validation dataset. We further explored policy performance by analyzing the time in range (TIR) (70-180 mg/dL) in relation to the difference of GLUCOSE’s dosing recommendations and clinician-administered doses ( Fig. 2 ). We calculated the cumulative differences as the model’s predicted insulin doses minus the observed insulin doses over the first 24 hours of ICU stay: Cumulative differences are positive when the RL model recommends more insulin than what was administered, while negative differences indicate that the model predicted a lower insulin dose compared to what was observed. Estimation of Feature Importance The interpretability of machine learning models is crucial in clinical care, where the rationale behind a model’s predictions must be clear to ensure patient safety and informed decision-making. SHAP, a method grounded in cooperative game theory, assesses the contribution of each feature to a model’s prediction by analyzing all possible combinations of feature values ( 28 ). In this study, we employ Permutation SHAP to estimate these contributions, as it provides a model-agnostic framework for elucidating model outputs. Human Evaluations We further assessed the clinical validity of GLUCOSE in three separate phases of human evaluations. In the first phase, we compared the hourly insulin dosing recommended by GLUCOSE to that administered to patients in both internal testing and external validation datasets. Two senior endocrinologists (RS, AG), each with over ten years of clinical experience, provided their own hourly insulin dosing recommendations for 10 randomly selected patients in each cohort. Using the average hourly insulin doses recommended by endocrinologists as reference, we compared the insulin doses recommended by GLUCOSE to those administered to these patients. In the second phase, we had two senior cardiac intensivists (>5 years’ experience) (PM, VN), two junior cardiac intensivists (<5 years’ experience) (DP, JG), and two cardiac intensive care unit nurse practitioners (AC, DG) provide their recommendations for hourly insulin doses in the first day after cardiac surgery for the same patients. We then compared the GLUCOSE recommended doses to those recommended by these clinicians, again using the average endocrinologist recommendations as references. In the third phase, a panel of two senior intensivists (AS, GK) conducted a blinded evaluation of GLUCOSE against other clinician recommended insulin dosing schemes. Senior intensivists were selected for this phase because, in practice, these frontline clinicians are frequently responsible for making rapid decisions regarding glucose control in critically ill patients. They evaluated each dosing scheme for each patient using the following 5-point Likert scale questions: 1. Q1 (Safety) - How much risk for hypoglycemia does this regimen put the patient at? 1. Very high risk 2. High risk 3. Medium risk 4. Low risk 5. Minimal risk 2. Q2 (Effectiveness) - How effective is this regimen in bringing the glucose level within an acceptable range? 1. Not effective at all 2. Slightly effective 3. Moderately effective 4. Very effective 5. Extremely effective 3. Q3 (Acceptability) - How acceptable would this regimen be to you in clinical settings? 1. Strongly unacceptable 2. Unacceptable 3. Neutral 4. Acceptable 5. Strongly acceptable Evaluation of Model Recommendations in Excluded Patient Subsets We also evaluated the GLUCOSE ‘s recommendations using the part of external validation dataset that was excluded from the primary analysis due to presence of ambiguous insulin administration data, which prevented us from making direct comparisons or calculating OPE. Patients for which exact doses of insulin, vasopressors, or inotropes could not be resolved were separated and underwent the same exclusion criteria and preprocessing used for the primary external validation cohort. Any ambiguous data was zero-filled. Statistical Analysis We performed comparisons of categorical features using Chi-squared test and continuous features using t-test and Kruskal-Wallis test, as appropriate. All significance levels were set at α=0.05. To compare insulin doses administered by clinicians and GLUCOSE before hypo- and hyperglycemic episodes, we used Mann-Whitney U tests given the skewed distributions. To evaluate the accuracy of the insulin dosing schemes, we calculated the mean absolute error (MAE) between the insulin doses recommended by various dosing schemes to those provided by endocrinologists. MAEs were determined by subtracting the endocrinologists’ recommendations from the doses suggested by clinicians and the GLUCOSE system for each hourly dose administered or recommended for the 20 retrospectively reviewed patients. We then performed a t-test to identify any significant differences in MAE between the observed clinicians’ dosing and the endocrinologist’s suggested dosing, and MAE between GLUCOSE’s suggested dosing and the endocrinologist’s suggested dosing. To assess differences in the average insulin doses across subsets of excluded patients, we used ANOVA tests for group-wise assessment and two sided t-tests for pairwise assessment. All statistical tests were conducted using Python 3.8.2 using SciPy ( 56 ). Data Availability All datasets are available online as below - Study uses data from MIMIC-IV ( https://physionet.org/content/mimiciv/3.1/ ) and eICU-CRD ( https://eicu-crd.mit.edu ) datasets ACKNOWLEDGEMENTS This study was supported by NIH/NIDDK grant 5K08DK131286 (AS). This work was supported in part through the computational and data resources and staff expertise provided by Scientific Computing and Data at the Icahn School of Medicine at Mount Sinai and supported by the Clinical and Translational Science Awards (CTSA) grant UL1TR004419 from the National Center for Advancing Translational Sciences. Research reported in this publication was also supported by the Office of Research Infrastructure of the National Institutes of Health under award number S10OD026880 and S10OD030463. The content is solely the responsibility of the authors and does not necessarily represent the official views of the National Institutes of Health. REFERENCES 1. ↵ Najmaii S , Redford D , Larson DF . Hyperglycemia as an effect of cardiopulmonary bypass: intra-operative glucose management . J Extra Corpor Technol . 2006 ; 38 ( 2 ): 168 – 73 . OpenUrl PubMed 2. ↵ Galindo RJ , Fayfman M , Umpierrez GE . Perioperative Management of Hyperglycemia and Diabetes in Cardiac Surgery Patients . Endocrinol Metab Clin North Am . 2018 ; 47 ( 1 ): 203 – 22 . OpenUrl PubMed 3. ↵ Moorthy V , Sim MA , Liu W , Chew STH , Ti LK . Risk factors and impact of postoperative hyperglycemia in nondiabetic patients after cardiac surgery: A prospective study . Medicine (Baltimore ). 2019 ; 98 ( 23 ): e15911 . OpenUrl PubMed 4. ↵ Lowden E , Schmidt K , Mulla I , Andrei AC , Cashy J , Oakes DJ , et al. EVALUATION OF OUTCOMES AND COMPLICATIONS IN PATIENTS WHO EXPERIENCE HYPOGLYCEMIA AFTER CARDIAC SURGERY . Endocr Pract . 2017 ; 23 ( 1 ): 46 – 55 . OpenUrl PubMed 5. ↵ Greco G , Ferket BS , D’Alessandro DA , Shi W , Horvath KA , Rosen A , et al. Diabetes and the Association of Postoperative Hyperglycemia With Clinical and Economic Outcomes in Cardiac Surgery . Diabetes Care . 2016 ; 39 ( 3 ): 408 – 17 . OpenUrl Abstract / FREE Full Text 6. ↵ Ruan J , Huang W , Jiang J , Hu C , Li Y , Peng Z , et al. Association between hyperglycemia at ICU admission and postoperative acute kidney injury in patients undergoing cardiac surgery: Analysis of the MIMIC-IV database . Journal of Intensive Medicine . 2024 . 7. Schmeltz LR , DeSantis AJ , Thiyagarajan V , Schmidt K , O’Shea-Mahler E , Johnson D , et al. Reduction of surgical mortality and morbidity in diabetic patients undergoing cardiac surgery with a combined intravenous and subcutaneous insulin glucose management strategy . Diabetes Care . 2007 ; 30 ( 4 ): 823 – 8 . OpenUrl Abstract / FREE Full Text 8. ↵ Ascione R , Rogers CA , Rajakaruna C , Angelini GD . Inadequate blood glucose control is associated with in-hospital mortality and morbidity in diabetic and nondiabetic patients undergoing cardiac surgery . Circulation . 2008 ; 118 ( 2 ): 113 – 23 . OpenUrl Abstract / FREE Full Text 9. ↵ Giakoumidakis K , Nenekidis I , Brokalaki H . The correlation between peri-operative hyperglycemia and mortality in cardiac surgery patients: a systematic review . Eur J Cardiovasc Nurs . 2012 ; 11 ( 1 ): 105 – 13 . OpenUrl CrossRef PubMed 10. ↵ Lazar HL , McDonnell M , Chipkin SR , Furnary AP , Engelman RM , Sadhu AR , et al. The Society of Thoracic Surgeons practice guideline series: Blood glucose management during adult cardiac surgery . Ann Thorac Surg . 2009 ; 87 ( 2 ): 663 – 9 . OpenUrl CrossRef PubMed Web of Science 11. ↵ Williams JB , Peterson ED , Albrecht ÁS , Li S , Hirji SA , Ferguson T , et al. Glycemic control in patients undergoing coronary artery bypass graft surgery: Clinical features, predictors, and outcomes . Journal of Critical Care . 2017 ; 42 : 328 – 33 . OpenUrl PubMed 12. ↵ Johnston LE , Kirby JL , Downs EA , LaPar DJ , Ghanta RK , Ailawadi G , et al. Postoperative Hypoglycemia Is Associated With Worse Outcomes After Cardiac Operations . Ann Thorac Surg . 2017 ; 103 ( 2 ): 526 – 32 . OpenUrl PubMed 13. ↵ Martinez M , Santamarina J , Pavesi A , Musso C , Umpierrez GE . Glycemic variability and cardiovascular disease in patients with type 2 diabetes . BMJ Open Diabetes Res Care . 2021 ; 9 ( 1 ). 14. ↵ Rodbard D . Glycemic variability: measurement and utility in clinical medicine and research--one viewpoint . Diabetes Technol Ther . 2011 ; 13 ( 11 ): 1077 – 80 . OpenUrl CrossRef PubMed 15. ↵ Kaelbling LP , Littman ML , Moore AW . Reinforcement learning: A survey . Journal of artificial intelligence research . 1996 ; 4 : 237 – 85 . OpenUrl 16. ↵ Levine S , Kumar A , Tucker G , Fu J . Offline reinforcement learning: Tutorial, review, and perspectives on open problems . arXiv preprint arXiv : 200501643 . 2020 . 17. ↵ Bellemare MG , Dabney W , Munos R , editors. A distributional perspective on reinforcement learning. International conference on machine learning ; 2017 : PMLR . 18. ↵ Dabney W , Rowland M , Bellemare M , Munos R , editors. Distributional reinforcement learning with quantile regression. Proceedings of the AAAI conference on artificial intelligence ; 2018 . 19. ↵ Dabney W , Ostrovski G , Silver D , Munos R , editors. Implicit quantile networks for distributional reinforcement learning. International conference on machine learning ; 2018 : PMLR . 20. ↵ Johnson AEW , Bulgarelli L , Shen L , Gayles A , Shammout A , Horng S , et al. MIMIC-IV, a freely accessible electronic health record dataset . Scientific Data . 2023 ; 10 ( 1 ): 1 . OpenUrl CrossRef PubMed 21. ↵ Pollard TJ , Johnson AEW , Raffa JD , Celi LA , Mark RG , Badawi O . The eICU Collaborative Research Database, a freely available multi-center database for critical care research . Scientific Data . 2018 ; 5 ( 1 ): 180178 . OpenUrl PubMed 22. ↵ Fujimoto S , Gu SS . A minimalist approach to offline reinforcement learning . Advances in neural information processing systems . 2021 ; 34 : 20132 – 45 . OpenUrl 23. ↵ Komorowski M , Celi LA , Badawi O , Gordon AC , Faisal AA . The Artificial Intelligence Clinician learns optimal treatment strategies for sepsis in intensive care . Nature Medicine . 2018 ; 24 ( 11 ): 1716 – 20 . OpenUrl CrossRef PubMed 24. ↵ Peine A , Hallawa A , Bickenbach J , Dartmann G , Fazlic LB , Schmeink A , et al. Development and validation of a reinforcement learning algorithm to dynamically optimize mechanical ventilation in critical care . npj Digital Medicine . 2021 ; 4 ( 1 ): 32 . OpenUrl PubMed 25. ↵ Lee H , Yoon H-K , Kim J , Park JS , Koo C-H , Won D , et al. Development and validation of a reinforcement learning model for ventilation control during emergence from general anesthesia . npj Digital Medicine . 2023 ; 6 ( 1 ): 145 . OpenUrl PubMed 26. ↵ Le H , Voloshin C , Yue Y , editors. Batch policy learning under constraints. International Conference on Machine Learning ; 2019 : PMLR . 27. ↵ Paine TL , Paduraru C , Michi A , Gulcehre C , Zolna K , Novikov A , et al. Hyperparameter selection for offline reinforcement learning . arXiv preprint arXiv : 200709055 . 2020 . 28. ↵ Lundberg SM , Lee S-I . A unified approach to interpreting model predictions . Advances in neural information processing systems . 2017 ; 30 . 29. ↵ Fitzgerald O , Perez-Concha O , Gallego-Luxan B , Rudd L , Jorm L . Curation and description of a blood glucose management and nutritional support cohort using the eICU collaborative research database . medRxiv . 2023 :2023.04. 20.23288845. 30. ↵ Navaratnarajah M , Rea R , Evans R , Gibson F , Antoniades C , Keiralla A , et al. Effect of glycaemic control on complications following cardiac surgery: literature review . J Cardiothorac Surg . 2018 ; 13 ( 1 ): 10 . OpenUrl PubMed 31. ↵ van den Berghe G, Wouters P , Weekers F , Verwaest C , Bruyninckx F , Schetz M , et al. Intensive insulin therapy in critically ill patients . N Engl J Med . 2001 ; 345 ( 19 ): 1359 – 67 . OpenUrl CrossRef PubMed Web of Science 32. Moghissi ES , Korytkowski MT , DiNardo M , Einhorn D , Hellman R , Hirsch IB , et al. American Association of Clinical Endocrinologists and American Diabetes Association consensus statement on inpatient glycemic control . Diabetes Care . 2009 ; 32 ( 6 ): 1119 – 31 . OpenUrl FREE Full Text 33. Wiener RS , Wiener DC , Larson RJ . Benefits and risks of tight glucose control in critically ill adults: a meta-analysis . Jama . 2008 ; 300 ( 8 ): 933 – 44 . OpenUrl CrossRef PubMed Web of Science 34. ↵ Dellinger RP , Levy MM , Carlet JM , Bion J , Parker MM , Jaeschke R , et al. Surviving Sepsis Campaign: international guidelines for management of severe sepsis and septic shock: 2008 . Intensive Care Med . 2008 ; 34 ( 1 ): 17 – 60 . OpenUrl CrossRef PubMed Web of Science 35. ↵ Zaykov AN , Mayer JP , DiMarchi RD . Pursuit of a perfect insulin . Nature Reviews Drug Discovery . 2016 ; 15 ( 6 ): 425 – 39 . OpenUrl CrossRef PubMed 36. ↵ Rodbard D . Glycemic Variability: Measurement and Utility in Clinical Medicine and Research—One Viewpoint . Diabetes Technology & Therapeutics . 2011 ; 13 ( 11 ): 1077 – 80 . OpenUrl CrossRef PubMed 37. ↵ Umpierrez G , Cardona S , Pasquel F , Jacobs S , Peng L , Unigwe M , et al. Randomized controlled trial of intensive versus conservative glucose control in patients undergoing coronary artery bypass graft surgery: GLUCO-CABG trial . Diabetes care . 2015 ; 38 ( 9 ): 1665 – 72 . OpenUrl Abstract / FREE Full Text 38. ↵ Investigators N-SS . Intensive versus conventional glucose control in critically ill patients . New England Journal of Medicine . 2009 ; 360 ( 13 ): 1283 – 97 . OpenUrl CrossRef PubMed Web of Science 39. ↵ Fisher BM , Gillen G , Hepburn DA , Dargie HJ , Frier BM . Cardiac responses to acute insulin-induced hypoglycemia in humans . Am J Physiol . 1990 ; 258 ( 6 Pt 2 ): H1775 – 9 . OpenUrl 40. ↵ Adler GK , Bonyhay I , Failing H , Waring E , Dotson S , Freeman R . Antecedent hypoglycemia impairs autonomic cardiovascular function: implications for rigorous glycemic control . Diabetes . 2009 ; 58 ( 2 ): 360 – 6 . OpenUrl Abstract / FREE Full Text 41. ↵ Mohseni S . Neurologic damage in hypoglycemia . Handb Clin Neurol . 2014 ; 126 : 513 – 32 . OpenUrl CrossRef PubMed 42. ↵ Manor-Shulman O , Beyene J , Frndova H , Parshuram CS . Quantifying the volume of documented clinical information in critical illness . J Crit Care . 2008 ; 23 ( 2 ): 245 – 50 . OpenUrl CrossRef PubMed Web of Science 43. ↵ Wang G , Liu X , Ying Z , Yang G , Chen Z , Liu Z , et al. Optimized glycemic control of type 2 diabetes with reinforcement learning: a proof-of-concept trial . Nature Medicine . 2023 ; 29 ( 10 ): 2633 – 42 . OpenUrl PubMed 44. ↵ Fox I , Lee J , Pop-Busui R , Wiens J , editors. Deep reinforcement learning for closed-loop blood glucose control. Machine Learning for Healthcare Conference ; 2020 : PMLR . 45. ↵ Zhu T , Li K , Herrero P , Georgiou P . Basal glucose control in type 1 diabetes using deep reinforcement learning: An in silico validation . IEEE Journal of Biomedical and Health Informatics . 2020 ; 25 ( 4 ): 1223 – 32 . OpenUrl 46. ↵ Raghu A , Komorowski M , Celi LA , Szolovits P , Ghassemi M , editors. Continuous state-space models for optimal sepsis treatment: a deep reinforcement learning approach. Machine Learning for Healthcare Conference ; 2017 : PMLR . 47. ↵ Jacobi J , Bircher N , Krinsley J , Agus M , Braithwaite SS , Deutschman C , et al. Guidelines for the use of an insulin infusion for the management of hyperglycemia in critically ill patients . Crit Care Med . 2012 ; 40 ( 12 ): 3251 – 76 . OpenUrl CrossRef PubMed Web of Science 48. ↵ Kumar A , Zhou A , Tucker G , Levine S . Conservative q-learning for offline reinforcement learning . Advances in Neural Information Processing Systems . 2020 ; 33 : 1179 – 91 . OpenUrl 49. ↵ Hong Z-W , Agrawal P , Combes RTd, Laroche R . Harnessing mixed offline reinforcement learning datasets via trajectory weighting . arXiv preprint arXiv:230613085. 2023 . 50. ↵ Emerson H , Guy M , McConville R . Offline reinforcement learning for safer blood glucose control in people with type 1 diabetes . Journal of Biomedical Informatics . 2023 ; 142 : 104376 . OpenUrl PubMed 51. ↵ Van Hasselt HP , Guez A , Hessel M , Mnih V , Silver D . Learning values across many orders of magnitude . Advances in neural information processing systems . 2016 ; 29 . 52. ↵ Gu W , Wang S . An Improved Strategy for Blood Glucose Control Using Multi-Step Deep Reinforcement Learning . arXiv preprint arXiv:240307566. 2024 . 53. ↵ Seno T , Imai M . d3rlpy: An offline deep reinforcement learning library . Journal of Machine Learning Research . 2022 ; 23 ( 315 ): 1 – 20 . OpenUrl 54. ↵ Voloshin C , Le HM , Jiang N , Yue Y . Empirical study of off-policy policy evaluation for reinforcement learning . arXiv preprint arXiv:191106854. 2019 . 55. ↵ Tang S , Wiens J , editors. Model selection for offline reinforcement learning: Practical considerations for healthcare settings . Machine Learning for Healthcare Conference ; 2021 : PMLR. 56. ↵ Virtanen P , Gommers R , Oliphant TE , Haberland M , Reddy T , Cournapeau D , et al. SciPy 1.0: fundamental algorithms for scientific computing in Python . Nature methods . 2020 ; 17 ( 3 ): 261 – 72 . OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted January 02, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following GLUCOSE: A Distributional Reinforcement Learning Model for Optimal Glucose Control After Cardiac Surgery Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share GLUCOSE: A Distributional Reinforcement Learning Model for Optimal Glucose Control After Cardiac Surgery Jacob M. Desman , Zhang-Wei Hong , Moein Sabounchi , Ashwin S. Sawant , Jaskirat Gill , Ana C Costa , Gagan Kumar , Rajeev Sharma , Arpeta Gupta , Paul McCarthy , Veena Nandwani , Doug Powell , Alexandra Carideo , Donnie Goodwin , Sanam Ahmed , Umesh Gidwani , Matthew Levin , Robin Varghese , Farzan Filsoufi , Robert Freeman , Avniel Shetreat-Klein , Alexander W Charney , Ira Hofer , Lili Chan , David Reich , Patricia Kovatch , Roopa Kohli-Seth , Monica Kraft , Pulkit Agrawal , John A. Kellum , Girish N. Nadkarni , Ankit Sakhuja medRxiv 2025.01.01.25319851; doi: https://doi.org/10.1101/2025.01.01.25319851 Share This Article: Copy Citation Tools GLUCOSE: A Distributional Reinforcement Learning Model for Optimal Glucose Control After Cardiac Surgery Jacob M. Desman , Zhang-Wei Hong , Moein Sabounchi , Ashwin S. Sawant , Jaskirat Gill , Ana C Costa , Gagan Kumar , Rajeev Sharma , Arpeta Gupta , Paul McCarthy , Veena Nandwani , Doug Powell , Alexandra Carideo , Donnie Goodwin , Sanam Ahmed , Umesh Gidwani , Matthew Levin , Robin Varghese , Farzan Filsoufi , Robert Freeman , Avniel Shetreat-Klein , Alexander W Charney , Ira Hofer , Lili Chan , David Reich , Patricia Kovatch , Roopa Kohli-Seth , Monica Kraft , Pulkit Agrawal , John A. Kellum , Girish N. Nadkarni , Ankit Sakhuja medRxiv 2025.01.01.25319851; doi: https://doi.org/10.1101/2025.01.01.25319851 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Cardiovascular Medicine Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4436) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15229) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6600) Geriatric Medicine (668) Health Economics (997) Health Informatics (4538) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (542) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15917) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3333) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9232) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a0105e5c8b784193',t:'MTc3OTY2ODM2Ng=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();