Staffable ICU early-warning with calibrated multi-centre validation and alert-budget guardrails

Staffable ICU early-warning with calibrated multi-centre validation and alert-budget guardrails | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Staffable ICU early-warning with calibrated multi-centre validation and alert-budget guardrails Sanwal Ahmad Zafar , Wei Qin , Toufique Ahmed Soomro , Areeba Ali Khan , Muhammad Salman Faisal doi: https://doi.org/10.1101/2025.10.30.25339143 Sanwal Ahmad Zafar 1 School of Mechanical Engineering, Shanghai Jiao tong University , Shanghai, China Find this author on Google Scholar Find this author on PubMed Search for this author on this site Wei Qin 1 School of Mechanical Engineering, Shanghai Jiao tong University , Shanghai, China Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: wqin{at}sjtu.edu.cn Toufique Ahmed Soomro 2 Artificial Intelligence and Cyber Futures Institute, Charles Sturt University , Bathurst, NSW, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site Areeba Ali Khan 3 Services Institute of Medical Sciences , Lahore, Pakistan Find this author on Google Scholar Find this author on PubMed Search for this author on this site Muhammad Salman Faisal 4 Stephenson Cancer Center, OU Health University, Oklahoma Medical Centre , Oklahoma, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract We present a staffable ICU early-warning system with calibrated multi-centre validation and alert-budget guardrails. A multi-horizon mortality model was developed on a large public ICU database and externally validated across independent hospitals at fixed post-admission landmarks (6/12/18/24 h). Evaluation was prespecified to emphasize actionability: discrimination, calibration (slope/intercept, Brier, ECE), decision-curve net benefit, and operating-point performance. To make deployment resource-aware, we translate threshold choice into alerts per 100 admissions and map alerts to clinician review time and net monetary benefit (NMB), yielding guardrails that bound workload while preserving utility. We further provide a silent-trial readout that estimates lead time prior to activation, supporting governance and cautious piloting. Across centres and horizons, the model maintained strong discrimination, tight calibration, and positive net benefit at clinically relevant thresholds. Guardrail selection produced workload-constrained operating points with stable subgroup performance. By coupling calibrated multicentre validation with alert-budget guardrails and deployment-proximal readouts, this study reframes ICU early-warning from maximizing AUC to delivering a staffable, value-aware program ready for monitored roll-out and periodic recalibration. Introduction Clinical deterioration remains a major driver of morbidity, mortality, and resource use in the ICU. Machine-learning (ML) early-warning systems (EWS) have shown promising discrimination in retrospective studies, yet bedside usefulness depends as much on calibrated risk and actionable operating points as on rank order performance. Contemporary reporting guidance emphasizes complementing AUROC with calibration, clinical decision utility, and transparent description of intended use and thresholding, together with clear assessments of bias and applicability for ML-based prediction [ 1 , 2 ]. In parallel, health-economic guidance encourages aligning evaluation with the decision context and explicitly considering opportunity costs when models trigger human review or downstream actions [ 3 – 5 ]. Two gaps limit translation. First, most EWS reports do not connect operating thresholds to staffable alert programs or quantify the workload those thresholds imply. Without an alert-budget perspective, even well-calibrated models can overwhelm teams with low-value alerts or underperform if set too conservatively. Second, deployment-proximal evaluation is often missing: before activation, services benefit from “silent-trial” assessment to estimate alert volume, timeliness, and missed events under local case-mix and staffing, and to confirm that thresholds deliver the intended balance of sensitivity, precision, and net clinical value. We externally validate a pragmatic ICU mortality model at fixed landmarks (6, 12, 18, and 24 h), following current reporting and risk-of-bias recommendations [ 1 , 2 ]. We report discrimination, calibration, decision-curve utility, and operating points carried unchanged to external data. We also introduce an alert-budget guardrail mapped to staff time, a concise NMB readout following economic reporting guidance [ 3 , 4 ], and examine temporal stability and site-level synthesis. Additionally, we include SHAP summaries for transparency ( Figure 1 ) . Download figure Open in new tab Figure 1. Sentinel-ICU pipeline overview. Model development workflow showing feature engineering at four horizons (6/12/18/24h), XGBoost modeling with isotonic calibration, dual thresholding strategy (τ γj for accuracy, τ CGT for operational planning), and deployment framework. Key 12h metrics: Brier=0.076, ECE=0.023. Our prespecified outcomes are: (i) discrimination and calibration at each landmark; (ii) decision-curve net benefit and horizon-specific operating-point performance; (iii) an operational readout at the 12-h guardrail linking alert rates to review workload and NMB; (iv) temporal stability and site-level heterogeneity; and (v) subgroup summaries to support equity and governance considerations. Taken together, the study is designed to move beyond “high AUROC” toward staffable, budget-constrained, and auditable ICU decision support, with all assumptions and threshold selection fully explained in Methods and outcomes reported concisely in result section. MATERIALS AND METHODS Operating thresholds and workload mapping We used two thresholds. τ γj (Youden’s J on calibrated risks, fixed from internal validation) for reporting accuracy metrics, and τ CGT (clinical guardrail threshold) set to meet a target alert budget (alerts per 100 admissions). Figure 1 (Thresholds & Operations) illustrates the roles of τ γj and τ CGT We report accuracy at τ γj and operational workload/economics at τ CGT , without model refitting or horizon leakage. Both thresholds were prespecified on the MIMIC-IV internal validation split and then carried unchanged to eICU-CRD; no threshold tuning was performed on external data. Alerts/100 were converted to minutes and clinician-hours/100 using 5-10 min/alert ( Table 1 ). View this table: View inline View popup Download powerpoint Table 1. Operational Summary at 12 h (τ CGT ) Clinical utility, economic evaluation and sub-group performance Clinical utility was summarized by decision-curve net benefit (thresholds 0.01-0.30; treat-all/none comparators) and by a simple net monetary benefit (NMB) scenario ( Figure 3C ). Alert counts were converted to reviewer workload as alerts × minutes per alert (5-10) ( Table 1 ), expressed as clinician-hours per 100 admissions, following CHEERS 2022 reporting guidance [ 3 – 5 ]. This approach follows established workload conversion and staffing equivalence guidelines [ 4 , 6 ]. NMB reported with base-case assumptions; full formula in paragraph S11. At the prespecified 12-h threshold we report sensitivity and PPV (95% CIs) across sex, age, and admission source with the threshold fixed; full tables/definitions are in S12-S13 and Table S5. Software and Reproducible Tools To reduce leakage and overfitting, we froze the feature set and hyperparameters before external testing. We also provide two runnable tools: Supplementary Tool S1, a lightweight FastAPI mortality-inference service and Supplementary Tool S2, an NMB forecaster that maps operating points. Frozen features, hash-locked splits, and repo tools, as well as supplementary tools and endpoints documented in S14. Study Cohorts and External Validation External validation in eICU-CRD used four post-admission windows (6/12/18/24 h) with window-specific inclusion; thresholds, workload conversion, decision-curve/NMB, and silent-trial definitions follow Methods, details in paragraphs S2-S4. We computed discrimination (AUROC/AUPRC), calibration (reliability curves, ECE/Brier, slope/CITL (calibration-in-the-large)), decision-curve net benefit [ 6 ], NMB, and subgroup performance (sex, age, admission type) ( Table 2 , Table 3 ). Figure S2(A) shows eICU AUROC by age subgroup, and Figure S2(B) eICU AUROC by gender subgroups. A silent-trial analysis estimated lead time among alerted events. Lead time was defined a priori as the interval from the first model alert at a given horizon to the recorded time of in-hospital death for true-positive cases; for survivors, lead time is undefined and not analyzed. We included explainability summaries (global SHAP and case-level waterfalls) for transparency only (not used for decisioning) [ 7 ]. Internal data were drawn from MIMIC-IV; external validation used eICU-CRD [ 1 , 8 ]. View this table: View inline View popup Download powerpoint Table 2. External performance by window (eICU) View this table: View inline View popup Download powerpoint Table 3. Prespecified operating-point metrics (95% CI) We conducted a retrospective observational study using de-identified electronic health records from two publicly available critical care databases. Development/internal validation: MIMIC-IV v2.2 (2008–2019) [ 9 , 10 ]; external validation: eICU-CRD (2014–2015) [ 11 , 12 ]. We adhered to each database’s usage guidelines and derived index ICU admissions for adults (≥18 y) with known in-hospital mortality; horizon-specific eligibility (details in paragraph S2). MIMIC-IV contains ICU admissions from a single academic medical center across multiple ICUs with rich, high-frequency clinical measurements and outcomes. eICU-CRD spans 208 ICUs across the United States with diverse settings and practice patterns, enabling assessment of transportability across sites. Cohort construction, feature extraction, and windowed prediction logic were harmonized between development and validation to preserve comparability across 6, 12, 18, and 24 h horizons after ICU admission. Feature importance across all windows is shown in Figure S4. We harmonized units/time-stamps across MIMIC-IV/eICU; full specifications in paragraph S1. Outcomes and Features The binary endpoint was in-hospital mortality. For external validation, outcomes were taken as recorded in eICU-CRD discharge disposition and verified against hospital mortality flags when available. Horizon-specific labels were defined in the standard way: the target is the probability of in-hospital death, conditional on information accrued up to the horizon (6, 12, 18, or 24 h post-ICU admission). Importantly, windows were prospective in construction, no data after the horizon were used to form predictors at that horizon. To reduce label ambiguity, we excluded admissions with missing discharge status or inconsistent mortality indicators. Dataset/source indicators were used only for diagnostics and were excluded from model inputs. Early-ICU vitals/labs and demographics; leakage-prone variables excluded as well as in-fold imputation and no cross-split leakage are detailed in paragraph S3 . Model Development and Validation Preprocessing (median imputation) and any class-imbalance strategy were fit on the training split only. For imbalance, we used either scale-pos-weight tuning or SMOTE on the training matrix (never across splits), with the chosen approach logged per horizon. Predictors were standardized only when needed for baseline comparators (e.g., logistic regression); tree-based models used raw scales. Details of imbalance handled by prespecified rule; and isotonic calibration internally and external calibration are reported in paragraph S5 . Calibration We applied isotonic regression calibration separately for each horizon on the internal validation split and held the calibrated models fixed for external testing. Calibration was assessed in external validation using Brier score, expected calibration error (ECE; equal-frequency bins), and reliability plots. Calibration-in-the-large (CITL) and calibration slope were estimated with bootstrap confidence intervals. No recalibration was performed on eICU; external calibration is reported descriptively. Prespecified threshold set internally and held fixed (paragraph S6 ). Temporal blocks and site pooling performed (details in paragraph S7 ). Baselines Throughout, we enforced leakage guards: (i) exclusion of post-horizon information (e.g., length-of-stay derived fields), (ii) strict in-fold fitting of imputer/SMOTE, (iii) fixed random seeds for splits, and (iv) reporting on external data without any threshold recalibration. We audited feature matrices before training to confirm that forbidden columns (e.g., icu_los_hours) and dataset indicators (source) were absent from X, and we re-checked the released artifacts accordingly. Regularized LR baseline included and leakage audits performed, details in paragraph S8 . Statistical analysis Continuous variables are reported as mean ± SD or median [IQR], and categorical variables as counts (%). Group differences between development and external cohorts were summarized descriptively. Discrimination was measured by AUROC and AUPRC at each horizon, with 95% CIs via nonparametric patient-level bootstrap (10,000 resamples). Because event prevalence is modest, AUPRC is reported alongside AUROC to reflect class imbalance. We summarize horizon-wise performance with a macro-average across 6/12/18/24 h (unweighted). At the defined operating thresholds, we report sensitivity, specificity, PPV, NPV, accuracy, and likelihood ratios with 95% CIs (bootstrap; exact intervals guarded when percentile bounds exceeded [0, 1 ]). Site-level AUROCs were pooled using a random-effects (DerSimonian–Laird) model with inverse-variance weighting; standard errors were derived from reported CIs when available, otherwise approximated via the Hanley–McNeil variance. We report pooled AUROC, 95% CI, I 2 , and number of sites per horizon. RESULTS Patient characteristics We evaluated a windowed ICU mortality model at four horizons, 6, 12, 18, and 24 h after ICU admission, in an external eICU-CRD cohort using window-specific inclusion to prevent horizon leakage. Window-level performance summaries are in Table 2 . External eICU cohorts per horizon with ~12–13% prevalence (see Table S1). Because deployment hinges on workload and timing, we quantified operational impact at the guardrail threshold. At 12 h, the alert rate was 61.4 per 100 admissions (PPV 0.304), translating to 5.1-10.2 clinician-hours per 100 admissions under 5-10 min/alert assumptions, with a median lead time of 138.6 h (IQR 54.6– 286.3) among alerted events; the base-case NMB peaked near τ CGT ( Figure 3D ; Table S4). Primary Performance Discrimination was high across all windows and tapered mildly with longer horizons. Window-level AUROC (AUPRC) was 0.857 (0.546) at 6 h, 0.864 (0.535) at 12 h, 0.835 (0.453) at 18 h, and 0.825 (0.432) at 24 h ( Figure 2A ). Precision–recall curves remained meaningfully above the no-skill baseline (class prevalence ≈12–13%) ( Figure 2C ). Download figure Open in new tab Figure 1a. Model performance in external validation (eICU-CRD). (A) ROC curves by prediction horizon with 95% CIs. (B) Calibration plot at 12h showing observed vs predicted risk. (C) Precision-recall curves; dashed lines indicate outcome prevalence. (D) Site-level AUROC distributions with pooled estimates. Download figure Open in new tab Figure 2. Clinical utility and operational metrics. (A) Decision curves showing net benefit vs treat-all/none strategies. (B) Subgroup calibration at 12h. (C) Net monetary benefit curves under different review time assumptions. (D) Relationship between alert threshold, workload, and PPV; vertical line marks operational guardrail (τ CGT =0.087). Calibration and Clinical Utility Calibration and reliability were acceptable across horizons; at 12 h, CITL and slope are in Table S2, the calibration plot is in Figure 2B , horizon-wise reliability curves (6/12/18/24 h) are in Figure S1, and subgroup reliability (10 equal-frequency bins) is in Figure 3B . Decision curves ( Figure 3A ) showed positive net benefit across clinically relevant thresholds (0.01–0.30); at 12 h, net benefit exceeded treat-all/none in the mid-risk region and remained robust in sensitivity analyses [ 4 , 7 ]. Download figure Open in new tab Figure 3. Temporal stability analysis. Performance metrics (AUROC, Brier score, ECE) across eight equal-count temporal blocks for each prediction horizon, showing stable discrimination without calendar-time degradation. Operating points and Subgroup stability (12 h) The validation threshold τ γj (chosen on calibrated risks in the internal split) was prespecified and carried unchanged to eICU-CRD. For operations, a 12 h guardrail τ CGT targeted a predefined alert budget. In external testing, τ γj was 0.143 (6 h), 0.360 (12 h), 0.250 (18 h), 0.112 (24 h). At 12 h, τ γj yielded sensitivity 0.618, spec 0.915 (PPV 0.505; NPV 0.945; Table S3). At 12 h, threshold-fixed sensitivity/PPV showed modest age-related variation consistent with prevalence; full 95% CIs are in Table S5. Temporal Stability Eight equal-count calendar blocks per horizon showed stable AUROC, Brier, and ECE without monotonic degradation (Figure 4A–D; Table S7; Figure S3). Site-level heterogeneity and Pooled discrimination Random-effects meta-analysis showed consistently strong pooled discrimination across windows (AUROC ≈ 0.83-0.87), with low-moderate heterogeneity at 6-12 h, negligible heterogeneity at 18 h, and higher dispersion at 24 h. The 12 h window had the highest pooled AUROC (0.869, 95% CI 0.857-0.881), whereas the near-zero I 2 at 18 h is compatible with within-site variance alone; the higher I 2 at 24 h likely reflects later-stay missingness and practice variation. Full window-specific estimates (pooled AUROC with 95% CI, k , and I 2 ) are shown in Figure 2D and detailed with site-level forest plots in Table S6. Random-effects pooling (DerSimonian–Laird) follows standard methods as described [ 13 , 14 ]. Operational impact at the Guardrail At the 12 h guardrail (τ CGT = 0.087), the model fired at 61.4 alerts/100 admissions with PPV = 0.304, yielding ~42 true and 96 false alerts per 1,000 admissions (Table S3; Table S4). In silent-trial assessment, alerts preceded event recognition by median 138.6 h (IQR 54.6–286.3), indicating substantial lead time before activation. Assuming 5-10 min review per alert, the review burden is ~5.1–10.2 clinician-hours/100 admissions. Under these assumptions, net monetary benefit (NMB) peaked near τ CGT . This explicit conversion from alert rate to staff time leading to economic proxy aligns with recommendations to verify burden and value during pre-deployment “shadow mode”/silent-trials [ 15 , 16 ]. Subgroup analyses At τ CGT and τ YJ , subgroup performance showed no clinically concerning drop-offs across sex, age bands, or admission type. Given operational salience, we also inspected a payer-type slice (Medicare/Medicaid/Private/Self-pay/Other) as an audit rather than a subgroup-optimized model; no material imbalances in precision or false-negative rates were observed at the guardrail point (Table S8). This aligns with calls to surface fairness-relevant outcomes during evaluation and governance of ICU AI [ 17 ]. Model Interpretation We include global SHAP (Figure S5) and exemplar case waterfalls (Figure S6) for transparency; explanations were not used for alerts or decisions [ 18 , 19 ]. SHAP summaries (Figure S7) provide additional insight into how the model prioritizes variables across different time windows and SHAP dependency plots (Figure S8) illustrate how individual features interact and influence the model’s prediction at different levels. Discussion We reported equity audits and transparency outputs; explanations were not used for decisioning [ 20 , 21 ]. Post-deployment, we will monitor prevalence, risk distributions, AUROC/AUPRC, and calibration (slope/intercept, ECE) on rolling windows. Recalibration will be triggered if AUROC drops by ≥0.03 or ECE increases by ≥0.05 compared with the external-validation baseline. Our subgroup analyses suggest stable sensitivity across sex and clinically interpretable variation across age bands and admission sources at a fixed, prespecified threshold . PPV differences tracked subgroup prevalence rather than model collapse. Across 6, 12, 18, and 24 h windows, discrimination sat within a narrow, clinically strong band, calibration was consistently tight by Brier and ECE, and decision-curve analysis showed positive net benefit across clinically plausible thresholds [ 21 ] ( Table 3 ) . Operating-point behavior aligned with clinical intuition: earlier windows favored sensitivity and NPV, whereas the 12 h window favored specificity and PPV. For example, at 6 h (τ γj = 0.143) sensitivity was0.79 with NPV 0.965; full, window-specific estimates with uncertainty are presented in Results. Together, these findings indicate stable early discrimination with strong rule-out performance and a predictable precision–recall trade-off as horizons lengthen. Using site-stratified meta-analysis on a large multicenter cohort, we observed low-to-moderate heterogeneity at 6 h and 12 h, minimal heterogeneity at 18 h, and moderate heterogeneity at 24 h; even at 24 h, pooled discrimination remained clinically strong (Table S6) . These data support cautious adoption with prospective monitoring and routine recalibration [ 13 ]. Temporal behavior was stable. Window-specific panels of AUROC, Brier, and ECE across equal-count temporal blocks showed no monotonic degradation; fluctuations were modest and centered on window means, suggesting that case-mix shifts during the evaluation period did not materially distort performance. The mild rise in Brier with longer horizons is consistent with greater outcome uncertainty as more future events are anticipated, while low ECE values indicate acceptable probability calibration in external validation. Taken together, small Brier and low ECE support using the probabilities without immediate recalibration in similar settings; however, we recommend ongoing monitoring and simple intercept/slope updates as needed. Net-benefit gains clustered in the 5–20% risk band typical of early-warning and escalation workflows. At 6 h, the threshold reflects a conservative, sensitivity-forward stance appropriate when the downside of a missed deterioration is high and interventions are comparatively low risk. By 12 h, a higher threshold increases PPV and specificity, which is desirable when downstream actions (e.g., family meetings, imaging, transfers) have higher opportunity costs. The 18 h and 24 h operating points preserved clinically meaningful detection while tempering avoidable alarms as clinical context evolves. Importantly, bounding alerts per 100 admissions via a guardrail directly addresses the operational burden by aligning notifications with staffing capacity; the silent-trial analysis further supported feasibility, with alerts preceding event recognition by a substantial median lead time. Consistent with Methods, the net monetary benefit (NMB) scenario peaked near the guardrail under the 5-10 min/alert handling assumption; assumptions and sensitivity ranges are detailed in Methods, and outcomes are summarized in Results. Our analysis hews to contemporary reporting norms, emphasizing calibrated risk and threshold-level trade-offs over rank discrimination alone, and adds deployment-oriented elements that clarify what it would cost to run the service and how much signal it would return [ 14 ]. Beyond AUROC, we report Brier/ECE, operating-point metrics with uncertainty, and decision curves; we enforce leakage guards (exclusion of length-of-stay proxies, strict in-fold preprocessing, per-horizon calibration). Unlike single-time-point score validations, the fixed-horizon lens clarifies when the model is most actionable and how thresholds should be tuned to workload and harm preferences at different times in the stay. This study has limitations. It is a retrospective validation on de-identified multicenter data; residual confounding, undocumented workflow differences, and selection effects remain possible. The window definitions are pragmatic and do not capture continuously updated predictions; they should be viewed as deployable checkpoints aligned with admission-to-early-stay workflows. The feature set is constrained to routinely available variables; unmeasured confounders (e.g., clinician gestalt, imaging) may limit ceiling performance. While calibration transferred well, local intercept/slope updates at go-live are prudent, especially where baseline mortality or admission practices differ. Finally, moderate heterogeneity at 24 h warrants local monitoring and, if necessary, site-specific threshold tuning. Clinical implications follow directly. For early screening (6–12 h), sensitivity-oriented thresholds that maximize NPV and net benefit pair well with low-harm, reversible care pathways (enhanced monitoring, early review). For later checkpoints (18–24 h), resource-aware thresholds can reduce avoidable alarms while preserving detection, particularly when capacity is tight. Prospective deployment should include drift monitoring (risk distribution, prevalence, calibration slope/intercept), pre-registered operating points, and lightweight recalibration triggers [ 13 ]. Future work will extend beyond fixed horizons to rolling predictions, add fairness and subgroup stability analyses, and test prospective impact on workflow and outcomes. Conclusions From 6-24 h after ICU admission, the model showed strong discrimination, small calibration error, low-to-moderate site heterogeneity, and consistent decision-curve benefit. Framing outputs as a staffable alert program, via guardrails, workload translation, silent-trial lead time, and a succinct economic view, clarifies what it would cost to run the service and how much signal it returns at a chosen threshold. These properties support careful clinical piloting with routine monitoring and recalibration. Data Availability Source code for the Sentinel-ICU analyses and demo services is available at GitHub ( https://github.com/Sjtu-Fuxilab/Sentinel-API ). An archived snapshot of the exact version used here, including model artifacts, will be deposited on Zenodo and cited with a DOI upon acceptance. Two runnable packages accompany this submission: Supplementary Tool S1, the Sentinel-ICU Mortality Inference API (FastAPI, v1.0), and Supplementary Tool S2, the net monetary benefit (NMB) forecaster app (FastAPI, v2.3). Mirror copies of both tools are hosted in the same GitHub repository. https://github.com/Sjtu-Fuxilab/Sentinel-API Funding Information Research supported by the Interdisciplinary Program of Shanghai Jiao Tong University, China (Project No. YG2025QNA31). Data Availability Data are publicly available on PhysioNet under credentialed access. We used MIMIC-IV v2.2 (DOI: 10.13026/6mm1-ek67) and eICU-CRD v2.0 (DOI: 10.13026/C2WM1R). Please also see the dataset descriptor articles in Scientific Data for MIMIC-IV (2023) and eICU-CRD (2018). PhysioNet platform citation: Goldberger et al., Circulation 2000 [ 22 ]. Code Availability Source code for the Sentinel-ICU analyses and demo services will be available at GitHub ( https://github.com/Sjtu-Fuxilab/Sentinel-API ). An archived snapshot of the exact version used here, including model artifacts, will be deposited on Zenodo and cited with a DOI upon acceptance. Two runnable packages accompany this submission: Supplementary Tool S1 , the Sentinel-ICU Mortality Inference API (FastAPI, v1.0), and Supplementary Tool S2 , the net monetary benefit (NMB) forecaster app (FastAPI, v2.3). Mirror copies of both tools are hosted in the same GitHub repository. Ethics approval and consent to participate This study used de-identified public datasets (MIMIC-IV v2.2 and eICU-CRD). According to our institutional policy, the analysis constitutes non-human subjects research and was exempt from IRB review. No patient consent was required. Authors’ Contributions SAZ conceived and designed the study; acquired and curated the data; implemented the software; performed the analyses; created the figures/tables; and drafted the manuscript. WQ supervised the study, contributed to study design and methodological oversight, provided resources, critically revised the manuscript, and served as the corresponding author ; WQ also secured funding. TAS reviewed the manuscript and contributed to design and methodology. AAK provided clinical interpretation and validation and critically revised the manuscript. MSF provided clinical oversight and resources, and critically revised the manuscript. All authors read and approved the final manuscript . Consent for publication Not applicable. Clinical trial number Not applicable. Competing interests The authors declare that they have no competing interests. Supplementary Information Supplementary Tables S1-S8 and Figures S1-S8 are available online. Abbreviations CI Confidence interval CITL Calibration-in-the-large DCA Decision-curve analysis ED Emergency department ECE Expected calibration error HER Electronic health record FP / FN False positive / false negative ICU Intensive care unit IQR Interquartile range K Number of sites included in a pooled/meta-analysis at a horizon LOS Length of stay NMB Net monetary benefit NPV / PPV Negative / positive predictive value PR curve Precision–recall curve ROC Receiver-operating characteristic SD Standard deviation SE / SP Sensitivity / specificity τ γj Youden’s J on calibrated risks, fixed from internal validation τ CGT Clinical guardrail threshold TP / TN True positive / true negative V, C, W, h Parameters in NMB: value per TP, cost per FP, staff cost/min, review minutes per alert References [1]. ↵ G. S. Collins , K. G. M. Moons , P. Dhiman , and et al. , “ TRIPOD+AI statement: updated guidance for reporting clinical prediction models that use regression or machine learning methods ,” BMJ , vol. 385 , p. e078378 , 2024 , doi: 10.1136/bmj-2023-078378 . OpenUrl FREE Full Text [2]. ↵ K. G. M. Moons , J. A. A. Damen , T. Kaul , and et al. , “ PROBAST+AI: an updated quality, risk of bias, and applicability assessment tool for prediction models using regression or artificial intelligence methods ,” BMJ , vol. 388 , p. e082505 , 2025 , doi: 10.1136/bmj-2024-082505 . OpenUrl FREE Full Text [3]. ↵ J. Elvidge et al. , “ Consolidated Health Economic Evaluation Reporting Standards for Interventions That Use Artificial Intelligence (CHEERS-AI) ,” (in eng), Value Health , vol. 27 , no. 9 , pp. 1196 – 1205 , Sep 2024 , doi: 10.1016/j.jval.2024.05.006 . OpenUrl CrossRef PubMed [4]. ↵ A. J. Vickers and E. B. Elkin , “ Decision Curve Analysis: A Novel Method for Evaluating Prediction Models ,” Medical Decision Making , vol. 26 , no. 6 , pp. 565 – 574 , 2006 , doi: 10.1177/0272989X06295361 . OpenUrl CrossRef PubMed Web of Science [5]. ↵ A. J. Vickers , B. Van Calster , and E. W. Steyerberg , “ Net benefit approaches to the evaluation of prediction models, molecular markers, and diagnostic tests ,” BMJ , vol. 352 , p. i6 , 2016 , doi: 10.1136/bmj.i6 . OpenUrl FREE Full Text [6]. ↵ T.-H. Chiang , H. Tai , and et al. , “ Dynamic accuracy of a time-varying early warning system in detecting physiological changes in CCU patients ,” Critical Care , 2025 , doi: 10.1186/s13054-025-05515-3 . OpenUrl CrossRef [7]. ↵ Z. Zhang , “ Decision curve analysis: a technical note ,” Annals of Translational Medicine , vol. 6 , no. 15 , p. 308 , 2018 , doi: 10.21037/atm.2018.07.02 . OpenUrl CrossRef PubMed [8]. ↵ M. Cecconi and et al. , “ Implementing Artificial Intelligence in Critical Care Medicine: a consensus of 22 ,” Critical Care , 2025 , doi: 10.1186/s13054-025-05532-2 . OpenUrl CrossRef [9]. ↵ A. E. W. Johnson , L. Bulgarelli , T. Pollard , S. Horng , L. A. Celi , and R. Mark . MIMIC-IV (version 2.2), PhysioNet , doi: 10.13026/6mm1-ek67 . OpenUrl CrossRef [10]. ↵ A. E. W. Johnson et al. , “ MIMIC-IV, a freely accessible electronic health record dataset ,” Scientific Data , vol. 10 , p. 1 , 2023 , doi: 10.1038/s41597-022-01899-x . OpenUrl CrossRef PubMed [11]. ↵ T. Pollard , A. Johnson , J. Raffa , L. A. Celi , O. Badawi , and R. Mark . eICU Collaborative Research Database (version 2.0), PhysioNet , doi: 10.13026/C2WM1R . OpenUrl CrossRef [12]. ↵ T. J. Pollard , A. E. W. Johnson , J. D. Raffa , L. A. Celi , R. G. Mark , and O. Badawi , “ The eICU Collaborative Research Database, a freely available multi-center database for critical care research ,” Scientific Data , vol. 5 , p. 180178 , 2018 , doi: 10.1038/sdata.2018.178 . OpenUrl CrossRef PubMed [13]. ↵ R. DerSimonian and N. Laird , “ Meta-analysis in clinical trials ,” Controlled Clinical Trials , vol. 7 , no. 3 , pp. 177 – 188 , 1986 , doi: 10.1016/0197-2456(86)90046-2 . OpenUrl CrossRef PubMed Web of Science [14]. ↵ J. P. T. Higgins , S. G. Thompson , J. J. Deeks , and D. G. Altman , “ Measuring inconsistency in meta-analyses ,” BMJ , vol. 327 , no. 7414 , pp. 557 – 560 , 2003 , doi: 10.1136/bmj.327.7414.557 . OpenUrl FREE Full Text [15]. ↵ S. Nasir and et al. , “ Achieving large-scale clinician adoption of AI-enabled decision support: current state and recommendations ,” BMJ Health & Care Informatics , vol. 31 , no. 1 , p. e100971 , 2024 , doi: 10.1136/bmjhci-2023-100971 . OpenUrl Abstract / FREE Full Text [16]. ↵ J. C. C. Kwong and et al. , “ The silent trial — the bridge between bench-to-bedside clinical AI evaluation ,” Frontiers in Digital Health , vol. 4 , p. 929508 , 2022 , doi: 10.3389/fdgth.2022.929508 . OpenUrl CrossRef PubMed [17]. ↵ D. van de Sande and et al. , “ Algorithmic fairness audits in intensive care medicine: artificial intelligence evaluated ,” Critical Care , vol. 26 , p. 144 , 2022 , doi: 10.1186/s13054-022-04197-5 . OpenUrl CrossRef PubMed [18]. ↵ X. Guan , J. Fang , Y. Zhao , and et al. , “ Prediction of new-onset atrial fibrillation in patients with acute myocardial infarction treated with percutaneous coronary intervention using machine learning and explainable artificial intelligence ,” Critical Care , vol. 28 , p. 104 , 2024 , doi: 10.1186/s13054-024-05040-9 . OpenUrl CrossRef PubMed [19]. ↵ G. Abgrall and et al. , “ Should AI models be explainable to clinicians? ,” Critical Care , 2024 , doi: 10.1186/s13054-024-05005-y . OpenUrl CrossRef PubMed [20]. ↵ R. Parsons , R. D. Blythe , S. M. Cramb , and S. M. McPhail , “ Integrating economic considerations into cutpoint selection may help align clinical decision support toward value-based healthcare ,” Journal of the American Medical Informatics Association , vol. 30 , no. 6 , pp. 1103 – 1113 , 2023 , doi: 10.1093/jamia/ocad042 . OpenUrl CrossRef PubMed [21]. ↵ R. Parsons , R. D. Blythe , A. G. Barnett , S. M. Cramb , and S. M. McPhail , “ predictNMB: An R package to estimate if or when a clinical prediction model is worthwhile ,” Journal of Open Source Software , vol. 8 , no. 84 , p. 5328 , 2023 , doi: 10.21105/joss.05328 . OpenUrl CrossRef [22]. ↵ A. L. Goldberger et al. , “ PhysioBank, PhysioToolkit, and PhysioNet: Components of a New Research Resource for Complex Physiologic Signals ,” Circulation , vol. 101 , no. 23 , pp. e215 – e220 , 2000 , doi: 10.1161/01.CIR.101.23.e215 . OpenUrl Abstract / FREE Full Text View the discussion thread. Back to top Previous Next Posted November 02, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Staffable ICU early-warning with calibrated multi-centre validation and alert-budget guardrails Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Staffable ICU early-warning with calibrated multi-centre validation and alert-budget guardrails Sanwal Ahmad Zafar , Wei Qin , Toufique Ahmed Soomro , Areeba Ali Khan , Muhammad Salman Faisal medRxiv 2025.10.30.25339143; doi: https://doi.org/10.1101/2025.10.30.25339143 Share This Article: Copy Citation Tools Staffable ICU early-warning with calibrated multi-centre validation and alert-budget guardrails Sanwal Ahmad Zafar , Wei Qin , Toufique Ahmed Soomro , Areeba Ali Khan , Muhammad Salman Faisal medRxiv 2025.10.30.25339143; doi: https://doi.org/10.1101/2025.10.30.25339143 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Intensive Care and Critical Care Medicine Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15227) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6597) Geriatric Medicine (668) Health Economics (997) Health Informatics (4534) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3332) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9230) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a0034ef86fcd0704',t:'MTc3OTUzMTQyMQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();