The Neurosurgical Uncertainty Index: Self-Doubting AI for rare or unexpected surgical complications

The Neurosurgical Uncertainty Index: Self-Doubting AI for rare or unexpected surgical complications | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search The Neurosurgical Uncertainty Index: Self-Doubting AI for rare or unexpected surgical complications Grace M. Thiong’o , Adegboyega Ogundokun doi: https://doi.org/10.1101/2025.05.01.25326833 Grace M. Thiong’o 1 The Philanthropic Surgeon, Not-for-Profit Organization Toronto ,ON, Canada MD/PHD Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: grace.muthoni{at}duck.Com Adegboyega Ogundokun 1 The Philanthropic Surgeon, Not-for-Profit Organization Toronto ,ON, Canada MSc, CCSP, CISSP, GICSP Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Rare or unexpected postoperative neurosurgical complications pose a challenge due to clinical variability and gaps in available data. We introduce the Neurosurgical Uncertainty Index (NUI), an uncertainty-aware AI framework that integrates bootstrap sampling for aleatoric uncertainty, isolation forest anomaly detection, and clinical calibration to predict and stratify risks for 13 complications. NUI distinguishes between data-driven and model-driven uncertainty and highlights cases that conventional models often miss. In a cohort of 80 patients, the hybrid Rare Event Score (anomaly × uncertainty) achieved critical risk stratification with an AUROC of 0.92 (95% CI 0.85–0.97) for complications requiring intervention, demonstrating precision 89% for critical cases (Score 1.5 nats) flagged 18% of predictions for review, preventing three overconfidence errors. Interpretable risk tiers are designed to integrate seamlessly with clinical workflows. By merging machine learning, neurosurgery, and epistemology, NUI promotes AI that acknowledges its limitations, with the aim of safer surgery. 1. Background High-stakes environments such as neurosurgery demand precision paired with humility. Although modern techniques like intraoperative MRI, robotic assistance, and neuromonitoring have revolutionized surgical execution, postoperative decision-making often remains dependent on older statistical paradigms.[ 1 ] Surgeons regularly confront the daunting question of how best to prepare for complications that defy expectation. Current AI systems are adept at forecasting frequent events—such as cerebrospinal fluid leaks occurring in 5–15% of cases—but they falter when rare, life-altering outcomes emerge.[ 2 ] Complications such as delayed subdural empyema, paradoxical edema in glioma resections, or shunt failures in the presence of normal imaging are not well predicted because the models tend to conflate data frequency with certainty [ 3 ],[ 4 ],[ 5 ]. Traditional risk scores rely on logistic regression with rigid probability bounds that treat uncertainty merely as noise rather than a signal [ 6 ],[ 7 ]. This reliance can lead to overconfident predictions, even in situations where the model has never encountered a specific complication [ 8 ]. In contrast, our approach is informed by Karl Popper’s philosophy that knowledge advances through the continuous testing and falsification of hypotheses [ 9 ]. Bridging the certainty-chasm and moving from prediction to falsification is a twofold challenge. First, rare complications - those with incidence < 1% - are often excluded from training datasets, rendering them invisible to conventional AI [ 10 ]. Second, unexpected complications, such as atypical presentations of common conditions (for example, a tumor patient experiencing stroke-like symptoms that mimic seizures), resist straightforward categorization even though they occur more frequently [ 11 ]. Notably, these categories intersect but are not equivalent: a complication can be both rare and predictable, as in the case of an intraoperative air embolism, or common yet unexpected, such as postoperative delirium in young adults. Existing frameworks fail to disentangle these dimensions, instead treating uncertainty as a single, monolithic challenge rather than as a nuanced clinical signal. The NUI Framework: Philosophy and Methodology The NUI framework re-imagines AI as a partner rather than an oracle. It identifies and communicates its own limitations by integrating three domains. In the realm of machine learning, bootstrap ensembles are used [ 12 ], specifically 100 resampled models, to generate probabilistic predictions that are evaluated using Shannon entropy [ 13 ]. This unsupervised anomaly detector identifies feature space outliers (e.g., a 25-year-old with geriatric biomarker profiles or in our study patient 44 with pulmonary embolism with U =0.19, A =0.75, R =0.14), where a high anomaly score did not align with clinical risk. From the neurosurgical perspective, clinical calibration is applied through procedure-specific modifiers, such as assigning an α value of 0.7 for ventriculoatrial shunts compared to 0.4 for biopsies, based on clinical opinion [ 4 ]. Additionally, pediatric ( 65 years) patients receive β = 1.2 risk amplification, reflecting physiological vulnerability [ 14 ],[ 15 ]. Finally, epistemological principles are translated into action by treating each prediction as a falsifiable hypothesis; high-entropy outputs trigger further review by human experts [ 16 ],[ 17 ],[ 18 ]. The cost of excessive confidence in AI is clinically significant and measurable [ 19 ],[ 20 ],[ 21 ],[ 4 ]. Current systems often don’t know what they don’t know; unable to sense when their training data do not adequately represent a case at hand [ 22 ]. In contrast, NUI tries to codify in a diagnostic lens a surgeon’s self-awareness or unease that ‘something is wrong’ before lab tests or imaging confirm it [ 23 ]. NUI’s innovation lies not in outperforming existing models on common tasks but in failing gracefully; redirecting clinician attention when its knowledge falters. At its core, NUI represents a paradigm shift from knowledge-as-possession to knowledge-as-process, embracing the idea that while knowledge can only be finite, our ignorance must necessarily be infinite [ 24 ]. 2. Related Work NUI is a foundation for developing self-doubting AI in clinical applications, building on the insights from previous authors who emphasize the importance of explainable AI for understanding complex systems [ 25 ]. Across neurosurgery and AI, these works grapple with uncertainty: Staartjes focus is on modeling it [ 26 ]; Senders and Ghassemi warn of blind spots and false certainty [ 4 ],[ 3 ]; Topol calls for humility [ 5 ]. Yet, few bridge uncertainty with decision support that truly reflects a model’s awareness of its own limits akin to human traits. 3. Methods This study adheres to the IDEAL framework (Idea, Development, Exploration, Assessment, Long-term study) for surgical innovation [ 27 ], aligning with Stage 2a (Development Phase) to refine and validate the Neurosurgical Uncertainty Index (NUI) in a proof-of-concept cohort. Our framework aims to emphasize iterative refinement and transparent uncertainty quantification to directly inform our hybrid risk stratification approach. It ensures rigor while acknowledging the exploratory nature of anomaly-driven AI. The code developed for our work is available in a GitHub repository ( https://github.com/surgeon-in-the-loop/Project-Neurosurgical-Uncertainty-Index-NUI-.git ). We encourage researchers to access our repository for further details and to adapt the code as needed for their own applications. 3.1. Data Acquisition and Curation We curated a multinational cohort of 80 neurosurgical patients (2019–2023) from open access studies on PubMed Central and Google Scholar, using automated parsers to extract structured tabular data as have previous authors [ 28 ]. Inclusion criteria required adult or pediatric craniotomies granular complication grading per Landriel Iban~ez classification [ 29 ] and tabulated individual patient data. Exclusion criteria removed, animal studies, and non-tabulated reports. The final cohort included tumor resections (54/80, 67.5%), ventriculoatrial (VA) shunts (13/80, 16.3%), and biopsies (6/80, 7.5%), with complications spanning postoperative hematoma (21/80, 26.3%), tumor progression (21/80, 26.3%), and rare events like subdural empyema (1/80, 1.3%). 3.2. Data Preprocessing Temporal variables were normalized to a range of [0,1] using the formula X ⍰ = X max − X min . For handling missing values, numerical features, such as preoperative albumin, underwent median imputation, while categorical gaps, like unspecified sex in 7 out of 80 records, were addressed using mode imputation. One-hot encoding was applied to transform 14 categorical variables (13 complications plus ‘N/A’ for none), into 42 binary features. This approach preserved all clinically relevant categories while avoiding information loss. To ensure the preservation of rare complications (defined as those with 5 or fewer cases), a class-aware splitting method was employed, allocating 20% of the data to the test set while maintaining the prevalence ratios. 3.3. Feature Engineering and Selection A Random Forest classifier (scikit-learn 1.2.2) ranked 42 clinical features by Gini importance. Top predictors included follow-up 19d (4.3%), Indication for Procedure Metastasis (3.8%) and Notes Resection (3.7%). Recursive Feature Elimination (RFE) reduced dimensionality to 15 variables. Principal Component Analysis (PCA) explained 32% cumulative variance across the first five components (PC1=10%, PC2=7%, PC3=5%, PC4=5%, PC5=5%). Partial dependence plots revealed VA shunts doubled hematoma risk versus biopsies (OR=2.1, p =0.02). 3.4. Model Development and Training Algorithms, specifically XGBoost and Random Forest were benchmarked to assess their effectiveness in handling class imbalance. For class balancing, SMOTE-NC was utilized to generate synthetic samples for complications with fewer than 50 cases, such as subdural empyema, using three nearest neighbors. In terms of cost-sensitive learning, class weights were assigned inversely proportional to the frequency of complications, applying a fourfold penalty for rare classes. Hyperparameter tuning was conducted using Bayesian optimization with Optuna across 432 configurations, focusing on maximizing the macro F1-score under stratified 3-fold cross-validation. The final parameters for the XGBoost model were set to: n estimators = 280, max depth = 5, and learning rate = 0.08. 3.5. Uncertainty Quantification Model uncertainty was quantified via 100 bootstrap-sampled XGBoost ensembles. Perpatient uncertainty ( H ( p )) was computed as Shannon entropy in equation (1): where C =13 complication classes. This entropy was calibrated using: Procedure-Specific Risk Factors – Biopsies ( α = 0.4), shunts ( α = 0.7), resections ( α = 0.6). Age Modifiers – Pediatric ( β = 1.2), geriatric ( β = 1.1) – as seen in equation (2): 3.6. Anomaly Detection An Isolation Forest with Mahalanobis scaling identified feature-space outliers. Anomaly scores (sraw) were normalized to [0,1] as shown in equation (3): The Rare Event Score ( R ) combined anomaly and uncertainty multiplicative (equation (4)): Risk tiers were defined based on the risk score ( R ). The Critical tier, with a risk score of R ≥ 0.8, included 23 out of 80 cases, accounting for 28.8% of the total. This category comprised conditions such as postoperative hematoma, which occurred in 12 of these cases, and tumor progression, which was noted in 8 cases. The Urgent tier, defined by a risk score between 0.6 and 0.8 (inclusive), included 15 out of 80 cases, representing 18.8%. This group featured conditions like pneumonia, which affected 4 of these cases, and focal motor seizures, which were present in 3 cases. In the Monitor tier, with a risk score ranging from 0.3 to 0.6, there were 7 out of 80 cases, or 8.8% of the total. This tier included subdural empyema, which was observed in 1 of these cases. Finally, the Routine tier, characterized by a risk score of less than 0.3, comprised 35 out of 80 cases, accounting for 43.8%. This category primarily consisted of uneventful recoveries, which occurred in 13 of these cases. Risk tiers, categorized as Critical, Urgent, Monitor, and Routine, were derived from the Rare Event Score, calculated as R = A × U , where A represents the normalized anomaly score and U denotes the Calibrated NUI, both bounded within the range of [0,1]. The thresholds for these tiers were optimized using Youden’s J statistic based on ROC curves. The Critical tier, defined by a risk score of R ≥0.8, was established to maximize precision for interventions, such as hematoma evacuation. The Urgent tier, with a risk score ranging from 0.6 to 0.8, aimed to balance sensitivity and specificity for early warnings. A precision-recall analysis was conducted, resulting in a precision of 89% and a recall of 85% for cases in the Critical tier. Finally, an error analysis was performed to audit false positives, exemplified by patient 52, who had a critical score but did not receive an intervention, allowing for further refinement of the thresholds. 3.7. Interpretability and Ethical Safeguards Interpretability and ethical safeguards were addressed through several key measures. SHAP analysis revealed that Notes Resection, with a SHAP value of 0.23, and operative duration, with a SHAP value of 0.18, were the top contributors to predictions of hematoma (as illustrated in Figure 4B ). To mitigate bias, the AIF360 tool was employed for reweighting, which successfully reduced the demographic parity disparity from Δ = 0.32 to Δ = 0.07. Additionally, uncertainty thresholds were established, flagging predictions with entropy greater than 1.5 nats for review. This measure effectively prevented three instances of overestimation. 3.8. Clinical Deployment and Validation The NUI framework was operationalized through a prototype Streamlit dashboard designed for real-time risk visualization and clinician feedback. While prospective clinical trials remain future work, preliminary technical validation demonstrated the system’s feasibility i.e., dashboard functionality and workflow integration. This phase-one implementation lays the groundwork for future clinical validation, where the framework’s ability to reduce missed rare complications will be prospectively assessed [ 30 ]. Internal validation was conducted using bootstrap resampling with 1,000 iterations, resulting in an area under the receiver operating characteristic curve (AUROC) of 0.88, with a 95% confidence interval of 0.83 to 0.93, and a calibration slope of 0.9, with a 95% confidence interval of 0.8 to 1.1. Calibration was further confirmed by a Brier score of 0.21, indicating probabilistic reliability. Additionally, the model demonstrated actionability, achieving a precision of 0.82 at the top five predictions for rare complications. 4. Results 4.1. Evaluation Approach Overall, our results evaluate the baseline model performance and then the utility of uncertainty, anomaly measures and a resulting hybrid metric to predict unexpected complications following surgery. We utilize a systemic approach starting with assessing the dataset characteristics and preprocessing outcomes, feature importance and dimensionality reduction, baseline model performance, uncertainty and anomaly-driven risk stratification and finally interpretability and bias mitigation. 4.2. Dataset Characteristics and Preprocessing Outcomes Our cohort comprised 80 neurosurgical patients (2019–2023), including 13 postoperative complications and one “nan” class (13/80, 16.3%) representing uneventful recoveries. Tumor resections (54/80, 67.5%), ventriculoatrial (VA) shunts (13/80, 16.3%), and biopsies (6/80, 7.5%) were the most frequent procedures. Postoperative hematoma (21/80, 26.3%) and tumor progression (21/80, 26.3%) dominated complications, while rare events like subdural empyema (1/80, 1.3%) and cerebral infarction (1/80, 1.3%) were critically underrepresented ( Figure 1A ). Demographics revealed a male predominance (43/80, 53.8%), female (30/80, 37.5%), and unspecified sex (7/80, 8.8%), with pediatric ( 65 years, 19/80, 23.8%) patients exhibiting 2.1× higher complication risk than adults (OR=2.1, 95% CI: 1.3–3.4). Figure1 A, B and C illustrate the data demographics. Download figure Open in new tab Figure 1: Dataset characteristics: (A) Complication frequencies, (B) Patients stratified into three age bins: pediatric (65 years). The y-axis shows normalized values in decimal form, reflecting risk scores (or odds ratios) rather than raw counts. Each bar Is divided to indicate the distribution of specific complications within the corresponding age group, highlighting that pediatric and geriatric patients exhibit a higher complication risk relative to adults. (C) Sex distribution by complication Data preprocessing addressed missing values (12.4% of entries) via median imputation for numerical features (e.g., follow-up duration) and mode imputation for categorical variables (e.g., unspecified sex). Follow-up durations (1–46 months) were normalized to [0,1], revealing a moderate inverse correlation with complication severity (r=–0.39, p < 0.001). One-hot encoding transformed 14 categorical variables (13 complications plus ‘N/A’ for none) into 42 binary features, preserving all clinically relevant categories while avoiding information loss. Figure2 illustrates the correlations heatmap and value counts for complications, respectively. Download figure Open in new tab Figure 2: Dataset preprocessing (A) Correlations Heat map, (B) Value Counts for Complications 4.3. Feature Importance and Dimensionality Reduction Recursive Feature Elimination (RFE) and Random Forest feature ranking identified temporal follow-up intervals (e.g., Follow-up 19d: 4.3%, Follow-up 12d: 4.3%) and procedurespecific indicators (e.g., Indication for Procedure Metastasis: 3.8%) as top predictors ( Figure3A ). Principal Component Analysis (PCA) reduced dimensionality, with the first five components explaining 32% cumulative variance (PC1=10%, PC2=7%, PC3=5%, PC4=5%, PC5=5%). SHAP dependence calculations revealed that VA shunts doubled hematoma risk compared to biopsies (OR=2.1, p =0.02), while metastatic indications tripled tumor progression risk (OR=3.8, 95% CI: 1.6–8.9). Figure3A is the PCA graph, whilst 3C depicts the clinical SHAP dependence plot. Download figure Open in new tab Figure 3: Feature Importance and Dimensionality Reduction (A) Feature importance, (B) Principal Component Analysis variance plot, (C) Clinical SHAP dependence plot - Clinical Class Registry is as follows: 0: Cardiac arrest 1: Cerebral infarction 2: Epileptic spasms 3: Focal motor seizures 4: Hydrocephalus 5: Pneumonia 6: Postop. Brain edema?herniation 7: Postop.I hematoma 8: Pulmonary embolism 9: Sepsis 10: Shunt failure 11: Subdural empyema 12: Tumor progression 13: nan 4.4. Baseline Model Performance XGBoost achieved strong discriminative performance for common complications (macro AUROC = 0.88) but failed entirely on rare events (e.g., tumor progression recall = 0%, F1 = 0.00), see Figure4A . Class imbalance contributed to overestimation in predictions, as 75% of misclassified rare cases had a prediction entropy of less than 0.5 nats. However, by integrating bootstrap uncertainty quantification, false security errors were reduced by 27% (ΔBrier = 0.11), while the area under the receiver operating characteristic curve (AUROC) remained stable at 0.85 ± 0.03. The final Neurosurgical Uncertainty Index (NUI) framework achieved the following values: Macro AUROC 0.88 (95% CI: 0.83–0.93), Rare Event Score AUROC (calculated in relation to the precision-recall tradeoff) 0.92 ± 0.04, Calibration Error 0.03 (Brier Score = 0.21) and Critical Recall 89% (20/23 critical cases requiring intervention). Of note, critical recall in our study measures the proportion of Critical-tier patients ( R ≥ 0.8) requiring intervention, not per-complication detection. In contrast, class-specific recalls (e.g., tumor progression) reflect baseline model limitations, addressed through uncertainty-aware risk stratification. The confusion matrix highlighted robust performance for focal motor seizure detection (precision = 0.91, recall = 1.00) but poor postop. hematoma specificity (precision = 0.33), likely due to heterogeneous presentations. Figure4A shows macro-AUC = 0.88 and class-specific recalls (which is different than critical recall), while Figure4B presents a Confusion Matrix that highlights the baseline model’s failure on rare classes, justifying the need for NUI. Download figure Open in new tab Figure 4: Baseline Model Performance (A) macro AUC=0.88 with balanced class-specific recalls (which is different than critical recall), (B) Clinical Decisions Confusion matrix. 4.5. Uncertainty and Anomaly-Driven Risk Stratification Bootstrap ensembles (n=100) quantified per-patient uncertainty via Shannon entropy, calibrated against procedure-specific risk factors (α = 0.7 for shunts vs. α = 0.4 for biopsies). Isolation Forests identified 14% of cases (11/80) as anomalous, with 78% (9/11) involving rare complications (e.g., cerebral infarction). The Rare Event Score in Table1 stratified patients into four tiers: critical (score ≥ 0.8, 23/80, 28.8%): postop. hematoma (12/23), tumor progression (8/23), sepsis (2/23), cerebral infarction (1/23); urgent (0.6 ≤ Score < 0.8, 15/80, 18.8%): pneumonia (4/15), focal motor seizures (3/15); monitor (0.3 ≤ Score < 0.6, 7/80, 8.8%): subdural empyema (1/7), hydrocephalus (1/7); routine (Score < 0.3, 35/80, 43.8%) and no complications (13/35) or low-risk events. The Score achieved 89% precision for critical interventions (e.g., hematoma evacuation) and a Brier score of 0.21, reflecting well-calibrated uncertainty. Figure 4 : NUI basal model performance output. 4.6. Interpretability and Bias Mitigation SHAP analysis attributed 23% of hematoma predictions to Notes_Resection (SHAP=0.23) and 18% to prolonged operative duration. AIF360 reweighting reduced demographic parity disparities from Δ = 0.32 to Δ = 0.07 (78% reduction). Entropy thresholds (> 1.5 nats) flagged 18% of predictions for review, preventing three critical misjudgment errors. 5. Conclusion 5.1. Redefining Uncertainty as a Clinical Compass Our findings suggest that NUI represents a paradigm shift in surgical risk prediction. Rather than viewing uncertainty as a flaw to be minimized, NUI embraces it as a diagnostic tool—a signal that invites further human evaluation. By incorporating measures of both aleatoric and epistemic uncertainty, the framework not only improves the detection of rare complications but also reduces errors associated with overconfidence by 27%. This aligns with the notion that scientific progress arises not from confirming hypotheses but from rigorously testing them [ 9 ]. This acknowledgment by NUI that its predictions are not infallible reflects the cautious approach that experienced surgeons use when making critical decisions. 5.2. The Hybrid Rare Event Score: Bridging Data and Intuition Central to NUI is the rare event score ( Table 1 ), a hybrid metric combining normalized anomaly detection ( A ) and calibrated uncertainty ( U ) multiplicatively – equation (4). This formulation prioritizes cases where both model ignorance (high entropy) and data rarity (anomalous features) converge – an improvement over traditional models that treat these dimensions independently [ 31 ]. Technical validation, scores > 0.8 achieved 89% precision for complications requiring intervention (e.g., hematoma evacuation), outperforming the ACS NSQIP calculator’s precision of 62% for similar endpoints [ 6 ]. Critically, the Rare Event Score distinguishes rarity from unpredictability. For example, in our cohort, common complications like postoperative hematoma (21/80 cases) often had moderate anomaly scores but low uncertainty due to well-established risk factors such as Notes Resection [SHAP=0.23]), resulting in lower Rare Event Scores. Conversely, rare events like subdural empyema (1/80 cases) triggered high anomaly and uncertainty scores (R ≥ 0.8), prioritizing them for urgent review. While the framework was not tested on ultra-rare complications outside our dataset (for example paradoxical brainstem herniation), its design principles theoretically extend to such scenarios, provided relevant features are captured for differing datasets. This nuance addresses a longstanding gap in surgical risk tools, which often merge incidence with clinical relevance [ 32 ]. View this table: View inline View popup Download powerpoint Table 1: Rare Event Score Risk Stratification 5.3. Clinical Translation: Closing Three Historical Gaps NUI directly tackles three limitations of conventional models. The first, rarity-agnostic design considers existing tools like the ACS NSQIP surgical risk calculator which excludes complications with < 1% incidence, rendering them blind to events like delayed subdural empyema [ 7 ],[ 6 ]. The second model, static calibration, considers traditional scores such as the American Society of Anesthesiologists physical status classification system which use fixed risk coefficients thus ignoring temporal shifts in patient status [ 33 ]. NUI’s dynamic calibration—via procedure-specific modifiers ( α ) and a temporal decay function in equation (5): enables real-time risk adjustments. For example, a shunt patient’s risk score decays 2.3 times faster post-discharge than a tumor resection case, reflecting differential recovery trajectories [ 34 ]. The third model addresses the concept of Ethical Blind Spots, highlighting that unchecked AI overconfidence disproportionately affects marginalized groups that are underrepresented in the training data [ 35 ]. Our bias-aware reweighting (AIF360) reduced demographic parity disparities from Δ=0.32 to Δ=0.07 (78% reduction) across age/gender subgroups, while entropy thresholds (>1.5 nats) prevented three critical errors during validation. 5.4. Limitations and the Path Forward While NUI shows promise, it is yet to be tested on larger more diverse datasets. Perhaps, federated learning across global consortia could be beneficial, although it is crucial to implement differential privacy safeguards in later project phases [ 36 ]. Additionally, as bootstrap ensembles increased training time by four times compared to baseline models, future work could focus on optimizing computational costs through deep ensembles, such as stochastic weight averaging, or by utilizing Bayesian neural networks, which can approximate uncertainty with fewer resources [ 37 ],[ 38 ]. Lastly, regarding calibration assumptions, NUI tests the hypothesis that entropy reliably acts as a proxy for model ignorance, a premise that may be challenged by noisy or adversarial inputs [ 39 ]. Integrating conformal prediction, such as prediction intervals for uncertainty, could enhance the model’s robustness [ 40 ]. Future studies will advance through IDEAL Stage 2b (Exploration) and Stage 3 (Assessment), prospectively validating NUI in multi-institutional cohorts while addressing ethical and computational scalability challenges. 5.5. Future Directions: Toward Neuro-symbolic Uncertainty Three pathways could extend NUI’s utility. The first is multi-modal integration which involves fusing tabular data with intraoperative imaging, such as MRI diffusion changes, or genomics, like APOE4 status related to delirium risk, that could enrich anomaly detection. Early experiments with vision transformers have shown promise in linking radiographic “unknowns” to clinical uncertainty [ 41 ]. The second, dynamic meta-learning could be achieved through a neuro-symbolic architecture that combines deep learning with rule-based expert systems, such as IF-THEN rules for shunt failure, to refine the Rare Event Score. For example, symbolic logic might be used to override anomaly flags in cases where definitive lab confirmations are available. Lastly, longitudinal adaptation through temporal validation using recurrent latent variables could model complication trajectories, for example the progression from brain edema to herniation, thereby enabling preemptive interventions. Pilot tests using postoperative EEG streams have shown reasonably high accuracy in predicting delayed seizures; accordingly, blending NUI insights with intuitive, rule-based reasoning could help us improve postoperative patient outcomes [ 42 ]. 5.6. Philosophical Implications: The Virtue of Doubt NUI implements a Bayesian-Popperian ethos; it is Bayesian in its probabilistic risk stratification, and Popperian in its anomaly-driven falsification [ 24 ]. This synthesis reflects the everyday reality of clinical practice where behind every data point is a patient and a family hoping for the best possible care [ 43 ]. By making ignorance explicit through its quantification, calibration, and then acting on it NUI aligns AI with the iterative, self-correcting spirit of the scientific method. 5.7. Final thoughts In an era where AI’s clinical role is often reduced to prediction, NUI redefines its purpose: augmenting doubt as much as providing answers. Our framework demonstrates that uncertainty-aware systems need not sacrifice accuracy, instead, they can enhance it by redirecting attention to the ‘known unknowns’ that matter most. For the neurosurgeon facing a deteriorating patient with inconclusive tests, NUI offers more than a risk score; it offers a partner that says, ‘I cannot be certain here, take a closer look’. Thus NUI bridges the gap between machine intelligence and human wisdom, fostering a future where AI’s greatest strength is knowing its limits. Data Availability All data produced in the present study are available upon reasonable request to the authors https://github.com/surgeon-in-the-loop/Project-Neurosurgical-Uncertainty-Index-NUI-.git Footnotes grace.muthoni{at}duck.Com gravity.Repeat5b{at}icloud.Com References [1]. ↵ Manjila S , Rosa B , Price K , Manjila R , Mencattelli M , Dupont PE . Robotic Instruments Inside the MRI Bore: Key Concepts and Evolving Paradigms in Imaging-enhanced Cranial Neurosurgery . World Neurosurgery 2023 ; 176 : 127 – 39 . doi: 10.1016/j.wneu.2023.01.025 . OpenUrl CrossRef PubMed [2]. ↵ Hajikarimloo B , Sabbagh Alvani M , Koohfar A , Goudarzi E , Dehghan M , Hojjat SH , et al. Clinical Application of Artificial Intelligence in Prediction of Intraoperative Cerebrospinal Fluid Leakage in Pituitary Surgery: A Systematic Review and Meta-Analysis . World Neurosurgery 2024 ; 191 : 303 – 313 .e1. doi: 10.1016/j.wneu.2024.09.015 . OpenUrl CrossRef PubMed [3]. ↵ Ghassemi M , Oakden-Rayner L , Beam AL . The false hope of current approaches to explainable artificial intelligence in health care . The Lancet Digital Health 2021 ; 3 : e745 – 50 . doi: 10.1016/S2589-7500(21)00208-9 . OpenUrl CrossRef [4]. ↵ Senders JT , Zaki MM , Karhade AV , Chang B , Gormley WB , Broekman ML , et al. An introduction and overview of machine learning in neurosurgical care . Acta Neurochirurgica 2018 ; 160 : 29 – 38 . doi: 10.1007/S00701-017-3385-8 . OpenUrl CrossRef PubMed [5]. ↵ Topol EJ . High-performance medicine: the convergence of human and artificial intelligence . Nature Medicine 2019 25:1 2019 ; 25 : 44 – 56 . doi: 10.1038/s41591-018-0300-7 . OpenUrl CrossRef PubMed [6]. ↵ Bilimoria KY , Liu Y , Paruch JL , Zhou L , Kmiecik TE , Ko CY , et al. Development and Evaluation of the Universal ACS NSQIP Surgical Risk Calculator: A Decision Aide and Informed Consent Tool for Patients and Surgeons . Journal of the American College of Surgeons 2013 ; 217 : 833 – 833 . doi: 10.1016/J.JAMCOLLSURG.2013.07.385 . OpenUrl CrossRef PubMed [7]. ↵ Cohen ME , Ko CY , Bilimoria KY , Zhou L , Huffman K , Wang X , et al. Optimizing ACS NSQIP modeling for evaluation of surgical quality and risk: patient risk adjustment, procedure mix adjustment, shrinkage adjustment, and surgical focus . Journal of the American College of Surgeons 2013 ; 217 . doi: 10.1016/J.JAMCOLLSURG.2013.02.027 . OpenUrl CrossRef [8]. ↵ Rajpurkar P , O’Connell C , Schechter A , Asnani N , Li J , Kiani A , et al. CheXaid: deep learning assistance for physician diagnosis of tuberculosis using chest x-rays in patients with HIV . NPJ Digital Medicine 2020 ; 3 : 115 . OpenUrl PubMed [9]. ↵ Popper KR . Conjectures and refutations!z: the growth of scientific knowledge . Recording for the Blind & Dyslexic ; 2008 . [10]. ↵ Obermeyer Z , Powers B , Vogeli C , Mullainathan S. Dissecting racial bias in an algorithm used to manage the health of populations . Science 2019 ; 366 : 447 – 53 . doi: 10.1126/SCIENCE.AAX2342/SUPPL_FILE/AAX2342_OBERMEYER_SM.PDF . OpenUrl Abstract / FREE Full Text [11]. ↵ Hashimoto DA , Rosman G , Rus D , Meireles OR . Artificial Intelligence in Surgery: Promises and Perils . Annals of Surgery 2018 ; 268 : 70 – 6 . doi: 10.1097/SLA.0000000000002693 . OpenUrl CrossRef PubMed [12]. ↵ Efron B. Bootstrap Methods: Another Look at the Jackknife . Https://DoiOrg/101214/Aos/1176344552 1979 ; 7 : 1 – 26 . doi: 10.1214/AOS/1176344552 . OpenUrl CrossRef [13]. ↵ Shannon CE . A Mathematical Theory of Communication . Bell System Technical Journal 1948 ; 27 : 379 – 423 . doi: 10.1002/J.1538-7305.1948.TB01338.X . OpenUrl CrossRef [14]. ↵ Tariciotti L , Fiore G , Carapella S , Remore LG , Schisano L , Borsa S , et al. A frailty-adjusted stratification score to predict surgical risk, post-operative, long-term functional outcome, and quality of life after surgery in intracranial meningiomas . Cancers 2022 ; 14 : 3065 . OpenUrl PubMed [15]. ↵ Huang J , Shlobin NA , DeCuypere M , Lam SK . Deep learning for outcome prediction in neurosurgery: a systematic review of design, reporting, and reproducibility . Neurosurgery 2022 ; 90 : 16 – 38 . OpenUrl PubMed [16]. ↵ Gelman A , Hwang J , Vehtari A. Understanding predictive information criteria for Bayesian models . Statistics and Computing 2014 ; 24 : 997 – 1016 . OpenUrl CrossRef [17]. ↵ Blei DM , Kucukelbir A , McAuliffe JD . Variational Inference: A Review for Statisticians . Journal of the American Statistical Association 2017 ; 112 : 859 – 77 . doi: 10.1080/01621459.2017.1285773 . OpenUrl CrossRef [18]. ↵ Lakatos I. The Methodology of Scientific Research Programmes: Philosophical Papers . The Methodology of Scientific Research Programmes 1978 . doi: 10.1017/CBO9780511621123 . OpenUrl CrossRef [19]. ↵ Hoebel KV , Bridge CP , Ahmed S , Akintola O , Chung C , Huang RY , et al. Expert-centered Evaluation of Deep Learning Algorithms for Brain Tumor Segmentation . Radiology Artificial Intelligence 2024 ; 6 . doi: 10.1148/RYAI.220231 . OpenUrl CrossRef [20]. ↵ Croskerry P. The Importance of Cognitive Errors in Diagnosis and Strategies to Minimize Them: Academic Medicine 2003 ; 78 : 775 – 80 . doi: 10.1097/00001888-200308000-00003 . OpenUrl CrossRef PubMed Web of Science [21]. ↵ Eva KW , Norman GR . Heuristics and biases - a biased perspective on clinical reasoning . Med Educ 2005 ; 39 : 870 – 2 . doi: 10.1111/j.1365-2929.2005.02258.x . OpenUrl CrossRef PubMed [22]. ↵ Hüllermeier E , Waegeman W. Aleatoric and epistemic uncertainty in machine learning: an introduction to concepts and methods . Machine Learning 2021 ; 110 : 457 – 506 . doi: 10.1007/S10994-021-05946-3/FIGURES/17 . OpenUrl CrossRef [23]. ↵ Gawande Atul . The checklist manifesto!z: how to get things right 2011 : 215 – 215 . [24]. ↵ Martin JA , Popper KR . Objective knowledge: an evolutionary approach . The Philosophical Review 1972 ; 84 : 103 – 103 . doi: 10.2307/2184085 . OpenUrl CrossRef [25]. ↵ Lundberg SM , Erion G , Chen H , DeGrave A , Prutkin JM , Nair B , et al. From local explanations to global understanding with explainable AI for trees . Nature Machine Intelligence 2020 2:1 2020 ; 2 : 56 – 67 . doi: 10.1038/s42256-019-0138-9 . OpenUrl CrossRef PubMed [26]. ↵ Staartjes VE , Stumpo V , Kernbach JM , Klukowska AM , Gadjradj PS , Schröder ML , et al. Machine learning in neurosurgery: a global survey . Acta Neurochirurgica 2020 ; 162 : 3081 – 91 . OpenUrl PubMed [27]. ↵ McCulloch P , Cook JA , Altman DG , Heneghan C , Diener MK , IDEAL Group . IDEAL framework for surgical innovation 1: the idea and development stages . The BMJ 2013 ; 346 : f3012 – f3012 . doi: 10.1136/BMJ.F3012 . OpenUrl CrossRef [28]. ↵ Shortliffe EH , Sepúlveda MJ . Clinical Decision Support in the Era of Artificial Intelligence . JAMA 2018 ; 320 : 2199 – 200 . doi: 10.1001/JAMA.2018.17163 . OpenUrl CrossRef PubMed [29]. ↵ Ibaez FAL , Hem S , Ajler P , Vecchi E , Ciraolo C , Baccanelli M , et al. A new classification of complications in neurosurgery . World Neurosurgery 2011 ; 75 : 709 – 15 . doi: 10.1016/J.WNEU.2010.11.010 . OpenUrl CrossRef PubMed [30]. ↵ Thiong’o G. Project: Neurosurgical Uncertainty Index (NUI) 2025 . doi: 10.5281/ZENODO.14963708 . OpenUrl CrossRef [31]. ↵ Chandola V , Banerjee A , Kumar V. Anomaly detection: A survey . ACM Comput Surv 2009 ; 41 : 1 – 58 . doi: 10.1145/1541880.1541882 . OpenUrl CrossRef [32]. ↵ Obermeyer Z , Emanuel EJ . Predicting the Future — Big Data, Machine Learning, and Clinical Medicine . New England Journal of Medicine 2016 ; 375 : 1216 – 9 . doi: 10.1056/NEJMP1606181/SUPPL_FILE/NEJMP1606181_DISCLOSURES.PDF . OpenUrl CrossRef PubMed [33]. ↵ Owens WD , Felts JA , Spitznagel EL . ASA physical status classifications: a study of consistency of ratings . Anesthesiology 1978 ; 49 : 239 – 43 . doi: 10.1097/00000542-197810000-00003 . OpenUrl CrossRef PubMed Web of Science [34]. ↵ Hosainey SAM , Lykkedrang BL , Meling TR . Long-term risk of shunt failure after brain tumor surgery . Neurosurg Rev 2022 ; 45 : 1589 – 600 . doi: 10.1007/s10143-021-01648-5 . OpenUrl CrossRef PubMed [35]. ↵ Rajkomar A , Hardt M , Howell MD , Corrado G , Chin MH . Ensuring Fairness in Machine Learning to Advance Health Equity . Annals of Internal Medicine 2018 ; 169 : 866 – 72 . doi: 10.7326/M18-1990 . OpenUrl CrossRef PubMed [36]. ↵ Sheller MJ , Edwards B , Reina GA , Martin J , Pati S , Kotrotsou A , et al. Federated learning in medicine: facilitating multi-institutional collaborations without sharing patient data . Scientific Reports 2020 10:1 2020 ; 10 : 1 – 12 . doi: 10.1038/s41598-020-69250-1 . OpenUrl CrossRef PubMed [37]. ↵ Lakshminarayanan B , Pritzel A , Blundell C. Simple and scalable predictive uncertainty estimation using deep ensembles . Advances in Neural Information Processing Systems 2017 ; 30 . [38]. ↵ Gal Y , Ghahramani Z. Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning 2016 : 1050 – 9 . [39]. ↵ Szegedy C , Zaremba W , Sutskever I , Bruna J , Erhan D , Goodfellow I , et al. Intriguing properties of neural networks . arXiv Preprint arXiv:13126199 2013 . [40]. ↵ Angelopoulos AN , Bates S. A Gentle Introduction to Conformal Prediction and Distribution-Free Uncertainty Quantification 2021 . [41]. ↵ Chen RJ , Ding T , Lu MY , Williamson DFK , Jaume G , Song AH , et al. Towards a general-purpose foundation model for computational pathology . Nature Medicine 2024 ; 30 : 850 – 62 . doi: 10.1038/S41591-024-02857-3 . OpenUrl CrossRef PubMed [42]. ↵ Kiral-Kornek I , Roy S , Nurse E , Mashford B , Karoly P , Carroll T , et al. Epileptic Seizure Prediction Using Big Data and Deep Learning: Toward a Mobile System . EBioMedicine 2018 ; 27 : 103 – 11 . doi: 10.1016/J.EBIOM.2017.11.032 . OpenUrl CrossRef PubMed [43]. ↵ Ioannidis JPA . Why Most Published Research Findings Are False . PLOS Medicine 2005 ; 2 : e124 – e124 . doi: 10.1371/JOURNAL.PMED.0020124 . OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted May 02, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following The Neurosurgical Uncertainty Index: Self-Doubting AI for rare or unexpected surgical complications Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share The Neurosurgical Uncertainty Index: Self-Doubting AI for rare or unexpected surgical complications Grace M. Thiong’o , Adegboyega Ogundokun medRxiv 2025.05.01.25326833; doi: https://doi.org/10.1101/2025.05.01.25326833 Share This Article: Copy Citation Tools The Neurosurgical Uncertainty Index: Self-Doubting AI for rare or unexpected surgical complications Grace M. Thiong’o , Adegboyega Ogundokun medRxiv 2025.05.01.25326833; doi: https://doi.org/10.1101/2025.05.01.25326833 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Surgery Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15227) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6597) Geriatric Medicine (668) Health Economics (997) Health Informatics (4534) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3332) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9230) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a002adb6efb73fe2',t:'MTc3OTUyNDgxNg=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();