Model uncertainty quantification: A post hoc calibration approach for heart disease prediction

Model uncertainty quantification: A post hoc calibration approach for heart disease prediction | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Model uncertainty quantification: A post hoc calibration approach for heart disease prediction View ORCID Profile Peter Adebayo Odesola , View ORCID Profile Adewale Alex Adegoke , View ORCID Profile Idris Babalola doi: https://doi.org/10.1101/2025.09.28.25336834 Peter Adebayo Odesola 1 Southampton Solent University , Southampton, United Kingdom Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Peter Adebayo Odesola Adewale Alex Adegoke 2 Westminster Foundation for Democracy London , United Kingdom Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Adewale Alex Adegoke Idris Babalola 3 Department of Health and Social Care , London, United Kingdom Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Idris Babalola For correspondence: eidreiz01{at}gmail.com Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract We investigate whether post-hoc calibration improves the clinical trustworthiness of heart-disease predictions beyond conventional accuracy metrics. Using a structured clinical dataset (1,025 records; 85/15 train-test split), we benchmarked six classifiers logistic regression, SVM, k-nearest neighbors, naïve Bayes, random forest, and XGBoost on accuracy, ROC-AUC, precision, recall, and F1, and then evaluated probability quality before and after Platt (sigmoid) and isotonic calibration using Brier score, expected calibration error (ECE), log loss, Spiegelhalter’s Z-test, and reliability diagrams. Baseline discrimination was high (e.g., SVM: accuracy 92.9%, ROC-AUC 99.4%, F1 92.8%), and ensembles achieved perfect test-set scores (random forest and XGBoost: 100% across metrics), prompting calibration analysis. Isotonic calibration consistently improved probability quality for most models: random forest Brier from 0.007 to 0.002, ECE 0.051 to 0.011, log loss 0.056 to 0.012; naive Bayes Brier from 0.162 to 0.132, ECE 0.145 to 0.118, log loss 1.936 to 0.446; SVM ECE 0.086 to 0.044 and log loss 0.142 to 0.133. Platt scaling helped some models but occasionally worsened calibration (e.g., KNN ECE 0.035 to 0.081). Reliability diagrams corroborated these trends, with isotonic yielding curves closer to the 45° diagonal line, while Spiegelhalter’s test moved toward non-significance for several models’ post-calibration. Ultimately, Isotonic calibration delivered the most consistent gains in probability reliability while preserving discrimination, strengthening the interpretability and clinical actionability of model outputs. 1. Introduction 1.1. Background Heart disease continues to be the major leading cause of death globally. It was recorded that heart disease was responsible for an estimated 19.8 million deaths in 2022 [ 1 ]. However, early and accurate prediction plays a significant role in the prevention of adverse results and reduction in healthcare costs. Machine learning (ML) models are increasingly adopted for diagnostic and prognostic tasks in cardiology due to their ability to uncover complex patterns in large clinical datasets [ 2 ]. Early ML research on heart disease cohorts primarily focused on classification accuracy, with studies routinely reporting performance above 97% using supervised classifiers [ 3 ]. These models have the capacity to learn non-linear relationships and high-dimensional interactions between contributing factors such as age, cholesterol, blood pressure, and electrocardiogram results. For example, algorithms such as Random Forest and Gradient Boosting have demonstrated superior performance to identify subtle indicators of cardiovascular abnormalities compared to traditional rule-based systems [ 4 ]. This makes them powerful techniques for risk stratification and preventive care. However, there could be possibility that the models often provide high predictive performance, while probabilistic outputs can be poorly calibrated. That is, the confidence scores they assign do not always align with actual probabilities of disease presence [ 5 ]. In high-stakes domains such as healthcare system, well-calibrated predictions are more important to guide the appropriate treatment decisions and manage clinical risks efficiently. Miscalibrated models may lead to overconfident or underconfident decisions, ultimately compromising patient safety [ 6 ]. This has prompted a growing interest in uncertainty quantification and post hoc calibration methods, which can adjust the model’s output probabilities without retraining the original model [ 7 ]. The importance of these methods has increased in response to an increasing demand for transparent and trustworthy AI systems in clinical settings, particularly with the rise of explainable AI initiatives [ 8 ]. Furthermore, recent research has proven that visual tools such as reliability diagrams and calibration metrics such as Expected Calibration Error (ECE), Brier score, and log loss are important in evaluating how well a model is calibrated [ 9 ]. While accuracy and AUROC (Area Under the Receiver Operating Characteristic curve) remain popular metrics for model evaluation, they are insufficient for assessing how well a model estimates uncertainty. These metrics provide both quantitative and visual representations of uncertainty and prediction quality, which are vital for gaining the confidence of clinical stakeholders. 1.2 Motivation and Problem Statement One of the major challenges faced by the medical health sector is the inability to detect early stages of problems related to the heart. When making decisions in the clinical sector, uncalibrated predictions may be misleading. For example, if a model predicts that a patient has a 90% chance of developing heart disease, clinicians must trust that this probability truly reflects clinical reality, otherwise this could lead to incorrect decisions and poor outcomes for the patient. In many studies, calibration and uncertainty quantification in medical AI systems are often overlooked, leading to a gap between predictive performance and clinical trust [ 6 ]. However, this paper addresses that gap by evaluating the calibration of several popular classifiers using post hoc techniques. 1.3 Scope and Contributions This study aims to evaluate and compare uncertainty estimation of heart disease prediction models. The research is guided by the following questions: 1. How do post-hoc calibration methods (Platt scaling and isotonic regression) affect the uncertainty, calibration quality, and prediction confidence of machine learning models for heart disease classification? 2. What are the baseline levels of calibration and uncertainty (ECE, Brier score, log loss, Spiegelhalter’s Z-score) for heart disease prediction before and after post-hoc calibration? 3. How does each model (e.g., Random Forest, XGBoost, SVM, KNN and Naive Bayes) perform in terms of probability calibration for heart disease before and after applying post hoc calibration? Below, we delineate the contributions of this work in light of the research questions above. We conduct a systematic, model-agnostic evaluation of post-hoc calibration for heart-disease prediction, quantifying how Platt (sigmoid) and isotonic mapping alter probability quality without retraining the base models. Beyond headline discrimination metrics, we emphasize clinically relevant probability fidelity, calibration, sharpness, and statistical goodness-of-fit. This study makes four (4) contributions, summarized as follows: 1. A side-by-side pre/post analysis of six machine learning classifiers using reliability diagrams plus Brier, ECE, log loss, Spiegelhalter’s Z/p, and sharpness to provide complementary views of probability quality for heart disease prediction. 2. Empirical demonstration that isotonic calibration most consistently improves probability estimates, whereas Platt scaling helps some models but can worsen others. 3. Despite perfect test-set discrimination for some model, reliability diagrams reveal overconfidence pre-calibration, demonstrating why discrimination alone is insufficient for clinical use. 4. Analysis of variance in predicted probabilities shows calibration-induced smoothing and overconfidence correction, clarifying confidence reliability trade-offs relevant to clinical interpretation. 1.4 Related Works 1.4.1 Machine Learning in Heart Disease Prediction: Calibration and Reliability Considerations Machine learning (ML) techniques have been widely applied to predict cardiovascular disease outcomes, typically using patient risk factor data to classify the presence or risk of heart disease. For example, in heart disease prediction using supervised machine learning algorithms: Performance analysis and comparison, [ 10 ] evaluated several classifiers (KNN, decision tree, random forest, etc.) on a Kaggle heart disease dataset. They reported perfect performance with random forests achieving 100% accuracy (along with 100% sensitivity and specificity). However, their evaluation emphasized accuracy and did not include any probability calibration or uncertainty quantification. Similarly, [ 11 ] evaluation of Heart Disease Prediction Using Machine Learning Methods with Elastic Net Feature Selection compared logistic regression (LR), KNN, SVM, random forest (RF), AdaBoost, artificial neural network (ANN), and multilayer perceptron on the Kaggle dataset used in this study. They found RF to attain ∼99% accuracy and AdaBoost ∼94% on the full feature set and observed SVM performing best after SMOTE class-balancing and feature selection. Like [ 10 ], this study focused on accuracy improvements and other discrimination metrics, with no model calibration applied. Another work by [ 12 ], they also utilized the Kaggle dataset we explored. They evaluated a wide range of classifiers including RF, decision tree (DT), gradient boosting (GBM), KNN, AdaBoost, LR, ANN, QDA, LDA, SVM and reported extremely high accuracy for ensemble methods. In fact, their RF model reached 100% training accuracy (and ∼99% under cross-validation). Despite reporting precision, recall, F1-score, and ROC-AUC for each model, this work too did not report any calibration metrics or uncertainty estimates; the focus remained on discrimination performance. Beyond the popular Kaggle/UCI datasets, researchers have explored ML on other heart disease cohorts. For instance, [ 13 ] in A Machine Learning Model for Detection of Coronary Artery Disease s applied ML to the Z-Alizadeh Sani dataset (303 patients from Tehran’s Rajaei cardiovascular center). They employed six algorithms (DT, deep neural network, LR, RF, SVM, and XGBoost) to predict coronary artery disease (CAD). After Pearson-correlation feature selection, the best results were achieved by SVM and LR, each attaining 95.45% accuracy with 95.91% sensitivity, 91.66% specificity, F1≈0.969, and AUROC ≈0.98. Notably, although this study achieved excellent discrimination, it did not incorporate any post-hoc probability calibration or uncertainty analysis, the evaluation centered on accuracy and ROC curves alone. [ 14 ] took a different approach by leveraging larger, real-world data. In an interpretable LightGBM model for predicting coronary heart disease: Enhancing clinical decision-making with machine learning, they trained a LightGBM model on a U.S. CDC survey dataset (BRFSS 2015) and validated on two external cohorts (the Framingham Heart Study and the Z-Alizadeh Sani data). The LightGBM achieved about 90.6% accuracy (AUROC ∼81.1%) on the BRFSS training set, with slightly lower performance on Framingham (85% accuracy, ∼67% AUROC) and Z-Alizadeh (80% accuracy). While Deng et al. prioritized model interpretability (using SHAP values) and reported standard metrics like accuracy, precision, recall, and AUROC, they did not report any calibration-specific metrics (e.g. no ECE, Brier score, or reliability diagrams), nor did they apply Platt scaling or isotonic regression in their pipeline. Several recent studies have pushed accuracy to very high levels by combining datasets or using advanced ensembles, yet still largely ignore calibration. [ 15 ] proposed a hybrid approach for predicting heart disease using machine learning and an explainable AI method, where they combined a private hospital dataset with a public one and used feature selection plus ensemble methods. Their best model (an XGBoost classifier on a selected feature subset SF-2) achieved 97.57% accuracy with 96.61% sensitivity, 90.48% specificity, 95.00% precision, F1=92.68%, and 98% AUROC. Despite this impressive performance, no probability calibration was mentioned; the study’s contributions focused on maximizing accuracy and explaining feature impacts (via SHAP) rather than assessing prediction uncertainty. Using a clinical and biometric dataset (n=571) with a man-in-the-loop paradigm for assessing coronary artery disease, [ 16 ] compared standard ML classifiers; best accuracy reached ≈83% with expert input, but the work emphasized explainability over probabilistic calibration. To address the need for diverse and comprehensive research, we conducted a lightweight systematic review and surveyed a range of peer reviewed studies on ML for heart disease prediction in the last 5–10 years with focus on a minimum of 5,000 cohort patients built into the experimental setup. Table 1 summarizes key studies, including their data sources, ML approaches, and whether model calibration was evaluated (and how). View this table: View inline View popup Table 1. Recent ML-based heart disease prediction studies (2017-2025) - Summary of data, methods, and calibration evaluation. (Calibration metrics: HL = Hosmer–Lemeshow test; ECE = Expected Calibration Error; O/E = observed-to-expected ratio; Brier = Brier score.) Each study is cited with its year and reference number (e.g., 2025 [ 17 ] means the study was published in 2025 and is reference [ 17 ] in the reference list). 1.4.2 Gaps in Research Despite abundant work on ML-based heart disease prediction, there are clear gaps in the literature regarding probability calibration and uncertainty quantification. First, most studies prioritize discriminative performance (accuracy, F1, AUROC, etc.) and devote little or no attention to how well the predicted probabilities reflect true risk. As shown above, prior works seldom report calibration metrics like ECE or Brier score, nor do they plot reliability diagrams. For example, none of the 10+ studies reviewed applied calibration methods such as Platt scaling or isotonic regression to their classifiers, except for only one study [ 28 ]. This indicates a lack of focus on calibration quality, an important aspect if these models are to be used in clinical decision-making where calibrated risk predictions are crucial. Second, there is a lack of unified evaluation across multiple models and calibration techniques. Prior research typically evaluates a set of ML models on a dataset (as in comparative studies) but stops at reporting raw performance metrics. No study to date has systematically taken multiple classification models for heart disease and evaluated them before and after post-hoc calibration. This means it remains unclear how different algorithms (e.g. an SVM vs. a random forest) compare in terms of probability calibration (not just classification accuracy), and whether simple calibration methods can significantly improve their reliability. Furthermore, the interplay between model uncertainty (e.g. variance in predictions) and calibration has not been explored in this domain. Third, most heart disease prediction papers do not report uncertainty metrics or advanced calibration statistics. Metrics such as the Brier score (which combines calibration and refinement), the ECE (Expected Calibration Error), or even more domain-specific checks like Spiegelhalter’s Z-test for calibration, are virtually absent from prior studies. Sharpness (the concentration of predictive distributions) and other uncertainty measures are also not discussed. This leaves a research gap in understanding how confident we can be in these model predictions and where they might be over or under-confident. For instance, none of the reviewed studies provide reliability diagrams to visually inspect calibration; as a result, a model claiming 95% accuracy might still make poorly calibrated predictions (overestimating or underestimating risk). To the best of our knowledge, no prior work has offered a comprehensive evaluation of pre and post-calibration metrics across multiple models on the specific Kaggle heart disease dataset (1,025 records) used in this study. While several papers have used this or similar data for model comparison, none have examined calibration changes (ECE, log-loss, Brier, sharpness, Spiegelhalter’s Z-test, calibration curves) resulting from post-hoc calibration methods (Platt scaling, isotonic regression). In short, existing studies have left a critical question unanswered: if we calibrate our heart disease prediction models, do their confidence estimates become more trustworthy, and how does this vary by model? Addressing this gap is the focus of our work. We provide a thorough assessment of multiple classifiers before and after calibration, using a suite of calibration and uncertainty metrics not previously applied in this context, thereby advancing the evaluation criteria for heart disease ML models beyond conventional accuracy-based measures. 2 Materials and Methods 2.1 Research Methodology Overview This study employs a structured machine learning workflow to predict heart disease risk based on clinical and demographic variables. As outlined in Figure 1 , the process begins with the heart disease dataset, followed by data preprocessing, model selection and training, performance evaluation, and post-hoc calibration. Two calibration techniques (i.e Platt Scaling and Isotonic Regression) are applied to refine probabilistic outputs, with effectiveness assessed. Download figure Open in new tab Figure 1. Workflow Diagram for Heart Disease Prediction and Calibration Pipeline 2.2 Description of Dataset The Heart Disease dataset used in this study was sourced from Kaggle. It was originally sourced by merging data from four medical centers Cleveland, Hungary, Switzerland and VA Long Beach bringing the sample size to 1,025 records including 713 males (69.6%) and 312 females (30.4%), ages ranging between 29 - 77 years (median age ∼56). The dataset contains 14 variables encompassing demographic, clinical and diagnostic test features. Descriptions of the dataset are outlined in Table 2 . View this table: View inline View popup Table 2. Data description for heart disease The dataset was inspected for missing values and none was identified. The outcome variable (Heart Disease) was approximately balanced, with 51.3% of records labeled Disease and 48.7% labeled No Disease as shown in Figure 2 . Download figure Open in new tab Figure 2. Heart disease distribution 2.3 Data Preprocessing In this study, the dataset was separated into 13 predictors (i.e patient risk factors) and the 1 outcome feature (i.e the presence or risk of heart disease). Predictors were further divided into two groups: numerical features (e.g Age, RestingBP, Cholesterol) and categorical features (e.g ChestPainType, RestingECG, Thalassemia, Sex). We scale numerical features using a RobustScaler approach, which centres values around the median and spreads them according to the interquartile range. This method was selected due to it being less sensitive to outliers and skewness [ 30 ]. For categorical features, a One-Hot Encoding approach was applied, converting each category into binary (0/1) variables. This ensured that all categories were represented in a machine-readable format. 2.4 Model Selection In this work, we benchmark six models (spanning linear, non-linear and ensemble model architecture) to classify patients based on the presence or absence of heart disease. Models selected include Logistic Regression (LR), Support Vector Machines (SVM), Random Forest (RF), Extreme Gradient Boosting (XGBoost), K-Nearest Neighbors (KNN), and Naive Bayes (NB). The data was split into training (85%) and testing (15%) sets. We trained each model on the preprocessed training data and evaluated on the held-out test data. Logistic Regression (LR) Logistic Regression is a supervised machine learning model well-suited for binary classification, such as determining the presence or absence of heart disease. LR calculates the probability of a class (e.g., disease or no disease) by applying a sigmoid function to a weighted sum of predictor variables. Its strengths include simplicity, efficiency, and the ability to interpret coefficients as odds ratios, which is valuable in clinical settings for understanding feature importance and risk factors. Logistic Regression has a proven track record in medical research for risk stratification and is easily calibrated for probability estimation [ 31 ]. Support Vector Machines (SVM) Support Vector Machines are powerful, supervised classification models that work by finding the optimal hyperplane that separates classes in the feature space. SVMs excel at handling high-dimensional data and can model nonlinear relationships through kernel tricks, making them highly effective for complex medical datasets. Their ability to maximize the margin between classes reduces the likelihood of misclassification, which is especially useful when distinguishing subtle differences between patients with and without heart disease. SVMs are known for their robustness in real-world clinical prediction tasks [ 32 ] Random Forest (RF) Random Forest is an ensemble algorithm that builds multiple decision trees during training and aggregates their outputs via majority voting for classification. It is especially effective at capturing nonlinear relationships and interactions among risk factors in heart disease prediction. The ensemble nature of RF mitigates overfitting and variance, providing more reliable and stable predictions on diverse patient populations. Its embedded feature importance scores help clinicians identify key predictors of heart disease, further supporting its use in healthcare analytics [ 33 ] Extreme Gradient Boosting (XGBoost) XGBoost is a gradient boosting framework that creates a series of weak learners (usually decision trees) and optimizes them sequentially. It is renowned for combining high predictive accuracy with speed and efficiency, making it a top performer in medical classification challenges. XGBoost handles missing data gracefully and is robust to outliers, both common in clinical datasets. Its sophisticated regularization techniques reduce overfitting, and its model interpretability tools are advantageous for validating results in heart disease risk prediction [ 34 ]. K-Nearest Neighbors (KNN) K-Nearest Neighbors is a non-parametric classification method that predicts the class of a sample based on the majority class among its k closest neighbors in feature space. KNN is intuitive, easy to implement, and doesn’t make assumptions about data distribution, making it suitable for heterogeneous clinical datasets. KNN is effective at leveraging local patterns, which can be helpful for identifying at-risk heart disease patients based on similarity to previously observed cases. However, it can be sensitive to feature scaling and less efficient with very large datasets [ 35 ]. Naive Bayes (NB) Naive Bayes is a probabilistic classification algorithm that applies Bayes’ theorem, assuming feature independence. Its simplicity and computational efficiency make it attractive for medical tasks with many categorical variables. Despite its “naive” independence assumption, NB often performs surprisingly well for heart disease prediction because it can handle missing values, is robust with noisy data, and quickly estimates posterior probabilities. This makes it valuable for real-time risk assessment and decision support in clinical environments [ 36 ]. 2.5 Model Performance Metric We evaluated classification performance using Accuracy, ROC-AUC, Precision, Recall, and F1-score. Let TP, FP, TN, and FN denote true positives, false positives, true negatives, and false negatives, respectively. Accuracy Defined as , accuracy reflects the share of correctly classified cases in the test set. In clinical screening contexts where disease prevalence may be low accuracy depends on the decision threshold and can mask deficiencies under class imbalance, yielding seemingly strong performance while missing many positive cases [ 37 ]. ROC-AUC The receiver-operating-characteristic area summarizes discrimination across all thresholds; it equals the probability that a randomly selected positive receives a higher score than a randomly selected negative and ranges from 0.5 (no discrimination) to 1.0 (perfect). ROC-AUC is broadly used in clinical prediction for its threshold-agnostic view of separability, though it does not reflect calibration or the clinical costs of specific error types [ 38 ]. Precision Given by , quantifies how reliable positive alerts are among patients flagged as having heart disease, the fraction truly positive. As thresholds are lowered to capture more cases, precision typically decreases, illustrating the trade-off clinicians face between false alarms and case finding [ 39 ]. Recall Defined as , measures the proportion of truly diseased patients the model detects (sensitivity). Raising recall generally requires a lower threshold, which increases false positives and reduces precision; selecting an operating point should therefore reflect clinical consequences and disease prevalence [ 40 ]. F1-score The harmonic mean ,provides a single summary when both missed cases and false alarms matter. F1 is commonly reported in imbalanced biomedical tasks, though its interpretation should be complemented by other metrics given known limitations under skewed prevalence [ 41 ]. These metrics establish a consistent baseline for cross-model comparison and inform our subsequent calibration and uncertainty quantification analysis. 2.6 Post-Hoc Calibration Techniques Post-hoc calibration refers to techniques applied after model training that map raw scores to probabilities without changing the underlying classifier. In clinical settings where decisions hinge on risk estimates, these procedures use a held-out calibration set to fit a simple, typically monotonic mapping so that predicted probabilities better match observed event rates [ 9 ][ 42 ] [ 43 ]. In clinical text or imaging pipelines for heart-disease prediction, this is attractive: one can retain the trained model and its operating characteristics, then calibrate its outputs to yield probabilities that are more trustworthy for downstream decision thresholds, alerts, or shared decision-making [ 42 ][ 43 ]. For this study, we applied two post hoc calibration methods, Platt scaling (sigmoid calibration) and isotonic regression to adjust model outputs (such as prediction scores) into well-calibrated probabilities [ 7 ]. (i) Platt scaling works by fitting a smooth S-shaped (sigmoid) curve to the model’s scores using a separate validation set, so that predicted probabilities better match actual outcomes. This method is simple and efficient but assumes that the relationship between scores and probabilities follows a logistic pattern [ 9 ][ 44 ]. (ii) Isotonic regression is a more flexible, non-parametric method that does not assume any specific shape. Instead, it fits a step-like monotonic curve that can adapt to complex patterns in the data [ 45 ]. While this flexibility can better capture irregular relationships, it can also lead to overfitting if the validation dataset is small. In practice, Platt scaling is most useful when a sigmoid relationship is expected, while isotonic regression is preferred when the calibration pattern is unknown or more complex [ 9 ]. Using both methods together provides a robust calibration toolbox, ensuring reliable probability estimates across different models. This is particularly important in clinical applications such as heart disease prediction, where well-calibrated risk probabilities are essential for trustworthy model outputs and informed medical decision-making. 2.7 Model Uncertainty Quantification and Calibration Evaluation Metrics In this study, we measure the uncertainty of the models using these key calibration evaluation metrics: Brier Score, Expected Calibration Error (ECE), Log Loss, Spiegelhalter’s Z-score & p-value, Sharpness and reliability diagram. A combination of these metrics provides a holistic understanding of the effectiveness of each model in relation to quantifying model uncertainty. Brier Score The Brier Score measures the mean squared difference between predicted probabilities and the actual binary outcomes. Unlike accuracy which reduces predictions to “yes/no” and ignores the uncertainty behind probability values the Brier Score penalizes poorly calibrated or overly confident predictions. This makes it more informative for model uncertainty quantification, especially in clinical settings were knowing the probability of heart disease (and not just a binary label) aids risk discussions and decision-making. Lower Brier Scores indicate better calibrated and more reliable probability forecasts, a key aspect of clinical utility [ 46 ]. Expected Calibration Error (ECE) ECE summarizes how closely a model’s predicted probabilities match the observed frequencies of outcomes. It divides predictions into probability bins and measures mismatch between average predicted probability and actual outcome rate in each bin. In heart disease prediction, ECE helps verify if model confidence reflects real-world risks, ensuring patients with a predicted 70% heart disease risk, for example, actually face that risk. Lower ECE values indicate better calibrated models crucial for trusted clinical decision support [ 5 ]. Log Loss Log Loss (or cross-entropy loss) evaluates the uncertainty of probabilistic outputs by heavily penalizing confident but incorrect predictions. Log Loss is sensitive to how far predicted probabilities diverge from the actual class, providing a continuous measure of model reliability. For heart disease prediction, low Log Loss means the model rarely makes wildly overconfident errors, promoting safer, uncertainty-aware clinical interpretation [ 47 ]. Spiegelhalter’s Z-score & p-value Spiegelhalter’s Z-score tests overall calibration by comparing predicted probabilities to actual outcomes, normalized by their variance. A non-significant p-value suggests the model is well-calibrated; otherwise, the probabilistic forecasts may be systematically over- or under-confident. This calibration test is especially important in health applications, assuring clinicians that model probabilities are statistically valid reflections of true outcome chances [ 48 ]. Sharpness (variance of predicted probabilities) Sharpness measures the spread or concentration of predicted probabilities, independent of whether they’re correct. High sharpness means the model often predicts risks near 0 or 1, indicating confident, decisive forecasts. For heart disease prediction, greater sharpness is desirable only if paired with good calibration confident predictions should be correct. Thus, sharpness reveals how much intrinsic uncertainty the model expresses, helping physicians judge whether predictions are actionable or too vague for clinical use [ 49 ]. Reliability Diagram (calibration plot) A reliability diagram visualizes how predicted probabilities align with observed event rates by plotting, across confidence bins, the empirical outcome frequency against the mean predicted probability. A perfectly calibrated model traces the 45° diagonal line; systematic deviations reveal over or under-confidence [ 9 ]. Reliability diagrams are standard in both forecast verification and machine-learning calibration, and they are visual check of probability accuracy while leaving discrimination unaffected. Practical caveats include sensitivity to binning and sample size, and the fact that the plot alone does not convey how many samples fall in each bin often addressed by adding a companion confidence histogram.[ 5 ][ 49 ] 2.8 Experimental Setup Table 3 summarizes the experiment setup, evaluation choices, and preprocessing decisions. View this table: View inline View popup Download powerpoint Table 3. Pipeline decisions for Baseline Classification Performance & Calibration 3. Results 31 Baseline Model Performance (Before Calibration) To establish a performance benchmark for probability calibration, six (6) classification models were trained and evaluated. Table 4 summarizes evaluation metrics including accuracy, ROC AUC, precision, recall, F1 score, and 5-fold cross-validated accuracy (mean ± standard deviation) on the test set. View this table: View inline View popup Table 4. Performance metrics of baseline classification models (before calibration). The 6 classifiers all achieved competitive results, with ensemble-based models (Random Forest and XGBoost) attaining perfect scores across all reported metrics on the held-out test set. SVM also performed robustly, yielding an accuracy of 92.9%, ROC AUC of 99.4%, and F1 score of 92.8%. Logistic Regression and KNN followed closely, whereas Naive Bayes, while comparatively lower in performance (accuracy = 81.8%), remained competitive with an F1 score of 82.5%. However, the perfect test-set performance by the ensemble models may suggest overfitting, especially given their slightly lower cross-validation accuracy (Random Forest: 98.7%; XGBoost: 98.2%), warranting further scrutiny via probabilistic calibration metrics and establishing a foundation for subsequent analysis of uncertainty quantification to determine whether high predicted probabilities genuinely reflect empirical frequencies. 32 Calibration and Uncertainty Before & After Post-Hoc Adjustment We plot reliability diagrams and partitioned predicted probabilities into five equal-width bins. Given a test set of 153 instances (15% of the 1,025-record dataset), this produces approximately 30 samples per bin, striking a balance between calibration resolution and statistical stability. This aligns with prior calibration studies which recommend ≥10 bins depending on dataset size and caution against excessive binning for modestly sized datasets to maintain interpretability and reduce sampling noise [ 50 ]. As shown in Figure 3 (Before Calibration), XGBoost and Logistic Regression are close to the diagonal line, suggesting good intrinsic calibration. Naive Bayes and KNN showed substantial deviations from perfect calibration, with Naive Bayes underestimating outcome at intermediate predicted probability levels confirming the well-known tendency of these models to produce poorly calibrated outputs [ 7 ][ 51 ] and KNN displaying slightly erratic behavior, indicative of high-variance estimates [ 9 ]. Ensemble models, despite perfect accuracy, were overconfident in high-probability regions especially Random Forest where the predicted values exceeded observed outcomes. In addition, we observed that the XGBoost model’s pre-calibration reliability diagram was already close to the diagonal, indicating well-calibrated initial probabilities. Download figure Open in new tab Figure 3. Reliability diagrams before calibra Application of sigmoid calibration improved the calibration of Logistic Regression and SVM bringing their reliability curves closer to the diagonal ( Figure 4 ). However, sigmoid calibration failed to correct significant non-linear calibration errors in Random Forest and KNN, likely due to its monotonic, logistic-form constraint [ 52 ][ 53 ]. Naive Bayes also showed improvement, though some mid-range under confidence persisted. We noted that the reliability curve became more erratic for XGBoost which distorted the well-calibrated probabilities in Figure 3 rather than improving them suggesting additional calibration was not beneficial. Download figure Open in new tab Figure 4. Reliability diagrams after sigmoid calibration In contrast, isotonic calibration produced more flexible corrections ( Figure 4 ), significantly aligning the predictions of SVM and Naive Bayes with observed event frequencies. Whereas KNN again remained unstable across bins. These findings suggest that while sigmoid calibration is suitable for models with nearly linear miscalibration, isotonic regression better handles complex, non-monotonic distortions in probabilistic estimates [ 54 ][ 55 ]. Download figure Open in new tab Figure 5. Reliability diagrams after isotonic calibration To complement the visual insights from reliability diagrams, we conducted a quantitative evaluation using six (6) calibration metrics: Brier Score, Expected Calibration Error (ECE), Log Loss, Sharpness (Variance), Spiegelhalter’s Z-test and Spiegelhalter’s P-value shown in Table 5 , to quantify calibration quality and predictive uncertainty before and after applying sigmoid and isotonic calibration. View this table: View inline View popup Table 5. Calibration Metrics by Model & Calibration Method As shown in Figure 6 , the Brier Score decreased across most models following post-hoc calibration, confirming improved probability calibration. Random Forest exhibited a marked reduction from 0.007 (uncalibrated) to 0.002 after isotonic calibration. Similarly, XGBoost achieved the lowest overall score (0.002) post-isotonic, despite already having a near-perfect uncalibrated score (0.001), indicating minimal benefit from further calibration. Naive Bayes, initially the poorest performer (0.162), improved substantially to 0.132 after both sigmoid and isotonic adjustments. K-Nearest Neighbors showed a modest but consistent improvement from 0.065 (uncalibrated) to 0.063 under both calibration methods. Logistic Regression and SVM also exhibited slight reductions, with isotonic performing marginally better than sigmoid in both cases. These findings suggest that isotonic calibration offers the most consistent improvement, particularly for tree-based and probabilistic models, though its benefit diminishes when models are already well-calibrated. Download figure Open in new tab Figure 6. Brier Scores Before and After Calibration The Expected Calibration Error (ECE) generally decreased after calibration ( Figure 7 ). Notably, Naive Bayes showed the largest reduction, from 0.145 (uncalibrated) to 0.118 (Isotonic). Random Forest similarly improved from 0.051 to 0.011 with isotonic calibration. Support Vector Machine also benefited, reducing from 0.086 to 0.044 post-isotonic. However, Logistic Regression and XGBoost exhibited minimal change, suggesting that they were already well-calibrated. K-Nearest Neighbors (KNN), in contrast, worsened with sigmoid calibration (from 0.035 to 0.081) before slightly recovering with isotonic (0.038). Isotonic regression outperformed sigmoid calibration across most models, demonstrating better reliability in probability estimates. Download figure Open in new tab Figure 7. Expected Calibration Error (ECE) Before and After Calibration Log Loss largely supports the trends observed in Brier Scores ( Figure 8 ). XGBoost exhibited the lowest log loss across all conditions, with minimal variation from 0.013 (None) to 0.014 (Isotonic) reinforcing its strong calibration by default. Random Forest benefited notably from calibration, improving from 0.056 (None) to 0.012 (Isotonic). Conversely, Naive Bayes demonstrated severe miscalibration in its raw form, with an uncalibrated log loss of 1.936, which was substantially reduced to 0.446 following isotonic regression. This highlights its tendency toward overconfident yet incorrect predictions. K-Nearest Neighbors (KNN) also showed calibration sensitivity, where sigmoid calibration worsened performance from 0.183 to 0.207, while isotonic returned it to baseline. Logistic Regression and SVM exhibited modest improvements, with SVM decreasing from 0.142 to 0.133 post-isotonic. Isotonic regression consistently lowered log loss, further confirming its advantage for improving model calibration under most conditions. Download figure Open in new tab Figure 8. Log Loss Before and After Calibration Spiegelhalter’s Z-Score and p-value ( Figure 9 ) offer a statistical measure of how well predicted probabilities match actual outcomes [ 48 ]. Ideally, a Z-score close to zero paired with a p-value > 0.05 indicates statistically sound calibration. Among all models, Random Forest and XGBoost consistently displayed near-zero Z-scores and high p-values across all calibration settings (p > 0.75), affirming stable and well-calibrated predictions. Naive Bayes, which was severely miscalibrated in earlier metrics, improved substantially after isotonic calibration (Z = 0.30, p = 0.77), compared to its uncalibrated state (Z = -0.60, p = 0.55). Similarly, Logistic Regression achieved near-zero Z-scores (e.g., Z = -0.04 post-isotonic) and strong p-values (≥ 0.88), confirming minimal deviation from expected reliability. Download figure Open in new tab Figure 9. Heatmaps of Spiegelhalter’s Z-score (left) and p-value (right) across models and calibration methods. K-Nearest Neighbors (KNN) exhibited negative Z-scores across all settings (e.g., Z = -1.21, -0.98), yet its p-values increased with calibration (up to p = 0.33), suggesting partial improvement. Support Vector Machine (SVM) had the highest Z-scores overall (1.02-1.11), and consistently low p-values (≈ 0.27-0.34), implying persistent miscalibration despite adjustments. These results further emphasize that Spiegelhalter’s test provides complementary validation to Brier, ECE, and Log Loss, with isotonic calibration yielding statistically significant reliability gains in most models. To explore the relationship between calibration and prediction quality, we plotted Expected Calibration Error (ECE) against the Brier Score for all model-calibration combinations ( Figure 10 ). Ideally, well-calibrated and accurate models should lie close to the diagonal line, where ECE and Brier Score are proportionally aligned. This was observed in XGBoost and Random Forest, which consistently clustered near the bottom-left corner, regardless of calibration method, indicating strong predictive reliability and calibration consistency. K-Nearest Neighbors (sigmoid) and Naive Bayes (uncalibrated) appeared well above the diagonal, indicating inflated ECE relative to their Brier Scores. This suggests that while these models may exhibit seemingly reasonable predictive accuracy (as measured by Brier Score), their probability estimates are poorly calibrated, which could undermine trust in probabilistic outputs. Download figure Open in new tab Figure 10. Scatter Plot of Brier Score vs. Expected Calibration Error (ECE) Across All Models and Calibration States This divergence highlights a key distinction: while Brier Score captures both calibration and classification performance, ECE specifically isolates calibration drift, making it an essential complement in evaluation. These findings support prior recommendations [ 5 ] to jointly assess calibration with multiple complementary metrics, thereby providing a multidimensional view of model trustworthiness, especially post-calibration. 3.3 Sharpness Analysis Figure 11 presents the sharpness of model predictions, measured as the variance of predicted probabilities. Higher variance indicates more confident predictions, while lower variance suggests more cautious, uniform output distributions. Random Forest and XGBoost exhibited the sharpest predictions, with Random Forest (Isotonic) reaching the highest variance at 0.240, followed closely by XGBoost (Isotonic) at 0.238 and XGBoost (uncalibrated) at 0.237. These results demonstrate the models’ inherent confidence, which was largely preserved post-calibration. Naive Bayes (uncalibrated) also showed relatively high sharpness at 0.224, but experienced a substantial reduction after calibration, dropping to 0.128 with isotonic and further down to 0.086 with sigmoid. This steep decline in sharpness suggests that Naive Bayes, while initially overconfident, became excessively conservative post-calibration especially under sigmoid, which applied the strongest regularisation. Download figure Open in new tab Figure 11. Variance of predicted probabilities (“sharpness”) across models and calibration methods. Logistic Regression followed a similar trend, with sharpness decreasing from 0.141 (uncalibrated) to 0.134 (Sigmoid) and 0.128 (Isotonic), indicating calibration-induced smoothing. K-Nearest Neighbors and Support Vector Machine exhibited moderate sharpness across all methods, though in both cases sigmoid calibration led to noticeable reductions compared to isotonic. For instance, KNN decreased from 0.187 (uncalibrated) to 0.157 (Sigmoid), and SVM dropped from 0.179 (uncalibrated) to 0.167 (Sigmoid). 4 Interpretation of Results This study demonstrates the significant impact of post-hoc calibration methods on model confidence, calibration quality, and statistical reliability in heart disease prediction. Isotonic regression consistently outperformed sigmoid calibration across most metrics, especially in reducing ECE and Log Loss. For example, Random Forest’s ECE dropped from 0.051 (uncalibrated) to 0.011 (isotonic), with Log Loss reducing from 0.056 to 0.012 ( Figure 7 - 8 ). Naive Bayes also showed strong gains, with Log Loss improving from 1.936 to 0.446 and its Z p-value increasing from 0.08 to 0.83 ( Figure 9 ), confirming isotonic regression’s effectiveness for overconfident models. These findings support existing theory that sigmoid calibration, being a parametric method, often suppresses extreme probabilities more aggressively than isotonic calibration, which is non-parametric and locally adaptive. While such smoothing can enhance calibration reliability, it may also trade off against the informativeness of predictions highlighting the need to balance confidence and calibration depending on the deployment context [ 9 ]. In clinical applications, where treatment decisions may hinge on predicted risk probabilities, miscalibrated models can yield overconfident but incorrect predictions, potentially leading to under-treatment or overtreatment of patients. For example, Naive Bayes pre-calibration predicted extreme probabilities but had poor alignment with actual outcomes (high Log Loss, low p-value), which post-calibration successfully corrected. This highlights the need for calibration pipelines in AI-assisted diagnostics to improve trustworthiness and reduce the risk of confidence-driven misdiagnosis [ 56 ]. Moreover, the use of p-values and Z-scores provides a statistical basis for interpreting model reliability beyond visual tools, such as reliability diagrams. Previous studies have employed post-hoc calibration in medical machine learning, but many rely solely on a single metric such as Brier Score or ECE [ 5 ], [ 57 ]. This work expands the scope by combining six (6) complementary metrics: Brier, ECE, Log Loss, Z-score, P-value and Sharpness across six (6) classifiers, offering a more multidimensional view of calibration performance. The incorporation of Sharpness ( Figure 11 ), reflecting the variance of predicted probabilities, adds an important lens into model confidence. For instance, sigmoid calibration consistently reduced sharpness more than isotonic, as observed in Naive Bayes (0.086 post-sigmoid vs. 0.128 post-isotonic), highlighting a trade-off between confidence and calibration smoothness. Few prior studies visualize calibration results this comprehensively across model types. Our comparative heatmaps ( Figures 9 ) and calibration-vs-confidence scatter plots ( Figure 10 ) allow practitioners to assess which models and calibrations offer both reliability and actionable probabilities, a balance rarely presented in calibration literature but essential for healthcare deployment [ 58 ]. The contribution of this work lies in demonstrating that calibration effectiveness is highly model-dependent and we present a multi-metric evaluation framework for assessing model calibration in binary heart disease prediction. Our findings show that some models such as Naive Bayes and Random Forest benefit from isotonic calibration, while others such as KNN deteriorate, especially under sigmoid. By introducing sharpness as an evaluative dimension, we go beyond correctness to assess confidence dispersion, an area still underreported in medical AI calibration research. This dual lens of both reliability and confidence is essential for clinical interpretability, risk stratification, and model auditability, especially when deploying decision-support tools in real-world healthcare systems. This study is limited by the size of the dataset (N=1,025), which may constrain generalizability to broader clinical populations. Although five-fold cross-validation was employed to mitigate variance, the absence of an external validation cohort restricts our ability to assess overfitting or calibration robustness in unseen data. Additionally, we did not test more advanced calibration methods such as Bayesian binning or temperature scaling, which could further refine probability estimates. Future research should explore these techniques on larger datasets and assess calibration dynamics over time or across demographic subgroups for fairness assurance. 5. Conclusion This study evaluated the calibration performance of six classification models for heart disease prediction, using a range of post-hoc techniques and uncertainty metrics. While most models exhibited high accuracy and ROC AUC, their probability estimates often lacked alignment with observed outcomes, underscoring the importance of calibration beyond traditional performance metrics. Post-hoc calibration particularly isotonic regression demonstrated consistent improvements in reducing Brier Score, Expected Calibration Error (ECE), and Log Loss across models, with Spiegelhalter’s Z-test further validating statistical calibration gains. Compared to sigmoid, isotonic calibration proved more effective for correcting non-linear miscalibrations, especially in probabilistically unstable models such as Naive Bayes and ensemble classifiers. However, models such as KNN remained challenging to calibrate regardless of method, highlighting the limits of post-hoc approaches for certain decision boundary behaviors. These findings reinforce the need to assess and calibrate predictive uncertainty when deploying machine learning models in clinical settings. Well-calibrated probabilities are essential for accuracy, for supporting trustworthy decision-making and risk communication. Given its flexibility and empirical advantage, isotonic regression is recommended as the preferred calibration method in healthcare AI applications. Data Availability All data produced in the present work are contained in the manuscript Conflict of Interest The authors declare that no funding was received from any affiliated institution for this research. The work was conducted independently and the views expressed are solely those of the authors. Data and Code Availability The datasets analyzed during the current study is publicly available at Kaggle ( https://www.kaggle.com/datasets/johnsmith88/heart-disease-dataset ). The code supporting the findings of this study is available from the corresponding author on reasonable request. About the authors Peter Adebayo Odesola holds a master’s degree in Artificial Intelligence and Data Science from Solent University (2022) and a postgraduate diploma in Education. His expertise spans data analytics, machine learning, and AI in healthcare, with projects on predictive modelling, automation, and visualisation that advance data-driven solutions in both academic and professional contexts. Adewale Alex Adegoke holds a master’s degree in Applied Artificial Intelligence and Data Science from Southampton Solent University, completed in 2023. During his time at the university, he contributed to various innovative research initiatives. Adewale has since worked as a Data Scientist at 10Alytics, where he applied advanced technologies and research methodologies to craft data-driven solutions for businesses. He currently serves as a Data Systems Manager at the Westminster Foundation for Democracy (WFD), UK. His research focuses on the application of advanced data analytics and machine learning techniques to solve real-world challenges. Idris Babalola is a Senior Data Scientist with the Department of Health and Social Care, UK. He has held previous part-time roles at Solent University United Kingdom as a Data scientist, MSc research supervisor as well as Associate lecturer in Computing. He holds a MSc in AI and Data science from Solent University (2022). His research interest lies in the use of AI for Healthcare utilising data science skills, NLP and large language models. References [1]. ↵ World Health Organization , “Cardiovascular diseases (CVDs),” July 2025 . [2]. ↵ D. Dey , P. J. Slomka , P. Leeson , D. Comaniciu , M. L. Bots , “ Artificial intelligence in cardiovascular imaging: JACC state-of-the-art review ,” Journal of the American College of Cardiology , vol. 73 , no. 11 , pp. 1317 – 1335 , 2019 , doi: 10.1016/j.jacc.2018.12.054 . OpenUrl FREE Full Text [3]. ↵ S. Srinivasan , S. Gunasekaran , S. K. Mathivanan , et al. , “ An active learning machine technique based prediction of cardiovascular heart disease from UCI-repository database ,” Scientific Reports , vol. 13 , art. no. 13588 , 2023 , doi: 10.1038/s41598-023-40717-1 . OpenUrl CrossRef [4]. ↵ S. F. Weng , J. Reps , J. Kai , J. M. Garibaldi , N. Qureshi , “ Can machine-learning improve cardiovascular risk prediction using routine clinical data? ,” PLOS ONE , vol. 12 , no. 4 , e0174944 , 2017 , doi: 10.1371/journal.pone.0174944 . OpenUrl CrossRef PubMed [5]. ↵ C. Guo , G. Pleiss , Y. Sun , K. Q. Weinberger , “ On calibration of modern neural networks ,” in Proceedings of the 34th International Conference on Machine Learning (ICML) , pp. 1321 – 1330 , 2017 , doi: 10.48550/arXiv.1706.04599 . OpenUrl CrossRef [6]. ↵ H. Jiang , B. Kim , M. Y. Guan , M. Gupta , “ To trust or not to trust a classifier ,” in Proceedings of the 32nd International Conference on Neural Information Processing Systems (NeurIPS 2018), Montreal, Canada , pp. 5546 – 5557 , 2018 . [7]. ↵ B. Zadrozny , C. Elkan , “ Transforming classifier scores into accurate multiclass probability estimates ,” Proceedings of the eighth ACM SIGKDD international conference on Knowledge discovery and data mining , pp. 694 – 699 , 2002 , doi: 10.1145/775047.775151 . OpenUrl CrossRef [8]. ↵ S. Tonekaboni , S. Joshi , M. D. McCradden , A. Goldenberg , “ What clinicians want: contextualizing explainable machine learning for clinical end use ,” in Proceedings of the 4th Machine Learning for Healthcare Conference (PMLR , vol. 106 ), pp. 359 – 380 , 2019 . URL: https://proceedings.mlr.press/v106/tonekaboni19a.html OpenUrl [9]. ↵ A. Niculescu-Mizil , R. Caruana , “ Predicting good probabilities with supervised learning ,” Proceedings of the 22nd International Conference on Machine Learning (ICML) , pp. 625 – 632 , 2005 , doi: 10.1145/1102351.1102430 . OpenUrl CrossRef [10]. ↵ Ali , M. M. , Paul , B. K. , Ahmed , K. , Bui , F. M. , Quinn , J. M. W. , & Moni , M. A. ( 2021 ). Heart disease prediction using supervised machine learning algorithms: Performance analysis and comparison . Computers in biology and medicine , 136 , 104672 . doi: 10.1016/j.compbiomed.2021.104672 OpenUrl CrossRef PubMed [11]. ↵ Ghosh , S. , & Islam , M. A. ( 2023 ). Performance Evaluation and Comparison of Heart Disease Prediction Using Machine Learning Methods with Elastic Net Feature Selection . American Journal of Applied Mathematics and Statistics , 11 ( 2 ), 35 - 49 . DOI: 10.12691/ajams-11-2-1 OpenUrl CrossRef [12]. ↵ Ahmad , G. N. , Shafiullah Fatima , H. , Abbas , M. , Rahman , O. , Imdadullah, & Alqahtani , M. S. ( 2022 ). Mixed Machine Learning Approach for Efficient Prediction of Human Heart Disease by Identifying the Numerical and Categorical Features . Applied Sciences , 12 ( 15 ), 7449 . doi: 10.3390/app12157449 OpenUrl CrossRef [13]. ↵ Sayadi , M. , Varadarajan , V. , Sadoughi , F. , Chopannejad , S. , & Langarizadeh , M. ( 2022 ). A Machine Learning Model for Detection of Coronary Artery Disease Using Noninvasive Clinical Parameters . Life (Basel, Switzerland) , 12 ( 11 ), 1933 . doi: 10.3390/life12111933 OpenUrl CrossRef PubMed [14]. ↵ Deng , L. , Lu , K. , & Hu , H. ( 2025 ). An interpretable LightGBM model for predicting coronary heart disease: Enhancing clinical decision-making with machine learning . PLOS ONE , 20 ( 9 ), e0330377 . doi: 10.1371/journal.pone.0330377 OpenUrl CrossRef PubMed [15]. ↵ El-Sofany , H. , Bouallegue , B. & El-Latif , Y.M.A. A proposed technique for predicting heart disease using machine learning algorithms and an explainable AI method . Sci Rep 14 , 23277 ( 2024 ). doi: 10.1038/s41598-024-74656-2 OpenUrl CrossRef PubMed [16]. ↵ Samaras A-D , Moustakidis S , Apostolopoulos ID , Papandrianos N , Papageorgiou E. Classification models for assessing coronary artery disease instances using clinical and biometric data: an explainable man-in-the-loop approach . Sci Rep . 2023 ; 13 : 6668 . doi: 10.1038/s41598-023-33500-9 . OpenUrl CrossRef PubMed [17]. ↵ Vu , T. , Kokubo , Y. , Inoue , M. , Yamamoto , M. , Mohsen , A. , Martin-Morales , A. , Dawadi , R. , Inoue , T. , Tay , J. T. , Yoshizaki , M. , Watanabe , N. , Kuriya , Y. , Matsumoto , C. , Arafa , A. , Nakao , Y. M. , Kato , Y. , Teramoto , M. , & Araki , M. ( 2025 ). Machine Learning Model for Predicting Coronary Heart Disease Risk: Development and Validation Using Insights From a Japanese Population-Based Study . JMIR cardio , 9 , e68066 . DOI: 10.2196/68066 OpenUrl CrossRef [18]. M. U. Rehman , S. Naseem , A. U. R. Butt , et al. , “ Predicting coronary heart disease with advanced machine learning classifiers for improved cardiovascular risk assessment ,” Sci. Rep ., vol. 15 , art. 13361 , 2025 . DOI: 10.1038/s41598-025-96437-1 OpenUrl CrossRef [19]. G. M. Rao , D. Ramesh , V. Sharma , et al. , “ AttGRU-HMSI: enhancing heart disease diagnosis using hybrid deep learning approach ,” Sci. Rep ., vol. 14 , art. 7833, 2024 . DOI: 10.1038/s41598-024-56931-4 OpenUrl CrossRef [20]. J. You , Y. Guo , J. J. Kang , et al. , “ Development of machine learning-based models to predict 10-year risk of cardiovascular disease: a prospective cohort study ,” Stroke Vasc. Neurol ., vol. 8 , no. 6 , pp. 475 – 485 , 2023 . DOI: 10.1136/svn-2023-002332 OpenUrl Abstract / FREE Full Text [21]. Li , C. , Liu , X. , Shen , P. , Sun , Y. , Zhou , T. , Chen , W. , Chen , Q. , Lin , H. , Tang , X. , & Gao , P. ( 2024 ). Improving cardiovascular risk prediction through machine learning modelling of irregularly repeated electronic health records . European Heart Journal -Digital Health , 5 ( 1 ), 30 – 40 . DOI: 10.1093/ehjdh/ztad058 OpenUrl CrossRef PubMed [22]. J. W. Hughes , J. Tooley , J. T. Soto , et al. , “ A deep learning-based electrocardiogram risk score for long term cardiovascular death and disease ,” npj Digit. Med ., vol. 6 , art. 169, 2023 . DOI: 10.1038/s41746-023-00916-6 OpenUrl CrossRef PubMed [23]. Xi , Y. , Wang , H. , & Sun , N. ( 2022 ). Machine learning outperforms traditional logistic regression and offers new possibilities for cardiovascular risk prediction: A study involving 143,043 Chinese patients with hypertension . Frontiers in cardiovascular medicine , 9 , 1025705 . DOI: 10.3389/fcvm.2022.1025705 OpenUrl CrossRef PubMed [24]. S. Y. Cho , S. H. Kim , S. H. Kang , et al. , ( 2021 ) “ Pre-existing and machine learning-based models for cardiovascular risk prediction ,” Sci. Rep ., vol. 11 , art. 8886. DOI: 10.1038/s41598-021-88257-w OpenUrl CrossRef [25]. R. Khera , J. Haimovich , N. C. Hurley , et al. , “ Use of machine learning models to predict death after acute myocardial infarction ,” JAMA Cardiol ., vol. 6 , no. 6 , pp. 633 – 641 , 2021 . DOI: 10.1001/jamacardio.2021.0122 OpenUrl CrossRef [26]. L. R. Guarneros-Nolasco , N. A. Cruz-Ramos , G. Alor-Hernández , L. Rodríguez-Mazahua , J.L. Sánchez-Cervantes , “ Identifying the main risk factors for cardiovascular diseases prediction using machine learning algorithms ,” Mathematics , vol. 9 , no. 20 , art. 2537 , 2021 . DOI: 10.3390/math9202537 OpenUrl CrossRef [27]. L. Yang , H. Wu , X. Jin , et al. , “ Study of cardiovascular disease prediction model based on random forest in eastern China ,” Sci. Rep ., vol. 10 , art. 5245 , 2020 . DOI: 10.1038/s41598-020-62133-5 OpenUrl CrossRef PubMed [28]. ↵ A. M. Alaa , T. Bolton , E. Di Angelantonio , J. H. F. Rudd , M. van der Schaar , “ Cardiovascular disease risk prediction using automated machine learning: a prospective study of 423,604 UK Biobank participants ,” PLoS ONE , vol. 14 , no. 5 , e0213653 , 2019 . DOI: 10.1371/journal.pone.0213653 OpenUrl CrossRef PubMed [29]. S. F. Weng , J. Reps , J. Kai , J. M. Garibaldi , N. Qureshi , “ Can machine-learning improve cardiovascular risk prediction using routine clinical data? ” PLoS ONE , vol. 12 , no. 4 , e0174944 , 2017 . DOI: 10.1371/journal.pone.0174944 OpenUrl CrossRef PubMed [30]. ↵ P. J. Rousseeuw and C. Croux , “ Alternatives to the Median Absolute Deviation ,” Journal of the American Statistical Association , vol. 88 , no. 424 , pp. 1273 – 1283 , Dec . 1993 . DOI: 10.1080/01621459.1993.10476408 . OpenUrl CrossRef [31]. ↵ C. Y. J. Peng , K. L. Lee , G. M. Ingersoll , “ An introduction to logistic regression analysis and reporting ,” Journal of Educational Research , vol. 96 , no. 1 , pp. 3 – 14 , 2002 , doi: 10.1080/00220670209598786 . OpenUrl CrossRef Web of Science [32]. ↵ T. S. Furey , N. Cristianini , N. Duffy , D. W. Bednarski , M. Schummer , D. Haussler , “ Support vector machine classification and validation of cancer tissue samples using microarray expression data ,” Bioinformatics , vol. 16 , no. 10 , pp. 906 – 914 , 2000 , doi: 10.1093/bioinformatics/16.10.906 . OpenUrl CrossRef PubMed Web of Science [33]. ↵ L. Breiman , “ Random Forests ,” Machine Learning , vol. 45 , no. 1 , pp. 5 – 32 , Oct . 2001 . DOI: 10.1023/A:1010933404324 . OpenUrl CrossRef [34]. ↵ T. Chen , C. Guestrin , “ XGBoost: A scalable tree boosting system ,” in Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD) , pp. 785 – 794 , 2016 , doi: 10.1145/2939672.2939785 . OpenUrl CrossRef [35]. ↵ T. Cover , P. Hart , “ Nearest neighbor pattern classification ,” IEEE Transactions on Information Theory , vol. 13 , no. 1 , pp. 21 – 27 , 1967 , doi: 10.1109/TIT.1967.1053964 . OpenUrl CrossRef [36]. ↵ G. H. John and P. Langley , “ Estimating Continuous Distributions in Bayesian Classifiers ,” in Proceedings of the Eleventh Conference on Uncertainty in Artificial Intelligence (UAI’95), Montreal, Quebec, Canada , 1995 , pp. 338 – 345 . DOI: 10.5555/2074158.2074196 . OpenUrl CrossRef [37]. ↵ Saito , T. , & Rehmsmeier , M. ( 2015 ). The precision-recall plot is more informative than the ROC plot when evaluating binary classifiers on imbalanced datasets . PLOS ONE , 10 ( 3 ), e0118432 . DOI: 10.1371/journal.pone.0118432 OpenUrl CrossRef PubMed [38]. ↵ Fawcett , T. ( 2006 ). An introduction to ROC analysis . Pattern Recognition Letters , 27 ( 8 ), 861 – 874 . DOI: 10.1016/j.patrec.2005.10.010 OpenUrl CrossRef Web of Science [39]. ↵ Davis , J. , & Goadrich , M. ( 2006 ). The relationship between precision-recall and ROC curves . Proceedings of the 23rd International Conference on Machine Learning (ICML) , 233 – 240 . DOI: 10.1145/1143844.1143874 OpenUrl CrossRef [40]. ↵ Steyerberg , E. W. ( 2019 ). Clinical Prediction Models: A Practical Approach to Development, Validation, and Updating (2nd ed .). Springer . DOI: 10.1007/978-3-030-16399-0 OpenUrl CrossRef PubMed [41]. ↵ Chicco , D. , & Jurman , G. ( 2020 ). The advantages of the Matthews correlation coefficient (MCC) over F1 score and accuracy in binary classification evaluation . BMC Genomics , 21 , 6 . DOI: 10.1186/s12864-019-6413-7 OpenUrl CrossRef PubMed [42]. ↵ Penso , C. , Frenkel , L. , & Goldberger , J. ( 2024 ). Confidence calibration of a medical imaging classification system that is robust to label noise . IEEE Transactions on Medical Imaging , 43 ( 6 ), 2050 – 2060 . DOI: 10.1109/TMI.2024.3353762 OpenUrl CrossRef PubMed [43]. ↵ Van Calster , B. , McLernon , D.J. , van Smeden , M. et al. Calibration: the Achilles heel of predictive analytics . BMC Med 17 , 230 ( 2019 ). DOI: 10.1186/s12916-019-1466-7 OpenUrl CrossRef PubMed [44]. ↵ Kull , M. , Filho , T.D. , & Flach , P.A. ( 2017 ). Beyond sigmoids: How to obtain well-calibrated probabilities from binary classifiers with beta calibration . Electronic Journal of Statistics , 11 , 5052 – 5080 . DOI: 10.1214/17-EJS1338SI OpenUrl CrossRef [45]. ↵ R. E. Barlow and H. D. Brunk , “ The Isotonic Regression Problem and Its Dual ,” Journal of the American Statistical Association , vol. 67 , no. 337 , pp. 140 – 147 , Mar . 1972 . DOI: 10.1080/01621459.1972.10481216 OpenUrl CrossRef [46]. ↵ M. Assel , D. Sjoberg , A. Vickers , “ The Brier score does not evaluate the clinical utility of diagnostic tests or prediction models ,” Diagnostic and Prognostic Research , vol. 1 , 2017 , doi: 10.1186/s41512-017-0020-3 . OpenUrl CrossRef PubMed [47]. ↵ D. R. Cox , “ The Regression Analysis of Binary Sequences ,” Journal of the Royal Statistical Society. Series B (Methodological) , vol. 20 , no. 2 , pp. 215 – 242 , 1958 . DOI: 10.1111/j.2517-6161.1958.tb00292.x OpenUrl CrossRef [48]. ↵ D. J. Spiegelhalter , “ Probabilistic Prediction in Patient Management and Clinical Trials ,” Statistics in Medicine , vol. 5 , no. 5 , pp. 421 – 433 , Sep . 1986 . DOI: 10.1002/sim.4780050506 OpenUrl CrossRef PubMed Web of Science [49]. ↵ T. Gneiting , F. Balabdaoui , and A. E. Raftery , “ Probabilistic Forecasts, Calibration and Sharpness ,” Journal of the Royal Statistical Society: Series B (Statistical Methodology) , vol. 69 , no. 2 , pp. 243 – 268 , Apr . 2007 . DOI: 10.1111/j.1467-9868.2007.00587.x OpenUrl CrossRef [50]. ↵ G. Bröcker , L. A. Smith , “ Increasing the reliability of reliability diagrams ,” Weather and Forecasting , vol. 22 , no. 3 , pp. 651 – 661 , 2007 , doi: 10.1175/WAF993.1 . OpenUrl CrossRef [51]. ↵ Meelis Kull , Telmo M. Silva Filho , Peter Flach ( 2017 ) “Beyond sigmoids: How to obtain well-calibrated probabilities from binary classifiers with beta calibration,” Electronic Journal of Statistics, Electron . J. Statist . 11 ( 2 ), 5052 – 5080 . DOI: 10.1214/17-EJS1338SI OpenUrl CrossRef [52]. ↵ Lin , HT. , Lin , CJ. & Weng , R.C. ( 2007 ) A note on Platt’s probabilistic outputs for support vector machines . Mach Learn 68 , 267 – 276 . DOI: 10.1007/s10994-007-5018-6 OpenUrl CrossRef [53]. Böken , B. ( 2021 ). On the appropriateness of Platt scaling in classifier calibration . Inf. Syst ., 95 , 101641 . DOI: 10.1016/J.IS.2020.101641 OpenUrl CrossRef [54]. ↵ Naeini , M.P. ,, Cooper , G.F. ,, Hauskrecht , M. Obtaining well calibrated probabilities using Bayesian binning . In Proceedings of the 29th AAAI Conference on Artificial Intelligence (AAAI 2015); Austin, TX, USA , 2015 ; pp. 2901 – 2907 . DOI: 10.1609/aaai.v29i1.9602 OpenUrl CrossRef [55]. ↵ Yingxiang Huang , Wentao Li , Fima Macheret , Rodney A Gabriel , Lucila Ohno-Machado , A tutorial on calibration measurements and calibration models for clinical prediction models , Journal of the American Medical Informatics Association , Volume 27 , Issue 4 , April 2020 , Pages 621 – 633 , doi: 10.1093/jamia/ocz228 OpenUrl CrossRef PubMed [56]. ↵ E. W. Steyerberg , A. J. Vickers , N. R. Cook , T. Gerds , M. Gonen , N. Obuchowski , M. W. Kattan , “ Assessing the performance of prediction models: a framework for traditional and novel measures ,” Epidemiology , vol. 21 , no. 1 , pp. 128 – 138 , 2010 , doi: 10.1097/EDE.0b013e3181c30fb2 . OpenUrl CrossRef PubMed Web of Science [57]. ↵ X. Jiang , M. Osl , J. Kim , L. Ohno-Machado , “ Calibrating predictive model estimates to support personalized medicine ,” Journal of the American Medical Informatics Association , vol. 19 , no. 2 , pp. 263 – 274 , 2012 , doi: 10.1136/amiajnl-2011-000291 . OpenUrl CrossRef PubMed [58]. ↵ V. Kuleshov , N. Fenner , S. Ermon , “ Accurate uncertainties for deep learning using calibrated regression ,” in Proceedings of the 35th International Conference on Machine Learning (ICML) , pp. 2796 – 2804 , 2018 , doi: 10.48550/arXiv.1807.00263 . OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted September 30, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Model uncertainty quantification: A post hoc calibration approach for heart disease prediction Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Model uncertainty quantification: A post hoc calibration approach for heart disease prediction Peter Adebayo Odesola , Adewale Alex Adegoke , Idris Babalola medRxiv 2025.09.28.25336834; doi: https://doi.org/10.1101/2025.09.28.25336834 Share This Article: Copy Citation Tools Model uncertainty quantification: A post hoc calibration approach for heart disease prediction Peter Adebayo Odesola , Adewale Alex Adegoke , Idris Babalola medRxiv 2025.09.28.25336834; doi: https://doi.org/10.1101/2025.09.28.25336834 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Cardiovascular Medicine Subject Areas All Articles Addiction Medicine (570) Allergy and Immunology (863) Anesthesia (301) Cardiovascular Medicine (4442) Dentistry and Oral Medicine (444) Dermatology (383) Emergency Medicine (609) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1511) Epidemiology (15231) Forensic Medicine (30) Gastroenterology (1126) Genetic and Genomic Medicine (6610) Geriatric Medicine (668) Health Economics (998) Health Informatics (4542) Health Policy (1370) Health Systems and Quality Improvement (1613) Hematology (543) HIV/AIDS (1266) Infectious Diseases (except HIV/AIDS) (15924) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (147) Nephrology (668) Neurology (6608) Nursing (346) Nutrition (999) Obstetrics and Gynecology (1146) Occupational and Environmental Health (957) Oncology (3338) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1693) Pharmacology and Therapeutics (692) Primary Care Research (712) Psychiatry and Clinical Psychology (5450) Public and Global Health (9240) Radiology and Imaging (2203) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (596) Sexual and Reproductive Health (714) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a020bc5b8a804807',t:'MTc3OTgzOTk4OQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();