Conformal Prediction and Venn-ABERS Calibration for Reliable Machine Learning-Based Prediction of Bacterial Infection Focus

Conformal Prediction and Venn-ABERS Calibration for Reliable Machine Learning-Based Prediction of Bacterial Infection Focus | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Conformal Prediction and Venn-ABERS Calibration for Reliable Machine Learning-Based Prediction of Bacterial Infection Focus View ORCID Profile Jacob Bahnsen Schmidt , View ORCID Profile Karen Leth Nielsen , View ORCID Profile Dmytro Strunin , View ORCID Profile Nikolai Søren Kirkby , View ORCID Profile Jesper Qvist Thomassen , Steen Christian Rasmussen , View ORCID Profile Ruth Frikke-Schmidt , View ORCID Profile Frederik Boëtius Hertz , View ORCID Profile Allan Peter Engsig-Karup doi: https://doi.org/10.1101/2025.01.21.25320878 Jacob Bahnsen Schmidt 1 Department of Clinical Microbiology, Rigshospitalet , Blegdamsvej 9, Copenhagen, 2100, Region Hovedstaden, Denmark 2 Department of Applied Mathematics and Computer Science, Technical University of Denmark , Asmussens Allé, 303B, Lyngby, 2800, Region Hovedstaden, Denmark Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Jacob Bahnsen Schmidt For correspondence: Jacob.bahnsen.schmidt{at}regionh.dk Karen Leth Nielsen 1 Department of Clinical Microbiology, Rigshospitalet , Blegdamsvej 9, Copenhagen, 2100, Region Hovedstaden, Denmark Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Karen Leth Nielsen Dmytro Strunin 1 Department of Clinical Microbiology, Rigshospitalet , Blegdamsvej 9, Copenhagen, 2100, Region Hovedstaden, Denmark Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Dmytro Strunin Nikolai Søren Kirkby 1 Department of Clinical Microbiology, Rigshospitalet , Blegdamsvej 9, Copenhagen, 2100, Region Hovedstaden, Denmark Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Nikolai Søren Kirkby Jesper Qvist Thomassen 3 Department of Clinical Biochemistry, Rigshospitalet , Blegdamsvej 9, Copenhagen, 2100, Region Hovedstaden, Denmark Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Jesper Qvist Thomassen Steen Christian Rasmussen 1 Department of Clinical Microbiology, Rigshospitalet , Blegdamsvej 9, Copenhagen, 2100, Region Hovedstaden, Denmark Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ruth Frikke-Schmidt 3 Department of Clinical Biochemistry, Rigshospitalet , Blegdamsvej 9, Copenhagen, 2100, Region Hovedstaden, Denmark 4 Department of Clinical Medicine, University of Copenhagen , Blegdamsvej 3B, Copenhagen, 2200, Region Hovedstaden, Denmark Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Ruth Frikke-Schmidt Frederik Boëtius Hertz 1 Department of Clinical Microbiology, Rigshospitalet , Blegdamsvej 9, Copenhagen, 2100, Region Hovedstaden, Denmark 5 Department of Immunology & Microbiology, University of Copenhagen , Blegdamsvej 3B, Copenhagen, 2200, Region Hovedstaden, Denmark Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Frederik Boëtius Hertz Allan Peter Engsig-Karup 2 Department of Applied Mathematics and Computer Science, Technical University of Denmark , Asmussens Allé, 303B, Lyngby, 2800, Region Hovedstaden, Denmark Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Allan Peter Engsig-Karup Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Finding the focus of bacterial infections can be challenging, especially for hospitalised patients. Conventional microbiological diagnostic methods are either time consuming, expensive, or difficult to interpret due to contamination. The aim of this study was to apply machine learning (ML) to reliably predict the focus of bacterial infections. This study utilised a dataset including samples from 10,153 patients, collected from November 1, 2019, to June 3, 2023, at Rigshospitalet, Denmark. The dataset contains microbiological findings, biochemical data, and vital parameters. The dataset was analysed using ML. The ML-outputs were calibrated using Venn-ABERS calibration and model uncertainty was addressed using conformal risk control. The best performing model was the XGBoost model achieving a Log loss of 0.219 ± 0.050 (mean ± SD.) and an AUC of 0.93 ± 0.051. Combining the model with methods from the conformal prediction framework achieves predictive capabilities that surpasses similar studies, while also accounting for model uncertainty by providing statistically robust uncertainty estimates via calibrated probabilistic predictions. 1. Introduction The global mortality rate from bacterial infections remains alarmingly high. A study estimated that 7.7 million deaths worldwide in 2019 were caused by 33 different genera and species of bacteria 1 . The mortality rate associated with these infections was 52.2 deaths per 100,000 population in high-income countries. Many of these deaths are due to delayed antibiotic therapy or inadequate treatment of infections, in many cases due to resistance of the causative bacteria 2 . Clinical microbiological diagnostics provide clinicians with important information, such as the identity of the pathogen and its antimicrobial resistance profile for the treatment of infectious diseases. Today’s microbiological diagnostics primarily rely on phenotypic testing, which can be slow and time-consuming due to the need for bacterial growth 3 , 4 . This delay in diagnosis often leads to the use of broad-spectrum antibiotics. However, narrow-spectrum antibiotics are usually more effective against specific types of bacteria and have fewer side effects compared to broad-spectrum antibiotics 3 . Ensuring that the most effective antibiotics are administered as early as possible, which in turn improves patient outcomes and ultimately lowers the global mortality rate from bacterial infections. Furthermore, reducing the unnecessary use of antibiotics and allowing for a faster transition from broad-spectrum antibiotics to narrow-spectrum antibiotics helps mitigate the emergence of multi-drug resistance and preserve the effectiveness of existing antibiotics. In addition to the early and accurate identification of the bacterial pathogen and its antimicrobial resistance profile, it is crucial to determine whether the isolated pathogen is truly causing the disease or is merely a contaminant from the clinical specimen. Clinicians must assess patient history, vital parameters, and laboratory test results to decide if the patient has an infection or is at risk for developing severe infections before administering antibiotics. If treatment is necessary, they must decide which antibiotic to use, whether it should be given orally or intravenously, and the appropriate dosage. Not only the identity of the pathogen and its resistance profile, but also the severity of the disease and the site of infection significantly influences the selection, effectiveness, and management of antibiotic therapy. Consequently, making treatment decisions and choosing the appropriate antibiotic can be complex, requiring clinical expertise and additional laboratory or imaging information. Recent antibiotic stewardship initiatives, such as the “Start smart then focus toolkit” from ( www.gov.uk ) attempt to guide clinicians in rational antibiotic use. Machine learning (ML) could be a valuable predictive tool, providing diagnosticians with decision support in the early stages of diagnosing bacterial infections. Previous studies have shown promising results in predicting bacterial infections using data-driven ML approaches. A study by Marandi et al. 5 focused on predicting bloodstream infection (BSI) primarily based on biochemical data. Their best performing model, LightGBM, achieved an AUC of 0.69, with a sensitivity of 0.57 and specificity of 0.70. Rawson et al. 6 used support vector machines (SVM) to predict the onset of community-acquired bacterial infections within the first 72 hours of hospital admission. Using six commonly available blood parameters, they achieved an AUC of 0.84, with a sensitivity of 0.89 and specificity of 0.63 at a likelihood cut-off of 0.81. Slightly adjusting the cut-off to 0.82 resulted in a sensitivity of 0.44 and specificity of 0.93, indicating potential model uncertainty and calibration issues. In this study, we aimed to develop a data-driven approach using classical ML-models combined with the conformal prediction framework 7 , to create a reliable, fast and cost-effective datadriven ML-model for supporting clinicians and diagnosticians in diagnosing bacterial infections. Classical ML-models such as XGBoost are algorithms that learn patterns in data to make predictions, similar to how a clinician might consider multiple risk factors when assessing a patient. The conformal prediction framework allows such models to provide a measure of confidence or uncertainty with each prediction, helping to indicate how reliable the prediction is in a given case. This method addresses known issues related to model uncertainty in the field of ML, providing more robust and interpretable predictions. By leveraging electronic health records and biochemical data, our goal was to create a foundation for a future datadriven ML model that, by utilizing the conformal prediction framework, can offer accurate predictions with well-calibrated confidence levels, ultimately enhancing decision-making in clinical settings and improving patient outcomes. 2. Methods 2.1. Data Collection This study utilised a retrospective cohort design, including patients of all age groups who had positive bacterial cultures sampled at Rigshospitalet (Copenhagen, Denmark) between November 1, 2019 and June 3, 2023. Rigshospitalet serves as a tertiary hospital primarily for referral and highly specialised treatment. The dataset was produced using Danish electronic health records, i.e., patient journals, by combining data from the electronic patient journals (Sundhedsplatformen, Epic Systems Corporation) and the MADS database (Department of Microbiology Data Systems, in Danish Mikrobiologisk Afdelings Data System). Doublets, defined as subsequent isolates stemming from the same patient with the same outcome within 30 days, were removed from the dataset. The dataset was compiled as a cross-sectional dataset to simplify the analysis while ensuring that only the most recent observations were used, allowing the model to be applicable to new cases without requiring access prior recorded data.Biochemical test results from blood samples and vital parameters were gathered up to seven days prior to the positive conventional cultures, to ensure that the model is trained on data relevant to the bacterial infection in question. The biochemical test results and vital parameters were aggregated using the mean on each day per patient. This resulted in a dataset consisting of 73 features ( Table 1 ). 58 describing the patient’s biochemical test results, ten describing the patient’s vital parameters, two describing the patient’s age and sex, and three features gained from feature engineering. Furthermore, data transformations were applied to each feature to obtain Gaussian distributed features, as described in S6. For the purpose of supervised machine learning (ML), the dataset was labelled using a table describing the findings related to the bacterial isolates. Specifically, ML labels were generated using a combination of the sample origin and type of bacteria found in the sample (S15 shows the ML-label assigned to each combination of bacteria and origin). View this table: View inline View popup Table 1. Table of summary statistics (Median, [IQR] and number of observations) grouped by the infection type (For binary variables only the number of observations is given). Variable describes the name of the feature in the dataset and description is a short description of the feature. Airway is airway infections; BSI is blood stream infections; and UTI is urinary tract infections. A full table describing all the variables in all infection focuses is available in S16. The labels in this study included three foci of infection: airway infections (airways), blood stream infections (BSI), urine tract infections (UTI). For each focus of infection, we categorized bacteria into two groups: (1) typical pathogens (hereafter referred to as “pathogens”) and (2) organisms that are usually not pathogens or are common contaminants (hereafter referred to as “PS thus PSBSI, PSUTI, PSairways”). This classification was based on recent publications, adjusted to local guidelines, and considered the setting of a referral hospital. We used the following sources to inform our categorization 8 – 10 . This approach allowed us to differentiate between organisms likely to cause true infections and those more likely to represent contamination or colonization in this specific context. Due to the species included for sequencing, the class PSBSI was excluded from the analysis as this class only included 11 positive blood samples. Figure 1 provides an overview of the study design. Download figure Open in new tab Figure 1. Overview of the Machine Learning pipeline used in this study from data collection to model evaluation. 2.2. Machine Learning The dataset includes patients with a positive bacterial culture isolated from a wide range of body sites. The ML problem was framed as a seven dimensional one-vs-rest binary classification task. The goal was to create a model that could distinguish between seven different classes (different foci of infections and colonizations), each of them treated separately in a one-vs-rest approach, where the model learns to identify a particular focus of infection/colonization (the “one”) and distinguish it from the combined group of the remaining foci of infection/colonization (the “rest”). The ML-models were evaluated using 5-fold cross-validation, with an 80-20% split between training and test sets. This study tested the use of a Support Vector Machine 11 (SVM) and a Gradient Boosting Machine (GBM). These were implemented using Scikit-learn version 1.3.2 and python 3.8 - AzureML. Additionally, the XGBoost classifier 12 version 1.3.3 was used, which is a scalable tree boosting algorithm building on the ideas of Friedman 2001 13 . XGBoost is an ensemble method that utilises a forest of simple classification trees. And can handle missing data, so no further preprocessing was necessary when training the XGBoost model. The model was optimised using a small grid search (sklearn’s GridSearchCV) and the optimal hyperparameters was found to be the default settings of the XGBoost model (for information on the SVM setup see S3). Generally, ML-methods struggle with addressing model uncertainty. To tackle this issue, this study employed methods from the conformal prediction framework 7 , 14 , using a calibration dataset of 1000 observations. First the probabilistic model outputs were calibrated using the Venn-ABERS (VA) calibration technique 15 . VA calibration aims to adjust the probabilistic model outputs, to better match the ground truth. In other words, if the model outputs a 42% probability of airway infection, this should accurately reflect the real-world likelihood. VA calibration uses a two-fold isotonic regression: one describes the probability of belonging to the class, and the other describes the probability of not belonging to the class. These regression models create a prediction interval ( p 0 , p 1) for an observation belonging to the positive class, with a guarantee that the true probability lies within this interval 16 . The adjusted probability outputs are calculated using the two isotonic regression models 17 and the formula in Eq. (1) . In practice, it is desirable to produce a model that can provide uncertainty guaranties. Therefore, this study employed the Conformal Risk Control (CRC) procedure from the conformal prediction framework 18 . CRC is a technique used to ensure reliable predictions that contain most of the true labels, while maintaining a user-specified error rate. In other words, CRC aims at creating prediction sets that are both accurate and meaningful. The intuition behind CRC can be described in a handful of steps 14 , 18 . First the user chooses an acceptable error rate called α . This should be thought of as the maximum error rate that the user can tolerate to accept the ML prediction in a given application. Secondly, prediction sets for new observations are generated, ideally including the true labels of the observations. The third step is the balancing act, which ensures that the average error of the prediction sets, generated by a suitably chosen loss function is less than or equal to the chosen error rate. In the context of multi-label classification, as in this study, where each patient can have multiple focuses of infection, the goal is to produce a classifier that can assign accurate probabilities to each focus of infection and then choose thresholds, such that only predictions or focuses of infection with adequately high probability scores are included in the prediction set. This is an important part of the CRC procedure and essentially helps keeping the prediction sets small and meaningful, such that only the most likely outcomes are included in the prediction set. Intuitively this also means that the size of the prediction sets wary with the difficulty of the new observation. If a new observation is diagnostically easy, then the resulting prediction sets will be small. However, if a new observation is difficult, then the prediction sets will be comparatively larger. Put differently, the model signals uncertainty in a human like fashion by generating prediction sets of different sizes based on how certain the model is. The CRC framework requires a user chosen loss function. This study used the False negative rate (FNR) as the loss function, to control the sensitivity, setting the user specified error rate α = 0.1, which essentially ensures a sensitivity of at least 90% on average. For a more thorough and technical description of the mathematical implementation of CRC see S4. This study has also assumed a random pool based active learning 19 (AL) strategy for the sampling in order to estimate how much data is required to train similar models in the future. We also performed an ablation study as part of a sensitivity analysis, removing all hematopoietic patients and patients with ( LEU < 1 ⋅ 10 9 / L ) from the dataset. A low leucocyte count may associate with increased risk of BSI, especially among patients with cancer or hematopoietic stem cell transplantation recipients and the presence of this population in our dataset may impact the model. 2.3. Model Evaluation The models were evaluated using area under the receiver operating characteristics curve (AUC), as well as precision, sensitivity, specificity, F1-score, and multi class Brier score. Finally, this study examined the best performing model using Shapley additive explanations (SHAP). SHAP is a model agnostic tool that uses game theory to interpret the output of machine learning models 20 , that can be used with caution for interpretability 21 . 3. Results This study included 10,153 hospitalized patients from both sexes and all age groups and from whom a bacterial isolate was recovered and processed at the Department of Clinical Microbiology (DCM) at Rigshospitalet between November 2019 and June 2023. From these 10,153 patients, 20,037 positive bacterial isolates of a diverse array of species (listed in S15) were processed at the DCM (on average 1.97 isolates per patient) and paired with 30,123 sets of biochemical test results. From these biochemical test results, 6,138 were related to isolates from the respiratory tract, 2,968 related to the blood stream, 13,052 related to the urinary tract and 9,922 related to other body sites. The patients were categorized into groups based on the sample site of the clinical isolate. The first group comprised patients with a clinically apparent infection of the respiratory tract (1,228), the second with a blood stream infection (2,968) and the third with a urinary tract infection (11,430). There were 55 patients who were categorized as having a UTI as well as a respiratory tract infection, 364 patients were categorized with both a BSI and a UTI and 28 patients with simultaneous BSI and respiratory tract infection. There were also patients from whom bacterial organisms that are usually not pathogens or are common contaminant were recovered from the respiratory tract (4,910), the urinary tract (1,622) or the blood stream (11). These patients were categorized into the three patient groups of airway, urinary tract and blood stream contamination / colonization. The group of BSI contaminations was however removed due to data sparsity. An overview of the patient group sizes is outlined in Figure 2 , as an Euler diagram and a table, describing both the absolute number and overlaps in each of the 6 resulting infection classes. Download figure Open in new tab Figure 2. (a) shows an overview of the dataset, as an Euler diagram, where each ellipse represents an infection focus. The Euler diagram does not show all intersections between infection focuses, but rather the most important ones based on correlation between groups. The Euler diagram is fitted to the dataset using the eulerr R library. Figure 2(b) shows a summary of the dataset, showing the number of observations in each infection category. Airway includes airway infections, BSI is blood stream infections, UTI is urine tract infections, Other is an ensemble of other types of infection. PS denotes organisms that are usually not pathogens and common contaminants. This term is used to define observations that potentially could be contamination rather than infection. It should also be noted that the column % of samples sum to more than 100%, as patients can have multiple focuses of infection. In the next step, 73 features were used as input for a model to predict the patient’s site of infection (respiratory tract, bloodstream, or urinary tract). These features included 58 biochemical test results, ten vital parameters, age, sex, and three features obtained through feature engineering ( Table 1 ). An overview of the model performance is presented in Table 2 . The XGBoost model achieved the highest AUC and F1-score (mean ± std.) of 0.93 ± 0.051 and 0.71 ± 0.053 respectively. XGBoost outperforms the SVM on nearly all performance metrics. The sensitivity of the naïve XGBoost model was 0.63 ± 0.101 (mean ± std.), meaning it correctly predicted the label in 63% of the positive cases. The precision of the XGBoost model was 0.83 ± 0.094, indicating that if the model makes ten predictions, roughly eight will be correct. Additionally, the XGBoost model had a lower multi-class Brier score (mean ± std.) of 0.389 ± 0.103 and a Log loss of 0.219 ± 0.050, suggesting better calibration of probabilistic model outputs compared to the SVM. Applying the VA calibration substantially improved the F1-score for both models, indicating a better overall balance between precision and sensitivity. Using the CRC method achieved the desired sensitivity of at least 90%, but this increased sensitivity came with a trade-off in precision and specificity. View this table: View inline View popup Table 2. Comparison table, showing the mean (std) micro averaged performance metrics across the 5-fold cross validation for each model, in both their naïve and VA calibrated states, as well as the model performance when the output have been optimized to satisfy the conformal risk control guarantee with an α = 0.1. PPV: Positive Predictive Value (Precision), SEN: Sensitivity, SPC: Specificity, F1: F1-score (measure of predictive performance – harmonic mean between PPV and SPC), MBS: Multi-class brier score (measure the accuracy of probabilistic predictions, lower score is better), AUC: area under the receiver operating characteristics curve, CPU TT: mean CPU time during training in minutes (std. in seconds), SVM: support vector machine, VA: Venn-ABERS calibration, CRC: conformal risk control, XGB: XGBoost, p-value: p-value for the test of difference in naïve models calculated using a permutation test with 1000 permutations. Figure 4a illustrates the receiver operating characteristics (ROC) curves of the XGBoost model. The XGBoost model was the best performing model based on AUC, multi class Brier Score, Log loss, and F1-score. The model generally shows strong performance in predicting the patients’ site of infection, with the highest AUC observed for predicting airway infections and urine contamination, while the lowest AUC is seen in the “Other infections” class. The class that achieves the best balance between sensitivity and specificity is the Urine infections as this class has the highest F1-score. Figure 4b-d depicts the confusion matrices for the XGBoost model in its naïve, VA calibrated, and CRC-VA calibrated states. It is evident that VA calibration substantially increases the number of true positives (TP). Controlling the sensitivity to at least 90% on average using CRC results in an increased number of TPs, but also increases the prediction set size, as indicated by the elevated number of False Positives (FP). Download figure Open in new tab Figure 3. (a) Naïve XGBoost (b) VA calibrated XGBoost. Micro averaged calibration curves for the XGBoost model, showing the mean predicted probability vs. the fraction of positives. (a) significantly underestimates the infection probability and is therefore untrustworthy, while (b) is well calibrated and therefore provides trustworthy probability estimates. The blue lines represent airway infections (airways) and common contaminants of the airways (PSairways). The red line represents blood stream infections (BSI). The yellow lines represent urinary tract infections (UTI) and common contaminants of the urinary tract (PSUTI). The green line represents other types of infection (Other). Download figure Open in new tab Figure 4. Overview of the XGBoost model performance on the test sets. (a) receiver operating characteristics (ROC) for each infection class using 5-fold cross validation. (b), (c), (d) Confusion matrices for the XGBoost model in its naïve, VA calibrated, and CRC-VA calibrated form respectively. The confusion matrices are summed across the 5-fold cross validation (for reference a theoretically perfect confusion matrix can be seen in S8) From the confusion matrices it is clear that both calibration and CRC uncertainty based prediction sets increase the number of correct predictions. VA: Venn-ABERS, CRC: conformal risk control, Airways: airway infections, BSI: blood stream infection, UTI: urine tract infection, PS: organisms that are usually not pathogens and common contaminants, AUC: area under the ROC curve. The CRC method mathematically signals uncertainty in a manner similar to human intuition. Figure 5 displays the prediction set sizes at different levels of α . When α = 0.01, corresponding to a sensitivity of at least 99% on average, the model includes a larger proportion of the label space in the prediction set. Higher levels of α results in smaller prediction sets with a larger risk of missing the true class. At α = 0.3 corresponding to a sensitivity of at least 70% on average, the median prediction set size is one. It is also evident that the CRC method can produce empty prediction sets when the predicted probability of each class is below the probability threshold, indicating lack of confidence in predicting a class label. Download figure Open in new tab Figure 5. Prediction set sizes at different levels of α when applying the conformal risk control method. α corresponds to the user specified error tolerance, from left to right the figure shows prediction set sizes at 1, 5, 10, 20 and 30% error rates. Here It can be seen that large user tolerated error rates results in bolder or overconfident prediction sets, while small user tolerated error rates result in large prediction sets. Active learning (AL is a framework 19 aimed at maximising model performance while minimizing the number of annotated samples required. Based on the available dataset, we estimate the amount of data needed to achieve the expected results using a pool based AL strategy. Figure 6 shows that the XGBoost model requires approximately 2,000 positive samples per class to approach the maximal obtainable sensitivity. This can also be seen in figure S 14 which depicts the relationship between the std. of micro averaged performance metrics from the XGBoost model. Download figure Open in new tab Figure 6. Sensitivity of the Naïve XGBoost model plotted vs. the number of positives in the training dataset for each infection class, when the model is trained with different training dataset sizes. Here it is seen that model performance starts to stagnate at approximately 1,000-2,000 positive observations. Airways is airway infections, BSI is blood stream infections, UTI are urine tract infections, PS denotes organisms that are usually not pathogens and common contaminants,, and the “Other” class is an ensemble of other types of infection Similar results for other performance metrics can be seen in S7-S13. The ablation study, removing hematopoietic patients and patients with low leucocyte count yielded similar results based on the performance metrics, only varying AUC by a few percentage points (absolute difference in AUC was 1.17 ± 1.17 pp. ± std.). The SHAP-plots in Figure 7 illustrate feature importance across the different classes of infection. Figure 7a ranks features by their SHAP-value, highlighting their importance. It is evident that the model heavily relies on information about the patient’s ability to breathe and oxygenate their blood, as O2 therapy and blood oxygen saturation are amongst the most important features. Download figure Open in new tab Figure 7. SHAP summary plots displaying feature importance of the 9 most important features as determined by SHAP-value. The SHAP-value is equivalent to the log-odds of predicting the class. Here it can be seen that the model generally highlights known risk factors associated with each infection focus. Sex is encoded categorically as, male=1, female=0. airways: airway infection, BSI: blood stream infection, UTI: urine tract infection, PS: oranisms that are usually not pathogens and common contaminants, O2Therapy: O2 Therapy, EWS: Early warning score, age: patient age in years, Saturation: blood O2 saturation percentage, LDH: lactate dehydrogenase, CREA: creatinine, eGFR: estimated glomerular filtration rate, DiastolicBP: diatolic blood pressure, LEU: leukocyte count, LYMFO: lymphocyte count, Temperature: patient body temperature, CARB: Carbamide, BILI: bilirubin, ALB: albumin, HB: hemoglobin, CRP: C-reactive protein, HR: Heart rate, SystolicBP: Systolic blood pressure, INR: International normalized ratio blood test, BASO: Basophilocyte count, GLU: Glucose, NEUTRO: Neutrophil count. Figure 7c depicts the feature importance related to BSIs. The two most important features for predicting BSIs are the leukocyte (LEU) count and the hemoglobin concentration (HB). Specifically, a low LEU count corresponds to a high risk of BSI compared to all other classes of infection. While bacterial infections typically cause elevated LEU counts 22 , this result may be explained by the large group of immunocompromised patients at Rigshospitalet. Furthermore, the patients’ temperature appears to be a predictor of BSI. This aligns with the literature, as patients with BSI are known to have elevated body temperatures, except in cases where the blood circulation is affected, which can cause hypothermia 23 . Figure 7d shows that the greatest risk factor when predicting UTI is the patient sex, with female encoded as zero and male as one. Furthermore, high levels of C-reactive protein (CRP) correspond to a lower chance of UTI compared to the other classes of infection, while a high concentration of albumin (ALB) indicates a higher likelihood of predicting UTI. 4. Discussion The aim of this study was to develop a data-driven predictive probabilistic model that can learn from historic real patient data and predict the focus of infection within a reasonable margin of error and account for how uncertainty in the data translate to predictive uncertainty to improve model reliability. The best performing ML-model was the XGBoost model, achieving a Log loss of 0.219 and an AUC of 0.93, which is infrequently observed within the field of bacterial infections and indicative of good model performance. This model also showed a decent balance between sensitivity and specificity as seen by the F1-score of 0.71, meaning that the model performance is acceptable, both when measured on how well it predicts the correct label and how well it excludes the incorrect labels. Metrics such as AUC, F1-score, etc. that are based on classification outcomes are however known to be misleading 24 . For example, a model can achieve high AUC while still producing poorly calibrated or overconfident probability estimates and can therefore not be used as sole indicators of performance. In contrast, metrics like Log loss and Brier score directly considers the predicted probabilities and calibration hereof. Log loss penalizes incorrect and overconfident predictions more heavily, while Brier score measure the mean squared difference between predicted probabilities and actual outcomes across all classes. These metrics are therefore more appropriate when the aim is to produce reliable, real-world probabilistic predictions. The XGBoost model also outperforms the SVM when compared on Log loss and Brier score. The naïve model state did, however, prove to be somewhat badly calibrated, nevertheless using Venn-ABERS (VA) calibration a reasonably well calibrated model with a multi class brier score of 0.366 and a Log loss of 0.205 was obtained. Applying conformal risk control (CRC) in this setting guarantees a sensitivity of at least 90% on average, meaning that in nine out of ten cases, the correct prediction is included in the prediction set, while also not creating excessively large prediction sets. Compared to similar studies in the field of bacterial infections VA-calibration and CRC results in a considerably improved model performance. Furthermore, it was found, through active learning that studies should include a minimum of 1,000 positive observations pr. class and preferably 2,000 observations to achieve satisfactory model quality and robust uncertainty quantification. A dataset of this magnitude may be difficult to obtain in most hospital settings. However, considering the active learning results, this dataset size is essential for training a robust ML-model using XGBoost, when applied to patient data with the purpose of predicting bacterial infection focus. Adopting ML to clinical practices requires a high level of trustworthy artificial intelligence 25 (AI). In a clinical setting clinicians and diagnosticians are responsible for human lives. For the clinicians and diagnosticians to trust the predictions made by AI and ML-models, the models need to be built with transparency, robustness, and a high level of explainability 25 . To achieve this one should address known biases in the dataset, potential strengths, and weaknesses of the ML-models, and provide explanations for the model outputs. The dataset in this study comes with a handful of natural biases. The data is recorded at Rigshospitalet in Denmark. Rigshospitalet is a highly specialised tertiary hospital, patients therefore often present with numerous comorbidities. This makes diagnostics increasingly difficult, but it also introduces biases towards sick people in the dataset. The input features were transformed to resemble the gaussian distribution, since this is a common choice for input features. However, we acknowledge that an ablation study assessing the impact of this transformation would provide further insight. The ML-labels used in this study could also introduce biases. The labels are generated using the sample origin and the bacterial species. However, this combination is likely not perfectly true to reality and could potentially introduce a level of epistemic errors into the dataset. To increase trustworthiness and diagnostic value of the ML-models the labels should be redefined using other more strict definitions of contamination. The findings align well with previous knowledge, expected findings, and demonstrate that the ML-modelling can support identifying clinically relevant factors for predicting the different foci for bacterial infections. The ML-model’s heavy reliance on O2 therapy and blood oxygen saturation for predicting respiratory infections, the identification of leukocyte counts and hemoglobin concentration as important features for BSI prediction, and the identification of patient sex as the most important factor for UTI prediction is clinically sensible and consistent with epidemiological data. The inverse relationship between CRP levels and UTI prediction, as well as the positive association with albumin levels, may reflect the model’s ability to distinguish UTIs from more severe systemic infections. This nuanced interpretation of biomarkers demonstrates the ML-model’s sophistication in differentiating between infection types. These results are promising since the active learning framework suggests that approximately 2,000 positive samples per class are needed to achieve near-optimal model performance. This information is valuable for future studies, as it provides a clear guideline for dataset size requirements. It also indicates that the current study likely had sufficient data to develop a robust model. The naïve models proved to be badly calibrated. This issue was handled using VA-calibration. Badly calibrated models could potentially decrease the trustworthiness of the model and ML in general, as users empirically will start to doubt the ML-models if it consistently outputs either lower or higher chance of bacterial infection than what the user observes. The usage of tools in this study are chosen with clinical application in mind. Specifically, the usage of conformal predictions to generate statistically robust prediction sets is of potentially high value in a clinical setting. Many ML-methods do not provide uncertainty estimates for their predictions. Applying the conformal risk control (CRC) method generates prediction sets of varying size as seen in Figure 5 . This is clinically useful as the framework not only provides statistically sound uncertainty guaranties for the user specified performance metrics, but also allows the ML-models to generate prediction sets of varying size, which essentially signals uncertainty in a human like fashion and is known to significantly improve accuracy in human-in-the-loop systems 26 , where humans make decisions based on expert knowledge and ML-outputs. Prediction sets can be a great way of signalling uncertainty, but the size of the prediction sets needs to be discussed. For example, using CRC it is possible to guarantee a certain level of sensitivity. Controlling the sensitivity to 99% for the XGBoost model results, by definition, in few predictions sets where the correct label is excluded. This is however achieved by outputting half the label space for median observations. In practice large prediction sets do not provide any direct diagnostic value to the clinicians or diagnosticians, it does however indicate high uncertainty, which warrants further examination of the patient. Reducing the sensitivity to 70% mostly resulted in prediction set sizes of two or less. This size is potentially more useful as it significantly narrows the potential search field. The issue with this setting is that the model essentially guarantees an error rate of approximately 30%. When using CRC in practice it is therefore important to find the right balance between prediction set size and error rate, such that the prediction sets are narrow enough that clinicians and diagnosticians can extract actual value from them, but still wide enough to provide a reasonably low error rate. Initially CRC might not seem like a powerful tool solely based on performance metrics, however considering that CRC provides any model with statistically robust uncertainty guaranties on the sole condition that new observations stem from the same distribution as the one provided by the calibration dataset, it should perhaps not only be considered a powerful tool, but a necessity when applying ML in a health care setting. In terms of explainability the XGBoost model has been analysed using SHAP-plots. SHAP-plots are widely used as a tool for explainable AI. SHAP-analysis assumes feature independence, this is almost never achievable in real life applications, and the SHAP-analysis should therefore be interpreted with a pinch of scepticism 21 . From the SHAP-analysis it is also clear that the dataset contains human introduced biases, as O2 therapy is deemed as the most important feature when predicting airway infections. This feature is controlled by the diagnosticians and is therefore naturally biased towards the human decision maker. Nevertheless, the majority of the features deemed important by the model are expected and known biomarkers of bacterial infection (age 27 , LDH 28 , LEU 22 , temperature 23 , sex, and CRP). Our study has several limitations. It is a retrospective design, which may limit the ability to control for confounding factors, and the model was not validated on an external dataset from a different hospital or time period, which would have provided stronger evidence of its generalizability. The low prevalence of positive blood cultures in the dataset may affect the model’s performance, particularly its positive predictive value. Additionally, the model does not incorporate important clinical information such as comorbidities or presence of indwelling devices, which are known risk factors for bloodstream infections. However, this study combines advanced machine learning techniques (XGBoost) with conformal prediction methods (VA calibration and CRC) to create a more reliable and interpretable model. The use of CRC to generate prediction sets of varying sizes based on uncertainty is an approach developed for, inter alia, clinical decision support, mimicking human-like signalling of uncertainty and the comprehensive analysis of bacterial infections using a wide range of biochemical and vital sign data is more extensive than previous studies in this field. The integration of trustworthy AI principles, addressing model uncertainty and providing explainable results, represents a significant step towards clinical application of ML in infection diagnosis. Thus, this study provides valuable insights into the data requirements for developing similar models in the future, which is crucial for advancing the field of ML in clinical microbiology. 5. Conclusion The XGBoost model achieved high performance (F1-score: 0.71, AUC: 0.93, Log loss: 0.219, MBS: 0.389) in predicting the focus of bacterial infections using electronic health record data and biochemical markers. Venn-ABERS calibration and conformal risk control significantly improved model calibration and guaranteed a sensitivity of at least 90% on average. We estimate that approximately 2,000 positive samples per infection class are needed to achieve optimal model performance, as determined by active learning analysis. The model identified clinically relevant predictors for different infection types, such as oxygen therapy for respiratory infections and leukocyte count for bloodstream infections, which aligns well with clinical observations. Conformal prediction methods can generate variable-size predictions sets, signalling uncertainty in a human-like fashion and potentially improve clinical decision-making. Despite limitations such as its retrospective design and lack of external validation, this datadriven ML-approach of combining classical ML-models like XGBoost with conformal prediction methods provides valuable insights into data requirements and methodological approaches for developing trustworthy AI systems in clinical microbiology. These findings may have significant implications for improving antibiotic stewardship and patient outcomes in the management of bacterial infections. Author contribution Conceptualization: Jacob Bahnsen Schmidt, Frederik Boëtius Hertz, Karen Leth Nielsen, Allan Peter Engsig-Karup Data curation: Jacob Bahnsen Schmidt, Dmytro Strunin, Nikolai Søren Kirkby, Steen Christian Rasmussen Formal analysis: Jacob Bahnsen Schmidt Investigation: Jacob Bahnsen Schmidt Methodology: Jacob Bahnsen Schmidt, Frederik Boëtius Hertz, Karen Leth Nielsen, Allan Peter Engsig-Karup Ethical approval: Frederik Boëtius Hertz. Resources: Frederik Boëtius Hertz, Karen Leth Nielsen Software: Jacob Bahnsen Schmidt, Dmytro Strunin Formal analysis: Jacob Bahnsen Schmidt Supervision: Frederik Boëtius Hertz, Karen Leth Nielsen, Allan Peter Engsig-Karup Validation: Jacob Bahnsen Schmidt, Dmytro Strunin, Jesper Qvist Thomassen, Steen Christian Rasmussen Visualization: Jacob Bahnsen Schmidt Project administration: Karen Leth Nielsen, Frederik Boëtius Hertz, Allan Peter Engsig-Karup Writing – Original Draft Preparation: Jacob Bahnsen Schmidt, Frederik Boëtius Hertz. Writing – Review & Editing: Jacob Bahnsen Schmidt, Frederik Boëtius Hertz, Karen Leth Nielsen, Dmytro Strunin, Nikolai Søren Kirkby, Jesper Qvist Thomassen, Steen Christian Rasmussen, Ruth Frikke-Schmidt, Allan Peter Engsig-Karup Materials & Correspondence: Jacob Bahnsen Schmidt Funding This work was funded by Department of Clinical Microbiology, Rigshospitalet, Denmark, Beta-Health Foundation ID:1188), Den Frie Forskningsfond (DFF) (3101-00040B), and the Novo Nordisk Foundation (NNF Laureate Grant 18OC0033946). The funders had no role in designing and conducting the study, analysis, and interpretation of the data, or the writing, review, and approval of the manuscript. Ethical approval There were no ethical concerns for this study. This was an observational, non-intervention study. The extraction of data from registries was approved by the local data protection agency (Journal-nr.: R-21015888) and registered in Pactius (P-2020-743). Data Sharing All relevant data are within the manuscript and its Supporting Information files. We are unable to share sensitive data instances due to compliance with the General Data Protection Regulation (GDPR) and Danish legislation. These regulations mandate that the privacy and confidentiality of individuals must be protected, and sensitive data should not be made publicly available without appropriate safeguards. Therefore, we cannot provide access to the sensitive data instances used in this study to ensure compliance with these regulations. Code Availability Access to the code used for the statistical design and ML-process can be made available on reasonable request to the authors. Competing Interest The authors declare no competing interests. Data Availability All relevant data are within the manuscript and its Supporting Information files. We are unable to share sensitive data instances due to compliance with the General Data Protection Regulation (GDPR) and Danish legislation. These regulations mandate that the privacy and confidentiality of individuals must be protected, and sensitive data should not be made publicly available without appropriate safeguards. Therefore, we cannot provide access to the sensitive data instances used in this study to ensure compliance with these regulations. Footnotes We have updated the manuscript to improve both clarity and model evaluation, in response to reviewer feedback and based on our own assessment. One key change is the inclusion of Log Loss as an additional performance metric. This offers a more informative view of predictive uncertainty and complements the existing evaluation framework. Alongside this, we have placed less emphasis on AUC as the primary metric for model comparison, shifting the focus to a broader set of performance indicators that provide a more nuanced assessment of model behavior. We have also expanded the performance metrics table to include not only global metrics, but also class-specific results. This addition allows for a clearer understanding of how the model performs across different classes, which we believe is important for a more complete interpretation of the results. In addition to these updates, we have revised and clarified several parts of the manuscript text, particularly in sections describing the evaluation methods and the reasoning behind metric selection. References 1. ↵ Ikuta , K. S. et al. Global mortality associated with 33 bacterial pathogens in 2019: a systematic analysis for the Global Burden of Disease Study 2019 . The Lancet 400 , 2221 – 2248 ( 2022 ). OpenUrl 2. ↵ Cassini , A. et al. Attributable deaths and disability-adjusted life-years caused by infections with antibiotic-resistant bacteria in the EU and the European Economic Area in 2015: a population-level modelling analysis . Lancet Infect Dis 19 , 56 – 66 ( 2019 ). OpenUrl CrossRef PubMed 3. ↵ Melander , R. J. , Zurawski , D. V & Melander , C. Narrow-spectrum antibacterial agents . Medchemcomm 9 , 12 – 21 ( 2018 ). OpenUrl CrossRef PubMed 4. ↵ Gibson , B. , Wilson , D. J. , Feil , E. & Eyre-Walker , A . The distribution of bacterial doubling times in the wild . Proceedings of the Royal Society B 285 , 20180789 ( 2018 ). 5. ↵ Marandi , R. Z. , et al. Prediction of bloodstream infection using machine learning based primarily on biochemical data . ( 2024 ). 6. ↵ Rawson , T. M. et al. Supervised machine learning for the prediction of infection on admission to hospital: a prospective observational cohort study . Journal of Antimicrobial Chemotherapy 74 , 1108 – 1115 ( 2019 ). OpenUrl PubMed 7. ↵ Shafer , G. & Vovk , V . A Tutorial on Conformal Prediction . Journal of Machine Learning Research 9 , ( 2008 ). 8. ↵ Kouri , T. T. et al. The EFLM European Urinalysis Guideline 2023 . Clinical Chemistry and Laboratory Medicine (CCLM) 62 , 1745 – 1747 ( 2024 ). OpenUrl 9. Markussen , D. L. et al. Diagnostic Stewardship in Community-Acquired Pneumonia With Syndromic Molecular Testing . JAMA Netw Open 7 , e240830 ( 2024 ). OpenUrl 10. ↵ Antibiotic Treatment for 7 versus 14 Days in Patients with Bloodstream Infections . New England Journal of Medicine ( 2024 ) doi: 10.1056/NEJMoa2404991 . OpenUrl CrossRef 11. ↵ Pedregosa , F. et al. Scikit-learn: Machine Learning in Python . Journal of Machine Learning Research 12 , 2825 – 2830 ( 2011 ). OpenUrl 12. ↵ Chen , T. & Guestrin , C. Xgboost: A scalable tree boosting system . in Proceedings of the 22nd acm sigkdd international conference on knowledge discovery and data mining 785 – 794 ( 2016 ). 13. ↵ Friedman , J. H . Greedy function approximation: a gradient boosting machine . Ann Stat 1189 – 1232 ( 2001 ). 14. ↵ Angelopoulos , A. N. & Bates , S. A gentle introduction to conformal prediction and distribution-free uncertainty quantification . arXiv preprint arXiv:2107.07511 ( 2021 ). 15. ↵ Vovk , V. & Petej , I. Venn-abers predictors . arXiv preprint arXiv:1211.0025 ( 2012 ). 16. ↵ Manokhin , V . Practical Guide to Applied Conformal Prediction in Python . ( Packt Publishing Ltd ., Birmingham , 2023 ). 17. ↵ Manokhin , V. Multi-class probabilistic classification using inductive and cross Venn–Abers predictors . in Proceedings of the Sixth Workshop on Conformal and Probabilistic Prediction and Applications (eds. Gammerman , A. , Vovk , V. , Luo , Z. & Papadopoulos , H. ) vol. 60 228 – 240 ( PMLR , 2017 ). OpenUrl 18. ↵ Angelopoulos , A. N. , Bates , S. , Fisch , A. , Lei , L. & Schuster , T. Conformal risk control . arXiv preprint arXiv:2208.02814 ( 2022 ). 19. ↵ Settles , B . Active learning literature survey . Computer Sciences Technical Report 1648 ( 2009 ). 20. ↵ Lundberg , S. M. et al. From local explanations to global understanding with explainable AI for trees . Nat Mach Intell 2 , 56 – 67 ( 2020 ). OpenUrl CrossRef PubMed 21. ↵ Huang , X. & Marques-Silva , J. The Inadequacy of Shapley Values for Explainability . arXiv preprint arXiv:2302.08160 ( 2023 ). 22. ↵ Honda , T. , Uehara , T. , Matsumoto , G. , Arai , S. & Sugano , M . Neutrophil left shift and white blood cell count as markers of bacterial infection . Clinica Chimica Acta 457 , 46 – 53 ( 2016 ). OpenUrl CrossRef 23. ↵ Kushimoto , S. et al. The impact of body temperature abnormalities on the disease severity and outcome in patients with severe sepsis: an analysis from a multicenter, prospective survey of severe sepsis . Crit Care 17 , 1 – 9 ( 2013 ). OpenUrl CrossRef PubMed 24. ↵ Lobo , J. M. , Jiménez-Valverde , A. & Real , R. AUC: a misleading measure of the performance of predictive distribution models . Global Ecology and Biogeography 17 , 145 – 151 ( 2008 ). OpenUrl 25. ↵ Li , B. et al. Trustworthy ai: From principles to practices . ACM Comput Surv 55 , 1 – 46 ( 2023 ). OpenUrl 26. ↵ Cresswell , J. C. , Sui , Y. , Kumar , B. & Vouitsis , N. Conformal Prediction Sets Improve Human Decision Making . arXiv preprint arXiv:2401.13744 ( 2024 ). 27. ↵ Meyer , K. C. Lung infections and aging . Ageing Res Rev 3 , 55 – 67 ( 2004 ). OpenUrl CrossRef PubMed Web of Science 28. ↵ Drent , M. , Cobben , N. A. , Henderson , R. F. , Wouters , E. F. & van Dieijen-Visser , M . Usefulness of lactate dehydrogenase and its isoenzymes as indicators of lung damage or inflammation . European Respiratory Journal 9 , 1736 – 1742 ( 1996 ). OpenUrl Abstract / FREE Full Text 29. Madsen , H. & Thyregod , P . Introduction to General and Generalized Linear Models . ( Chapman & Hall , Lyngby , 2011 ). View the discussion thread. Back to top Previous Next Posted April 23, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Conformal Prediction and Venn-ABERS Calibration for Reliable Machine Learning-Based Prediction of Bacterial Infection Focus Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Conformal Prediction and Venn-ABERS Calibration for Reliable Machine Learning-Based Prediction of Bacterial Infection Focus Jacob Bahnsen Schmidt , Karen Leth Nielsen , Dmytro Strunin , Nikolai Søren Kirkby , Jesper Qvist Thomassen , Steen Christian Rasmussen , Ruth Frikke-Schmidt , Frederik Boëtius Hertz , Allan Peter Engsig-Karup medRxiv 2025.01.21.25320878; doi: https://doi.org/10.1101/2025.01.21.25320878 Share This Article: Copy Citation Tools Conformal Prediction and Venn-ABERS Calibration for Reliable Machine Learning-Based Prediction of Bacterial Infection Focus Jacob Bahnsen Schmidt , Karen Leth Nielsen , Dmytro Strunin , Nikolai Søren Kirkby , Jesper Qvist Thomassen , Steen Christian Rasmussen , Ruth Frikke-Schmidt , Frederik Boëtius Hertz , Allan Peter Engsig-Karup medRxiv 2025.01.21.25320878; doi: https://doi.org/10.1101/2025.01.21.25320878 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Infectious Diseases (except HIV/AIDS) Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15229) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6600) Geriatric Medicine (668) Health Economics (997) Health Informatics (4536) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (541) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3332) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9232) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a0084e030ae558d3',t:'MTc3OTU4MzgxMQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();