Ovarian cancer recurrence prediction: comparing confirmatory to real world predictors with machine learning

Ovarian cancer recurrence prediction: comparing confirmatory to real world predictors with machine learning | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Ovarian cancer recurrence prediction: comparing confirmatory to real world predictors with machine learning D. Katsimpokis , A.E.C. van Odenhoven , M.A.J.M. van Erp , H.H.B. Wenzel , M.A. van der Aa , M.M.H. van Swieten , H.P.M. Smedts , J.M.J. Piek doi: https://doi.org/10.1101/2025.03.04.25321571 D. Katsimpokis 1 Department of Research & Development, Netherlands Comprehensive Cancer Organisation (IKNL) , Utrecht, the Netherlands PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: d.katsimpokis{at}iknl.nl A.E.C. van Odenhoven 2 Department of Gynaecology and Obstetrics and Catharina Cancer Institute, Catharina Hospital Eindhoven , the Netherlands MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site M.A.J.M. van Erp 3 Department of Gynaecology and Obstetrics, Amphia Hospital Breda , the Netherlands MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site H.H.B. Wenzel 1 Department of Research & Development, Netherlands Comprehensive Cancer Organisation (IKNL) , Utrecht, the Netherlands PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site M.A. van der Aa 1 Department of Research & Development, Netherlands Comprehensive Cancer Organisation (IKNL) , Utrecht, the Netherlands PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site M.M.H. van Swieten 1 Department of Research & Development, Netherlands Comprehensive Cancer Organisation (IKNL) , Utrecht, the Netherlands PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site H.P.M. Smedts 3 Department of Gynaecology and Obstetrics, Amphia Hospital Breda , the Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site J.M.J. Piek 2 Department of Gynaecology and Obstetrics and Catharina Cancer Institute, Catharina Hospital Eindhoven , the Netherlands MD, PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Introduction Ovarian cancer is one of the deadliest cancers in women, with a 5-year survival rate of 17-28% in advanced stage (FIGO IIB-IV) disease and is often diagnosed at advanced stage. Machine learning (ML) has the potential to provide a better survival prognosis than traditional tools, and to shed further light on predictive factors. This study focuses on advanced stage ovarian cancer and contrasts expert-derived predictive factors with data-driven ones from the Netherlands Cancer Registry (NCR) to predict progression-free survival. Methods A Delphi questionnaire was conducted to identify fourteen predictive factors which were included in the final analysis. ML models (regularized Cox regression, Random Survival Forests and XGBoost) were used to compare the Delphi expert-based set of variables to a real-world data (RWD) variable set derived from the NCR. A traditional, non-regularized, Cox model was used as the benchmark. Results While regularized Cox regression models with the RWD variable set outperformed the traditional Cox regression with the Delphi variables (c-index: 0.70 vs. 0.64 respectively), the XGBoost model showed the best performance overall (c-index: 0.75). The most predictive factors for recurrence were treatment types and outcomes as well as socioeconomic status, which were not identified as such by the Delphi questionnaire. Conclusion Our results highlight that ML algorithms have higher predictive power compared to the traditional Cox regression. Moreover, RWD from a cancer registry identified more predictive variables than a panel of experts. Overall, these results have important implications for AI-assisted clinical prognosis and provide insight into the differences between AI-driven and expert-based decision-making in survival prediction. 1. Introduction Ovarian cancer is the fifth deadliest cancer in women worldwide and has the highest mortality rate of all gynecological cancers [ 1 ]. In the Netherlands, approximately 1300 women are diagnosed with ovarian cancer annually, of whom around 1000 will die from the disease, which is usually diagnosed at a late stage [ 2 ]. The five-year survival rate is only 17-28% for women with advanced stage (Federation of International Gynaecology and Obstetrics [FIGO] classification stages IIB-IV) [ 3 , 4 ]. Even after initial treatment, approximately 70% to 80% of patients develop recurrent disease [ 5 ]. Despite decades of research on risk factors for ovarian cancer recurrence, such as FIGO classification and tumor characteristics, it is still challenging to accurately predict progression-free survival for (individual) ovarian cancer patients [ 6 – 8 ]. In recent years, great promise has come from the field of artificial intelligence (AI) and machine learning (ML), where evidence for timely detection and accurate cancer prognosis has grown tremendously [ 9 ]. By learning statistical associations from large amounts of data, ML models can show high accuracy of diagnostic prediction, sometimes comparable to or better than health-care professionals (e.g., by spotting cancerous masses in magnetic resonance imaging data) and time-to-event modeling (e.g., overall/progression-free survival) thereby paving the way towards AI-assisted healthcare [ 10 , 11 ]. Specifically in the field of gynecological cancers, the use of AI has also gained ground. [ 12 ] For example, a recent meta-analysis of 34 imaging studies using AI to detect ovarian cancer across different imaging modalities, showed that algorithms had equal and, sometimes, higher diagnostic accuracy than clinicians [ 13 ]. ML models have also been developed to differentiate benign from malignant ovarian neoplasms, to predict overall survival after diagnosis, and to identify biomarkers and demographic factors contributing to overall survival [ 14 – 16 ]. Most studies related to AI and ML in ovarian cancer have focused on the histopathology, diagnosis and performance. However, prediction of progression-free survival and its related predictive factors has received less attention [ 12 ]. The current study aims to identify predictive factors for ovarian cancer recurrence and evaluate the performance of ML models in comparison to traditional models, such as Cox regression. We specifically focused on patients with advanced-stage disease who face the most challenging survival profile. We featured two sets of predictive variables in our analysis: one set selected by medical experts through a Delphi consensus process, and a larger set derived from real world data (RWD) obtained from the Netherlands Cancer Registry (NCR). This work has important implications for AI-assisted clinical prognosis and provides insight into the differences between AI and medical expert-based variable selection for survival prediction. 2. Data and Methods 2.1. Data source We extracted data on ovarian cancer from the NCR, which is the national population-based cancer registry of the Netherlands, with nationwide coverage since 1989. The registry offers cancer incidence information, along with clinical, socio-demographic and (primary) treatment data. Information on vital status and date of death is updated annually using an automated link with the national Personal Records Database (BRP), and is now up to date until February 1st, 2024. 2.2. Data sample Our initial sample consisted of patients diagnosed with epithelial ovarian cancer in the Netherlands from 2015 – 2017 (n = 3667). We excluded 63 patients with incomplete vital status information. We focused on advanced stages (FIGO stage: IIB – IV; although some where clinically suspect of having early stage disease), because these patients have a much higher mortality rate than those with early stage disease, thereby further excluding 731 patients. Subsequently, we restricted the sample further in order to avoid two common biases. Firstly, we excluded 489 patients that did not receive any therapies, as it would act as a competing risk and, therefore, bias cumulative risk estimators [ 17 ]. In addition, our sample included information on (the timing of) various types of therapy (i.e. neoadjuvant, primary and adjuvant; primary and interval debulking surgery). Since some therapies can take longer to complete than others, they would present inflated survivorship estimates in comparison to the shorter therapy regimes only because people need to survive long enough to finish them, leading to immortal time bias [ 18 ]. Our data showed evidence of immortal time bias (see Appendix A). Since the vast majority (>85%) of the patients had completed their primary therapy within the first 5 months after diagnosis, we excluded a further 355 patients who had disease recurrence before the first 5 months and we started following patients from 5 months after diagnosis. After exclusions, the final sample consisted of 2029 patients. A flow-chart can be found in Figure 1 . Download figure Open in new tab Figure 1. Flow chart of patient selection. 2.3. Delphi consensus making A Delphi questionnaire was conducted to reach expert consensus [ 19 ]. The expert panel consisted of 40 medical doctors (oncologists, gynecologic oncologists, surgical oncologists), 20 of which ended up filling in the questionnaires. We asked participants to answer the following question: “in your experience as a medical specialist, what factors play a role in survival and disease free survival in patients with high stage (FIGO IIb-IV) ovarian cancer?” Out of all responces, we identified 20 possible prognostic factors for recurrent disease. The information for six factors was not available in the NCR. Thus, fourteen expert-identified prognostic factors were included in the analysis (see Table 1 ). Detailed information about the full Delphi process and the outcomes of the questionnaire are presented in the supplementary material. All data were evaluated and analysed by two clinical investigators (AvO, MvE). View this table: View inline View popup Download powerpoint Table 1. Delphi variables agreed by experts as predictive factors, presented in alphabetical order. The second column indicates the presence of information in the NCR. The last column lists NCR variables that encodes this information. “✔” shows presence in the NCR, while “–” shows absence or practically absence (more than 90%) of information in the NCR. Abbreviations: PCI: Peritoneal Cancer Index. CA-125: Cancer Antigen 125, ASA: American Society of Anesthesiologists. CCI: Charlston Comorbidity Index score. 2.4. Data variables 2.4.1. Delphi approach This set includes fourteen predictor variables from the Delphi consensus. These variables constitute the theoretical (i.e., expert-based) minimum number of predictors for progression-free ovarian cancer survival. These variables were defined without seeing the data, thereby constituting a confirmatory test [ 20 ]. 2.4.2. Data-driven approach For the data-driven approach, we relied on a larger RWD set of prognostic factors. In this case, the following NCR variables were included on top of the Delphi variables: pretreatment CA-125, ovariectomy (no vs. yes), ovariectomy with hysterectomy (no vs. yes), staging surgery (no vs. yes), result of debulking (complete vs. optimal vs. suboptimal vs. no debulking), and type of debulking (interval vs. primary vs. no debulking), type of chemotherapy (preoperative vs. postoperative vs. pre&postoperative vs. no chemotherapy vs. yes, but no surgery), hormonal therapy (no vs. yes), intraperitoneal chemotherapy (no vs. yes), hyperthermic intraperitoneal chemotherapy (no vs. yes), differentiation grade (levels: 1-3), laterality (bilateral vs. unilateral vs. extra-ovarian), socio-economic status (levels: 1-10) . 2.5. Machine learning modeling 2.5.1. Cox-based models For the Delphi approach, we used the semi-parametric Cox proportional hazards model with the Delphi variables. This makes the model easy to interpret because it remains agnostic about the baseline hazard function while treating the predictors in a regression-type form. However, the model relies on proportional hazards assumptions, which can be restrictive and may not always hold true in practice [ 21 ]. In our RWD approach, we applied two adaptations of the Cox model with the full set of variables: (i) the Cox model with the Lasso regularization, and (ii) the Cox model with the Elastic Net, which combines Lasso and Ridge Regularizations. Regularization is crucial for two reasons. First, it has been shown to enhance the performance of the Cox model [ 22 , 23 ]. Second, it prevents overfitting by shrinking non-predictive coefficients to zero, effectively performing variable selection. However, compared to Lasso, Elastic Net tends to perform more stable variable selection in the presence of correlated variables [ 24 ]. Therefore, we included both models in subsequent analyses for comparison. 2.5.2. Model fitting The full model fitting procedure is described in Appendix B. In short, missing values were imputed using the Missforest algorithm [ 25 ]. The models were fitted using internal validation based on bootstrapping (n = 500) to adjust for generalization error [ 26 ]. Model parameters were optimized during the internal validation step using a randomized cross-validation parameter search. Bootstrapping was used to achieve stable variable selection estimates. 2.5.3. More advanced machine learning modeling While Cox-based models are easily interpretable, we also explored Random Survival Forests (RSFs) and the extreme gradient boosting trees model (XGBoost), which have demonstrated state-of-the-art performance in order to evaluate their marginal predictive advantage within the RWD approach [ 27 – 30 ]. Both RSFs and XGBoost are not restricted by the proportional hazards assumption, allowing them to capture non-linear relationships in the data. Moreover, since XGBoost can natively handle missing values, we fitted imputed and non-imputed versions of the RWD to this model to investigate the impact of imputation on performance. Appendix B provides more details. 3. Results Table 2 gives the descriptive statistics of the cohort (i.e. variables of the Delphi and the data-driven approach). 77% of patients (i.e., no recurrence = 467; recurrence = 1562) showed disease recurrence after the start of follow-up, which began 5 months after the initial diagnosis, with mean time-to-recurrence at 600 days (standard deviation: 570 days) with a maximum follow-up time of 2043 days. View this table: View inline View popup Table 2. Descriptive statistics of predictor variables (in alphabetical order). In case of categorical or ordinal variables, the count and percentage of each level are given. In case of numerical variables the mean and standard deviation (“SD”) are given. The additional variables of the RWD approach are indicated with a star (*) in the first column. Abbreviations respectively: SD: Standard deviation, ASA: American Society of Anesthesiologists, BMI: Body Mass Index, NA: Not applicable, BRCA: BReast CAncer gene, CCI: Charlson Comorbidity Index, FIGO: Federation of International Gynaecology and Obstetrics, WHO: World Health Organisation, NOS: Not Otherwise Specified, HIPEC: Hyperthermic intraperitoneal chemotherapy, CA-125: Cancer Antigen-125, SES: socio economic status. The comparison of the three Cox models indicated that the baseline (Delphi) model revealed lower predictive accuracy than the RWD approach (cf. Table 3 ). This shows that the additional cancer registry variables have an added predictive value beyond the Delphi variables. Conversely, some predictors identified in the Delphi process may not be very predictive in the presence of the RWD variable set. Although both regularization models achieved the same performance, we picked the Elastic Net model for variable selection, because it produces more stable coefficients than the Lasso model in the presence of correlated variables. Notably, calibration of the Elastic Net model indicated no gross misfit (see Appendix C). View this table: View inline View popup Download powerpoint Table 3. Model performance adjusted for generalization error (i.e., overfitting) based on bootstrapping (500 iterations). Mean and 95% quantile-based confidence intervals are given. Numbers indicate the concordance (c-)index. “NAs” indicate whether the data set contained missing values or whether these were imputed beforehand. Note that only the Cox model (baseline) was fitted to the Delphi variables alone, while the rest of the models were fitted to the RWD variable set. The best performing model is indicated in bold. In comparison to the data-driven Cox-based models, the XGBoost model, which was also fitted on the RWD, achieved higher performance than the Lasso and Elastic Net models, both in the version with imputed data and in the version without data imputation (cf. Table 3 ). Data imputation did not change the performance of XGBoost, implying that the MissForest imputation method did not decrease performance. It is noteworthy that the XGBoost model showed a substantial improvement of performance over the baseline Cox regression model with the Delphi variables. Finally, RSFs scored lower than the XGBoost model, but performed better in comparison to the (regularized) Cox regression models. Variable selection showed a variety of results (cf. Figure 2 ). The top ten predictive variables showed a mix of variables grouped into three categories, relating to 1) intervention type and outcome (i.e., type of surgery and results of debulking, presence of postoperative chemotherapy, number of platinum cycles), 2) clinical characteristics (i.e., differentiation grade, FIGO stage, endometrioid cancer type, BRCA mutation) and 3) demographics (i.e., socio-economic status). Interestingly, only five of the top ten variables were identified by the Delphi panel, namely FIGO stage, endometrioid cancer type, BRCA2 or no BRCA mutation and number of platinum cycles, highlighting the importance of the data-driven variables. Moreover, two of the Delphi variables, e.g. the number of positive lymph nodes and time from chemotherapy to surgery, had low predictive power regarding disease recurrence. Lastly, other variables identified by the Delphi consensus, such as age and BMI, were found to be moderately predictive. Download figure Open in new tab Figure 2. Variable selection from the Elastic Net model. Each dot indicates the proportion that each predictor was non-zero in the bootstrap samples, and the range indicates the estimated 95% Wilson-based confidence interval. The higher the percentage, the more important the variable is for prediction, and vice versa. Abbreviations: FIGO: Federation of International Gynaecology and Obstetrics, BRCA: BReast CAncer gene, CCI: Charlson Comorbidity Index, CCI: Charlson Comorbidity Index, ASA: American Society of Anesthesiologists Chemo: chemotherapy, NOS: not otherwise specified 4. Discussion We applied and compared several machine learning models in order to predict progression-free survival of patients with ovarian cancer. We compared a confirmatory Delphi-based set of predictor variables with RWD drawn from the NCR that included the Delphi variables (i.e., nested comparison). Our results show that regularized Cox regression models based on the RWD variable set outperformed the traditional Cox regression based on the Delphi set (c-indices: 0.64 vs. 0.70 respectively). This means that the traditional Cox regression would correctly discriminate the time-to-event of a random pair of patients only 64% of the time, in contrast to the 70% of the regularized version based on the RWD set. Furthermore, variable selection on the regularized Cox model (i.e., Elastic Net) showed that many predictive variables for recurrence were related to types and results of therapy as well as socioeconomic status, all of which were not included in the Delphi set. Specifically, out of the ten most predictive variables, only five variables were identified by the Delphi consensus. Thirdly, the state-of-the-art XGBoost model achieved the best performance (c-index: 0.75) on the RWD variable list, outperforming both regularized Cox regression models and RSFs. The superior model performance of the RWD approach underlines the importance of using data-driven, cancer registry variables for predicting ovarian cancer recurrence. Although the Delphi variables represent the (minimal) expert-based set of expected predictive factors for recurrence, not all of them were highly predictive (e.g., age), and some showed little to no predictive value (e.g., number of cancer-positive lymph nodes). Furthermore, many factors not mentioned in the Delphi consensus showed a high degree of predictiveness (e.g., socio-economic status). On the other hand, some variables that were not included in the Delphi consensus (e.g., pretreatment CA-125) played indeed a minimal role in recurrence prediction. Our results indicate that while expert-based knowledge plays a valuable role in predicting ovarian cancer recurrence, it has limitations and should therefore be complemented with RWD from cancer registries. Future research can further enrich these data sets by incorporating primary healthcare data as well as more intricate demographic and time-varying information, to improve predictive accuracy. However, it is important to note that the predictive value of the factors analyzed in this study reflects the statistical correlations among these variables, rather than causal relationships. Establishing causality would require counterfactual intervention and a deeper understanding of the causal structure of the study system. [ 31 ] Such causal understanding, which is absent from ML models, may further explain why variable selection results from ML models and from the Delphi questionnaire did not completely overlap. This research question, which is beyond the scope of the current paper, could shed light on the medical decision-making during the Delphi process. Variable selection through the Cox-based Elastic Net is easy to interpret. Due to their complexity, more advanced models such as RSFs and XGBoost require explainability methods to understand how the models use the predictor variables to perform prediction. Techniques such as Shapley values [ 32 ] can provide valuable insights into the contribution of each variable to the model’s prediction, but still require complex computations and are prone to overinterpretation (e.g., as causal indicators) or misinterpretation due to confirmation bias [ 33 , 34 ]. In our current study, we opted to perform variable selection with the easily explainable Cox-based Elastic Net, since the performance of the XGBoost model was only marginally higher than the Cox-based models. The superior performance of the XGBoost model is in line with recent literature suggesting that the model can outperform Cox-based models and other ensemble methods, such as RSFs, for time-to-event modeling [ 35 ]. Although the XGBoost technically outperformed the regularized Cox regression models, its advantage in predictive performance was not substantial. This result may stem from the fact that the proportional hazards assumption of the Cox model seems to not be grossly violated in our data (e.g., the calibration of the Cox models was good; cf. Appendix D). Higher sample sizes (i.e., many tens of thousands patient records) might be required for such non-linear patterns to be effectively used by XGBoost. The current study has many methodological strengths. Firstly, the internal validation method based on bootstrapping validates the model building procedure and provides more stable performance metrics, including overfitting and variable selection, compared to simpler train-test split techniques [ 26 ]. Secondly, our sample size is larger than that in previous studies and we used a wide range of predictive factors, ranging from demographic, clinical and primary therapy information, thereby employing the strength of NCR information [ 36 , 37 ]. Thirdly, we used an a priori expert-informed set of variables (i.e., the Delphi consensus) in order to have a theoretical baseline of the minimal set of predictive factors for recurrence disease before we see the data. This allowed us to separate a confirmatory test of this minimal set of variables from the expanded RWD set. The study has also limitations. First of all, our model is only valid for prediction with follow-up starting at five months after diagnosis and for those patients that received at least some primary therapy, in order to avoid competing risk bias and immortal time bias, as the ML models employed cannot natively correct for these. Future research could employ more advanced methods, such as simulation to model competing risks. In addition, models such as time-varying-coefficient Cox regression could account for the different durations of the different combinations of primary therapies [ 38 ]. A second limitation of the current study concerns the variables suggested by the Delphi consensus but missing from the NCR (i.e., motivation of the patient, response on chemotherapy, level of PCI, CA-125 levels after chemotherapy, involvement of mesentery and quality of surgical team, ascites). In addition, some of the variables suggested by the Delphi (e.g., motivation of the patient or quality of surgical team) cannot be easily put down on a measurable scale. Future research could endeavor to find proxies for such variables, for example in patient self-reports on quality of life and external hospital reviews. 5. Conclusion Our work provides evidence that ML algorithms, such as XGBoost, Random Survival Forests and regularized Cox regression are better in predicting progression-free survival for ovarian cancer than the traditional Cox model. In addition, expert-derived predictors can be enhanced through secondary (cancer registry) data, which can provide better predictive results when combined with ML variable selection. Data Availability Data are not directly available, but need to be requested from Integraal Kankercentrum Nederland (IKNL) through a formal application procedure ( www.iknl.nl ) 7. Statements and Declarations Funding The authors declare that no funds, grants, or other support were directed to the preparation of this manuscript. Competing Interests For Dimitris Katsimpokis: Institutional research funding from AstraZeneca, Roche and Janssen. All other authors have no relevant financial or non-financial interests to disclose. Ethics approval According to the Dutch Central Committee on Research involving Human Subjects, no ethical approval is needed for this study, as it is a retrospective study, which uses data from the NCR. 6. Acknowledgements DK thanks Maarten Bijlsma for his generous theoretical guidance throughout the project. Also thanks to all medical specialists’ who made the Delphi possible. Footnotes ↵ * Joint first author 8. References 1. ↵ Sung , H. , et al. , Global Cancer Statistics 2020: GLOBOCAN Estimates of Incidence and Mortality Worldwide for 36 Cancers in 185 Countries . CA : A Cancer Journal for Clinicians , 2021 . 71 ( 3 ): p. 209 – 249 . OpenUrl 2. ↵ Integraal Kankercentrum , N. , Epitheliaal ovariumcarcinoom - Rapport ovarium carcinoom Nederland . 2019 . 3. ↵ Doubeni , C.A. , A.R.B. Doubeni , and A.E. Myers , Diagnosis and Management of Ovarian Cancer . 2016 . 4. ↵ American Cancer, S ., Key statistics for Ovarian Cancer . 2023 . 5. ↵ Bryant A , H.S., Kunonga PT , Gajjar K , Craig D , Vale L , Winter-Roach BA , Elattar A , Naik R , Impact of residual disease as a prognostic factor for survival in women with advanced epithelial ovarian cancer after primary surgery . Cochrane Database of Systematic Revieuws , 2022 ( 9 ). 6. ↵ Chan , J.K. , et al. , Prognostic factors for high-risk early-stage epithelial ovarian cancer: A Gynecologic Oncology Group study . Cancer , 2008 . 112 ( 10 ): p. 2202 – 2210 . OpenUrl CrossRef PubMed Web of Science 7. Chang , L.C. , et al. , Prognostic factors in epithelial ovarian cancer: A population-based study . PLoS ONE , 2018 . 13 ( 3 ). 8. ↵ Ivana Rizzuto , C.S., Jayanta Chatterjee , Jane Borley , Thomas Glass Hopkins , Hani Gabra , Sadaf Ghaem-Maghami , Les Huson , Sarah P. Blagden , Risk of Ovarian Cancer Relapse Score: A Prognostic Algorithm to Predict Relapse Following Treatment for Advanced Ovarian Cancer . International Journal of Gynecological Cancer , 2015 . 25 ( 3 ). 9. ↵ Buch , V.H. , I. Ahmed , and M. Maruthappu , Artificial intelligence in medicine: Current trends and future possibilities, in British Journal of General Practice . 2018 , Royal College of General Practitioners . p. 143 – 144 . 10. ↵ Xiaoxuan Liu *, L.F., Aditya U Kale , Siegfried K Wagner , Dun Jack Fu , Alice Bruynseels , Thushika Mahendiran , Gabriella Moraes , Mohith Shamdas , Christoph Kern , Joseph R Ledsam , Martin K Schmid , Konstantinos Balaskas , Eric J Topol , Lucas M Bachmann , Pearse A Keane , Alastair K Denniston , A comparison of deep learning performance against health-care professionals in detecting diseases from medical imaging: a systematic review and meta-analysis . The Lancet 2019 . 1 . 11. ↵ Arturo Moncada-Torres , M.C.v.M., Mathijs P. Hendriks Sabine Siesling & Gijs Geleijnse , Explainable machine learning can outperform Cox regression predictions and provide insights in breast cancer survival . Nature 2011 . 11 . 12. ↵ Akazawa , M. and K. Hashimoto , Artificial intelligence in gynecologic cancers: Current status and future challenges – A systematic review, in Artificial Intelligence in Medicine . 2021 , Elsevier B.V . 13. ↵ Xu , H.-L. , et al. , Artificial intelligence performance in image-based ovarian cancer identification: A systematic review and meta-analysis . eClinicalMedicine , 2022 . 53 : p. 101662 – 101662 . OpenUrl PubMed 14. ↵ Lu , M. , et al. , Using machine learning to predict ovarian cancer . International Journal of Medical Informatics , 2020 . 141 : p. 104195 – 104195 . OpenUrl CrossRef PubMed 15. Anna H. Koch , L.S.J., Caroline L. P. Muntinga , T. A. Gootzen , Nienke M. A. van de Kruis , Joost Nederend , Tim Boers , Fons van der Sommen and Jurgen M. J. Piek , Analysis of computer-aided diagnostics in the preoperative diagnosis of ovarian cancer: a systematic review . Insights into Imaging , 2023 . 14 . 16. ↵ Liu , L.L.a.Z.P. , Detecting prognostic biomarkers of breast cancer by regularized Cox proportional hazards models . Journal of Translational Medicine , 2021 . 19 . 17. ↵ Peter C. Austin , D.S.L., Jason P. Fine , Introduction to the Analysis of Survival Data in the Presence of Competing Risks . Circulation , 2016 . 18. ↵ Linda E Lévesque , J.A.H., Abbas Kezouh , Samy Suissa , Problem of immortal time bias in cohort studies: example using statins for preventing progression of diabetes . BMJ , 2010 . 19. ↵ Daniel Beiderbeck , N.F., Heiko A. von der Gracht , Sascha L. Schmidt , Vera M. Schweitzer , Preparing, conducting, and analyzing Delphi surveys: Cross-disciplinary practices, new directions, and advancements . MethodsX , 2021 . 8 . 20. ↵ Eric-Jan Wagenmakers , R.W., Denny Borsboom , and a.R.A.K. Han L. J. van der Maas , An Agenda for Purely Confirmatory Research . Perspectives on Psychological Science , 2012 . 21. ↵ Mats J. Stensrud , D.M.A.H., Why Test for Proportional Hazards? JAMA Guide to Statistics andMethods , 2020 . 323 . 22. ↵ Hoa Helen Zhang , W.L. , Adaptive Lasso for Cox’s proportional hazards model . Biometrika , 2007 . 94 ( 3 ): p. 691 – 703 . OpenUrl CrossRef Web of Science 23. ↵ Francisco M. Ojeda , C.M. , Daniela Börnigen , David-Alexandre Trégouët , Arne Schillert , Matthias Heinig , Tanja Zeller , Renate B. Schnabel , Comparison of Cox Model Methods in a Low-Dimensional Setting with Few Events . Genomics, Proteomics & Bioinformatics , 2016 . 14 ( 4 ): p. 235 – 243 . OpenUrl PubMed 24. ↵ Hui Zou , T.H. , Regularization and Variable Selection Via the Elastic Net . Journal of the Royal Statistical Society Series B: Statistical Methodology , 2005 . 67 ( 2 ): p. 301 – 320 . OpenUrl CrossRef Web of Science 25. ↵ Daniel J. Stekhoven , P.B., MissForest—non-parametric missing value imputation for mixed-type data . Bioinformatics 2012 . 28 ( 1 ): p. 112 – 118 . OpenUrl CrossRef PubMed Web of Science 26. ↵ Frank E. Harrell , J. , Regression Modeling Strategies . With Applications to Linear Models, Logistic and Ordinal Regression, and Survival Analysis . 2015 . 27. ↵ Wansu Chen , B.Z., Christie Y. Jeon , Fagen Xie , Yu-Chen Lin , Rebecca K. Butler , Yichen Zhou , Tiffany Q. Luong , Eva Lustigova , Joseph R. Pisegna , Bechien U. Wu , Machine learning versus regression for prediction of sporadic pancreatic cancer . Pancreatology , 2023 . 23 ( 4 ): p. 396 – 402 . OpenUrl PubMed 28. Arturo Moncada-Torres , M.C.v.M. , Mathijs P. Hendriks , Sabine Siesling & Gijs Geleijnse Explainable machine learning can outperform Cox regression predictions and provide insights in breast cancer survival . Nature - Scientific Reports , 2021 . 11 . 29. Xifeng Wu , H.T., Qingfeng Hu , Shan Pou Tsai , David Ta-Wei Chu , Chi-Pang Wen , Novel machine learning algorithm in risk prediction model for pan-cancer risk: application in a large prospective cohort . BMJ Oncology , 2024 . 3 ( 1 ). 30. ↵ Ivy Y. Zhang , G.R.H., Bo Qin & Jun Deng Long-term survival and second malignant tumor prediction in pediatric, adolescent, and young adult cancer survivors using Random Survival Forests: a SEER analysis . Nature - Scientific Reports , 2023 . 13 . 31. ↵ Miguel A. Hernan , J.H.a.B.H., A Second Chance to Get Causal Inference Right: A Classification of Data Science Tasks . Chance , 2019 . 32 . 1 . 32. ↵ Lundberg , S.M. , A Unified Approach to Interpreting Model Predictions . ArXiv , 2017 . 33. ↵ Marzyeh Ghassemi , L.O.-R., Andrew L Beam , The false hope of current approaches to explainable artificial intelligence in health care . Lancet Digit Health , 2021 . 3 . 34. ↵ Lipton , Z.C. , The mythos of model interpretability in machine learning, the concept of interpretability is both important and slippery . 2018 . 35. ↵ Arturo Moncada-Torres , M.C.v.M., Mathijs P. Hendriks Sabine Siesling & Gijs Geleijnse , Explainable machine learning can outperform Cox regression predictions and provide insights in breast cancer survival . Nature , 2021 . 11 . 36. ↵ M. Timmermans , M.A.v.d.A., R.I. Lalisang , P.O. Witteveen , K.K. Van de Vijver , R.F. Kruitwagen , G.S. Sonke , Interval between debulking surgery and adjuvant chemotherapy is associated with overall survival in patients with advanced ovarian cancer . Gynecologic Oncology , 2018 . 150 ( 3 ): p. 446 – 450 . OpenUrl CrossRef PubMed 37. ↵ J.C.J. Elzakkers , M.A.v.d.A., A.M. van Altena , J.A. de Hullu and M.G. Harmsen , Further insights into the role of tumour characteristics in survival of young women with epithelial ovarian cancer . 2019 . 155 ( 2 ): p. 213 – 219 . OpenUrl 38. ↵ Hernán , M.A. , Sterne , Jonathan A. C. , Higgins , Julian P. T , Shrier , Ianf , Hernández-Díaz , Sonia , A Structural Description of Biases That Generate Immortal Time . Epidemiology , 2025 . 36 ( 1 ): p. 107 – 114 . OpenUrl PubMed View the discussion thread. Back to top Previous Next Posted March 11, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Ovarian cancer recurrence prediction: comparing confirmatory to real world predictors with machine learning Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Ovarian cancer recurrence prediction: comparing confirmatory to real world predictors with machine learning D. Katsimpokis , A.E.C. van Odenhoven , M.A.J.M. van Erp , H.H.B. Wenzel , M.A. van der Aa , M.M.H. van Swieten , H.P.M. Smedts , J.M.J. Piek medRxiv 2025.03.04.25321571; doi: https://doi.org/10.1101/2025.03.04.25321571 Share This Article: Copy Citation Tools Ovarian cancer recurrence prediction: comparing confirmatory to real world predictors with machine learning D. Katsimpokis , A.E.C. van Odenhoven , M.A.J.M. van Erp , H.H.B. Wenzel , M.A. van der Aa , M.M.H. van Swieten , H.P.M. Smedts , J.M.J. Piek medRxiv 2025.03.04.25321571; doi: https://doi.org/10.1101/2025.03.04.25321571 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Epidemiology Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (299) Cardiovascular Medicine (4422) Dentistry and Oral Medicine (443) Dermatology (382) Emergency Medicine (607) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1507) Epidemiology (15217) Forensic Medicine (30) Gastroenterology (1123) Genetic and Genomic Medicine (6584) Geriatric Medicine (667) Health Economics (997) Health Informatics (4524) Health Policy (1367) Health Systems and Quality Improvement (1611) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15909) Intensive Care and Critical Care Medicine (1103) Medical Education (622) Medical Ethics (145) Nephrology (667) Neurology (6583) Nursing (345) Nutrition (998) Obstetrics and Gynecology (1143) Occupational and Environmental Health (956) Oncology (3330) Ophthalmology (970) Orthopedics (369) Otolaryngology (420) Pain Medicine (435) Palliative Medicine (129) Pathology (663) Pediatrics (1690) Pharmacology and Therapeutics (691) Primary Care Research (710) Psychiatry and Clinical Psychology (5437) Public and Global Health (9218) Radiology and Imaging (2194) Rehabilitation Medicine and Physical Therapy (1369) Respiratory Medicine (1195) Rheumatology (593) Sexual and Reproductive Health (709) Sports Medicine (529) Surgery (709) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9ff729898d8d52ad',t:'MTc3OTQwNDA1OQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();