Topological embedding and directional feature importance in ensemble classifiers for multi-class classification

Topological embedding and directional feature importance in ensemble classifiers for multi-class classification | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Topological embedding and directional feature importance in ensemble classifiers for multi-class classification Eloisa Rocha Liedl , Shabeer Mohamed Yassin , Melpomeni Kasapi , View ORCID Profile Joram M. Posma doi: https://doi.org/10.1101/2024.08.01.605982 Eloisa Rocha Liedl 1 Section of Bioinformatics, Division of Systems Medicine, Department of Metabolism, Digestion and Reproduction, Imperial College London , W12 0NN, United Kingdom 2 Department of Surgery and Cancer, Imperial College London , W12 0NN, United Kingdom Find this author on Google Scholar Find this author on PubMed Search for this author on this site Shabeer Mohamed Yassin 1 Section of Bioinformatics, Division of Systems Medicine, Department of Metabolism, Digestion and Reproduction, Imperial College London , W12 0NN, United Kingdom 3 Centre for Integrative Systems Biology and Bioinformatics (CISBIO), Department of Life Sciences, Imperial College London , SW7 2AZ, United Kingdom Find this author on Google Scholar Find this author on PubMed Search for this author on this site Melpomeni Kasapi 1 Section of Bioinformatics, Division of Systems Medicine, Department of Metabolism, Digestion and Reproduction, Imperial College London , W12 0NN, United Kingdom 4 Section of Nutrition, Department of Metabolism, Digestion and Reproduction, Imperial College London , W12 0NN, United Kingdom Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: jmp111{at}ic.ac.uk mk218{at}imperial.ac.uk Joram M. Posma 1 Section of Bioinformatics, Division of Systems Medicine, Department of Metabolism, Digestion and Reproduction, Imperial College London , W12 0NN, United Kingdom Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Joram M. Posma For correspondence: jmp111{at}ic.ac.uk mk218{at}imperial.ac.uk Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Cancer is the second leading cause of disease-related death worldwide, and machine learning-based identification of novel biomarkers is crucial for improving early detection and treatment of various cancers. A key challenge in applying machine learning to high-dimensional data is deriving important features in an interpretable manner to provide meaningful insights into the underlying biological mechanisms We developed a class-based directional feature importance (CLIFI) metric for decision tree methods and demonstrated its use for the The Cancer Genome Atlas proteomics data. We incorporated this metric into four algorithms, Random Forest (RF), LAtent VAriable Stochastic Ensemble of Trees (LAVASET), and Gradient Boosted Decision Trees (GBDTs), and a new extension incorporating the LAVA step into GBDTs (LAVABOOST). Both LAVA methods incorporate topological information from protein interactions into the decision function. The different models’ performance in classifying 28 cancers resulted in F1-scores of 93% (RF), 92% (LAVASET), 89% (LAVABOOST) and 86% (GBDT), with no method outperforming all others for individual cancer type prediction. The CLIFI metric allowed the visualisation of the model decision making functions, and the distributions indicated heterogeneity in several proteins (MYH11, ER α , BCL2) for different cancer types (including brain glioma, breast, kidney, thyroid and prostate cancer). We have developed an integrated, directional feature importance metric for multi-class decision tree-based classification models that facilitates interpretable feature importance assessment. The CLIFI metric can be used in conjunction with incorporating topological information into the decision functions of models to add inductive bias for improved interpretability. Availability All codes are available for data curation from https://github.com/EloisaRL/TCGA-proteomics-pipeline and the LAVASET (v1.0) package from https://github.com/melkasapi/LAVASET . Introduction Cancer has risen from third place in 2010 to second place in 2019 as the leading cause of disease-related death worldwide, and it is forecasted that the global cancer burden will continue to grow for the next two decades ( 1 – 3 ). As a result, the application of machine learning (ML) methods in oncology has expanded ( 4 ). In addition to data analysis automation, ML models often show higher accuracy in diagnosis and survival predictions than traditional clinical methodology ( 5 , 6 ). Given the shift in cancer medical practice to personalised and targeted treatments, classification of cancer patients is moving towards the need of identifying subpopulations within one cancer type ( 7 , 8 ). Ensemble methods, a subfield of ML, are commonly used for this classification task ( 9 – 11 ), as they excel in maintaining performance with high-dimensional datasets that have a small number of samples relative to the number of features (small n , large p ), unlike other ML methods - including deep learning algorithms that require more samples for effective training ( 12 ). Such high-dimensional datasets are common in cancer research, including genomic, epigenetic, transcriptomic, proteomic, and clinical data. Examples of ensemble methods include Random Forests (RFs) ( 13 ) and Gradient Boosted Decision Trees (GBDTs) ( 14 ). Despite gradient boosting showing superior prediction performance in previous cancer prediction studies compared to RF, support vector machines, and logistic regression algorithms ( 15 – 17 ), the existence of heterogeneity limits the application of RF and GBDT in cancer. While binary classification models can still be easily interpreted in how predictions are being made, by using Gini values, multi-class classification algorithms are not ( 18 ). High-dimensional datasets often contain correlated features (feature interactions), which can lead the model to overfitting, by capturing noise or redundant features. Hence, they compromise model generalisation by assigning importance to non-biologically meaningful features ( 18 ). There are techniques that can address the problem of correlated features e.g. Boruta algorithm ( 19 ), a novel RF algorithm, but they do not explicitly incorporate domain knowledge of correlations. The Boruta algorithm was utilised to identify microRNAs distinguishing normal and ovarian cancer patients, yielding more differentially expressed microRNAs compared to previous studies ( 20 ). Subsequent pathway analysis validated the ML findings, demonstrating concordance with existing literature ( 20 ). While these algorithms are effective, biological pathway interpretation and validation are necessary to verify and interpret the results. Topological ML algorithms incorporating this type of information into the model architecture can address the multicollinearity problem and make the model more interpretable thereby reducing the need for extensive post hoc validation analyses. Informative feature importance values depend on two factors: class-specificity and directionality. ML models in cancer research commonly utilise binary classification models with Gini-based feature importance assignment, which considers only the magnitude of features’ influence on predictions ( 10 , 21 , 22 ). Applying multi-class classification algorithms offer greater versatility as they enable prediction of multiple classes using a single model. In cancer research, ensemble methods with RF and GBDTs are rarely used for multi-class classification, instead a binary one versus all approach is usually employed to identify classspecific important features. For example, Ortiz-Ramon et al. used two models to discriminate brain metastases patients, based on their primary site of origin: a multiclass RF model with 87% accuracy and a one versus all model with 82% accuracy ( 23 ). They then, identified class-specific Gini-based feature importance by analysing the one vs all models. While the Gini-based feature importance did help identify differentiating biomarkers, it did not clarify whether high or low values of the biomarkers are associated with each class. The need for class-specific and directional feature importance assignment is essential for broader application of multi-class ensemble models. Directional feature importance calculations are available for ensemble models, for example permutation importance ( 13 ) and SHapley Additive exPlanations (SHAP) ( 24 ); however, they are post-hoc and not incorporated within the methods. To the best of our knowledge, no such method exists for RF or GBDT that is integrated within algorithm. Our contribution aims to address two gaps. The first is to introduce an integrated, directional feature importance metric for decision tree-based models (such as RF and GBDT) to facilitate feature importance assessment for multi-class classification. We demonstrate the feature importance measure on the small-scale Fisher Iris data before applying it on a large dataset. The second is to expand on recently published work, LAVASET (LAtent VAriable Stochastic Ensemble of Trees) ( 25 ), that demonstrated how data-specific properties can be integrated in the ensemble model for better interpretability for correlated, temporal and spatial features. We extend this paradigm to GBDTs, producing LAVABOOST (LAtent VAriable gradient BOOSTed decision trees), and apply the 4 algorithms for multi-class classification in The Cancer Genome Atlas (TCGA) proteomics dataset ( 26 ) of 28 different cancers with incorporation of topological information of protein-protein interactions. Materials and Methods Data Data - Fisher Iris The original Fisher Iris data was obtained from scikit-learn and used to demonstrate the directional feature importance. Here, we added 3 random noise features of different distributions (Gaussian, uniform, and bimodal) to the original features (sepal length, sepal width, petal length, petal width) to test the feature importance for features with different conditions of (random) distributions (we refer to this dataset as ‘Iris’). Additionally, we randomly shuffled the values for each feature to break the correlation pattern of the Iris data to demonstrate the directional feature importance output for models without predictive power, this dataset is referred to as ‘Iris-permuted’. Data - TCGA Proteomic profiling data of 28 cancer types was obtained from TCGA project. These publicly available datasets were generated from tumour tissue analysis conducted by the University of Texas MD Anderson Cancer Centre using reversed-phase protein arrays ( 26 ). A total of 243 glioblastoma multiforme (GBM), 435 brain lower grade glioma (LGG), 354 head and neck squamous cell carcinoma (HNSC), 381 thyroid carcinoma (THCA), 126 oesophageal carcinoma (ESCA), 226 sarcoma (SARC), 352 skin cutaneous melanoma (SKCM), 365 lung adenocarcinoma (LUAD), 328 lung squamous cell carcinoma (LUSC), 62 mesothelioma (MESO), 90 thymoma (THYM), 919 breast invasive carcinoma (BRCA), 357 stomach adenocarcinoma (STAD), 184 liver hepatocellular carcinoma (LIHC), 120 pancreatic adenocarcinoma (PAAD), 46 adrenocortical carcinoma (ACC), 82 pheochromocytoma and paraganglioma (PCPG), 478 kidney renal clear cell carcinoma (KIRC), 63 kidney chromophobe (KICH), 216 kidney renal papillary cell carcinoma (KIRP), 363 colon adenocarcinoma (COAD), 132 rectum adenocarcinoma (READ), 432 ovarian serous cystadenocarcinoma (OVCA), 440 uterine corpus endometrial carcinoma (UCEC), 172 cervical squamous cell carcinoma and endocervical adenocarcinoma (CESC), 352 prostate adenocarcinoma (PRAD), 122 testicular germ cell tumours (TGCT), 343 bladder urothelial carcinoma (BLCA), were included in the initial dataset. All samples were obtained from the TCGA portal (version 39.0, release date 04/Dec/2023) at https://portal.gdc.cancer.gov/ . Data Processing - TCGA Proteins with more than 50% missing values were removed. Missing values were imputed using the k-Nearest Neighbours Imputer ( scikit-learn, v1.3.2 ) with k=5. The protein list was further refined to include only those that had been validated by Western blotting in a TCGA pan-cancer independent study ( 27 ). Compared to the prior study, we included additional cancer types. To assess whether this affected clustering ability, an unsupervised analysis of the data with and without the additional cancer types showed no clear differences. Consequently, the final dataset comprised of 7,783 samples with 113 proteomic features. Machine Learning Classifiers Random Forest and LAVASET The LAVASET package provides implementation for both the RF and LAVASET algorithms ( 25 ), based on the principles of Breiman’s original RF algorithm ( 13 ). LAVASET utilises the original C++ implementation for computational efficiency, with main functions written in Python for easier user readability. The traditional Classification And Regression Trees (CART) algorithm ( 28 ) is used to build individual decision trees. The modifiable hyperparameters for both algorithms are the number of trees, number samples per tree, and the maximum number of features. LAVASET in addition has a distance parameter which defines the threshold strength of feature interactions to be considered in the embedding. The ‘LAVA’ step occurs before the feature is chosen for the best split (see ( 25 ) for complete details). In brief, Principal Component Analysis (PCA) is used to identify the first right singular vector (loading) of the decomposition of the feature selected and its correlated features. The loading is then used to transform the original features into a single score (left singular value). This is performed for all features (and their neighbours/correlated features) selected at the splitting step. Therefore, the feature dataset is transformed to consist of latent variables instead of the original feature values. Extension of Latent Variable Embedding to Boosting Algorithms The multiclass classification gradient boosting decision tree (GBDT) algorithm developed in this study was adapted from Matt Bowers’ implementation of Algorithm 6 from greedy function approximation ( 14 ). This implementation uses a one-against-all approach to reduce the problem into K binary problems, where K is the number of classes. Our implementation of GBDT, as well as LAVABOOST (Latent Variable embedding into GBDTs), uses the traditional CART algorithm ( 28 ) with the original C++ implementation to build individual decision trees rather than the scikit-learn DecisionTreeRegressor function. The sequential trees in GBDTs and LAVABOOST predict the errors of the previous prediction therefore, to prevent biases, the predictions are initialised to 0. LAVABOOST has a number of prerequisites and hyperparameters that can be optimised (see section). A distance matrix is provided by the user containing the strength of interactions between features in the input data set (see section). The features the algorithm considers correlated are those with an interaction strength less than or equal to the threshold set by the user, the threshold is set by the ‘distance’ parameter. The modifiable optimisation parameters include the number of estimators (boosting rounds), learning rate, number samples per tree, and the maximum number of features. When the number of samples per tree or maximum number of features is set to None , all samples/features, respectively, are used. Distance Metric for Latent Variable Embedding For use in LAVASET and LAVABOOST only, we produced a distance matrix representing the protein interactions. The Search Tool for the Retrieval of Interacting Genes/Proteins database (STRINGdb) version 12 ( https://string-db.org/ ) ( 29 ) was used to identify the protein-protein interactions of all proteins in the TCGA dataset. For any TCGA protein name not found in the STRINGdb database, we used the gene name of each protein (listed in ( 27 )) to find protein interactions between all 113 features. Protein interactions were defined by self-interactions (distance d=0), proteins linked to the same gene (d=1), protein-coding gene-gene interactions (d=2, evaluated using confidence level of 0.7), and no interaction (d=100). Directional Feature Importance The directional features importance calculation occurs after the optimal feature (or latent feature for LAVASET and LAVABOOST) has been selected with the optimal split. For all splits, the Gini coefficient values are calculated. For RF and GBDT, the Gini is attributed to the chosen feature for the optimal split. For LAVASET and LAVABOOST, a normalised Gini feature importance is calculated for each original feature using the absolute value of the loadings (normalised to a sum of 1). Below we describe the newly proposed CLass-based Integrated directional Feature Importance (CLIFI) calculation for ensemble models such as RF, GBDT, LAVASET and its novel extension LAVABOOST. The benefit of this calculation is that it is integrated into the algorithm and therefore feature importances of the models can be directly inferred without need for additional (post-hoc) techniques. The G-test ( Eqn. 1 ), a complimentary approach to Kullback-Leibler divergence, to compare the observed and expected frequencies of categorical data (i.e. the distribution of groups after splitting). Where O i is the observed count for the i th category and likewise, E i is the expected count for the same category under the null hypothesis. We use a modified version of the G-test to quantify the distribution of samples in a class in each of the splits, where the observed count in Equation 1 is replaced with the frequency in the left split ( L i ) for class i ( Eqn. 2 ), and the expected count is replaced by half the frequency in the parent node . The same process is done for frequency in the right split ( R i , Eqn. 3 ). These two quantities are partial G-tests ( M ij ), and summing these results is the combined G-test value for that class. This assumes that P i , and either (or both) of L i and R i , are non-zero. A perfect split is represented by 2 × P i × ln 2, therefore we normalise M by the perfect split to scale it between 0 (equal split) and 1 (perfect split) and subsequently multiply with P i /A i to account for the number of samples of the class considered for the split relative to all samples in that tree ( A i , ‘ancestor’ node). The directionality of the feature for class i is indicated by the sign of M i 2 − M i 1 . Combining this, and simplifying, results in Equation 4 for the CLIFI of feature j for class i : A positive CLIFI value for a feature signifies that the direction of the split is weighted more towards the right node (higher split values) than the left (lower split values). The magnitude of the association is given by the numerical value itself, where the closer is it to 1 (or −1), the more it is associated with higher (or lower) values of that feature for RF and LAVASET models. The aggregated CLIFI (aCLIFI) value is the sum of the CLIFI values across all trees for a specific feature and class, and for comparison between models the normalised aggregated CLIFI (naCLIFI) values represent a division by the highest absolute aggregated CLIFI value across all features and classes. For GBDT and LAVABOOST the above cannot be used directly because each class label ( i ) predicted by the model is an error label, therefore, samples of the same class (noted by h ) may have different error labels. Therefore, the CLIFI for these models is calculated by scaling the CLIFI value for the error label by the proportion of samples in the real class ( Eqn. 5 ), where x represents the distinct error labels associated with class h , and S is the number of samples within class h with error label i divided by the total number of samples in class h . Model Evaluation 30% of the data was set aside for testing, with the remaining 70% split 80:20 into training and validation sets. Individual class frequencies were balanced across the subsets of data. The training set (ca. 56% of the total data) was used to calculate individual models with different hyperparameter settings, and the validation set (ca. 14% of the total data) was used to determine the optimal hyperparameters. The optimal model was then used to predict the left-out test set (30%). For the TCGA data, RF, LAVASET, and GBDT were run 100 distinct times and LAVABOOST 10 distinct times with different random states to evaluate the model robustness. The optimal hyperparameters for RF and LAVASET models were models with 150 trees and ‘sqrt’ of number of features for each split. The values trialed for number of trees were 100 to 180 in steps of 10. Both models used 80% of samples per tree. Optimal parameters for GBDT and LAVABOOST were 130 estimators (boosting rounds), a learning rate of 0.1, ‘sqrt’ of number of features for each split, and including all samples (‘None’). The values trialed for number of trees were 10 to 150 in steps of 10, and the values trialed for learning rate were 0.1, 0.2, and 0.3. The distance parameter for LAVABOOST and LAVASET was set to 2. Classification performance was assessed based on accuracy, precision, recall and F1-score, with the mean ± standard deviation reported for each model, for the test set. The proximity matrix was used as input to UMAP (number of components= 2 , init= random , random state= 0 ) ( 30 ) to visualise the similarity between samples from different cancer types. The Iris dataset was split 80:20 into training and validation sets. The optimal parameters for the Iris data were ‘sqrt’ of number of features per split for RF and GBDT, 100 trees for RF, and 7 boosting rounds with 0.1 learning rate for GBDT. Evaluation and Visualisation of CLIFI Values Class feature importance assignment was evaluated for LAVASET, RF, LAVABOOST, and GBDT using the python package NetworkX. A template protein layout on a hexagonal grid was defined based on the protein interactions, thus interacting proteins are closer together, and was used for all algorithms. The feature importance results present the top 10 proteins with the highest CLIFI value (and higher than the average CLIFI value of the positively assigned proteins), with the same was done for the negatively assigned proteins (10 lowest CLIFI values that are lower than the average CLIFI of negative CLIFI values), with LAVASET and LAVABOOST also showing protein interactions between these proteins. For visualisation purposes that was restricted to the top 10 to allow all edges to be drawn without node-edge crossings. Paired and unpaired t-tests were used to assess differences between methods in model performance. A Kruskal-Wallis test was used to evaluate whether features’ CLIFI values were significantly different between classes. For those features with significant differences, a Mann-Whitney U test (Wilcoxon rank sum test) was used for pairwise comparisons and these were visualised in a heatmap. All p-values were adjusted for multiple testing using the Benjamini-Hochberg method. Compute, Software Versioning, and Code Availability All calculations were performed on an HP Z6 G4 workstation with 16-core Intel(R) Xeon(R) Silver 4110 CPU @ 2.10GHz with 128GB RAM. Combining the TCGA study datasets and defining the protein interactions were implemented with R in (v4.3.2) R Studio (v2023.09.1+494). These codes are available from https://github.com/EloisaRL/TCGA-proteomics-pipeline . All other analyses were implemented in Python (v3.12.1) with libraries and packages: lavaset (v0.1.1), matplotlib (v3.8.2), networkx (v3.2.1), numpy (v1.26.2), pandas (v2.1.4), plotly (v5.19.0), scikit-learn (v1.3.2), seaborn (v0.13.2), umap (v0.1.1), and umap-learn (v0.5.5). The code for GBDT and LAVABOOST, and implementation of the direction feature importance for all 4 algorithms (including RF and LAVASET), have been added as separate branch to the LAVASET repository ( https://github.com/melkasapi/LAVASET , v1.0.0). Results Classification Performance Iris Data We show that our algorithms show consistent performance with respect to the example datasets, where the permuted version has low classification r ates, a nd f or t he real data we show that RF outperforms GBDT (p < 0.0001) ( Table 1 ). For the demonstration of the CLIFI values, we choose to show these for the RF models for both the Iris (predictive) and Iris-permuted (not predictive) datasets. View this table: View inline View popup Download powerpoint Table 1. Classification performance of RF and GBDT for the Iris and Iris-permuted datasets. Scores for accuracy, precision (weighted), recall (weighted), F1-score (weighted) are given across 20 random initialisations of the models. The RF was run with 100 trees, GBDT with 7 boosting rounds and learning rate of 0.1. Values shown are the mean ± standard deviation, represented as percentages. Values in bold indicate the highest performances for each metric. TCGA Data For the TCGA dataset (28 classes), RF outperformed the other methods ( Table 2 ) though while its difference with LAVASET is significant (p < 0 .001) t he difference is small. Both RF and LAVASET outperform GBDT and LAVABOOST, with LAVABOOST outperforming GB-DTs (p < 0.001). LAVABOOST appears to have larger variability in its performance compared to LAVASET. The class-specific performance varies between 0% to 100% for all algorithms (see), with no model consistently outperforming other models on all cancer types. For example, while RF and LAVASET have a considerable higher overall accuracy, for glioblastoma, brain lower grade glioma, thyroid, and testicular germ cell cancer the boosting methods have better performance. View this table: View inline View popup Download powerpoint Table 2. Classification performance of RF, LAVASET, GBDT, and LAVABOOST. Scores for accuracy, precision (weighted), recall (weighted), F1-score (weighted) across 100 random initialisations of RF, LAVASET, and GBDT, and 10 random initialisations of LAVABOOST. Values shown are the mean ± standard deviation. Values in bold indicate the highest performances for each metric. Feature Importance Assessment Iris Data The CLIFI value importance assignment was first tested on the Iris dataset to serve as a benchmark for demonstrating the interpretability. Figure 1A shows the output from using the Gini coefficient indicating t hat sepal length (SL), petal length (PL), and petal width (PW) are most predictive. In the CLIFI distribution plot ( Fig 1B ) PL, PW and SL have the broadest spread of CLIFI values, i.e. they deviate most from 0, hence these are likely to be predictive in the model. Figure 1C-E shows the distribution of CLIFI values for the 3 noise variables for each class (Setosa, Versicolour, Virginica) is centered at 0, and while come noise variables had higher Gini coefficients than the real sepal width (SW) feature there are no significant differences between groups in the CLIFIs. The largest differences are seen for the real variables ( Fig 1F-I ). These distributions clearly show the relationship between individual features and the class information. Download figure Open in new tab Fig. 1. Visualisation of feature importance for Iris and Iris-permuted dataset for the RF model. (A) Normalised Gini coefficients for the Iris dataset. (B) Aggregated distribution of CLIFI values across all classes, CLIFI distributions that deviate from a normal distribution around 0 may have class information. (C) Distribution of CLIFI values for Gaussian noise. (D) Distribution of CLIFI values for uniform noise. (E) Distribution of CLIFI values for bimodal noise. (F) Distribution of CLIFI values for sepal length. (G) Distribution of CLIFI values for sepal width. (H) Distribution of CLIFI values for petal length. (I) Distribution of CLIFI values for petal width. (J) Normalised Gini coefficients for the Iris-permuted dataset. (K) Aggregated distribution of CLIFI values across all classes. (L) Distribution of CLIFI values for sepal length. (M) Distribution of CLIFI values for sepal width. (N) Distribution of CLIFI values for petal length. (O) Distribution of CLIFI values for petal width. Pairwise comparison within a class were tested. *** = p < 0.0005, ** = p < 0.005, * = p < 0.05. While the Gini coefficients are still non-zero ( Fig 1 J ) for Iris-permuted, the CLIFI values are all centered around 0 ( Fig 1K ) indicating an equal split. The differences in the permuted features ( Fig 1L-M ) are all centered around 0 and considerably more narrow than those from the real model ( Fig 1F-I ), hence features with a wider dispersion of CLIFI values away from zero (toward either −1 or 1) signifies greater predictive importance in class separation. TCGA Data - Feature Importance Visualisation Analysis of the topmost important proteins across all cancers show that RF and LAVASET have the capability to detect proteins with similar average CLIFI magnitudes of importance across multiple cancer types, e.g. progesterone receptor (PR) in RF, and cyclin B1 and ER α (oestrogen receptor alpha) in both RF and LAVASET. We visualised the aggregated CLIFI values (normalised to the maximum value to allow comparison between methods) as heatmaps in Figure 2 . Download figure Open in new tab Fig. 2. Heatmap of the top ten most important proteins across all 28 cancer types. Proteins selected based on the highest absolute normalised aggregated CLIFI value. (A) RF, (B) LAVASET, (C) GBDT, and (D) LAVABOOST. Colour bar shows the feature importance gradient, from negative to positive CLIFI-values, with white indicating a zero value. Notably, within these proteins with high naCLIFI magnitude concordance, there are certain cancer types with significantly different naCLIFI values when compared to the majority. For example, Figure 2A shows that while the average naCLIFI values for PR is close to 1 in most cancer types, in lung adenocarcinoma, lung squamous cell, and kidney renal clear cell, the average naCLIFI value is close to −1. Additionally, Figure 2B shows that while the average naCLIFI value for Cyclin B1 is close to 1 in most cancer types, in kidney renal clear cell the average naCLIFI value is close to −1. However, this is not seen in the GBDT or LAVABOOST ( Fig 2C,D ), instead most proteins have a high CLIFI value magnitude for some cancer types while the others have varying magnitudes. For example, the MYH11 protein has a high magnitude of naCLIFI values for thymoma ( 0) in all models, with the boosting models showing greater variability due to these predicting errors of prior trees. TCGA Data - Interpretability of Feature Importance in Latent Variable Embedding Models While the heatmaps show the overall importance, these plots fail to display the relations between features. LAVASET and LAVABOOST integrate topological information, hence the outputs are conditional on the mapping of features. Figure 3 shows the top proteins for four cancer types for which LAVASET outperformed all other methods. The edges show the neighbours in the distance matrix for calculating the latent embedding to facilitate interpreting the feature relations. Download figure Open in new tab Fig. 3. Visualisation of important features (node colour) and protein-protein interactions (edges) for selected cancer types from the LAVASET multi-class classification model. (A) Brain lower grade glioma (F1 = 0.97). (B) Thyroid cancer (F1 = 0.98). (C) Uterine endometrial cancer (F1 = 0.95). (D) Prostate adenocarcinoma (F1 = 1.00). Edges are only shown if labelled proteins have interactions. It can be seen that while MYH11 (smooth muscle myosin heavy chain 11) has interactions with BCL2 (B-cell lymphoma 2) and p38MAPK (p38 mitogen-activated protein kinase), and ER α with RAB25 (Ras-related protein in brain 25) and SMAD1 (Mothers Against Decapentaplegic Homolog 1), this is independent from the signs of the naCLIFI values. Incorporating the topological information allows for interpretation of the model in terms of (joint) function and role rather than on an individual protein basis (as with RF and GBDT). More relations between proteins are incorporated into the model, however here we only visualise the top few for simplicity. TCGA Data - Individual Feature Importance Evaluation Similar to the Iris data, the CLIFI values allow for the visualisation of feature importance across all classes for individual features, with the distribution indicating the sign of association with the split ( Fig 4 ). As example, we show several proteins with different distributions of CLIFI values. Download figure Open in new tab Fig. 4. Visualisation with violin plots of five selected top proteins, and one unimportant, from the LAVASET multi-class classification model. From left to right: BCL2, ER α , NOTCH1, MYH11, TFRC, MTOR. For example, the distribution of CLIFI values for BCL2 indicates that it is importance in the classification of several cancers, each of which appear to have bi- or multimodal distributions (hinting at heterogeneity), and specifically with thyroid and kidney renal papillary cell (higher expression) and oesophageal, mesothelioma, liver, pancreatic and adrenocortical (lower expression) cancers. TFRC (transferrin receptor) shows CLIFI distributions trending in opposite directions for lung squamous cell carcinoma (higher) and lung adenocarcinoma (lower) indicating potentially discriminatory potential between different cancers of the same organ. Other proteins such as ER α and MYH11 have clear patterns for specific cancers with higher and lower expression of these proteins, with the bimodal distributions more evident that for other proteins. NOTCH1 (Neurogenic locus notch homolog protein 1) has CLIFI values largely centered around 0 except for brain lower grade gliomas, however while it may appear its expression may not relate to differences between other cancers, its distributions are considerably different from the MTOR (mammalian target of rapamycin) protein which is not associated with any class separation in any model. We have observed that the CLIFI values relate to the protein expression levels in the original data. For example for MYH11 ( Fig 5 ) prostate and stomach cancer have the highest CLIFI values matching the protein expression levels. Likewise, oesophageal, thymoma, cervical, and testicular cancers have CLIFI values indicating lower expression levels. Though mesothelioma, liver, prostate and some other cancer types exhibit bimodal (or multimodal) distributions pointing towards possible heterogeneity in the proteomic signature of these cancers. Download figure Open in new tab Fig. 5. Comparison of protein expression level for MYH11 with CLIFI value from LAVASET model. TCGA Data - Visualisation of Heterogenity in Classification of Cancer Types The model performance is given with performance values for each cancer types, however this fails to show the homo- and heterogeneity of the data. We calculate a proximity matrix from the output from the ensemble model which represents the frequency that pairs of samples end up in the same leaf node in the classification model. We graphically represent the proximities as input to UMAP (2 dimensions) in Figure 6 . The CLIFI values for several proteins indicated that breast cancer display bimodal distributions, and the UMAP plot shows that there appear to be multiple breast and stomach cancer subgroups based on the proteomics-based proximity matrix from LAVASET and visualises the similarity between cancer types based on the model predictions. Download figure Open in new tab Fig. 6. UMAP visualisation of the LAVASET proximity matrix with each cancer type labelled showing heterogeneity of some cancer types in the model prediction. A > B indicates cluster contains more samples from cancer type A than cancer B, ‘/’ indicates completely overlapped samples within a cluster. While ovarian and uterine cancer had similar CLIFI values to breast cancer for some proteins (e.g. ER α ), overall they cluster separately from breast cancer. The UMAP visualisation also shows that colon and rectal cancer (that are indistinguishable) are similar to two clusters of stomach cancer samples, the two lung cancers are closely related to head and neck cancers, and there are 2 clusters of brain cancers (one with majority of glioblastoma samples, another with majority of lower grade gliomas), all of which relates to the relative proximity of these cancers in the body. Discussion We have introduced a new directional feature importance metric, CLIFI, that is integrated within the algorithm. The distribution of CLIFI values contain information on class separation and can be represented both on a global and classspecific level. Therefore, combined with being embedded within the training procedure, it provides unique insights that other methods such as the Gini coefficient and SHAP values do not, such as allowing the identification of heterogeneity within a class for specific features. Secondly, we have extended the LAVA framework to gradient boosting methods and introduced the LAVABOOST algorithm here for datasets with correlated features and multiple classes. While our results indicate that for these data RF and LAVASET outperform GBDTs and LAVABOOST, all methods at least outperform others for some cancer types in the TCGA proteomics data. Since the CLIFI calculation is based on measuring the change in class distribution, it is not dependent on the class prediction accuracy and provides a new level of interpretability of stochastic ensemble models. Unlike the Gini coefficient, also arising from the training model, we demonstrated that CLIFI values are (significantly) higher for real features than random ones for the Iris example dataset, and that if the feature contains little class information (Iris-permuted) the CLIFI values have narrow intervals and center around 0, whereas Gini coefficients are still large. The same was shown for the real TCGA data where unimportant features have narrow CLIFI distributions around 0 (e.g. MTOR) and predictive features such as BCL2, ER α , NOTCH1, and MYH11 deviate from this. Although, this may suggest that CLIFI is correlated with accuracy, both RF and LAVASET assign a strikingly high importance to ER α for classifying rectum adenocarcinoma, with overall F1-scores of 0 and 4.65% for RF and LAVASET, respectively. CLIFI resembles the manner in which the model makes its predictions. In the case of rectum adenocarcinoma, these patients are predicted with colon adenocarcinoma patients since both have very similar importance assignments in the heatmaps and the precision of colon adenocarcinoma is noticeably lower compared to other cancers (71-72% versus 84-100%). Another benefit of using a feature importance calculation based on the change of class distribution is that it is unaffected by unbalanced classes, unlike other commonly used methods such as the Gini coefficient and SHAP values. Finally, the CLIFI distribution identifies the feature profile used by the model for specific class predictions, and it can be used to identify features with bi- or multimodal distributions within a class, even though overall the class could appear as homogenous. While our aim here was to demonstrate the new methodology, the use of the publicly available TCGA data ( 26 ) allow comparison of model outputs with previously published literature. For example, for uterine corpus endometrial carcinoma LAVASET ( 25 ) outperforms the other methods and the proteins with the highest magnitude of average naCLIFI values (MYH11, ER α , and cyclin B1) have all been found to be expressed in the same direction that is indicated by the CLIFI values by others in endometrial cancers ( 31 – 33 ) previously. The original LAVASET method was applied on different datasets, but with distances defined by the data itself (spatial or temporal) ( 25 ). Here we integrated protein interaction data into this step to allow the method to make the decisions based on topological (pathway) information. Analysis of the top proteins with the most interactions identifies pathways which are also important for disease progression in cancer, such as epidermal growth factor signalling (via EGFR and EGFRpY1173) ( 34 ), the MAPK pathway (via p38MAPK and MAPKpT202Y204) ( 35 ), and BCL-regulated apoptotic pathway (via BCL-2 and BCL-xL) ( 36 ). However, not all proteins associated with each pathway are identified a s being important. Since we used a multi-class model, the feature importance profile will highlight the most differentiating proteins, and thus will not be an exact match to proteomic profiles identified in biological studies. Additionally, proteins can be part of multiple pathways, hence this impacts on the ‘neighbours’ used to calculate the latent feature embedding in ‘LAVA’. While the UMAP visualisation of model proximities has shown some heterogeneity and subclasses within some cancer types (e.g. breast and stomach cancer), analysis of the CLIFI values allows to identity the features responsible for this (bimodal distributions of ER α and others). However, despite MYH11 being the most important protein for prostate cancer prediction and the CLIFI values indicating a bimodal distribution, the UMAP shows only one cluster. While differential ER α expression has been reported in prostate cancer patients ( 37 ) however, to the best of our knowledge, differential MYH11 expression has not thereby warranting further investigation. The CLIFI distribution will allow further a posteriori analysis, such as tests for normality, deviation from 0, and others using conventional, univariate statistical tests. When comparing CLIFI values between models as done here, they should be normalised. Whereas for the more common case where only a single model is calculated, the aggregated CLIFI values can be used on their own. We have also extended the ‘LAVA’ framework to boosting methods, but observed that LAVASET outperformed LAVABOOST, therefore. However, for these data, RF and LAVASET appear to be the most predictive models, hence their CLIFI values are potentially of more interest. However, CLIFI can be implemented in other methods such as XGBoost ( 38 ) which may have higher performance. For our data, RF and LAVASET had the highest performance with F1-scores of 92.29% and 91.83%, respectively. However, optimising the additional parameters of XGBoost (sample random state) had an F1 of 92.53%. Since the LAVABOOST outperformed GBDT, it is plausible that the performance of XGBoost may increase further with the addition of the ‘LAVA’ step and the integration of CLIFI will improve its interpretability. In summary, we have contributed an integrated, directional feature importance metric for decision tree-based models to facilitate feature importance assessment for multi-class classification. This metric can be used together with incorporating topological information into the decision functions of tree-based algorithms. The incorporation of topological information has been extended here to protein interactions but can be used with any type of network information that links together individual features to add inductive bias into the model for improved interpretability. Finally, several findings from analysing the individual directional feature importances match with existing literature, while other observations have not, generating potentially new testable hypotheses for cancer heterogeneity and subsequent clinical diagnostic studies. AUTHOR CONTRIBUTIONS Conceptualisation, M.K. and J.M.P.; Data curation, E.R.L. and S.M.Y.; Formal analysis, E.R.L. and S.M.Y.; Funding acquisition, J.M.P.; Investigation, E.R.L. and S.M.Y.; Methodology, M.K. and J.M.P.; Project administration, M.K. and J.M.P.; Resources, J.M.P.; Software, E.R.L. and M.K.; Supervision, M.K. and J.M.P.; Validation, E.R.L. and J.M.P.; Visualisation, E.R.L. and J.M.P.; Writing – original draft, E.R.L.; Writing – review & editing, M.K. and J.M.P. All authors have read and agreed to the published version of the manuscript. FUNDING M.K. is supported by the Horizon Europe Research and Innovation Programme funded DOMINO project (grant number 101060218). J.M.P. is supported by the Horizon Europe project CoDiet and Medical Research Council (MRC) funded GI-tools project (MR/V012452/1). The CoDiet project is funded by the European Union under Horizon Europe grant number 101084642. CoDiet research activities taking place at Imperial College London and the University of Nottingham are supported by UK Research and Innovation (UKRI) under the UK government’s Horizon Europe funding guarantee (grant number 101084642). The authors declare no conflict of interest. The funders had no role in the design of the study; in the collection, analyses, or interpretation of data; in the writing of the manuscript, or in the decision to publish the results. Appendix Class-specific Classification Performance View this table: View inline View popup Download powerpoint Table 3. Classification report of RF and LAVASET when classifying 28 cancer types for a single initialisation. Values in bold indicate the highest performance (expressed as percentage) for each metric per cancer type across the 4 methods (see Table 4 for results for GBDT and LAVABOOST). View this table: View inline View popup Download powerpoint Table 4. Classification report of GBDT and LAVABOOST when classifying 28 cancer types for a single initialisation. Values in bold indicate the highest performance (expressed as percentage) for each metric per cancer type across the 4 methods (see Table 3 for results for RF and LAVASET). Footnotes https://github.com/EloisaRL/TCGA-proteomics-pipeline https://github.com/melkasapi/LAVASET Bibliography 1. ↵ F. Bray , A. Jemal , N. Grey , J. Ferlay , and D. Forman . Global cancer transitions according to the human development index (2008-2030): a population-based study . Lancet Oncol , 13 ( 8 ): 790 – 801 , 2012 . doi: 10.1016/S1470-2045(12)70211-5 . OpenUrl CrossRef PubMed Web of Science 2. H. Sung , J. Ferlay , R. L. Siegel , M. Laversanne , I. Soerjomataram , A. Jemal , and F. Bray . Global cancer statistics 2020: Globocan estimates of incidence and mortality worldwide for 36 cancers in 185 countries . CA Cancer J Clin , 71 ( 3 ): 209 – 249 , 2021 . doi: 10.3322/caac.21660 . OpenUrl CrossRef PubMed 3. ↵ Global Burden of Disease Cancer, Collaboration , J. M. Kocarnik , K. Compton , F. E. Dean , W. Fu , B. L. Gaw , J. D. Harvey , H. J. Henrikson , D. Lu , A. Pennini , R. Xu , E. Ababneh , M. Abbasi-Kangevari , H. Abbastabar , S. M. Abd-Elsalam , A. Abdoli , A. Abedi , H. Abidi , H. Abolhassani , I. A. Adedeji , Q. E. S. Adnani , S. M. Advani , M. S. Afzal , M. Aghaali , B. O. Ahinkorah , S. Ahmad , T. Ahmad , A. Ahmadi , S. Ahmadi , T. Ahmed Rashid , Y. Ahmed Salih , G. T. Akalu , A. Aklilu , T. Akram , C. J. Akunna , H. Al Hamad , F. Alahdab , Z. Al-Aly , S. Ali , Y. Alimohamadi , V. Alipour , S. M. Aljunid , M. Alkhayyat , A. Almasi-Hashiani , N. A. Almasri , S. A. A. Al-Maweri , S. Almustanyir , N. Alonso , N. Alvis-Guzman , H. Amu , E. W. Anbesu , R. Ancuceanu , F. Ansari , A. Ansari-Moghaddam , M. H. Antwi , D. Anvari , A. E. Anyasodor , M. Aqeel , J. Arabloo , M. Arab-Zozani , O. Aremu , H. Ariffin , T. Aripov , M. Arshad , A. Artaman , J. Arulappan , Z. Asemi , M. Asghari Jafarabadi , T. Ashraf , P. Atorkey , A. Aujayeb , M. Ausloos , A. F. Awedew , B. P. Ayala Quintanilla , T. Ayenew , M. A. Azab , S. Azadnajafabad , A. Azari Jafari , G. Azarian , A. Y. Azzam , A. D. Badiye , S. Bahadory , A. A. Baig , J. L. Baker , S. Balakrishnan , M. Banach , T. W. Barnighausen , F. Barone-Adesi , F. Barra , A. Barrow , M. Behzadifar , U. I. Belgaumi , W. M. M. Bezabhe , Y. M. Bezabih , D. S. Bhagat , A. S. Bhagavathula , N. Bhardwaj , P. Bhardwaj , S. Bhaskar , K. Bhattacharyya , et al. Cancer incidence, mortality, years of life lost, years lived with disability, and disability-adjusted life years for 29 cancer groups from 2010 to 2019: A systematic analysis for the global burden of disease study 2019 . JAMA Oncol , 8 ( 3 ): 420 – 444 , 2022 . doi: 10.1001/jamaoncol.2021.6987 . OpenUrl CrossRef 4. ↵ K. Kourou , T. P. Exarchos , K. P. Exarchos , M. V. Karamouzis , and D. I. Fotiadis . Machine learning applications in cancer prognosis and prediction . Comput Struct Biotechnol J , 13 : 8 – 17 , 2015 . doi: 10.1016/j.csbj.2014.11.005 . OpenUrl CrossRef PubMed 5. ↵ S. M. McKinney , M. Sieniek , V. Godbole , J. Godwin , N. Antropova , H. Ashrafian , T. Back , M. Chesus , G. S. Corrado , A. Darzi , M. Etemadi , F. Garcia-Vicente , F. J. Gilbert , M. Halling-Brown , D. Hassabis , S. Jansen , A. Karthikesalingam , C. J. Kelly , D. King , J. R. Ledsam , D. Melnick , H. Mostofi , L. Peng , J. J. Reicher , B. Romera-Paredes , R. Sidebottom , M. Suleyman , D. Tse , K. C. Young , J. De Fauw , and S. Shetty . International evaluation of an ai system for breast cancer screening . Nature , 577 ( 7788 ): 89 – 94 , 2020 . doi: 10.1038/s41586-019-1799-6 . OpenUrl CrossRef PubMed 6. ↵ Ewelina Bębas , Marta Borowska , Marcin Derlatka , Edward Oczeretko , Marcin Hladuński , Piotr Szumowski , and Malgorzata Mojsak . Machine-learning-based classification of the histological subtype of non-small-cell lung cancer using mri texture analysis . Biomedical Signal Processing and Control , 66 : 102446 , 2021 . doi: 10.1016/j.bspc.2021.102446 . OpenUrl CrossRef 7. ↵ J. Yan , Z. Liu , S. Du , J. Li , L. Ma , and L. Li . Diagnosis and treatment of breast cancer in the precision medicine era . Methods Mol Biol , 2204 : 53 – 61 , 2020 . doi: 10.1007/978-1-0716-0904-0 {\_}5. OpenUrl CrossRef 8. ↵ M. Chen , S. J. Copley , P. Viola , H. Lu , and E. O. Aboagye . Radiomics and artificial intelligence for precision medicine in lung cancer treatment . Semin Cancer Biol , 93 : 97 – 113 , 2023 . doi: 10.1016/j.semcancer.2023.05.004 . OpenUrl CrossRef 9. ↵ Z. Yan , J. Li , Y. Xiong , W. Xu , and G. Zheng . Identification of candidate colon cancer biomarkers by applying a random forest approach on microarray data . Oncol Rep , 28 ( 3 ): 1036 – 42 , 2012 . doi: 10.3892/or.2012.1891 . OpenUrl CrossRef PubMed 10. ↵ R. Toth , H. Schiffmann , C. Hube-Magg , F. Buscheck , D. Hoflmayer , S. Weidemann , P. Lebok , C. Fraune , S. Minner , T. Schlomm , G. Sauter , C. Plass , Y. Assenov , R. Simon , J. Meiners , and C. Gerhauser . Random forest-based modelling to detect biomarkers for prostate cancer progression . Clin Epigenetics , 11 ( 1 ): 148 , 2019 . doi: 10.1186/s13148-019-0736-8 . OpenUrl CrossRef PubMed 11. ↵ E. M. Gashimova , A. Z. Temerdashev , D. V. Perunov , V. A. Porkhanov , I. S. Polyakov , and E. V. Dmitrieva . Selectivity of exhaled breath biomarkers of lung cancer in relation to cancer of other localizations . Int J Mol Sci , 24 ( 17 ), 2023 . doi: 10.3390/ijms241713350 . OpenUrl CrossRef 12. ↵ B. Dou , Z. Zhu , E. Merkurjev , L. Ke , L. Chen , J. Jiang , Y. Zhu , J. Liu , B. Zhang , and G. W. Wei . Machine learning methods for small data challenges in molecular science . Chem Rev , 123 ( 13 ): 8736 – 8780 , 2023 . doi: 10.1021/acs.chemrev.3c00189 . OpenUrl CrossRef 13. ↵ Leo Breiman . Random forests . Machine Learning , 45 ( 1 ): 5 – 32 , 2001 . ISSN 0885-6125 . doi: 10.1023/A:1010933404324 . OpenUrl CrossRef 14. ↵ Jerome H Friedman . Greedy function approximation: a gradient boosting machine . Annals of Statistics , pages 1189 – 1232 , 2001 . doi: 10.1214/aos/1013203451 . OpenUrl CrossRef PubMed 15. ↵ C. Hou , X. Zhong , P. He , B. Xu , S. Diao , F. Yi , H. Zheng , and J. Li . Predicting breast cancer in chinese women using machine learning techniques: Algorithm development . JMIR Med Inform , 8 ( 6 ): e17364 , 2020 . doi: 10.2196/17364 . OpenUrl CrossRef 16. Z. Huang , C. Hu , C. Chi , Z. Jiang , Y. Tong , and C. Zhao . An artificial intelligence model for predicting 1-year survival of bone metastases in non-small-cell lung cancer patients based on xgboost algorithm . Biomed Res Int , 2020 : 3462363 , 2020 . doi: 10.1155/2020/3462363 . OpenUrl CrossRef 17. ↵ H. Chen , N. Wang , X. Du , K. Mei , Y. Zhou , and G. Cai . Classification prediction of breast cancer based on machine learning . Comput Intell Neurosci , 2023 : 6530719 , 2023 . doi: 10.1155/2023/6530719 . OpenUrl CrossRef 18. ↵ Baptiste Gregorutti , Bertrand Michel , and Philippe Saint-Pierre . Correlation and variable importance in random forests . Statistics and Computing , 27 : 659 – 678 , 2017 . doi: 10.1007/s11222-016-9646-1 . OpenUrl CrossRef 19. ↵ Miron B Kursa and Witold R Rudnicki . Feature selection with the boruta package . Journal of Statistical Software , 36 : 1 – 13 , 2010 . doi: 10.18637/jss.v036.i11 . OpenUrl CrossRef PubMed 20. ↵ F. Hamidi , N. Gilani , R. Arabi Belaghi , H. Yaghoobi , E. Babaei , P. Sarbakhsh , and J. Malakouti . Identifying potential circulating mirna biomarkers for the diagnosis and prediction of ovarian cancer using machine-learning approach: application of boruta . Front Digit Health , 5 : 1187578 , 2023 . doi: 10.3389/fdgth.2023.1187578 . OpenUrl CrossRef 21. ↵ J. W. Chen and J. Dhahbi . Lung adenocarcinoma and lung squamous cell carcinoma cancer classification, biomarker identification, and gene expression analysis using overlapping feature selection methods . Sci Rep , 11 ( 1 ): 13323 , 2021 . doi: 10.1038/s41598-021-92725-8 . OpenUrl CrossRef 22. ↵ T. Kolisnik , A. K. Sulit , S. Schmeier , F. Frizelle , R. Purcell , A. Smith , and O. Silander . Identifying important microbial and genomic biomarkers for differentiating rightversus left-sided colorectal cancer using random forest models . BMC Cancer , 23 ( 1 ): 647 , 2023 . doi: 10.1186/s12885-023-10848-9 . OpenUrl CrossRef 23. ↵ R. Ortiz-Ramon , A. Larroza , S. Ruiz-Espana , E. Arana , and D. Moratal . Classifying brain metastases by their primary site of origin using a radiomics approach based on texture analysis: a feasibility study . Eur Radiol , 28 ( 11 ): 4514 – 4523 , 2018 . doi: 10.1007/s00330-018-5463-6 . OpenUrl CrossRef PubMed 24. ↵ Scott M. Lundberg and Su-In Lee . A unified approach to interpreting model predictions . In Proceedings of the 31st International Conference on Neural Information Processing Systems, NIPS’17 , page 4768 – 4777 , Red Hook, NY , USA , 2017 . Curran Associates Inc . ISBN 9781510860964 . doi: 10.5555/3295222.3295230 . OpenUrl CrossRef 25. ↵ M. Kasapi , K. Xu , T. M. D. Ebbels , D. P. O’Regan , J. S. Ware , and J. M. Posma . LAVASET: Latent Variable Stochastic Ensemble of Trees. an ensemble method for correlated datasets with spatial, spectral, and temporal dependencies . Bioinformatics , 40 ( 3 ), 2024 . doi: 10.1093/bioinformatics/btae101 . OpenUrl CrossRef 26. ↵ J. Li , Y. Lu , R. Akbani , Z. Ju , P. L. Roebuck , W. Liu , J. Y. Yang , B. M. Broom , R. G. Verhaak , D. W. Kane , C. Wakefield , J. N. Weinstein , G. B. Mills , and H. Liang . Tcpa: a resource for cancer functional proteomics data . Nat Methods , 10 ( 11 ): 1046 – 7 , 2013 . doi: 10.1038/nmeth.2650 . OpenUrl CrossRef PubMed 27. ↵ Rehan Akbani , Patrick Kwok Shing Ng , Henrica MJ Werner , Maria Shahmoradgoli , Fan Zhang , Zhenlin Ju , Wenbin Liu , Ji-Yeon Yang , Kosuke Yoshihara , and Jun Li . A pan-cancer proteomic perspective on the cancer genome atlas . Nature communications , 5 ( 1 ): 3887 , 2014 . doi: 10.1038/ncomms4887 . OpenUrl CrossRef PubMed 28. ↵ Leo Breiman , Jerome H. Friedman , Richard A. Olshen , and Charles J. Stone . Classification And Regression Trees . Routledge , October 1984 . ISBN 9781315139470 . doi: 10.1201/9781315139470 . OpenUrl CrossRef 29. ↵ D. Szklarczyk , R. Kirsch , M. Koutrouli , K. Nastou , F. Mehryary , R. Hachilif , A. L. Gable , T. Fang , N. T. Doncheva , S. Pyysalo , P. Bork , L. J. Jensen , and C. von Mering . The string database in 2023: protein-protein association networks and functional enrichment analyses for any sequenced genome of interest . Nucleic Acids Res , 51 ( D1 ): D638 – D646 , 2023 . doi: 10.1093/nar/gkac1000 . OpenUrl CrossRef PubMed 30. ↵ Leland McInnes , John Healy , Nathaniel Saul , and Lukas Großberger . Umap: Uniform manifold approximation and projection . Journal of Open Source Software , 3 ( 29 ): 861 , September 2018 . ISSN 2475-9066 . doi: 10.21105/joss.00861 . OpenUrl CrossRef 31. ↵ K. Milde-Langosch , A. M. Bamberger , C. Goemann , E. Rossing , G. Rieck , B. Kelp , and T. Loning . Expression of cell-cycle regulatory proteins in endometrial carcinomas: correlations with hormone receptor status and clinicopathologic parameters . J Cancer Res Clin Oncol , 127 ( 9 ): 537 – 44 , 2001 . doi: 10.1007/s004320100256 . OpenUrl CrossRef PubMed Web of Science 32. A. C. Rodriguez , Z. Blanchard , K. A. Maurer , and J. Gertz . Estrogen signaling in endometrial cancer: a key oncogenic pathway with several open questions . Horm Cancer , 10 ( 2-3 ): 51 – 63 , 2019 . doi: 10.1007/s12672-019-0358-9 . OpenUrl CrossRef PubMed 33. ↵ M. F. B. Jamaluddin , Y. A. Ko , A. Ghosh , S. M. Syed , Y. Ius , R. O’Sullivan , J. K. Netherton , M. A. Baker , P. Nahar , K. Jaaback , and P. S. Tanwar . Proteomic and functional characterization of intra-tumor heterogeneity in human endometrial cancer . Cell Rep Med , 3 ( 9 ): 100738 , 2022 . doi: 10.1016/j.xcrm.2022.100738 . OpenUrl CrossRef 34. ↵ M. A. Khalifa , R. S. Mannel , S. D. Haraway , J. Walker , and K. W. Min . Expression of egfr, her-2/neu, p53, and pcna in endometrioid, serous papillary, and clear cell endometrial adenocarcinomas . Gynecol Oncol , 53 ( 1 ): 84 – 92 , 1994 . doi: 10.1006/gyno.1994.1092 . OpenUrl CrossRef PubMed Web of Science 35. ↵ A. Liu , D. Zhang , X. Yang , and Y. Song . Estrogen receptor alpha activates mapk signaling pathway to promote the development of endometrial cancer . J Cell Biochem , 120 ( 10 ): 17593 – 17601 , 2019 . doi: 10.1002/jcb.29027 . OpenUrl CrossRef 36. ↵ X. Ma , Y. Zhao , Y. Li , H. Lu , and Y. He . Relevance of bcl-x expression in different types of endometrial tissues . J Exp Clin Cancer Res , 29 ( 1 ): 14 , 2010 . doi: 10.1186/1756-9966-29-14 . OpenUrl CrossRef PubMed 37. ↵ S. Mishra , Q. Tai , X. Gu , J. Schmitz , A. Poullard , R. J. Fajardo , D. Mahalingam , X. Chen , X. Zhu , and L. Z. Sun . Estrogen and estrogen receptor alpha promotes malignancy and osteoblastic tumorigenesis in prostate cancer . Oncotarget , 6 ( 42 ): 44388 – 402 , 2015 . doi: 10.18632/oncotarget.6317 . OpenUrl CrossRef PubMed 38. ↵ Tianqi Chen and Carlos Guestrin . Xgboost: A scalable tree boosting system . In Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, KDD ‘16 , page 785 – 794 , New York, NY, USA , 2016 . Association for Computing Machinery . ISBN 9781450342322 . doi: 10.1145/2939672.2939785 . OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted August 04, 2024. Download PDF Data/Code Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Topological embedding and directional feature importance in ensemble classifiers for multi-class classification Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Topological embedding and directional feature importance in ensemble classifiers for multi-class classification Eloisa Rocha Liedl , Shabeer Mohamed Yassin , Melpomeni Kasapi , Joram M. Posma bioRxiv 2024.08.01.605982; doi: https://doi.org/10.1101/2024.08.01.605982 Share This Article: Copy Citation Tools Topological embedding and directional feature importance in ensemble classifiers for multi-class classification Eloisa Rocha Liedl , Shabeer Mohamed Yassin , Melpomeni Kasapi , Joram M. Posma bioRxiv 2024.08.01.605982; doi: https://doi.org/10.1101/2024.08.01.605982 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Bioinformatics Subject Areas All Articles Animal Behavior and Cognition (7644) Biochemistry (17725) Bioengineering (13915) Bioinformatics (42031) Biophysics (21486) Cancer Biology (18635) Cell Biology (25548) Clinical Trials (138) Developmental Biology (13397) Ecology (19937) Epidemiology (2067) Evolutionary Biology (24361) Genetics (15619) Genomics (22538) Immunology (17763) Microbiology (40468) Molecular Biology (17206) Neuroscience (88733) Paleontology (667) Pathology (2842) Pharmacology and Toxicology (4834) Physiology (7657) Plant Biology (15172) Scientific Communication and Education (2046) Synthetic Biology (4304) Systems Biology (9831) Zoology (2272)