Benchmarking of automated cancer cell annotation methods for scRNA-seq data reveals Consensus annotation as the preferred method

Benchmarking of automated cancer cell annotation methods for scRNA-seq data reveals Consensus annotation as the preferred method | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Benchmarking of automated cancer cell annotation methods for scRNA-seq data reveals Consensus annotation as the preferred method View ORCID Profile Tamim Abdelaal , View ORCID Profile Christiaan Klijn , Matt Hancock doi: https://doi.org/10.1101/2025.01.13.632750 Tamim Abdelaal 1 Discovery Data Science, Genmab B.V. , Utrecht, The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Tamim Abdelaal For correspondence: taab{at}genmab.com Christiaan Klijn 1 Discovery Data Science, Genmab B.V. , Utrecht, The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Christiaan Klijn Matt Hancock 1 Discovery Data Science, Genmab B.V. , Utrecht, The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Preview PDF Abstract Targeted cancer therapies have shown therapeutic advantages due to tumor-specific drug activity. Single-cell RNA-sequencing has been widely used in cancer studies to define different cellular identities. However, accurate identification of tumor vs other normal cells is essential to define novel tumor-specific targets. Recent methods have been developed to perform the task of tumor cell annotation, which can be divided into two categories: CNV-based methods, which use transcriptome measurements to infer copy number variations and identify cells with alterations as tumor, and Reference-based methods which train a classifier using previously annotated tumor data and use it to annotate new datasets. We benchmarked the state-of-the-art method of each category, SCEVAN and scATOMIC, respectively, together with Consensus annotation method where a cell is considered tumor if both methods agree on that. Across 20 cancer datasets spanning 9 cancer types with a total of 379 samples, the Consensus annotation outperformed other methods in terms of precision score. SCEVAN works well when clear CNVs are detected, otherwise cells are randomly split between normal and tumor producing many false positives. While scATOMIC efficiently detects all normal cells except normal epithelial cells, where all epithelial cells (normal and malignant) are considered cancerous. The Consensus annotation method overcomes the limitations of both methods, being able to detect normal epithelial cells together with other normal cell types as non-tumor, also annotating malignant epithelial cells as tumor even with weak CNVs. This produces overall higher precision with the least number of false positives, leading to confident tumor-specific potential therapeutic targets. Implementation is available in the MACE R package through GitLab ( https://gitlab.com/genmab-public/mace/ ). 1 Introduction Understanding the complex interactions within tumor microenvironments (TMEs) is crucial for elucidating the mechanisms of cancer proliferation, progression, and therapeutic resistance [ 1 - 3 ]. Single-cell RNA sequencing (scRNA-seq) has emerged as a powerful tool to dissect these complicated systems by providing comprehensive transcriptomic profiles of individual cells within a tumor, facilitating the detailed study of cellular heterogeneity and the dynamic interplay between malignant cells and their surrounding immune and stromal components [ 4 - 6 ]. Accurate cell type annotation in scRNA-seq data is pivotal for deriving meaningful biological insights, which can be performed manually or automated [ 7 ]. Traditional manual annotation methods, which rely heavily on pre-existing knowledge and canonical gene markers, have significant limitations, including subjectivity and incomplete markers knowledge. On the other hand, automated classifier-based methods have been developed, offering more consistent and scalable annotations [ 8 , 9 ]. In the context of cancer, it is important to dissect malignant tumor cells from the surrounding normal immune and stromal cells [ 10 ]. While designing targeted cancer therapies, accurate identification of malignant tumor cells is essential to enable tumor-specific drug activity, avoiding damaging or depletion of other normal cells in the TME. Malignancy refers to the presence of cancerous cells that exhibit uncontrolled growth, invasion into surrounding tissues, and the potential to metastasize to distant organs. Defining malignant cells in single-cell genomics poses significant challenges due to the heterogeneity within tumors and the dynamic nature of cancer progression. Malignant cells often exhibit genomic instability, characterized by mutations, copy number variations (CNVs), and other genetic alterations that differentiate them from normal cells [ 11 ]. However, the tumor microenvironment (TME) is composed of a complex mixture of malignant cells, immune cells, stromal cells, and other non-cancerous components, making it difficult to distinguish malignant cells with high precision [ 12 ]. Additionally, the variability in gene expression profiles among malignant cells, driven by factors such as clonal evolution and environmental influences, further complicates their identification [ 13 ]. Several computational methods have been developed to identify tumor cells using scRNA-seq data, which can be divided into two broad categories. First, methods relying on training a classifier using previously annotated cancer reference datasets, hence Reference-based methods. Next, using the trained classifier model to annotate new datasets. Reference-based methods include Seurat [ 14 ], scmap [ 15 ], SingleR [ 6 ], CHETAH [ 16 ], scType [ 17 ] and SingleCellNet [ 18 ] among others. Although these methods generally perform the task of cell identification well, those are not designed to differentiate the complex TME of cancer studies. Therefore, a recent method scATOMIC [ 19 ] was designed specifically to identify cells from cancer scRNA-seq datasets, showing overall superior performance especially for the cancer cells compartment. Thus, scATOMIC represents the current state-of-the-art Reference-based method. The second category of methods identifying tumor cells are relying on inference of copy number variation (CNV) from scRNA-seq gene expression data, hence CNV-based methods. Various methods including inferCNV [ 5 ], CopyKAT [ 20 ] and SCEVAN [ 21 ], all depending on the assumption of defining tumor cells having genomic instabilities compared to a normal diploid genome for normal cells. Recently, SCEVAN showed superior performance to other methods, representing the current state-of-the-art CNV-based method. Also, SCEVAN can independently detect a set of confident normal cells to be used to define the normal diploid genomic profile, i.e. it does not require a set of normal cells as an extra input. Currently, it is not clear which category of methods is more effective when identifying malignant tumor cells in scRNA-seq data analysis. Further, an independent evaluation is required to compare methods across various cancer types, guiding users to which method fits best their data, and guiding further developments in the field. In this study, we benchmarked the state-of-the-art method from each category: Reference-based (scATOMIC) and CNV-based (SCEVAN), for the task of precise identification of tumor vs normal cells using scRNA-seq data. The evaluation was performed using 20 cancer scRNA-seq studies covering 9 different cancer types spanning a total of 379 patient samples. As results show, the performance of the two methods varied across cancer types, hence, we designed a Consensus annotation method which carried the benefits and overcomes the limitations of both Reference-based and CNV-based methods. The Consensus annotation method showed superior tumor cell annotation performance producing overall higher precision across all datasets. 2 Methods 2.1 Tumor cell identification methods We evaluated the state-of-the-art methods scATOMIC [ 19 ], denoted as the Reference-based method, and SCEVAN [ 21 ] denoted as the CNV-based method, for Ma lignant Ce ll (MACE) annotation. Where the MACE annotation is defined as the task of annotating tumor vs normal cells using scRNA-seq dataset. Both methods are publicly available as R packages, and were applied using their default settings, unless otherwise stated. During the benchmarking, we provided the input scRNA-seq datasets as raw count data. The annotation was performed one patient sample at a time to avoid differences across individuals. Finally, the predicted annotation by each method is evaluated against the ground-truth cell type annotation provided by the authors of each dataset/study ( Fig. 1A ). Download figure Open in new tab Fig. 1 (A) Benchmarking pipeline. Cancer scRNA-seq dataset is provided as input raw count data to each Ma lignant Ce ll (MACE) classification method per patient samples. Each method predicts tumor vs normal cell annotation which is further evaluated against the ground-truth cell type annotation of each dataset. (B) Consensus annotation pipeline . Combination of the Reference-based and the CNV-based methods, the Reference-based method generates the initial cell annotation, while the CNV-based refine the initial tumor annotation further into tumor and normal epithelial cells. The Reference-based method, scATOMIC, provides a hierarchy of cell type annotations. For the task of predicting tumor vs normal cells, we stop at a high level in the hierarchy at layer 2. Cells classified as ‘Non Stromal Cell’ are considered ‘tumor’ cells, other cell type annotations are considered ‘normal’ cells, and any unclassified cell annotation is left as ‘unclassified’. The CNV-based method, SCEVAN, directly provides a class annotation for each cell either being ‘tumor’ or ‘normal’. Both methods filter out low-quality cells during their preprocessing, these left out cells are annotated as ‘filtered’. These filtered cells were also left out during the evaluation against the ground-truth annotation. We added a third method combining the Reference-based and the CNV-based methods, denoted as the Consensus annotation method ( Fig. 1B ). As suggested by the method name, the Consensus annotation identify a tumor cell where both Reference-based and CNV-based methods agree on that ‘tumor’ annotation. The detailed steps of the Consensus annotation are: 1) Performing the Reference-based annotation on all cells within the input sample, to obtain an initial tumor vs normal cell annotation, where the tumor annotation represents the epithelial cells. 2) Performing the CNV-based annotation only on the tumor cells identified by the Reference-based method to further split them into normal epithelial cells, having normal diploid genomic profiles, and tumor cells with abnormal genomic CNV profiles. Here, we provide the CNV-based method with a sample of 50 cells from the ‘normal’ annotation by the Reference-based method, representing the reference normal diploid genomic profile. This is mainly to speed-up the computation time and skip the internal step of the CNV-based method to identify these normal cells by itself. 3) The final annotations by the Consensus method are: ‘normal’ cells identified by the Reference-based method, ‘normal Epithelial’ cells identified as tumor cells by the Reference-based method and redefined as normal cells by the CNV-based method, and ‘tumor’ cells identified as tumor cells by both methods. Filtered out low-quality cells are annotated as ‘filtered’. 2.2 Datasets We used a total of 20 cancer scRNA-seq datasets to evaluate all methods. Datasets are covering 9 different cancer types having a combined number of 379 cancer samples (summarized in Table 1 ). All datasets have been obtained from the original publications, including the raw count gene expression data, along with the cell type annotation provided by the authors of each study. Only cancer samples were retained from each study, blood or adjacent normal samples were excluded. View this table: View inline View popup Download powerpoint Table 1 Overview of the datasets used during this study 2.3 Data preprocessing The raw count gene expression data was used as input to all methods. For the Reference-based method, scATOMIC, cells expressing less than 200 genes or having more than 25% mitochondrial genes were filtered out. For the CNV-based method, SCEVAN, cells expressing less than 200 genes were filtered out, as well as genes expressed in less than 10% of the cells. 2.4 Evaluation We considered the tumor vs normal annotation task as a binary classification, with ‘tumor’ being our positive label. Therefore, we evaluated the prediction performance of each method using the Precision score: Where TP is the true positives representing actual tumor cells correctly annotated as tumor cells, and FP is false positives representing normal cells wrongly annotated as tumor cells. Since the annotation was performed per patient sample, the precision score was calculated per patient sample as well. 2.5 Implementation details For the Reference-based method, scATOMIC, we applied the run_scATOMIC and create_summary_matrix functions, while setting the input parameters fine_grained_T, use_CNVs , and modify_results to False . We obtained the cell annotations from the ‘layer_2’ output. For the CNV-based method, SCEVAN, we applied the pipelineCNA function, while setting the input parameters SUBCLONES and plotTree to False . We obtained the cell annotations from the ‘class’ output. We developed wrapper functions, compatible with Seurat objects, to perform the Reference-based, CNV-based and Consensus tumor vs normal annotation. The implementation is available as the MACE R package through GitLab ( https://gitlab.com/genmab-public/mace/ ). 3 Results 3.1 Annotation performance varies per cancer type We started by benchmarking the Reference-based and the CNV-based methods for the task of annotating tumor vs normal cells, using all 20 scRNA-seq datasets covering 9 different cancer types. We compared the predicted cell annotations provided by each method against the cell type annotation provided by the authors of each study and calculated the precision score per patient sample. From the authors annotation, we considered the malignant/cancer cells population as the ground-truth ‘tumor’ and all other cell populations as ‘normal’. We based the benchmarking results on the precision score, although other classification metrics could be used. However, in the context of targeted therapy on cancer cells, it is more important to define a purified set of tumor cells, instead of annotating more tumor cells with contamination from other normal cells. Therefore, we aimed to find the method producing the best precision score to control and minimize the false positive predictions. Based on the precision score, we observed that results vary across cancer types. The Reference-based method produced better average performance for Head and Neck, Breast, Colorectal, Lung and Kidney cancers. Where more samples had higher precision score for the Reference-based method compared to the CNV-based method ( Fig. 2 ). However, for Pancreas and Liver cancers, the CNV-based method produced higher performance. Finally, both methods had comparable performance for Skin and Bladder cancers. Moreover, the precision score produced in most samples is relatively high, however, large performance variability is observed for the CNV-based method for Head and Neck, Breast, Lung and Kidney cancers. Similarly, for the Reference-based method when applied on Pancreatic cancer samples. Overall, this shows that on average both methods can produce good annotation performance depending on the cancer type. Download figure Open in new tab Fig. 2 Benchmarking results for the Reference-based vs the CNV-based methods. Scatterplots showing the precision score of the Reference-based and the CNV-based methods across 9 cancer types. Each point represents a patient sample, where more points on the upper triangle shows higher Reference-based performance, while more points in the lower triangle shows higher CNV-based performance. 3.2 Consensus annotation combines the power of other methods We investigated the limitations of the Reference-based and the CNV-based methods, by examining when and why cells are wrongly annotated. Taking the Peng_et_al_PDAC dataset as an example, where the Reference-based method performed poorly. According to the authors cell type annotation, two ductal cell populations were identified, where the Ductal cell type 2 is considered the malignant population based on copy number variation profile ( Fig. 3A ). Comparing with the Reference-based tumor annotation, all immune and stromal cell types were correctly annotated as ‘normal’ cells. However, together with the Ductal cell type 2, the Reference-based method included the Acinar cells and Ductal cell type 1 as ‘tumor’ cells, which are considered mis-annotations ( Fig. 3A ). The latter two populations were correctly separated from the ‘tumor’ cells by the CNV-based method, which more precisely annotated only the Ductal cell type 2 as ‘tumor’ ( Fig. 3A ). Download figure Open in new tab Fig. 3 (A-B) UMAP visualization [ 39 ] of (A) the Peng_et_al_PDAC and (B) the Qian_et_al_Breast datasets colored according to the authors cell type annotation, in comparison with the MACE annotation produced by the Reference-based, the CNV-based and the Consensus methods. (C) Boxplot comparing the performance of all methods, based on the precision score, showing that the Consensus method produced the most precise annotation across 9 different cancer types. On the other hand, investigating the Qian_et_al_Breast dataset as an example of poor performance for the CNV-based method. The authors annotated the tumor cells as the Cancer cell population ( Fig. 3B ). The CNV-based method almost randomly splits all cells between tumor and normal cells, including many wrong ‘tumor’ annotations from immune and stromal cell types ( Fig. 3B ). However, the Reference-based method performed well by segregating the Cancer cell population as ‘tumor’ from all other ‘normal’ cells ( Fig. 3B ). Building on this information, we combined both methods into Consensus annotation to retain the power and overcome the limitation of each method. The motivation is to have a precise tumor vs normal annotation method, which: 1) can accurately separate potential epithelial cancer cells from all other normal cells in the TME, and 2) can further separate the epithelial cells compartment into malignant/tumor and normal cells based on CNV profiles, thus retaining the normal epithelial cells as non-malignant. The Consensus method uses the Reference-based method to achieve the first task of identifying epithelial vs other normal cells. Followed by the CNV-based method applied only on those epithelial cells (see Methods), together with examples of normal cells for normal CNV diploidy profile, to further separate normal from malignant epithelial cells. For the Peng_et_al_PDAC dataset, the Consensus method was able to separate the Acinar cells and the Ductal cell type 1 as ‘normal’, benefiting from the CNV-based method ( Fig. 3A ). Further, for the Qian_et_al_Breast dataset, the Consensus method annotated tumor cells only from the Cancer cell population, benefiting from the Reference-based method. Also, it reverts all the wrong ‘tumor’ annotations made by the CNV-based method to ‘normal’ cells ( Fig. 3B ). Across all datasets, the Consensus annotation method outperforms other (separately applied) methods, producing the highest average precision score across all cancer types ( Fig. 3C ). This shows that the Consensus method combines the benefits of both Reference-based and CNV-based methods, offering a new solution to precisely annotate tumor cells using scRNA-seq data. 3.3 Consensus method defines tumor cells with valid CNV profiles Next to producing the best annotation performance, we aimed to validate that the final tumor epithelial cells identified by the Consensus method have valid CNVs, in comparison to their normal epithelial counterpart. Inspecting the CNV profiles of the Consensus method indeed showed clear CNV profile differences between tumor and normal epithelial cells. As observed by example patient samples from the Peng_et_al_PDAC and Qian_et_al_Breast datasets, tumor epithelial cells are experiencing various genomic deletions (loss) and duplications (gain) across multiple genomic locations ( Fig. 4 , left column). In contrast, the normal epithelial cells show a homogenous diploid CNV profile as expected. Download figure Open in new tab Fig. 4 CNV profiles of tumor vs normal cells. Heatmaps showing the CNV profiles produced by the Consensus method (left column) and the CNV-based method (right column) across samples from the Peng_et_al_PDAC dataset (top two rows) and the Qian_et_al_Breast dataset (bottom two rows). Each heatmap shows cells x genomic locations (rows x columns), colored according to the copy number variation (blue=loss, red=gain, white=normal diploid). The vertical sidebar to each heatmap indicates the tumor vs normal predicted annotation. More importantly, we compared the CNV profiles produced by the Consensus method with the CNV profiles produced by the CNV-based method for the same samples ( Fig. 4 , right column). Recall that the CNV-based method is applied across all cells, not only the epithelial cells compartment. For the Peng_et_al_PDAC samples, the tumor cells annotated by the CNV-based method had clear CNV aberrations compared to normal cells. These clear CNVs suggest the high performance of the CNV-based method on the Peng_et_al_PDAC dataset. The Consensus method was able to retain and emphasize these CNVs as well. However, the CNV-based method produced weak CNV profiles for the Qian_et_al_Breast samples, which do not support a clear separation between tumor and normal cells, explaining the poor annotation performance with many false positive tumor annotations from normal cell types (as shown in Fig. 3B ). On the contrary, while inferring the CNV profiles on the epithelial cells only, the Consensus method was able to detect clear CNV profiles for the same Qian_et_al_Breast samples, supporting tumor vs normal epithelial annotation and avoiding false tumor annotation from other normal cell types. Overall, this shows that the Consensus method is able to produce precise MACE annotations, with clear CNV profiles supporting the malignancy of the annotated tumor cells. 4. Discussion We present a benchmarking study for automated cancer annotation methods using scRNA-seq data. Across 20 cancer datasets covering 9 different cancer types, we compared the state-of-the-art method from the Reference-based category, scATOMIC, and the CNV-based category, SCEVAN. The benchmarking results showed differences in annotation performance depending on the cancer type. Further, we introduced a new Consensus annotation strategy, combining the two previous methods, showing overall superior annotation performance across all cancer types, when compared to applying each method separately. The Reference-based method can accurately separate epithelial cells from all other cell types. Where in solid tumors, epithelial cells are the main source of malignant cells. However, the Reference-based method cannot further separate normal epithelial cells from those who turned into a malignant state. This limitation is the main cause of the poor annotation performance across cancer types known for strong CNVs like pancreatic cancer. The CNV-based method can handle cancer types with strong and clear CNVs, producing precise tumor annotations. However, the assumption that all cancer cells undergo copy number aberration is not necessarily valid across all cancer types, where weak CNV profiles are obtained. The final step of the CNV-based method SCEVAN, similar to earlier CNV-based methods, is to cluster cells according to their obtained CNV profiles into two clusters using hierarchical clustering. The cluster with most initial/confident normal cells (providing the baseline normal diploid profile) is consider all normal, while the other cluster is considered tumor. With weak CNV profiles, this final clustering step, forcing the data into two clusters, randomly splits the cells between tumor and normal across all cells in the TME. Moreover, when applied on a completely normal tissue sample, the CNV-based will always produce some tumor annotations, as a result of this forced clustering. However, the Reference-based method will accurately report no tumor cell in the case of a normal tissue sample. The Consensus annotation method combines the power of each method and overcomes their individual limitations. The Reference-based method ensures accurate annotation of epithelial cells. Also, this ensures to report no tumor cells when applied to a completely normal tissue sample with no epithelial cells. Secondly, applying the CNV-based method only on the epithelial cells compartment avoids wrong tumor annotations across other normal cell types in the TME. Thirdly, it ensures to separate normal epithelial cells, with diploid CNV profiles, from tumor epithelial cells with copy number abnormalities. As previously mentioned, we focused on the precision score to evaluate the MACE annotation of all methods, to ensure low false positive annotations, i.e. minimal contamination of normal cells mis-annotated as tumor. We argue that a false positive annotation is more harmful than a false negative annotation, with tumor cells being mis-annotated as normal. However, optimizing for high precision may produce a trade-off with lower recall score, indicating higher false negative annotations. We relied on the authors cell type annotations as the ground-truth for evaluating the MACE annotations. However, it is important to mention that the authors annotations represent a limitation in the current benchmark. As the evaluation presented is dependent and affected by the methodologies used by the authors to generate the original cell type annotations. For example, datasets where authors annotated tumor cells based on CNV-based methods had on average better performance for the CNV-based method in this benchmark. The Consensus annotation showed precise MACE annotations using scRNA-seq data. Possible improvement can be achieved relying on spatial transcriptomics data, which is growing in the field of single-cell analysis and oncology. With spatial transcriptomics, next to the gene expression data, the spatial distribution of different cells can be used to further inform and improve the final MACE annotation. Altogether, we present a benchmarking of automated cancer cell annotation methods using scRNA-seq data. We suggest a new Consensus annotation method combining the power of the current state-of-the-art methods and producing precise tumor vs normal cell annotation in the context of solid tumors. References 1. ↵ Angelova , M. , et al. , Evolution of Metastases in Space and Time under Immune Selection . Cell , 2018 . 175 ( 3 ): p. 751 – 765 e16 . OpenUrl CrossRef PubMed 2. Maynard , A. , et al. , Therapy-Induced Evolution of Human Lung Cancer Revealed by Single-Cell RNA Sequencing . Cell , 2020 . 182 ( 5 ): p. 1232 – 1251 e22 . OpenUrl CrossRef PubMed 3. ↵ Salemme , V. , et al. , The Crosstalk Between Tumor Cells and the Immune Microenvironment in Breast Cancer: Implications for Immunotherapy . Front Oncol , 2021 . 11 : p. 610303 . OpenUrl CrossRef PubMed 4. ↵ Valdes-Mora , F. , et al. , Single-Cell Transcriptomics in Cancer Immunobiology: The Future of Precision Oncology . Front Immunol , 2018 . 9 : p. 2582 . OpenUrl CrossRef PubMed 5. ↵ Patel , A.P. , et al. , Single-cell RNA-seq highlights intratumoral heterogeneity in primary glioblastoma . Science , 2014 . 344 ( 6190 ): p. 1396 – 401 . OpenUrl Abstract / FREE Full Text 6. ↵ Aran , D. , et al. , Reference-based analysis of lung single-cell sequencing reveals a transitional profibrotic macrophage . Nat Immunol , 2019 . 20 ( 2 ): p. 163 – 172 . OpenUrl CrossRef PubMed 7. ↵ Clarke , Z.A. , et al. , Tutorial: guidelines for annotating single-cell transcriptomic maps using automated and manual methods . Nat Protoc , 2021 . 16 ( 6 ): p. 2749 – 2764 . OpenUrl CrossRef PubMed 8. ↵ Abdelaal , T. , et al. , A comparison of automatic cell identification methods for single-cell RNA sequencing data . Genome Biol , 2019 . 20 ( 1 ): p. 194 . OpenUrl CrossRef PubMed 9. ↵ Xie , B. , et al. , Automatic cell type identification methods for single-cell RNA sequencing . Comput Struct Biotechnol J , 2021 . 19 : p. 5874 – 5887 . OpenUrl CrossRef PubMed 10. ↵ Fan , J. , K. Slowikowski , and F. Zhang , Single-cell transcriptomics in cancer: computational challenges and opportunities . Exp Mol Med , 2020 . 52 ( 9 ): p. 1452 – 1465 . OpenUrl CrossRef PubMed 11. ↵ Hanahan , D. and R.A. Weinberg , Hallmarks of cancer: the next generation . Cell , 2011 . 144 ( 5 ): p. 646 – 74 . OpenUrl CrossRef PubMed Web of Science 12. ↵ Tirosh , I. , et al. , Dissecting the multicellular ecosystem of metastatic melanoma by single-cell RNA-seq . Science , 2016 . 352 ( 6282 ): p. 189 – 96 . OpenUrl Abstract / FREE Full Text 13. ↵ McGranahan , N. and C. Swanton , Clonal Heterogeneity and Tumor Evolution: Past, Present, and the Future . Cell , 2017 . 168 ( 4 ): p. 613 – 628 . OpenUrl CrossRef PubMed 14. ↵ Stuart , T. , et al. , Comprehensive Integration of Single-Cell Data . Cell , 2019 . 177 ( 7 ): p. 1888 – 1902 e21 . OpenUrl CrossRef PubMed 15. ↵ Kiselev , V.Y. , A. Yiu , and M. Hemberg , scmap: projection of single-cell RNA-seq data across data sets . Nat Methods , 2018 . 15 ( 5 ): p. 359 – 362 . OpenUrl CrossRef PubMed 16. ↵ de Kanter , J.K. , et al. , CHETAH: a selective, hierarchical cell type identification method for single-cell RNA sequencing . Nucleic Acids Res , 2019 . 47 ( 16 ): p. e95 . OpenUrl CrossRef PubMed 17. ↵ Ianevski , A. , A.K. Giri , and T. Aittokallio , Fully-automated and ultra-fast cell-type identification using specific marker combinations from single-cell transcriptomic data . Nat Commun , 2022 . 13 ( 1 ): p. 1246 . OpenUrl CrossRef PubMed 18. ↵ Tan , Y. and P. Cahan , SingleCellNet: A Computational Tool to Classify Single Cell RNA-Seq Data Across Platforms and Across Species . Cell Syst , 2019 . 9 ( 2 ): p. 207 – 213 e2 . OpenUrl CrossRef PubMed 19. ↵ Nofech-Mozes , I. , et al. , Pan-cancer classification of single cells in the tumour microenvironment . Nat Commun , 2023 . 14 ( 1 ): p. 1615 . OpenUrl CrossRef PubMed 20. ↵ Gao , R. , et al. , Delineating copy number and clonal substructure in human tumors from single-cell transcriptomes . Nat Biotechnol , 2021 . 39 ( 5 ): p. 599 – 608 . OpenUrl CrossRef PubMed 21. ↵ De Falco , A. , et al. , A variational algorithm to detect the clonal copy number substructure of tumors from scRNA-seq data . Nat Commun , 2023 . 14 ( 1 ): p. 1074 . OpenUrl CrossRef PubMed 22. Elyada , E. , et al. , Cross-Species Single-Cell Analysis of Pancreatic Ductal Adenocarcinoma Reveals Antigen-Presenting Cancer-Associated Fibroblasts . Cancer Discov , 2019 . 9 ( 8 ): p. 1102 – 1123 . OpenUrl Abstract / FREE Full Text 23. Hwang , W.L. , et al. , Single-nucleus and spatial transcriptome profiling of pancreatic cancer identifies multicellular dynamics associated with neoadjuvant treatment . Nat Genet , 2022 . 54 ( 8 ): p. 1178 – 1191 . OpenUrl CrossRef PubMed 24. Peng , J. , et al. , Single-cell RNA-seq highlights intra-tumoral heterogeneity and malignant progression in pancreatic ductal adenocarcinoma . Cell Res , 2019 . 29 ( 9 ): p. 725 – 738 . OpenUrl CrossRef PubMed 25. Steele , N.G. , et al. , Multimodal Mapping of the Tumor and Peripheral Blood Immune Landscape in Human Pancreatic Cancer . Nat Cancer , 2020 . 1 ( 11 ): p. 1097 – 1112 . OpenUrl CrossRef PubMed 26. Horny , K. , et al. , Mesenchymal-epithelial transition in lymph node metastases of oral squamous cell carcinoma is accompanied by ZEB1 expression . J Transl Med , 2023 . 21 ( 1 ): p. 267 . OpenUrl CrossRef PubMed 27. Kurten , C.H.L. , et al. , Investigating immune and non-immune cell interactions in head and neck tumors by single-cell RNA sequencing . Nat Commun , 2021 . 12 ( 1 ): p. 7338 . OpenUrl CrossRef PubMed 28. Puram , S.V. , et al. , Single-Cell Transcriptomic Analysis of Primary and Metastatic Tumor Ecosystems in Head and Neck Cancer . Cell , 2017 . 171 ( 7 ): p. 1611 – 1624 e24 . OpenUrl CrossRef PubMed 29. Qian , J. , et al. , A pan-cancer blueprint of the heterogeneous tumor microenvironment revealed by single-cell profiling . Cell Res , 2020 . 30 ( 9 ): p. 745 – 762 . OpenUrl CrossRef PubMed 30. Wu , S.Z. , et al. , A single-cell and spatially resolved atlas of human breast cancers . Nat Genet , 2021 . 53 ( 9 ): p. 1334 – 1347 . OpenUrl CrossRef PubMed 31. Pelka , K. , et al. , Spatially organized multicellular immune hubs in human colorectal cancer . Cell , 2021 . 184 ( 18 ): p. 4734 – 4752 e20 . OpenUrl CrossRef PubMed 32. Lee , H.O. , et al. , Lineage-dependent gene expression programs influence the immune landscape of colorectal cancer . Nat Genet , 2020 . 52 ( 6 ): p. 594 – 603 . OpenUrl CrossRef PubMed 33. Khaliq , A.M. , et al. , Refining colorectal cancer classification and clinical stratification through a single-cell atlas . Genome Biol , 2022 . 23 ( 1 ): p. 113 . OpenUrl CrossRef PubMed 34. Kim , N. , et al. , Single-cell RNA sequencing demonstrates the molecular and cellular reprogramming of metastatic lung adenocarcinoma . Nat Commun , 2020 . 11 ( 1 ): p. 2285 . OpenUrl CrossRef PubMed 35. Jerby-Arnon , L. , et al. , A Cancer Cell Program Promotes T Cell Exclusion and Resistance to Checkpoint Blockade . Cell , 2018 . 175 ( 4 ): p. 984 – 997 e24 . OpenUrl CrossRef PubMed 36. Ma , L. , et al. , Tumor Cell Biodiversity Drives Microenvironmental Reprogramming in Liver Cancer . Cancer Cell , 2019 . 36 ( 4 ): p. 418 – 430 e6 . OpenUrl CrossRef PubMed 37. Bi , K. , et al. , Tumor and immune reprogramming during immunotherapy in advanced renal cell carcinoma . Cancer Cell , 2021 . 39 ( 5 ): p. 649 – 661 e5 . OpenUrl CrossRef PubMed 38. Chen , Z. , et al. , Single-cell RNA sequencing highlights the role of inflammatory cancer-associated fibroblasts in bladder urothelial carcinoma . Nat Commun , 2020 . 11 ( 1 ): p. 5077 . OpenUrl CrossRef PubMed 39. ↵ McInnes , L. , J. Healy , and J. Melville , UMAP: Uniform Manifold Approximation and Projection for Dimension Reduction . arXiv , 2020 . View the discussion thread. Back to top Previous Next Posted January 16, 2025. Download PDF Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Benchmarking of automated cancer cell annotation methods for scRNA-seq data reveals Consensus annotation as the preferred method Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Benchmarking of automated cancer cell annotation methods for scRNA-seq data reveals Consensus annotation as the preferred method Tamim Abdelaal , Christiaan Klijn , Matt Hancock bioRxiv 2025.01.13.632750; doi: https://doi.org/10.1101/2025.01.13.632750 Share This Article: Copy Citation Tools Benchmarking of automated cancer cell annotation methods for scRNA-seq data reveals Consensus annotation as the preferred method Tamim Abdelaal , Christiaan Klijn , Matt Hancock bioRxiv 2025.01.13.632750; doi: https://doi.org/10.1101/2025.01.13.632750 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Bioinformatics Subject Areas All Articles Animal Behavior and Cognition (7629) Biochemistry (17660) Bioengineering (13881) Bioinformatics (41913) Biophysics (21436) Cancer Biology (18578) Cell Biology (25482) Clinical Trials (138) Developmental Biology (13372) Ecology (19889) Epidemiology (2067) Evolutionary Biology (24302) Genetics (15599) Genomics (22483) Immunology (17728) Microbiology (40365) Molecular Biology (17163) Neuroscience (88540) Paleontology (666) Pathology (2830) Pharmacology and Toxicology (4821) Physiology (7637) Plant Biology (15130) Scientific Communication and Education (2045) Synthetic Biology (4290) Systems Biology (9818) Zoology (2269)