CATHe2: Enhanced CATH Superfamily Detection Using ProstT5 and Structural Alphabets

CATHe2: Enhanced CATH Superfamily Detection Using ProstT5 and Structural Alphabets | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results CATHe2: Enhanced CATH Superfamily Detection Using ProstT5 and Structural Alphabets View ORCID Profile Orfeú Mouret , View ORCID Profile Jad Abbass doi: https://doi.org/10.1101/2025.06.22.660903 Orfeú Mouret a School of Computer Science and Mathematics , Kingston-upon-Thames, KT1 2EE, UK b Telecom Physique Strasbourg, Strasbourg University , 300 Bd Sébastien Brant, CS 10413, Strasbourg, 67412, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Orfeú Mouret Jad Abbass a School of Computer Science and Mathematics , Kingston-upon-Thames, KT1 2EE, UK Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Jad Abbass For correspondence: j.abbass{at}kingston.ac.uk Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Motivation The CATH database is a free publicly available online resource that provides annotations about the evolutionary and structural relationships of protein domains. Due to the flux of protein structures coming mainly from the recent breakthrough of AlphaFold and therefore the non-feasibility of manual intervention, the CATH team recently developed an automatic CATH superfamily classifier called CATHe, that uses a feed-forward network classifier with protein Language Model (pLM) embeddings as input. Using the same dataset, in this paper, we present, CATHe2 that improves on CATHe by switching the old pLM ProtT5 for one of the most recent versions called ProstT5, and by introducing domain 3D information as input to the classifier, in the form of Structural Alphabet representation, namely 3Di sequence embeddings. Finally, CATHe2 implements a new version of the feed-forward network (FNN, i.e, non-recurrent neural network) classifier architecture, fine-tuned to perform at the CATH superfamily prediction task. Results The best CATHe2 model reaches an accuracy of 92.2 ± 0.7% with an F1 score of 82.3 ± 1.3% which constitutes an improvement of 9.9% on the F1 score and 6.6% on the accuracy, from the previous CATHe version (85.6 ± 0.4% accuracy and 72.4 ± 0.7% F1 score) on its largest dataset (~ 1700 superfamilies). This model uses ProstT5 AA sequence and 3Di sequence embeddings as input to the classifier, but a simplified version requiring only AA sequences, already improves CATHe’s F1 score by 6.7 ± 1.3% and accuracy by 6.6 ± 0.7% on its largest dataset. Availability & Implementation The code is available on https://GitHub.com/Mouret-Orfeu/CATHe2 . Datasets: https://doi.org/10.5281/zenodo.14534966 Contact orfeu.mouret.pro{at}outlook.fr , j.abbass{at}kingston.ac.uk 1. Introduction Proteins are essential to life, forming the foundation of virtually all biological processes. They are complex nanomachines with specific functions which largely depend on their unique 3D structure. Figuring out what shapes proteins fold into is known as the “protein folding problem”, and has stood as a grand challenge in biology since 1972 [ 1 , 2 ]. Alphafold2 is a 2020 automated algorithm which reached experimental performance in the protein folding problem, which is now considered solved for a large part of known proteins [ 3 ]. Since then, a database of Alphafold inferred 3D protein conformations also called CSM (Computed Structure Model) has been filled with over 200 million protein structure predictions. This database is the Alphafold database or AFDB for short [ 4 ]. As protein evolution gives rise to families of structurally related proteins, within which sequence identities can be extremely low, structure-based classifications are needed to identify unanticipated relationships in known structures, now present in overwhelming numbers. CATH is a classification system that fills this need, and even allows inference of domain function when paired with FunFam subclassification [ 5 ]. The corresponding CATH database is updated daily with the release of a snapshot version of the most up-to-date classifications. A full-release version called CATH-Plus including a greater depth of information about CATH domains is also uploaded annually, benefiting from ever improving automatic classification algorithms [ 6 , 7 , 8 ]. The last CATH-Plus version available at the time of writing is 4.4.0 according to the CATH website [ 9 ], this version includes 6,573 superfamilies. After the major bioinformatics revolution with AlphaFold2 (AF2) [ 3 ], bioinformaticians are now witnessing a second revolution with the rise of protein Language Models (pLMs) [ 10 , 11 , 12 , 13 ]. After the surprising huge leap of performance in NLP (Natural Language Processing) allowed by LLMs (Large Language Models) in recent years, protein processing soon took advantage of this new technique to process Amino Acid (AA) sequences, resulting in automatic and powerful feature extractors such as the ProtT5 pLM and CATHe classifier [ 14 ] (in this paper, ‘ProtT5’ refers to ProtT5-XL-U50 from the T5 models presented in the ProtTrans paper [ 15 ]). With new pLMs, pLM usage, and larger protein 3D conformation databases, the remote homologue detection and homology based inference (HBI) tool landscape has been rapidly evolving, making use of fast and sensitive large scale remote homologue detection using structural information [ 16 , 17 , 18 ] or better feature extraction from primary sequences. While standard sequence profile and Hidden Markov Model (HMM)-based alignment tools such as MMSeqs2 [ 19 ] and HHsuite [ 20 , 21 ] have been successful, they reach their limits in the twilight or midnight zones of sequence similarity. In contrast, structural alignment tools like Foldseek [ 18 ] leverage 3D protein information to perform better at detecting remote homology, and are significantly faster than earlier 3D alignment methods such as DALI [ 22 ]. These techniques are the state of the art in their own category, but the new wave of techniques inspired by pLMs is already being regarded as more promising as they can be faster and more sensitive in some cases. Embeddingbased annotation transfer (EAT) transfers protein annotation to a close protein using distance test, like Euclidean distance or cosine similarity between pLM embeddings [ 23 , 24 , 25 , 26 ]. EAT can be coupled with Contrastive Learning (CL) to extend the distance between embedding distribution clusters and thus improving EAT results [ 25 , 27 ]. Protein Embeddings Based Alignment (EBA or PEbA) uses pLM embeddings alignment to find homologues and has been very successful, with a lot of work using it to beat former state-of-the-art methods in various domains [ 28 , 29 , 30 ]. Protein artificial neural network (ANN) classifiers, such as those used in CATHe and this study, are also valuable tools for homology-based inference (HBI). These classifiers perform HBI by using supervised learning to develop internal representations for homology detection. All of the previous techniques are based on 3 possible types of input. First primary sequences, that can be vectorized in many different ways, pLM embeddings is one of them. Secondly evolutionary information, including Multiple Sequence Alignment (MSA), (HMM based techniques use MSA but some pLM based techniques too [ 31 , 32 , 33 , 34 ]), and the last input type is structural information, which can take multiple forms like full and precise 3D shape or just contact information, secondary structure annotation, or 1D representation like with Structural Alphabets (SA) [ 35 ]. All this structural information can be inferred information or ground truth and used as is or embedded using pLM for more complex feature extraction [36, 37, 38, 39, 26]. Recent structural alignment tools are promising, potentially more meaningful and accurate for HBI since structure is more important for protein function than primary sequences, plus, most pairs of proteins with similar structures populate the midnight zone [ 40 ]. Nowadays, many new techniques revolve around finding shortcuts to use 3D information without having to compute the 3D shape of proteins or skipping MSA construction, and thus speeding up potential subsequent structural alignment [ 41 , 42 , 43 ]. Using both structural and primary sequence information, i.e hybrid-information based methods have been shown to outperform other categories of methods [ 44 ] for remote homology detection and HBI. As primary sequences have been already studied extensively current research is focused on making the most out of protein 3D structure. Most sensitive and fast remote homology detection approaches today use embedding-based and structureaware tools such as Foldseek, ESMFold [ 45 ] or AlphaFoldMultimer [ 46 ] for instance, to produce relevant data for subsequent HBI techniques like EBA which seems to be one of the most effective. Some techniques like Foldseek-TM [ 18 ] use shortcuts to be even faster but less sensitive [ 28 ]. However, there is a way to be even faster, without losing in sensitivity, i.e, using discriminative methods [ 10 ]. Discriminative methods realize HBI tasks without the need for any kind of alignment, as with the ANN classifiers of CATHe and CATHe2. This is why automated protein annotation with ANN classifiers is a good long-term solution for large scale specific HBI tasks. Although it is less transparent, ANN classifiers are potentially faster and more accurate as no alignment is required and the classifier learns on its own how to find homologs, whatever the complexity of the homology relation. Given the right amount of representative data, sufficiently complex ANN classifiers can learn the proper way to classify (and annotate) proteins based on their most relevant properties extracted by a trained pLM, benefiting from transfer learning. In this paper, we introduce CATHe2, where both AA sequence embeddings and 3D structure information are used to perform the HBI of CATH annotations with an ANN classifier. The 3D information which is also embedded thanks to the 3Di SA developed for Foldseek and the ProstT5 pLM. The first version of CATHe relied on both pLM AA sequence embedding and ANN classifier usage to improve on the automatic CATH annotation task up to an F1 score of 72.4% on the corresponding dataset. CATHe2 improves on this method, using newer pLMs, adding protein 3D information as input (in the form of SA embeddings) and fine tuning the previous ANN classifier architecture for a final model F1 score of 82.3% on almost the same dataset. But CATHe and CATHe2 are not the only CATH annotation algorithms tested in recent years, many different pLMs were tested with different metrics, across many papers using different techniques and approaches for CATH annotation inference. As shown at the end of the Results paragraph, the current ranking of HBI techniques for CATH annotations evidenced by literature review seems to be classifiers >EBA >EAT+CL >EAT. Similarly, a ranking of pLM tested for this task should resemble ProstT5 >ProtT5 >TM-Vec >Ankh large >ESM2 >ESM1b although this ranking depends strongly on HBI technique. Almost all of these pLMs were also tested for CATHe2. 2. Materials and methods 2.1. Data and datasets To conduct as accurate and fair comparison as possible, CATHe2 datasets are based on the CATHe ones [ 14 ]. In CATHe, two SF datasets were used. The large SF dataset contains the 1,773 SF which had at least 2 non-redundant corresponding sequences in PDB at the time of CATHe development. This SF dataset represented 87.6% of the CATH v4.3 database in terms of domain number. The smaller SF dataset is a subset of the bigger one, only keeping the most populated 50 SF, representing 37.32% of the CATH v4.3 domain sequences in total. For each SF in the corresponding SF dataset, at least one domain sequence was placed in the test and validation set. Then, training sets were built using CATH-Gene3D sequences corresponding to the respective test and validation set SFs. In addition, sequences from PDB corresponding to the 3,456 SF without 2 non-redundant sequences in PDB were included via the ‘other’ part of the training dataset. All these other SF were under the single SF label ‘other’ in the test and validation set, as there were not enough sequences in PDB to populate the test and validation set with each of these SF. The sample sizes of the corresponding training datasets are 1,039,135 domains and 528,863 domains for the large and small training set respectively. For reference, CATH database contains 6,573 SF at the time of writing, but most of them are admittedly very rare as, in 2023 the 200 most highly populated superfamilies in CATH alone represent 62% of AF2 domains [ 7 ]. The CATHa dataset building process include a sequence identity filter in order to assesse the CATHe model using a dataset of remote homologues having less than 20% sequence identity to any domain in the training set. Although CATHe datasets were left as is for CATHe2 as much as possible, some of the dataset protein domains were removed to test and train CATHe2 due to the use of 3Di sequences, however, as discussed below, the missing proteins can be seen as negligible when performance is compared. For the small dataset, there was no loss whatsoever. In the large dataset, the inability to find the PDB files to compute 3Di properly for some domains resulted in the loss of 137,698 domains in this dataset, and 32 superfamilies among the 1773 formerly present in this dataset. These lost SF were among the least represented in the training set (please see Figure 1 ), which implies they were rare SF in the first place. Moreover, the ‘other’ SF representing all other SF than the 1772 present in the CATHe main dataset was not included in the CATHe2 dataset, due to a lack of information on these domains. It is worth mentioning that in the experiment where only AA sequence was considered, the dataset of CATHe2 does not suffer any superfamily loss compared to CATHe. Download figure Open in new tab Figure 1: Lost SF by support in CATHe training set This graph presents the support size of some superfamilies in the CATHe training set, on a logarithmic scale. Orange bars represent lost SF for the 3Di sequence dataset, meaning the SF lost due to unavailability of 3D structure for 3Di computation. As mentioned in Data and datasets, the lost SFs had low support in this dataset. For the sake of further and advanced experiments, we have created some smaller instances of the CATHe datasets by applying two different filters resulting in CATH SF loss, although none of these filters affected the number of SF lost in the final CATHe2 models. The first filter tested was the predicted local distance difference test (pLDDT) threshold filter. One of the most interesting features generated by AF2 is a confidence score, i.e., pLDDT [ 47 ]. Since the accuracy of the CSM directly affects the accuracy of the 3Di sequences and consequently the accuracy of the models, we have tried to remove as much as possible the ‘low quality’ structures in the hope to improve the models. Since the 3Di sequences from CATHe training dataset are computed on CSM from AFDB, which provides a confidence score, namely the pLDDT score, we have computed the overall pLDDT of all domains to be considered in the filter. The pLDDT threshold filter removes domains from the training set for which the pLDDT is under a certain threshold (pLDDT thresholds tested for CATHe2 models are: 0, 4, 14, 24, 34, 44, 54, 64, 74, 84); the SF completely removed from the training set by this filter were also removed from the test and validation set, see Figure 2 . Needless to say, a high threshold for such a filter not only leads to loss of SFs but also the reduction of the size of each class representing SFs. The second filter tested was a support threshold. As seen in Figure 1 , the size of classes (size called support of the classes, a class being an SF here), is not only quite unbalanced, but also some SFs comprises extremely low number of domains (< 10). This affects the ability of the models to train on such SFs, therefore, a support threshold filter was tested to try removing low support SF and improve the classifier performance. Support thresholds tested for CATHe2 models were 0 and 10, where 0 refers to not applying the filter. The SF completely removed from the training set by this filter were also removed from the test and validation set. A rather high number of SF was lost with support threshold = 10 (these lost SF include 167 SF which had already a support inferior or equal to 10 in CATHe raw training set, and 5 SF for which support was lowered under 10 by unfound 3D structure for 3Di usage). For this reason, no models trained with support threshold 10 have made it to the final version of CATHe2 although performance metrics were higher, see Section 3 for more details. Figure 2 summarises these two filtering processes and Figure 3 shows training set size in domains and SF relatively to the different filter thresholds. Download figure Open in new tab Figure 2: Overview of the dataset filtering process Filtering process of CATHe2 datasets. Each filter is optional, pLDDT threshold and support threshold have adjustable values and many combinations were tested to improve model performance as much as possible without losing too much SF in the process. Download figure Open in new tab Figure 3: Training set size evolution with the different filters These plots show training set size for the different combinations of thresholds for the pLDDT filter and the support filter tested for CATHe2. The size is either counted with domain number of SF number. 2.2. 3Di processing Structural Alphabets (SA) are sets of clusters grouping similar 3D conformations of short strings of AA. To each cluster is associated a letter, the set of clusters constituting an alphabet. They are used to represent the full 3D shape of proteins in one dimension, by building a sequence of SA letters, allowing for various 1D processing of 3D information. Most SA, such as CLE [ 48 ], 3D-BLAST [ 49 ] and Protein Blocks (PB) [ 50 ], discretize the conformations of short stretches of usually 3–5 Cα atoms. The SA chosen for this study is called 3D interaction (3Di) [ 18 ], it does not describe the backbone but rather tertiary interactions. The 20 states (or 20 letters) of the 3Di alphabet describe the geometric conformation of each residue, with its spatially closest residue. 3Di has three key advantages over traditional backbone structural alphabets. First, a weaker dependency between consecutive letters, secondly, a more evenly distributed state frequencies, both enhancing information density and reducing false positives, thirdly, the highest information density is encoded in conserved protein cores and the lowest in non-conserved coil/loop regions, whereas the opposite is true for backbone structural alphabets. In CATHe2, 3Di are computed using Foldseek [ 18 ] on PDB files from the RCSB Protein Data Bank (PDB) [ 51 ] for the test set and validation set. As the training set is composed of domains from the AlphaFold Protein Structure Database (AFDB) [ 52 ], Computed Structure Models (CSM) PDB files are extracted from there to compute 3Di sequences for this dataset. See Figure 4 for the whole CATHe2 input creation process. Download figure Open in new tab Figure 4: AA and 3Di sequence processing flowchart CATHe datasets are composed of protein domains, not full protein sequences, therefore these domains had to be aligned to PDB and AFDB pdb file sequences, then trimmed, in order to keep only the 3D structure corresponding to the CATHe dataset domains. For this alignment, the Pairwise Aligner method from the Bio library was used [ 53 ]. These trimmed 3D structures were then used to compute 3Di sequences for ProstT5 to transform into 3Di sequence embeddings. Finally, combinations of 3Di embeddings and AA sequence embeddings were tested to train the CATHe2 classifier in order to get the highest F1 score possible. Meanwhile other hyperparameters than CATHe2 input type were tested like dataset filter values or classifier architecture. 2.3. Models CATHe2 is an FNN classifier for which 7 hyperparameters were fine tuned to achieve the best results possible. First, the number of layer blocks; a layer block referring to a sequence of adjacent layers in this order: dense layer, LeakyReLu layer (Leaky Rectified Linear Unit layer), batch normalization layer, dropout layer. The other hyperparameters were the dropout rate for the dropout layer, the dense layer size, the pLDDT threshold, the support threshold, the input type (AA embeddings only, 3Di embeddings only or the concatenation of both) and finally, the pLM used to embed AA sequences, which are presented in this section. It is to note that apart from ProtT5, the pLMs described below are all recent and were launched after the publication of CATHe paper. ProtT5 (i.e, ProtT5-XL-U50) is the medium sized T5 encoder and the best-performing one from the ProtTrans paper [ 15 ]. This model is purely AA sequence based, meaning no structure information was used to train it. It was trained on BFD100 [ 47 ] and fine-tuned on Uniref50 [ 54 ]. Although ProtT5 embeddings were already used in CATHe, it is tested again for CATHe2 with a different code for generating embeddings, a better fine-tuned classifier architecture alongside 3Di embeddings in input. In this study ProtT5 proves again to be an excellent pLM for protein feature extraction by yielding very good results despite being rather old (2020). The ProstT5 pLM is a fine-tuned version of ProtT5 trained on both 3Di and AA span denoising task and bi-directional translation [ 27 ]. As a result, the encoder of ProstT5 can produce embeddings for both AA and 3Di sequences. The 3Di sequences used to train ProstT5 were computed from protein structures in AFDB [ 27 , 55 ]. ProstT5 is already being used in many works, often for 3Di prediction from primary sequences, bypassing full 3D prediction [ 56 , 57 ]. Moreover, in the ProstT5 paper, a CATH annotation benchmark is also carried out, using EAT and EAT+CL between embeddings from various inputs (AA, 3Di, ProstT5 predicted 3Di (p3Di), and the concatenation of AA and p3Di) comparing results with various pLM embeddings. The best results being consistently found to be with the ProstT5 pLM. In the CATHe2 code ProstT5 is called ProstT5 full as a half precision version called ProstT5 half is also tested for AA embedding computation (float16 weights instead of float32). TM-Vec is part of the tool duo TM-Vec+DeepBLAST whose purpose was to enable faster remote homology detection using structural alignment and deep learning [ 58 ]. TM-Vec produces structure-aware protein sequence embeddings, designed to help predict the TM-score [ 59 ], a measure of structural similarity between two protein sequences without the intermediate computation of their structures, even for remote homologs that fall below the 10% sequence identity. The model was trained on sequences from the CATH and SwissModel [ 60 ] structural databases. This model is actually based on ProtT5, with extra layers to help predict TM-scores [ 58 , 55 ]. For their CATH annotation benchmark, the TM-Vec team used EAT to find that TM-Vec separates CATH structural classes more clearly than the default ProtT5. Their results show that across each level of the CATH hierarchy, TM-Vec outperformed FoldSeek [ 18 ], MMseqs2 [ 19 ] and finally ProtTucker [ 25 ] which combines ProtT5 embeddings, EAT, and Contrastive Learning, for CATH annotation inference with HBI [ 25 ]. ESM2 was introduced in 2023 alongside the ESMFold tool that predicts protein structure from AA sequences only [ 45 ]. ESM2 is a purely AA sequence based pLM with an attention mechanism to learn pairwise interactions between amino acid sequences. It was trained on the Uniref50 and Uniref90 databases to learn large amounts of information and representations from protein sequences using masked language modelling (MLM). The same model was trained on multiple scales, ranging from 8 million parameters to 15 billion parameters. It is the 15 billion parameter version that was tested in this study (also called ESM2 15B) as it was shown to perform better overall [ 45 , 55 ]. To fit this large model into the available memory space for CATHe dataset embedding, only a half precision version of ESM2 15B was tested (float16 weights instead of float32). This pLM was included in CATHe pLM tests as it was a recent pLM that had been shown to capture the primary sequence features related to 3D conformation. Plus it is very large and the proportionality between language model size and the richness of its learned representations has been validated in many fields [ 23 ], however, in this context, larger is not always better [ 15 ]. Even if ESM2 15B had shown incredible results, which is not the case (see Section 3 ), using it for the final inference model of CATHe2 would not have been practical due to the enormous amount of memory required just to load it, even in half precision. To use such heavy pLMs in regular settings, memory efficient techniques have to be employed like the ones used in ESME [ 61 ]. Ankh is a purely AA sequence based pLM using a T5like architecture with a 48-layer transformer trained with MLM on UniRef50 sequences [ 23 , 62 , 11 ]. According to the corresponding paper, Ankh is a pLM that focuses on protein-specific optimization rather than relying on large model size, i.e, involving empirical, knowledge-guided optimization to improve performance without a proportional increase in computational resources. There are actually two Ankh models that were tested in this project, Ankh large, which is the larger version of Ankh, and Ankh base which is smaller. Ankh was also benchmarked on a CATH annotation task and yielded rather good results compared to ESM2 and ProtT5, outperforming them for the mean accuracy across all CATH levels [ 23 ]. 3. Results In order to have a comprehensive evaluation of different pLMs with different hyperparameters, we have run many preliminary experiments on CATHe’s large dataset in order to come up with the ideal combination. To do this, some comparisons have been made with fixed classifier structure and dataset filters to compare all pLMs, and ProstT5 full was selected to fine tune other hyperparameters on (fine tuning all hyperparameters for each pLM would have been too computationally costly). Then a grid search has been performed to approach the best classifier structure and dataset filter values, guided by training loss analysis. Some more insights for this fine tuning and for the preliminary experiments are available in the supplementary material to this paper. Based on the above, we have chosen the best combination found of all hyperparameters and filters to present a summary of the results. Although balanced accuracy and MCC were also computed for CATHe2 models, we have chosen to focus on F1 score since the dataset is highly unbalanced (95% confidence windows were computed using bootstrapping on the CATHe test set with 1000-folds and the standard deviation from this bootstrapping method was then multiplied with 1.96 to obtain the 95% confidence interval values). All F1 scores for all models tested for CATHe2 are available in the /results/perf dataframe.csv file in the GitHub project. Table 1 presents those results on the CATHe large dataset. View this table: View inline View popup Download powerpoint Table 1: CATHe2 final models performance These results suggest ProstT5 full >ProstT5 half >ESM2 15B >ProtT5 >Ankh large >Ankh base >TM-Vec in the CATHe2 context, but it is important to note that each model and version of models can perform differently relative to the others, based on the other hyperparameters. Moreover, comparing the performance of different pLM embedding inputs in the same ANN classifier like in Table 1 is questionable as each pLM produces embeddings which are of different sizes and probably need different ANN classifier architecture to show maximum performance. However, ProstT5 full AA+3Di embedding based version of CATHe outperforms significantly other CATHe2 models and previous models, i.e., CATHe (F1 Score of 82.3% Vs 72.4%) [ 14 ]. As shown in Table 1 , the fourth row, even with ProtT5 using our improved ANN algorithm and adjusted hyperparameters, the F1 Score didn’t exceed 80.6%, which shows the added value of ProstT5 over ProtT5. The ProstT5 full embedding, AA only model available for inference in CATHe2 has the following hyperparameters: Nb Layer Block: 2, Dropout: 0.1 Input Type: AA only, Layer size: 1024, pLDDT threshold: 0, Support threshold: 0, and has an F1 score of 78.7% ± 1.3 on CATHe large dataset. The introduction of 3Di embeddings in addition to AA embeddings improved classifier performance whereas 3Di embeddings on their own clearly underperform. This implies that AA sequences have a lot more useful information for CATHe2 classifiers than 3Di sequences. This was expected since the classification of CATH’s superfamilies is mainly based on sequence similarity, that is, evolutionary information. Not to add that the 3Di SA, similar to other structural alphabets, does not fully capture the structural information, in other words, it is not a one-to-one relationship (one SA sequence could correspond to more than one structure). In this section, we will present the results of the advanced and further experiments, i.e., the pLDDT and support filters. As shown in Figure 5 and as mentioned earlier, the combination of 3Di and AA inputs yielded consistently better results than AA only trained models for the same hyperparameter combination, especially when applying a pLDDT threshold around 24, whereas 3Di only models performed much worse consistently even with a high pLDDT threshold. Surprisingly, high pLDDT thresholds resulted in poor F1 scores. The reason behind this is the loss of many domains in the training dataset. When applying a ‘strict’ pLDDT filter (84 for example), the size of each class/SF decreases significantly, which naturally affects the accuracy of the model. The conclusion here is that the negative effect of decreasing the size of the training dataset outweighs the positive effect of removing domains whose 3D structure is of medium quality. This was also expected, because as shown in Figure 5 (AA+3Di Vs AA Vs 3Di), the values that the 3Di has added is relatively quite small, becauseagain – superfamily annotations are heavily based on evolutionary information, consequently, one wouldn’t expect a tangible improvement when removing ‘accurate’ AA sequences whose 3Di sequence is not ‘that accurate’. As for the second filter, i.e., the support filter, results are shown in Figure 6 . due to a higher number of SF lost with support threshold (172 SF were lost with a threshold equal to 10), no models trained with it made it to the final version of CATHe2 despite achieving a very high performance of 88.0 ± 1% F1 score and accuracy of 93.8 ± 0.6% on the large dataset. This result shows again that training data sample size is very important to improve model performance and implies that a good part of future development should be focused on the gathering of larger well curated datasets. Download figure Open in new tab Figure 5: CATHe2 F1 score input type comparison along pLDDT threshold for a ProstT5 model For AA only input there is only a single data point, for pLDDT threshold 0, because pLDDT is a 3D CSM confidence score, whereas no 3D models are used with only AA inputs. Moreover, the training set size mentioned for this AA only data point is not the one mentioned by the second y axis, but rather the full initial training dataset containing 1,039,135 domains. Hyperparameters for this model comparison are model embedding: ProstT5 full, Support threshold: 0, Nb layer block = 2, Layer size = 1024, Dropout = 0.3. Results are taken from CATHe2 large dataset, with no support threshold. Download figure Open in new tab Figure 6: F1 score comparison for support threshold for the best hyperparameter combination Contrary to the first plot, the hyperparameters here correspond to the best combination found for CATHe2 on its large dataset, i.e ProstT5 full AA+3Di embeddings with Dropout = 0.3, Layer size = 2048 and Nb layer block = 2 Compared to the previous ANN classifier architecture, CATHe2 best models have more layers (2 layer blocks instead of 1) and larger dense layers (2048 nodes instead of 128). This increase of classifier complexity has been shown to improve results up to a certain point. To conclude on CATHe2 results, Table 2 presents a comprehensive performance review of pLM and HBI technique combinations for CATH annotation inference ranked from best to worst, according to the literature found on this topic. The performance metric here is the mean accuracy across all CATH levels. (When no input type is mentioned, it means only AA embeddings were used). This ranking is up to discussion as the accuracy is not so relevant for unbalanced datasets, F1 score would be more informative. Moreover (apart from CATHe and CATHe2 which predict the whole CATH annotation, i.e, the SF annotation) these are averages over multiple predicted CATH level annotation, plus each result relies on different datasets across multiple papers. Finally, the same technique with the same pLM trained and tested on the same datasets can yield varying results across replications with sometimes as much as a 7% standard deviation. Still, this ranking of HBI techniques for CATH annotation tends to show that classifiers >EBA >EAT+CL >EAT for CATH annotation transfer. This is also up to discussion as each technique has different variants depending on input type, distance metric, alignment algorithm, etc. Moreover, each technique requires a different computation time, whether it is for model construction or inference on new data. This general ranking is nonetheless supported by other papers [ 63 ]. From this paper review, classifiers seem to perform better at CATH HBI, and ProstT5 appears to be particularly effective for CATH annotation transfer. Note that the good placement of CATHe and CATHe2 classifiers could be explained by the fact that they are restricted on around 1700 CATH annotations only, whereas other HBI techniques can query a larger set of possible annotations. View this table: View inline View popup Download powerpoint Table 2: CATH annotation HBI technique performance ranking 4. Discussion 4.1. Critical Analysis of the Study In this paper, we proposed CATHe2, an updated version of CATHe (the CATH superfamilies automated classifier produced by the CATH team). Using roughly the same large dataset, we were able to attain an F1 score of 82.3% vs 72.4% (accuracy of 92.2% vs 85.6%). After a thorough study and comparison using different pLMs, hyperparameters, and methodologies, we propose the CATHe2 classifier with 2 layer blocks, 0.3 dropout rate for the dropout layers, and 2048 nodes for the dense layers. The input of this final model is a concatenation of ProstT5 AA sequence embeddings and ProstT5 3Di embeddings (pLDDT threshold of 24 for the CSM used to compute 3Di sequences). Although the CATHe2 dataset is quite large and is representative of the most populated SFs (The 200 most highly populated superfamilies in CATH alone represent 62% of AF2 domains [ 7 ]), CATHe2 is bounded to infer only CATH superfamilies present in these ~ 1700 SF, and therefore cannot predict a superfamily that was not in the training set, in particular, it lacks the SF labeled ‘other’ present in former CATHe. The absence of this class could explain the performance improvement of CATHe2 over CATHe, but it is likely to explain only a small part of this improvement as a ProtT5 model trained with the same hyperparameter as CATHe, without the ‘other’ SF has only an F1 score of 73.8 ± 1.3%, compared to the 72.4 ± 0.7% of CATHe and the 82.3 ± 1.3% of the final CATHe2 model. Beyond the removal of the ‘other’ superfamily detection, we had to remove 32 SF (out of 1773), due to the absence of their corresponding structures, missing those SF from the model is likely to explain only a small part of CATHe2 F1 score improvement too. Indeed, a classifier similar to the CATHe one, trained on AA sequences only, but without those 32 SF (without the ‘other’ class), improved by 1.5% only compared to the same model but trained with those SFs (still without the ‘other’ class). 4.2. Avenues for improvement In this part, many new techniques and avenues for improvements are proposed to help future research on CATH annotation automation. A first promising improvement lead would be to implement Light Attention [ 63 ] when computing pLM embeddings per protein, instead of the average pooling on residue embeddings like in CATHe and CATHe2. As always, training the classifier with a larger, well curated dataset, is also expected to improve results, to make this easier, 3Di sequences could be inferred from AA sequence only using ProstT5 associated CNN [ 27 ] or ESM2 embeddings [ 41 ]. Training the pLMs again during classifier training is another promising lead. It may orient embedding computation toward the specialized task of CATH annotation prediction and thus improving results [ 64 ], although back propagation training for specific task fine tuning is not always successful and can lead to catastrophic forgetting or overfitting [ 15 , 65 ]. Not to mention that such task is computationally very heavy for large models, especially without using complexity reduction methods like reduced precision or low ranking adaptations (LoRA) [ 66 , 67 , 68 , 69 ]. For CATHe2 the concatenation of AA embeddings from one pLM and 3Di embeddings from ProstT5 was tested. To go further, multiple AA embeddings from multiple pLMs and the 3Di embeddings from ProstT5 could be concatenated (or even 3Di embeddings from multiple 3Di trained pLMs). Specially, concatenating AA embeddings from ProstT5 and ProtT5 as the ProstT5 paper suggests, could improve performance [ 27 ]. Although 3Di sequences were used in this study, it may be interesting to test protein blocks (PB) [ 50 ] instead as the 3Di alphabet was not developed to maximize remote homology search sensitivity, but to model tertiary interactions in protein structures [ 18 ]. Even when using 3Di, a bilingual model like ProstT5 may not be optimal. Thus, a potential improvement on CATHe2 would be to use AA sequence embeddings from an AA sequence specialized pLM like ProtT5 and a 3Di (or PB) sequence specialized pLM which, to our knowledge, has not been developed yet. The pLMs tested for CATHe2 were just a promising subset of all current pLMs suited for the task. Other promising pLMs exist, ProSST for instance is a good candidate, as it is a hybrid AAstructure pLM that produces its own 3Di like structural representation with an advanced structure quantization method and a better attention formulation to leverage the structure cues [ 70 ]. Finally, SaProt embeddings could be used to improve CATHe2 results too. Very similar to ProstT5, SaProt is based on ESM2 650M instead of ProtT5. For SaProt, AA and 3Di were fused to create a novel structural alphabet on which it was trained [ 71 ]. SaProt could be one of the best pLM to introduce structural information as suggested by other researchers [ 38 ]. Many other pLMs could be tested, extensive lists can be found in the literature [ 11 , 12 ]. Lastly, other avenues for improvement include using embeddings from another hidden layer than the last one, using embedding denoising [ 10 ], and using other classifier architectures like CNN classifiers or even a combination of CNN classifiers, which seem to be more efficient than the simple FNN classifier used for CATHe and CATHe2 [72]. Declaration of interest The authors declare no conflict of interest. CRediT authorship contribution statement Orfeu Mouret: Methodology, Software, Data curation, Visualization, Investigation, WritingOriginal draft, Conceptualization. Jad Abbass: Conceptualization, Supervision, Writing-Reviewing and Editing. Funding This research did not receive any specific grant from funding agencies in the public, commercial, or not-for-profit sectors Declaration of Generative AI and AI-assisted technologies in the writing process During the preparation of this work the author(s) used ChatGPT-4o in order to assist with phrasing and to correct spelling. After using this tool/service, the author(s) reviewed and edited the content as needed and take(s) full responsibility for the content of the publication. Acknowledgments Orfeu Mouret would like to thank Jad Abbass and Kingston University staff in general for their support. Footnotes https://github.com/Mouret-Orfeu/CATHe2 https://zenodo.org/records/14970431 References [1]. ↵ A. Cb , The formation and stabilization of protein structure , Biochem J 128 ( 4 ) ( 1972 ) 737 – 749 . doi: 10.1042/bj1280737 . OpenUrl FREE Full Text [2]. ↵ C. B. Anfinsen , Principles that govern the folding of protein chains , Science 181 ( 4096 ) ( 1973 ) 223 – 230 . doi: 10.1126/science.181.4096.223 . OpenUrl FREE Full Text [3]. ↵ J. Jumper , R. Evans , A. Pritzel , T. Green , M. Figurnov , O. Ronneberger , K. Tunyasuvunakool , R. Bates , A. Žídek , A. Potapenko , A. Bridgland , C. Meyer , S. A. A. Kohl , A. J. Ballard , High accuracy protein structure prediction using deep learning , Nature 596 ( 2021 ) 583 – 589 . doi: 10.1038/s41586-021-03819-2 . OpenUrl CrossRef PubMed [4]. ↵ G. DeepMind , EMBL-EBI, Alphafold protein structure database ( 2025 ). URL https://alphafold.ebi.ac.uk/ [5]. ↵ S. Das , D. Lee , I. Sillitoe , N. L. Dawson , J. G. Lees , C. A. Orengo , Functional classification of cath superfamilies: a domain-based approach for protein function annotation , Bioinformatics 31 ( 21 ) ( 2015 ) 3460 – 3467 . doi: 10.1093/bioinformatics/btv398 . OpenUrl CrossRef PubMed [6]. ↵ I. Sillitoe , N. Bordin , N. Dawson , V. P. Waman , P. Ashford , H. M. Scholes , C. S. M. Pang , L. Woodridge , C. Rauer , N. Sen , M. Abbasian , S. Le Cornu , S. D. Lam , K. Berka , I. Varekova , R. Svobodova , J. Lees , C. A. Orengo , Cath: increased structural coverage of functional space , Nucleic Acids Research 49 ( D1 ) ( 2020 ) D266 – D273 . doi: 10.1093/nar/gkaa1079 . OpenUrl CrossRef PubMed [7]. ↵ N. Bordin , I. Sillitoe , V. Nallapareddy , C. Rauer , S. D. Lam , V. P. Waman , N. Sen , M. Heinzinger , M. Littmann , S. Kim , S. Velankar , M. Steinegger , B. Rost , C. Orengo , Alphafold2 reveals commonalities and novelties in protein structure space for 21 model organisms , Communications Biology 6 ( 1 ) ( 2023 ) 160 . doi: 10.1038/s42003-023-04488-9 . OpenUrl CrossRef PubMed [8]. ↵ V. P. Waman , N. Bordin , R. Alcraft , R. Vickerstaff , C. Rauer , Q. Chan , Sillitoe, H. Yamamori , C. Orengo , Cath 2024: Cath-alphaflow doubles the number of structures in cath and reveals nearly 200 new folds , Journal of Molecular Biology 436 ( 17 ) ( 2024 ) 168551 . doi: 10.1016/j.jmb.2024.168551 . OpenUrl CrossRef PubMed [9]. ↵ O. group, Cath/gene3d ( 2025 ). URL https://www.cathdb.info/ [10]. ↵ M. Kilinc , K. Jia , R. L. Jernigan , Major advances in protein function assignment by remote homolog detection with protein language models – a review , Current Opinion in Structural Biology 90 ( 2025 ) 102984 . doi: 10.1016/j.sbi.2025.102984 . OpenUrl CrossRef PubMed [11]. ↵ W. Fan , Y. Zhou , S. Wang , Y. Yan , H. Liu , Q. Zhao , L. Song , Q. Li , Computational protein science in the era of large language models (llms) ( 2025 ). arxiv: 2501.10282 . URL https://arxiv.org/abs/2501.10282 [12]. ↵ M. Wang , Z. Ren , Deep learning methods for protein representation and function prediction: A comprehensive overview , unpublished preprint ( 2025 ). Doi: 10.2139/ssrn.5096181 . OpenUrl CrossRef [13]. ↵ W. Ruan , Y. Lyu , J. Zhang , J. Cai , P. Shu , Y. Ge , Y. Lu , S. Gao , Y. Wang , P. Wang , L. Zhao , T. Wang , Y. Liu , L. Fang , Z. Liu , Z. Liu , Y. Li , Z. Wu , J. Chen , H. Jiang , Y. Pan , Z. Yang , J. Chen , S. Liang , W. Zhang , T. Ma , Y. Dou , J. Zhang , X. Gong , Q. Gan , Y. Zou , Z. Chen , Y. Qian , S. Yu , J. Lu , K. Song , X. Wang , A. Sikora , G. Li , X. Li , Q. Li , Y. Wang , L. Zhang , Y. Abate , L. He , W. Zhong , R. Liu , C. Huang , W. Liu , Y. Shen , P. Ma , H. Zhu , Y. Yan , D. Zhu , T. Liu , Large language models for bioinformatics ( 2025 ). arxiv: 2501.06271 . URL https://arxiv.org/abs/2501.06271 [14]. ↵ V. Nallapareddya , N. Bordin , I. Sillitoe , M. Heinzinger , M. Littmann , V. P. Waman , N. Sen , B. Rost , C. Orengo , Cathe: detection of remote homologues for cath superfamilies using embeddings from protein language models , Bioinformatics 39 ( 1 ) ( 2023 ) btad029 . doi: 10.1093/bioinformatics/btad029 . OpenUrl CrossRef PubMed [15]. ↵ A. Elnaggar , M. Heinzinger , C. Dallago , G. Rehawi , Y. Wang , L. Jones , T. Gibbs , T. Feher , C. Angerer , M. Steinegger , D. Bhowmik , B. Rost , Prottrans: Toward understanding the language of life through selfsupervised learning , IEEE Transactions on Pattern Analysis and Machine Intelligence 44 ( 10 ) ( 2022 ) 7112 – 7127 . doi: 10.1109/TPAMI.2021.3095381 . OpenUrl CrossRef PubMed [16]. ↵ Z. Zhang , C. Wang , M. Xu , V. Chenthamarakshan , A. Lozano , P. Das , J. Tang , A systematic study of joint representation learning on protein sequences and structures ( 2023 ). arxiv: 2303.06275 . URL https://arxiv.org/abs/2303.06275 [17]. ↵ P. M. Borujeni , R. Salavati , Functional domain annotation by structural similarity , NAR Genomics and Bioinformatics 6 ( 1 ) ( 2024 ) qae005. doi: 10.1093/nargab/lqae005 . OpenUrl CrossRef [18]. ↵ M. van Kempen , S. S. Kim , C. Tumescheit , M. Mirdita , J. Lee , C. L. M. Gilchrist , J. Söding , M. Steinegger , Fast and accurate protein structure search with foldseek , Nature Biotechnology 42 ( 2 ) ( 2024 ) 243 – 246 . doi: 10.1038/s41587-023-01773-0 . OpenUrl CrossRef PubMed [19]. ↵ M. Steinegger , J. Söding , Mmseqs2 enables sensitive protein sequence searching for the analysis of massive data sets , Nature Biotechnology 35 ( 11 ) ( 2017 ) 1026 – 1028 . doi: 10.1038/nbt.3988 . OpenUrl CrossRef PubMed [20]. ↵ M. Steinegger , M. Meier , M. Mirdita , H. Vöhringer , S. J. Haunsberger , J. Söding , Hh-suite3 for fast remote homology detection and deep protein annotation , BMC Bioinformatics 20 ( 1 ) ( 2019 ) 473 . doi: 10.1186/s12859-019-3019-7 . OpenUrl CrossRef PubMed [21]. ↵ M. Remmert , A. Biegert , A. Hauser , J. Söding , Hhblits: lightning-fast iterative protein sequence searching by hmm-hmm alignment , Nature Methods 9 ( 2 ) ( 2012 ) 173 – 175 . doi: 10.1038/nmeth.1818 . OpenUrl CrossRef PubMed Web of Science [22]. ↵ L. Holm , Dali server: structural unification of protein families , Nucleic Acids Research 50 ( W1 ) ( 2022 ) W210 – W215 . doi: 10.1093/nar/gkac387 . OpenUrl CrossRef PubMed [23]. ↵ A. Elnaggar , H. Essam , W. Salah-Eldin , W. Moustafa , M. Elkerdawy , C. Rochereau , B. Rost , Ankh: Optimized protein language model unlocks general-purpose modelling ( 2023 ). arxiv: 2301.06568 . URL https://arxiv.org/abs/2301.06568 [24]. ↵ M. Kilinc , K. Jia , R. L. Jernigan , Improved global protein homolog detection with major gains in function identification , Proceedings of the National Academy of Sciences 120 ( 9 ) ( 2023 ). Doi: 10.1073/pnas.2211823120 . OpenUrl CrossRef PubMed [25]. ↵ M. Heinzinger , M. Littmann , I. Sillitoe , N. Bordin , C. Orengo , B. Rost , Contrastive learning on protein embeddings enlightens midnight zone , NAR Genomics and Bioinformatics 4 ( 2 ) ( 2022 ) qac043 . doi: 10.1093/nargab/lqac043 . OpenUrl CrossRef [26]. ↵ T. Bepler , B. Berger , Learning the protein language: Evolution, structure, and function , Cell Systems 12 ( 6 ) ( 2021 ) 654 – 669.e3 . doi: 10.1016/j.cels.2021.05.017 . OpenUrl CrossRef PubMed [27]. ↵ M. Heinzinger , K. Weissenow , J. G. Sanchez , A. Henkel , M. Mirdita , M. Steinegger , B. Rost , Bilingual language model for protein sequence and structure , NAR Genomics and Bioinformatics 6 ( 4 ) ( 2024 ) qae150. doi: 10.1093/nargab/lqae150 . OpenUrl CrossRef [28]. ↵ L. Pantolini , G. Studer , J. Pereira , J. Durairaj , G. Tauriello , T. Schwede , Embedding-based alignment: combining protein language models with dynamic programming alignment to detect structural similarities in the twilight-zone , Bioinformatics 40 ( 1 ) ( 2024 ) btad786 . doi: 10.1093/bioinformatics/btad786 . OpenUrl CrossRef PubMed [29]. ↵ B. G. Iovino , Y. Ye , Protein embedding based alignment , BMC Bioinformatics 25 ( 1 ) ( 2024 ) 85 . doi: 10.1186/s12859-024-05699-5 . OpenUrl CrossRef PubMed [30]. ↵ K. Kaminski , J. Ludwiczak , K. Pawlicki , V. Alva , S. Dunin-Horkawicz , plm-blast: distant homology detection based on direct comparison of sequence representations from protein language models , Bioinformatics 39 ( 10 ) ( 2023 ) btad579 . doi: 10.1093/bioinformatics/btad579 . OpenUrl CrossRef PubMed [31]. ↵ K. Erckert , B. Rost , Assessing the role of evolutionary information for enhancing protein language model embeddings , Scientific Reports 14 ( 1 ) ( 2024 ) 20692 . doi: 10.1038/s41598-024-71783-8 . OpenUrl CrossRef PubMed [32]. ↵ U. Lupo , D. Sgarbossa , A.-F. Bitbol , Protein language models trained on multiple sequence alignments learn phylogenetic relationships , Nature Communications 13 ( 1 ) ( 2022 ) 6298 . doi: 10.1038/s41467-022-34032-y . OpenUrl CrossRef PubMed [33]. ↵ C. Gong , A. Klivans , J. M. Loy , T. Chen , Q. Liu , D. J. Diaz , Evolutioninspired loss functions for protein representation learning , in: Forty-first International Conference on Machine Learning , 2024 . URL https://openreview.net/forum?id=y5L8W0KRUX [34]. ↵ R. M. Rao , J. Liu , R. Verkuil , J. Meier , J. Canny , P. Abbeel , T. Sercu , A. Rives , Msa transformer , in: Proceedings of the 38th International Conference on Machine Learning, Vol. 139 of Proceedings of Machine Learning Research , PMLR , 2021 , pp. 8844 – 8856 . URL https://proceedings.mlr.press/v139/rao21a.html [35]. ↵ N. Labiosa , A. Kohli , C. Chung , C. Korban , Hybrid transformer and neural network configuration for protein classification using amino acids , bioRxiv ( 2024 ). Doi: 10.1101/2024.10.31.621421 . OpenUrl Abstract / FREE Full Text [36]. V. Gligorijević , P. D. Renfrew , T. Kosciolek , J. K. Leman , D. Berenberg , T. Vatanen , C. Chandler , B. C. Taylor , I. M. Fisk , H. Vlamakis , R. J. Xavier , R. Knight , K. Cho , R. Bonneau , Structure-based protein function prediction using graph convolutional networks , Nature Communications 12 ( 1 ) ( 2021 ) 6101 . doi: 10.1038/s41467-021-23303-9 . OpenUrl CrossRef PubMed [37]. H. Ghazikhani , G. Butler , Toot-plm-p2s: Incorporating secondary structure information into protein language models , unpublished preprint ( 2024 ). Doi: 10.1101/2024.08.13.607781 . OpenUrl Abstract / FREE Full Text [38]. ↵ D. Frolova , M. A. Pak , A. Litvin , I. Sharov , D. N. Ivankov , I. Oseledets , Mulan: Multimodal protein language model for sequence and structure encoding , unpublished preprint ( 2024 ). Doi: 10.1101/2024.05.30.596565 . OpenUrl Abstract / FREE Full Text [39]. J. Zhao , W. Zhuang , J. Song , Y. Li , S. Lu , Pre-training protein bi-level representation through span mask strategy on 3d protein chains ( 2024 ). arxiv: 2402.01481 . URL https://arxiv.org/abs/2402.01481 [40]. ↵ S. K, H. M, S. M, R. B , Nearest neighbor search on embeddings rapidly identifies distant protein relations , Frontiers in Bioinformatics 2 ( 2022 ) 1033775 . doi: 10.3389/fbinf.2022.1033775 . OpenUrl CrossRef PubMed [41]. ↵ S. R. Johnson , M. Peshwa , Z. Sun , Sensitive remote homology search by local alignment of small positional embeddings from protein language models , eLife 12 ( 2024 ) RP91415 . doi: 10.7554/eLife.91415 . OpenUrl CrossRef PubMed [42]. ↵ Z. Zhang , J. Lu , V. Chenthamarakshan , A. Lozano , P. Das , J. Tang , Structure-informed protein language model ( 2024 ). arxiv: 2402.05856 . URL https://arxiv.org/abs/2402.05856 [43]. ↵ Y. Zhang , W. Zhang , Rprot-vec: A deep learning approach for fast protein structure similarity calculation ( 2025 ). URL https://www.biorxiv.org/content/10.1101/2025.01.25.634852v1.abstract [44]. ↵ B. Lin , X. Luo , Y. Liu , X. Jin , A comprehensive review and comparison of existing computational methods for protein function prediction , Briefings in Bioinformatics 25 ( 4 ) ( 2024 ) bbae289 . doi: 10.1093/bib/bbae289 . OpenUrl CrossRef PubMed [45]. ↵ Z. Lin , H. Akin , R. Rao , B. Hie , Z. Zhu , W. Lu , N. Smetanin , R. Verkuil , O. Kabeli , Y. Shmueli , A. dos Santos Costa , M. Fazel-Zarandi , T. Sercu , S. Candido , A. Rives , Evolutionary-scale prediction of atomic-level protein structure with a language model , Science 379 ( 6637 ) ( 2023 ) 1123 – 1130 . doi: 10.1126/science.ade2574 . OpenUrl CrossRef PubMed [46]. ↵ R. Evans , M. O’Neill , A. Pritzel , N. Antropova , A. Senior , T. Green , A. Žídek , R. Bates , S. Blackwell , J. Yim , O. Ronneberger , S. Bodenstein , M. Zielinski , A. Bridgland , A. Potapenko , A. Cowie , K. Tunyasuvunakool , R. Jain , E. Clancy , P. Kohli , J. Jumper , D. Hassabis , Protein complex prediction with alphafold-multimer , bioRxiv ( 2022 ). Doi: 10.1101/2021.10.04.463034 . OpenUrl Abstract / FREE Full Text [47]. ↵ J. Jumper , R. Evans , A. Pritzel , T. Green , M. Figurnov , O. Ronneberger , K. Tunyasuvunakool , R. Bates , A. Žídek , A. Potapenko , A. Bridgland , C. Meyer , S. A. A. Kohl , A. J. Ballard , A. Cowie , B. Romera-Paredes , S. Nikolov , R. Jain , J. Adler , T. Back , S. Petersen , D. Reiman , E. Clancy , M. Zielinski , M. Steinegger , M. Pacholska , T. Berghammer , S. Bodenstein , D. Silver , O. Vinyals , A. W. Senior , K. Kavukcuoglu , P. Kohli , D. Hassabis , Highly accurate protein structure prediction with alphafold , Nature 596 ( 2021 ) 583 – 589 . doi: 10.1038/s41586-021-03819-2 . OpenUrl CrossRef PubMed [48]. ↵ S. Wang , W.-M. Zheng , Clepaps: Fast pair alignment of protein structures based on conformational letters , Journal of Bioinformatics and Computational Biology 06 ( 02 ) ( 2008 ) 347 – 366 . doi: 10.1142/S0219720008003461 . OpenUrl CrossRef [49]. ↵ J.-M. Yang , C.-H. Tung , Protein structure database search and evolutionary classification , Nucleic Acids Research 34 ( 13 ) ( 2006 ) 3646 – 3659 . doi: 10.1093/nar/gkl395 . OpenUrl CrossRef PubMed Web of Science [50]. ↵ A. de Brevern , C. Etchebest , S. Hazout , Bayesian probabilistic approach for predicting backbone structures in terms of protein blocks , Proteins: Structure, Function, and Bioinformatics 41 ( 3 ) ( 2000 ) 271 – 287 . doi: 10.1002/1097-0134(20001115)41:33.0.CO;2-Z . OpenUrl CrossRef [51]. ↵ H. Berman , J. Westbrook , Z. Feng , G. Gilliland , T. Bhat , H. Weissig , Shindyalov , P. Bourne, The protein data bank, Nucleic Acids Research 28 ( 1 ) ( 2000 ) 235 – 242 . doi: 10.1093/nar/28.1.235 . URL https://www.rcsb.org/ OpenUrl CrossRef PubMed Web of Science [52]. ↵ M. Varadi , S. Anyango , M. Deshpande , S. Nair , C. Natassia , G. Yordanova , D. Yuan , O. Stroe , G. Wood , A. Laydon , A. Žídek , T. Green , K. Tunyasuvunakool , S. Petersen , J. Jumper , E. Clancy , R. Green , A. Vora , M. Lutfi , M. Figurnov , A. Cowie , N. Hobbs , P. Kohli , G. Kleywegt , E. Birney , D. Hassabis , S. Velankar , Alphafold protein structure database: massively expanding the structural coverage of protein-sequence space with high-accuracy models , Nucleic Acids Research 50 ( D1 ) ( 2021 ) D439 – D444 . doi: 10.1093/nar/gkab1061.biopython . doi:10.1093/nar/28.1.235. URL https://biopython.org/docs/1.76/api/Bio.Align.html OpenUrl CrossRef [53]. ↵ B. E. Suzek , Y. Wang , H. Huang , P. B. McGarvey , C. H. Wu , the UniProt Consortium , Uniref clusters: a comprehensive and scalable alternative for improving sequence similarity searches , Bioinformatics 31 ( 2015 ) 926 – 932 . doi: 10.1093/bioinformatics/btu739 . OpenUrl CrossRef PubMed [54]. ↵ C. Li , Z. Flamholz , L. Kelly , Comparing protein language models using remote homology detection for phages ( 2025 ). URL https://openreview.net/forum?id=IEZjjDX0iC [55]. ↵ D. Kim , S. Park , M. Steinegger , Unicore enables scalable and accurate phylogenetic reconstruction with structural core genes , unpublished preprint ( 2024 ). URL https://www.biorxiv.org/content/early/2024/12/22/2024.12.22.629535 [56]. ↵ J. Puentes Mozo , Dual-modality transformer-based approach for viral protein classification integrating protein language models and 3di fasta representations , Tech. rep., Fundación Universitaria de Ciencias de la Salud ( 2024 ). URL https://hdl.handle.net/1992/74920 [57]. ↵ T. Hamamsy , J. T. Morton , R. Blackwell , D. Berenberg , N. Carriero , V. Gligorijevic , C. E. M. Strauss , J. K. Leman , K. Cho , R. Bonneau , Protein remote homology detection and structural alignment using deep learning , Nature Biotechnology 42 ( 6 ) ( 2024 ) 975 – 985 . doi: 10.1038/s41587-023-01917-2 . OpenUrl CrossRef [58]. ↵ Y. Zhang , J. Skolnick , Scoring function for automated assessment of protein structure template quality , Proteins: Structure, Function, and Bioinformatics 57 ( 4 ) ( 2004 ) 702 – 710 . doi: 10.1002/prot.20264 . OpenUrl CrossRef PubMed Web of Science [59]. ↵ A. Waterhouse , M. Bertoni , S. Bienert , G. Studer , G. Tauriello , R. Gumienny , F. T. Heer , T. A. de Beer , C. Rempfer , L. Bordoli , R. Lepore , T. Schwede , Swiss-model: homology modelling of protein structures and complexes , Nucleic Acids Research 46 ( W1 ) ( 2018 ) W296 – W303 . doi: 10.1093/nar/gky427 . OpenUrl CrossRef PubMed [60]. ↵ M.H. Çelik , X. Xie , Efficient inference , training, and fine-tuning of protein language models, unpublished preprint ( 2024 ). Doi: 10.1101/2024.10.22.619563 . OpenUrl Abstract / FREE Full Text [61]. ↵ C. J-Y, W. J-F, H. Y, L. X-H, Q. Y-R, S. C-L , Evaluating the advancements in protein language models for encoding strategies in protein function prediction: a comprehensive review , Frontiers in Bioengineering and Biotechnology 13 ( 2025 ) 1506508 . doi: 10.3389/fbioe.2025.1506508 . OpenUrl CrossRef [62]. ↵ H. Stärk , C. Dallago , M. Heinzinger , B. Rost , Light attention predicts protein location from the language of life , Bioinformatics Advances 1 ( 1 ) ( 2021 ) vbab035 . doi: 10.1093/bioadv/vbab035 . OpenUrl CrossRef PubMed [63]. ↵ R. Schmirler , M. Heinzinger , B. Rost , Fine-tuning protein language models boosts predictions across diverse tasks , Nature Communications 15 ( 1 ) ( 2024 ) 7407 . doi: 10.1038/s41467-024-51844-2 . OpenUrl CrossRef PubMed [64]. ↵ Y. Luo , Z. Yang , F. Meng , Y. Li , J. Zhou , Y. Zhang , An empirical study of catastrophic forgetting in large language models during continual finetuning , unpublished preprint ( 2025 ). arxiv: 2308.08747 . URL https://arxiv.org/abs/2308.08747 [65]. ↵ A. Kumar , Fine tune large protein model (prottrans) using huggingface , hands on tutorial and result analysis ( 2024 ). URL https://training.galaxyproject.org/training-material/topics/statistics/tutorials/fine_tuning_protTrans/tutorial.html [66]. ↵ S. Zhang , J. K. Liu , Seqproft: Applying lora finetuning for sequence-only protein property predictions ( 2024 ). arxiv: 2411.11530 . URL https://arxiv.org/abs/2411.11530 [67]. ↵ S. Sledzieski , M. Kshirsagar , M. Baek , R. Dodhia , J. L. Ferres , B. Berger , Democratizing protein language models with parameter-efficient finetuning , Proceedings of the National Academy of Sciences 121 ( 26 ) ( 2024 ) e2405840121 . doi: 10.1073/pnas.2405840121 . OpenUrl CrossRef PubMed [68]. ↵ E. J. Hu , yelong shen , P. Wallis , Z. Allen-Zhu , Y. Li , S. Wang , L. Wang , W. Chen , Lora: Low-rank adaptation of large language models , 2022 . URL https://openreview.net/forum?id=nZeVKeeFYf9 [69]. ↵ M. Li , P. Tan , X. Ma , B. Zhong , H. Yu , Z. Zhou , W. Ouyang , B. Zhou , L. Hong , Y. Tan , Prosst: Protein language modeling with quantized structure and disentangled attention , unpublished preprint ( 2024 ). Doi: 10.1101/2024.04.15.589672 . URL https://www.biorxiv.org/content/early/2024/05/17/2024.04.15.589672.1 OpenUrl Abstract / FREE Full Text [70]. ↵ J. Su , C. Han , Y. Zhou , J. Shan , X. Zhou , F. Yuan , Saprot: Protein language modeling with structure-aware vocabulary , unpublished preprint ( 2024 ). Doi: 10.1101/2023.10.01.560349 . OpenUrl Abstract / FREE Full Text [71]. ↵ R. Vitale , L. A. Bugnon , E. L. Fenoy , D. H. Milone , G. Stegmayer , Evaluating large language models for annotating proteins , Briefings in Bioinformatics 25 ( 3 ) ( 2024 ) bbae177 . doi: 10.1093/bib/bbae177 . OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted June 26, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following CATHe2: Enhanced CATH Superfamily Detection Using ProstT5 and Structural Alphabets Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share CATHe2: Enhanced CATH Superfamily Detection Using ProstT5 and Structural Alphabets Orfeú Mouret , Jad Abbass bioRxiv 2025.06.22.660903; doi: https://doi.org/10.1101/2025.06.22.660903 Share This Article: Copy Citation Tools CATHe2: Enhanced CATH Superfamily Detection Using ProstT5 and Structural Alphabets Orfeú Mouret , Jad Abbass bioRxiv 2025.06.22.660903; doi: https://doi.org/10.1101/2025.06.22.660903 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Bioinformatics Subject Areas All Articles Animal Behavior and Cognition (7635) Biochemistry (17697) Bioengineering (13894) Bioinformatics (41951) Biophysics (21455) Cancer Biology (18593) Cell Biology (25509) Clinical Trials (138) Developmental Biology (13380) Ecology (19903) Epidemiology (2067) Evolutionary Biology (24322) Genetics (15611) Genomics (22509) Immunology (17737) Microbiology (40398) Molecular Biology (17183) Neuroscience (88619) Paleontology (667) Pathology (2833) Pharmacology and Toxicology (4825) Physiology (7644) Plant Biology (15158) Scientific Communication and Education (2046) Synthetic Biology (4296) Systems Biology (9825) Zoology (2271)