Fine-Tuning Protein Language Models Enhances the Identification and Interpretation of the Transcription Factors

Fine-Tuning Protein Language Models Enhances the Identification and Interpretation of the Transcription Factors | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Fine-Tuning Protein Language Models Enhances the Identification and Interpretation of the Transcription Factors View ORCID Profile Mir Tanveerul Hassan , View ORCID Profile Saima Gaffar , View ORCID Profile Hamza Zahid , View ORCID Profile Sang Jun Lee doi: https://doi.org/10.1101/2025.11.27.691010 Mir Tanveerul Hassan 1 Jeonbuk RICE Intelligence Innovation Research Center, Jeonbuk National University , Jeonju, 54896, South Korea Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Mir Tanveerul Hassan Saima Gaffar 2 Division of Electronics and Information Engineering, Jeonbuk National University , Jeonju, 54896, South Korea Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Saima Gaffar Hamza Zahid 2 Division of Electronics and Information Engineering, Jeonbuk National University , Jeonju, 54896, South Korea Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Hamza Zahid Sang Jun Lee 2 Division of Electronics and Information Engineering, Jeonbuk National University , Jeonju, 54896, South Korea Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Sang Jun Lee For correspondence: sj.lee{at}jbnu.ac.kr Abstract Full Text Info/History Metrics Supplementary material Preview PDF Abstract Transcription factors (TFs) are pivotal regulators of gene expression and play essential roles in diverse cellular activities. The three-dimensional organization of the genome and transcriptional regulation are predominantly orchestrated by TFs. By recruiting the transcriptional machinery to gene enhancers or promoters, TFs can either activate or repress transcription, thereby controlling gene activity and various biological pathways. Accurate identification of TFs is vital for elucidating gene regulatory mechanisms within cells. However, experimental identification remains labor-intensive and time-consuming, highlighting the necessity for efficient computational approaches. In this study, we present a two-layer predictive framework utilizing protein language models (pLMs) via full fine-tuning and parameter-efficient fine-tuning. The initial layer robustly classifies and identifies transcription factors, while the subsequent layer predicts TFs with a binding preference for methylated DNA (TFPMs). Our approach further incorporates attention weights and protein sequence motifs to enhance interpretability and predictive capability. By leveraging attention mechanisms, we highlight biologically relevant regions of the protein sequences that contribute most strongly to the predictions. Additionally, motif analysis facilitates the identification of conserved sequence patterns that are critical for TF recognition and function. Across both TF and TFPM classification tasks, the inclusion of these features allowed our methods to consistently surpass contemporary models, as demonstrated by independent test results. Keypoints Developed a two-layer predictive framework using protein language models (pLMs) with both full fine-tuning and parameter-efficient fine-tuning methods. The first layer accurately identifies transcription factors (TFs), and the second layer predicts TFs with binding preference for methylated DNA (TFPMs). Integrated attention weights and protein sequence motifs to enhance model interpretability by highlighting biologically relevant sequence regions and conserved patterns. Achieved superior performance compared to state-of-the-art methods, validated by independent testing. Mir Tanveerul Hassan obtained his M.Tech. in Computer Science from the University of Kashmir, India, in 2020, and later earned his Ph.D. in Electronics and Information Engineering from Jeonbuk National University, Jeonju, South Korea. He is currently serving as a postdoctoral fellow at the Jeonbuk RICE Intelligence Innovation Research Center. His research interests encompass computational biology, bioinformatics, and pattern recognition. Saima Gaffar received her B.Tech. and M.Tech. degrees in Computer Science from the University of Kashmir, Srinagar, India, and her Ph.D. in Electronics and Information Engineering from Jeonbuk National University, South Korea. Her research focuses on bioinformatics, computational biology, deep learning, and image processing. Hamza Zahid received his B.S. degree in Mechatronics Engineering from the University of Engineering and Technology, Peshawar, Pakistan. He is currently pursuing the integrated M.S. and Ph.D. degrees in Electronics and Information Engineering at Jeonbuk National University, South Korea. His primary research interests include the applications of artificial intelligence in computational drug discovery. Sang Jun Lee received his B.S., M.S., and Ph.D. degrees in Electrical Engineering from POSTECH, South Korea. Following his doctoral studies, he worked as a senior researcher at the Samsung Advanced Institute of Technology (SAIT). He is currently an Associate Professor in the Division of Electronics and Information Engineering at Jeonbuk National University, South Korea. His research interests include image analysis, deep learning, and medical image processing. Introduction Transcription factors (TFs) are key regulatory proteins that play a central role in cell-fate determination by governing gene expression. They are distributed throughout the genome, and their abundance generally scales with genome size [ 1 ]. In fact, larger genomes tend to encode a greater number of TFs. It is estimated that approximately 10% of the human genome encodes TFs, making them one of the most prevalent protein families in humans. Acting as critical regulators of transcription, TFs interact with promoter regions to control gene expression by recruiting RNA polymerases to the core promoter, thereby either enhancing or repressing transcription. Although all cells share the same genetic blueprints, TFs drive cellular diversity by selectively modulating the expression of distinct gene sets, thus orchestrating processes such as differentiation. They regulate multiple target genes across distinct developmental stages, yet their activity is confined by specific DNA binding motifs. Mutations within TFs have been implicated in numerous diseases [ 2 ]. Essentially, TFs serve as molecular switches that govern the spatial, temporal, and quantitative dynamics of gene expression, ensuring precise transcriptional control. Accurate identification of TFs thus represents both a fundamental biological challenge and a major therapeutic opportunity for drug discovery. Structurally, TFs possess three primary functional domains: a DNA-binding domain, a transcriptional regulatory interface, and a signaling region [ 3 , 4 ], which enable effective binding to gene promoters [ 5 ]. DNA methylation was initially recognized as a mechanism for influencing TF binding affinity, predominantly targeting cytosines within CpG dinucleotides [ 6 ]. Initially, methylation was regarded as a transcriptional repressive mark, especially when localized within CpG islands at promoters, where it obstructed transcription factor access and resulted in gene silencing. With advances in high-throughput and third-generation sequencing technologies, such as bisulfite sequencing, PacBio single-molecule real-time sequencing, and Oxford Nanopore sequencing, a much broader landscape of DNA modifications has been identified [ 7 ]. More recent studies further demonstrated that certain TFs, even in the absence of methyl-CpG binding domains (MBDs), retain the capacity to bind methylated DNA [ 8 , 9 , 10 ]. Elucidating these interactions is vital for understanding the biological diversity conferred by DNA methylation-dependent regulation [ 11 , 12 , 13 ]. Conventionally, TF discovery has relied on wet-lab approaches, including SELEX-based methods [ 14 ], MITOMI [ 15 ], and ChIP-based assays [ 16 ], which have enabled the identification of many TFs cataloged in public databases [ 17 , 18 , 19 , 20 ]. However, these experimental strategies are resource-intensive in terms of both time and cost and remain inadequate for characterizing TFs across all tissues and species. Computational methods, particularly those driven by artificial intelligence, have emerged as powerful complements to experimental discovery. By exploiting models trained on known TF sequences, novel TF candidates can be computationally predicted and later validated through experiments, thereby streamlining the identification pipeline and dramatically reducing the associated costs. Liu et al. (2020) [ 21 ] introduced TFPred, the first computational framework for distinguishing TFs from non-TFs, utilizing classical hand-crafted encodings such as composition/transition/distribution (CTD), amino acid composition (AAC), and dipeptide composition (DPC) with a support vector machine (SVM) classifier. However, these descriptors lacked the ability to capture sequential information, overlooking the positional arrangement of amino acids that can critically influence protein structure and function. To address this, Nguyen (2022) [ 22 ] proposed PSSM+CNN, wherein protein sequences were transformed into position-specific scoring matrices (PSSM) followed by classification with a deep convolutional neural network. Nonetheless, the accuracy of PSSM is contingent on sequence alignment quality, which introduces potential variability. Subsequently, Zheng et al. [ 23 ] adapted capsule networks (Capsule TF), inspired by Sabour et al. [ 24 ], while Li et al. (2022) [ 25 ] developed Li RNN, which directly processed protein sequences as residue triplets through long short-term memory (LSTM) networks. The latest model is the TFProtBert [ 26 ], using ProtBert embeddings as an input for the machine learning models, as the embedding vectors capture general semantic similarity but not task-specific distinctions. Despite their innovations, these neural models often suffered from overfitting due to limited training data and the complexity of tuning numerous hyperparameters. Recent progress in transformer architectures and large language models (LLMs) has revolutionized protein sequence analysis. By drawing parallels between proteins and natural language, LLMs can model both local and long-range dependencies among amino acids, uncovering complex structural and functional relationships. This paradigm treats amino acid sequences as linguistic tokens, enabling pretrained models to capture rich contextual information that traditional encodings often overlook. This study uses ESM2 and ProtBert models for protein analysis. ESM2, trained on large protein datasets like UniRef50 via masked language modeling, captures rich biochemical and structural context without needing evolutionary alignments, aiding tasks like structure and function prediction. ProtBert, based on BERT, is pretrained on 217 million sequences from UniRef100, masking 15% of amino acids, and treats each protein as an independent document without next sentence prediction. The full model fine-tuning and Parameter-Efficient FineTuning (PEFT) pLMs can achieve and show better results due to the inherent nature of large language models (LLMs) to capture context and subtle biological dependencies within protein sequences. Building on this insight, we propose leveraging pLMs with both full and parameter-efficient fine-tuning strategies to overcome the challenges of traditional methods. This approach demonstrates superior effectiveness in transcription factor (TF) classification and enables accurate prediction of TF interactions with methylated DNA, providing a powerful tool for understanding gene regulation mechanisms. The architecture of the proposed method is depicted in Figure 1 . Download figure Open in new tab Figure 1. The architecture of the full and PEFT fine-tuning in the ESM2 models. (A) The ESM-2 backbone model uses the protein sequences to identify TF and NTF initially, and subsequently separates TF into TFPM and TFPNM categories. The n layers represent of the number of layers/depth of the model, where n = 6, 12, 30, 33, or 36 depending on the architecture of the ESM-2 model. (B) In the PEFT framework, LoRA introduces additional trainable low-rank matrices (r) into the self-attention module, enabling efficient fine-tuning without modifying the full model parameters. To further interpret model behavior, we compare the attention weights from different layers and heads with known top motifs. This analysis aims to reveal how the model’s attention aligns with biologically relevant sequence patterns, providing insight into which residues and motifs are prioritized during transcription factor classification and interaction predictions. This enhanced understanding links computational attention patterns to established biological motifs, shedding light on the mechanistic basis of model decisions. Methods and material Data The benchmark training and testing datasets were sourced from the study by Liu et al. [ 21 ]. The original dataset contained 601 human and 129 mouse transcription factors (TFs) with affinity for methylated DNA [ 27 , 6 ], along with 286 TFs preferring non-methylated DNA [ 3 ]. To improve dataset quality, Liu et al. (TF dataset 2020) [ 21 ] applied several preprocessing steps. Sequences containing ambiguous amino acids such as “X”, “B”, or “Z” were first removed, followed by redundancy reduction using the CD-HIT clustering tool [ 28 , 29 ] at a 25% sequence identity threshold. Additionally, sequences shorter than 50 residues were excluded. After these refinements, 522 TFs were retained as positive samples. For negative samples, an equal number of non-TFs were randomly selected from the UniProt database under the following criteria: reviewed status, experimental evidence at the protein level, sequence length greater than 50 amino acids, and absence of DNA-binding transcriptional activity. These selected sequences also exhibited less than 25% identity, as ensured by CD-HIT. Thus, the final TF dataset comprised 522 positive and 522 negative entries. For constructing the first-layer model, identifying transcription factors against non-transcription factors (TF vs NTF), the TF dataset was divided into training and independent test sets at an 80:20 ratio. The TF training set consisted of 416 positive and 416 negative sequences, while the TF independent set contained 106 positives and 106 negatives. For the second-layer model, targeting the identification of TFs with a preference for methylated DNA (TFPM dataset), the TFPM training set included 146 non-methylationassociated TFs (negative class) and 146 methylation-associated TFs (positive class). The TFPM independent test set consisted of 37 negatives and 69 positives. Feature Encoding Feature extraction, or feature encoding, is a critical step in building robust machine learning models and directly influences their predictive performance. Among various approaches, evolutionary information captured through position-specific scoring matrix (PSSM) [ 30 ] profiles provides a richer representation than raw sequence data alone. Consequently, PSSM-based feature descriptors have become widely adopted as essential primary inputs for model construction, bridging an important gap in bioinformatics research. These descriptors have consistently enhanced prediction accuracy in diverse protein-related tasks, including fold recognition, structural class prediction, protein-protein interaction analysis, subcellular localization, RNA-binding site identification, and functional annotation. We extracted PSSM matrices using PSI-BLAST [ 31 ] run against UniProt-50 for each protein sequence. Each protein sequence of length L yielded a matrix of dimensions L × 20. We processed the extracted PSSM matrices using row and/or column transformations and converted them in a 1D feature vectors representing different properties of the original PSSM matrix. In total we extracted seven different types of PSSM profiles and are defined as: AAC PSSM: This feature, known as the auto-covariance transformation, is computed for column j by first calculating the mean of the column. Next, the deviation from this mean is obtained for the elements at rows i and i + g , which are then multiplied together. The summation is performed by iterating i from 1 to L − g . Since j ranges from 1 to 20 and g ranges from 1 to 10, the resulting representation yields a feature vector of length 200, as defined below: where L is the length of the protein sequence, S i , j represents the PSSM score at position i for amino acid j . This feature captures the dependency and spatial correlation of evolutionary information along the sequence, enhancing the representation of protein characteristics. AB_PSSM: This feature encompasses two distinct types of feature vectors. Initially, each protein sequence is segmented into 20 equally sized sections, referred to as blocks, and within each block, the corresponding row vectors from the PSSM are summed. The resultant vector for each block is then normalized by dividing by the block’s length, which corresponds to 5% of the total protein length. Arranging these 20 normalized vectors sequentially yields the first feature vector with a length of 400. For the second type, the mean of the positive values in each column of the PSSM is calculated for each block and each amino acid. These 20 average values, one for each block, are concatenated for each of the 20 amino acid types, producing a vector of length 20 per amino acid; combining these across all amino acids results in a second feature vector of length 400. Trigram PSSM: This feature vector, extracted from the PSSM, has a length of 8,000. It is computed by multiplying elements from three consecutive rows and three distinct columns of the PSSM. This operation is applied across all possible sets of three consecutive rows, and the resulting products are summed to yield a single feature corresponding to the selected three columns. Since there are 20 possible columns, the total length of the feature vector becomes 20 × 20 × 20 = 8,000. It can be defined as: where P i,m represents the PSSM score at row i and column m, L is the length of the protein sequence, and m, n, r ∈ { 1, …, 20 } correspond to the amino acid columns. DP PSSM: This feature, associated with dipeptide composition (DPC), was initially introduced for predicting the structural class of proteins. To compute this descriptor, elements from two consecutive rows and two distinct columns in the PSSM are multiplied together. This multiplication is performed across all relevant pairs of rows and columns. It can be defined as: where P i,m and P i +1, n are elements from two consecutive rows (i and i+1) and two distinct columns (m and n) of the PSSM matrix, and L is the length of the protein sequence. PSe PSSM: This feature was initially introduced for the prediction of membrane proteins and their respective types [ 32 ]. The PSe PSSM descriptor forms a 320-dimensional vector, where the first 20 elements represent the average values of the 20 rows in the PSSM matrix. The remaining components are calculated as follows: for each column, the mean squared difference between the i th and ( i + lag ) th elements is computed, with lag taking integer values from 1 to 15. Consequently, the total length of the resulting feature vector is 20 × 15 + 20 = 320 The process of generating this feature is expressed in Equation 4 . where j = 1, 2, …, 20, lag = 1, 2, …, 15 k = 20 + j + 20(lag − 1) Smoothed PSSM: Smoothed PSSM refers to a representation of a position-specific scoring matrix in which local sequence context is incorporated by averaging or summing evolutionary scores over a window of neighboring residues. Instead of using the raw scores for each position independently, a smoothing (or sliding) window is applied to the PSSM so that each position’s score reflects not only its own evolutionary information but also that of its surrounding residues. This approach captures local patterns and contextual conservation, which can improve prediction of functional regions or protein attributes. If the window extends beyond sequence boundaries, zero vectors are typically appended to maintain consistency in the profile length. For a residue at position i , the Smoothed PSSM value is calculated by incorporating the scores of neighboring residues within a specified window. Mathematically, this can be represented as: where w denotes the window radius and P j is the PSSM score at position j . If the summation window extends beyond the sequence termini, zero-padding is applied to maintain the length of the resulting profile vector. K separated bigram PSSM: K separated bigram PSSM is a feature extracted from the Position Specific Scoring Matrix, which extends the concept of dipeptide composition (DPC). For each pair of distinct columns m and n , it considers pairs of residues separated by a fixed gap of length k along the sequence. Mathematically, the feature for columns m and n with separation k is calculated by: where P i,m and P i + k,n are PSSM scores at positions i and i + k for columns m and n , respectively, and L is the protein length. The special case when k = 1 corresponds exactly to the dipeptide composition (DPC) feature. Apart from the PSSM profiles, we extracted sequence-level embeddings from the pre-trained models and used the same as an input to a 128 node custom-head classifier. The embeddings were extracted from the last hidden layer of a model. Pre-trained pLMs The pLMs employed in this study varied considerably in both architecture and scale ( Table 1 ), ranging from 8 million parameters (ESM2-8M) to 3 billion parameters (ESM2-3B). The ESM2 models are encoder-only architectures based on RoBERTa64 and trained with an unsupervised masked language modeling objective [ 33 ]. In contrast, the other pLM, Prot-Bert, is built on BERT architecture [ 34 ], an encoder large language model pretrained using a span-masking objective [ 35 , 34 ]. All models were initialized from pre-trained checkpoints publicly available on HuggingFace. The pre-trained models were utilized to extract the sequence-level embeddings from the last hidden layer. The size of the sequence-level embeddings is mentioned in Table 1 . Extraction of embeddings from the pre-trained ESM2-3B model failed due to the model exceeding available memory limits, even with an effective batch size of 1. The final classification task was carried out by a 128 node two layer custom head classifier and the same setting was carried on the all pre-trained models for the standard evaluation. View this table: View inline View popup Download powerpoint Table 1. Details of the models and their respective embedding size Model training Top-level comparisons were made between pre-trained and fine-tuned models as follows. For the pre-trained results, we generated embeddings for all sequences. For tasks operating at the protein level, sequence embeddings were reduced to a single fixed-length vector by averaging across the sequence dimension, preserving the original embedding size. This vector was then passed through a prediction head, a custom head classifier with a fully connected layer containing 128 neurons. Training stopped when the plateau in training loss was reached, with each experiment repeated five times using different random seeds. For fine-tuning, we attached an identical fully connected layer (size 128) to each pLM encoder to serve as the prediction head. In the case of ProtBert, the final hidden states were average-pooled across the sequence length during training for protein-level tasks. Following the recommendations of the ESM2 authors [ 36 ], the prediction head for ESM2 was linked exclusively to the first token (the special token). Each experiment was run three times with different random seeds, and training was stopped once both the training and validation losses plateaued. To improve computational efficiency, Parameter-Efficient Fine-Tuning (PEFT) was applied across all models using the Low-Rank Adaptation (LoRA) approach [ 37 ]. For the smaller ESM2 variants (up to the 650M model), we additionally evaluated full fine-tuning of the entire network For both embedding-based and fine-tuning approaches, we reported the performance of the checkpoint yielding the lowest validation loss for each run. The improvements were calculated as: where Δ ( k ) represents the best performance of the model k . All models and embedding extraction were carried out on an Nvidia GeForce RTX 3080 GPU coupled with an Intel i9 processor with 128GB memory using PyTorch 2.2.0+cu118. The GPU is good enough to run and train all models but full fine-tuning ESM2 3B due to its sheer size. Evaluation metrics Machine learning, deep learning, and graph neural networks find widespread application in bioinformatics [ 38 , 39 , 40 , 41 , 42 ], chemoinformatics [ 43 , 44 ], and image analysis [ 45 , 46 , 47 ]. Evaluating the performance of computational models is critical to establish their predictive accuracy and reliability. To this end, a standard suite of evaluation metrics is employed, including accuracy (Acc), specificity (Sp), sensitivity (Sn), precision (Pr), F1 score (F1), area under the receiver operating characteristic curve (AUC), and Matthews correlation coefficient (MCC). These metrics are extensively utilized in bioinformatics research for providing a comprehensive understanding of model behavior, highlighting both strengths and weaknesses [ 48 , 49 , 50 , 51 , 52 ]. Definitions of these metrics are provided as follows: where T P represents true positive, T N represents true negative, F P represents false positive, and F N represents false negative. Results Performance of baseline models on raw PSSM features We assessed the performance of each PSSM-based feature representation individually across five distinct machine learning classifiers—RF, XGB, ETC, LGBM, and CatBoost—using a 5-fold cross-validation strategy. Seven feature encodings—AAC PSSM, AB PSSM, Tri gram PSSM, DP PSSM, Smoothed PSSM, K separated bigram PSSM, and PSe PSSM—were evaluated against these five conventional machine learning classifiers, frequently used in the field of bioinformatics [ 53 , 54 , 55 ], in the identification of transcription factors, and the detailed results are mentioned in Supplementary Tables S1 and S2. From the Figure 2 , the cross-validation MCC is in the range of 0.307-0.805, showing variable results across different feature encodings. Among the seven PSSM features, Trigram PSSM shows the highest CV-MCC value of 0.805 against the ETC classifier and the least CV-MCC score of 0.602 against the XGB classifier, thus showing the variation of almost 20% in the CV-MCC scores across different classifiers. The DP PSSM shows consistent performance with a little variation across the different classifiers, with values ranging from 0.726-0.750. The Smoothed PSSM encoding was the least effective feature encoding type irrespective of the type of the classifier, with CV-MCC ranging from 0.307-0.580. Among the different classifier algorithms, the XGB model was the least performing one, with values ranging from 0.307-0.743 and the ETC showed better results than the other classifiers against all PSSM features but two. Download figure Open in new tab Figure 2. Heatmap representing MCC score of seven PSSM feature encodings against five machine learning models (a) cross-validation MCC score (b) independent MCC score On the independent dataset, the classifiers show the similar pattern as can be seen from the Figure 2 . The Smoothed PSSM showed the least performance across the different classifiers, with values ranging from 0.307-0.500, akin to the cross-validation performance. Here again, DP PSSM shows the consistent performance with values ranging from 0.7260.783. The CatBoost classifier showed the highest independent MCC value of 0.801 against the K separated bigram PSSM. The XGB model showed the least performance among the five classifiers, with values ranging from 0.307-0.745. From Supplementary Tables S1 and S2, all PSSM features except Smoothed PSSM showed consistent performance in terms of accuracy across the different classifiers on the main benchmark and independent datasets. Performance of the pre-trained models Before fine-tuning the protein-language models, we extracted the embeddings from five models of moderate size, which include ESM2-8M, ESM2-35M, ESM2-150M, ESM2-650M, and ProtBert. For evaluation, we constructed a custom classifier head of a 128-D dense layer, rather than using the default classifier head provided by the Hugging Face, for more control and consistency. The performance of the custom head classifier on the independent dataset embeddings showed that the ESM2-650M model is performing best as compared to the other models, which is understandably due to the 650M parameter size, the highest among these five protein models ( Table 2 ). The custom head classifier achieved an accuracy of 0.932±0.003 and an MCC of 0.831±0.038, and the ESM2-8M model showed the least performance as compared to the other models, achieving the best accuracy of 0.857±0.008 and an MCC value of 0.716±0.014, reflecting its meager parameter size as the main reason for its below-par performance. The ProtBert model was somewhere intermediate in performance as compared to the other four models with an accuracy of 0.906±0.220 and an MCC score of 0.814±0.039. View this table: View inline View popup Download powerpoint Table 2. Performance comparison of the pre-trained models on the independent TF set Performance of the full fine-tuned models All five models were fully fine-tuned along with the custom head classifier of the 128-D dense layer to enhance its prediction and interpretation. Each model ran for a total of 10 epochs, and the effective batch size was set as 8. For larger models we used the gradient accumulation steps parameter to avoid out-of-memory issues due to resource constraints. The training accuracy-loss curve is shown in Figure 3 . It can be seen from Figure 3 , the training loss and the validation loss do converge in all cases of the ESM2-based models. The performance of the five protein-language models on the independent dataset is mentioned in Table 3 . The ESM2-650M model achieved an accuracy of 0.948±0.015 and an MCC score of 0.898±0.030, the highest among the five pLMs. The ESM2-8M model, with the least number of parameters, also showed good performance, achieving an accuracy of 0.910±0.019 and an MCC value of 0.820±0.038. To assess the stability of the ESM2 models, we employed bootstrapping with 1,000 resampled test sets and evaluated performance across each iteration. The bootstrapped performance results demonstrate that the ESM2 models are both robust and highly stable across repeated resampling of the test set. The narrow standard deviations observed for all key metrics indicate that the model’s behavior does not fluctuate significantly with changes in sample composition. This low variability confirms that the model’s predictions are not driven by chance patterns or specific subsets of the data, but instead reflect consistently reliable discrimination between transcription factor and non-transcription-factor sequences. View this table: View inline View popup Download powerpoint Table 3. Performance comparison of the fully fine-tuned models on the independent TF dataset Download figure Open in new tab Figure 3. Training accuracy-loss curve of the four evolutionary pLMs. The green represents training accuracy, blue represents the training loss, and orange represents the validation loss curve. (a) accuracy-loss curve of the full fine-tuned ESM2-8M (b) accuracy-loss curve of the full fine-tuned ESM2-35M (c) accuracy-loss curve of the full fine-tuned ESM2-150M (d) accuracy-loss curve of the full fine-tuned ESM2-650M The performance of the ProtBert, a 450M model, is quite disappointing, as the model was not able to learn anything even with hyperparameter tuning, with the training loss curve remaining flat, thus showing BERT-based models’ inability to learn any meaningful information or features. The results demonstrated ( Table 3 ) that the fine-tuned models, except ProtBert, showed significant improvements over the pre-trained models ( Table 2 ). The improvement in the fine-tuned models is largely because of the inherent nature of the large language models to learn the intrinsic characteristics of the protein sequences. Using LoRA Apart from the fine-tuning full layers, we used the PEFT technique to understand the impact on the performance of the protein-language models. We tried LoRA with variable rank values ranging from 4 to 32. The effective batch size was again kept at 8, the same as in fully fine-tuned models, for effective comparison. The LoRA technique does reduce the training time and GPU memory usage considerably but has limited impact on the performance of the protein-language models. The performance of the protein-language models using LoRA is depicted in Table 4 . The LoRA doesn’t bring any significant improvement in the performance of the protein models over fully fine-tuned models, as can be seen from Tables 3 and 4 . Again, the ProtBert models don’t show any learning process even with LoRA and hyperparameter optimization, thus revealing their inability to learn anything because of their BERT-based structure. The Evolutionary Scale Modeling 2-based models showed a good learning process, with the ESM2-650M achieving the highest accuracy of 0.943±0.016 and the highest MCC score of 0.887±0.032. The ESM2-8M was the least performing model with an accuracy of 0.900±0.020 and an MCC score of 0.801±0.041, slightly less than the fully fine-tuned corresponding model. With LoRA using a fraction of the original parameters, we were able to train the ESM2-3B model as well, achieving the highest accuracy of 0.934±0.017 and an MCC score of 0.868±0.034, with the rank set as 4, thus showing the ability of the LoRA technique to run large language models on small GPUs. View this table: View inline View popup Download powerpoint Table 4. Performance assessment of LoRA models on the independent TF dataset The improvements in the fully fine-tuned and LoRA models over the pre-trained models are depicted in Figure 4 . The improvements were calculated as per the equation 7 . The fully fine-tuned and LoRA models showed improvements over the pre-trained models across different metrics ( Figure 4 ). The fully fine-tuned models showed overall improvement across all metrics over pre-trained models. The ESM2-8M and ESM235M models were able to show variable improvements across different metrics over pre-trained models. The ESM2-150M and ESM2-650M models also showed modest improvements across all metrics but specificity. The specificity values decreased by 0.94% in the ESM2-150M model and 1.25% in the ESM2-650M model. In the case of LoRA models, the largest overall gain occurs in the ESM2 8M model, especially in specificity (+8.15%) and MCC score (+8.47%). The ESM2-35M shows strong balanced gains, notably in F1 (+5.70%) and MCC (+6.65%). The ESM2-150M and ESM2-650M also showed improvements, but gains are smaller, with modest improvements spread across all metrics. The AUC improves only for ESM2-650M (+0.61%), while it slightly decreases in ESM2-35M and ESM2-150M. The low standard deviations indicate that the performance of ESM2 models are stable across different resampled versions of the test set, meaning the results are not overly sensitive to which samples happen to be in the test split. The accuracy-loss curves of the LoRA ESM2 models are depicted in Figure 5 . Download figure Open in new tab Figure 4. The values represent improvement in the fine-tuned and LoRA models over Pre-trained models in percentage terms. The blue tiles represent the positive improvement of above 2%, yellow tiles represent improvement of 0%-2%, and purple tiles represents the negative improvement. * denotes the LoRA models. Download figure Open in new tab Figure 5. Training accuracy-loss curve of the five ESM2 models using PEFT technique LoRA. The green represents training accuracy, blue represents the training loss, and orange represents the validation loss curve. (a) accuracy-loss curve of the LoRA ESM2-8M (b) accuracy-loss curve of the LoRA ESM2-35M (c) accuracy-loss curve of the LoRA ESM2-150M (d) accuracy-loss curve of the LoRA ESM2-650M (e) accuracy-loss curve of the LoRA ESM2-3B The primary factors influencing computational resource demands during model training were the parameter sizes of the pLMs and the quadratic scaling of attention mechanisms, which increased requirements for longer protein sequences. Modern GPUs with over 10GB of memory are generally capable of handling all pLMs evaluated in this study, except the ESM2-3B model. With these approaches, most pLMs could be fine-tuned even on older GPUs with as little as 10GB of memory. Interestingly, both full fine-tuning and parameterefficient LoRA fine-tuning required similar amounts of GPU memory for smaller models, differing only in training speed ( Figure 6 ). In contrast, generating embeddings required much less GPU memory, making it suitable for datasets with very long sequences. The runtime values of the fully fine-tuned and LoRA models are mentioned in Supplementary Table S3. Download figure Open in new tab Figure 6. Fine-tuning training speed. The training speeds of full fine-tuning (blue) and LoRA (red) are displayed on a normalized scale, using ESM2-650M LoRA fine-tuning as the reference point with a baseline value of 1 (x). The corresponding speed-up achieved by each model (olive) is shown on a linear scale. All experiments used transcription factor protein sequences of length 1024 in a per-protein training setup. The LoRA technique is more beneficial as it runs faster than the full fine-tuned models. In the case of the largest model, ESM2-650M, LoRA fine-tuning was approximately 3 times faster than full fine-tuning. Performance evaluation with the existing predictors We evaluated the performance of our fully fine-tuned ESM2-650M model with the existing state-of-the-art predictors in the literature, namely TFPred [ 21 ], Li RNN [ 25 ], PSSM+CNN [ 22 ], Capsule TF [ 23 ], and TFProtBert [ 26 ]. From Table 5 , it is clear that the full fine-tuned model, ESM2-650M, is achieving the best possible results among the existing methods, with an accuracy of 0.948, which is higher than the rest of the lot in the range of 3.6%-11.8%, and with an MCC score of 0.899, which is also significantly higher than the existing methods, thus showing the robust nature of the full fine-tuned model. The fully fine-tuned model also has the highest sensitivity, specificity, and ROC-AUC values among the other models ( Table 5 ). View this table: View inline View popup Download powerpoint Table 5. Performance comparison between the existing models and the best fully fine-tuned model on the independent dataset. † represents the full fine-tuned model. Second layer performance In the second layer, the transcription factors as predicted in the first layer were further classified into transcription factors preferring binding to the methylated DNA (TFPM) and the transcription factors preferring binding to the non-methylated DNA (TFPNM). The second layer performance of the full fine-tuned and the LoRA models using ESM2 variants on the independent dataset (TFPM independent set) is mentioned in Supplementary Tables S4 and S5, respectively. The performance of the ESM2-8M full fine-tuned model is slightly better than the LoRA 8M model, as can be seen from the Supplementary Tables S4 and S5. The ESM2-8M fully fine-tuned achieved the highest accuracy of 0.593 ± 0.043 and an MCC score of 0.149 ± 0.067, and the LoRA 35M model achieved the highest accuracy of 0.495 ± 0.046 and an MCC score of 0.111 ± 0.089. The performance, on the TFPM independent dataset, of the fully fine-tuned ESM2-8M model against the existing methods, Li RNN [ 25 ] and TFProtBert [ 26 ], is shown in the Table 6 . The fully fine-tuned ESM2-8M shows around 4%-29% improvement in the accuracy metric compared to the existing methods, whereas in the case of the MCC metric, the fully fine-tuned method shows improvement in the range of 8%-63%. These results show the robustness of the fully fine-tuned ESM2-8M model in identifying TFPMs and TFPNMs as compared to the existing methods, with a great scope of improvement. View this table: View inline View popup Download powerpoint Table 6. The performance comparison between the existing methods and the best model on the second layer in identifying TFPMs and TFPNMs. † represents the fully fine-tuned model. Attention Maps and Motifs To investigate how the top model’s attention mechanism captures functionally important amino acid fragments, we adapted a technique from prior work [ 56 , 57 ]. We examined how the attention-based amino acid confidence scores from the best full fine-tuned model, ESM2-650M, aligned with the top known motifs extracted using the STREME web server [ 58 ] from the TF dataset (positive TF samples). As shown in Figure 7 , the amino acids or fragments that received the highest attention corresponded closely to those that frequently occur in the motif. The top motif in the positive benchmark TF dataset identified by the STREME [ 58 ] is “ERKRRRNR”, and its corresponding attention map is depicted in Figure 7a . This suggests that the model’s attention mechanism effectively identifies biologically meaningful regions, providing a plausible explanation for its predictions. Furthermore, full fine-tuning further refines the model’s ability to capture functional interactions among amino acids. This results in a more distinct and varied attention pattern across relevant amino acid pairs, enhancing the model’s performance in sequence classification. Download figure Open in new tab Figure 7. Visualization of attentions for various protein sequences corresponding to the top six motifs. The protein sequences are too long to be used for attention visualization, so we extracted only the high attention part from the original sequences corresponding to the motifs for better visualization. For each of the six highest-scoring motifs, the corresponding high-attention regions from representative protein sequences are extracted and shown with their 1D attention heatmaps. The top panels display the sequence logos generated from the attention-enriched subsequences, while the bottom panels show the amino-acid positions and their normalized attention weights across different transformer layers and heads. Figure 8 depicts the attention pattern of the transcription factor protein sequence containing the top motif “ERKRRRNR”. The attention pattern, as shown in Figure 8a , depicts stronger attention (darker blue regions), occurring primarily in the middle of the sequence, especially among and around the repeated “RKRRRR” segment. This data indicates that these amino acids are being focused on more intensely by this attention head, either attending to themselves or to each other. Layer 8, Head 16 of the model appears to focus on and within the central, arginine/lysine-rich motif of the protein, likely attending to biologically significant features or repetitive sequence regions. This pattern suggests that this head plays a role in identifying related motifs, which are characterized by clusters of basic residues. Figure 8b shows the darkest blue regions are concentrated strictly along the main diagonal, indicating that most of the attention in this head is focused on each amino acid attending to itself or its immediate neighbors. Unlike Layer 8, Head 16, which captured strong inter-residue attention, especially around repetitive motifs, Layer 11, Head 3 in this map predominantly reinforces local or self-attention, acting conservatively rather than extracting or aggregating broader features. Together, the two heads reveal complementary attention behaviors, one specialized in motif-level enrichment and the other maintaining global positional structure. Download figure Open in new tab Figure 8. The attention heatmap corresponding to the protein sequence containing the top motif “ERKRRRNR” shows the nature of the attention pattern in different layers and heads. (a) Layer 8, Head 16 shows a strong localized attention concentration around the motif region, indicating that this head specifically captures the arginine-rich segment and its surrounding contextual residues (b) Layer 11, Head 3 displays a more diagonal, self-attention–dominant pattern, reflecting position-wise identity attention while still highlighting mild emphasis around the motif. Conclusion In this research, we utilized the pLM ESM2 models (Evolutionary Scale Modeling 2) and ProtBert. The comparison between pre-trained models, full fine-tuned models, and LoRA models revealed the fact that the full fine-tuned and LoRA models achieved better results than the pre-trained models on the account of their ability to adapt more effectively to downstream tasks through fine-tuning. This adaptation allows them to specialize and improve performance on specific target tasks compared to the generalist pre-trained models, which are trained on broad data and not specifically optimized for those tasks. Fine-tuning, including LoRA’s parameter-efficient approach, enables updating model parameters to capture task-specific patterns, thus improving task accuracy and generalization beyond what is achievable by just using the pre-trained representations alone. The full fine-tuning is resourceand time-intensive, whereas LoRA is resource-effective, and the time complexity is slightly better than the full fine-tuning. The full fine-tuned ESM-650M model, on the TF independent set, achieved the highest accuracy of 0.948 ± 0.015 and an MCC score of 0.898 ± 0.030, while the LoRA ESM2-650M model reached an accuracy of 0.943 ± 0.016 and an MCC of 0.887 ± 0.032, showing only slight performance variation. Notably, the LoRA model completes training in less than one-third of the time required by full fine-tuning, compensating for its marginally lower accuracy. This advantage is more pronounced in larger models like ESM2-3B, which cannot be fully fine-tuned on local GPUs due to its size but can be efficiently trained using LoRA with rank 4, demonstrating LoRA’s practical effectiveness for large pLMs. This highlights LoRA’s ability to deliver competitive performance with significantly reduced computational resources compared to full fine-tuning. These results demonstrate the effectiveness of the full and PEFT fine-tuning over the pre-trained models and the machine learning techniques, reflected when compared to the existing methods on both independent datasets (TF and TFPM independent sets). Full fine-tuning of ProtBert using transcription data failed to achieve good results primarily due to optimization difficulties during training. Such failures often stem from vanishing gradients in early training stages, where the model’s parameter updates become very small and the training does not progress effectively. This issue leads to poor convergence and performance stagnation on the new task. Our analysis using the fully fine-tuned ESM2-650M model demonstrates that the attention mechanisms/weights captured by the different heads and layers effectively highlight biologically meaningful amino acid fragments, particularly those corresponding to experimentally validated motifs such as “ERKRRRNR.” By comparing attention patterns with STREME-derived motifs, we show that full fine-tuning enhances the model’s ability to capture functional interactions within transcription factor sequences. The complementary behaviors of different attention heads and layers, with one emphasizing motif-level relationships and another reinforcing local structural consistency, provide interpretable insights into the model’s decision-making process. These findings support the reliability of the proposed framework and underscore the value of attention-based interpretation in protein sequence classification tasks. In the future work, we could involve using larger and more advanced models, like the recent ESM-2 variants, as they have been shown to improve performance in protein-related tasks. Additionally, integrating retrieval-based methods like RAG-ESM or leveraging enhanced structural and functional annotations may further improve model interpretability and accuracy. Addressing the limitations of resource constraints and model complexity, such as developing more efficient or resource-friendly versions, is also a promising direction for future research. Funding This work was supported by the Institute of Information & Communications Technology Planning & Evaluation (IITP)Innovative Human Resource Development for Local Intellectualization program grant funded by the Korea government (MSIT)(IITP-2025-RS-2024-00439292). Data availability The data, source code, and trained models are made available at GitHub at https://github.com/mirtanveer/Fine_Tune_TF . The full and PEFT fine-tuning of the ESM2 models and the inference/prediction on the TF independent dataset using the LoRA-ESM2-650M trained model can be carried out using the Google Colab notebooks attached to the GitHub https://github.com/MirTanveer/Fine-Tune-TF/tree/main/Notebooks . Declaration of Competing Interest No competing interests whatsoever. Supplementary Data Some partial experimental results are described in the supplementary material. Funder Information Declared Institute of Information & Communications Technology Planning & Evaluation (IITP) , IITP-2025-RS-2024-00439292 Footnotes Mir Tanveerul Hassan obtained his M.Tech. in Computer Science from the University of Kashmir, India, in 2020, and later earned his Ph.D. in Electronics and Information Engineering from Jeonbuk National University, Jeonju, South Korea. He is currently serving as a postdoctoral fellow at the Jeonbuk RICE Intelligence Innovation Research Center. His research interests encompass computational biology, bioinformatics, and pattern recognition. Saima Gaffar received her B.Tech. and M.Tech. degrees in Computer Science from the University of Kashmir, Srinagar, India, and her Ph.D. in Electronics and Information Engineering from Jeonbuk National University, South Korea. Her research focuses on bioinformatics, computational biology, deep learning, and image processing. Hamza Zahid received his B.S. degree in Mechatronics Engineering from the University of Engineering and Technology, Peshawar, Pakistan. He is currently pursuing the integrated M.S. and Ph.D. degrees in Electronics and Information Engineering at Jeonbuk National University, South Korea. His primary research interests include the applications of artificial intelligence in computational drug discovery. Sang Jun Lee received his B.S., M.S., and Ph.D. degrees in Electrical Engineering from POSTECH, South Korea. Following his doctoral studies, he worked as a senior researcher at the Samsung Advanced Institute of Technology (SAIT). He is currently an Associate Professor in the Division of Electronics and Information Engineering at Jeonbuk National University, South Korea. His research interests include image analysis, deep learning, and medical image processing. References 1. ↵ Sarah K Kummerfeld and Sarah A Teichmann . Dbd: a transcription factor prediction database . Nucleic acids research , 34 ( Suppl 1 ): D74 – D81 , 2006 . OpenUrl CrossRef PubMed Web of Science 2. ↵ John H Bushweller . Targeting transcription factors in cancer—from undruggable to reality . Nature Reviews Cancer , 19 ( 11 ): 611 – 624 , 2019 . OpenUrl CrossRef PubMed 3. ↵ Yimeng Yin , Ekaterina Morgunova , Arttu Jolma , Eevi Kaasinen , Biswajyoti Sahu , Syed Khund-Sayeed , Pratyush K Das , Teemu Kivioja , Kashyap Dave , Fan Zhong , et al. Impact of cytosine methylation on dna binding specificities of human transcription factors . Science , 356 ( 6337 ): eaaj2239 , 2017 . OpenUrl Abstract / FREE Full Text 4. ↵ Zijie Shen and Quan Zou . Basic polar and hydrophobic properties are the main characteristics that affect the binding of transcription factors to methylation sites . Bioinformatics , 36 ( 15 ): 4263 – 4268 , 2020 . OpenUrl PubMed 5. ↵ Matthew T Weirauch , Ally Yang , Mihai Albu , Atina G Cote , Alejandro Montenegro-Montero , Philipp Drewe , Hamed S Najafabadi , Samuel A Lambert , Ishminder Mann , Kate Cook , et al. Determination and inference of eukaryotic transcription factor sequence specificity . Cell , 158 ( 6 ): 1431 – 1443 , 2014 . OpenUrl CrossRef PubMed Web of Science 6. ↵ Guohua Wang , Ximei Luo , Jianan Wang , Jun Wan , Shuli Xia , Heng Zhu , Jiang Qian , and Yadong Wang . Medreaders: a database for transcription factors that bind to methylated dna . Nucleic acids research , 46 ( D1 ): D146 – D151 , 2018 . OpenUrl PubMed 7. ↵ Qian Liu , Li Fang , Guoliang Yu , Depeng Wang , Chuan-Le Xiao , and Kai Wang . Detection of dna base modifications by deep recurrent neural network on oxford nanopore sequencing data . Nature communications , 10 ( 1 ): 2449 , 2019 . OpenUrl PubMed 8. ↵ Shaohui Hu , Jun Wan , Yijing Su , Qifeng Song , Yaxue Zeng , Ha Nam Nguyen , Jaehoon Shin , Eric Cox , Hee Sool Rho , Crystal Woodard , et al. Dna methylation presents distinct binding sites for human transcription factors . elife , 2 : e00726 , 2013 . OpenUrl CrossRef PubMed 9. ↵ Ishminder K Mann , Raghunath Chatterjee , Jianfei Zhao , Ximiao He , Matthew T Weirauch , Timothy R Hughes , and Charles Vinson . Cg methylated microarrays identify a novel methylated sequence bound by the cebpb— atf4 heterodimer that is active in vivo . Genome research , 23 ( 6 ): 988 – 997 , 2013 . OpenUrl Abstract / FREE Full Text 10. ↵ Sofia Gkountela , Francesc Castro-Giner , Barbara Maria Szczerba , Marcus Vetter , Julia Landin , Ramona Scherrer , Ilona Krol , Manuel C Scheidmann , Christian Beisel , Christian U Stirnimann , et al. Circulating tumor cell clustering shapes dna methylation to enable metastasis seeding . Cell , 176 ( 1 ): 98 – 112 , 2019 . OpenUrl CrossRef PubMed 11. ↵ Huifang Zhang , Qinqin Gao , Shuo Tan , Jia You , Cong Lyu , Yunpeng Zhang , Mengmeng Han , Zhaosu Chen , Jialun Li , Hailin Wang , et al. Set8 prevents excessive dna methylation by methylation-mediated degradation of uhrf1 and dnmt1 . Nucleic Acids Research , 47 ( 17 ): 9053 – 9068 , 2019 . OpenUrl CrossRef PubMed 12. ↵ Shasha Yin , Liu Liu , Charles Brobbey , Viswanathan Palanisamy , Lauren E Ball , Shaun K Olsen , Michael C Ostrowski , and Wenjian Gan . Prmt5-mediated arginine methylation activates akt kinase to govern tumorigenesis . Nature communications , 12 ( 1 ): 3444 , 2021 . OpenUrl PubMed 13. ↵ Huamin Wang , Xiang Hu , Mingyan Huang , Juan Liu , Yan Gu , Lijia Ma , Qi Zhou , and Xuetao Cao . Mettl3-mediated mrna m6a methylation promotes dendritic cell activation . Nature communications , 10 ( 1 ): 1898 , 2019 . OpenUrl PubMed 14. ↵ Emmanuelle Roulet , Stéphane Busso , Anamaria A Camargo , Andrew JG Simpson , Nicolas Mermod , and Philipp Bucher . High-throughput selex–sage method for quantitative modeling of transcription-factor binding sites . Nature biotechnology , 20 ( 8 ): 831 – 835 , 2002 . OpenUrl CrossRef PubMed Web of Science 15. ↵ Sylvie Rockel , Marcel Geertz , and Sebastian J Maerkl . Mitomi: a microfluidic platform for in vitro characterization of transcription factor–dna interaction . Gene Regulatory Networks: Methods and Protocols , pages 97 – 114 , 2012 . 16. ↵ Takuya Yashiro , Mutsuko Hara , Hideoki Ogawa , Ko Okumura , and Chiharu Nishiyama . Critical role of transcription factor pu. 1 in the function of the ox40l/tnfsf4 promoter in dendritic cells . Scientific reports , 6 ( 1 ): 34825 , 2016 . OpenUrl PubMed 17. ↵ Edgar Wingender , Peter Dietze , Holger Karas , and Rainer Knüppel . Transfac: a database on transcription factors and their dna binding sites . Nucleic acids research , 24 ( 1 ): 238 – 241 , 1996 . OpenUrl CrossRef PubMed Web of Science 18. ↵ Diego Mauricio Rianõ-Pachón, Slobodan Ruzicic , Ingo Dreyer , and Bernd Mueller-Roeber . Plntfdb: an integrative plant transcription factor database . BMC bioinformatics , 8 : 1 – 10 , 2007 . OpenUrl CrossRef PubMed 19. ↵ Qi-Hui Zhu , An-Yuan Guo , Ge Gao , Ying-Fu Zhong , Meng Xu , Minren Huang , and Jinchu Luo . Dptf: a database of poplar transcription factors . Bioinformatics , 23 ( 10 ): 1307 – 1308 , 2007 . OpenUrl CrossRef PubMed Web of Science 20. ↵ Qiong Zhang , Wei Liu , Hong-Mei Zhang , Gui-Yan Xie , Ya-Ru Miao , Mengxuan Xia , and An-Yuan Guo . htftarget: a comprehensive database for regulations of human transcription factors and their targets . Genomics, Proteomics and Bioinformatics , 18 ( 2 ): 120 – 128 , 2020 . OpenUrl 21. ↵ Meng-Lu Liu , Wei Su , Jia-Shu Wang , Yu-He Yang , Hui Yang , and Hao Lin . Predicting preference of transcription factors for methylated dna using sequence information . Molecular Therapy-Nucleic Acids , 22 : 1043 – 1050 , 2020 . OpenUrl 22. ↵ Quang H Nguyen , Hoang V Tran , Binh P Nguyen , and Trang TT Do . Identifying transcription factors that prefer binding to methylated dna using reduced g-gap dipeptide composition . ACS omega , 7 ( 36 ): 32322 – 32330 , 2022 . OpenUrl PubMed 23. ↵ Peijie Zheng , Yue Qi , Xueyong Li , Yuewu Liu , Yuhua Yao , and Guohua Huang . A capsule network-based method for identifying transcription factors . Frontiers in Microbiology , 13 : 1048478 , 2022 . OpenUrl PubMed 24. ↵ Sara Sabour , Nicholas Frosst , and Geoffrey E Hinton . Dynamic routing between capsules . Advances in neural information processing systems , 30 , 2017 . 25. ↵ Hongfei Li , Yue Gong , Yifeng Liu , Hao Lin , and Guohua Wang . Detection of transcription factors binding to methylated dna by deep recurrent neural network . Briefings in bioinformatics , 23 ( 1 ): bbab533 , 2022 . OpenUrl PubMed 26. ↵ Saima Gaffar , Kil To Chong , and Hilal Tayara . Tfprotbert: Detection of transcription factors binding to methylated dna using protbert latent space representation . International Journal of Molecular Sciences , 26 ( 9 ): 4234 , 2025 . OpenUrl PubMed 27. ↵ Alex Graves and Jürgen Schmidhuber . Framewise phoneme classification with bidirectional lstm and other neural network architectures . Neural networks , 18 ( 5-6 ): 602 – 610 , 2005 . OpenUrl CrossRef PubMed 28. ↵ Ying Huang , Beifang Niu , Ying Gao , Limin Fu , and Weizhong Li . Cd-hit suite: a web server for clustering and comparing biological sequences . Bioinformatics , 26 ( 5 ): 680 – 682 , 2010 . OpenUrl CrossRef PubMed Web of Science 29. ↵ Quan Zou , Gang Lin , Xingpeng Jiang , Xiangrong Liu , and Xiangxiang Zeng . Sequence clustering in bioinformatics: an empirical study . Briefings in bioinformatics , 21 ( 1 ): 1 – 10 , 2020 . OpenUrl PubMed 30. ↵ Thomas D Schneider and R Michael Stephens . Sequence logos: a new way to display consensus sequences . Nucleic acids research , 18 ( 20 ): 6097 – 6100 , 1990 . OpenUrl CrossRef PubMed Web of Science 31. ↵ S. F. Altschul , T. L. Madden , A. A. Schäffer , J. Zhang , Z. Zhang , W. Miller , and D. J. Lipman . Gapped blast and psi-blast: a new generation of protein database search programs . Nucleic Acids Research , 25 ( 17 ): 3389 – 3402 , 1997 . OpenUrl CrossRef PubMed Web of Science 32. ↵ Kuo-Chen Chou and Hong-Bin Shen. Memtype-2l: A web server for predicting membrane proteins and their types by incorporating evolution information through pse-pssm . Biochemical and Biophysical Research Communications , 360 ( 2 ): 339 – 345 , 2007 . OpenUrl CrossRef PubMed Web of Science 33. ↵ Roshan Rao , Jason Liu , Tristen Verkuil , John Meier , John Canny , Pieter Abbeel , and Tom Sercu . Msa transformer . In International Conference on Learning Representations , 2021 . 34. ↵ Colin Raffel , Noam Shazeer , Adam Roberts , Katherine Lee , Sharan Narang , Michael Matena , Yanqi Zhou , Wei Li , and Peter J Liu . Exploring the limits of transfer learning with a unified text-to-text transformer . Journal of Machine Learning Research , 21 ( 140 ): 1 – 67 , 2020 . OpenUrl PubMed 35. ↵ Jacob Devlin , Ming-Wei Chang , Kenton Lee , and Kristina Toutanova . Bert: Pre-training of deep bidirectional transformers for language understanding . In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics , pages 4171 – 4186 . Association for Computational Linguistics , 2019 . 36. ↵ Zeming Lin , Halil Akin , Roshan Rao , Brian Hie , Zhongkai Zhu , Wenting Lu , Nikita Smetanin , Robert Verkuil , Ori Kabeli , Yaniv Shmueli , et al. Evolutionary-scale prediction of atomic-level protein structure with a language model . Science , 379 ( 6637 ): 1123 – 1130 , 2023 . OpenUrl CrossRef PubMed 37. ↵ Edward J. Hu , Yelong Shen , Phillip Wallis , Zeyuan AllenZhu , Yuanzhi Li , Shean Wang , Lu Wang , and Weizhu Chen . Lora: Low-rank adaptation of large language models . arXiv preprint arxiv: 2106.09685 , 2021 . 38. ↵ Maryam Rasool , Kil To Chong , and Hilal Tayara . A multimodule graph-based neural network for accurate drugtarget interaction prediction via genomic, proteomic, and structural data fusion . International Journal of Biological Macromolecules , page 145907 , 2025 . 39. ↵ Sabir Ali , Waleed Alam , Hilal Tyara , and Kil To Chong . An accurate prediction of drug–drug interactions and side effects by using integrated convolutional and bilstm networks . Chemometrics and Intelligent Laboratory Systems , 257 : 105304 , 2025 . OpenUrl 40. ↵ Mir Tanveerul Hassan , Hilal Tayara , and Kil To Chong . Meta-il4: An ensemble learning approach for il-4-inducing peptide prediction . Methods , 217 : 49 – 56 , 2023 . OpenUrl CrossRef PubMed 41. ↵ Mir Tanveerul Hassan , Hilal Tayara , and Kil To Chong . ianoxpep: a machine learning model for the identification of anti-oxidative peptides using ensemble learning . IEEE/ACM Transactions on Computational Biology and Bioinformatics , 2024 . 42. ↵ Saima Gaffar , Mir Tanveerul Hassan , Hilal Tayara , and Kil To Chong . If-aip: a machine learning method for the identification of anti-inflammatory peptides using multifeature fusion strategy . Computers in Biology and Medicine , 168 : 107724 , 2024 . OpenUrl PubMed 43. ↵ Waqar Ahmad , Hilal Tayara , and Kil To Chong . Attention-based graph neural network for molecular solubility prediction . ACS omega , 8 ( 3 ): 3236 – 3244 , 2023 . OpenUrl PubMed 44. ↵ Bilal Ahmad Mir , Hilal Tayara , and Kil To Chong . Sb-net: Synergizing cnn and lstm networks for uncovering retrosynthetic pathways in organic synthesis . Computational Biology and Chemistry, page 108130 , 2024 . 45. ↵ Muhammad Farhan , Hassan Eesaar , Afaq Ahmed , Kil To Chong , and Hilal Tayara . Transforming highway safety with autonomous drones and ai: A framework for incident detection and emergency response . IEEE Open Journal of Vehicular Technology , 2025 . 46. ↵ Jiho Choi , Gyeongyeon Hwang , Yewon Ji , Hakyoung Yoon , and Sang Jun Lee . Pfsh-net: Parallel frequency-spatial hybrid network for segmentation of kidney stones in precontrast computed tomography images of dogs . Computers in Biology and Medicine , 186 : 109609 , 2025 . OpenUrl PubMed 47. ↵ Gyutae Hwang , Jiwoo Jeong , and Sang Jun Lee . Sfa-net: Semantic feature adjustment network for remote sensing image segmentation . Remote Sensing , 16 ( 17 ): 3278 , 2024 . OpenUrl 48. ↵ Basir Akbar , Hilal Tayara , and Kil To Chong . Unveiling dominant recombination loss in perovskite solar cells with a xgboost-based machine learning approach . Iscience , 2024 . 49. ↵ Hamza Zahid , Kil To Chong , and Hilal Tayara . Integrating graph convolution and attention mechanism for kinase inhibition prediction . Molecules , 30 ( 13 ): 2871 , 2025 . OpenUrl PubMed 50. ↵ Basir Akbar , Kil To Chong , and Hilal Tayara . Ensemble insights: Unlocking the recombination losses in perovskite solar cells using stacked classifier . Engineering Applications of Artificial Intelligence , 153 : 110909 , 2025 . OpenUrl 51. ↵ Mir Tanveerul Hassan , Hilal Tayara , and Kil To Chong . An integrative machine learning model for the identification of tumor t-cell antigens . BioSystems , 237 : 105177 , 2024 . OpenUrl PubMed 52. ↵ Mir Tanveerul Hassan , Hilal Tayara , and Kil To Chong . Naii-pred: An ensemble-learning framework for the identification and interpretation of sodium ion inhibitors by fusing multiple feature representation . Computers in Biology and Medicine , 178 : 108737 , 2024 . OpenUrl CrossRef PubMed 53. ↵ Mir Tanveerul Hassan , Hilal Tayara , and Kil To Chong . Possum: identification and interpretation of potassium ion inhibitors using probabilistic feature vectors . Archives of Toxicology , pages 1 – 11 , 2024 . 54. ↵ Saima Gaffar , Hilal Tayara , and Kil To Chong . Stackaagp: Computational prediction and interpretation of anti-angiogenic peptides using a meta-learning framework . Computers in Biology and Medicine , 174 : 108438 , 2024 . OpenUrl PubMed 55. ↵ Hamza Zahid , Hilal Tayara , and Kil To Chong . Harnessing machine learning to predict cytochrome p450 inhibition through molecular properties . Archives of Toxicology , pages 1 – 12 , 2024 . 56. ↵ Ting Li , Henghui Fan , Jianping Zhao , Xiaomei Yang , and Junfeng Xia . Multipep-dlcl: recognition of multifunctional therapeutic peptides through deep learning with labelsequence contrastive learning . Briefings in Bioinformatics , 26 ( 3 ): bbaf274 , 2025 . OpenUrl PubMed 57. ↵ Yitian Fang , Mingshuang Luo , Zhixiang Ren , Leyi Wei , and Dong-Qing Wei . Cela-mfp: a contrast-enhanced and label-adaptive framework for multi-functional therapeutic peptides prediction . Briefings in Bioinformatics , 25 ( 4 ), 2024 . 58. ↵ Timothy L Bailey . Streme: accurate and versatile sequence motif discovery . Bioinformatics , 37 ( 18 ): 2834 – 2840 , 2021 . OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted November 28, 2025. Download PDF Supplementary Material Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Fine-Tuning Protein Language Models Enhances the Identification and Interpretation of the Transcription Factors Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Fine-Tuning Protein Language Models Enhances the Identification and Interpretation of the Transcription Factors Mir Tanveerul Hassan , Saima Gaffar , Hamza Zahid , Sang Jun Lee bioRxiv 2025.11.27.691010; doi: https://doi.org/10.1101/2025.11.27.691010 Share This Article: Copy Citation Tools Fine-Tuning Protein Language Models Enhances the Identification and Interpretation of the Transcription Factors Mir Tanveerul Hassan , Saima Gaffar , Hamza Zahid , Sang Jun Lee bioRxiv 2025.11.27.691010; doi: https://doi.org/10.1101/2025.11.27.691010 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Bioinformatics Subject Areas All Articles Animal Behavior and Cognition (7618) Biochemistry (17633) Bioengineering (13856) Bioinformatics (41841) Biophysics (21399) Cancer Biology (18529) Cell Biology (25422) Clinical Trials (138) Developmental Biology (13352) Ecology (19860) Epidemiology (2067) Evolutionary Biology (24282) Genetics (15582) Genomics (22462) Immunology (17700) Microbiology (40295) Molecular Biology (17140) Neuroscience (88419) Paleontology (666) Pathology (2823) Pharmacology and Toxicology (4813) Physiology (7632) Plant Biology (15107) Scientific Communication and Education (2042) Synthetic Biology (4284) Systems Biology (9808) Zoology (2267)