Why variant effect predictors and multiplexed assays agree and disagree

Why variant effect predictors and multiplexed assays agree and disagree | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Why variant effect predictors and multiplexed assays agree and disagree View ORCID Profile Benjamin J. Livesey , View ORCID Profile Joseph A. Marsh doi: https://doi.org/10.1101/2025.07.31.667868 Benjamin J. Livesey 1 MRC Human Genetics Unit, Institute of Genetics and Cancer, University of Edinburgh , Edinburgh, UK Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Benjamin J. Livesey For correspondence: blivesey{at}exseed.ed.ac.uk joseph.marsh{at}ed.ac.uk Joseph A. Marsh 1 MRC Human Genetics Unit, Institute of Genetics and Cancer, University of Edinburgh , Edinburgh, UK Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Joseph A. Marsh For correspondence: blivesey{at}exseed.ed.ac.uk joseph.marsh{at}ed.ac.uk Abstract Full Text Info/History Metrics Supplementary material Preview PDF Abstract Computational variant effect predictors (VEPs) and multiplexed assays of variant effect (MAVEs) are two key tools for assessing the functional consequences of genetic variants. While their outputs are often concordant, there are also many differences. Here, we analyse missense MAVE data from 37 different human proteins, comparing them to five state-of-the-art VEPs in order to quantify and explain their points of agreement and disagreement. We find that discordance is not random but reflects fundamental differences in how each method infers functional impact. VEPs, which rely heavily on sequence conservation and basic structural features, tend to overcall pathogenicity at buried and hydrophobic residues, while underestimating impact in disordered regions and on charged surface residues. MAVEs, by contrast, capture context-specific mechanisms more accurately, but can miss pathogenic variants when the assay fails to reflect disease biology, or be subject to high levels of experimental noise. By comparing both global patterns and specific clinically relevant variants, we show how protein features, assay design, and variant type shape predictive discordance. Our findings provide a framework for interpreting when and why VEPs and MAVEs diverge and point toward strategies for improving variant interpretation through integrated, mechanism-aware approaches. Introduction Genetic sequencing is becoming an ever more useful tool for medical diagnosis as advances have continued to make it more accessible and less expensive to perform. Unfortunately, our understanding of the mechanisms underlying the pathogenicity of the many novel variants being identified in disease-relevant proteins has not experienced the same amount of progress. This has led to a widening gap between the number of variants observed in sequencing data and the number with assigned clinical interpretations. As of December 2024, 57% of the single nucleotide variants (SNVs) reported in ClinVar ( Landrum et al , 2018 ) that can be mapped to a UniProt primary isoform ( The UniProt Consortium, 2025 ) protein sequence are of uncertain clinical significance, and that percentage is likely to rise as more sequencing is carried out. This also represents only a small fraction of SNVs possible in the sequence space of the human proteome. Assays for protein variant function are a commonly employed tool to provide evidence for variant classification, however single-variant functional assays are too expensive and time-consuming to adequately fill the gap between the number of novel variants being observed and the need to identify those of clinical significance. To this end, advances have been made in the development and refinement of alternative high-throughput strategies to identify potentially pathogenic variants in the human population. One such approach is the use of computational variant effect predictors (VEPs), also sometimes known as pathogenicity predictors. VEPs make use of various features such as sequence conservation, structural context and residue properties to predict the likelihood of a variant affecting protein function. While the methodology behind VEPs is diverse, they can be roughly divided into three classes based on their use of human population data. The largest class of VEPs is clinical-trained predictors. These VEPs include human population data in their models, often through training on labelled variants from databases such as ClinVar and HGMD ( Stenson et al , 2020 ) but also potentially by exposure to human allele frequency data, often as a predictive feature. While clinical-trained predictors tend to perform well in benchmarks, they are also the most vulnerable to data circularity , and thus measurements of their performance may be inflated ( Grimm et al , 2015 ). Population-free VEPs have been steadily improving in performance over the past several years. These methods make no use of human population data either as training labels or features, often putting more emphasis on evolutionary conservation and structural features of proteins. Population-free predictors are not vulnerable to data circularity in their assessment and are competitive with clinical trained supervised predictors at the cutting edge ( Meier et al , 2021 ; Laine et al , 2019 ; Frazer et al , 2021 ). The third category, population-tuned VEPs make some use of human population data for model tuning or optimisation, but do not use it as training labels or include allele frequencies as a feature. This is the smallest of the three classes and may be somewhat vulnerable to data circularity depending on the exact model training procedures, although the risk is smaller than with clinical-trained models. Compared to wet-lab functional characterisation, VEPs are highly cost effective (largely free-to-use) and produce results extremely rapidly, making them suitable for large-scale variant prioritisation. Determining the accuracy and reliability of VEP predictions is the subject of ongoing research like the Critical Assessment of Genome Interpretation (CAGI) ( Critical Assessment of Genome Interpretation Consortium, 2024 ), and is highly dependent on the selected benchmark dataset ( Livesey & Marsh, 2025 ). Another approach is to use multiplexed assays of variant effect (MAVEs) ( Fowler & Fields, 2014 ), a general term for a high-throughput assay in which the relative function of large numbers of genetic variants can be measured. MAVEs that focus on evaluating the relative fitness of protein coding missense variants may also be referred to as deep mutational scanning (DMS). In a typical MAVE for a protein coding gene, a library of variants is created in the gene of interest and then used to transform cells in which the function of the resultant mutant protein is linked to some measurable attribute such as cellular growth rate or reporter gene expression. The assay used to determine variant fitness can be fine-tuned to the specific protein or disease being assessed, for example, DNA binding activity for the PAX6 transcription factor ( McDonnell et al , 2024 ) or protease activity of the caspase proteins ( Roychowdhury & Romero, 2022 ). Many MAVEs use the relative growth rate of the variant-bearing cells as a proxy for fitness, if the function of the target gene is essential ( Weile et al , 2017 ; Jiang, 2019 ; Sun et al , 2020 ). The relative fitness of each variant can be calibrated by using the measurements of null and wild type variants. Unlike VEPs, MAVEs represents a true measurement of protein activity or abundance in the cell following mutation, however it takes considerable expertise, time and specialist equipment to carry out, although it is considerably less time consuming and expensive than performing separate functional assays for each individual variant. Several MAVEs have reported extremely good performance for the classification of variants from ClinVar ( Findlay et al , 2018 ; Grønbæk-Thygesen et al , 2024 ; Gersing et al , 2023 ). Both MAVEs and VEPs are different tools that can be used to address the same issue: the annotation of variants of uncertain clinical significance (VUS). Indeed, in the American College of Medical Genetics and Association for Molecular Pathology (ACMG/AMP) guidelines for variant interpretation, the two different annotation sources can be combined with one another to improve evidence strength ( Richards et al , 2015 ). Currently VEPs are only capable of providing supporting evidence for pathogenicity while functional assays such as MAVEs can provide up to strong evidence. Significant advances in both fields have been made since the 2015 guidelines were published, and work is under way to produce a set of guidelines with notably improved evidence strengths for VEPs ( Pejaver et al , 2022 ; Badonyi & Marsh, 2025 ). With the understanding that evidence for the reclassification of many VUS are likely to come from both MAVE and VEP sources in the near future, it is important to know how the two approaches differ for the purpose of combining evidence from them. Can we really treat functional measurements from a MAVE and functional predictions from a VEP as independent of each other if they are both attempting to estimate the same value? We have recently shown that VEPs, in particular recently developed population-free methods show a strong correlation with MAVE datasets across numerous proteins ( Livesey & Marsh, 2025 ). We used this relationship as the basis of three VEP benchmarking studies, highlighting the progress in VEP development across several years. The best-performing predictors approached a Spearman’s correlation of 0.8 with some MAVE datasets. While both approaches are attempting to address the same fundamental questions of variant effects, the extremely different methodologies employed are naturally going to result in contradicting predictions in some proteins and protein regions. It is important both for future computational method development and integration of the two methods as separate lines of evidence to know where and why the two approaches disagree on the effects of variants on protein function. In this study we compare and contrast the performances of 37 MAVE datasets and 5 representative VEPs to determine the extent and causes of discordance between the predictions made by the different approaches. We identify protein regions in which MAVEs and VEPs tend to differ in their predictions and general properties that may be predictive of such regions in other proteins. Finally, we assess several cases of known pathogenic and benign variants where MAVEs and VEPs produce contradicting results and deduce the reasons for the discrepancies. Results and discussion MAVE and VEP dataset selection and normalisation The MAVE datasets used in this study are the same 36 MAVE datasets we previously used to benchmark 96 VEPs ( Livesey & Marsh, 2025 ) with the addition of a BRCA1 dataset ( Findlay et al , 2018 ), which was excluded from the benchmark due to the large overlap between scored variants and variant databases ( Table 1 ). The mutagenesis technique, fitness assay, cellular model system and general technique employed by the MAVE experiments is diverse, although yeast-based functional complementation assays are one of the most common ( Weile et al , 2017 ; Gersing et al , 2023 ; Sun et al , 2020 ). Other notable experimental setups include VAMP-seq assays for protein stability ( Clausen et al , 2024 ; Matreyek et al , 2018 ), a saturation genome editing approach in a human cell line ( Findlay et al , 2018 ), binding assays ( Bandaru et al , 2017 ; Weng et al , 2024 ) and several unique assays tailored for protein-specific functions ( Miller et al , 2022 ; Amorosi et al , 2021 ). These datasets were curated to ensure that the fitness assay employed is relevant to normal protein function or stability. A single score-set is selected to be representative of each MAVE target protein, even in cases where multiple groups produced results for a single protein. The selected score set is the one with the highest median correlation to 96 VEPs in the benchmark study. View this table: View inline View popup Table 1. Summary of 37 MAVE datasets. Compared to modern VEPs; older, well-known VEPs such as SIFT and PolyPhen-2 are largely redundant, so we selected five state-of-the-art VEPs to perform our analysis. We previously found each of these methods to be either immune or highly resistant to the phenomenon of data circularity, so making comparisons using annotated disease variants should not materially affect the results. AlphaMissense is an advanced language model built on AlphaFold2 that makes use of protein structure and is tuned with human population frequencies ( Cheng et al , 2023 ). ESCOTT is a combination of the conservation-based statistical GEMME model ( Laine et al , 2019 ) and further structural features ( Tekpinar et al , 2024 ). ESM-1v is a pure language model designed specifically for zero-shot variant effect prediction (i.e. no protein-specific training) ( Meier et al , 2021 ). SaProt is a model that combines the 3D structural context of residues with their sequence context, producing a hybrid sequence/structure language model ( Su et al , 2024 ). VARITY_R is a top-performing clinical-trained predictor that is relatively insensitive to data circularity and sources training data from several databases ( Wu et al , 2021 ). Together, these five predictors represent a diverse methodological cross-section of VEP techniques at the cutting edge of current methodology. They are also all either population-free predictors or, in the case of VARITY and AlphaMissense, not particularly sensitive to data circularity. This is important as data circularity may influence VEP outputs based on previously encountered clinical data, potentially skewing comparisons with MAVE data, especially in the context of previously observed clinically relevant variants. MAVE and VEP performance is heterogenous across proteins. To demonstrate this, we calculated the area under the receiver operating characteristic curve (AUROC) for each of the five selected predictors in addition to the MAVE data. We used ClinVar pathogenic variants with a 1-star or higher review status as the pathogenic dataset and gnomAD as the putatively benign dataset. While gnomAD is not a purely benign set of variants, we refer to it as ‘putatively benign’ as it provides a representation of genetic variation in a healthy human population. While gnomAD certainly contains recessive and low-penetrance pathogenic variants, particularly among variants with low allele frequency, it better reflects the real-life clinical application of VEPs than other purely benign alternatives, that being distinguishing rare pathogenic variants from rare benign variants. Datasets like ClinVar-benign, as well as containing far fewer variants, are primarily composed of common variants in the human population, as allele frequency is a major determining factor in the benign classification criteria. The AUROC calculation was bootstrapped 1000 times by sampling with replacement from both datasets independently to obtain a distribution ( Figure 1 ). Depending on the protein and/or MAVE assay, the MAVE data can either perform worse than the VEPs ( HMBS , GCK , MTHFR ), similarly to the VEPs ( GCH1 , PAX6 , PPARG ) or better ( MSH2 , TP53 , BRCA1 ) in separating the pathogenic variants from the putatively benign. The five VEPs selected for this analysis have similar levels of performance, and their outputs also strongly correlate with one another. They have a much lower, although still significant, correlation to the MAVE assay data ( Figure S1 ). This demonstrates that VEPs are much more similar to each other than to the MAVEs, despite their varying methodologies and are likely to identify similar types of pathogenic variants to one another. Download figure Open in new tab Figure 1. The area under the receiver operating characteristic curve (AUROC) for the five VEPs and MAVE dataset across nine proteins. The pathogenic dataset was comprised of ClinVar pathogenic and likely pathogenic variants with a review status of 1 star or higher. The putatively benign dataset was composed of gnomAD variants not present in the pathogenic set. Only proteins with at least 10 variants from each dataset were considered. The distributions plotted were produced by bootstrapping the AUROC calculation 1000 times, sampling with replacement independently from both datasets. The same variants were used to calculate the AUROC for each VEP and the MAVE data on each bootstrap. The first challenge in characterising areas of proteins where VEPs and MAVEs disagree is making the various scoring systems employed comparable. Most commonly, VEPs assign scores on a 0-1 scale, where the number indicates the probability of a mutation being damaging to protein function, however this is far from the only scoring system. MAVE datasets most frequently use a relative fitness scale with wild-type-like variants scoring 1 and null variants 0, which is the opposite of the most common VEP scale but also allows for worse-than-null and fitter-than-wild type variants unlike most VEPs. Comparing scores that use such a diverse range of scales is challenging, which is why we decided to use normalised rank scores. To perform normalised ranking, we took each score scale and first ensured that lower values represent the fitter variants by deducting every score from 1 if the scale was inverted. We then ordered the scores by value and assigned each variant a rank on a per-protein basis. The ranks were then normalised between 0 and 1, providing a simple and comparable scale for all datasets. In cases where multiple scores are being compared and some are missing data, the missing data is removed from all score sets before applying the normalisation, ensuring that rankings are always performed with the same variants in each score set. A comparison can be made between any two MAVE or VEP score sets for a particular protein by deducting one set of normalised rank scores from the other, resulting in values close to 0 where the two sets agree perfectly and values up to 1 or -1 in the case of extreme discordance between the scores. Regions of discordance as measured by the difference between normalised rank scores can then be associated with protein structural features, sites and domains, providing insight into the features of proteins that MAVEs and VEPs are most likely to disagree on. This approach is illustrated in Figure 2 , where predictions from the ESM-1v VEP are compared to a set of deep mutational scan results for the low density lipoprotein receptor adaptor protein 1 ( Jiang, 2019 ), and plotted alongside important protein features. Download figure Open in new tab Figure 2. An illustration of how comparisons are made between VEP and MAVE datasets on different scales. A . The MAVE dataset for the protein LDLRAP1 is converted into ranks and normalised between 0 and 1. B . The predictions from ESM-1v for LDLRAP1 are also converted into ranks and normalised between 0 and 1. C. The data in section (B) is deducted from the data in section (A) to give the difference in normalised ranks across the protein. The black line represents a moving average across the protein for a window of 100 variants (∼5 residues) with a stride size of 25 variants. Higher scores represent variants that the MAVE finds to be more impactful on protein function than the MAVE, while lower scores indicate the opposite. D . The mean of the normalised difference, averaged across all five VEPs (black line) is compared to the location of protein features (top) and disordered regions as indicated by AlphaFold pLDDT (bottom). The first notable difference between MAVE and VEP datasets can be observed in the distribution of normalised rank scores. As illustrated in Figure 2A and 2B , scores for different amino acid substitutions at each protein position in the MAVE data tend to be dispersed across a wider range of values than for the ESM-1v predictor. This pattern remains consistent across other VEPs and MAVE datasets ( Figure S2 ) with normalised MAVE rank scores showing greater standard deviation at each residue position and also across small windows of 10 amino acids than the VEPs. VARITY was excluded from this figure, as it only produces predictions for variants possible by single nucleotide substitutions (∼30% of all variants) and thus naturally demonstrates lower positional deviation than the other VEPs. Interestingly one dataset stood out as having particularly low standard deviation per-position, the MAVE set for PPARG ( Majithia et al , 2016 ), with an average standard deviation of only 0.0763, compared to the MAVE average of 0.196 (excluding PPARG ). Insight into amino acid level predictions It is well known that mutations involving certain amino acids are much more likely to be damaging to protein function than others ( Petukh et al , 2015 ). For example, aromatic amino acids are very bulky and hydrophobic, often driving the formation of the hydrophobic core of globular proteins. Mutations of these large residues to those with different properties, or inclusion of these residues on the protein surface can lead to altered folding pathways or misfolding ( Chiti & Dobson, 2006 ). Cysteine residues are vital in the formation of disulphide bridges, particularly in extracellular proteins and those localised to oxidising compartments such as the endoplasmic reticulum. Additionally, the unique conformational constraints of proline and the flexibility of glycine mean that substitution at these positions often lead to significant changes in protein backbone structure. We investigated the differences between MAVEs and VEPs in terms of their tendency to label mutations involving specific amino acids as more or less damaging to protein function. To perform this analysis, we pooled data from the five representative VEPs and compared them to the MAVE data. Our results indicate that there are some significant differences between the tendency of VEPs and MAVEs to identify specific amino acid changes as damaging. We present the Hedges G effect size of the differences in the normalised rank scores for the pooled MAVE data against the pooled VEP data for every combination of wild type and mutant amino acid in Figure 3 as a heatmap. Positive (red) effect sizes indicate variants that the MAVE tends to label as more damaging than the VEP while negative (blue) effect sizes indicate variants that the VEPs identify as more damaging. The intensity of the colour indicates the magnitude of the effect. Download figure Open in new tab Figure 3. A heatmap of the effect sizes between normalised VEP and MAVE ranks for all amino acid substitutions. Positive, red effect sizes indicate substitutions that MAVEs fund more pathogenic than the VEPs while negative, blue effect sizes indicate substitutions that VEPs find more pathogenic than the MAVEs. White squares indicate substitutions where both methods are concordant. Synonymous mutations are not included in this analysis and the grey boxes indicate blank squares. There are several interesting trends that can be observed on this heatmap. First, the plot is generally mirrored along the diagonal, indicating a consistent magnitude and direction of the effect size regardless of the orientation of the wild type and mutant residue. I.E., an isoleucine to valine mutation and a valine to isoleucine mutation are both predicted more damaging by MAVEs than VEPs by about the same amount on average. The upper left quadrant of the heatmap, representing substitutions between charged and polar residues shows almost all positive effect sizes (except cysteine). This indicates that MAVEs tends to find these substitutions between residues with somewhat similar properties to be more damaging than VEPs do. Similarly, there is also a cluster of small, hydrophobic residues on the bottom right (I, L, V, A) where substitutions between them also demonstrate a similar positive effect size and thus a tendency to be labelled as more damaging by MAVE data than by VEPs. Almost any mutation involving the bulky, hydrophobic amino acids (W, Y, F) as either the wild type or the mutant results in negative effect sizes and are thus found more damaging by the VEPs than by the MAVE. A similar effect can also be observed in the bottom left quadrant, representing variants where small, hydrophobic residues are mutated to charged or polar ones. The overall pattern is fairly clear: mutations that are considered to be conventionally damaging to protein function, i.e., hydrophobic to charged, small to bulky or bulky to small are routinely predicted as more damaging by the VEPs than MAVEs. Mutations that are conventionally considered as less likely to be disease-causing, i.e., charged to charged or small/hydrophobic to small/hydrophobic are found to be more damaging by the MAVEs than the VEPs. This general pattern is consistent across different subsets of the top-performing VEPs and MAVE datasets, demonstrating that the trend is not being driven by a small set of MAVE datasets or VEPs. Our speculation as to the reason for this pattern is as follows. MAVEs make no prior assumption about the effects of amino acid substitutions but bases the outcome purely on an independent fitness assay that is unique for each protein and fitness definition. VEPs have to make assumptions (during their training process) about the types of variants that would typically be damaging in order to be generalisable across many different proteins. At the core of most VEPs is a multiple sequence alignment, providing residue-level conservation features to the predictor, that will generally reflect known evolutionary trends of bulky and property-altering substitutions being less favourable than those between residues with similar properties. By learning these trends, VEPs become better able to generalise their predictions across multiple diverse proteins but may be unable to identify occasional exceptions to these trends in different protein contexts that MAVEs are able to identify consistently. In other words, VEPs are likely overcalling substitutions that are typically considered the most damaging and undercalling those that are most often phenotypically neutral while MAVEs are producing a more balanced overview. This also ties in with our observation that MAVEs tends to assign a greater range of scores to variants at a single position in a protein than VEPs do. The prediction made by VEPs is based on their previous experience (training), which includes the fact that some wild type amino acids are less tolerant of certain mutations than others in general. MAVEs are unique for each protein, leading to a larger range of predictions at each position that is unconstrained by prior assumptions of residue-level mutational tolerance. Protein features with discordant VEP/MAVE predictions In each of the 37 MAVE datasets, we identified the subset of variants with the largest gap between the five representative VEPs and MAVEs in terms of normalised rank differences. We term the 5% of variants with the largest normalised rank difference where the VEP assigns the variants as more damaging than the MAVEs as VEP-damaging variants and those the MAVE assigns as more damaging as MAVE-damaging variants. We also identified the 5% of variants with the smallest absolute normalised rank difference, representing those where both MAVE and VEPs concur about the relative functional impact of the variants (concordant variants). VEP-damaging variants are enriched among buried residues and interface residues We used a simple, previously described method to classify residues as belonging to the protein surface, interior or interface and break down interface residues into core, support or rim ( Levy, 2010 ) using biological assemblies in the PDB ( Berman et al , 2000 ). We found previously that interface rim residues resembled protein surface residues more than the interface core or support in terms of properties and pathogenic mutation enrichment ( Livesey & Marsh, 2022 ), so we grouped them with the protein surface residues. VEP-damaging variants occur most frequently in protein interior and interface regions, while MAVE-damaging variants are depleted ( Figure 4A ). The opposite pattern is observed in protein surface residues where VEP-damaging variants are depleted while MAVE-damaging variants are enriched. This finding also ties in with our previous observation that the substitution of bulky hydrophobic residues, which are most commonly found in the hydrophobic core of globular proteins, are often predicted as more damaging by VEPs than MAVEs. Charged and polar residues are more common on the protein surface than the interior, and mutations of these are frequently predicted as more damaging by MAVEs than VEPs. Download figure Open in new tab Figure 4. Characteristics of discordantly predicted mutations. A. The proportion of variants from the VEP-pathogenic, concordant and MAVE-pathogenic classes occurring at the protein surface, interior and interface residues (interface rim residues are included in the surface group). B. The distribution of relevant solvent accessibility within the three classes. C . The distributions of AlphaFold-derived predicted local distance difference test (pLDDT) values within the three classes. D. The proportion of variants from each class that that occur within helices, strands and loops as defined by DSSP. We also calculated the relative solvent accessibility (RSA) of each residue using AlphaFold-predicted structures of every protein and compared the RSA of the three classes ( Figure 4B ). Residues where VEP-damaging mutations occur have a lower RSA than the other two classes, indicating that this class contains the highest proportion of buried residues. MAVE-damaging mutations have the highest RSA and are thus likely primarily surface residues. Those variants where MAVEs and VEPs are concordant are intermediate to the other two classes, likely indicating a mix of interior and surface residues. MAVE-damaging mutations are enriched in disordered regions We obtained predicted local distance difference test (pLDDT) values from AlphaFold2-generated PDB files. pLDDT is a per-residue measurement of confidence in the AlphaFold2 structural prediction, and regions of low pLDDT correspond to disordered regions of proteins. While the median pLDDT for all three groups of variants is above 90, meaning that the residue positions were confidently predicted on average, the MAVE-damaging group has both the lowest median (92.27) and a distribution where the third quartile extends to much lower pLDDT values than the other two groups ( Figure 4C ). It has been observed that VEPs tend to produce poorer predictions of pathogenicity within intrinsically disordered regions of proteins ( Fawzy & Marsh, 2025 ). Disordered regions are usually less well conserved throughout evolution than structured protein regions. Evolutionary conservation is also a vital component of almost all VEPs, and sometimes the only feature employed (particularly for ESM-1v), under the assumption that variants in poorly conserved regions are more likely to be benign. While it is true that disordered regions of proteins are highly enriched in benign variants relative to pathogenic ones, VEPs tend to excessively classify the majority of variants in disordered regions as benign. MAVEs are under no such constraints in disordered regions, so it is likely that MAVE data are closer to the biological reality of variant effects in disordered regions. The observed trend is being driven by VEPs preferentially predicting the majority of variants in disordered regions as benign, resulting in variants found even moderately damaging by the MAVEs being discordantly predicted and appearing in the MAVE-damaging variants. We also used the Define Secondary Structure of Proteins (DSSP) algorithm to classify residues as belonging to a helix, a strand or a loop (no defined secondary structure) ( Figure 4D ). VEP-damaging variants were enriched relative to MAVE-damaging variants in both helices and strands, while MAVE-damaging variants were enriched in loops. This pattern is also likely related to protein disorder, as most disordered regions will be included within the loop class, while helices and sheets are unlikely to harbour disordered regions. Characteristics of discordantly predicted protein regions In order to determine the protein regions that corresponded to the greatest levels of discordance between the VEPs and the MAVEs, we extracted protein feature annotations from UniProt including notable domains, motifs and protein-protein interaction sites, we also used InterPro to annotate all conserved regions and residues from the Conserved Domain Database (CDD). We plotted the mean normalised difference between the MAVE and VEPs across the protein sequence and structure in order to determine the protein features with the greatest discordance between predictions of pathogenicity ( Figure 5 and Supplemental file SF1). Also shown are the pLDDT scores per residue obtained from AlphaFold, which is a useful proxy for sequence disorder. Download figure Open in new tab Figure 5. The correspondence of normalised rank difference to protein sequence and structural features. Each panel displays the Alphafold2-derived structure of the protein with residues coloured by the mean normalised difference between VEPs and MAVEs. Higher (red) values indicate residues where the MAVE considers variants more pathogenic than the VEPs, while lower (blue) values indicate residues where the VEP predicts variants to be more pathogenic than the MAVE. Alongside is a plot, where the rectangles at the top represent protein domains, regions and conserved residues. The black like represents the mean normalised difference between VEPs and DMS within a scanning window covering 100 datapoints (at least 5 residues). The heatmap at the bottom of each plot represents the AlphaFold2 predicted local distance difference test (pLDDT) scores across the protein with blue representing greater confidence in the structure prediction, and red representing lower confidence. A. Structure and plot for HMGCR . B. Structure and plot for CBS C. Structure and plot for GCK . D. Structure and plot for ASPA . Structures and plots for the remaining 33 proteins are available in supplemental file S1. Within all residues covered by each extracted annotation, we calculated the overall average of the mean normalised difference value across all possible variants between the MAVE and the VEPs. Thus, regions with a negative average difference value are predicted to be more pathogenic by the VEPs, and regions with a positive average difference value are found to be more pathogenic by the MAVE. The full results of this analysis are available in Table S1. In several cases, entire protein domains were preferentially predicted as pathogenic by either the MAVE or the VEPs, demonstrating a systematic, domain wide difference between the predictions and measurements made by the two methods. This was most noticeable in the sterol-sensing domain of HMGCR (residues 61-218) which has an average difference of -0.22, indicating that the VEPs identify variants in this region as significantly more pathogenic than the MAVE ( Figure 5A ). A similar effect was notable in the CBS domain of CBS (418-476) where the average difference was - 0.12 ( Figure 5B ). There are also several domains that exhibit a positive average difference, albeit to a lesser extent including the PPARG nuclear receptor ligand binding domain (238-503) with an average difference of 0.07 and the GCK hexokinase small subdomain (67-203) with an average difference of 0.08 ( Figure 5C ). Of the twelve proteins in our dataset with Uniprot-annotated disordered regions present, all had positive average differences within those regions ranging from 0.03 ( GCH1 ) to 0.22 ( CBS, Figure 5B ), indicating that variants in these regions are consistently found more pathogenic by the MAVEs than by the VEPs. Disordered regions of proteins typically have low sequence constraint and harbour notably fewer damaging variants and more functionally neutral ones than ordered regions ( Ahmed et al , 2022 ). These two attributes mean that VEPs tend to perform poorly at identifying the few truly damaging variants present within these regions ( Fawzy & Marsh, 2025 ) as sequence conservation is a core component of almost every VEP (and all of those used for this analysis). Furthermore, the high frequency of neutral variants may also lead to supervised VEPs that have been trained using variants in these regions learning to associate disordered regions with benign protein variation. It is therefore likely that this pattern observed in disordered regions is being driven by poor VEP performance in those regions and a preference to label most variants as benign. MAVEs performance is independent of evolutionary conservation, and thus MAVEs are likely providing more reliable measurements of variant function in regions of disorder. Transmembrane helices are annotated in three of the proteins in our dataset ( HMGCR , VKORC1 and ADRB2 ). All transmembrane regions (with the sole exception of ADRB2 transmembrane helix 4) have a negative difference value, indicating that variants in these regions are predicted as more pathogenic by VEPs than by the MAVEs ( Figure 5A ). Transmembrane helices span the hydrophobic interior of a lipid bilayer and are thus predominantly composed of hydrophobic residues, even more so than helices on the protein interior ( Hildebrand et al , 2004 ). While not classified as such in our protein interface data, transmembrane helices can be thought of as large protein-ligand interfaces. The similar amino acid composition of transmembrane regions to the protein interior is likely resulting in the VEPs treating these features like part of the protein interior where variants that significantly alter properties of hydrophobic residues may have a deleterious impact on protein folding. While variants in transmembrane helices have the potential to impact membrane integration, they are far less likely to affect the normal protein folding pathways than variants in helices within the protein interior and are thus likely less damaging than predicted by the VEPs. The MAVEs likely represent a more balanced overview of the functional impact of variants in transmembrane domains. Conserved active site residues are among the most concordantly predicted residues among all of the MAVE datasets. 15 proteins have annotated active sites within the UniProt and InterPro annotations, and of these, the 9 have average difference values between 0.07 and -0.06 ( HMGCR , SRC , NUDT15 , MAPK1 , PDE3A , PTEN , GCH1 , CASP3 and CASP7 ). This makes sense as active site residues tend to be highly conserved, thus variants would impact both MAVE measurements of activity and VEP predictions. 5 other proteins ( ASPA , PRKN , VKORC1, TPK1 and MTHFR ) all have low negative average difference values at the annotated active sites. For three of these proteins, the enrichment is easily explained by looking at the fitness assay employed during the MAVE assay. PRKN , ASPA and VKORC1 were all assayed used VAMP-seq, a method that assesses relative fitness by quantifying protein expression (identifying structurally disruptive mutations) and does not directly quantify protein activity ( Matreyek et al , 2018 ). Thus, catalytically impaired variants will not be identified by the MAVE assay unless they also cause the protein to misfold. This mismatch between the MAVE readout and the fitness predictions being made by the VEPs demonstrates that caution should be used when employing MAVE as a true fitness standard as the definition of protein fitness in the assay may not match precisely with mechanisms of pathogenicity in the protein. An extreme case of this is shown in Figure 5D for ASPA where variants in residues within and in close proximity to active site are all predicted more pathogenic by the VEPs than by the DMS. In TPK1 and MTHFR , the reasons for the negative average difference at active site residues is less clear. Large average difference values are also notable among dimerisation, protein interaction, metal and ligand binding sites (Table S1). Although the magnitude and direction of the difference varies from protein to protein. Analysis of discordantly predicted clinical variants While it is clear that there are systematic differences between MAVE and VEP measurements of variant effect, it is less clear which approach is producing the more reliable results without referring to a ‘gold standard’. For this purpose, we created two sets of labelled human variants for the 37 proteins. - The Clinvar-pathogenic dataset is composed of human single nucleotide polymorphisms annotated as Pathogenic or Likely pathogenic in the ClinVar database. We retained only records with a 1-star review status or higher. - The Clinvar-benign dataset is constructed similarly to the Clinvar-pathogenic one, but including only variants annotated as Benign and Likely benign. We pooled the normalised rank differences across each of the MAVE datasets. We investigated variants from the clinvar-pathogenic and clinvar-benign datasets with extremely high or low (>0.75 or <-0.75) mean normalised rank differences, being those with the greatest level of discordance between VEPs and MAVEs and attempt to identify a possible cause for the disagreement with respect to its pathogenic/benign label. Figure 6 shows the normalised MAVE scores plotted against the mean of the normalised VEP scores for nine proteins with discordantly predicted variants. We have performed a detailed analysis of the variants in the top left and bottom right of this plot, being those that are most discordantly predicted by MAVEs and VEPs. The normalised rank differences for all Clinvar-pathogenic and Clinvar-benign variants across the 37 MAVE datasets is available in Table S2. Download figure Open in new tab Figure 6. Normalised MAVE score plotted against the mean of the normalised VEP scores for ClinVar-pathogenic and ClinVar benign variants in nine proteins. The labelled variants are those that are most discordantly predicted as measured by the mean normalised difference between MAVEs and VEPs. SNCA α-synuclein is a protein that plays an important role in synaptic activity. Variants in α-synuclein can cause Parkinson’s disease and other neural degenerative conditions collectively called α-synucleinopathies ( Alafuzoff & Hartikainen, 2017 ). Pathogenic variants typically trigger aberrant aggregation of α-synuclein into structures such as Lewy bodies that impair normal cellular functions. The MAVE for α-synuclein activity was performed in yeast cells with the readout being the yeast growth rate, which is impaired by, and thus a proxy for, α-synuclein aggregation ( Newberry et al , 2020 ). While there is only a total of five ClinVar variants in α-synuclein after filtering for entry quality, it contains the ClinVar-pathogenic variant with the highest mean normalised rank difference (0.853), A53T. This is a variant found pathogenic by the MAVE but predicted as benign by all VEPs. The A53T mutation is well known to have a high aggregation potential and is implicated in Parkinson’s disease ( Lee et al , 2002 ). Using an alignment of 1051 sequences related to α-synuclein generated by MMSeqs2 ( Steinegger & Söding, 2017 ), we found that threonine is actually the most common residue at the equivalent position to alanine 53 in the human sequence (401 occurrences vs only 84 for alanine) ( Figure S3A ). The most likely explanation is that VEPs are failing to correctly predict this mutation because the pathogenic mutant residue is common at this position through evolution. Given that this a well-known pathogenic mutation with extensive characterisation, the VEPs are unquestionably incorrect in their assessment of variant effect, likely because this mutation violates their baseline assumption that evolutionary conservation is a good proxy for variant effect. It may be that this variant is only pathogenic in the context of humans due to our relatively long lifespan, given that Parkinson’s disease has a late age-of-onset, typically appearing after the age of 60. BRCA1 Breast cancer type 1 susceptibility protein performs a central role in the homology-directed repair of DNA double-strand breaks ( Moynahan et al , 1999 ). In addition, BRCA1 also participates in non-homologous end-joining DNA repair ( Baldeyron et al , 2002 ) and performs several other functions relevant to maintaining genome stability, it is classed as a tumour suppressor. Pathogenic variants in BRCA1 increase the risk of breast and/or ovarian cancer, typically through loss of protein function. The MAVE for BRCA1 used a saturation genome editing approach to introduce variants into the protein’s N-terminal RING domain and C-terminal BRCT domains in the HAP1 human cell line ( Findlay et al , 2018 ). BRCA1 is an essential gene in HAP1 cells, and loss of function impairs cellular growth, permitting a readout of protein function by measuring cellular growth rate. Of the 245 ClinVar variants in BRCA1 , only three exceed the 0.75 threshold: A1823T, A1823S and D1818G, all of which are pathogenic. The MAVE concurs with the ClinVar classification and also finds all three pathogenic, while the VEPs assign all three as benign. The codon encoding A1823 straddles two exons, so the variants at that location may impact splicing. Pangolin ( Zeng & Li, 2022 ) predicts a 65% chance of splice donor loss on a threonine mutation (NM_007294.4(BRCA1):c.5467G>A (p.Ala1823Thr)) and a 67% chance of splice loss on a serine mutation (NM_007294.4(BRCA1):c.5467G>T (p.Ala1823Ser)). A study has found that D1818G also impacts splicing and results in skipping of exon 22 ( Rouleau et al , 2010 , 22). It is likely the VEPs are failing to account for the impact on splice sites, resulting in incorrect benign predictions for these variants. Because this MAVE was performed using saturation genome editing in a human cell line, the protein was able to be studied in its endogenous context, with native splicing machinery. A similar assay performed in yeast, or even using expression from a plasmid in human cell lines would almost certainly be insensitive to such splice-impacting variants, similarly to the VEPs. SHOC2 Leucine-rich repeat protein SHOC-2 is a scaffold protein and a core component of the SHOC2-MRAS-PP1c (SMP) complex that regulates MAP kinase signalling. Pathogenic mutations in SHOC2 are known to be a cause of several RASopathies, most notably Noonan syndrome-like disorder with loose anagen hair 1 ( Garavelli et al , 2015 ). This condition is typified by characteristic, slow growing, sparce and easily pluckable hair in addition to the developmental, cognitive and cardiac defects that characterise RASopathies. The MAVE for SHOC2 was performed in a KRAS-mutant human cancer cell line by introducing the mutant sequence with a viral vector and knocking out the native SHOC2 gene ( Kwon et al , 2022 ). Fitness was determined by the relative abundance of each variant at the endpoint of the assay. SHOC2 has nine variants in ClinVar, which largely show good concordance with VEP predictions of pathogenicity. The one exception to this is S2G which is a ClinVar pathogenic mutation, found pathogenic by the MAVE, but predicted benign by the VEPs. This particular variant is known to introduce an N-myristoylation site into the protein, resulting in aberrant targeting to the plasma membrane ( Cordeddu et al , 2009 ). With the exception of Serine at position 2, the N-terminus of SHOC2 satisfies the consensus sequence for N-myristoyltransferase targeting. As part of its normal function, SHOC2 is translocated from the cytoplasm into the nucleus under growth factor stimulation, but is unable to effectively do so when targeted to the plasma membrane by N-myrisoylation, explaining the poor variant fitness in the MAVE. It is also unsurprising that VEPs failed to identify this pathogenic gain-of-function mutant given the small number of proteins to which such a site can be introduced by a single missense variant. MSH2 The primary role of DNA mismatch repair protein Msh2 is to locate and repair DNA mismatches, which is a function it performs as a heterodimer with MSH6, forming the MutSα complex. Alternatively, MSH2 can dimerise with MSH3, forming the MutSβ complex which repairs larger-scale insertions and deletions ( Traver et al , 2015 ). Pathogenic variants in MSH2 cause Lynch syndrome, an autosomal dominant disease that greatly increases cancer susceptibility. Many of the variants that cause Lynch syndrome result in decreased mismatch repair activity ( Ollila et al , 2006 ). The MAVE for MSH2 was carried out in human HAP1 cells, with endogenous expression of MSH2 knocked out, mutant MSH2 was introduced using a lentiviral construct ( Jia et al , 2021 ). Selection was performed using a purine analogue that is toxic to cells with a functional mismatch repair pathway, resulting in higher growth rates for cells with reduced MSH2 activity (I.E., a reverse-survival assay, where cells with reduced protein fitness are better able to proliferate than the wild type). A single variant in mismatch repair protein MSH2 exceeds the concordance threshold, S612P. This is a likely benign variant in ClinVar which is identified as benign by the MAVE, but as pathogenic by the VEPs. This variant occurs in an alpha helix, which proline mutations tend to disrupt, and in an alignment of 13624 related sequences, only five occurrences of proline were observed at this position, demonstrating strong residue constraint ( Figure S3B ). We suggest that the MAVE may actually be incorrect and that this may actually be a mis-labelled variant in ClinVar because, in addition to the likely helix disruption and constraint at this residue, no individuals bearing this variant are present in gnomAD and numerous other variants in the same helix are ClinVar-pathogenic (A604D, D603Y/G/V, A600D/V). A closer look at the category assignment shows that the MSH2 MAVE study is actually used as functional evidence supporting the classification of likely benign (VCV002447337.2 - ClinVar - NCBI), further complicating the comparison and raising the question of data circularity when using the ClinVar classification as the gold-standard in this case. MTHFR Methylenetetrahydrofolate reductase is a metabolic enzyme that catalyses the conversion of 5,10-methylenetetrahydofolate to 5-methyltetrahydrofolate, which serves as a methyl group donor for the conversion of homocysteine into methionine ( Froese et al , 2018 ). Variants that cause a severe loss of MTHFR function (usually defined as <20% of wild type function remaining) cause homocystinuria, an autosomal recessive disease that results in developmental delay and severe intellectual disability ( Burda et al , 2015 ). The MAVE for MTHFR makes use of the ability of the protein to functionally complement its yeast homologue MET13, which is essential for yeast growth unless supplemental methionine is added to the growth medium ( Weile et al , 2021 ). Variant function was measured by assessing the yeast growth rate via a Tile-seq approach. Y506D and P572L are likely pathogenic and pathogenic/likely pathogenic variants in ClinVar respectively, found benign by the MAVE and predicted as pathogenic by the VEPs. Y506 is a core residue at a dimerisation interface. This particular variant was not remarked on by the authors of the MAVE study but it was noted that dimerisation interface residues scored no lower than other residues at the surface of the same domain ( Weile et al , 2021 ). This was attributed to dimerisation not being strictly required for MTHFR activity, although the pathogenic classification of the variant implies that some potentially unaccounted-for function of the protein is impacted by mutation of the interface. One patient study found that a compound heterozygote bearing the Y506D variant along with a partial duplication in the other allele (p.Leu89_Pro101dup) had only 8.1% MTHFR activity remaining ( Burda et al , 2015 ). P572 is a fully buried residue, it is reported as pathogenic and likely pathogenic in ClinVar by two reporters, and additionally has been found to severely reduce MTHFR activity in a human expression system (3.1 ±0.4% activity remaining) ( Burda et al , 2017 ). It is likely that, in both cases that the VEPs have successfully identified a pathogenic variant that was overlooked by the MAVE, possibly due to experimental noise, or a difference between the yeast expression system and human gene context. GCK Hexokinase-4 is a metabolic enzyme that phosphorylates hexose sugars at the first stage of glycolysis ( Takeda et al , 1993 ). It also acts as a glucose sensor in pancreatic beta cells. Pathogenic variants in GCK are associated with defects in glucose metabolism, specifically maturity-onset diabetes of the young 2 (MODY2) which is an early-onset autosomal dominant disease where insulin secretion is impaired. The MAVE for GCK activity was carried out using yeast complementation in a yeast strain in which the three yeast hexokinase genes had been deleted ( Gersing et al , 2023 ). This strain was unable to grow on glucose medium, but was rescued by the activity of plasmid-expressed human GCK. There are two likely pathogenic variants in GCK that are predicted pathogenic by the VEPs but found to be benign by the MAVE: A208P (reviewed by expert panel) and L185P. Neither variant is present in gnomAD, and all other variants at L185 are found by the MAVE to result in severe loss-of-function. Most other variants as A208 also result in a loss-of-function. It is highly likely that the benign MAVE results are due to experimental noise in this case and are not accurately reflecting protein function. CARD11 Caspase recruitment domain-containing protein 11 is a signalling protein involved in the adaptive immune response following the activation of T and B cell surface receptors. It is activated by phosphorylation, homooligomerises and recruits BCL10 and MALT1 , leading to the phosphorylation of downstream proteins and eventual release of NF-kappa-B ( Gaide et al , 2001 ). CARD11 variants can cause B-cell expansion with NFKB and T-cell anergy, an autosomal dominant condition characterised by splenomegaly (enlarged spleen) and polyclonal B-cell expansion ( Snow et al , 2012 ). CARD11 variants can also cause both dominant and recessive forms of immunodeficiency ( Greil et al , 2013 ; Ma et al , 2017 ). The MAVE for CARD11 activity used a saturation genome editing approach in a human B cell lymphoma cell line, TMD8, where growth is dependent on NF-κB signalling (and therefore upstream CARD11 and B-cell receptor signalling as well) ( Meitlis et al , 2020 ). As CARD11 activity is vital to growth in this cell line, relative fitness was determined by the abundance of each variant following selection. C49Y is a pathogenic mutation in ClinVar that is found benign by the MAVE and predicted as pathogenic by the VEPs. C49 is a hotspot for gain-of-function variants in CARD11 , where multiple variants increase downstream NF-κB activity ( Meitlis et al , 2020 ). The difference between MAVE and VEP predictions at this location is caused by a disconnect in readouts between the two approaches. VEPs are (accurately) predicting the variant as pathogenic, while the MAVE is (accurately) capturing the increased downstream activity caused by the variant – however this fitter-than-wild type phenotype just looks benign in a direct comparison of the scores due to the rank normalisation. In this particular fitness assay, gain-of-function variant do not negatively impact cellular fitness as they would in the context of the whole organism. A solution that can help flag pathogenic gain-of-function variants is to use an approach employed by Weile et al. and ‘flip’ these scores by setting them to their reciprocal score (assuming a 1-0 scale), we have found that this approach improves MAVE correlation with VEPs in a previous study ( Livesey & Marsh, 2020 ). ASPA and PRKN Aspartoacyclase is an enzyme that catalyses the deacetylation of N-acetylaspartic acid to form acetate and L-aspartate, it is thought to regulate the concentration of N-acetylaspartic acid in brain tissue ( Herga et al , 2006 ). Pathogenic variants, resulting in severe loss of enzyme activity cause Canavan disease, a rare lethal neurodegenerative condition that often results in death in early childhood ( Mendes et al , 2017 ). E3 ubiquitin ligase parkin functions as part of an E3 ubiquitin ligase complex, and catalyses the covalent attachment of ubiquitin proteins onto its targets ( Shimura et al , 2000 ). PRKN plays a role in the degradation of misfolded proteins by attaching chains of ubiquitin, targeting them to aggresomes. Variants in this protein that impair its ligase activity can lead to both to classic Parkinson disease and an autosomal recessive form of Parkinson disease ( Hedrich et al , 2002 , 50; West et al , 2002 ). The MAVEs for both ASPA and PRKN were performed using the VAMP-seq technique which assesses variant abundance through quantified expression of a GFP fusion protein ( Grønbæk-Thygesen et al , 2024 ; Clausen et al , 2024 ). Unlike most other MAVE assays discussed here, VAMP-seq does not measure protein activity, but rather protein stability since unstable variants will be rapidly degraded and therefore not contribute to the GFP signal produced by the cell. R71S and M434T in ASPA and PRKN respectively are both ClinVar pathogenic mutations found benign in the MAVEs and predicted as pathogenic by VEPs. Because VAMP-seq was used, stability altering mutations were identified, but not activity-altering mutations that did not also disrupt protein stability, which is likely the case for these two variants. The mutation in ASPA alters a residue that binds the proteins substrate, likely abrogating the binding ( Bitto et al , 2007 ). The mutation in PRKN occurs in close proximity to the active site and may also affect substrate binding. It is clear that VEPs can fail in situations where their assumption of evolutionary conservation being tied to phenotypic impact is violated. This could occur in proteins that have developed relatively new functions compared to their homologues throughout evolution or in ‘new’ and thus poorly evolutionarily conserved areas of proteins. Similar effects are likely in protein sequences that are under strong positive selection as in numerous immune related proteins. VEPs are known to best predict loss-of-function variants ( Gerasimavicius et al , 2022 ), which makes their failure to identify the N-terminal gain-of-function mutation in SHOC2 and the splicing variants in BRCA1 understandable. MAVE data are subject to experimental noise, some of which will be cleaned up during replication or various statistical filtering procedures, but at least some low-quality results will inevitably make it into the final datasets. While it is difficult to positively confirm whether or not experimental noise is the cause of any given case of discordance with VEP predictions, it seems likely that many of the cases where the MAVE results diverge from the ClinVar label and the VEP accurately predicts the label may be due to noise in the MAVE results. The other major cause of MAVE prediction failure is when the assessed protein function or readout from the MAVE does not match the mechanism(s) underlying the disease being investigated. We see this in both ASPA and PRKN , that the VAMP-seq assay for protein stability fails to identify variants that reduce protein activity but maintain stability. This downside can be ameliorated by performing multimodal assays instead of studying variant effects in a single context, using a single assay to produces fitness scores. Multimodal assays can be tailored to the unique mechanisms underlying each disease related to a protein, and have the potential to provide significant insight into the pathogenicity of variants that standard MAVE assays may fail to identify. Conclusion Understanding both the strengths and limitations of variant interpretation methods is essential for improving clinical classification of variants of uncertain significance. Our analysis highlights key systematic differences between VEPs and MAVEs. VEPs tend to overcall pathogenicity in structurally buried regions, including protein interiors, interfaces, and transmembrane domains, while undercalling potentially pathogenic variants on protein surfaces or within intrinsically disordered regions. MAVEs, in contrast, can provide direct functional evidence but are limited by experimental noise and the choice of assay, which may not capture all disease-relevant mechanisms. Expanding the use of MAVEs to assay distinct phenotypes is critical. A single assay may miss important variant effects. For example, VAMP-seq may overlook catalytic impairment at active sites. Applying different assay modalities allows protein functions to be dissected more precisely, offering insights into gain-of-function, dominant-negative, or splicing mechanisms that would otherwise go undetected. Saturation genome editing (SGE) represents a major step forward in this area, enabling variant assessment within native genomic contexts and avoiding artefacts introduced by overexpression systems. With the continued improvement of computational variant effect predictor technology, correct prediction of cases similar to those highlighted here where the VEP failed to correctly identify a ClinVar labelled variant will become more important. A remarkable amount of progress has been made by focussing purely on evolutionary conservation, and VEPs that solely make use of this feature remain among the top performers to date (although in the case of SNCA, the over-reliance on conservation also proved a hinderance). Several recent predictors have benefitted from the addition of protein structural features on top of conservation ( Gerasimavicius et al , 2025 ), implying that the path to further progress may be from integration of alternative sources of informative features. One such feature that may lead to improvements is the addition of a splice-site impact predictor, which would have allowed the VEPs to correctly identify the three splice-impacting variants in BRCA1 as pathogenic. From the analysis of mutation level preferences, and the performance of VEPs in both disordered regions and transmembrane helices, it does appear that MAVEs tend to be the more reliable methods in general. Most of the quirks of VEP predictions we identified are rooted in their requirement to generalise their predictions to a diverse array of proteins, leading to good performance in most regions at the expense of poor performance in unusual or poorly characterised regions, that do not occur often enough or contain enough clinically relevant variants to significantly reduce overall performance metrics. Further evidence of this is seen in the VEPs inability to identify several disease mechanisms that do not fit the standard loss-of-function paradigm among the most discordantly predicted variants and their general tendency to perform worse at gain-of-function and dominant-negative variants. While the generation of MAVE datasets for all human proteins and isoforms is an aspirational goal of the Atlas of Variant Effects alliance (AVE), it is for the moment, not yet attainable ( Fowler et al , 2023 ). The process of performing a mutational scan from conceptualisation to assay design, lab skills, troubleshooting, sequencing and post-processing is a highly skill-intensive process. Currently, relatively few groups are capable of performing the process end-to-end and generating high-quality datasets. So, for now, VEPs allow us to quickly and easily fill in the gaps with predictions that are still largely accurate. In conclusion, while modern VEPs and MAVEs show a strong correlation, and tend to be in concordance for most clinically relevant variants, the cases where major discordance emerges sheds light on the weaknesses of both methods. By taking onboard the findings made here, there is scope for both MAVEs and VEPs to improve and better-reflect the true fitness of a greater proportion of variants. Methods Data acquisition and sources Deep mutational scan datasets were obtained from several sources including their primary publication ( Table 1 ), MaveDB ( Esposito et al , 2019 ) and ProteinGym ( Notin et al , 2023 ). For the five variant effect predictors, AlphaMissense results were downloaded from their repository ( https://zenodo.org/records/8208688 ). SaProt was run locally using instructions in their GitHub repository ( https://github.com/westlake-repl/SaProt ). ESM-1v was also run locally using their ‘zero shot variant effect prediction’ notebook as a template ( https://github.com/facebookresearch/esm/tree/main/examples/variant-prediction ). ESCOTT predictions were downloaded from the repository ( https://zenodo.org/records/10670914 ). VARITY predictions were downloaded in bulk from the VARITY webserver ( http://varity.varianteffect.org/ ). Pathogenic and benign variants were labelled using ClinVar (06/08/2024 release). Putatively benign variants were labelled using gnomAD version 4.1. Protein feature annotations were obtained from UniProt ( https://www.uniprot.org/ ) and from InterPro ( https://www.ebi.ac.uk/interpro/ ). AlphaFold predicted local distance difference test (pLDDT) scores were extracted from the AlphaFold PDB-format files ( https://alphafold.ebi.ac.uk/ ). Relative solvent exposed surface area (RSA) was calculated from first biological assembly files in the protein databank (PDB) ( https://www.rcsb.org/ ). ROC AUC calculation Area under the receiver operating characteristic curve (AUROC) was calculated using the roc_auc_score() function of the sklearn python package (0.18.1). Prior to AUROC calculation, any dataset with an inverted scale was deducted from 1, to ensure that a score of 0 represented wild type-like variants and 1 represented the greatest fitness effects. For the generation of Figure 1 , the pathogenic dataset was obtained from ClinVar, filtered for pathogenic and likely pathogenic variants with a 1-star review status or higher. The putatively benign dataset composed of gnomAD variants (passing the internal filter). The AUROC calculation was bootstrapped 1000 times with independent selection with replacement from the pathogenic and putatively benign datasets. Rank normalisation To perform rank normalisation, any dataset with an inverted scale was deducted from 1, to ensure that a score of 0 represented wild type-like variants and 1 represented the greatest fitness effects. Scores were converted into ranks using the dataframe.rank() function of the pandas python package (1.1.5). The default behaviour of the rank function was used, which assigns the average rank of the group to any tied values. Following ranking, the ranked data was normalised min-max normalised between values of 0 and 1. Where multiple columns of data are compared to each other, all values missing from at least one column were removed from all columns prior to ranking to ensure that ranks were all calculated with the same variants. Interface classification The classifications of surface, interior and interface were assigned as previously described ( Levy, 2010 ) by RSA. RSA was calculated from the first biological assembly of each protein in the PDB. Interfaces are defined as areas where the RSA changes between the monomeric and complex assembly structures. Surface residues have an RSA of 25% or higher while interior residues have an RSA less than 25%. Among interface residues, support residues have an RSA of less than 25% in both the monomer and complex. Interface rim residues have an RSA of 25% or higher in both the monomer and complex. Core interface residues have an RSA of 25% or higher in the monomer but less than 25% in the complex. We previously found that interface rim residues were very similar to surface residues in terms of properties and pathogenic variant enrichment, so all calculations using these definitions include the interface rim as surface resides ( Livesey & Marsh, 2022 ). Supplementary Figures Download figure Open in new tab Figure S1. Average Spearman’s correlation between each of the VEPs and the MAVE data across all 37 proteins. Spearman’s correlation in each pairwise comparison is averaged across each of the 37 MAVE datasets. Download figure Open in new tab Figure S2. Average standard deviation of MAVE and VEP score sets. The average standard deviation of the normalised rank scores of all possible amino acid substitutions at each position in a protein plotted against the average standard deviation of all possible amino acid substitutions in 10 amino acid bins for each protein. Download figure Open in new tab Figure S3. Logo plots representing the number of each amino acid present at each position in a multiple sequence alignment of three proteins. Multiple sequence alignments were generated by MMseqs2, the size of each letter represents the total number of that particular amino acid at each aligned position. The differences in column size are due to the removal of gap characters and non-standard amino acid notations from the analysis. A. A representation of the α-synuclein alignment around the location of pathogenic variant A53T. B. A representation of the MSH2 alignment around the location of allegedly benign variant S612P. C. A representation of the TPK1 alignment around the location of pathogenic variant G48V. Acknowledgements This project was supported by core funding from the Medical Research Council to the MRC Human Genetics Unit (MC_UU_00035/9). Funder Information Declared Medical Research Council , MC_UU_00035/9 References 1. Ahler E , Register AC , Chakraborty S , Fang L , Dieter EM , Sitko KA , Vidadala RSR , Trevillian BM , Golkowski M , Gelman H , et al. ( 2019 ) A Combined Approach Reveals a Regulatory Mechanism Coupling Src’s Kinase Activity, Localization, and Phosphotransferase-Independent Functions . Mol Cell 74 : 393 – 408 .e20 OpenUrl CrossRef PubMed 2. ↵ Ahmed SS , Rifat ZT , Lohia R , Campbell AJ , Dunker AK , Rahman MS & Iqbal S ( 2022 ) Characterization of intrinsically disordered regions in proteins informed by human genetic diversity . PLOS Computational Biology 18 : e1009911 OpenUrl 3. ↵ Alafuzoff I & Hartikainen P ( 2017 ) Alpha-synucleinopathies . Handb Clin Neurol 145 : 339 – 353 OpenUrl PubMed 4. ↵ Amorosi CJ , Chiasson MA , McDonald MG , Wong LH , Sitko KA , Boyle G , Kowalski JP , Rettie AE , Fowler DM & Dunham MJ ( 2021 ) Massively parallel characterization of CYP2C9 variant enzyme activity and abundance . The American Journal of Human Genetics 108 : 1735 – 1751 OpenUrl CrossRef PubMed 5. ↵ Badonyi M & Marsh JA ( 2025 ) acmgscaler: An R package and Colab for standardised gene-level variant effect score calibration within the ACMG/AMP framework . 2025.05.16.654507 doi: 10.1101/2025.05.16.654507 [PREPRINT] OpenUrl Abstract / FREE Full Text 6. ↵ Baldeyron C , Jacquemin E , Smith J , Jacquemont C , De Oliveira I , Gad S , Feunteun J , Stoppa-Lyonnet D & Papadopoulo D ( 2002 ) A single mutated BRCA1 allele leads to impaired fidelity of double strand break end-joining . Oncogene 21 : 1401 – 1410 OpenUrl CrossRef PubMed Web of Science 7. ↵ Bandaru P , Shah NH , Bhattacharyya M , Barton JP , Kondo Y , Cofsky JC , Gee CL , Chakraborty AK , Kortemme T , Ranganathan R , et al. ( 2017 ) Deconstruction of the Ras switching cycle through saturation mutagenesis . eLife 6 : e27810 OpenUrl CrossRef PubMed 8. ↵ Berman HM , Westbrook J , Feng Z , Gilliland G , Bhat TN , Weissig H , Shindyalov IN & Bourne PE ( 2000 ) The Protein Data Bank . Nucleic Acids Research 28 : 235 – 242 OpenUrl CrossRef PubMed Web of Science 9. ↵ Bitto E , Bingman CA , Wesenberg GE , McCoy JG & Phillips GN ( 2007 ) Structure of aspartoacylase, the brain enzyme impaired in Canavan disease . Proc Natl Acad Sci U S A 104 : 456 – 461 OpenUrl Abstract / FREE Full Text 10. Brenan L , Andreev A , Cohen O , Pantel S , Kamburov A , Cacchiarelli D , Persky NS , Zhu C , Bagul M , Goetz EM , et al. ( 2016 ) Phenotypic characterization of a comprehensive set of MAPK1/ERK2 missense mutants . Cell Rep 17 : 1171 – 1183 OpenUrl CrossRef PubMed 11. ↵ Burda P , Schäfer A , Suormala T , Rummel T , Bürer C , Heuberger D , Frapolli M , Giunta C , Sokolová J , Vlášková H , et al. ( 2015 ) Insights into Severe 5,10-Methylenetetrahydrofolate Reductase Deficiency: Molecular Genetic and Enzymatic Characterization of 76 Patients . Human Mutation 36 : 611 – 621 OpenUrl CrossRef PubMed 12. ↵ Burda P , Suormala T , Heuberger D , Schäfer A , Fowler B , Froese DS & Baumgartner MR ( 2017 ) Functional characterization of missense mutations in severe methylenetetrahydrofolate reductase deficiency using a human expression system . Journal of Inherited Metabolic Disease 40 : 297 – 306 OpenUrl CrossRef PubMed 13. ↵ Cheng J , Novati G , Pan J , Bycroft C , Žemgulytė A, Applebaum T , Pritzel A , Wong LH , Zielinski M , Sargeant T , et al. ( 2023 ) Accurate proteome-wide missense variant effect prediction with AlphaMissense . Science 381 : eadg7492 OpenUrl CrossRef PubMed 14. Chiasson MA , Rollins NJ , Stephany JJ , Sitko KA , Matreyek KA , Verby M , Sun S , Roth FP , DeSloover D , Marks DS , et al. Multiplexed measurement of variant abundance and activity reveals VKOR topology, active site and human variant impact . eLife 9 : e58026 15. ↵ Chiti F & Dobson CM ( 2006 ) Protein Misfolding, Functional Amyloid, and Human Disease . Annual Review of Biochemistry 75 : 333 – 366 OpenUrl CrossRef PubMed Web of Science 16. ↵ Clausen L , Voutsinos V , Cagiada M , Johansson KE , Grønbæk-Thygesen M , Nariya S , Powell RL , Have MKN , Oestergaard VH , Stein A , et al. ( 2024 ) A mutational atlas for Parkin proteostasis . Nature Communications 15 : 1541 OpenUrl PubMed 17. ↵ Cordeddu V , Di Schiavi E , Pennacchio LA , Ma’ayan A , Sarkozy A , Fodale V , Cecchetti S , Cardinale A , Martin J , Schackwitz W , et al. ( 2009 ) Mutation in SHOC2 promotes aberrant protein N-myristoylation and underlies Noonan-like syndrome with loose anagen hair . Nat Genet 41 : 1022 – 1026 OpenUrl CrossRef PubMed Web of Science 18. ↵ Critical Assessment of Genome Interpretation Consortium ( 2024 ) CAGI, the Critical Assessment of Genome Interpretation, establishes progress and prospects for computational genetic variant interpretation methods . Genome Biol 25 : 53 OpenUrl CrossRef PubMed 19. ↵ Esposito D , Weile J , Shendure J , Starita LM , Papenfuss AT , Roth FP , Fowler DM & Rubin AF ( 2019 ) MaveDB: an open-source platform to distribute and interpret data from multiplexed assays of variant effect . Genome Biology 20 : 223 OpenUrl CrossRef PubMed 20. ↵ Fawzy M & Marsh JA ( 2025 ) Assessing variant effect predictors and disease mechanisms in intrinsically disordered proteins . 2025.04.01.646619 doi: 10.1101/2025.04.01.646619 [PREPRINT] OpenUrl Abstract / FREE Full Text 21. ↵ Findlay GM , Daza RM , Martin B , Zhang MD , Leith AP , Gasperini M , Janizek JD , Huang X , Starita LM & Shendure J ( 2018 ) Accurate classification of BRCA1 variants with saturation genome editing . Nature 562 : 217 – 222 OpenUrl CrossRef PubMed 22. ↵ Fowler DM , Adams DJ , Gloyn AL , Hahn WC , Marks DS , Muffley LA , Neal JT , Roth FP , Rubin AF , Starita LM , et al. ( 2023 ) An Atlas of Variant Effects to understand the genome at nucleotide resolution . Genome Biology 24 : 147 OpenUrl CrossRef PubMed 23. ↵ Fowler DM & Fields S ( 2014 ) Deep mutational scanning: a new style of protein science . Nat Methods 11 : 801 – 807 OpenUrl CrossRef PubMed Web of Science 24. ↵ Frazer J , Notin P , Dias M , Gomez A , Min JK , Brock K , Gal Y & Marks DS ( 2021 ) Disease variant prediction with deep generative models of evolutionary data . Nature 599 : 91 – 95 OpenUrl CrossRef PubMed 25. ↵ Froese DS , Kopec J , Rembeza E , Bezerra GA , Oberholzer AE , Suormala T , Lutz S , Chalk R , Borkowska O , Baumgartner MR , et al. ( 2018 ) Structural basis for the regulation of human 5,10-methylenetetrahydrofolate reductase by phosphorylation and S-adenosylmethionine inhibition . Nat Commun 9 : 2261 OpenUrl CrossRef PubMed 26. ↵ Gaide O , Martinon F , Micheau O , Bonnet D , Thome M & Tschopp J ( 2001 ) Carma1, a CARD-containing binding partner of Bcl10, induces Bcl10 phosphorylation and NF-kappaB activation . FEBS Lett 496 : 121 – 127 OpenUrl CrossRef PubMed Web of Science 27. ↵ Garavelli L , Cordeddu V , Errico S , Bertolini P , Street ME , Rosato S , Pollazzon M , Wischmeijer A , Ivanovski I , Daniele P , et al. ( 2015 ) Noonan syndrome-like disorder with loose anagen hair: a second case with neuroblastoma . Am J Med Genet A 167A : 1902 – 1907 OpenUrl CrossRef 28. Garvie CW , Wu X , Papanastasiou M , Lee S , Fuller J , Schnitzler GR , Horner SW , Baker A , Zhang T , Mullahoo JP , et al. ( 2021 ) Structure of PDE3A-SLFN12 complex reveals requirements for activation of SLFN12 RNase . Nat Commun 12 : 4375 OpenUrl CrossRef PubMed 29. ↵ Gerasimavicius L , Livesey BJ & Marsh JA ( 2022 ) Loss-of-function, gain-of-function and dominant-negative mutations have profoundly different effects on protein structure . Nat Commun 13 : 3895 OpenUrl CrossRef PubMed 30. ↵ Gerasimavicius L , Teichmann SA & Marsh JA ( 2025 ) Leveraging protein structural information to improve variant effect prediction . Curr Opin Struct Biol 92 : 103023 OpenUrl CrossRef PubMed 31. ↵ Gersing S , Cagiada M , Gebbia M , Gjesing AP , Coté AG , Seesankar G , Li R , Tabet D , Weile J , Stein A , et al. ( 2023 ) A comprehensive map of human glucokinase variant activity . Genome Biology 24 : 97 OpenUrl CrossRef PubMed 32. ↵ Greil J , Rausch T , Giese T , Bandapalli OR , Daniel V , Bekeredjian-Ding I , Stütz AM , Drees C , Roth S , Ruland J , et al. ( 2013 ) Whole-exome sequencing links caspase recruitment domain 11 (CARD11) inactivation to severe combined immunodeficiency . J Allergy Clin Immunol 131 : 1376 – 1383 .e3 OpenUrl CrossRef 33. ↵ Grimm DG , Azencott C-A , Aicheler F , Gieraths U , MacArthur DG , Samocha KE , Cooper DN , Stenson PD , Daly MJ , Smoller JW , et al. ( 2015 ) The Evaluation of Tools Used to Predict the Impact of Missense Variants Is Hindered by Two Types of Circularity . Human Mutation 36 : 513 – 523 OpenUrl CrossRef PubMed 34. ↵ Grønbæk-Thygesen M , Voutsinos V , Johansson KE , Schulze TK , Cagiada M , Pedersen L , Clausen L , Nariya S , Powell RL , Stein A , et al. ( 2024 ) Deep mutational scanning reveals a correlation between degradation and toxicity of thousands of aspartoacylase variants . Nat Commun 15 : 4026 OpenUrl CrossRef PubMed 35. ↵ Hedrich K , Marder K , Harris J , Kann M , Lynch T , Meija–Santana H , Pramstaller PP , Schwinger E , Bressman SB , Fahn S , et al. ( 2002 ) Evaluation of 50 probands with early-onset Parkinson’s disease for Parkin mutations . Neurology 58 : 1239 – 1246 OpenUrl CrossRef PubMed 36. ↵ Herga S , Berrin J-G , Perrier J , Puigserver A & Giardina T ( 2006 ) Identification of the zinc binding ligands and the catalytic residue in human aspartoacylase, an enzyme involved in Canavan disease . FEBS Letters 580 : 5899 – 5904 OpenUrl PubMed 37. ↵ Hildebrand PW , Preissner R & Frömmel C ( 2004 ) Structural features of transmembrane helices . FEBS Letters 559 : 145 – 151 OpenUrl CrossRef PubMed Web of Science 38. Huttinger ZM , Haynes LM , Yee A , Kretz CA , Holding ML , Siemieniak DR , Lawrence DA & Ginsburg D ( 2021 ) Deep mutational scanning of the plasminogen activator inhibitor-1 functional landscape . Sci Rep 11 : 18827 OpenUrl CrossRef PubMed 39. ↵ Jia X , Burugula BB , Chen V , Lemons RM , Jayakody S , Maksutova M & Kitzman JO ( 2021 ) Massively parallel functional testing of MSH2 missense variants conferring Lynch syndrome risk . Am J Hum Genet 108 : 163 – 175 OpenUrl CrossRef PubMed 40. ↵ Jiang RJ ( 2019 ) Exhaustive Mapping of Missense Variation in Coronary Heart Disease-related Genes . 41. Jones EM , Lubock NB , Venkatakrishnan A , Wang J , Tseng AM , Paggi JM , Latorraca NR , Cancilla D , Satyadi M , Davis JE , et al. ( 2020 ) Structural and functional characterization of G protein–coupled receptors with deep mutational scanning . eLife 9 : e54895 OpenUrl CrossRef PubMed 42. Kotler E , Shani O , Goldfeld G , Lotan-Pompan M , Tarcic O , Gershoni A , Hopf TA , Marks DS , Oren M & Segal E ( 2018 ) A Systematic p53 Mutation Library Links Differential Functional Impact to Cancer Mutation Pattern and Evolutionary Conservation . Mol Cell 71 : 178 – 190 .e8 OpenUrl CrossRef PubMed 43. ↵ Kwon JJ , Hajian B , Bian Y , Young LC , Amor AJ , Fuller JR , Fraley CV , Sykes AM , So J , Pan J , et al. ( 2022 ) Structure–function analysis of the SHOC2–MRAS–PP1C holophosphatase complex . Nature 609 : 408 – 415 OpenUrl CrossRef PubMed 44. ↵ Laine E , Karami Y & Carbone A ( 2019 ) GEMME: A Simple and Fast Global Epistatic Model Predicting Mutational Effects . Molecular Biology and Evolution 36 : 2604 – 2619 OpenUrl CrossRef PubMed 45. ↵ Landrum MJ , Lee JM , Benson M , Brown GR , Chao C , Chitipiralla S , Gu B , Hart J , Hoffman D , Jang W , et al. ( 2018 ) ClinVar: improving access to variant interpretations and supporting evidence . Nucleic Acids Res 46 : D1062 – D1067 OpenUrl CrossRef PubMed 46. ↵ Lee MK , Stirling W , Xu Y , Xu X , Qui D , Mandir AS , Dawson TM , Copeland NG , Jenkins NA & Price DL ( 2002 ) Human α-synuclein-harboring familial Parkinson’s disease-linked Ala-53 → Thr mutation causes neurodegenerative disease with α-synuclein aggregation in transgenic mice . Proc Natl Acad Sci U S A 99 : 8968 – 8973 OpenUrl Abstract / FREE Full Text 47. ↵ Levy ED ( 2010 ) A Simple Definition of Structural Regions in Proteins and Its Use in Analyzing Interface Evolution . Journal of Molecular Biology 403 : 660 – 670 OpenUrl CrossRef PubMed Web of Science 48. ↵ Livesey BJ & Marsh JA ( 2020 ) Using deep mutational scanning to benchmark variant effect predictors and identify disease mutations . Mol Syst Biol 16 : e9380 OpenUrl CrossRef PubMed 49. ↵ Livesey BJ & Marsh JA ( 2022 ) The properties of human disease mutations at protein interfaces . PLOS Computational Biology 18 : e1009858 OpenUrl 50. ↵ Livesey BJ & Marsh JA ( 2025 ) Variant effect predictor correlation with functional assays is reflective of clinical classification performance . Genome Biology 26 : 104 OpenUrl CrossRef PubMed 51. van Loggerenberg W , Sowlati-Hashjin S , Weile J , Hamilton R , Chawla A , Sheykhkarimli D , Gebbia M , Kishore N , Frésard L , Mustajoki S , et al. ( 2023 ) Systematically testing human HMBS missense variants to reveal mechanism and pathogenic variation . Am J Hum Genet 110 : 1769 – 1786 OpenUrl CrossRef PubMed 52. ↵ Ma CA , Stinson JR , Zhang Y , Abbott JK , Weinreich MA , Hauk PJ , Reynolds PR , Lyons JJ , Nelson CG , Ruffo E , et al. ( 2017 ) Germline hypomorphic CARD11 mutations in severe atopic disease . Nat Genet 49 : 1192 – 1201 OpenUrl CrossRef PubMed 53. ↵ Majithia AR , Tsuda B , Agostini M , Gnanapradeepan K , Rice R , Peloso G , Patel KA , Zhang X , Broekema MF , Patterson N , et al. ( 2016 ) Prospective functional classification of all possible missense variants in PPARG . Nat Genet 48 : 1570 – 1575 OpenUrl CrossRef PubMed 54. ↵ Matreyek KA , Starita LM , Stephany JJ , Martin B , Chiasson MA , Gray VE , Kircher M , Khechaduri A , Dines JN , Hause RJ , et al. ( 2018 ) Multiplex Assessment of Protein Variant Abundance by Massively Parallel Sequencing . Nat Genet 50 : 874 – 882 OpenUrl CrossRef PubMed 55. ↵ McDonnell AF , Plech M , Livesey BJ , Gerasimavicius L , Owen LJ , Hall HN , FitzPatrick DR , Marsh JA & Kudla G ( 2024 ) Deep mutational scanning quantifies DNA binding and predicts clinical outcomes of PAX6 variants . Molecular Systems Biology 20 : 825 – 844 OpenUrl CrossRef PubMed 56. ↵ Meier J , Rao R , Verkuil R , Liu J , Sercu T & Rives A ( 2021 ) Language models enable zero-shot prediction of the effects of mutations on protein function . In Advances in Neural Information Processing Systems pp 29287 – 29303 . Curran Associates, Inc. 57. ↵ Meitlis I , Allenspach EJ , Bauman BM , Phan IQ , Dabbah G , Schmitt EG , Camp ND , Torgerson TR , Nickerson DA , Bamshad MJ , et al. ( 2020 ) Multiplexed Functional Assessment of Genetic Variants in CARD11 . The American Journal of Human Genetics 107 : 1029 – 1043 OpenUrl CrossRef PubMed 58. ↵ Mendes MI , Smith DE , Pop A , Lennertz P , Fernandez Ojeda MR , Kanhai WA , van Dooren SJ , Anikster Y , Barić I , Boelen C , et al. ( 2017 ) Clinically Distinct Phenotypes of Canavan Disease Correlate with Residual Aspartoacylase Enzyme Activity . Hum Mutat 38 : 524 – 531 OpenUrl CrossRef PubMed 59. Mighell TL , Evans-Dutson S & O’Roak BJ ( 2018 ) A Saturation Mutagenesis Approach to Understanding PTEN Lipid Phosphatase Activity and Genotype-Phenotype Relationships . Am J Hum Genet 102 : 943 – 955 OpenUrl CrossRef PubMed 60. ↵ Miller PG , Sathappa M , Moroco JA , Jiang W , Qian Y , Iqbal S , Guo Q , Giacomelli AO , Shaw S , Vernier C , et al. ( 2022 ) Allosteric inhibition of PPM1D serine/threonine phosphatase via an altered conformational state . Nat Commun 13 : 3778 OpenUrl CrossRef PubMed 61. ↵ Moynahan ME , Chiu JW , Koller BH & Jasin M ( 1999 ) Brca1 controls homology-directed DNA repair . Mol Cell 4 : 511 – 518 OpenUrl CrossRef PubMed Web of Science 62. ↵ Newberry RW , Leong JT , Chow ED , Kampmann M & DeGrado WF ( 2020 ) Deep mutational scanning reveals the structural basis for α-synuclein activity . Nat Chem Biol 16 : 653 – 659 OpenUrl CrossRef PubMed 63. ↵ Notin P , Kollasch A , Ritter D , van Niekerk L , Paul S , Spinner H , Rollins N , Shaw A , Orenbuch R , Weitzman R , et al. ( 2023 ) ProteinGym: Large-Scale Benchmarks for Protein Fitness Prediction and Design . Advances in Neural Information Processing Systems 36 : 64331 – 64379 OpenUrl 64. ↵ Ollila S , Sarantaus L , Kariola R , Chan P , Hampel H , Holinski–Feder E , Macrae F , Kohonen–Corish M , Gerdes A , Peltomäki P , et al. ( 2006 ) Pathogenicity of MSH2 Missense Mutations Is Typically Associated With Impaired Repair Capability of the Mutated Protein . Gastroenterology 131 : 1408 – 1417 OpenUrl CrossRef PubMed 65. ↵ Pejaver V , Byrne AB , Feng B-J , Pagel KA , Mooney SD , Karchin R , O’Donnell-Luria A , Harrison SM , Tavtigian SV , Greenblatt MS , et al. ( 2022 ) Calibration of computational tools for missense variant pathogenicity classification and ClinGen recommendations for PP3/BP4 criteria . Am J Hum Genet 109 : 2163 – 2177 OpenUrl CrossRef PubMed 66. ↵ Petukh M , Kucukkal TG & Alexov E ( 2015 ) On human disease-causing amino acid variants: statistical study of sequence and structural patterns . Hum Mutat 36 : 524 – 534 OpenUrl CrossRef PubMed 67. ↵ Richards S , Aziz N , Bale S , Bick D , Das S , Gastier-Foster J , Grody WW , Hegde M , Lyon E , Spector E , et al. ( 2015 ) Standards and guidelines for the interpretation of sequence variants: a joint consensus recommendation of the American College of Medical Genetics and Genomics and the Association for Molecular Pathology . Genet Med 17 : 405 – 423 OpenUrl CrossRef PubMed 68. ↵ Rouleau E , Lefol C , Moncoutier V , Castera L , Houdayer C , Caputo S , Bièche I , Buisson M , Mazoyer S , Stoppa-Lyonnet D , et al. ( 2010 ) A missense variant within BRCA1 exon 23 causing exon skipping . Cancer Genet Cytogenet 202 : 144 – 146 OpenUrl PubMed 69. ↵ Roychowdhury H & Romero PA ( 2022 ) Microfluidic deep mutational scanning of the human executioner caspases reveals differences in structure and regulation . Cell Death Discov 8 : 1 – 8 OpenUrl PubMed 70. ↵ Shimura H , Hattori N , Kubo S i , Mizuno Y , Asakawa S , Minoshima S , Shimizu N , Iwai K , Chiba T , Tanaka K , et al. ( 2000 ) Familial Parkinson disease gene product, parkin, is a ubiquitin-protein ligase . Nat Genet 25 : 302 – 305 OpenUrl CrossRef PubMed Web of Science 71. Silverstein RA , Sun S , Verby M , Weile J , Wu Y , Gebbia M , Fotiadou I , Kitaygorodsky J & Roth FP ( 2022 ) A systematic genotype-phenotype map for missense variants in the human intellectual disability-associated gene GDI1 . 2021.10.06.463360 doi: 10.1101/2021.10.06.463360 [PREPRINT] OpenUrl Abstract / FREE Full Text 72. ↵ Snow AL , Xiao W , Stinson JR , Lu W , Chaigne-Delalande B , Zheng L , Pittaluga S , Matthews HF , Schmitz R , Jhavar S , et al. ( 2012 ) Congenital B cell lymphocytosis explained by novel germline CARD11 mutations . J Exp Med 209 : 2247 – 2261 OpenUrl Abstract / FREE Full Text 73. ↵ Steinegger M & Söding J ( 2017 ) MMseqs2 enables sensitive protein sequence searching for the analysis of massive data sets . Nat Biotechnol 35 : 1026 – 1028 OpenUrl CrossRef PubMed 74. ↵ Stenson PD , Mort M , Ball EV , Chapman M , Evans K , Azevedo L , Hayden M , Heywood S , Millar DS , Phillips AD , et al. ( 2020 ) The Human Gene Mutation Database (HGMD®): optimizing its use in a clinical diagnostic or research setting . Hum Genet 139 : 1197 – 1207 OpenUrl CrossRef PubMed 75. ↵ Su J , Han C , Zhou Y , Shan J , Zhou X & Yuan F ( 2024 ) SaProt: Protein Language Modeling with Structure-aware Vocabulary . 2023.10.01.560349 doi: 10.1101/2023.10.01.560349 [PREPRINT] OpenUrl Abstract / FREE Full Text 76. Suiter CC , Moriyama T , Matreyek KA , Yang W , Scaletti ER , Nishii R , Yang W , Hoshitsuki K , Singh M , Trehan A , et al. ( 2020 ) Massively parallel variant characterization identifies NUDT15 alleles associated with thiopurine toxicity . Proceedings of the National Academy of Sciences 117 : 5394 – 5401 OpenUrl Abstract / FREE Full Text 77. ↵ Sun S , Weile J , Verby M , Wu Y , Wang Y , Cote AG , Fotiadou I , Kitaygorodsky J , Vidal M , Rine J , et al. ( 2020 ) A proactive genotype-to-patient-phenotype map for cystathionine beta-synthase . Genome Med 12 : 13 OpenUrl CrossRef PubMed 78. ↵ Takeda J , Gidh-Jain M , Xu LZ , Froguel P , Velho G , Vaxillaire M , Cohen D , Shimada F , Makino H & Nishi S ( 1993 ) Structure/function studies of human beta-cell glucokinase. Enzymatic properties of a sequence polymorphism, mutations associated with diabetes, and other site-directed mutants . Journal of Biological Chemistry 268 : 15200 – 15204 OpenUrl Abstract / FREE Full Text 79. ↵ Tekpinar M , David L , Henry T & Carbone A ( 2024 ) PRESCOTT: a population aware, epistatic and structural model accurately predicts missense effect. 2024.02.03.24302219 doi: 10.1101/2024.02.03.24302219 [PREPRINT] OpenUrl Abstract / FREE Full Text 80. ↵ The UniProt Consortium ( 2025 ) UniProt: the Universal Protein Knowledgebase in 2025 . Nucleic Acids Research 53 : D609 – D617 OpenUrl CrossRef PubMed 81. ↵ Traver S , Coulombe P , Peiffer I , Hutchins JRA , Kitzmann M , Latreille D & Méchali M ( 2015 ) MCM9 Is Required for Mammalian DNA Mismatch Repair . Molecular Cell 59 : 831 – 839 OpenUrl PubMed 82. VCV002447337.2 - ClinVar - NCBI 83. ↵ Weile J , Kishore N , Sun S , Maaieh R , Verby M , Li R , Fotiadou I , Kitaygorodsky J , Wu Y , Holenstein A , et al. ( 2021 ) Shifting landscapes of human MTHFR missense-variant effects . The American Journal of Human Genetics 108 : 1283 – 1300 OpenUrl CrossRef PubMed 84. ↵ Weile J , Sun S , Cote AG , Knapp J , Verby M , Mellor JC , Wu Y , Pons C , Wong C , van Lieshout N , et al. ( 2017 ) A framework for exhaustively mapping functional missense variants . Mol Syst Biol 13 : 957 OpenUrl Abstract / FREE Full Text 85. ↵ Weng C , Faure AJ , Escobedo A & Lehner B ( 2024 ) The energetic and allosteric landscape for KRAS inhibition . Nature 626 : 643 – 652 OpenUrl CrossRef PubMed 86. ↵ West A , Periquet M , Lincoln S , Lücking CB , Nicholl D , Bonifati V , Rawal N , Gasser T , Lohmann E , Deleuze J-F , et al. ( 2002 ) Complex relationship between Parkin mutations and Parkinson disease . Am J Med Genet 114 : 584 – 591 OpenUrl CrossRef PubMed Web of Science 87. ↵ Wu Y , Li R , Sun S , Weile J & Roth FP ( 2021 ) Improved pathogenicity prediction for rare human missense variants . Am J Hum Genet 108 : 1891 – 1906 OpenUrl CrossRef PubMed 88. ↵ Zeng T & Li YI ( 2022 ) Predicting RNA splicing from DNA sequence using Pangolin . Genome Biology 23 : 103 OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted August 01, 2025. Download PDF Supplementary Material Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Why variant effect predictors and multiplexed assays agree and disagree Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Why variant effect predictors and multiplexed assays agree and disagree Benjamin J. Livesey , Joseph A. Marsh bioRxiv 2025.07.31.667868; doi: https://doi.org/10.1101/2025.07.31.667868 Share This Article: Copy Citation Tools Why variant effect predictors and multiplexed assays agree and disagree Benjamin J. Livesey , Joseph A. Marsh bioRxiv 2025.07.31.667868; doi: https://doi.org/10.1101/2025.07.31.667868 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Bioinformatics Subject Areas All Articles Animal Behavior and Cognition (7624) Biochemistry (17651) Bioengineering (13871) Bioinformatics (41882) Biophysics (21424) Cancer Biology (18566) Cell Biology (25461) Clinical Trials (138) Developmental Biology (13365) Ecology (19867) Epidemiology (2067) Evolutionary Biology (24290) Genetics (15590) Genomics (22476) Immunology (17714) Microbiology (40331) Molecular Biology (17148) Neuroscience (88483) Paleontology (666) Pathology (2828) Pharmacology and Toxicology (4817) Physiology (7635) Plant Biology (15114) Scientific Communication and Education (2044) Synthetic Biology (4286) Systems Biology (9815) Zoology (2268)