Are our 95% CIs only worth 45% confidence?

Are our 95% CIs only worth 45% confidence? A heterogeneity-aware method for extending inference | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Are our 95% CIs only worth 45% confidence? A heterogeneity-aware method for extending inference View ORCID Profile Ulrich Knief , View ORCID Profile Wolfgang Forstmeier doi: https://doi.org/10.1101/2025.08.05.668631 Ulrich Knief 1 Evolutionary Biology and Ecology, Institute of Biology I (Zoology), University of Freiburg , 79104 Freiburg, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Ulrich Knief For correspondence: ulrich.knief{at}biologie.uni-freiburg.de wolfgang.forstmeier{at}bi.mpg.de Wolfgang Forstmeier 2 Department of Ornithology, Max Planck Institute for Biological Intelligence , 82319 Seewiesen, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Wolfgang Forstmeier For correspondence: ulrich.knief{at}biologie.uni-freiburg.de wolfgang.forstmeier{at}bi.mpg.de Abstract Full Text Info/History Metrics Supplementary material Preview PDF Abstract When multiple studies on the same research question, or multiple analyses of the same dataset are summarized in a meta-analysis, our confidence intervals (CIs) are put to a relentless test of reliability. While we might hope that 95% of the CIs would contain the meta-analytic mean value (and therefore presumably also the true value), a recent meta-analysis of 512 meta-analyses in ecology and evolution suggests that only a sobering 45% of them do. As this paradox of overconfidence continues to confuse researchers, we attempt to explain where most of the heterogeneity in findings might be coming from. Much of it is unsurprising, because conventional 95% CIs reflect only sampling noise and ignore other sources of error such as arbitrary analysis decisions (“model uncertainty”). Being aware of multiple sources of error beyond sampling noise, the replication crisis logically follows from an anticonservative statistical practice that allows for overinterpretation beyond the legitimate inference space, which in fact is narrower than commonly acknowledged. We explain how to calculate extended confidence intervals (CI ext ) that also cover other sources of biological and analytical heterogeneity, and we clarify which CI ext is valid for which extended inference space. We further show how multiple versions of analysis of the same dataset can be merged into a many-analyses meta-analysis (MAMA) which yields a CI that accounts for two sources of error. At the same time, we recognize that high heterogeneity estimates can be artefactual, arising from “comparing apples and oranges” (either in terms of statistical metric or biological interpretation). The estimate of only 45% deserved confidence therefore appears far too pessimistic. Overall, we argue for greater caution in interpreting both confidence intervals and heterogeneity estimates, and for clearer separation of comparable from non-comparable evidence. Introduction Doing science can be a frustrating and sobering business! Imagine that we just published a study showing that factor X has a statistically significant effect on our variable of interest Y . We estimated that this effect is approximately of magnitude m , and we further estimated a 95% confidence interval (CI lower ; CI upper ), which, in 95% of all imaginary repetitions of the study, should contain the true mean effect µ (the true, but unknown, magnitude of the effect of X on Y ). Now, here comes the frustrating part: it will not take long, and a colleague will publish another study on the same topic, finding a very different effect size that lies outside our proclaimed confidence interval, or, even worse, another colleague will publish a re-analysis of our own data, showing that our claim about magnitude m is no longer tenable when choosing a subtly different way of data analysis. All this is part of our daily business of doing science, and we are highly versed in finding post-hoc explanations for why the other study or the other way of data analysis may be invalid, and therefore does not threaten our initial findings and conclusions. Such daily drama of attempting to verify the findings of others can also be summarized quantitatively. By doing meta-analysis of repeated research findings on the same topic, we can quantify how often the proclaimed confidence intervals actually end up containing the true mean value that emerges from all the evidence combined. A recent meta-analysis of 512 published meta-analyses in the fields of ecology and evolution ( Yang et al . 2025 ) reported high levels of heterogeneity in research findings. As we will show, this suggests that, on average, only 45% of the time do the proclaimed confidence intervals contain the true value. In the present study, we will explain how heterogeneity in research findings is typically quantified (using the statistical measure of I 2 ) and how this measure is related to the proportion of CIs that should contain the true mean (here, the 45% mentioned above). We then explain how the discrepancy between 95% and 45% confidence can be understood and interpreted as the very expectable result of components of error that we hitherto have ignored in our statistical calculations. Conventional statistical practice, which systematically ignores certain sources of error ( Simmonds et al . 2024 ), is bound to yield overconfident results that will often fail to replicate ( Yarkoni 2022 ). Our review aims at not just raising awareness of neglected components of error, but also at enabling researchers to quantify, partly eliminate, and account for these components of error. While a range of sophisticated modelling tools have been developed that allow the specialist to incorporate particular additional source of error (see Simmonds et al . 2024 for an excellent review), we aim for a simple and easy-to-implement approach that, if desired, can cover all sources of error that may cause heterogeneity in study results. We explain how to calculate extended confidence intervals (CI ext ) that should be able to keep what they promise, namely 95% confidence for a reasonably large inference space ( sensu Anderson & McLean 1974 ; McLean et al . 1991 ). We also scrutinize the reasons why multiple analysts of the same dataset often disagree in their conclusions ( Botvinik-Nezer et al . 2020 ; Breznau et al . 2022 ; Gould et al . 2025 ; Huntington-Klein et al . 2021 ; Schweinsberg et al . 2021 ; Silberzahn et al . 2018 ) and we show how to summarize multiple analyses of the same data set. We recommend the tools of “directed acyclic graphs” (DAGs, Borger & Ramesh 2025 ; Cinelli et al . 2024 ; Wysocki et al . 2022 ) and “specification-curve analysis” ( Simonsohn et al . 2020 ) for settling disagreement on analysis decisions, and we highlight the risk of “comparing apples and oranges” in meta-analysis. We hence also call for caution in estimating and interpreting heterogeneity ( I 2 ). Three sources of error in individual studies When carrying out a study, we typically go through the three phases of (A) design, (B) data collection, and (C) data analysis ( Holzmeister et al . 2024 ). The decisions that are taken during each of these phases introduce some amount of uncertainty about our study’s conclusions (see Fig. 1 ). This uncertainty is about whether the choices that we took are actually the most representative choices for the broader research question at hand ( Yarkoni 2022 ). Conventionally, yet problematically, only one of these three components of error in Fig. 1 , namely the estimated sampling noise, is being used for the calculation of P -values and confidence intervals, while implicitly treating the other two factors as invariant. Variation that lies in these latter sources of uncertainty is, at best, discussed in the context of a sensitivity analysis ( Thabane et al . 2013 ; Young & Holsteen 2017 ), but these sources of error are not (yet) part of conventional statistical analysis — a situation that could be changed. Download figure Open in new tab Figure 1. An illustration of the three sources of variation contributing to the total variation between studies. When choosing a study system ( A ), a set of study individuals ( B ), and a way of data analysis ( C ) we haphazardly sample from distributions of possible effects on the outcome variable. These populations of possible effects at each step are represented by piles of open circles, one of which is chosen in a particular study. The three sampling effects (orange, blue, and red arrows) will add up to a sum of effects (in D ) that varies more widely than each of the three single components. In our example, all three components of sampling error have the same direction (which need not be the case), adding up to an empirical estimate m (dashed purple line). The orange, blue, and red squares illustrate the sampling variances (the square of the respective standard deviation, σ ), which add up to the total sampling variance (three-coloured square), which determines the width of the distribution in ( D ). The conventional 95% CI (in blue) is based on the distribution in ( B ) only, and, in ( D ), is not sufficiently wide to ensure that the true population mean of µ has a 95% probability of falling within the blue interval. In contrast, the extended CI (orange) in ( D ) will contain µ in 95% of all cases. As indicated, such extended CIs have a wider range of interpretational validity. During study design, we may decide to study a certain population of a certain species ( Fig. 1A ) in order to address a more general question (e.g. “does treatment X prolong the lifespan of animals?”). By taking this choice (rather than picking a different population or a different species), we have basically drawn a sample from an imaginary (and idealized) Gaussian distribution of true population-specific effects, which we use as a simple approximation for illustrative purposes. For instance, populations in which the treatment strongly prolongs lifespan will be on the far-right end of this distribution, while populations in which the treatment has little or even opposite effect on lifespan will be in the centre or on the left end, respectively. Hence, we sample from a pool of true effects that are intrinsic to each study system, and we are uncertain whether our choice of population is representative for animals in general (close to the mean of the distribution). There may be additional choices of study design (e.g. the implementation of treatment X ), that feed into the same pool of error variance in design (but for simplicity, we here refrain from splitting this variance into several subcomponents). The magnitude of this variance in study system (or study design) effects (i.e. the width of the Gaussian distribution in Fig. 1A ) is initially unknown to us until multiple estimates become available for meta-analysis. During data collection, we choose a subset of subjects (e.g. individuals) from the whole population in order to examine them ( Fig. 1B ). By quantifying the variability between subjects, we estimate the sampling noise , i.e. the uncertainty that comes from drawing these rather than an alternative set of subjects from the population. In practice, conventional 95% CIs are calculated based on this sampling noise ( Brunswik 1949 ). Again, we may happen to draw a set of individuals whose sensitivity to the treatment lies close to the population mean µ or further away from it. Note that the same logic of assessing sampling noise applies in the situation where a given set of individuals is randomly assigned to treatment groups. During data analysis, we typically select one particular way of estimating the effect of interest (effect of treatment on lifespan), from an entire pool of alternative options (aka “the garden of forking paths”; Gelman & Loken 2013 ) that would yield similar or different values for the effect of interest ( Fig. 1C ). This variation in analysis outcomes can be investigated using specification curves ( Simonsohn et al . 2020 ) or multiverse analyses ( Cantone & Tomaselli 2024 ; Steegen et al . 2016 ), or it may come to light when the statistical analyses are repeated independently by different researchers ( Botvinik-Nezer et al . 2020 ; Breznau et al . 2022 ; Gould et al . 2025 ; Huntington-Klein et al . 2021 ; Schweinsberg et al . 2021 ; Silberzahn et al . 2018 ), who inevitably make different choices in the process of analysing the data (e.g. treatment of outliers, data transformation, inclusion of fixed and random effects, model selection procedures). This variance in outcomes arising from analysis decisions is also widely known as “model uncertainty” ( Young & Holsteen 2017 ). The variance could be further subdivided (variance arising from alternative analyses that are fully legitimate versus from analyses that are simply incorrect and not defendable; Del Giudice & Gangestad 2021 ), but for now we treat “legitimate uncertainty” and “statistical incompetence” jointly as one source of variance. Taking all three sources of variation together (choice of study system and methods, individuals, way of analysis), we end up with a much broader expected distribution of outcomes (see Fig. 1D ) compared to the sampling noise alone ( Fig. 1B ) that we conventionally use for the calculation of the uncertainty in our study outcome (i.e. the classical 95% CI). Hence, while we report the conventional 95% CIs derived from our statistical model ( Fig. 1B ), we underestimate the genuine real-world 95% CIs, which would also include and and thus be derived from the sum of all three sources: . In this traditional way ( Fig. 1B ), we arrive at conclusions that are overconfident, if we think that they readily apply to a broader inference space — the real world that we are seeking to understand ( Borm et al . 2009 ; Yarkoni 2022 ). In our graphical example in Fig. 1 , we chose all three sources of variance to be of equal size (red, blue, and orange squares), implying that the three idealized Gaussian distributions are of the same width. In that special case, and assuming the independence of the three errors, the sum of all three variances is exactly three times as large (the area of the three-coloured square in Fig. 1D ), and the width of the distribution of the sum of the three errors in Fig. 1D is wider by a factor of (i.e. the edge length of the three-coloured square, compared to the edge length of the blue square). With this in mind, we can now draw a range of confidence intervals around the study outcome m , that take the various sources of error into account. In blue (B), we indicate the conventional 95% CI, which refers only to the expected noise of sampling individuals from the study population. This 95% CI is hence valid for sampling new individuals from the same population and for analysing the data in exactly the same way ( Gurevitch & Hedges 1999 ). If we then widen this 95% CI (here by a factor of or , if we want to consider two (B + C, in red) or all three sources of variance (A + B + C, in orange)), we can obtain extended CIs that would also be valid across multiple ways of analysis (C), or even across multiple study system effects (A). To implement this concept of extended CIs in practice, we require information on the width of the distributions included in the chosen inference space (e.g. alone, or ), which we can obtain from meta-analyses once multiple estimates are available from several study systems or from multiple analyses of a given dataset . For most empirical studies it will be sufficient to account for the latter ( , besides , of course), but keeping the former in mind is helpful for being clear about the legitimate inference space ( sensu Anderson & McLean 1974 ; McLean et al . 1991 ). Before we proceed with quantification of error components and confidence intervals, a brief note of justification may be required. We acknowledge that the three listed sources of error are of such different nature, that our approach of summing them up may be somewhat bewildering for a serious statistician. For instance, variation caused by analysis decisions has been described as “possibilistic” rather than as “probabilistic” ( Hall et al . 2022 ), and we agree that calculating the average outcome of a large number of badly done analyses will typically not help with getting closer to the truth ( Del Giudice & Gangestad 2021 ). However, from the point of view of an empirical researcher, there is a need for pragmatism. After discarding all bad choices of methods, it appears sensible to trust in the central tendency of results averaged across a large number of sensible choices, and to settle for the most robustly supported average conclusion. Our approach is hence meant to serve those who are looking for a simple and pragmatic solution to dealing with multiple sources of error. Quantifying these errors with meta-analysis A so-called random-effects meta-analysis can be used to learn more about the magnitude of these errors. For those readers who have never conducted such an analysis, we encourage them to gain some hands-on experience using our most basic tutorial in the Supplementary Materials (Part 4) and to explore the work by Harrison (2011) . Whenever one has multiple estimates of an effect ( m i , where i refers to the i th study), and each estimate comes with a measure of uncertainty (standard error SE or confidence interval CI), one can not only derive a meta-analytic mean estimate with its own CI, but one can also test to what extent the entire set of estimates is in agreement or disagreement. The latter is done by quantifying heterogeneity. Speaking simplistically, the analysis examines whether the effects ( m i ) actually vary more or less than expected from their standard errors. As explained above, these standard errors are based on conventional statistics and solely reflect the uncertainty due to sampling noise in Fig. 1B . If we now find that the scatter in estimated effects ( m i ) exceeds the expectations from sampling noise B alone, we infer that there is additional variance from A and/or C. This amount of additional variance (the sum of the red and orange squares in Fig. 1 ) is commonly referred to as heterogeneity variance τ ( Olsson-Collentine et al . 2020 ). As the magnitude of τ depends on the scale on which the effect sizes m i are measured, it is more common to quantify heterogeneity in the form of I 2 , which describes the proportion of the total variation in study estimates (i.e. the three-coloured square in Fig. 1 ) that is due to heterogeneity (red plus orange squares) ( Higgins & Thompson 2002 ; Higgins et al . 2003 ). Specifically, which is equivalent to subtracting the sampling noise from the total variance and dividing this by the total variance . Note that, at this point, there is no way of distinguishing between the variances and , i.e. they can only be assessed jointly relative to the magnitude of variance . The magnitude of I 2 is not directly influenced by the number of studies included in a meta-analysis, but it strongly depends on the sample sizes used in the included studies ( Rücker et al . 2008 ). This is because large samples help with averaging out the error, such that the sampling noise of each included study (blue square in Fig. 1B ) strongly declines with increasing sample size. And when the sampling noise can be kept small, the heterogeneity will appear to be relatively larger (i.e. the smaller the blue square, the greater the relative contributions of the orange and red squares). To illustrate this with some useful examples, an I 2 = 0.67 would correspond to the situation in Fig. 1 (with jointly making up two thirds of the total error), and I 2 = 0.9 would imply that variances jointly are nine times larger than variance . If the test for heterogeneity yields I 2 = 0, then there appears to be no variance due to study system effects and the choice of analysis , because the spread in the effect estimates m i happens to be equal or less than expected from their standard errors. While I 2 is bound between 0 and 1, the special situation of I 2 = 0 can be further examined using tests for the statistical significance of I 2 . If one obtains I 2 = 0 and P = 0.50, then the spread in estimates m i is exactly equal to the expectation from sampling noise, while P -values above 0.5 indicate less spread in estimates than expected by chance. In an extreme case, if one obtains I 2 = 0 and P = 0.99, then the spread in estimates m i is apparently much less than expected by chance alone, a situation that might sometimes be suggestive of data fabrication (e.g. when the agreement in results all coming from the same author appears “too good to be true”; Francis 2012 ). Most frequently, however, we are confronted with a situation where I 2 is rather large, for which there may be three possible explanations, that we will go through next. Main reasons for high levels of heterogeneity (1) Heterogeneity due to differences in study systems Whenever meta-analyses summarize studies that come from a wide range of study systems, it may seem most plausible to explain heterogeneity in effect sizes ( I 2 > 0) as a result of study system differences (e.g. Gurevitch & Hedges 1999 ), such as species differences in their sensitivity to the treatment ( Fig. 1A ). This often-made proposition, however, can also be tested systematically, for instance by adding species identity as a random effect (if there are repeated effect size estimates for the same species). In some cases, species identity is indeed an important factor (e.g. Rios Moura et al . 2021 ), and it may be interesting to examine the general role of phylogenetic inertia in such cases (for a tutorial see Mizuno et al . 2026 ). In other studies, species identity does not explain much of the observed variation in effect sizes (e.g. Cally et al . 2019 ; de Boer et al . 2021 ; Dougherty 2021 ; Pollo et al . 2025 ; Wang et al . 2019 ). Some authors have interpreted such lack of consistent differences between species optimistically as implying that the result of the meta-analysis is universally valid for all species (e.g. Pollo et al . 2025 ), yet one might also interpret this more pessimistically as implying a near-zero repeatability of research findings at the species level across studies. If species differences are not the main reason for heterogeneity between studies, then this may suggest a larger role for hard-to-tackle environmental conditions or methods of treatment that may differ between studies. (2) Heterogeneity due to the choice of analysis Recent “multi-team studies” have quantified the variance due to choice-of-analysis by having several teams of analysts independently analyse the same dataset and by quantifying the heterogeneity between their analysis outcomes ( Botvinik-Nezer et al . 2020 ; Gould et al . 2025 ; Huntington-Klein et al . 2021 ; Schweinsberg et al . 2021 ; Silberzahn et al . 2018 ), which then mirrors our uncertainty about which model represents the correct way of analysis ( Young & Holsteen 2017 ). Most of these studies have revealed an astonishingly large amount of heterogeneity originating from the “garden of forking paths” that consists of supposedly arbitrary analysis decisions ( Gelman & Loken 2013 ). These studies seem to suggest that the error due to the “choice of analysis” ( Fig. 1C ) alone can cause dramatic heterogeneity in outcomes. However, there is also the view ( Auspurg & Brüderl 2024a , b; Del Giudice & Gangestad 2021 ) that much of this apparent heterogeneity may arise from plainly incompetent data analysis by some contributing analysts and from meta-analysts summarizing effect size measures that are simply not comparable (the problem of “comparing apples and oranges”). We will come back to this controversy towards the end of our review, explaining why we suspect that truly arbitrary analysis variants (fully legitimate alternatives) typically introduce only modest amounts of heterogeneity in outcomes. In meta-analyses that summarize published studies, the effects of choice-of-analysis could also introduce considerable heterogeneity, if a fraction of the published studies engaged in “ P -hacking” via a targeted search for the way-of-analysis that yielded the most-extreme and typically significant outcomes ( Simmons et al . 2011 ). To illustrate that P -hacking can induce heterogeneity, we here use a simple simulation. In each of 10,000 simulated datasets with a true effect size of zero (see Supplementary Material Part 2 ), we selected the most significant result among 128 alternative analyses (7 binary decisions allowing for 2 7 = 128 variants) and compared it to a randomly chosen model from the same set. This increased heterogeneity in effect sizes from I 2 = 0.02 to I 2 = 0.61 (with increasing nearly 50-fold). The choice to not publish non-significant findings ( Rosenthal 1979 ) may also contribute to heterogeneity, especially in research fields where studies with strong positive and studies with strong negative effects are easier to publish than studies with non-significant effects. To quantify these effects empirically, it may be valuable to compare heterogeneity between meta-analyses of published studies and meta-analyses of unpublished data (e.g. Wang et al . 2019 ). Where unbiased data is not available, a range of techniques exists that may help with detecting and potentially eliminating publication bias ( Nakagawa et al . 2022 ). (3) Spurious heterogeneity due to underestimated sampling noise An often-overlooked possible explanation for heterogeneity is that sampling noise may have been underestimated. We diagnose heterogeneity if the multiple study outcomes disagree with each other, and they do so confidently (based on their SEs or CIs). This can also happen when these SEs or CIs were simply too small, i.e. over-confident. The main reason for why SEs or CIs may often be too narrow is that analysts overestimate the effective sample size in their dataset by not recognizing important sources of non-independence that result in pseudo-replication (e.g. random effects, spatial and temporal autocorrelation) or overdispersion (see Bryan et al . 2021 ; Colegrave & Ruxton 2018 ; Forstmeier et al . 2017 ; Knief & Forstmeier 2021 ; Schielzeth & Forstmeier 2009 ; Voelkl et al . 2020 ; Yarkoni 2022 ), and this typically leads to confidence intervals around m i that are too narrow. Since meta-analysis evaluates whether the variation in study outcomes m i is larger than expected from sampling noise alone, any overestimation of the sample size (and corresponding underestimation of CI width) will lead to a meta-analysis finding heterogeneity in study outcomes that is spurious because it actually is a consequence of incorrect data analysis. Interestingly, quantifying I 2 in meta-analysis can also be used to identify and correct underestimated confidence intervals. As an example, Maldonado-Chaparro et al . (2021) carried out 10 replicates of the same aviary experiment (each aviary containing 28 birds), and entered the 10 resulting effect size estimates (Pearson r ) with their standard errors into a meta-analysis. The finding of I 2 = 0.9865 suggested that their initial CIs based on a widely-used randomization procedure ( Butts 2008 ; for a more general treatment see Hart et al . 2022 ) had been too narrow by a factor of 8.6, showing that much wider CIs are needed for making inferences from one replicate aviary to the next, which is the minimal inference space that a study of general interest is supposed to cover. An intuitive way to evaluate the magnitude of reported SEs (or CIs) is to ask what sample size would be required to obtain uncertainty estimates of comparable size under independent sampling. For example, the SE of a Pearson correlation coefficient r can be approximated as , assuming that the N data points are independent ( Sokal & Rohlf 1995 ). Rearranging this expression allows us to estimate the sample size implied by a given SE. In the aviary study, this calculation suggests that the narrow aviary-level CIs produced by the randomization procedure imply an effective sample size on the order of 1,740 independent observations per aviary (range: ∼1,070–2,580), which is an order of magnitude larger than even the raw input data ( N = 196 for each aviary). In contrast, the 8.6-fold wider extended CIs (calculated as described in the next paragraph) correspond to a mean effective sample size of approximately 25.5 per aviary (range: ∼16–37), close to the actual number of 28 individuals per aviary. This suggests that, in this empirical study, the randomization procedure tended to overestimate the effective sample size by roughly 65–70-fold (pseudo-replication). More generally, translating confidence intervals (or standard errors) into implied effective sample sizes may provide a useful diagnostic tool for evaluating whether these uncertainty estimates are compatible with the underlying sampling design. If they correspond to implausibly large effective sample sizes relative to the actual sample size, this indicates pseudo-replication or an overly narrow inference space. It may also provide a useful way for evaluating whether randomisation procedures adequately account for dependence structures and maintain appropriate type I error control ( Hart et al . 2022 ; Winkler et al . 2015 ). In turn, heterogeneity-based approaches such as the extended confidence intervals proposed below may offer one possible way of correcting overconfident inference. Calculating extended confidence intervals After having quantified heterogeneity in meta-analyses, there is the possibility of extending the conventional 95% CIs to cover all levels of uncertainty in individual studies. In other words, we can calculate extended CIs that are just wide enough, such that a new meta-analysis using the extended CIs would yield a heterogeneity estimate I 2 of sharp zero (and a P -value = 0.50). For the extended 95% CIs we suggest the naming convention “95% CI ext ” to clearly distinguish them from the conventional 95% CIs. The conventional 95% CIs to our parameter estimate m are given by where z 0.025 denotes the standard normal critical value (approximately 1.96 for a two-sided 95% interval), assuming that the estimator is approximately normally distributed (e.g. for large sample sizes). We calculate the standard error SE as ( Borenstein et al . 2021 ; see Fig. 1 for the meaning of subscripts). We suggest to retain all conventional techniques for calculating P -values and CIs based on sampling noise alone, but to judge these at more stringent “heterogeneity-aware” thresholds. Thus, we need to extend the conventional 95% CIs to 95% CI ext by increasing the standard error from to . This can be done by multiplying the conventional 95% CIs with (we formally derive y in the Supplementary Material Part 1 ), which means that we obtain the extended 95% CIs as This formula was independently derived by Borm et al . (2009) . While we assume a normal distribution and the use of a z -score, our calculations are generally valid for all cases where a normal approximation is appropriate (i.e. most common analyses with reasonable sample sizes and well-chosen statistics; Borm et al . 2009 ; IntHout et al . 2016 ). Throughout, variance components are estimated using a plug-in approach and treated as fixed quantities, such that the resulting intervals represent large-sample approximations. Our proposed measure of CI ext constitutes a more conservative confidence interval that acknowledges additional sources of uncertainty, quantified empirically using the I 2 metric of heterogeneity. Note that these extended confidence intervals differ from so-called prediction intervals. While both approaches are informed by between-study heterogeneity, prediction intervals in meta-analysis leave the within-study standard errors unchanged and describe the dispersion of true effects across studies (corresponding to the distribution in Fig. 1A ), and hence the range within which the true effect of a new study is expected to fall. They are therefore mainly determined by the between-study variance and, unlike confidence intervals for the mean effect, do not approach zero as the number of studies increases. Prediction intervals in primary studies, in turn, describe the variability of individual observations within a population, rather than the uncertainty in the population mean ( Borenstein et al . 2021 ). In contrast, our extended confidence intervals quantify uncertainty in individual study estimates after accounting for additional sources of variation arising from study systems and analytical choices, and explicitly rescale the standard errors of individual studies to incorporate these additional sources of uncertainty. Having defined extended confidence intervals, we can also calculate the corresponding α -thresholds (i.e. “significance”-thresholds) that would keep the type I error rate at the desired level of, say, 5%. where, Φ(x) is the cumulative distribution function of the standard normal distribution. Limits of setting stricter α -thresholds for given I 2 values Table 1 and Figure 2 provide an overview of how the key parameters in hypothesis testing (width of CI and critical α -threshold) would need to be adjusted in order to account for a given level of heterogeneity in study outcomes. For small and medium levels of heterogeneity ( I 2 < 0.6), these adjustments appear relatively minor, yet as we get to high levels of heterogeneity (e.g. I 2 = 0.9), the adjustments become so extreme that they no longer appear sensible. For instance, Yang et al . (2025) reported that the median level of heterogeneity in meta-analyses in ecology and evolution was as high as I 2 = 0.908. In this scenario, conventional 95% CIs would, on average, cover the true population parameter in only 45% of the cases — and hence the title of our paper (see Table 1 , Fig. 2A ). To bring this value up to the desired 95%, extended confidence intervals would have to be wider by a factor of 3.3-fold ( Table 1 , Fig. 2B ). Accordingly, the critical α -threshold would have to be lowered from 0.05 to a sobering α ext = 6.4 × 10 −11 ( Table 1 , Fig. 2C ). In most cases, such adjustment would do more harm than good for scientific progress. Firstly, such extreme α -thresholds would unlikely be helpful for overcoming the reliability crisis, because overly strict criteria might primarily incentivise bad scientific practice ( Amrhein et al . 2019 ; Smaldino & McElreath 2016 ; Ware & Munafò 2015 ; Wasserstein et al . 2019 ). Secondly, and more importantly, we argue that I 2 -values of such magnitude should generally make us sceptical to begin with. As we will discuss towards the end of our review, we should check whether heterogeneity has been overestimated, and whether the meta-analysis suffers from the issue of “comparing apples and oranges”, which one could also paraphrase as the meta-analysis targeting an inference space that is larger than what is biologically meaningful and statistically sound. View this table: View inline View popup Download powerpoint Table 1. Derivation of the global type I error rate, the size of confidence intervals and extended α thresholds given heterogeneity I 2 . These derivations are identical to those in Borm et al . (2009) . For orientation, we also provide heterogeneity estimates from published meta-analyses and multi-analyst studies. “ y ” refers to the correction factor by which the standard errors and CIs would have to be multiplied in order to be sufficiently wide to keep the type I error rate at 5%. “ z score given I 2 ” shows how the realized threshold for declaring effects as significant gets lower when I 2 is high, leading to a high rate of false-positive findings (“Type I error rate given I 2 ”). Accordingly, the proportion of CIs covering the true mean µ goes down, and these expected values correspond well with observed numbers of CIs that contain µ . “Extended z score” shows the critical threshold for declaring significance that should be used to keep type I errors at a 5% rate, and the “Extended α threshold” shows critical P -values that would be needed for keeping type I errors at 5%. Calculations are implemented in the Supplementary Material Part 4. The pnorm() function in R ( R Core Team 2024 ) is the cumulative distribution function of the standard normal distribution Φ(x). Download figure Open in new tab Figure 2. Relationship between heterogeneity I 2 and ( A ) the percentage of conventional 95% CIs covering the true population parameter, ( B ) the extension factor y required to convert conventional to extended 95% CIs, and ( C ) the adjusted α -threshold for the extended 95% CIs. At I 2 = 0, the conventional 95% CIs are equal to the extended 95% CIs and the α -threshold does not need to be adjusted (i.e., it is at 0.05). As soon as there is heterogeneity between studies due to study system effects or the choice of analysis, the confidence intervals need to be adjusted by y to continue to cover 95% of the true population parameter, resulting in lower α -thresholds. Stars indicate the conventional 95% Cis covering the true population parameter, the extension factor y , and the adjusted α -threshold given a heterogeneity I 2 of 0.908 as reported by Yang et al . (2025) . Merging multiple versions of analysis by use of MAMA Figure 3 illustrates how we can, in principle, eliminate most of the uncertainty about which choice of data analysis is the most representative among numerous reasonable alternative ways of analysis. We here use the “ Eucalyptus dataset” of Gould et al . (2025) as an example, in which a large number of analysis teams examined the very same dataset to estimate the magnitude of a specific effect. From the 75 effect size estimates, we can see that the choices taken by some teams appear to support a significant negative effect size, the choices taken by others suggest a significant positive effect size, yet most choices yield intermediate effects that are close to zero. If we assume that all of these analyses are sensible and valid, we can summarize all these alternative choices into a meta-analytic mean value (the dashed purple line in Fig. 3 ), in the hope that this will eliminate most of the uncertainty, and will get us closer to the most-representative way of analysis (close to µ in Fig. 1C ). Of course, it is fair and necessary to question on a case-to-case basis whether some of the 75 analyses are misleading and should be excluded from contributing to the meta-analytic mean value (see e.g. Del Giudice & Gangestad 2021 ). Here, we simply followed Gould et al . (2025) in excluding the four most extreme estimates from an initial set of 79 analyses. Download figure Open in new tab Figure 3. Example of a many-analyses meta-analysis (MAMA) based on the “ Eucalyptus dataset” of Gould et al . (2025) . The forest plot shows the 75 estimated effect sizes (and their 95% CIs) obtained from different analysts (after removal of 4 outlier results), all of which had been asked to examine the same research question by analyzing the same dataset ( Gould et al . 2025 ). The histogram at the top shows the variance in outcome measures in analogy to Figure 1 . The diversity in analysis outcomes can be summarized in a meta-analytical mean estimate (dashed purple line), accompanied by the appropriate MAMA 95% CI ext (in purple), which is slightly wider than the typical (median) CI from single analysts that are based on only. Note that conventional meta-analysis of such a dataset would lead to a highly misleading “META 95% CI*” (in blue), which would only be valid if these were 75 independent studies with independent data (rather than 75 analyses of the same data). Moreover, conventional meta-analysis would conclude that is responsible for approximately 1/3 of , yielding the incorrect measure of I 2 * = 0.66, while in fact, is not part of , but comes on top of it (yielding I 2 MAMA = 0.75). Note that makes only a small contribution to the size of MAMA 95% CI ext , because the error is averaged out across the N = 75 analyses (dividing the error variance by N in the calculation of SE MAMA ). Also note that I 2 -values reported in Gould et al . (2025) correspond to our I 2 * . They did not use this label, but explicitly recognized the need for new definitions in the present context (of summarizing many analyses of the same data set). While sending all our datasets to a large number of colleagues for obtaining multiple versions of analysis may be impractical, we can also explore different analysis choices by undertaking a so-called “multiverse analysis” ( Cantone & Tomaselli 2024 ; Steegen et al . 2016 ). For readers who are not yet familiar with this approach, we recommend the very intuitive tutorial by Philipp Masur ( Masur 2020 ), besides the more in-depth treatments by Cantone & Tomaselli (2024) and Rohrer et al . (2026) . In brief, there is software ( Masur & Scharkow 2020 ) that allows to go through all “reasonable” versions of data analysis. The output of such analysis is summarized in a curve of effect sizes that are sorted by their magnitude (like in Fig. 3 ). If each entry to that curve gets labelled with a dot-plot indicating the underlying analysis decisions, it becomes easy to identify which choices are most influential in yielding large versus small outcome measures (aka “specification-curve analysis”; Simonsohn et al . 2020 ). In practice, it may often turn out that highly influential analysis decisions are actually those which fundamentally alter the research question that is being asked ( Del Giudice & Gangestad 2021 ). This does not necessarily mean that such analyses are wrong. Rather, empirical research questions are often broader than any single statistical model, and different model specifications may represent different, yet biologically defensible, operationalisations of the same broad question. For example, controlling for a certain covariate, or not, can make a drastic difference to the outcome (and the biological question) depending on how the covariate is related to the predictor of interest and to the dependent variable ( Cinelli et al . 2024 ; Wysocki et al . 2022 ). Typically, for understanding the total causal effect of a treatment on a dependent variable, you want to control for covariates that are confounders (or confound-blockers; considered as “good controls”), but you must not control for mediators, proxies, or colliders (considered as “bad controls”; Cinelli et al . 2024 ; Wysocki et al . 2022 ). Models that contain such “bad controls”, in most cases, address a very peculiar question and not the relevant question of the total effect of treatment on the dependent variable. Averaging across such different choices (e.g. controlling for a mediator or not), and hence across different research questions, may end up being the “comparison of apples and oranges”, which we will illustrate with a specific example further below. We therefore fully agree with Del Giudice & Gangestad (2021) in recommending to limit the multiverse analysis to only those analysis choices that do not alter the underlying research question. Yet, one may also argue for including numerous or essentially all sensible ways of analysis, such that we can rightfully generalize across a broad inference space. Calculating an appropriate MAMA CI ext If we want to summarize the outcomes of multiple analyses of the same dataset using meta-analysis (like in Fig. 3 ), we have to realize that it is not appropriate to simply use the software-inbuilt formulas that refer to a different context, namely the meta-analysis of independent studies that differ in study design, data collection, and data analysis. In the following, we label all those parameters with an asterisk (*) that were calculated from an inappropriate formula, in order to make them distinguishable from the true values. First, the CI* of the meta-analytic mean value (shown in blue in Fig. 3 ) is much too small, because it is based on the wrong assumption that Fig. 3 summarizes 75 independent studies (each of which would represent an independent draw from and and ), rather than 75 analyses of the same dataset (75 draws from only). Second, the inbuilt formulas yield a measure of I 2 * , which incorrectly assumes that part of the variation in analysis outcomes was caused by sampling noise , while in fact all analyses shared almost the exact same sampling noise. In a conventional meta-analysis, the total variance in study outcomes is the sum of three sources of error , while here, all the variance between study outcomes originates from only (see the histogram in Fig. 3 ), equals zero, and the sampling noise (being a matter of the sample size of the dataset, and reflected by the CIs provided by analysts) still requires to be considered in addition. With the examples of and (from Fig. 3 ), conventional analysis assumes that , being the sum of two components, namely and , yielding . However, in fact, , being the sum of and , yielding Beyond this example, we can generally compute I 2 MAMA as (for a formal derivation see the Supplementary Material Part 1 ). If we use the software-inbuilt I 2 * for calculating I 2 MAMA , this would restrict I 2 MAMA to the interval between 0.5 and 1 (which we can see by plugging in the lower ( I 2 * = 0) and upper ( I 2 * = 1) bound into equation (4). Yet, I 2 MAMA should reach from 0 to 1, because can be smaller than . Theoretically it can even approach zero, if all analysts followed the same analysis path, in which case I 2 MAMA = 0 (equivalent to I 2 * = 0, P → 1). We can make use of the fact that I 2 * can also be derived from Cochran’s Q ( Higgins & Thompson 2002 ), and in fact it commonly is ( Higgins et al . 2003 ). In that case I 2 * can also take negative values, leading to I 2 MAMA approaching zero, if the differences between analyses is small (see the Supplementary Material Part 1 for details and Fig. S1 ). We provide R code for calculating I 2 MAMA in the Supplementary Material Part 4 . Using a similar line of reasoning, which is formally laid out in the Supplementary Material Part 1 , we calculate the 95% CI ext for a many-analyses meta-analysis from SE MAMA as where N is the number of analyses summarized in the MAMA. Accordingly, This formulation treats as the typical within-analysis sampling variance across analyses. We show how to derive and , empirically validate the procedure of dividing by N , and provide R code in the Supplementary Material Part 4 . While all this may seem like an elegant solution to a major problem, the upcoming final part of our article actually suggests that there is often not much variance due to arbitrary analysis decisions that can and should be eliminated. Instead, it appears that much of the variance in arises from analysis decisions that shift the underlying biological question. A different question will often yield a different answer, and how precisely the research question is formulated should usually not be glossed over, as it risks averaging away the essential nuances. In the following final section, our perspective will take a rather dramatic turn. We raise the question of whether meta-analyses might frequently overestimate heterogeneity, partly for technical and partly for biological reasons. Using a specific example (the “blue tit dataset” from Gould et al . 2025 ), we also challenge the notion that arbitrary analysis decisions often exert a dramatic effect on the conclusions. What this example instead illustrates is that (1) there are many ways of making analytical mistakes, and that (2) two distinct experimental approaches to the same research question can yield drastically different answers. Do meta-analyses frequently overestimate the magnitude of the problem? We raised the question of whether conventional 95% CIs may only be worth 45% confidence, which was based on the median I 2 of 0.908 reported by Yang et al . (2025) . However, the estimate of only 45% deserved confidence might be overly pessimistic. Some meta-analyses may have overestimated heterogeneity by overestimating the effective sample size of empirical studies (for instance, by basing study CIs on the number of individuals rather than the true number of independent replicates; e.g. Cally et al . 2019 ). In such cases, a meta-analysis might work with smaller CIs than those reported in the original literature. Further, heterogeneity in outcomes may be inflated, if the conversion of parameter estimates to effect sizes fails, so that they are not on the same scale ( Auspurg & Brüderl 2024a , b ). Similarly, heterogeneity may be inflated when the meta-analysis includes effect sizes expressed as (log) odds ratios, instead of collapsible effect size estimates like risk ratios or the average marginal effects from logistic regression ( Greenland et al . 1999 ; Mood 2009 ). Finally, if meta-analysts regularly decide to summarize outcome measures that are biologically not comparable (“comparing apples and oranges”, see the example below), then this would also contribute to the high levels of heterogeneity observed by Yang et al . (2025) . A systematic analysis of these issues would be much needed before making any calls on whether heterogeneity is often over-estimated. How big is the effect of truly arbitrary analysis decisions? To explore the issue of choice-of-analysis more systematically, we carried out analyses on simulated datasets which allowed for 7 different binary analysis decisions, meaning that any given dataset could be analysed in 2 7 = 128 different ways (see Supplementary Materials Part 2 ). We found that the amount of variance introduced by these arbitrary analysis decisions was usually quite modest (median and ), but in rare cases could reach considerably high levels ( Fig. S2 ). Such a median level of variance due to analysis decisions would lead to heterogeneity of I 2 MAMA = 0.14 and would call for CI ext being wider by a factor of 1.08 than the conventional CI based on sampling noise alone, and for a heterogeneity-aware critical α of 0.034. More extensive simulation studies are needed to assess the generality of this result, extending it also to non-normally distributed data. There is also a need to find out whether such a rule of thumb (making CIs 8% wider) is usually sufficient to account for uncertainty in modelling decisions, or whether each study should do its own assessment of the consequences of uncertainty in the models that are being used. In any case, our result that arbitrary analysis decisions cause only little heterogeneity appears to stand in stark contrast to what many of us concluded from multi-team studies ( Botvinik-Nezer et al . 2020 ; Breznau et al . 2022 ; Gould et al . 2025 ; Huntington-Klein et al . 2021 ; Schweinsberg et al . 2021 ; Silberzahn et al . 2018 ), which mostly found that analyst teams differed drastically in their conclusions (i.e. in their effect size estimates). We therefore examine a specific example below. Revisiting the blue tit dataset — a case of remarkable disagreement among analysts According to Table 1 , one of the most spectacular cases of heterogeneity due to “choice of analysis” is the “blue tit dataset” of Gould et al . (2025) . In this multi-team study, 131 different analyses of the same dataset yielded results that were discordant to such an extent ( I 2 MAMA = 0.977), that our approach would recommend extending the CIs of single analyses by a mind-blowing factor of 6.5 (see Table 1 ) in order to encompass all the uncertainty about how to correctly analyse this data. This would imply that the very large and highly significant effect size estimate that one of us contributed to this study (Z r = -0.637 ± 0.052 SE) and reported as falling | z | = 12.3 standard errors away from zero, would now fall just short of the extended critical z -score (| z | = 12.8, Table 1 ) and therefore be no longer considered significantly different from zero, simply because many colleagues apparently disagree with there being an effect. To better understand what can lie behind such major disagreement, we here present some targeted versions of analysis that may help clarify part of the debate. We do this by following the advice of Del Giudice & Gangestad (2021) , namely to distinguish between equivalent (E) and non-equivalent (N) analysis decisions. In case of type (E) decisions, we have no a priori conceptual reasons that would clearly favour one version of analysis over the other, we can hence speak of truly arbitrary choices between equally valid options (“principled equivalence”). In contrast, principled non-equivalence means that we know before-hand that one analysis variant is inferior to the other (e.g. low vs. high validity of a trait), or that the two variants of analysis are actually testing two conceptually different hypotheses (e.g. inclusion of a covariate that results in overcontrol-bias; Arif & MacNeil 2023 ). In between those extremes of (E) and (N), will be cases of uncertainty (U) regarding equivalence. For our re-analysis of the blue-tit data set we chose 6 models that differ in type (E) decisions regarding the treatment of “good controls”, missing data, outlier removal, and data transformation. As a drastic contrast, we chose 8 models that differ in type (N) decisions, namely the inclusion of covariates which can be referred to as “bad controls” ( Cinelli et al . 2024 ). These include conditioning on a mediator (or descendants of a mediator; causing “overcontrol bias”), conditioning on alternative outcome measures, and conditioning on components of the predictor variable. We also created two models with type (U) decisions, for which equivalence may be debated (see Supplementary Materials Part 3 for details). Gould et al . (2025) provided data from a brood-size manipulation experiment in wild blue tits ( Cyanistes caeruleus ) breeding in nest boxes to multiple independent analysis teams. Each team was asked to answer the question: “To what extent is the growth of nestling blue tits influenced by competition with siblings?” This question was deliberately defined loosely, thereby allowing for multiple analytical pathways. The dataset contained information on 3,720 nestlings from 497 clutches. As an experimental treatment, nestlings had been partly swapped between broods, leading to net brood size reductions and enlargements, and roughly a quarter of the broods remained “unmanipulated”. Information was provided on nestling growth (per nestling: body mass at day 14, tarsus length at day 14) and nestling survival (per brood: post-treatment brood size, day 14 brood size, number of fledglings). In the present analysis, we narrow the focus and define Net Manipulation (the net change in number of chicks) as our continuous treatment variable (ranging from -4 to +4). We believe that in an experimental context, the primary goal is to estimate the effect of the experimental treatment, rather than natural variation. We consider Net Manipulation therefore the most direct operationalization of what is meant by “competition with siblings” in the original question. We further use Day 14 Chick Mass as our primary outcome variable. Based on these decisions, we constructed a directed acyclic graph (DAG; see Fig. S3 ), which illustrates the causal pathways from brood size manipulation to chick body mass at day 14 ( Borger & Ramesh 2025 ; Pearl 2009 ). We do acknowledge that slightly different versions of DAGs could be drawn and that this example represents a comparatively clear-cut case; in many empirical settings, distinctions between equivalent and non-equivalent analyses will be less clear, and part of the observed heterogeneity may reflect genuinely different but defensible ways of addressing a broader research question (see also Gould et al . 2025 ). Who is to blame for heterogeneity? The legitimate garden, incompetence, or diversity of fruits? When focussing on the entire blue tit dataset (all broods, including those where brood size was enlarged, reduced, or left unmanipulated; left panel of Fig. 4 ) we see that models that differ only in type (E) decisions yield nearly identical effect size estimates (effect of the net brood-size manipulation on offspring body mass). In contrast, when changing the model non-equivalently (N), such that it deliberately contains a problematic mediator variable (e.g. the post-manipulation clutch size; see Fig. S3 ) or alternative outcome measures as a covariate, it is possible to weaken the otherwise clear treatment effect. Note that we designed the latter kind of models such that they blur the targeted hypothesis, in order to demonstrate possible effects of poorly designed models. Download figure Open in new tab Figure 4. Effects of analytical decisions on estimated treatment effects in the “blue tit dataset” of Gould et al . (2025) . Effect size estimates Z r are Fisher-transformed effect sizes of the net brood size manipulation (number of nestlings removed from or added to each nest) on offspring size (mass or tarsus length). The three panels illustrate the effect of different data selection decisions: left, all nests included; centre, enlarged and unmanipulated broods only; right, reduced and unmanipulated broods only. In each panel, blue dots represent six models based on analytical decisions that we consider truly arbitrary (reflecting principled equivalence sensu Del Giudice & Gangestad (2021) ). The two yellow dots represent models where equivalence may be debated. The ten red dots correspond to models based on principled non-equivalence, where conditioning on a mediator, descendant of a mediator or alternative outcome measures alters the biological hypothesis being tested. Such models are intended to be non-sensible. For details on each model see Supplementary Materials Part 3 , which also contains the results for using an alternative measure of effect size, which is better suited for biological interpretation ( Fig. S4 ), as well as robustness checks based on alternative operationalizations of the outcome and treatment variables ( Figs. S5, S6 ). We then applied the same set of 18 models to two subsets of the data: one containing only enlarged and unmanipulated broods (centre panel of Fig. 4 ), and one containing only nests where brood size had been either reduced or left unmanipulated (right panel of Fig. 4 ). The most striking (post-hoc) observation here is that brood-size reductions appear to have no effect on offspring mass, while brood-size enlargements have clear negative effects on offspring mass. In this example, we can see three clear patterns: Analysis decisions that are truly arbitrary (within the legitimate garden of forking paths; “principled equivalence”) appear to have only minor effects on the analysis outcome (little scatter of blue dots within each of the three panels of Fig. 4 ), thereby confirming our earlier conclusion ( Fig. S2 ) also for this particular data set. Analyses that are purposefully incompetently done can make even large true effects disappear, and thereby induce much heterogeneity in outcomes (wide scatter of red dots in the left and central panels in Fig. 4 ). Speaking with the benefit of hindsight, when meta-analysts jointly evaluate the effects of brood-size enlargements and reductions within one analysis, they may be “comparing apples and oranges” (large difference between panels in Fig. 4 ). To confirm the robustness of these patterns, we repeated this series of analyses also for a different effect size measure ( Supplementary Materials Part 3 Fig. S4 ), for a different outcome variable (tarsus length; Fig. S5 ), and for an alternative classification of the treatment variable (as two-level factor; Fig. S6 ), and we obtained qualitatively the same results. While this extreme example demonstrates that analyst incompetence and the comparison of “apples and oranges” can have very large effects, there is probably also a role for decisions of uncertain equivalence (U) and a need to discuss such analysis decisions on a case-to-case basis. In the end, it always comes down to defining an intended inference space, which might sometimes also comprise a wider diversity of fruits (apples and oranges) ( Yarkoni 2022 ). Practical recommendations In order to substantiate the main conclusions of an empirical study more reliably, we encourage data analysts to explore the (dataset-specific) garden of forking paths of arbitrary analysis decisions by means of multiverse analysis ( Cantone & Tomaselli 2024 ; Simonsohn et al . 2020 ; Steegen et al . 2016 ). As a first step, one has to reach clarity about which model specifications can be considered as being about equally “reasonable” alternatives (e.g. concerning data transformation, error structure, outlier removal, inclusion of covariates and interactions, outcome measures). This clarity can be reached by constructing a directed acyclic graph (DAG) that illustrates the causal pathways between treatment and outcome, along with potential confounders, mediators, and colliders — each of which may influence the a priori study hypothesis if included in statistical models ( Arif & MacNeil 2023 ; Borger & Ramesh 2025 ; Lundberg et al . 2021 ; Pearl 2009 ). We particularly recommend the excellent tutorials by Wysocki et al . (2022) and Cinelli et al . (2024) for deciding which covariates must and which must not be included in a sensible model. Decisions of “principled equivalence” ( Del Giudice & Gangestad 2021 ) should be identified in advance based on the DAG. Once consensus on the decision framework is reached, all planned analyses (e.g., all combinations of 5 binary analysis decisions, yielding 2 5 = 32 versions of the model) can be performed using tools of multiverse analysis ( Masur & Scharkow 2020 ). A specification-curve plot can then reveal which analysis decisions turn out to be most influential for the study outcome and should therefore receive particular attention regarding whether they represent genuine decisions of “principled equivalence”. Parameter estimates and their CIs can be summarized using the described MAMA procedure (as shown in Fig. 3 ) to obtain CI ext (shown in purple in Fig. 3 ). The resulting CI ext is expected to be slightly wider than the typical (i.e., median) width of the CIs across the input models, with the degree of widening depending on the number of models included in the averaging. When interpreting these intervals, researchers should remain explicit about the inference space to which they apply. They are valid for the specific study system, unless heterogeneity in study system effects ( Fig. 1A ) has also been incorporated into the calculation of extended CIs (according to our equation (2)). Finally, the most optimistic takeaway from our otherwise perhaps sobering review is that I 2 estimates exceeding 0.9, and the associated concern that individual research findings may have severely reduced reliability, with nominal 95% CIs effectively capturing only 45%, may in fact be overly pessimistic. A substantial proportion of heterogeneity may instead stem from the use of inappropriate statistical procedures, either at the level of individual studies or during meta-analytic synthesis. At the individual level, greater theoretical grounding and clearly formulated hypotheses are needed; at the synthesis level, care must be taken to avoid pooling fundamentally heterogeneous studies (i.e. “apples and oranges”). Inappropriate statistical procedures can take many forms: conceptual naivety (e.g. conditioning on colliders, using of predictive models for causal inference; Arif & MacNeil 2022 ; Borger & Ramesh 2025 ), insufficient methodological rigor (e.g. failing to account for dependence between data points or overdispersion), or questionable research practices such as P -hacking or undisclosed post-hoc hypothesizing. Meta-analyses (and many-analyst syntheses) are by no means immune to analogous problems, and are further complicated by selecting studies that are comparable in terms of the hypothesis being tested, by choosing an appropriate effect size measure, and by methodological challenges associated with standardizing effect sizes across studies ( Auspurg & Brüderl 2024a , b ). Greater awareness of these issues may help reducing the magnitude of I 2 and may help increasing our confidence in the underlying data and analysis. Finally, meta-analyses may particularly advance our thinking about biological processes if they succeed at explaining what makes apples differently affected from oranges. When different operationalisations of a broad biological question lead to high heterogeneity, this heterogeneity can be informative: understanding its sources helps refine the underlying biological hypotheses rather than merely signalling a lack of reproducibility. Summary of conceptual insights and concluding remarks Empirical studies typically have multiple sources of error beyond sampling noise ( Fig. 1 ). The legitimate inference space is limited by the sources of error that are accounted for ( Fig. 1 ), hence, typically it is very narrow, covering just a single way of data analysis. The replication crisis logically follows from extrapolation to larger inference spaces than legitimate. Confidence intervals and the α threshold can be adjusted to account for multiple sources of error and to expand the legitimate inference space ( Fig. 2 ). Multiverse analysis should become a standard procedure for empirical studies with complex data. Many-analyses meta-analysis (MAMA) is a unifying tool for incorporating the multiverse into extended CIs and for looking beyond the usual inference space ( Fig. 3 ). Meta-analysis is also useful for verifying that CIs derived from randomization procedures correspond to the intended inference space (checking validity beyond a single dataset). Published estimates of heterogeneity ( I 2 ) should be regarded with caution, because meta-analysts may have been unaware of the many factors that can lead to inflated heterogeneity estimates. Truly arbitrary analysis decisions (of principled equivalence) seem to induce much less heterogeneity than most of the recent literature seems to suggest ( Fig. 4 ). Meta-analysis, and even MAMA, should avoid pooling fundamentally heterogeneous studies or approaches ( Fig. 4 ), as doing so fosters the misleading impression of low confidence in individual research findings. We use statistical models to distinguish meaningful patterns from noise in our data, and we are tempted to generalise our findings from specific populations of individuals or situations to broader research questions ( Yarkoni 2022 ). However, conventional 95% CIs only account for sampling noise , and thus limit us to the picture of the real world that we capture in our specific experimental settings and analyses. As a result, even when we control for all dependencies between datapoints, conventional 95% CIs do not include the real-world variation that we intentionally remove by holding certain experimental design factors constant ( Brunswik 1949 ; Gelman 2015 , 2018 ). Our approach of using extended confidence intervals raises awareness about how limited the legitimate inference space of many published research findings actually is, and it provides a tool for looking beyond that narrow horizon. Author contributions Conceptualization: WF and UK. Simulations and mathematical derivations: UK. Writing: UK and WF. Competing interests The authors declare no competing interests. Acknowledgements We are grateful to Shinichi Nakagawa, Markus Neuhäuser, Julia Rohrer and Holger Schielzeth for their thoughtful and valuable feedback on an earlier version of this manuscript. We further thank colleagues at the Max Planck Institute for Biological Intelligence for constructive feedback and discussions. Footnotes This version clarifies the derivations of CIext, I2MAMA, and SEMAMA. We revised the discussion of inference space, prediction intervals, and variance decomposition, clarified the role of analytical equivalence and non-equivalence in many-analyst studies, and expanded the discussion of pseudo-replication. Literature ↵ Amrhein V , Greenland S , McShane B ( 2019 ) Scientists rise up against statistical significance . Nature 567 , 305 – 307 . OpenUrl CrossRef PubMed ↵ Anderson VL , McLean RA ( 1974 ) Design of experiments: a realistic approach. Marcel Dekker, Inc ., New York, USA . ↵ Arif S , MacNeil MA ( 2022 ) Predictive models aren’t for causal inference . Ecology Letters 25 , 1741 – 1745 . OpenUrl CrossRef PubMed ↵ Arif S , MacNeil MA ( 2023 ) Applying the structural causal model framework for observational causal inference in ecology . Ecological Monographs 93 , e1554 . OpenUrl CrossRef ↵ Auspurg K , Brüderl J ( 2024a ) Claims about “a hidden universe of uncertainty” in social research are exaggerated: How many-analyst studies risk overestimating uncertainty . MetaArXiv doi: 10.31222/osf.io/uc84k . OpenUrl CrossRef ↵ Auspurg K , Brüderl J ( 2024b ) Toward a more credible assessment of the credibility of science by manyanalyst studies . Proceedings of the National Academy of Sciences 121 , e2404035121 . OpenUrl PubMed ↵ Borenstein M , Hedges LV , Higgins JPT , Rothstein HR ( 2021 ) Introduction to meta-analysis . John Wiley & Sons Ltd , Oxford, UK . ↵ Borger MJ , Ramesh A ( 2025 ) Let’s DAG in: how directed acyclic graphs can help behavioural ecology be more transparent . Proceedings of the Royal Society B: Biological Sciences 292 , e20250963 . OpenUrl ↵ Borm GF , Lemmers O , Fransen J , Donders R ( 2009 ) The evidence provided by a single trial is less reliable than its statistical analysis suggests . Journal of Clinical Epidemiology 62 , 711 - 715 .e711. OpenUrl CrossRef PubMed Web of Science ↵ Botvinik-Nezer R , Holzmeister F , Camerer CF , Dreber A , Huber J , Johannesson M , … Schonberg T ( 2020 ) Variability in the analysis of a single neuroimaging dataset by many teams . Nature 582 , 84 – 88 . OpenUrl CrossRef PubMed ↵ Breznau N , Rinke EM , Wuttke A , Nguyen HHV , Adem M , Adriaans J , … Żółtak T ( 2022 ) Observing many researchers using the same data and hypothesis reveals a hidden universe of uncertainty . Proceedings of the National Academy of Sciences 119 , e2203150119 . OpenUrl CrossRef PubMed ↵ Brunswik E ( 1949 ) Systematic and representative design of psychological experiments; with results in physical and social perception . Berkeley Symposium on Mathematical Statistics and Probability 1 , 143 – 202 . OpenUrl ↵ Bryan CJ , Tipton E , Yeager DS ( 2021 ) Behavioural science is unlikely to change the world without a heterogeneity revolution . Nature Human Behaviour 5 , 980 – 989 . OpenUrl PubMed ↵ Butts CT ( 2008 ) Social network analysis with sna . Journal of Statistical Software 24 , 1 – 51 . OpenUrl PubMed ↵ Cally JG , Stuart-Fox D , Holman L ( 2019 ) Meta-analytic evidence that sexual selection improves population fitness . Nature Communications 10 , e2017 . OpenUrl ↵ Cantone GG , Tomaselli V ( 2024 ) Theory and methods of the multiverse: an application for panel-based models . Quality & Quantity 58 , 1447 – 1480 . OpenUrl ↵ Cinelli C , Forney A , Pearl J ( 2024 ) A crash course in good and bad controls . Sociological Methods & Research 53 , 1071 – 1104 . OpenUrl ↵ Colegrave N , Ruxton GD ( 2018 ) Using biological insight and pragmatism when thinking about pseudoreplication . Trends in Ecology & Evolution 33 , 28 – 35 . OpenUrl PubMed ↵ de Boer RA , Vega-Trejo R , Kotrschal A , Fitzpatrick JL ( 2021 ) Meta-analytic evidence that animals rarely avoid inbreeding . Nature Ecology & Evolution 5 , 949 – 964 . OpenUrl PubMed ↵ Del Giudice M , Gangestad SW ( 2021 ) A traveler’s guide to the multiverse: promises, pitfalls, and a framework for the evaluation of analytic decisions . Advances in Methods and Practices in Psychological Science 4 , 1 – 15 . OpenUrl ↵ Dougherty LR ( 2021 ) Meta-analysis reveals that animal sexual signalling behaviour is honest and resource based . Nature Ecology & Evolution 5 , 688 – 699 . OpenUrl PubMed ↵ Forstmeier W , Wagenmakers EJ , Parker TH ( 2017 ) Detecting and avoiding likely false-positive findings – a practical guide . Biological Reviews 92 , 1941 – 1968 . OpenUrl ↵ Francis G ( 2012 ) Too good to be true: publication bias in two prominent studies from experimental psychology . Psychonomic Bulletin & Review 19 , 151 – 156 . OpenUrl PubMed ↵ Gelman A ( 2015 ) The connection between varying treatment effects and the crisis of unreplicable research: a Bayesian perspective . Journal of Management 41 , 632 – 643 . OpenUrl CrossRef ↵ Gelman A ( 2018 ) The failure of null hypothesis significance testing when studying incremental changes, and what to do about it . Personality and Social Psychology Bulletin 44 , 16 – 23 . OpenUrl CrossRef PubMed ↵ Gelman A , Loken E ( 2013 ) The garden of forking paths: why multiple comparisons can be a problem, even when there is no “fishing expedition” or “p-hacking” and the research hypothesis was posited ahead of time . www.stat.columbia.edu/~gelman/research/unpublished/p_hacking.pdf , 1 – 17 . ↵ Gould E , Fraser HS , Parker TH , Nakagawa S , Griffith SC , Vesk PA , … Whelan S ( 2025 ) Same data, different analysts: variation in effect sizes due to analytical decisions in ecology and evolutionary biology . BMC Biology 23 , e35 . OpenUrl ↵ Greenland S , Robins JM , Pearl J ( 1999 ) Confounding and collapsibility in causal inference . Statistical Science 14 , 29 – 46 . OpenUrl CrossRef Web of Science ↵ Gurevitch J , Hedges LV ( 1999 ) Statistical issues in ecological meta-analyses . Ecology 80 , 1142 – 1149 . OpenUrl CrossRef Web of Science ↵ Hall BD , Liu Y , Jansen Y , Dragicevic P , Chevalier F , Kay M ( 2022 ) A survey of tasks and visualizations in multiverse analysis reports . Computer Graphics Forum 41 , 402 – 426 . OpenUrl CrossRef ↵ Harrison F ( 2011 ) Getting started with meta-analysis . Methods in Ecology and Evolution 2 , 1 – 10 . OpenUrl ↵ Hart JDA , Weiss MN , Brent LJN , Franks DW ( 2022 ) Common permutation methods in animal social network analysis do not control for non-independence . Behavioral Ecology and Sociobiology 76 , e151 . OpenUrl ↵ Higgins JPT , Thompson SG ( 2002 ) Quantifying heterogeneity in a meta-analysis . Statistics in Medicine 21 , 1539 – 1558 . OpenUrl CrossRef PubMed Web of Science ↵ Higgins JPT , Thompson SG , Deeks JJ , Altman DG ( 2003 ) Measuring inconsistency in meta-analyses . BMJ 327 , 557 – 560 . OpenUrl FREE Full Text ↵ Holzmeister F , Johannesson M , Böhm R , Dreber A , Huber J , Kirchler M ( 2024 ) Heterogeneity in effect size estimates . Proceedings of the National Academy of Sciences 121 , e2403490121 . OpenUrl PubMed ↵ Huntington-Klein N , Arenas A , Beam E , Bertoni M , Bloem JR , Burli P , Chen N , Grieco P , Ekpe G , Pugatch T , Saavedra M , Stopnitzky Y ( 2021 ) The influence of hidden researcher decisions in applied microeconomics . Economic Inquiry 59 , 944 – 960 . OpenUrl CrossRef ↵ IntHout J , Ioannidis JP , Borm GF ( 2016 ) Obtaining evidence by a single well-powered trial or several modestly powered trials . Statistical Methods in Medical Research 25 , 538 – 552 . OpenUrl CrossRef PubMed ↵ Knief U , Forstmeier W ( 2021 ) Violating the normality assumption may be the lesser of two evils . Behavior Research Methods 53 , 2576 – 2590 . OpenUrl CrossRef PubMed ↵ Lundberg I , Johnson R , Stewart BM ( 2021 ) What is your estimand? Defining the target quantity connects statistical evidence to theory . American Sociological Review 86 , 532 – 565 . OpenUrl CrossRef Magnusson K ( 2014 ) Interpreting confidence intervals . https://rpsychologist.com/d3/ci/ . ↵ Maldonado-Chaparro AA , Forstmeier W , Farine DR ( 2021 ) Relationship quality underpins pair bond formation and subsequent reproductive performance . Animal Behaviour 182 , 43 – 58 . OpenUrl CrossRef ↵ Masur P ( 2020 ) How to do specification curve analyses in R: introducing ‘specr’ . https://philippmasur.de/2020/01/02/how-to-do-specification-curve-analyses-in-r-introducing-specr/ . ↵ Masur PK , Scharkow M ( 2020 ) specr: conducting and visualizing specification curve analyses. v1.0.0 . ↵ McLean RA , Sanders WL , Stroup WW ( 1991 ) A unified approach to mixed linear models . TheAmerican Statistician 45 , 54 – 64 . OpenUrl ↵ Mizuno A , Williams C , Lagisz M , Senior AM , Nakagawa S ( 2026 ) A unified framework for phylogenetic and spatial meta-analysis: concepts, implementation, and practical guidance . EcoEvoRxiv doi: 10.32942/x2wt1b . OpenUrl CrossRef ↵ Mood C ( 2009 ) Logistic regression: why we cannot do what we think we can do, and what we can do about it . European Sociological Review 26 , 67 – 82 . OpenUrl ↵ Nakagawa S , Lagisz M , Jennions MD , Koricheva J , Noble DWA , Parker TH , Sánchez-Tójar A , Yang Y , O’Dea RE ( 2022 ) Methods for testing publication bias in ecological and evolutionary metaanalyses . Methods in Ecology and Evolution 13 , 4 – 21 . OpenUrl ↵ Olsson-Collentine A , Wicherts JM , van Assen MALM ( 2020 ) Heterogeneity in direct replications in psychology and its association with effect size . Psychological Bulletin 146 , 922 – 940 . OpenUrl CrossRef PubMed ↵ Pearl J ( 2009 ) Causality: models, reasoning, and inference , 2 edn. Cambridge University Press , New York, USA . ↵ Pollo P , Lagisz M , Macedo-Rego RC , Mizuno A , Yang Y , Nakagawa S ( 2025 ) Synthesis of nature’s extravaganza: an augmented meta-meta-analysis on (putative) sexual signals . Ecology Letters 28 , e70215 . OpenUrl PubMed ↵ R Core Team ( 2024 ) R: a language and environment for statistical computing . version 4.4.1. ↵ Rios Moura R , Oliveira Gonzaga M , Silva Pinto N , Vasconcellos-Neto J , Requena GS ( 2021 ) Assortative mating in space and time: patterns and biases . Ecology Letters 24 , 1089 – 1102 . OpenUrl CrossRef PubMed ↵ Rohrer JM , Hullman J , Gelman A ( 2026 ) What’s a multiverse good for anyway? PsyArXiv doi: 10.31234/osf.io/37g29_v1 . OpenUrl CrossRef ↵ Rosenthal R ( 1979 ) The file drawer problem and tolerance for null results . Psychological Bulletin 86 , 638 – 641 . OpenUrl CrossRef Web of Science ↵ Rücker G , Schwarzer G , Carpenter JR , Schumacher M ( 2008 ) Undue reliance on I2 in assessing heterogeneity may mislead . BMC Medical Research Methodology 8 , e79 . OpenUrl ↵ Schielzeth H , Forstmeier W ( 2009 ) Conclusions beyond support: overconfident estimates in mixed models . Behavioral Ecology 20 , 416 – 420 . OpenUrl CrossRef PubMed Web of Science ↵ Schweinsberg M , Feldman M , Staub N , van den Akker OR , van Aert RCM , van Assen MALM , … Luis Uhlmann E ( 2021 ) Same data, different conclusions: radical dispersion in empirical results when independent analysts operationalize and test the same hypothesis . Organizational Behavior and Human Decision Processes 165 , 228 – 249 . OpenUrl Senior AM , Grueber CE , Kamiya T , Lagisz M , O’Dwyer K , Santos ESA , Nakagawa S ( 2016 ) Heterogeneity in ecological and evolutionary meta-analyses: its magnitude and implications . Ecology 97 , 3293 – 3299 . OpenUrl CrossRef PubMed ↵ Silberzahn R , Uhlmann EL , Martin DP , Anselmi P , Aust F , Awtrey E , … Nosek BA ( 2018 ) Many analysts, one data set: making transparent how variations in analytic choices affect results . Advances in Methods and Practices in Psychological Science 1 , 337 – 356 . OpenUrl ↵ Simmonds EG , Adjei KP , Cretois B , Dickel L , González-Gil R , Laverick JH , Mandeville CP , Mandeville EG , Ovaskainen O , Sicacha-Parada J , Skarstein ES , O’Hara B ( 2024 ) Recommendations for quantitative uncertainty consideration in ecology and evolution . Trends in Ecology & Evolution 39 , 328 – 337 . OpenUrl PubMed ↵ Simmons JP , Nelson LD , Simonsohn U ( 2011 ) False-positive psychology: undisclosed flexibility in data collection and analysis allows presenting anything as significant . Psychological Science 22 , 1359 – 1366 . OpenUrl CrossRef PubMed ↵ Simonsohn U , Simmons JP , Nelson LD ( 2020 ) Specification curve analysis . Nature Human Behaviour 4 , 1208 – 1214 . OpenUrl PubMed ↵ Smaldino PE , McElreath R ( 2016 ) The natural selection of bad science . Royal Society Open Science 3 , e160384 . OpenUrl ↵ Sokal RR , Rohlf FJ ( 1995 ) Biometry . W. H. Freeman , New York, USA . ↵ Steegen S , Tuerlinckx F , Gelman A , Vanpaemel W ( 2016 ) Increasing transparency through a multiverse analysis . Perspectives on Psychological Science 11 , 702 – 712 . OpenUrl CrossRef PubMed ↵ Thabane L , Mbuagbaw L , Zhang S , Samaan Z , Marcucci M , Ye C , Thabane M , Giangregorio L , Dennis B , Kosa D , Debono VB , Dillenburg R , Fruci V , Bawor M , Lee J , Wells G , Goldsmith CH ( 2013 ) A tutorial on sensitivity analyses in clinical trials: the what, why, when and how . BMC Medical Research Methodology 13 , e92 . OpenUrl ↵ Voelkl B , Altman NS , Forsman A , Forstmeier W , Gurevitch J , Jaric I , Karp NA , Kas MJ , Schielzeth H , Van de Casteele T , Würbel H ( 2020 ) Reproducibility of animal research in light of biological variation . Nature Reviews Neuroscience 21 , 384 – 393 . OpenUrl PubMed ↵ Wang D , Forstmeier W , Valcu M , Dingemanse NJ , Bulla M , Both C , Duckworth RA , Kiere LM , Karell P , Albrecht T , Kempenaers B ( 2019 ) Scrutinizing assortative mating in birds . PLOS Biology 17 , e3000156 . OpenUrl PubMed ↵ Ware JJ , Munafò MR ( 2015 ) Significance chasing in research practice: causes, consequences and possible solutions . Addiction 110 , 4 – 8 . OpenUrl CrossRef ↵ Wasserstein RL , Schirm AL , Lazar NA ( 2019 ) Moving to a world beyond “p < 0.05” . The American Statistician 73 , 1 – 19 . OpenUrl CrossRef ↵ Winkler AM , Webster MA , Vidaurre D , Nichols TE , Smith SM ( 2015 ) Multi-level block permutation . NeuroImage 123 , 253 – 268 . OpenUrl CrossRef PubMed ↵ Wysocki AC , Lawson KM , Rhemtulla M ( 2022 ) Statistical control requires causal justification . Advances in Methods and Practices in Psychological Science 5 , 1 – 19 . OpenUrl ↵ Yang Y , Noble DWA , Spake R , Senior AM , Lagisz M , Nakagawa S ( 2025 ) A pluralistic framework for measuring, interpreting and decomposing heterogeneity in meta-analysis . Methods in Ecology and Evolution 16 , 2710 – 2725 . OpenUrl ↵ Yarkoni T ( 2022 ) The generalizability crisis . Behavioral and Brain Sciences 45 , e1 . OpenUrl CrossRef PubMed ↵ Young C , Holsteen K ( 2017 ) Model uncertainty and robustness: a computational framework for multimodel analysis . Sociological Methods & Research 46 , 3 – 40 . OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted May 10, 2026. Download PDF Supplementary Material Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Are our 95% CIs only worth 45% confidence? A heterogeneity-aware method for extending inference Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Are our 95% CIs only worth 45% confidence? A heterogeneity-aware method for extending inference Ulrich Knief , Wolfgang Forstmeier bioRxiv 2025.08.05.668631; doi: https://doi.org/10.1101/2025.08.05.668631 Share This Article: Copy Citation Tools Are our 95% CIs only worth 45% confidence? A heterogeneity-aware method for extending inference Ulrich Knief , Wolfgang Forstmeier bioRxiv 2025.08.05.668631; doi: https://doi.org/10.1101/2025.08.05.668631 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Ecology Subject Areas All Articles Animal Behavior and Cognition (7636) Biochemistry (17704) Bioengineering (13898) Bioinformatics (41967) Biophysics (21460) Cancer Biology (18599) Cell Biology (25525) Clinical Trials (138) Developmental Biology (13384) Ecology (19909) Epidemiology (2067) Evolutionary Biology (24326) Genetics (15613) Genomics (22512) Immunology (17740) Microbiology (40423) Molecular Biology (17191) Neuroscience (88645) Paleontology (667) Pathology (2835) Pharmacology and Toxicology (4825) Physiology (7646) Plant Biology (15158) Scientific Communication and Education (2046) Synthetic Biology (4302) Systems Biology (9825) Zoology (2271)