Statistics in biology: a survey of the three major multidisciplinary journals

Statistics in biology: a survey of the three major multidisciplinary journals | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Statistics in biology: a survey of the three major multidisciplinary journals View ORCID Profile Jean Xavier Mazoit doi: https://doi.org/10.1101/2025.02.04.636422 Jean Xavier Mazoit 1 Animal ethics committee CEEA26, Université Paris-Saclay Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Jean Xavier Mazoit For correspondence: jean-xavier.mazoit{at}gmail.com Abstract Full Text Info/History Metrics Preview PDF Abstract This article presents the results of a survey of the three major multidisciplinary journals (Nature, Proceedings of the National Academy of Science, Science). Fifty articles involving experiments necessitating the agreement of an ethics committee were searched in each of the three journals to June 30th 2023. Because PNAs announced that a Statistical Review Committee was created and working since September 2023, a further set of 50 articles was retrieved and analyzed from 2024 January 1st. The following items were checked in articles: an explicit statement of approval from an animal ethics committee, the calculation of the appropriate samples size needed and how randomization and blinding were performed, the minimum sample size reported, the presence of repeats, and the methods used to limit type I and type II errors. No clear experimental design was fully reported in any article. The major problems were 1) extremely small sample sizes (<4/group) in nearly half of the articles, 2) confusion between biological repeats and technical replications, and 3) lack of correction for multiple comparisons. These errors led to major inflation of type I and type II errors. In conclusion, only 10 percent of the articles analyzed presented correct statistical methodology. Introduction Statistics are notoriously wrong in biological journals including the major ones ( 1 - 3 ). While medical journals have regularly increased their standards in the past 50-60 years, biological journals have maintained the same low level of statistical analysis. The amount of money involved in health industry and in reimbursement of medical expenses as well as the pressure of regulatory agencies are likely a major cause of the rigorous approach in medical journals ( 4 , 5 ). Statistics courses are usually provided in medical schools and most medical journals have consultant statisticians who are systematically or on-demand involved in the reviewing process. Conversely, most biologists have a low background in statistics and even consider statistics negligible if not simply annoying. This attitude causes major mistakes in articles that may induce erroneous scientific and ethical conclusions. For example, a frequent mistake consists to sample one mouse, seed several plates and consider the data obtained as biological repeats rather than technical replicates ( 6 ). Experiments involving animals are particularly sensitive to these mistakes ( 7 ). This may pose ethical and scientific issues. Therefore, the present study was performed with the aim to quantify the most important statistical problems encountered in the three major multidisciplinary journals. Methods Articles involving experiments on animals and necessitating the agreement of an ethics committee were searched in the three major multidisciplinary journals (Nature, PNAS [Proceedings of the National Academy of Science] and Science) to June 30th 2023. Although the aim of this study was only descriptive, the minimum number of articles to be analyzed in each group was calculated considering a two * three contingency table with an effect size (Cohen w) of 0.3 (medium), a power of 0.8 and an alpha risk of 0.0167 (0.05 corrected). Finally, because of possible exclusions, 50 articles/journal were analyzed. In addition, because PNAS established a Statistical Review Committee in September 2023 ( 8 ), an additional data set from January 1st 2024 was retrieved. The following items were checked in articles: The explicit statement of approval from an animal ethics committee, the reference to the ARRIVE checklist ( 9 ), the calculation of the appropriate number of animals or study units needed (power analysis) and how randomization and blinding were performed. When between-group statistical comparisons were performed, the minimum number of subjects or units in each group, the presence of repeats (biological or technical), the methods used for analyzing results, and when multigroup analysis was performed, the methods used to limit type 1 error inflation were noted. Finally, the report of statistics and if the actual p values were reported rather than a vague p<0.05. In addition, because a great number of experiments were performed on very small samples, we calculated the power (1-β) of Student t tests (two-tailed) as a function of the number of subjects (equal number in each group) and of Cohen effect size d, using Gpower v. 3.1 for illustrative purpose ( 10 ). Results First data set ( Fig. 1 and table 1 ) Fifty consecutive articles involving animals were retrieved from 2023 March 15 th – June 28 th (Nature), from 2023 March 16 th – June 27 th (PNAS) and from 2022 November 25 th – 2023 June 30 th (Science). A total of 6 articles were excluded from analysis because their authors did not perform any statistical analysis or because the study was done on very small samples due to ethical reasons ( Fig. 1 ). Less than 40 percent of articles presented almost correct statistical analyses (Nature n=12/47, PNAS n=18/49, Science n=18/48). View this table: View inline View popup Download powerpoint Table 1. Statistical analysis features in the three journals. Download figure Open in new tab Figure 1. Flowchart of the study. Animal experiments and design No clear experimental design was reported in any journal. Similarly, less than 10% ARRIVE statement was cited and sample size calculation was mentioned in less than 10 % of articles. Randomization and blinding were performed mostly for microscopic slides reading, and performed in one third of the articles. Between group comparisons The two major problems were extremely small sample size and confusion between biological repeats and technical replications. A test of normality was not performed in the vast majority of studies. The minimum number of subjects per group was less than four in 55 %, 37 % and 48 % of the articles in Nature, PNAS and Science respectively. In addition to impair the validity of the comparison (type I error), these very small sample sizes (2 subjects/group in three cases) led to a major risk of type II error. Power as a function of sample size and effect size is reported in Fig. 2 , which shows that a power of 0.8 is far from being reached with samples sizes of 3 subjects/group. In addition, when one consider reasonable values of Cohen d, i.e. less than 4, only a minimum of 5 subjects/group allows to reach a power of 0.8; and even then, only with an alpha error set at 0.05. As a consequence, experiments with less than 5 subjects/group have a very high rate of false negative. Download figure Open in new tab Figure 2. Power of a Student t test (two-tailed) as a function of effect size (Cohen d) and sample size (n = 3 to 6). Three conditions of type I errors (alpha) are represented: 0.05, 0.01, 0.001. The two dashed lines represent 80% and 90% power. Clearly, with reasonable values of effect size (Cohen d ≤ 4), less than 5 subjects/group is insufficient to attain a power of 80%. Even with these conditions, the problem of type I error remains. Data were generated using Gpower 3.1. Repeats Repeats and replicates are most often misunderstood. This confusion between technical and biological repeats was noticed 14 times in Nature, 15 times in Science and 12 times in PNAS. In addition, proper analysis technique (i.e. nested ANOVA or mixed-effect model) was noted only 13 times. The absence of correction for multiple comparisons was noticed frequently. In addition, False Discovery Rate (FDR) correction was sometimes used in place of Family-Wise Error Rate (FWER) correction to compare very small samples. Survival analysis was followed by a logrank test in nearly all cases. The test always lacked continuity correction in case of extremely small samples and correction for multiple comparisons when more than two groups were compared. Repeated measures are another issue. For example, when the same mouse is observed several times, the correlation between these measures was not taken into account. This problem was observed in 13, 5 and 12 articles in Nature, PNAS and Science respectively. Additional data set ( Fig. 1 and table 1 ) Fifty consecutive articles published in PNAS involving experiments in animals were retrieved from 2024 January 1st – 31st. A total of five articles were excluded from analysis because their authors did not perform any statistics (three articles), or because the studies involved non human primates with a limited sampling size (n=2) due to ethical reasons (two articles) ( Fig. 1 ). Only twenty of the remaining 45 articles presented almost correct statistical analyses. Unfortunately, despite the establishment of the committee, the problems were similar (item PNAS2, table 1 ). Discussion The present survey shows that improper statistical analysis is common in life sciences. This diagnostic is not new ( 1 - 3 , 8 ); however the extent of the problem appears of utmost importance. Indeed, this poses ethical issues, not only because of the problem of inadequate animal testing, but also because of the great impact these articles published in major journals may have on societal choices and future orientations of applied research. Animal experiments lack previous calculation of the right sample size and complete randomization and blinding are never included in a proper experimental design. The paradox is that the choice of too small samples is often favoured by ethics committees. Researchers multiply the experiments at the expense of the quality of research. An example is the multiplication of technical replications, which are often believed to be biological repeats (this mistake was noticed in 75 % of the 63 cases of explicit replicates reported). Together with analysis on extremely small samples (less than 5 subjects/group), this leads to major type I and type II error inflation. Type I errors Samples of extremely small size (less than 5 subjects/group, see table 1 ) whether analyzed using parametric or non parametric tests is a key issue ( 11 , 12 ). Some authors are aware of the problem and perform repeats. However, the confusion between biological repeats and technical replications makes the problem worse ( Table 1 ) ( 6 ). By performing repeats, most authors believe that this ascertains the robustness or their results. This is wrong. Firstly, they usually only reproduce their experimental bias. A proper repeated experiment should be performed in an independent laboratory, far from the initial one. Secondly, it is important to note that the probability that a second experiment reach also statistical significance is lower than expected by the researchers ( 13 , 14 ). For example, the probability that a second experiment reach p=0.05 after an initial experiment significant at p=0.05 is only 0.5; and after a first p=0.01 experiment is approximately seven in ten. In addition, what if the second experiment is not significant ? To pool the data ? Certainly not ! In this case, it is better to perform an analysis with an inter-occasion variability parameter. However, an initial power analysis leading to a moderately greater sample size should have been a much better choice. This leads to more robust results with less subjects (animals). The rational of technical replications, for example in culture plates, is to synthesize the data in a statistic like the mean or the median, and then to decrease the measurement error. However, often there is no clear boundary between biological and technical repeats. Thus, we made the following choice: cell lines or cells dissociated for the purpose of the study (dissociated neurons from embryos for example) have be considered as a population in which samples were taken. Conversely, repeated seeds from one mouse are clearly technical replications and in any case biological repeats. In this regard, there is a semantic confusion between experimental and statistical samples, even in forms that the authors must fill when submitting their manuscript. This brings us to the problem of study design. A clear experimental plan should be designed before the start of experiments and described in the report. Type II errors When comparing very small samples, power of the test is extremely low (Fig. Together with under publication of negative results, this inflation of false negative rate is a major bias in research in general. In addition, the “winner’s curse” effect inflate the magnitude of the effect size when a “significant” effect is discovered ( 3 , 15 ). Fortunately, when the experiment is repeated by another team, the effect size decreases because of regression to the mean ( 16 ). This is another reason to recommend repeats, but in other, independent laboratories. Corrections for multiple comparisons are often ignored or wrongly considered. Because researchers want to describe each tree, rather than the whole forest, all possible post-hoc tests are performed after an analysis of variance (ANOVA). In this case, a non corrected Student t test is the choice in almost half of the articles ( table 1 ). Some authors used the False Discovery Rate (FDR) correction in place of Family-Wise Error Rate (FWER) correction to compare very small samples. Procedures to control FDR have been conceived for large-scale testing and they are exploratory rather than confirmatory ( 17 ). Log rank tests also suffer from the same lack of correction for multiple comparison as well as for very small samples comparisons ( 18 ). In one case, the authors deliberately did not correct the risk, but let the reader judge by giving the raw probabilities. In the same way, it is recommended to give the full probability rather than a vague p less than 0.05 ( 19 ). Recently, some researchers have recommended considering statistical significance at 0.005 ( 20 ). However, in the eternal debate between the Ronald Fisher and Jersy Neyman approach, we are obliged to be pragmatic: it appears mandatory 1) to perform a power analysis before initiating the study, but report full probability values and 2) to increase the size of samples - less than five subjects/group should be an exception. At last, longitudinal models, i.e., models in which the same subject is observed more than once are inappropriately analyzed three times out of four. In these models, repeated observations are correlated and the number of degree of freedom is, therefore, less than in a classical ANOVA. The use of mixed-effect models may solve many problems. These models correctly and easily apprehend technical replicates nested in biological repeats, repeated measures and importantly, unbalanced data sets. A second data set was retrieved four months after the announcement in PNAS that a Statistical Review Committee was created and working. Unfortunately, the situation is similar as before, likely because reviewers - who are supposed to seize the committee, and authors have the same lack of statistical knowledge. In conclusion, only 10 percent of articles analyzed present appropriate statistical analysis. To remedy this problem, it may be recommended that consultants in methodology or statistics review most, if not all, articles before publication. Indeed, the problem will not be solved by the wave of a magic wand. This may be a difficult and costly task, but medical journals have been able to meet the challenge. It is a question of credibility for the “great” journals. Footnotes Conflicts of interest. None. Funding. None. References 1. ↵ R. Nuzzo , Scientific method: statistical errors . Nature . 13, 506 ( 7487 ): 150 – 2 ( 2014 ). doi: 10.1038/506150a . OpenUrl CrossRef PubMed Web of Science 2. E.C. Hayden , Weak statistical standards implicated in scientific irreproducibility . Nature ( 2013 ). doi: 10.1038/nature.2013.14131 OpenUrl CrossRef 3. ↵ J.P.A. Ioannidis , Why most published research findings are false . PLoS Med . 2 ( 8 ): e124 . ( 2005 ). OpenUrl CrossRef PubMed 4. ↵ https://www.fda.gov/media/71336/download . Guidance for Industry E9 Statistical Principles for Clinical Trials (accessed 2024 March 12th ). 5. ↵ https://www.ema.europa.eu/en/documents/scientific-guideline/ich-e-9-statistical-principles-clinical-trials-step-5_en.pdf (accessed 2024 March 12th ). 6. ↵ D.L. Vaux , F. Fidler , G. Cumming , Replicates and repeats--what is the difference and is it significant? A brief discussion of statistics and experimental design . EMBO Rep . 13 : 291 – 6 ( 2012 ). doi: 10.1038/embor.2012.36 . OpenUrl FREE Full Text 7. ↵ S.C. Landis et al. , A call for transparent reporting to optimize the predictive value of preclinical research . Nature . 11; 490 ( 7419 ): 187 – 91 ( 2012 ). doi: 10.1038/nature11556 . OpenUrl CrossRef PubMed Web of Science 8. ↵ A.E. Raftery et al. , PNAS establishes a Statistical Review Committee . Proc Natl Acad Sci U S A . 21; 120 ( 47 ): e2317870120 ( 2023 ). OpenUrl CrossRef PubMed 9. ↵ https://arriveguidelines.org/ (accessed 2024 March 12th ). 10. ↵ F. Faul , E. Erdfelder , A.G. Lang , A. Buchner , G*Power 3: A flexible statistical power analysis program for the social, behavioral, and biomedical sciences . Behavior Research Methods , 39 , 175 – 191 ( 2007 ). OpenUrl CrossRef PubMed Web of Science 11. ↵ J.C.F. de Winter , “ Using the Student’s t-test with extremely small sample sizes ,” Practical Assessment, Research, and Evaluation : 18 ; 10 . ( 2013 ). doi: 10.7275/e4r6-dj05 OpenUrl CrossRef 12. ↵ A.K. Dwivedi , I Mallawaarachchi , L.A. Alvarado , Analysis of small sample size studies using nonparametric bootstrap test with pooled resampling method . Stat Med . 36 : 2187 – 2205 ( 2017 ). doi: 10.1002/sim.7263 . OpenUrl CrossRef PubMed 13. ↵ P.R. Killeen , An alternative to null-hypothesis significance tests . Psychol Sci . 16 : 345 – 53 ( 2005 ). doi: 10.1111/j.0956-7976.2005.01538.x . OpenUrl CrossRef PubMed Web of Science 14. ↵ Cumming , G. Replication and p Intervals. p Values Predict the Future Only Vaguely, but Confidence Intervals Do Much Better . Perspect Psychol Sci . 2008; 3 : 286 – 300 ( 2008 ). doi: 10.1111/j.1745-6924.2008.00079.x . OpenUrl CrossRef PubMed 15. ↵ K.S. Button et al. , Power failure: why small sample size undermines the reliability of neuroscience . Nat Rev Neurosci . 14 : 365 – 76 ( 2013 ). doi: 10.1038/nrn3475 . OpenUrl CrossRef PubMed 16. ↵ L. Held , S. Pawel , S. Schwab , Replication Power and Regression to The Mean . Significance . 17 : 10 – 11 ( 2020 ). doi: 10.1111/1740-9713.01462 . OpenUrl CrossRef PubMed 17. ↵ Y. Benjamini , Y. Hochberg , Controlling the False Discovery Rate: a Practical and Powerful Approach to Multiple Testing . J. R. Statist. Soc. B 57 , No. 1 , pp. 289 – 300 ( 1995 ). OpenUrl CrossRef 18. ↵ A.M. Kellerer , D. Chmelevsky , Small-Sample Properties of Censored-Data Rank Tests . Biometrics . 39 : 675 – 682 ( 1983 ). OpenUrl CrossRef Web of Science 19. ↵ R.L. Wasserstein , N.A. Lazar , The ASA Statement on p-Values Context, Process, and Purpose . The American Statistician . 70 : 129 – 33 ( 2016 ). OpenUrl CrossRef PubMed 20. ↵ D.J. Benjamin et al. , Redefine statistical significance . Nat Hum Behav . 2 : 6 – 10 ( 2018 ). doi: 10.1038/s41562-017-0189-z . OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted February 06, 2025. Download PDF Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Statistics in biology: a survey of the three major multidisciplinary journals Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Statistics in biology: a survey of the three major multidisciplinary journals Jean Xavier Mazoit bioRxiv 2025.02.04.636422; doi: https://doi.org/10.1101/2025.02.04.636422 Share This Article: Copy Citation Tools Statistics in biology: a survey of the three major multidisciplinary journals Jean Xavier Mazoit bioRxiv 2025.02.04.636422; doi: https://doi.org/10.1101/2025.02.04.636422 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Scientific Communication and Education Subject Areas All Articles Animal Behavior and Cognition (7625) Biochemistry (17652) Bioengineering (13874) Bioinformatics (41890) Biophysics (21429) Cancer Biology (18567) Cell Biology (25467) Clinical Trials (138) Developmental Biology (13365) Ecology (19874) Epidemiology (2067) Evolutionary Biology (24294) Genetics (15591) Genomics (22478) Immunology (17717) Microbiology (40331) Molecular Biology (17153) Neuroscience (88496) Paleontology (666) Pathology (2828) Pharmacology and Toxicology (4817) Physiology (7635) Plant Biology (15114) Scientific Communication and Education (2044) Synthetic Biology (4286) Systems Biology (9817) Zoology (2268)