Selective reporting of outcomes and results in interrupted time series studies of health interventions: a methodological study

Selective reporting of outcomes and results in interrupted time series studies of health interventions: a methodological study | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Selective reporting of outcomes and results in interrupted time series studies of health interventions: a methodological study View ORCID Profile Phi-Yen Nguyen , View ORCID Profile Simon L. Turner , View ORCID Profile Elizabeth Korevaar , View ORCID Profile Matthew J. Page , View ORCID Profile Joanne E. McKenzie doi: https://doi.org/10.1101/2025.11.06.25339702 Phi-Yen Nguyen 1 Methods in Evidence Synthesis Unit, School of Public Health and Preventive Medicine, Monash University , St Kilda, Victoria, 3004, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Phi-Yen Nguyen Simon L. Turner 1 Methods in Evidence Synthesis Unit, School of Public Health and Preventive Medicine, Monash University , St Kilda, Victoria, 3004, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Simon L. Turner Elizabeth Korevaar 1 Methods in Evidence Synthesis Unit, School of Public Health and Preventive Medicine, Monash University , St Kilda, Victoria, 3004, Australia 2 Centre for Epidemiology and Biostatistics, Melbourne School of Population and Global Health, Faculty of Medicine, Dentistry and Health Sciences, University of Melbourne , Melbourne, Victoria, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Elizabeth Korevaar Matthew J. Page 1 Methods in Evidence Synthesis Unit, School of Public Health and Preventive Medicine, Monash University , St Kilda, Victoria, 3004, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Matthew J. Page Joanne E. McKenzie 1 Methods in Evidence Synthesis Unit, School of Public Health and Preventive Medicine, Monash University , St Kilda, Victoria, 3004, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Joanne E. McKenzie For correspondence: joanne.mckenzie{at}monash.edu Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF ABSTRACT Objective Selective reporting bias occurs when authors report outcomes or results based on the P value, magnitude or direction of the results. Selective reporting has not been examined in interrupted time series (ITS) studies. Therefore, we investigated ( 1 ) discrepancies in outcome reporting, ( 2 ) completeness of results reporting, and ( 3 ) evidence of selective reporting bias in ITS studies. Design We systematically searched for published peer-reviewed protocols of ITS studies of health interventions in 22 databases, and corresponding results reports addressing the protocol’s primary ITS research questions. For each primary research question, we identified outcomes that were reported in the protocol and the corresponding results report(s), and all results reported for those outcomes. Main outcome measure We defined a discrepancy as any outcome that was added, omitted, or had its primacy reclassified (e.g., from primary to secondary) in the results reports compared to the protocol. Each result was classified as fully reported if it was reported with both an effect estimate and a measure of precision (such as a confidence interval). Each result was also classified as favourable or not favourable to the interruption, based on its statistical significance and direction of effect. Results Our search for ITS protocols returned 4,590 abstracts. After excluding ongoing studies, protocols without published results for the primary research question, and records excluded for other reasons, we identified 44 ITS protocols (published 2010-2022) with 46 corresponding results reports. Among outcomes assessed for discrepancies, 52% (202/388) had a discrepancy, affecting 74% (31/42) of studies. Non-reporting of outcomes was prevalent, with 24% (132/553) of outcomes defined in the protocol not reported in the results report, and 60% (25/42) of protocols having at least one outcome not reported in the result reports. Only 28% (56/202) of discrepancies were justified by authors in the results reports. The association between a result favouring the interruption (based on statistical significance and direction of effect) and the result being fully reported was uncertain (OR=1.06 [95% CI 0.74 to 1.53]). Conclusion Non-reporting of outcomes and discrepancies in outcome reporting were prevalent. Pre-specifying outcomes in protocols and registries helps mitigate selective reporting. However, outcomes should be described in sufficient detail for readers to detect any changes. Section 1: What is already known on this topic Selective outcome and result reporting bias ― occurs when the authors report a subset of the measured outcomes or results, based on whether the P value, magnitude of effect estimate or direction of effect supports the authors’ hypothesis. Selective reporting bias has not been examined in interrupted time series (ITS) studies. Section 2: What this study adds Our study highlights several issues with outcome reporting among ITS studies. We found that non-reporting of outcomes and discrepancies between outcomes reported in study protocols and outcomes reported in the final reports were prevalent. Moreover, outcomes were often too broadly defined in the protocol to allow an accurate assessment of selective reporting. We introduced a template for the elements that could be reported when describing an outcome in the context of ITS data. Section 3: How might these results change the focus of research or clinical practice? Our findings call for greater transparency in outcome reporting among investigators of ITS studies. To mitigate selective reporting bias, authors are encouraged to pre-specify outcomes in a protocol and describe outcomes in sufficient detail in any publication (protocol or results reports) to enable readers to detect any change in outcome reporting. Reporting may also be improved with guidance and stricter enforcement of outcome reporting by journal editors, peer reviewers and funders. INTRODUCTION An interrupted time series (ITS) study is a non-randomised design commonly used to evaluate interventions that target populations for which randomisation may be unethical, infeasible or impractical. The design is a key method for evaluating “natural experiments” in which changes to infrastructure, polices or services are introduced by governments and healthcare systems ( 1 ). In an ITS study, measurements of an outcome variable are often collected continuously over time and aggregated using summary statistics (e.g., means) within regular time intervals (e.g., monthly) for analysis. An ‘interruption’ separates the time series into pre- and post-interruption segments. Different models can be fitted to this data; most commonly, segmented regression models ( 2 – 4 ). Using this model, the underlying time trend in the aggregated outcome during the pre-interruption segment is estimated and extrapolated into the post-interruption segment, providing a counterfactual for what would have occurred in the absence of the interruption. This counterfactual is then compared with the trend estimated from the aggregated data in the post-interruption segment, from which various effect measures can be calculated; for example, immediate change in level at the time of the interruption and the difference in pre- and post-interruption slopes ( 5 ) ( Figure 1 ) ( 5 – 7 ). The segmented regression model can be extended to adjust for additional features such as autocorrelation, seasonality, and time-varying confounders ( 7 – 9 ).. Download figure Open in new tab Figure 1. A graph of an interrupted time series The ITS design is less susceptible to selection and confounding biases, compared with other non-randomised designs ( 5 ). For this and other reasons, the design has seen an increase in popularity in recent times ( 3 , 4 , 10 ), with approximately 1000 studies published up to the year 2000, and 19,000 published since. In systematic reviews that examine the effects of interventions that target populations, ITS studies can provide valuable evidence, particularly when evidence from randomised trials is limited or unavailable ( 11 ). Reporting bias can affect the validity of findings from systematic reviews. One common type of reporting bias ― selective outcome and result reporting bias ― occurs when the authors report a subset of the measured outcomes or analyses, based on whether the effect estimate’s magnitude, direction or P value supports the authors’ hypothesis ( 12 – 14 ). Withholding outcomes and results from publication can skew the evidence that informs systematic reviews and meta-analyses, potentially changing their conclusions about the benefits and harms of an intervention ( 15 ). Understanding the extent of selective reporting bias in ITS studies is important so that review authors, and users of ITS studies, are aware of this risk of bias when evaluating and interpreting evidence. Although selective reporting bias has been extensively investigated in samples of systematic reviews ( 16 , 17 ), randomised trials ( 18 – 20 ), and non-randomised studies ( 19 , 21 ), it has not been examined in ITS studies specifically. OBJECTIVES To examine the discrepancies between outcomes specified in ITS study protocols and those reported in the corresponding results reports; To examine how completely results are reported in ITS studies; To examine whether completeness of reporting is influenced by the statistical significance and direction of the results. METHODS This study is one of three studies investigating reporting biases among ITS studies, for which we have published a protocol ( 22 ) (Supplementary File S1). Here, we provide an overview of the methods and the results for the second study; studies one and three are published separately. Deviations from the methods outlined in the protocol are available in Supplementary File S2, and the data extraction form is available in Supplementary File S3. 1. Creating a database of ITS studies We searched for published peer-reviewed protocols of ITS studies from 22 databases, using a highly sensitive search filter for identifying ITS studies ( 23 ), as well as the corresponding results reports that addressed the primary ITS research question(s) from the protocol (see Supplementary Files S4 and S5 for details of search strategies, eligibility criteria, and processes for screening and determining the primary research question). ITS studies for which a protocol and a corresponding results report was found constituted the sample for this study. 2. Identifying, classifying and matching outcomes from protocols and results reports 2.1. Identifying eligible outcomes For each protocol and corresponding results report, two authors (PYN and one of SLT/EK/MJP) independently extracted all outcomes pertaining to the primary ITS research question, with outcomes extracted separately from the protocol and from the results report. We extracted: available details of the outcome (e.g., measurement tool, timing of outcome measurement, time interval of aggregation), and, whether the outcome was an impact outcome (i.e. expected to be affected by the interruption) or a control outcome (i.e. not expected to be affected by the interruption ( 24 )). Unless otherwise specified, when we use the term outcome, we are referring to “impact outcomes”. We adapted a previous framework to define outcomes in randomised control trials (RCTs) ( 25 , 26 ) so that it was more suitable for outcomes analysed in ITS studies. For each outcome, we summarised, when reported, the following five elements: domain, specific measure, time points, data type of individual measurements, and data type at aggregation ( Table 1 ). View this table: View inline View popup Table 1. Framework for outcome definition We classified the primacy of each outcome as: primary , if the authors labelled it as such, or the outcome was used in a sample size calculation, or it was the only outcome in the study; secondary, if the authors labelled it as such; could not be classified, if the authors did not specify if the outcome was primary or secondary. 2.2. Matching outcomes from protocol to report of result One author (PYN) assessed whether each outcome from a protocol could be matched with a corresponding outcome from the results report, based on elements of outcome definition (see Section 2.1 and Supplementary File S5, section 5.4 for details). Only elements that were present in both documents were assessed – a match was confirmed if this subset of common elements were in agreement between the protocol and the results report. Each outcome in the protocol could be matched in the results report to a single outcome ( one-to-one ), to multiple outcomes ( one-to-many , for example, ‘sale of opioids’ is specified in the protocol, with sales of the specific opioids ‘oxycodone’, ‘codeine’, and ‘fentanyl’ specified in the results report), or to none at all . Any complex cases of matching were discussed and resolved at team meetings (PYN/JEM/SLT/EK/MJP). 3. Identifying and extracting data on reported results Two authors (PYN and one of JEM/SLT/EK/MJP) extracted all eligible results pertaining to the included outcomes from each results report. Discrepancies were resolved through discussion. An “eligible result” was defined as any measure of the difference between two segments of interest, including: a numerical result (e.g., an effect estimate such as an immediate level-change, with its confidence interval (CI), standard error (SE), or P value), or a qualitative statement about the statistical significance and/or the direction of the result (e.g., “There was no significant change in road fatalities after the policy was implemented”). For each eligible result, we extracted all numerical results reported, or if no numerical result was presented, any qualitative statement about statistical significance and/or the direction of effect. Note that multiple results per outcome were possible arising from different effect measures (e.g., immediate level change, slope change). Using the approach proposed by Chan and colleagues ( 4 ), we classified each result as: fully reported , if sufficient data were reported to include a result in a meta-analysis, that is, an effect estimate and a measure of precision (CI or SE); partially reported , if insufficient data were reported to include a result in a meta-analysis (e.g., an effect estimate is reported without any measure of precision); or qualitatively reported, if only a qualitative statement or a P value was reported. Lastly, where possible, we classified each result as being favourable to the interruption or not. A result was classified as “favourable” if both (i) the direction of the effect was favourable to the interruption (i.e., the effect estimate indicated greater benefit or less harm compared with the counterfactual), and (ii) the result was statistically significant (i.e. P value <0.05, or if absent, the 95% CI excluded the null, or the authors stated the result was statistically significant). Results that did not meet either criteria were classified as “not favourable”. 4. Assessing discrepancies in outcome reporting Control outcomes and outcomes with one-to-many matches were excluded from the assessment of discrepancies. We excluded outcomes with one-to-many matches because it was impossible to establish from the protocol whether the matched outcomes were ever intended to be included. We defined a discrepancy in outcome reporting as any of the following (full details in Supplementary File S5, section 5.4): missing in results report , where an outcome specified in the protocol was not mentioned in the report of the results; missing in protocol , where an outcome mentioned in the report of the results was not pre-specified in the protocol; discrepancy in outcome primacy , where a primary outcome in the protocol was demoted to secondary or unclassified outcome in the results report; or, a secondary or unclassified outcome in the protocol was promoted to primary in the results report. 5. Statistical analysis We summarised results by calculating frequencies and percentages for categorical variables (e.g., results that are fully reported, partially reported, qualitatively reported) and medians and interquartile ranges (IQR) for continuous variables. In addition, we summarised and categorised reported reasons for outcome discrepancies. We conducted a sensitivity analysis to examine the impact on the percentage of outcomes with a discrepancy by assuming that all outcomes for which authors provided a justification were not discrepancies. This provides a “best-case scenario” for the magnitude of outcome discrepancy, as it assumes that all the justifications are valid, which may not be the case. To examine whether there was an association between a result being favourable to the interruption and its completeness of reporting (fully reported / partially or qualitatively reported), we fitted a marginal logistic regression model using generalised estimating equations, with an exchangeable working correlation, robust standard errors clustered by study, and fixed effects for: whether a result is favourable to the interruption or not, and potential confounders: type of funding (industry, non-industry [government, education institutions or not-for-profit entities] or no funding [as declared by authors]) ( 27 ) and outcome primacy (primary or secondary / unclassified) ( 16 ). Results are reported as odds ratios (OR) with 95% CIs. Analyses were undertaken in Stata version 18 ( 28 ). RESULTS 1. Overview of the search process & extracting outcomes/results Of the 148 full texts retrieved from the literature search, 120 met the inclusion criteria for an eligible ITS study protocol ( Figure 2 ). A search for results reports matching these 120 protocols identified 467 potentially eligible full texts, from which 46 met our inclusion criteria as eligible results reports. Of the 120 ITS studies with protocols, 44 studies (corresponding to 44 protocols and 46 results reports) constituted the sample for this study. From these studies, we identified 572 eligible outcomes and 860 eligible results corresponding to the identified primary ITS research question ( Figure 2 ). Download figure Open in new tab Figure 2. Flowchart showing processes of literature search and screening 2. Characteristic of included studies Most studies were supported by non-industry funding (37/44, 84%), most were conducted in high-income countries (36/44, 82%), and the majority examined practice change interventions in a clinical setting (25/44, 57%) ( Table 2 ). Half of the studies were registered (22/44, 50%). Three-quarters of the studies outlined methods for other study designs in addition to the ITS study analysis (e.g., process evaluations) (33/44, 75%). View this table: View inline View popup Table 2. Characteristics of included studies 3. Characteristics of included outcomes We extracted 572 outcomes from the included studies; 516/572 (90%) outcomes were specified in the protocols and 433/572 (76%) outcomes were specified in the results report. Nearly all outcomes were “impact” outcomes (553/572, 97%), while 19/572 (3%) were “control” outcomes. 234/572 (41%) outcomes were one-to-one matched (i.e., an outcome in the protocol matched with a single outcome from the results report) and 165/572 (29%) were one-to-many matched (i.e., an outcome in the protocol matched with multiple outcomes in the results report). The remaining 173/572 (30%) had no match, including 117/572 (20%) outcomes that appeared only in the protocol and 56/572 (10%) outcomes that appeared only in the results report ( Table 3 ). View this table: View inline View popup Download powerpoint Table 3. Characteristics of included outcomes 4. Characteristics of included results We identified 860 eligible results corresponding to the identified primary ITS research question, with a median of 11 results (IQR 4 to 26) per study and 1 result (IQR 0 to 3) per outcome. Of the 860 results, 201/860 results (23%) were classified as favourable to the interruption, 641/860 results (75%) were classified as not favourable to the interruption, and 18/860 (2%) could not be classified due to lack of information. The median percentage of results favourable to the interruption per study was 20% (IQR 3% to 50%) ( Table 4 ). View this table: View inline View popup Download powerpoint Table 4. Patterns and discrepancies in reporting of outcomes 5. Completeness in reporting at the study, outcome and result level At the study level, in 12/44 (27%) studies, all of the outcomes had all of their results fully reported; in 7/44 (16%) studies, all of the outcomes had reported results, but to a varying degree of completeness. The remaining 25/44 (57%) studies had at least one outcome without any reported result ( Figure 3 ). Download figure Open in new tab Figure 3. Bar plot showing proportion of outcomes reported to different levels of completeness per study and overall At the outcome level, more than half of the outcomes had all of their results fully reported (315/553, 57%); 47/553 (8%) had at least one fully reported result but not all; 12/553 (2%) had only partially reported results but none that were fully reported, and 47/553 (8%) had only qualitatively reported results. A quarter of the outcomes (132/553, 24%) had no eligible results reported, or were not mentioned in the results report at all ( Table 4 ). At the result level, most of the results presented in the results reports were fully reported (741/860, 86%), with a median of 6 fully reported results per study (IQR 4 to 18) and 1 fully reported result per outcome (IQR 0 to 2). A small number of results were partially reported (28/860, 3%), and 91/860 results (11%) were qualitatively reported ( Table 5 ). In 31/44 (70%) studies, all of the reported results were fully reported; in the remaining 10/44 (23%) studies, the reported results were reported to varying degree of completeness ( Figure 4 ). Two studies only had qualitatively reported results and one study had no eligible result. Download figure Open in new tab Figure 4. Bar plot showing percentage of results reported to different levels of completeness per study and overall View this table: View inline View popup Table 5. Characteristics of results 6. Discrepancies in outcome reporting at the study and outcome level After excluding 19 control outcomes and 165 outcomes with one-to-many matches, 388 outcomes from 42 studies were assessed for discrepancies. At the study level, 31/42 (74%) had at least one outcome with discrepancy and 11/42 (26%) studies had no discrepancy. More than half of the studies were affected by outcomes missing in the results report (25/42, 60%), followed by outcomes missing in the protocol (11/42, 26%), or by discrepancies in outcome primacy (10/42, 24%). At the outcome level, approximately half (202/388, 52%) of the outcomes had a discrepancy in reporting. Specifically, 132/388 (34%) outcomes were specified in the protocol but missing in the results report; 55/388 (14%) outcomes were reported in the results report but missing in the protocol; 15/388 (4%) outcomes had a discrepancy in their outcome primacy. The remaining 186/388 outcomes (48%) were not discrepant. The median percentage of outcomes with any discrepancy per study was 27% (IQR 0% to 71%). The median percentage of outcomes per study with no discrepancy was 40% (IQR 0% to 83%) ( Table 4 ). Of the 202 outcomes with discrepancies, a justification was provided for the discrepancy for 56/202 (28%) outcomes from 11/31 (35%) studies ( Table 6 ). No justification was provided for a change in primacy of the outcome for any outcome with this type of discrepancy. The most common reasons for not reporting an outcome specified in the protocol were that rates of the outcomes were too low to undertake the analysis (affecting 22/202 [11%] outcomes in 4 [13%] studies) and the outcome was deemed unsuitable in evaluating the effect of intervention (affecting 20/202 [10%] outcomes in 3 [10%] studies). Outcomes were added in the results reports for various reasons, primarily because data for originally planned outcomes could not be collected (affecting 5/202 [2%] outcomes in 1 [3%] study). Our sensitivity analysis, in which we considered justified outcome discrepancies to no longer be discrepancies, yielded 38% (146/388) of the outcomes with a discrepancy in reporting, affecting 69% (29/42) studies (Supplementary File S6). View this table: View inline View popup Download powerpoint Table 6. Justification for outcome discrepancies 7. Selective reporting of results Of the 201 results favourable to the interruption, 183 were fully reported (91%), while of the 641 results not favourable to the interruption, 558 were fully reported (87%). The adjusted odds ratio (estimated from the logistic regression model) was OR=1.06 (95% CI 0.74 to 1.53) (Supplementary File S7) suggesting that the odds of a result being fully reported was slightly larger for results favourable to researchers’ hypotheses, but that there was uncertainty in this association. DISCUSSION 1. Principal findings and comparison with the literature Our study provides important insights about the extent of selective outcome reporting bias in ITS studies. We identified 44 ITS protocols and their published results, identifying 572 outcomes, and 860 results from these studies. Our findings highlight several issues with outcome reporting among ITS studies. First, we observed a high frequency of discrepancies in outcome reporting between the study protocols and their corresponding final reports. Three quarters (74%) of the studies had at least one outcome with a discrepancy. More than half (52%) of the outcomes assessed had a discrepancy, higher than that observed in studies investigating outcome discrepancies among systematic reviews (38% ( 17 )) and similar to that among randomised trials (54% ( 20 )). The most prevalent type of discrepancy was the omission of outcomes that were planned in the protocol, accounting for 34% of the assessed outcomes. This type of discrepancy could importantly affect an evidence base if the decision not to report the planned outcome is made based on the nature of the results. Justification for changes in outcome reporting was infrequent (only 28% of outcomes with discrepancies were given justifications). Expectedly however, most of the justifications related to practical issues (e.g., issues with the data collection or implementation). Ascertaining other reasons for changes (e.g., whether it was based on statistical significance of the outcome, whether the researchers considered the findings relevant to research question or important for readers) ( 29 ), may be more appropriately investigated by surveying the ITS authors. Second, many outcomes were too broadly defined in the protocol to allow assessment of discrepancies. Nearly 30% of outcomes in the protocol could be matched to multiple outcomes in the results reports; for example, in one protocol, the primary outcome was “the number of antibiotics prescriptions”, while the results report included separate outcomes for the number of prescriptions of different antibiotics (e.g., amoxicillin, cephalexin and ciprofloxacin). A broadly-defined outcome in the protocol leaves the door open for selective reporting, or presenting outcomes with significant findings as if they were planned outcomes ( 30 ). An example of a well-defined outcome, with reporting of each element, is provided in Table 1 . Third, non-reporting of outcomes is prevalent, with 57% of studies having at least one unreported outcome, and 24% of the outcomes having no results reported at all. The percentage of studies with unreported outcomes (57%) is slightly lower than that found among outcomes in randomised trials (60-88%) ( 31 , 32 ). We note that for these investigations, data from unreported outcomes was solicited from trial investigators ( 31 , 32 ), which was not done in our study. However, the frequency of incomplete reporting of outcomes in our study (18%) was substantially less compared with what has been found in randomised trials (31-65%). Lastly, our examination of the nature of the results (i.e. the direction and statistical significance) on completeness of reporting was uncertain due to the width of the confidence interval. Studies examining this association in randomised trials (using a similar classification of reporting completeness) ( 31 , 32 ), found large associations between statistical significance and complete reporting of results (odds ratios ranging from 2.4 to 4.7). Moreover, compared to randomised trials, ITS studies are not required to be registered and there were fewer ITS protocols published. Researchers publishing protocols are perhaps more likely to be aware of, and thus more adherent to, reporting requirements, including completely reporting all results. 2. Implications for practice Our findings suggest that authors of ITS studies do not report at least some of the planned outcomes. At best, this means that not all evidence is available for evidence synthesis; at worst, that the available evidence is biased, as will occur when the authors’ decision to report results has been based on the nature of those results. This highlights the need for registration of ITS studies and analyses, and publication of protocols and statistical analysis plans, which describe all outcomes in detail. Those who synthesise evidence from ITS studies should seek out registration details, a protocol or statistical analysis plan for each study, to assess for non-reporting of outcomes, even although they are not commonly reported ( 14 ). In addition, readers should be alert to missing outcomes that would be expected to be measured given the interruption and the context, such as blood pressures for interventions framed as “directed at individuals with hypertension” ( 14 ). Reporting sufficiently detailed outcomes will allow readers to detect any changes in the outcomes reported in the final publication. Our framework for defining an outcome in the context of ITS data provides a template for the elements that could be reported ( Table 1 ). It is inevitable that in some ITS studies, changes to outcomes will be required due to, for example, logistical or analytical challenges (e.g., planned data collection was not feasible). Reporting such changes, along with the rationale, is a recommendation in the CONSORT 2025 statement (item 10) ( 33 ), a reporting guideline for randomised trials that similarly applies to ITS studies. Stricter monitoring and enforcement of outcome reporting by journal editors, peer reviewers and funders may improve reporting of all planned outcomes ( 34 ). 3. Strengths and limitations To our knowledge, this is the first study to investigate selective outcome reporting in ITS studies. Many studies have investigated outcome reporting bias in clinical trials ( 12 , 35 , 36 ) or systematic reviews of RCTs ( 17 ), but few have been undertaken for non-randomised studies, potentially due to the difficulty in creating an inception cohort (i.e. a set of studies known to have been initiated, irrespective of their results) ( 19 ). Our framework to define outcomes was based on a previous framework developed for outcomes in clinical trials ( 25 , 33 ). A key difference with ITS outcomes is that the individual measurements are typically aggregated (using summary statistics) over units of time (e.g., months). We introduced new elements of outcome definitions to accommodate this difference, which allowed us to match the outcomes more accurately and consistently between protocols and results reports. Our study is not without limitations. Our sample includes only ITS studies with published protocols, which is a very small fraction of all ITS studies published. ITS studies with published protocols may be more completely reported than those without protocols. If this is true, our study might have underestimated the extent of outcome reporting discrepancies and incomplete reporting of results. Second, in many protocols, authors did not explicitly report what effect measures they planned to report. Therefore, we could not assess the extent of selective reporting at the result level (i.e. how many results were planned but not reported). CONCLUSION Even among ITS studies with published protocols, which likely represent best practice, non-reporting of outcomes and discrepancies in outcome reporting were prevalent. Outcomes should be pre-specified in a protocol or registry to mitigate selective reporting, and described in sufficient detail in any publication (protocol or results reports) to enable readers detect any change in outcome reporting. Data Availability All datasets and analytical code can be found on the Open Science Framework (DOI: osf.io/9jqh3 and osf.io/b837n). https://osf.io/9jqh3 https://osf.io/b837n Availability of data and materials All datasets and analytical code can be found on the Open Science Framework (DOI: osf.io/9jqh3 and osf.io/b837n ). Competing interests The authors declare no conflict of interest. Funding This work was supported by a Monash University’s Departmental Scholarship for P-YN’s Doctor of Philosophy (PhD) research program. MJP is supported by a NHMRC Investigator Grant (GNT2033917) and was supported by an Australian Research Council Discovery Early Career Researcher Award (DE200101618), the Research Support Package of Joanne E McKenzie’s NHMRC Investigator Grant (GNT2009612) and a Monash University Future Leader Postdoctoral Fellowship (FLPF23-1069865460) during the conduct of this research. SLT and EK were funded by the Research Support Package of Joanne E McKenzie’s NHMRC Investigator Grant (GNT2009612). JEM was supported by an NHMRC Investigator Grant (GNT2009612). Transparency statement The authors declare that the manuscript is an honest, accurate, and transparent account of the study being reported; that no important aspects of the study have been omitted; and that any discrepancies from the study as originally planned have been reported in the Supplementary Files. Patient and Public Involvement Patients and the public were not involved in the design and conduct of this methodological research study. Contributorship Statement All authors have reviewed and approved the final manuscript. All listed authors meet authorship criteria and that no other individuals meeting the criteria were omitted. JEM acted as the guarantor of this paper. PYN: conceptualisation, data curation, formal analysis, investigation, methodology, writing – original draft preparation EK: investigation, validation, writing – review and editing MJP, SLT: investigation, validation, supervision, writing – review and editing JEM: conceptualisation, methodology, validation, formal analysis, supervision, writing – review and editing ACKNOWLEDGEMENT Ethics approval and informed consent This study was conducted using publicly available data and did not involve human participants, identifiable personal data, animal subjects, or interventions requiring ethical approval. Data related to the included studies was deidentified and anonymised using unique identification numbers. REFERENCES 1. ↵ Craig P , Campbell M , Deidda M , Dundas R , Green J , Katikireddi SV , et al. Using natural experiments to evaluate population health and health system interventions: new framework for producers and users of evidence . BMJ . 2025 Mar 28; 388 : e080505 . OpenUrl FREE Full Text 2. ↵ Jandoc R , Burden AM , Mamdani M , Lévesque LE , Cadarette SM . Interrupted time series analysis in drug utilization research is increasing: systematic review and recommendations . J Clin Epidemiol . 2015 Aug 1; 68 ( 8 ): 950 – 6 . OpenUrl CrossRef PubMed 3. ↵ Hategeka C , Ruton H , Karamouzian M , Lynd LD , Law MR . Use of interrupted time series methods in the evaluation of health system quality improvement interventions: a methodological systematic review . BMJ Glob Health . 2020 Oct; 5 ( 10 ): e003567 . OpenUrl CrossRef PubMed 4. ↵ Ewusie JE , Soobiah C , Blondal E , Beyene J , Thabane L , Hamid JS. Methods , Applications and Challenges in the Analysis of Interrupted Time Series Data: A Scoping Review . J Multidiscip Healthc . 2020 May 13; 13 : 411 – 23 . OpenUrl PubMed 5. ↵ Bernal JL , Soumerai S , Gasparrini A . A methodological framework for model selection in interrupted time series studies . J Clin Epidemiol . 2018 Nov 1; 103 : 82 – 91 . OpenUrl CrossRef PubMed 6. Lopez Bernal J , Cummins S , Gasparrini A . Interrupted time series regression for the evaluation of public health interventions: a tutorial . Int J Epidemiol . 2017 Feb 1; 46 ( 1 ): 348 – 55 . OpenUrl CrossRef PubMed 7. ↵ Turner SL , Karahalios A , Forbes AB , Taljaard M , Grimshaw JM , McKenzie JE . Comparison of six statistical methods for interrupted time series studies: empirical evaluation of 190 published series . BMC Med Res Methodol . 2021 June 26; 21 ( 1 ): 134 . OpenUrl CrossRef PubMed 8. Turner SL , Forbes AB , Karahalios A , Taljaard M , McKenzie JE . Evaluation of statistical methods used in the analysis of interrupted time series studies: a simulation study . BMC Med Res Methodol . 2021 Dec 1; 21 ( 1 ): 1 – 18 . OpenUrl CrossRef PubMed 9. ↵ Schaffer AL , Dobbins TA , Pearson SA . Interrupted time series analysis using autoregressive integrated moving average (ARIMA) models: a guide for evaluating large-scale health interventions . BMC Med Res Methodol . 2021 Dec 1; 21 ( 1 ): 1 – 12 . OpenUrl CrossRef PubMed 10. ↵ Hudson J , Fielding S , Ramsay CR . Methodology and reporting characteristics of studies using interrupted time series design in healthcare . BMC Med Res Methodol . 2019 July 4; 19 ( 1 ): 137 . OpenUrl PubMed 11. ↵ Korevaar E , Karahalios A , Turner SL , Forbes AB , Taljaard M , Cheng AC , et al. Methodological systematic review recommends improvements to conduct and reporting when meta-analyzing interrupted time series studies . J Clin Epidemiol . 2022 May 1; 145 : 55 – 69 . OpenUrl CrossRef PubMed 12. ↵ Kirkham JJ , Dwan KM , Altman DG , Gamble C , Dodd S , Smyth R , et al. The impact of outcome reporting bias in randomised controlled trials on a cohort of systematic reviews . BMJ . 2010 Feb 15; 340 : c365 . 13. Hutton JL , Williamson PR . Bias in Meta-Analysis Due to Outcome Variable Selection Within Studies . J R Stat Soc Ser C Appl Stat . 2000 Sept 1; 49 ( 3 ): 359 – 70 . OpenUrl 14. ↵ Page MJ , Sterne JAC , Boutron I , Hróbjartsson A , Kirkham JJ , Li T , et al. ROB-ME: a tool for assessing risk of bias due to missing evidence in systematic reviews with meta-analysis . BMJ . 2023 Nov 20; 383 : e076754 . OpenUrl FREE Full Text 15. ↵ Hart B , Lundh A , Bero L . Effect of reporting bias on meta-analyses of drug trials: reanalysis of meta-analyses . BMJ . 2012 Jan 3; 344 : d7202 . 16. ↵ Norris SL , Holmer HK , Ogden LA , Fu R , Abou-Setta AM , Viswanathan MS , et al. Selective Outcome Reporting as a Source of Bias in Reviews of Comparative Effectiveness [Internet]. Rockville (MD): Agency for Healthcare Research and Quality (US) ; 2012 [cited 2025 June 20]. Available from: https://www.ncbi.nlm.nih.gov/books/NBK100614/ 17. ↵ Page MJ , McKenzie JE , Kirkham J , Dwan K , Kramer S , Green S , et al. Bias due to selective inclusion and reporting of outcomes and analyses in systematic reviews of randomised trials of healthcare interventions . Cochrane Database Syst Rev . 2014 Oct 1;2014( 10 ):MR000035. 18. ↵ Dwan K , Altman DG , Clarke M , Gamble C , Higgins JPT , Sterne JAC , et al. Evidence for the selective reporting of analyses and discrepancies in clinical trials: a systematic review of cohort studies of clinical trials . PLoS Med . 2014 June; 11 ( 6 ): e1001666 . OpenUrl CrossRef PubMed 19. ↵ Dwan K , Gamble C , Williamson PR , Kirkham JJ , Reporting Bias Group . Systematic review of the empirical evidence of study publication bias and outcome reporting bias - an updated review . PloS One . 2013 ; 8 ( 7 ): e66844 . OpenUrl CrossRef PubMed 20. ↵ Li G , Abbade LPF , Nwosu I , Jin Y , Leenus A , Maaz M , et al. A systematic review of comparisons between protocols or registrations and full reports in primary biomedical research . BMC Med Res Methodol . 2018 Jan 11; 18 ( 1 ): 9 . OpenUrl PubMed 21. ↵ Song F , Parekh-Bhurke S , Hooper L , Loke YK , Ryder JJ , Sutton AJ , et al. Extent of publication bias in different categories of research cohorts: a meta-analysis of empirical studies . BMC Med Res Methodol . 2009 Nov 26; 9 : 79 . 22. ↵ Nguyen PY , Turner S , Korevaar E , Forbes A , Page MJ , McKenzie JE . Investigation of reporting bias in interrupted time series (ITS) studies: a study protocol . F1000Research [Internet] . 2024 Nov 4 [cited 2025 Apr 11]; Available from: https://f1000research.com/articles/13-153/v2 23. ↵ Nguyen PY , McKenzie JE , Turner SL , Page MJ , McDonald S . Development of a search filter to retrieve reports of interrupted time series studies from MEDLINE and PubMed . Res Synth Methods . 2024 ; 15 ( 4 ): 627 – 40 . OpenUrl PubMed 24. ↵ Dusetzina SB , Brookhart MA , Maciejewski ML . Control Outcomes and Exposures for Improving Internal Validity of Nonrandomized Studies . Health Serv Res . 2015 Oct; 50 ( 5 ): 1432 – 51 . OpenUrl CrossRef PubMed 25. ↵ Zarin DA , Tse T , Williams RJ , Califf RM , Ide NC . The ClinicalTrials.gov Results Database 26. ↵ — Update and Key Issues . N Engl J Med . 2011 Mar 3; 364 ( 9 ): 852 – 60 . OpenUrl CrossRef PubMed Web of Science 26. ↵ Dickersin K , Li T , Mayo-Wilson E . The Benefits and Challenges of Using Multiple Sources of Information about Clinical Trials [Internet] . Patient-Centered Outcomes Research Institute (PCORI ); 2018 Mar [cited 2025 Sept 24]. Available from : doi: 10.25302/3.2018.ME.13035785 OpenUrl CrossRef 28. ↵ Holst M , Haslberger M , Yerunkar S , Strech D , Hemkens LG , Carlisle BG . Frequency of multiple changes to prespecified primary outcomes of clinical trials completed between 2009 and 2017 in German university medical centers: A meta-research study . PLoS Med . 2023 Oct; 20 ( 10 ): e1004306 . OpenUrl PubMed 28. ↵ StataCorp . Stata Statistical Software: Release 18. TX: StataCorp LLC.: College Station ; 2025 . 29. ↵ Smyth RMD , Kirkham JJ , Jacoby A , Altman DG , Gamble C , Williamson PR . Frequency and reasons for outcome reporting bias in clinical trials: interviews with trialists . BMJ . 2011 Jan 6; 342 : c7153 . 30. ↵ Baldwin JR , Pingault JB , Schoeler T , Sallis HM , Munafò MR . Protecting against researcher bias in secondary data analysis: challenges and potential solutions . Eur J Epidemiol . 2022 Jan 1; 37 ( 1 ): 1 – 10 . OpenUrl CrossRef PubMed 31. ↵ Chan AW , Krleza-Jerić K , Schmid I , Altman DG . Outcome reporting bias in randomized trials funded by the Canadian Institutes of Health Research . CMAJ Can Med Assoc J J Assoc Medicale Can . 2004 Sept 28; 171 ( 7 ): 735 – 40 . OpenUrl 32. ↵ Chan AW , Hróbjartsson A , Haahr MT , Gøtzsche PC , Altman DG . Empirical evidence for selective reporting of outcomes in randomized trials: comparison of protocols to published articles . JAMA . 2004 May 26; 291 ( 20 ): 2457 – 65 . OpenUrl CrossRef PubMed Web of Science 33. ↵ Hopewell S , Chan AW , Collins GS , Hróbjartsson A , Moher D , Schulz KF , et al. CONSORT 2025 explanation and elaboration: updated guideline for reporting randomised trials . BMJ . 2025 Apr 14; 389 : e081124 . OpenUrl FREE Full Text 34. ↵ Turner L , Shamseer L , Schulz KF , Moher D . Does use of the CONSORT Statement impact the completeness of reporting of randomised controlled trials published in medical journals? A Cochrane review . Syst Rev . 2012 Nov 29; 1 : 60 . 36. ↵ Silagy CA , Middleton P , Hopewell S . Publishing protocols of systematic reviews: comparing what was done to what was planned . JAMA . 2002 June 5; 287 ( 21 ): 2831 – 4 . OpenUrl CrossRef PubMed Web of Science 36. Dwan K , Kirkham JJ , Williamson PR , Gamble C . Selective reporting of outcomes in randomised controlled trials in systematic reviews of cystic fibrosis . BMJ Open . 2013 June 20; 3 ( 6 ): e002709 . OpenUrl Abstract / FREE Full Text View the discussion thread. Back to top Previous Next Posted November 07, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Selective reporting of outcomes and results in interrupted time series studies of health interventions: a methodological study Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Selective reporting of outcomes and results in interrupted time series studies of health interventions: a methodological study Phi-Yen Nguyen , Simon L. Turner , Elizabeth Korevaar , Matthew J. Page , Joanne E. McKenzie medRxiv 2025.11.06.25339702; doi: https://doi.org/10.1101/2025.11.06.25339702 Share This Article: Copy Citation Tools Selective reporting of outcomes and results in interrupted time series studies of health interventions: a methodological study Phi-Yen Nguyen , Simon L. Turner , Elizabeth Korevaar , Matthew J. Page , Joanne E. McKenzie medRxiv 2025.11.06.25339702; doi: https://doi.org/10.1101/2025.11.06.25339702 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Public and Global Health Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15227) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6597) Geriatric Medicine (668) Health Economics (997) Health Informatics (4534) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3332) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9230) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a0037eb9eab1593a',t:'MTc3OTUzMzM3Nw=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();