Quality and methodological heterogeneity of COVID-19 vaccine safety studies focusing on the myocarditis safety signal: A systematic review, meta-analysis and meta-regression

Quality and methodological heterogeneity of COVID-19 vaccine safety studies focusing on the myocarditis safety signal: A systematic review, meta-analysis and meta-regression | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Quality and methodological heterogeneity of COVID-19 vaccine safety studies focusing on the myocarditis safety signal: A systematic review, meta-analysis and meta-regression View ORCID Profile Tu P. H. Trang , Marloes T. Bazelier , Olaf H. Klungel , Sophie H. Bots doi: https://doi.org/10.1101/2025.06.18.25329834 Tu P. H. Trang 1 Division of Pharmacoepidemiology and Clinical Pharmacology, Utrecht Institute for Pharmaceutical Sciences, Utrecht University , Utrecht, The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Tu P. H. Trang For correspondence: t.p.h.trang{at}uu.nl Marloes T. Bazelier 1 Division of Pharmacoepidemiology and Clinical Pharmacology, Utrecht Institute for Pharmaceutical Sciences, Utrecht University , Utrecht, The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site Olaf H. Klungel 1 Division of Pharmacoepidemiology and Clinical Pharmacology, Utrecht Institute for Pharmaceutical Sciences, Utrecht University , Utrecht, The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site Sophie H. Bots 1 Division of Pharmacoepidemiology and Clinical Pharmacology, Utrecht Institute for Pharmaceutical Sciences, Utrecht University , Utrecht, The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Background The risk of myocarditis following COVID-19 vaccines has been widely investigated, yielding highly variable effect estimates. It remains unclear how much of this variation can be explained by methodological differences and study quality. This study aimed to evaluate the methodology of observational studies on myocarditis risk post-COVID-19 vaccination and identify sources of heterogeneity. Methods We systematically searched PubMed and EMBASE for observational studies reporting relative risks of myocarditis after COVID-19 vaccination published between Dec 2022 and Oct 2023. Risk of Bias (RoB) was assessed using the ROBINS-I tool. Meta-analysis and multivariable meta-regression were conducted for studies addressing comparable population-intervention-comparator-outcome (PICO) questions. Results We included 30 studies, comprising 35 study designs. We identified considerable variability in design elements including risk window length (7-288 days), reference period timing (three different ways), and control for COVID-19 disease (five different approaches). Thirteen (37%) study designs had serious or critical overall RoB. We observed strongest heterogeneity between studies in general population for dose 2 of BNT162b2 and mRNA-1273 compared to unvaccinated individuals/reference period (I 2 =96%, prediction interval (PI) of relative risk 0.40 – 20.20; I 2 =92%, PI 1.23 – 88.63, respectively). Meta-regression for the former PICO indicated that after adjusting for age and sex, effect estimates of samples with 28-day risk window were 0.56 times (95% CI 0.43-0.72) lower than those with 7-day risk window. The effect of study design, outcome definition, approaches to handle COVID-19 infection and overall RoB were not significant in the meta-regression. Conclusions We observed considerable variation in design specifications including risk window length, reference period timing, and control for COVID-19 disease. Significant heterogeneity in effect estimates was identified, especially for second doses. Design choices like risk window length may explain some of this heterogeneity. Future vaccine safety studies should include sensitivity analyses to explore the effect of design choices on their findings. Background The rapid development and distribution of Coronavirus disease 2019 (COVID-19) vaccines represents an unprecedented scientific accomplishment. The accelerated approval and massive roll-out of these vaccines also highlighted the importance of postmarketing safety monitoring to detect rare side effects for which the clinical trials were underpowered. Soon after vaccine roll-out, myocarditis was flagged as an important rare adverse event, and this signal was later confirmed by multiple real-world evidence studies across the world 1 , 2 . However, the effect estimates varied considerably between studies. For example, the reported risk of myocarditis after the second dose of BNT-162b2 compared to no vaccination ranged between 0.79 and 29.7 3 , 4 . Such heterogeneous findings pose challenges to benefit-risk assessment for policy decision-making, as well as undermine public trust in vaccine safety. The sources of heterogeneity in epidemiological studies can be roughly categorized into clinical diversity and methodological diversity 5 . In the context of vaccine safety, the clinical diversity entails the difference across studies in participant characteristics that affect their susceptibility to the adverse events, as well as the difference in vaccination policies and healthcare utilizations. Understanding these aspects helps to tailor the vaccination program to different populations and evaluate the transportability of evidence across different settings 6 . The methodological heterogeneity reflects the diversity in choice of designs and study parameters such as follow up length, selection of comparators, and covariate adjustments, as well as the degree of bias in each study 5 . Designing postmarketing vaccine safety study incurs special challenges, as vaccines are often administered by seasonal schedule which requires adjusting for time trends. In addition, vaccines are administered either to targeted populations or with high coverage, making it difficult to find appropriate control groups 7 . Various specialized methods have been developed and applied to accommodate these features, such as the case-centred analysis and self-controlled methods, yet their validity depends on meeting strict assumptions 7 . Prior simulation and empirical studies have suggested that the degree of biased results differs per study design, and that design parameters could partly explain the heterogeneity in study results 6 , 8 . These findings may however have limited generalizability to the unique situation of the COVID-19 pandemic, with constraints in healthcare utilization and strong time trends in vaccination. It is thus of interest to characterise the methodologies of COVID-19 vaccine safety studies and their impact on the heterogeneity of effect estimates. These insights could be helpful in assessing the consistency and robustness of evidence and informing evidence synthesis and future study design selection in similar contexts. Taking the potential risk of myocarditis following COVID-19 vaccination as a case study, we aimed to provide an overview of methodologies of inferential, comparative observational studies that evaluated COVID-19 vaccine safety signals. The first objective was to describe the study designs and analytical approaches used in observational studies on COVID-19 vaccine safety. The second objective was to assess the heterogeneity in findings across these studies and investigate the impact of study design and analysis choices on the observed heterogeneity. Methods Study design We conducted a systematic review of the methodology of observational studies investigating the risk of myocarditis following COVID-19 vaccination. Effect estimates were grouped using the PICO (Population, Intervention, Comparator, Outcome) framework and meta-analysed within their PICO group to quantify statistical heterogeneity. Sources of heterogeneity were investigated using meta-regression. We followed the Preferred Reporting Items for Systematic Reviews and Meta-Analyses 2020 (PRISMA) checklist 9 . The full study protocol is provided in Supplementary file 1. Eligibility criteria We included studies fulfilling all following criteria: 1) Observational study that aimed to quantify the risk of myocarditis (with or without pericarditis) after COVID-19 vaccination; 2) The exposure was defined as at least one dose of any COVID-19 vaccine, where the exposed could either be an individual or a specific time period within an individual; 3) The comparator was defined as either no exposure to any COVID-19 vaccine or exposure to a different COVID-19 vaccination regimen, where the unexposed could either be an individual or a specific time period within an individual; 4) A comparative effect measurement (ratio or difference) was reported between the exposure and comparator groups; 5) Published in English as a peer-reviewed scientific article. 6) Published between December 11 th , 2020 (first COVID-19 vaccine emergency use authorization) and October 9 th , 2023 10 7) Full-text available Studies aimed at safety signal detection (which employ data mining techniques without prior hypothesis), or signal strengthening (such as disproportionality analysis, ecological studies, and crude before-after comparison) were excluded. Editorials, letters to editor, and commentaries were also excluded. Search strategy Structured searches were conducted on PubMed and Embase on October 9 th , 2023 to identify relevant records. The search strings combined keywords and subject heading terms related to COVID-19 vaccines, safety, and comparison of effect measure (Supplementary file 1). Study selection The records were de-duplicated using Endnote X9 11 . Records were screened for inclusion by two independent reviewers (TT and SB) using ASReview, an open-source machine learning algorithm 12 . Discrepancies were resolved through discussion. The records were screened in two rounds with different machine-learning models to enhance the coverage of relevant records 13 , using the results of the first round as prior knowledge for the second round. Each screening round was stopped after 100 consecutive records were marked as “irrelevant”, or 50 consecutive irrelevant records after screening the first 1000 records. After the title and abstract screening, the full text of the potentially eligible articles were assessed by three reviewers (TT, SB, MB). Each reviewer screened 33% and independently double checked 33% of the total records. Discrepancies between reviewers were resolved by discussion among the research team. Data extraction Data extraction was performed by three reviewers (TT, SB, MB). Each reviewer extracted 33%, and one reviewer (TT) double checked randomly 50% of the total eligible studies. The following information was extracted from each study: study setting, data source and population; aspects of study design and statistical analysis, exposures, comparators, outcomes and effect estimates (Supplementary file 2). All primary analyses, secondary, sub-group and sensitivity analyses pertaining to the myocarditis outcome were extracted. Risk of bias assessment The risk of bias (RoB) of each eligible study was assessed using the Risk of Bias in Non-randomized Studies of Interventions (ROBINS-I) tool 14 . We adapted the tool to also cover assumptions specific to self-controlled and case control designs, as these were originally not included (Supplementary file 3). For self-controlled designs, we first checked whether the major validity assumptions as outlined by Whitaker et al (2006) were met 15 . Only self-controlled studies that fulfilled all validity assumptions underwent the ROBINS-I tool assessment. Each of three reviewers (TT, SB, MB) assessed 33% and independently double checked 33% of the total studies. Descriptive analysis Continuous variables are presented as median and interquartile range. Categorical variables are summarised with frequencies and percentages. A post-hoc analysis was conducted to assess the consistency of results in studies where two study designs and two approaches to handle COVID-19 infection were used. Three agreement metrics were evaluated for each pair of effect estimates 16 : 1) Statistical significance agreement: whether two estimates and their confidence intervals are on the same side of the null; 2) Estimate agreement: whether the point estimate of one design/approach is within the 95% CI of the other design/approach; and 3) Standardized difference agreement: whether the absolute value of the standardized difference is < 1.96). Meta-analysis Meta-analysis of primary analyses of the included studies were conducted to characterize the heterogeneity between studies. The effect estimates were grouped by PICO question: (P) General population or population with specific conditions, (I) Vaccine brand and dose, (C) unvaccinated individual/period or active comparator, and (O) outcome. As myocarditis was rare (<5% incidence), different ratio metrics (RR, OR, HR, IRR) could be combined 17 . Random-effect meta-analysis was conducted for PICOs with at least 3 studies. Tau-squared, I 2 and prediction interval were calculated to measure degree of heterogeneity. For studies that reported multiple effect estimates per PICO for overlapping samples (e.g. using different risk windows), a decision rule was applied to choose one single effect estimate per PICO per study 18 (Supplementary file 1). Univariate sub-group analyses were conducted for PICOs with at least three studies to explore the possible causes of heterogeneity in effect estimates between studies. The sub-groups included: 1) Age restriction of the study (general population, younger population only (50 years old); 2) Risk window length (within 14 days after vaccination, within 28 days after vaccination, or other lengths); 3) Study design (Cohort, self-controlled, or other designs); 4) Overall RoB (Low-moderate or serious-critical); 5) RoB of confounding domain (Low-moderate or serious-critical); 6) Outcome definition (myocarditis only, or myocarditis/pericarditis); and 7) Approach to handle COVID-19 infection in the analysis (No approach applied, approaches that measure direct effect of vaccination only (including censoring COVID-19 cases and excluding COVID-19 cases before and during the study), or other approaches). Contour-enhanced funnel plots and Egger’s test of asymmetry were generated for PICOs with at least 10 studies to investigate small-study effects and differentiate asymmetry that is due to non-reporting biases from that due to other factors 19 – 22 . Examination of sources of heterogeneity by meta-regression For PICOs with at least 10 studies without missing information on the explanatory variables and with considerable heterogeneity ( I 2 > 75%), multivariable random-effect meta-regression models were fitted to further investigate sources of heterogeneity, controlling for confounding effect of patient characteristics. The outcome was the log of the relative effect estimate. The explanatory variables included median age of the study sample, proportion of male in the study sample, study design, length of risk window, the approach to handle COVID-19 infection in the analysis, outcome definition, and overall RoB. All primary analyses, subgroup analyses and sensitivity analyses related to explanatory variables of interest were included in the model to increase statistical power and to estimate their between-study and within-study effects. Robust variance estimation method was employed to account for dependency between effect estimates from the same study, using the correlated and hierarchical effects working model 23 . The correlation between two effect sizes in the same study (p) is assumed to be constant and equal 0.7. The base meta-regression model included two case-mix variables, age, and sex (Model 1). Next, the effect of methodological factors (study design, risk window length, outcome definition, handling of COVID-19 infection, and RoB) was investigated one by one, adjusted for age and sex (Model 2 to 6). For variables that varied both within a study and between studies (e.g. age and sex distribution, length of risk window, as studies often have subgroup/ sensitivity analysis regarding these variables), both the study means and the study-mean-centred values of these two variables were included in the model, to estimate separately their between-study and within-study effect, respectively 24 , 25 . The regression coefficients (β), their associated 95% CI and the Wald-style F test p-value of each variable (indicating the overall significance of a variable) were reported 25 . Results Of all 10883 references identified, 266 full-text publications were reviewed. Thirty studies met the eligibility criteria and were included in systematic review (Supplementary figure S1) 3 , 4 , 26 – 53 . These studies comprised 35 designs (as five studies used two designs) and 36 distinct PICOs. General characteristics The eligible studies were mostly conducted in Western Europe (14 studies, 47%) and North America (7 studies, 23%) ( Table 1 ). Vaccination registries were the main data source for exposure ascertainment (20 studies, 67%), while the data sources for outcome and covariate ascertainment were more diverse. The two most common study designs for primary analysis were cohort (15 studies, 50%) and self-controlled designs (11 studies, 37%). Twenty-seven studies (87%) evaluated vaccine safety in the general population, while four (13%) targeted population with special conditions. View this table: View inline View popup Table 1. General characteristics of 30 eligible studies Among 36 distinct PICOs investigated in the included studies (Supplementary Table S1), most PICOs were on a specific dose of a specific vaccine (23 PICOs), with unvaccinated individuals or reference period as comparator (29 PICOs). Seven PICOs had active comparators, either comparing mRNA and BNT-162b2 26 , 32 , 34 , 35 , 42 or comparing heterologous and homologous vaccination courses 28 . The two PICOs examined by the most studies were effect of first dose and effect of second dose of BNT-162b2 compared to unvaccinated individual/period (14 studies per PICO). When a combination of doses was studied, it could be defined as “first and second dose” 29 , 32 , “first or second dose” 34 , 36 , 46 or “any dose” 31 , 33 , 38 , 39 , 43 . Diversity of study design parameters Definition of “unvaccinated individuals” and “reference period” as comparator There were five cohort studies that used “unvaccinated individuals” as comparator. One used historical controls 49 , two matched vaccinated and unvaccinated on the vaccination date 4 , 29 , and the other two assigned an artificial index date to unvaccinated controls 41 , 43 . For two case-control studies, one used hospital controls 39 , while one selected individuals without the outcome in the whole population 53 . Reference period within an individual was used as comparator in five cohort studies and all self-controlled studies (total 18 studies, 60%). Among them, ten studies selected all periods before, between and after risk period as reference period 3 , 27 , 31 , 33 , 40 , 44 – 46 , 50 , 52 , six studies only used period before first exposure 30 , 36 – 38 , 47 , 51 , and two only use period after the risk period 34 , 48 . Outcome definition and ascertainment Fifteen studies (50%) included only myocarditis as outcome, eleven studies (37%) included myocarditis and pericarditis as a composite outcome, and four studies (13%) included both definitions 27 , 33 , 34 , 39 . Risk window length There were 15 studies (50%) that used at least two different risk window lengths in their analysis. The most chosen risk window length was 28-day post-vaccination (15 studies, 50%), followed by 7-day post-vaccination (12 studies, 40%). Day 0 (day of vaccination) was included in the risk window in 13 studies (43%) 3 , 4 , 29 , 31 , 32 , 34 , 36 – 38 , 50 – 52 and excluded in 7 studies (23%) 28 , 30 , 45 , 47 – 49 , 53 . Two studies included both risk windows with and without day 0, 27 , 40 and three studies took day 0 as a separate risk window 33 , 44 , 46 . The risk window definition was unclear in five studies 35 , 39 , 41 – 43 . Control for confounding In all 35 study designs, age and sex were controlled for. Seasonality, comorbidities, outcome history and socioeconomic status were controlled for in 83%, 80%, 77% and 69% designs, respectively. COVID-19 infection could act as both confounder and mediator of the association between vaccination and myocarditis. In three out of 35 designs (11%), cases with COVID-19 infection before the study period were excluded 34 , 35 , 48 , and two designs excluded COVID-19 cases both before and during study period 33 . COVID-19 cases were censored in 7 designs (20%) 28 , 29 , 32 , 36 , 37 , 51 . In two designs (6%), COVID-19 infection was included as covariate in the model 42 , 53 . In 20 designs (57%), COVID-19 infection was not considered in the main analysis, among them 6 designs had sensitivity analysis that used one of the above approaches 3 , 27 , 30 , 40 , 44 , 50 . Risk of bias in the included studies Overall risk of bias was low for one (3%), moderate for 21 (60%), serious for 10 (29%), and critical for 3 (9%) designs ( Figure 1 ). Most of this variability could be attributed to the confounding domain, where 31% of designs were rated as either critical or serious risk of bias. All 13 included self-controlled studies met the major validity assumptions, and most (11 studies) implemented correct adjustments to ensure that exposure was not dependent on occurrence of the outcome. The full RoB assessment for each study is presented in Supplementary file 4. Download figure Open in new tab Figure 1. Proportion of risk of bias levels among 35 designs Studies that used two study designs There were four studies that used two study designs to answer the exact same PICO questions, resulting in a total of 31 pairs of effect estimates. Three studies 3 , 27 , 36 conducted SCCS and cohort, and one 33 conducted SCCS and case-control. Generally, the effect estimates from SCCS were higher than other designs ( Figure 2 ). In 7 pairs (23%) one of the three agreement metrics was not met. Download figure Open in new tab Figure 2. Comparison of pairs of effect estimates in studies that used two study designs to answer the exact same PICO questions. A) Pairs where both relative effect estimates were lower or equal to 2. B) Pairs where one of the relative effect estimates was higher than 2. Arrowheads indicate that the bounds of the confidence interval exceed the limit of the plot. Studies that used two approaches to handle COVID-19 infection There were 10 studies that applied two approaches to control for COVID-19 infection in the exact same PICO questions, resulting in a total of 40 pairs of effect estimates 3 , 27 , 32 , 36 , 38 , 40 , 44 , 45 , 48 , 50 . Most pairs met all agreement metrics; there were only two pairs (5%) without statistical significance agreement ( Figure 3 ). Download figure Open in new tab Figure 3. Comparison of pairs of effect estimates in study that used two approaches to handle COVID-19 infection in an exact same PICO question. A) Pairs where both relative effect estimates were lower or equal to 2. B) Pairs where one of the relative effect estimates was higher than 2. Arrowheads indicate that the bounds of the confidence interval exceed the limit of the plot. Meta-analysis We performed a meta-analysis for 12 PICOs answered in more than two studies (Supplementary Table S2). The funnel plots suggested low risk of publication bias (Supplementary Figure S2-S3). Heterogeneity was highest between estimates for the second dose of BNT-162b2 versus unvaccinated individuals/person-time in the general population (15 studies, I 2 = 95.6% (95% CI: 91.4 – 96.8)) and lowest between estimates for the second dose of CoronaVac versus unvaccinated individuals/person-time in the general population (3 studies, I 2 = 0% (95% CI 0-89.6)). For the first and second doses of both BNT-162b2 and mRNA-1273 versus unvaccinated individuals/person-time in the general population, we observed larger heterogeneity for the second dose compared to the first dose (indicated by higher I 2 and wider prediction interval), even though the estimates came from the same set of studies ( Figure 4 ). Download figure Open in new tab Figure 4. Meta-analysis of the risk of myocarditis following BNT-162b2 dose 1 (top left) and dose 2 (top right), and mRNA-1273 dose 1 (bottom left) and dose 2 (bottom right) compared with unvaccinated individuals/ reference periods, in general population. Note that the objective of the meta-analysis was to measure heterogeneity and not to estimate the pooled effect. Subgroup analysis showed no clear effect of study design, outcome definition, or risk of bias (overall or confounding only) on heterogeneity between studies (Supplementary Tables S3-S9). Subgroup analyses were also conducted for age (12 PICOs), risk window length (9 PICOs), and approaches to handling COVID-19 infection (10 PICOs). These suggested effect estimates were higher in younger populations compared to the general or older population (statistically significant in 4/12 PICOs), and in studies using shorter risk windows compared to longer ones (statistically significant in 5/9 PICOs). There was no clear trend in the effect of approaches used to handle COVID-19 infection. Meta-regression Meta-regression was performed for the only PICO with at least 10 studies: Dose 2 of BNT-162b2 compared with unvaccinated individuals/ reference period in general population ( Table 2 ). The within-study estimates showed that compared to female-only populations, the average effect estimate in male-only study populations was 1.34 times greater (95% CI 1.07 – 1.67). For age, the average effect estimates were 0.55 times (95% CI:0.33-0.94) and 0.33 times (95% CI 0.14-0.79) lower in populations with a median age of 30-49 years and >49 years, respectively, compared to populations with a median age under 30 years. For risk window length, the average effect estimates were 0.56 times (0.42-0.72) lower in studies with a 28-day risk window compared to those with a 7-day risk window. View this table: View inline View popup Table 2. Results of meta-regression models for PICO of Dose 2 of BNT-162b2 compared with unvaccinated individuals/ reference period in general population. Model 1 includes effect of age and sex, model 2 to 6 include methodological factors one by one, adjusting for age and sex. We were unable to estimate the between-study effect for sex due to overfitting and the between-study effect for age and risk-window length were not significant. The overall effects of sex, age, and risk window length were also not significant when tested using F-test ( Table 2 ). Discussion We systematically reviewed the methodology of 30 observational studies assessing the risk of myocarditis following COVID-19 vaccination, and employed meta-analysis and meta-regression to examine heterogeneity in effect estimates and the influence of design specifications. Substantial variability was observed in design choices, including risk window length, reference period timing, and approaches to account for COVID-19 infection. Approximately one-third of the studies were rated as having serious or critical risk of bias, primarily due to confounding and bias in outcome measurement. The highest heterogeneity in the effect estimates was observed for second doses of BNT-162b2 and mRNA-1273 compared to unvaccinated individuals/reference period in general population. Meta-regression suggested that longer risk windows were associated with smaller effect estimates, however its overall effect lacked significance. In recent years there has been increasing adoption of the self-controlled designs, specifically SCCS and SCRI, to evaluate vaccine safety 54 . Our findings are consistent with this trend, in which SCCS was the second most common design, after the cohort design. The SCCS/SCRI overcomes several typical challenges in vaccine safety studies. First, only cases are included, thereby avoiding the challenges of selecting unvaccinated controls particularly relevant in high-vaccine-coverage settings like those for COVID-19 vaccines. Second, the SCCS/SCRI inherently controls for all time-invariant confounders, mitigating risks associated with unmeasured confounding often encountered in between-person designs 55 . The SCCS/SCRI also minimizes selection bias and exposure misclassification bias as only vaccinated individuals are studied 55 . There are, however, certain caveats with this design. Only relative risk estimation is available for this design, and it is susceptible to measurement error related to timing of outcome and exposure, and subsequently the definition of risk and control periods. Furthermore, validity of this design requires fulfilment of several exposure and outcome criteria, or adaptation of the design 15 , 56 . Encouragingly, most SCCS studies in our review adhered to these assumptions, a notable improvement compared to earlier systematic reviews 57 , 58 . No single study design is universally optimal for all vaccine-outcome pairs. The selection of a suitable design should align with the characteristics of the outcome, risk of confounding and misclassification bias, and availability of data and resources 7 , 55 . Besides, it is also sensible to triangulate findings with more than one appropriate design. The agreement of results from different design could strengthen the evidence, and disagreement could shed light on the sources of bias 59 . Four studies in our review employed both SCCS and cohort or case-control designs and they mostly yielded consistent conclusions. It is important to note, however, that estimates from different designs reflect different estimands and need not be numerically identical. A prior simulation study also reported that the SCCS estimates were within 3% of a cohort gold standard, while a recent empirical evaluation study suggested that SCCS was less biased than cohort or case-control 8 , 60 . A noteworthy finding was the higher heterogeneity in the effect estimates for second doses of BNT-162b2 and mRNA-1273 compared to their first doses. The first and second dose of each vaccine were investigated by the same set of studies, meaning that the set of source populations, study designs and other design specification were similar between the two PICOs. This may denote that the factors specifically pertain to second dose could play a role in explaining the effect heterogeneity. One factor is the interval between first and second dose, ranging from 21 days to up to 84 days in different countries 36 , 61 – 63 . Studies have shown that longer inter-dose intervals could significantly reduce the risk of post-vaccination myocarditis 64 , 65 . However, we could not investigate this factor in our analysis due to limited reporting in original papers. Another factor could be the increased level of residual (time-varying) confounding, as the second dose risk window was further away from the control window before vaccination, and individuals who chose to receive second dose could be more different from those who did not get vaccinated. Researchers should therefore pay great attention to the risk of time-varying and selection bias when designing a multi-dose vaccine safety study 55 . Selecting an appropriate risk window is critical for unbiased effect estimation, particularly in SCCS/SCRI studies. If the chosen risk window is much longer or shorter or is placed so that it does not cover the true risk period, the relative incidence estimates could be biased toward the null 15 , 66 . The significant effects of several risk window categories on the magnitude of effect estimates identified in our meta-regression reaffirms the importance of careful consideration in defining risk window. Yet it is challenging to select the optimal risk window as clinical trials often provide little or no insight on the timing of specific adverse events. Vaccine safety studies, including those in our review, have often used pre-specified risk window based on literature, biological plausibility, expert opinions or convention 67 . It is recommended to include several (contiguous) periods, especially when there is uncertainty about the true risk period, albeit with potential increases in type I error 15 . Several data-driven methods to identify the optimal risk window have also been proposed, yet their adoption remains limited 66 , 68 , 69 . The included studies demonstrated broad variation in how they accounted for COVID-19 infection cases in their analyses: exclude or censor cases with COVID-19 infection, include COVID-19 infection as a covariate in the model, or ignore the cases altogether. COVID-19 infection represents a time-varying confounder that is affected by past exposure when considering the COVID-19 vaccine – myocarditis relationship, as COVID-19 infection also increased the risk of myocarditis, and the risk of having an infection could be different between the control window and the risk window 45 , 70 . Thus, both self-controlled studies and cohort and case-control studies, especially those comparing different time periods, are susceptible to this confounding bias. A quantitative bias analysis of COVID-19 vaccine-myocarditis relationship in SCRI design showed that a strong association between the time-varying confounder and the outcome would lead to overestimation of the true effect if the confounder was ignored 70 . While our findings suggest that different approaches to handle COVID-19 infection may not substantially alter effect estimates, additional research is needed to investigate their implications further. While our meta-regression models identified significant effects of certain levels of sex, age and risk window length, none of their overall effects was significant. These results should be interpreted cautiously considering the inherent drawbacks of meta-regressions. Due to the low number of studies in the meta-regression, the model may be underpowered to detect effects and be susceptible to confounding as only a limited number of variables could be included in the model 5 . Furthermore, with the lack of individual participant data, aggregate-data meta-regression is prone to ecological fallacy. We have tried to minimize this bias by using all available subgroup and sensitivity analyses and separating between-study and within-study effect (which is free from ecological bias) of certain variables 71 . However, the variability and incompleteness of reporting certain information across studies, such as age and sex distribution, hampered the granularity of our analysis. We thus call for a more comprehensive and uniform reporting of the participant and design characteristics in future studies, by using reporting checklist or core participant characteristics sets 72 . As suggested by Wang and colleagues (2022), researchers could also conduct sensitivity analyses over a designed matrix of alternative design specifications, with justification of the variations used in sensitivity analyses 6 . These practices would be particularly beneficial in evidence synthesis, enabling the standardization of results prior to meta-analysis to reduce heterogeneity, and the investigation of the sources of heterogeneity with higher precision and accuracy 72 . Besides the limitations of the meta-regression method, the scarcity of data limited our ability to investigate the role of other factors such as data sources, outcome ascertainment methods, and types of reference period. Prior research has suggested data differences could play a role in heterogeneity of real-world evidence studyies 6 . Additionally, there is currently no official quality assessment tool for self-controlled studies. We have adapted the ROBINS-I tool to accommodate the features of self-controlled designs, however this adapted version has not been validated. Developing standardized reporting guidelines and quality assessment frameworks tailored to these designs could address these gaps. Conclusions Our review identified substantial variability in design aspects such as risk window length, reference period timing, and accounting for COVID-19 disease, in studies investigating the myocarditis safety signal of COVID-19 vaccines. Considerable heterogeneity in effect estimates was observed, particularly for second doses. Design choices like risk window length may explain at least some of this heterogeneity. Future vaccine safety studies should include thorough sensitivity analyses to explore the effect of design and analytical choices on their findings, which could in turn strengthen the usefulness of real-world evidence in vaccine safety evaluation, informing evidence synthesis and benefit-risk assessments of vaccines. Data Availability All data produced in the present work are contained in the manuscript https://github.com/trangtph/Myocarditis_Vaccine_SR_public Supplementary material Supplementary file 1: Study protocol Supplementary file 2: Data extraction form Supplementary file 3: Adapted ROBINS-I tool for SCCS/SCRI studies Supplementary file 4: ROBINS-I assessment results Supplementary Figure S1: PRISMA low diagram Supplementary Figure S2. Contour-enhanced funnel plot for Dose 2 of BNT-162b2 compared with unvaccinated individuals/ reference period in general population. Supplementary Figure S3. Contour-enhanced funnel plot for Dose 2 of mRNA-1273 compared with unvaccinated individuals/ reference period in general population. Supplementary Table S1: PICOs investigated by included studies Supplementary Table S2: Summary of meta-analysis results Supplementary Table S3: Meta-analysis results by age subgroup Supplementary Table S4: Meta-analysis results by risk window length subgroup Supplementary Table S5: Meta-analysis results by study design subgroups Supplementary Table S6: Meta-analysis results by overall risk of bias (RoB) subgroups Supplementary Table S7: Meta-analysis results by confounding RoB subgroups Supplementary Table S8: Meta-analysis results by outcome definition subgroups Supplementary Table S9: Meta-analysis results by subgroups of approaches to handle COVID-19 infection in the analysis Data availability The data used for the analysis and the analysis codes are available at: https://github.com/trangtph/Myocarditis_Vaccine_SR_public Conflict of interest The authors have no conflict of interest to declare. Author contributions TT: Conceptualization, methodology, investigation, formal analysis, writing – original draft, visualisation, project administration; MB: conceptualisation, investigation, writing – review and editing, supervision; OK: conceptualisation, validation, writing – review and editing, supervision; SB: conceptualisation, investigation, writing – review and editing, supervision. Reference 1. ↵ Katoto , P. D. M. C. , et al. Systematic review and meta-analysis of myocarditis and pericarditis in adolescents following COVID-19 BNT162b2 vaccination . npj Vaccines 8 , 89 , doi: 10.1038/s41541-023-00681-3 ( 2023 ). OpenUrl CrossRef 2. ↵ European Medicines Agency . Comirnaty and Spikevax: possible link to very rare cases of myocarditis and pericarditis , ( 2021 ). 3. ↵ Stowe , J. , Miller , E. , Andrews , N. & Whitaker , H. J . Risk of myocarditis and pericarditis after a COVID-19 mRNA vaccine booster and after COVID-19 in those with and without prior SARS-CoV-2 infection: A self-controlled case series analysis in England . PLOS Medicine 20 , e1004245 , doi: 10.1371/journal.pmed.1004245 ( 2023 ). OpenUrl CrossRef PubMed 4. ↵ Lai , F. T. T. , et al. Adverse events of special interest following the use of BNT162b2 in adolescents: a population-based retrospective cohort study . Emerg Microbes Infect 11 , 885 – 893 , doi: 10.1080/22221751.2022.2050952 ( 2022 ). OpenUrl CrossRef PubMed 5. ↵ Douglas Altman et al. in Cochrane Handbook for Systematic Reviews of Interventions version 6.4 (ed Julian PT Higgins Jonathan J Deeks , Douglas G Altman ; on behalf of the Cochrane Statistical Methods Group ) ( 2023 ). 6. ↵ Wang , S. V. , Sreedhara , S. K. , Bessette , L. G. & Schneeweiss , S . Understanding variation in the results of real-world evidence studies that seem to address the same question . J Clin Epidemiol 151 , 161 – 170 , doi: 10.1016/j.jclinepi.2022.08.012 ( 2022 ). OpenUrl CrossRef PubMed 7. ↵ Baker , M. A. , et al. A Vaccine Study Design Selection Framework for the Postlicensure Rapid Immunization Safety Monitoring Program . American Journal of Epidemiology 181 , 608 – 618 , doi: 10.1093/aje/kwu322 ( 2015 ). OpenUrl CrossRef PubMed 8. ↵ Schuemie , M. J. , et al. Vaccine Safety Surveillance Using Routinely Collected Healthcare Data-An Empirical Evaluation of Epidemiological Designs . Front Pharmacol 13 , 893484 , doi: 10.3389/fphar.2022.893484 ( 2022 ). OpenUrl CrossRef 9. ↵ Page , M. J. et al. The PRISMA 2020 statement: An updated guideline for reporting systematic reviews . PLOS Medicine 18 , e1003583 , doi: 10.1371/journal.pmed.1003583 ( 2021 ). OpenUrl CrossRef PubMed 10. ↵ FDA Office of Media Affairs . FDA Takes Key Action in Fight Against COVID-19 By Issuing Emergency Use Authorization for First COVID-19 Vaccine , ( 2020 ). 11. ↵ EndNote (Clarivate, Philadelphia , PA, 2013 ). 12. ↵ van de Schoot , R. , et al. An open source machine learning framework for efficient and transparent systematic reviews . Nature Machine Intelligence 3 , 125 – 133 , doi: 10.1038/s42256-020-00287-7 ( 2021 ). OpenUrl CrossRef 13. ↵ ASReview . Active Learning Explained , ( 2022 ). 14. ↵ Sterne , J. A. , et al. ROBINS-I: a tool for assessing risk of bias in non-randomised studies of interventions . BMJ 355 , i4919 , doi: 10.1136/bmj.i4919 ( 2016 ). OpenUrl FREE Full Text 15. ↵ Whitaker , H. J. , Paddy Farrington , C. , Spiessens , B. & Musonda , P . Tutorial in biostatistics: the self-controlled case series method . Statistics in Medicine 25 , 1768 – 1797 , doi: 10.1002/sim.2302 ( 2006 ). OpenUrl CrossRef PubMed Web of Science 16. ↵ Heyard , R. , Held , L. , Schneeweiss , S. & Wang , S. V . Design differences and variation in results between randomised trials and non-randomised emulations: meta-analysis of RCT-DUPLICATE data . BMJ Medicine 3 , e000709 , doi: 10.1136/bmjmed-2023-000709 ( 2024 ). OpenUrl Abstract / FREE Full Text 17. ↵ VanderWeele , T. J . Optimal Approximate Conversions of Odds Ratios and Hazard Ratios to Risk Ratios . Biometrics 76 , 746 – 752 , doi: 10.1111/biom.13197 ( 2019 ). OpenUrl CrossRef 18. ↵ López-López , J. A. , Page , M. J. , Lipsey , M. W. & Higgins , J. P. T . Dealing with effect size multiplicity in systematic reviews and meta-analyses . Research Synthesis Methods 9 , 336 – 351 , doi: 10.1002/jrsm.1310 ( 2018 ). OpenUrl CrossRef 19. ↵ Sterne , J. A. C. , et al. Recommendations for examining and interpreting funnel plot asymmetry in meta-analyses of randomised controlled trials . BMJ 343 , d4002 , doi: 10.1136/bmj.d4002 ( 2011 ). OpenUrl FREE Full Text 20. Page MJ , H. J. , Sterne JAC. in Cochrane Handbook for Systematic Reviews of Interventions version 6.4 (ed Thomas J Higgins JPT , Chandler J , Cumpston M , Li T , Page MJ , Welch VA (editors)) Ch. 13, (Cochrane, 2023 ). 21. Gucciardi , D. F. , Lines , R. L. J. & Ntoumanis , N . Handling effect size dependency in meta-analysis . International Review of Sport and Exercise Psychology 15 , 152 – 178 , doi: 10.1080/1750984X.2021.1946835 ( 2022 ). OpenUrl CrossRef 22. ↵ Anzures-Cabrera , J. & Higgins , J. P . Graphical displays for meta-analysis: An overview with suggestions for practice . Res Synth Methods 1 , 66 – 80 , doi: 10.1002/jrsm.6 ( 2010 ). OpenUrl CrossRef PubMed 23. ↵ Pustejovsky , J. E. & Tipton , E . Meta-analysis with Robust Variance Estimation: Expanding the Range of Working Models . Prev Sci 23 , 425 – 438 , doi: 10.1007/s11121-021-01246-3 ( 2022 ). OpenUrl CrossRef PubMed 24. ↵ Tanner-Smith , E. E. & Tipton , E . Robust variance estimation with dependent effect sizes: practical considerations including a software tutorial in Stata and spss . Research Synthesis Methods 5 , 13 – 30 , doi: 10.1002/jrsm.1091 ( 2014 ). OpenUrl CrossRef 25. ↵ Yaremych , H. E. , Preacher , K. J. & Hedeker , D . Centering categorical predictors in multilevel models: Best practices and interpretation . Psychol Methods 28 , 613 – 630 , doi: 10.1037/met0000434 ( 2023 ). OpenUrl CrossRef PubMed 26. ↵ Abraham , N. , et al. Myocarditis and/or pericarditis risk after mRNA COVID-19 vaccination: A Canadian head to head comparison of BNT162b2 and mRNA-1273 vaccines . Vaccine 40 , 4663 – 4671 , doi: 10.1016/j.vaccine.2022.05.048 ( 2022 ). OpenUrl CrossRef 27. ↵ Andersson , N. W. , Thiesson , E. M. , Hansen , J. V. & Hviid , A. Safety of BA.4-5 or BA.1 bivalent mRNA booster vaccines: nationwide cohort study . BMJ 382 , e075015 , doi: 10.1136/bmj-2023-075015 ( 2023 ). OpenUrl Abstract / FREE Full Text 28. ↵ Andersson , N. W. , et al. Safety of heterologous primary and booster schedules with ChAdOx1-S and BNT162b2 or mRNA-1273 vaccines: nationwide cohort study . BMJ 378 , e070483 , doi: 10.1136/bmj-2022-070483 ( 2022 ). OpenUrl Abstract / FREE Full Text 29. ↵ Barda , N. , et al. Safety of the BNT162b2 mRNA Covid-19 Vaccine in a Nationwide Setting . New England Journal of Medicine 385 , 1078 – 1090 , doi : doi: 10.1056/NEJMoa2110475 ( 2021 ). OpenUrl CrossRef PubMed 30. ↵ Bots , S. H. , et al. Myocarditis and pericarditis associated with SARS-CoV-2 vaccines: A population-based descriptive cohort and a nested self-controlled risk interval study using electronic health care data from four European countries . Frontiers in Pharmacology 13 , doi: 10.3389/fphar.2022.1038043 ( 2022 ). OpenUrl CrossRef 31. ↵ Corrao , G. , et al. Increased risk of myocarditis and pericarditis and reduced likelihood of severe clinical outcomes associated with COVID-19 vaccination: a cohort study in Lombardy, Italy . BMC Infect Dis 22 , 844 , doi: 10.1186/s12879-022-07823-3 ( 2022 ). OpenUrl CrossRef 32. ↵ Dickerman , B. A. , et al. Comparative Safety of BNT162b2 and mRNA-1273 Vaccines in a Nationwide Cohort of US Veterans . JAMA Internal Medicine 182 , 739 – 746 , doi: 10.1001/jamainternmed.2022.2109 ( 2022 ). OpenUrl CrossRef PubMed 33. ↵ Fan , M. , et al. Risk of carditis after three doses of vaccination with mRNA (BNT162b2) or inactivated (CoronaVac) covid-19 vaccination: a self-controlled cases series and a case-control study . Lancet Reg Health West Pac 35 , 100745 , doi: 10.1016/j.lanwpc.2023.100745 ( 2023 ). OpenUrl CrossRef 34. ↵ Goddard , K. , et al. Risk of myocarditis and pericarditis following BNT162b2 and mRNA-1273 COVID-19 vaccination . Vaccine 40 , 5153 – 5159 , doi: 10.1016/j.vaccine.2022.07.007 ( 2022 ). OpenUrl CrossRef PubMed 35. ↵ Harris , D. A. , et al. Comparative Risks of Potential Adverse Events Following COVID-19 mRNA Vaccination Among Older US Adults . JAMA Network Open 6 , e2326852 – e2326852 , doi: 10.1001/jamanetworkopen.2023.26852 ( 2023 ). OpenUrl CrossRef 36. ↵ Husby , A. , et al. SARS-CoV-2 vaccination and myocarditis or myopericarditis: population based cohort study . BMJ 375 , e068665 , doi: 10.1136/bmj-2021-068665 ( 2021 ). OpenUrl Abstract / FREE Full Text 37. ↵ Karlstad , Ø. , et al. SARS-CoV-2 Vaccination and Myocarditis in a Nordic Cohort Study of 23 Million Residents . JAMA Cardiology 7 , 600 – 612 , doi: 10.1001/jamacardio.2022.0583 ( 2022 ). OpenUrl CrossRef PubMed 38. ↵ Knowlton , K. U. , et al. A Small but Significantly Greater Incidence of Inflammatory Heart Disease Identified After Vaccination for Severe Acute Respiratory Syndrome Coronavirus 2 . Open Forum Infect Dis 9 , ofab663 , doi: 10.1093/ofid/ofab663 ( 2022 ). OpenUrl CrossRef 39. ↵ Lai , F. T. T. , et al. Carditis After COVID-19 Vaccination With a Messenger RNA Vaccine and an Inactivated Virus Vaccine : A Case-Control Study . Ann Intern Med 175 , 362 – 370 , doi: 10.7326/m21-3700 ( 2022 ). OpenUrl CrossRef PubMed 40. ↵ Massari , M. , et al. Postmarketing active surveillance of myocarditis and pericarditis following vaccination with COVID-19 mRNA vaccines in persons aged 12 to 39 years in Italy: A multi-database, self-controlled case series study . PLoS Med 19 , e1004056 , doi: 10.1371/journal.pmed.1004056 ( 2022 ). OpenUrl CrossRef PubMed 41. ↵ Mevorach , D. , et al. Myocarditis after BNT162b2 mRNA Vaccine against Covid-19 in Israel . New England Journal of Medicine 385 , 2140 – 2149 , doi : doi: 10.1056/NEJMoa2109730 ( 2021 ). OpenUrl CrossRef PubMed 42. ↵ Naveed , Z. , et al. Comparative Risk of Myocarditis/Pericarditis Following Second Doses of BNT162b2 and mRNA-1273 Coronavirus Vaccines . J Am Coll Cardiol 80 , 1900 – 1908 , doi: 10.1016/j.jacc.2022.08.799 ( 2022 ). OpenUrl CrossRef PubMed 43. ↵ Nordström , P. , Ballin , M. & Nordström , A . Safety and effectiveness of monovalent COVID-19 mRNA vaccination and risk factors for hospitalisation caused by the omicron variant in 0.8 million adolescents: A nationwide cohort study in Sweden . PLoS Med 20 , e1004127 , doi: 10.1371/journal.pmed.1004127 ( 2023 ). OpenUrl CrossRef PubMed 44. ↵ Patone , M. , et al. Risks of myocarditis, pericarditis, and cardiac arrhythmias associated with COVID-19 vaccination or SARS-CoV-2 infection . Nature Medicine 28 , 410 – 422 , doi: 10.1038/s41591-021-01630-0 ( 2022 ). OpenUrl CrossRef PubMed 45. ↵ Patone , M. , et al. Risk of Myocarditis After Sequential Doses of COVID-19 Vaccine and SARS-CoV-2 Infection by Age and Sex . Circulation 146 , 743 – 754 , doi: 10.1161/circulationaha.122.059970 ( 2022 ). OpenUrl CrossRef PubMed 46. ↵ Ab Rahman , N. , et al. Risk of serious adverse events after the BNT162b2, CoronaVac, and ChAdOx1 vaccines in Malaysia: A self-controlled case series study . Vaccine 40 , 4394 – 4402 , doi: 10.1016/j.vaccine.2022.05.075 ( 2022 ). OpenUrl CrossRef PubMed 47. ↵ Rudan , I. , et al. BNT162b2 COVID-19 vaccination uptake, safety, effectiveness and waning in children and young people aged 12-17 years in Scotland . Lancet Reg Health Eur 23 , 100513 , doi: 10.1016/j.lanepe.2022.100513 ( 2022 ). OpenUrl CrossRef 48. ↵ Shoaibi , A. , et al. Evaluation of potential adverse events following COVID-19 mRNA vaccination among adults aged 65 years and older: Two self-controlled studies in the U.S . Vaccine 41 , 4666 – 4678 , doi: 10.1016/j.vaccine.2023.06.014 ( 2023 ). OpenUrl CrossRef PubMed 49. ↵ Sindet-Pedersen , C. , et al. Risk of Worsening Heart Failure and All-Cause Mortality Following COVID-19 Vaccination in Patients With Heart Failure: A Nationwide Real-World Safety Study . Circ Heart Fail 16 , e010617 , doi: 10.1161/circheartfailure.123.010617 ( 2023 ). OpenUrl CrossRef PubMed 50. ↵ Wan , E. Y. F. , et al. Safety of an inactivated, whole-virion COVID-19 vaccine (CoronaVac) in people aged 60 years or older in Hong Kong: a modified self-controlled case series . The Lancet Healthy Longevity 3 , e491 – e500 , doi: 10.1016/S2666-7568(22)00125-8 ( 2022 ). OpenUrl CrossRef 51. ↵ Yechezkel , M. , et al. Safety of the fourth COVID-19 BNT162b2 mRNA (second booster) vaccine: a prospective and retrospective cohort study . The Lancet Respiratory Medicine 11 , 139 – 150 , doi: 10.1016/S2213-2600(22)00407-6 ( 2023 ). OpenUrl CrossRef 52. ↵ Yamin , D. , et al. Safety of monovalent and bivalent BNT162b2 mRNA COVID-19 vaccine boosters in at-risk populations in Israel: a large-scale, retrospective, self-controlled case series study . The Lancet Infectious Diseases 23 , 1130 – 1142 , doi: 10.1016/S1473-3099(23)00207-4 ( 2023 ). OpenUrl CrossRef PubMed 53. ↵ Le Vu , S. , et al. Age and sex-specific risks of myocarditis and pericarditis following Covid-19 messenger RNA vaccines . Nature Communications 13 , 3633 , doi: 10.1038/s41467-022-31401-5 ( 2022 ). OpenUrl CrossRef PubMed 54. ↵ Wu , J. , Hauben , M. & Younus , M . Current Approaches in Postapproval Vaccine Safety Studies Using Real-World Data: A Systematic Review of Published Literature . Clinical Therapeutics 46 , 555 – 564 , doi: 10.1016/j.clinthera.2024.06.005 ( 2024 ). OpenUrl CrossRef PubMed 55. ↵ Lai , L. Y. , et al. Current Approaches to Vaccine Safety Using Observational Data: A Rationale for the EUMAEUS (Evaluating Use of Methods for Adverse Events Under Surveillance-for Vaccines) Study Design . Frontiers in Pharmacology 13 , doi: 10.3389/fphar.2022.837632 ( 2022 ). OpenUrl CrossRef 56. ↵ Whitaker , H. J. & Ghebremichael-Weldeselassie , Y . Self-Controlled Case Series Methodology . Annual Review of Statistics and Its Application 6 , 241 – 261 , doi: 10.1146/annurev-statistics-030718-105108 ( 2019 ). OpenUrl CrossRef 57. ↵ Gault , N. , et al. Self-controlled designs in pharmacoepidemiology involving electronic healthcare databases: a systematic review . BMC Medical Research Methodology 17 , 25 , doi: 10.1186/s12874-016-0278-0 ( 2017 ). OpenUrl CrossRef PubMed 58. ↵ Nie , X. , et al. Self-controlled case series design in vaccine safety: a systematic review . Expert Review of Vaccines 21 , 313 – 324 , doi: 10.1080/14760584.2022.2020108 ( 2022 ). OpenUrl CrossRef PubMed 59. ↵ Whitaker , H. J. , Hocine , M. N. & Farrington , C. P . The methodology of self-controlled case series studies . Statistical Methods in Medical Research 18 , 7 – 26 , doi: 10.1177/0962280208092342 ( 2009 ). OpenUrl CrossRef PubMed 60. ↵ Glanz , J. M. , et al. Four different study designs to evaluate vaccine safety were equally validated with contrasting limitations . Journal of Clinical Epidemiology 59 , 808 – 818 , doi: 10.1016/j.jclinepi.2005.11.012 ( 2006 ). OpenUrl CrossRef PubMed Web of Science 61. ↵ Kriss , J. L. , et al. COVID-19 Vaccine Second-Dose Completion and Interval Between First and Second Doses Among Vaccinated Persons - United States, December 14, 2020-February 14, 2021 . MMWR Morb Mortal Wkly Rep 70 , 389 – 395 , doi: 10.15585/mmwr.mm7011e2 ( 2021 ). OpenUrl CrossRef PubMed 62. Tsang , T. K. , et al. Evaluating the impact of extended dosing intervals on mRNA COVID-19 vaccine effectiveness in adolescents . Res Sq , doi: 10.21203/rs.3.rs-4518813/v1 ( 2024 ). OpenUrl CrossRef 63. ↵ European Centre for Disease Prevention and Control . Overview of the implementation of COVID-19 vaccination strategies and deployment plans in the EU/EEA , 6 May 2021. ( ECDC , Stockholm, 2021 ). 64. ↵ Fan , M. , et al. Risk of carditis among adolescents after extending the interdose intervals of BNT162b2 . npj Vaccines 9 , 31 , doi: 10.1038/s41541-023-00789-6 ( 2024 ). OpenUrl CrossRef 65. ↵ Le Vu , S. , et al. Influence of mRNA Covid-19 vaccine dosing interval on the risk of myocarditis . Nature Communications 15 , 7745 , doi: 10.1038/s41467-024-52038-6 ( 2024 ). OpenUrl CrossRef 66. ↵ Xu , S. , et al. Identifying optimal risk windows for self-controlled case series studies of vaccine safety . Statistics in Medicine 30 , 742 – 752 , doi: 10.1002/sim.4125 ( 2011 ). OpenUrl CrossRef PubMed 67. ↵ Kochhar , S. , et al. Defining the interval for monitoring potential adverse events following immunization (AEFIs) after receipt of live viral vectored vaccines . Vaccine 37 , 5796 – 5802 , doi: 10.1016/j.vaccine.2018.08.085 ( 2019 ). OpenUrl CrossRef PubMed 68. ↵ Campos , L. F. , Şentürk , D. , Chen , Y. & Nguyen , D. V. Bias and estimation under misspecification of the risk period in self-controlled case series studies . Stat 6 , 373 – 389 , doi: 10.1002/sta4.166 ( 2017 ). OpenUrl CrossRef PubMed 69. ↵ Xu , S. , Hambidge , S. J. , McClure , D. L. , Daley , M. F. & Glanz , J. M . A scan statistic for identifying optimal risk windows in vaccine safety studies using self-controlled case series design . Statistics in Medicine 32 , 3290 – 3299 , doi: 10.1002/sim.5733 ( 2013 ). OpenUrl CrossRef PubMed 70. ↵ Bots , S. H. , et al. Applying two approaches to detect unmeasured confounding due to time-varying variables in a self-controlled risk interval design evaluating COVID-19 vaccine safety signals, using myocarditis as a case example . Am J Epidemiol , doi: 10.1093/aje/kwae172 ( 2024 ). OpenUrl CrossRef 71. ↵ Thompson , S. G. & Higgins , J. P. T . How should meta-regression analyses be undertaken and interpreted? Statistics in Medicine 21 , 1559 – 1573 , doi: 10.1002/sim.1187 ( 2002 ). OpenUrl CrossRef PubMed Web of Science 72. ↵ Vuong , M. L. , Tu , P. H. T. , Duong , K. L. & Vo , T.-T . Development of minimum reporting sets of patient characteristics in epidemiological research: A methodological systematic review . Research Methods in Medicine & Health Sciences 5 , 49 – 62 , doi: 10.1177/26320843231191777 ( 2023 ). OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted June 18, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Quality and methodological heterogeneity of COVID-19 vaccine safety studies focusing on the myocarditis safety signal: A systematic review, meta-analysis and meta-regression Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Quality and methodological heterogeneity of COVID-19 vaccine safety studies focusing on the myocarditis safety signal: A systematic review, meta-analysis and meta-regression Tu P. H. Trang , Marloes T. Bazelier , Olaf H. Klungel , Sophie H. Bots medRxiv 2025.06.18.25329834; doi: https://doi.org/10.1101/2025.06.18.25329834 Share This Article: Copy Citation Tools Quality and methodological heterogeneity of COVID-19 vaccine safety studies focusing on the myocarditis safety signal: A systematic review, meta-analysis and meta-regression Tu P. H. Trang , Marloes T. Bazelier , Olaf H. Klungel , Sophie H. Bots medRxiv 2025.06.18.25329834; doi: https://doi.org/10.1101/2025.06.18.25329834 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Epidemiology Subject Areas All Articles Addiction Medicine (570) Allergy and Immunology (863) Anesthesia (301) Cardiovascular Medicine (4442) Dentistry and Oral Medicine (444) Dermatology (383) Emergency Medicine (609) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1511) Epidemiology (15231) Forensic Medicine (30) Gastroenterology (1126) Genetic and Genomic Medicine (6610) Geriatric Medicine (668) Health Economics (998) Health Informatics (4542) Health Policy (1370) Health Systems and Quality Improvement (1613) Hematology (543) HIV/AIDS (1266) Infectious Diseases (except HIV/AIDS) (15924) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (147) Nephrology (668) Neurology (6608) Nursing (346) Nutrition (999) Obstetrics and Gynecology (1146) Occupational and Environmental Health (957) Oncology (3338) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1693) Pharmacology and Therapeutics (692) Primary Care Research (712) Psychiatry and Clinical Psychology (5449) Public and Global Health (9240) Radiology and Imaging (2203) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (596) Sexual and Reproductive Health (714) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a0207653f8231b23',t:'MTc3OTgzNzEyMA=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();