Sensitivity of estimates of the effectiveness of REDD+ projects to matching specifications and moving from pixels to polygons as the unit of analysis

Sensitivity of estimates of the effectiveness of REDD+ projects to matching specifications and moving from pixels to polygons as the unit of analysis | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Sensitivity of estimates of the effectiveness of REDD+ projects to matching specifications and moving from pixels to polygons as the unit of analysis View ORCID Profile Alejandro Guizar-Coutiño , David Coomes , Tom Swinfield , View ORCID Profile Julia P G Jones doi: https://doi.org/10.1101/2024.05.22.595326 Alejandro Guizar-Coutiño 1 Department of Plant Sciences, University of Cambridge 2 The Biodiversity Consultancy, Cambridge , UK Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Alejandro Guizar-Coutiño For correspondence: ag997{at}cam.ac.uk David Coomes 1 Department of Plant Sciences, University of Cambridge Find this author on Google Scholar Find this author on PubMed Search for this author on this site Tom Swinfield 4 Cambridge Centre for Carbon Credits, University of Cambridge Find this author on Google Scholar Find this author on PubMed Search for this author on this site Julia P G Jones 5 School of Environmental and Natural Sciences, Bangor University 6 Department of Biology, Utrecht University Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Julia P G Jones Abstract Full Text Info/History Metrics Preview PDF Abstract There is a substantial interest in the potential of carbon credits generated by Reducing Emissions from tropical Deforestation and Degradation (REDD+) and traded on the voluntary carbon market for generating the finance needed to slow forest loss. However, such credits have become marred in controversy. Recent global-scale analysis using a range of methods for estimating the counterfactual rate of deforestation ex post suggest that many REDD+ projects have overestimated their effectiveness at reducing deforestation and consequently issued more credits than can be justified. All such methods include potentially arbitrary choices which can affect the estimate of the treatment effect. In addition, using pixels as the sampling unit, as some of the studies do, can introduce biases. One study which has been widely cited in the debate (Guizar-Coutiño et al. 2022) estimated avoided deforestation using statistical matching of pixels and a single set of matching options. We estimate avoided deforestation from the same set of projects using 7-hectare plots rather than pixels to sample deforestation and explore the sensitivity of the results to matching choices (exploring 120 matched sets in total). We filtered the results on three criteria: 1) post-matching covariate balance, 2) proportion of REDD+ samples that were successfully matched, and 3) similarity of trends in deforestation rates prior to REDD+ implementation (parallel trends). While one of the 44 REDD+ projects failed these quality control process, we estimate treatment effects for the remaining 43 projects. There was a substantial correlation between our new estimates and those published in Guizar-Coutiño et al. 2022 (0.72 measured in annual percent change, and 0.9 measured in total area change) and our headline estimate of 0.22% per yr (95% CI: 0.13 to 0.34) is essentially unchanged. At a time when confidence in the voluntary carbon markets is low, we hope these results provide reassurance that ex-post counterfactual estimates of avoided deforestation are consistent, helping accelerate their widespread adoption and rebuild trust in nature-based climate solutions. Introduction Forests sequester and store enormous amounts of carbon, making forest conservation an important nature-based solution to climate change 1 . REDD+ credits (Reducing Emissions from Deforestation and forest Degradation) are the most numerous of credits traded in the voluntary market 2 and there is very substantial interest in the potential of such credits to provide much-needed finance for tropical forest conservation 3 . However, the extent to which REDD+ projects have provided the reductions in deforestation or forest degradation they have claimed has become hotly debated 4 – 7 . To sell carbon credits on the voluntary carbon market, a REDD+ project needs to demonstrate additionality: i.e. that any reduction in deforestation is due to the project 8 . This requires an estimate of the counterfactual: how much deforestation would have occurred without the project 4 . REDD+ projects do this by using a variety of methodologies, the majority published and administered by the nonprofit organization VERRA, to estimate ‘baseline’ deforestation. Baselines are based on historical deforestation in project reference areas which are selected to be as similar as possible to project areas in terms of drivers of deforestation 9 . Deforestation in project areas is then compared with the baseline and, following verification, VCUs (voluntary carbon units) are issued which can then be traded on the voluntary market. This approach has advantages as allows a project to have some certainty up-front as to the number of credits a project can produce, however it has been criticized for putting a lot of weight on the ex-ante baseline estimates, rather than using information on real rates of deforestation in comparable areas during the project period to construct counterfactuals 9 , 10 . A suite of independent global-scale analyses using a variety of approaches have suggested that many REDD+ projects provide limited additional avoided deforestation. For example, Guizar-Coutiño et al. 11 found that 33 out of 40 projects studied did not avoid any deforestation. Even more concerning, there have been claims that projects have issued more voluntary carbon units than can be justified 10 , 12 . For example West et al. 10 , found that across 18 REDD+ projects for which they had data, projects had been used to offset almost three times more carbon emissions than their contribution to climate change mitigation, though these estimates are disputed 7 . While many analyses of the impact of conservation interventions on deforestation uses pixels as the unit of analysis (e.g. 13 – 15 ) there is increasing recognition that analyzing deforestation as binary, unrepeatable data (a pixel is deforested or not) can result in biases which are reduced by aggregating binary pixels spatially 16 . Where appropriate polygons exist, the unit of analysis can be pixels aggregated within land holdings 17 , or administrative boundaries 18 , 19 . However in many contexts appropriate units may not exist, or polygons reflecting such units may not be available. In such cases polygons of a given area placed at random can be used to sample the landscape 10 Given that forest conservation projects, including REDD+, are never randomly placed in a landscape, evaluating their impact on deforestation requires causal inference from observational data 20 . Areas where REDD+ projects are set up (treatment units) will inevitably differ in terms of confounders which influence both exposure to the intervention (in this case the location of REDD+ projects) and the outcome of interest (deforestation). Statistical matching is often used to ensure treatment and selected control units are as similar as possible with respect to these observable confounders 21 , 22 . However, there are a range of somewhat arbitrary modelling choices made when applying statistical matching which can influence the estimated treatment effect 23 , 24 . In this paper we revisit the same set of REDD+ projects as were presented in Guizar-Coutiño et al. 11 . While Guizar-Coutiño et al. 11 sample deforestation using pixels, we instead sample deforestation using circular polygons of 150 m radius, or 7.1-ha plots. We also explore how the choices made during statistical matching (applying 120 different combinations of matching options to each project) influence the estimated treatment effect. Methods The research pipeline is shown in Fig. 1 and developed in the following sections. Our outcome of interest is deforestation as reported in the Tropical Moist Forest (TMF) maps which present deforestation and degradation across the humid tropics at 30 m resolution from 1990 to 2019 25 . Download figure Open in new tab Figure 1. Schematic of the pipeline use to explore the impact of matching decisions on estimates of REDD+ effectiveness. Blocks indicate components of the analysis and are described in more detail in the methods. 1. Observational set-up Selection of REDD+ projects The VCS database is the leading accreditation registry for voluntary REDD+ projects 2 and the largest sources of data on voluntary REDD+ projects locations. We used the REDD+ database compiled by Guizar-Coutiño et al. 11 , which identified 81 “Reducing Deforestation and Degradation” projects established in the tropics 26 and successfully found boundary data for 71 of these. While new projects have been introduced since 2019, and others withdrawn, for consistency with 11 we analyze the same set of projects. We excluded projects with <80% evergreen forest cover at the start (because the TMF dataset only covers evergreen forest), and projects which operated for fewer than 5 years or commenced before 2000. In Guizar-Coutiño et al. 11 we included project PL1748 in the analyses despite not meeting the minimum number of years for our criteria. This is because we recorded its start date incorrectly during our original analysis. We include it here for consistency despite it only running for four years. Sampling within REDD+ areas and forested landscapes To assess the extent of reductions in deforestation due to REDD+, we compared changes in forest cover within treated areas with those taking place in similar forest patches within the wider landscape. We generated plots with a radius of 150 m on a 1 km grid within forested areas of the REDD+ projects and surrounding regions. Each plot was 7.1 hectares in size and contained up to 78 TMF pixels, which we considered adequate for characterizing local drivers of deforestation and our outcome as continuous variables 27 . We used the extent of forest cover in 1990 as delineated by the TMF 25 to position sampling plots, ensuring that all samples contained at least some portion of undisturbed moist forest cover when building the time-series of forest cover. We used the same approach to sample tropical moist forests plots in the wider landscape: we removed sampling locations that were not forested or that lay within a 15-km buffer zone around the project area, which may have been affected by local leakage of economic activities. We then random subsampled, until we had up to five samples in the wider landscape for each sampling plot located inside the REDD+ project, each separated by at least 1 km to reduce the spatial dependence 28 ; this created a pool of samples which was subsequently used to find matches for REDD+ project samples ( Fig. 2 ). Download figure Open in new tab Figure 2. Illustration of the sampling strategy, for a REDD+ project in Democratic Republic of the Congo (VCS 1359). Features shown include the boundary of the REDD+ area (solid boundary) and the 7.1-ha plots collected from treatment (orange) and landscape (pink) areas. A buffer surrounding the project boundaries (dotted boundary) is shown within which we did not potential controls. The extent of undisturbed forest cover (dark grey) was used as a reference for distributing sample plots. Plot-level forest disturbance estimates Annual estimates of forest cover and forest disturbances were extracted from the TMF database. This database provides a long-term characterization of forest disturbances and land cover change, on an annual basis spanning the period 1990 -2019. Our analyses focused on examining temporal patterns of three main forest classes: The undisturbed class, which refer to closed evergreen or semi-evergreen forest areas that have not been degraded or deforested; the degraded class, characterized as short-term disturbance (less than 2.5 years); and the deforested class, representing a long-term removal of forest cover 25 . For each year during the 1990 - 2019 period, we calculated the extent of undisturbed, and disturbed forests classes within sampling plots. From these metrics we derived our outcome of interest - the proportion of forest lost - defined as, where Δp t is the total area deforested in year t and p d is the area of undisturbed forest in the baseline or reference year (see 4. Exploring the impact of matching choices on estimated treatment effect) Drivers of deforestation We identified key covariates associated with deforestation 29 , some of which were also likely to have been associated with assignment to the treatment 30 , 31 . To characterize deforestation risks factors, we collected data on key and observable bioclimatic and socio-demographic determinants of deforestation (as in 29 Table S1). For each circular plot, we obtained mean estimates for elevation and slope 32 , distance to the nearest urban center 33 and distance to forest edge. To account for distance to forest edge, we produced annual time-series of the distance to the nearest deforested pixel, as characterized by the TMF time-series. First, for each pixel covered by the circular plot, we computed pixel-level estimates of the distance to the nearest pixel that had changed its status from undisturbed to deforested, or from degraded to deforested, during the observed year, for the period 1990-2019. We then produced plot-level annual rolling estimates of the mean distance to deforestation events in the previous five years, covering the period 1995-2019. 2. Matching design We used statistical matching to select a subset of samples in the wider landscape that were as similar as possible to samples in the project areas in terms of observable confounders: initial forest cover density, exposure to drivers of deforestation and trajectories of deforestation in the period prior to REDD+ (Table S1). We exact matched on country. We used a “base” set of covariates for all runs, which included: 1) elevation 32 , 2) slope 32 , 3) distance to population centers 33 , 4) distance to degraded areas over the five years prior to project commencement, and 5) extent of degraded area at the time of project implementation. To evaluate the sensitivity of estimates of avoided deforestation to matching specifications 24 , we ran matching runs with alternative specifications of: Algorithms: We used propensity score (PSM), Mahalanobis distance (MHN) and random forest (RFM) algorithms; Covariates: We first used the “base” set of covariates alone, then iteratively added the following covariates to that base set: extent of undisturbed forest in 1990 (forest area 1990), extent of undisturbed forest in the project implementation year (forest area), and extent of undisturbed forest in 1990 and implementation year (forest area + forest area 1990); resulting in 4 distinct covariate combinations. Calipers: We applied calipers of different sizes to constrain the selection of landscape observations prior to matching, so that the covariate distribution lay within 0.1, 0.3, 0.5, 0.7 and 0.9 SD of the covariate distribution in the treatment group. Matching with and without replacement: Matching iterations allowed for both, matching with replacement (e.g. allowing control observations to be paired with multiple treatment units), or without replacement (e.g. constraining one control observation per treatment unit). The combination of matching criteria resulted in 120 matching runs for each of the 44 selected REDD+ sites, giving a total of 5280 runs. 3. Assessing the quality of matched sets Matching runs were assessed against three criteria: Covariate balance between treated and control samples. Similarity between treated and control samples was determined by the overlap of the distributions of the covariate values. Matching runs were considered adequate if the absolute standardized mean difference between treated and matched control samples was less than 0.25 across all covariates included in the matching trial 22 . Proportion of project-area samples that were matched. Estimates of additionality can be biased unless a high proportion of treatment samples are successfully matched: only then is the “average treatment effect on the treated” estimated accurately 22 . Matching was considered adequate if at least 80% of treatment samples were paired with a control. Parallel trends . A key condition of difference-in-differences designs is that inferences cannot be drawn unless treatment and control groups follow similar outcome trajectories in the period prior to the intervention; this is known as the parallel trends condition 34 . Since our sampling strategy involves characterizing historical deforestation within the sampling plots, counterfactuals would be credible only if deforestation trends between treated and control plots prior to the introduction of REDD+ are comparable. We established the existence of parallel trends by ruling out significant differences in rates of forest loss prior to project implementation (exemplified in Fig. S1). To assess parallel trends, a linear model was fitted for each project: Where P it is the proportion of undisturbed forest area in the plot i in year t , and CI is the treatment group indicator ( i =1 for treated or 0 for control), relative to forest extent in 1990. Parameter {3 3 allows us to test for significant differences in deforestation trends between treated and control plots. We performed these tests using all available years before the project starting date. Departure from the parallel trends was assessed by testing whether the slope of the treatment and control groups were significantly different at p< 0.05; and matching was considered adequate if no departure was detected. 4. Calculating effect sizes Matched model effect sizes To estimate additionality for each matched set, we calculated annual forest loss rates by aggregating the extent of deforested area across all sampling plots for each group (e.g. treatment or control) in the five years before and after project implementation, divided by the extent of undisturbed forest across all sampling plots in each group in the 6 th year prior to project implementation (i.e., reference year). We then estimated a mean difference-in-differences in forest loss rates as: DD = (AT + AC) – (BT + BC), where DD is difference-in-differences, A and B refer to the before and after period and T and C refer to the treatment and control group rates of deforestation rates (% yr -1 ). Bootstrapped effect sizes We computed project-level mean difference-in-difference with 95% confidence intervals by bootstrapping difference-in-difference estimates of robustly matched sets. To produce a global estimate of the impact of REDD+ in reducing deforestation, we computed the mean of the site-level averages and estimated 95% confidence intervals by bootstrapping on the site-level averages. Results Sites included in the analysis Of the 71 mapped REDD+ projects, 22 had less than 80% evergreen forest cover and 5 did not meet our time frame criteria. We applied matching to the remaining 44 projects ( Fig. 3 ). Only 22% of all matched sets met all three of our quality-control criteria (1142 of 5280 matched sets). One of the 44 REDD+ projects failed to produce any high-quality matched set (project 1201). The remaining 43 REDD+ sites were located in America (n=34), Africa (n=6) and Asia-Pacific (n=3). Download figure Open in new tab Figure 3. Location of the REDD+ projects included in the analysis. Of the 71 mapped sites, 22 had less than 80% evergreen forest cover at the start of projects (purple), 5 did not meet our time frame criteria, leaving 44 projects that were examined with multiple matching approaches (34 in the Americas, 7 in Africa and 3 in Asia-Pacific). Successful matching models were returned for 43 projects (yellow), with plots size indicate the number of robust models available for each of them. Project PL1748 did not met the time frame criteria but was included in the analyses for consistency with Guizar-Coutiño et al. 2022. Matching parameters and quality of matched sets The quality of matched sets was more influenced by the choice of algorithms and by covariate selection, than by caliper size or matching with or without replacement. We note varying success rates across the three matching quality criteria (Fig. S2): 70% of sets failed the parallel trends criteria (i.e. they did not display similar outcome trajectories before the REDD+ project was established; Fig. S1), while 54% failed to achieve covariate balance between treated and control observations, and 31% failed to meet the criteria that 80% of treatment plots were successfully matched. Matched sets using the Mahalanobis distance method were much more likely to pass the three matching quality tests than sets matched with other methods ( Fig. 4 ). Moreover, including measures of forest area on different time periods alongside the base covariates increased the quality of matched sets. Neither the caliper size nor the choice of whether to match with replacement had much effect on whether a matched set passed the quality control tests. Download figure Open in new tab Figure 4. The number of matched sets that passed all three quality control criteria, subdivided by specifications made when matching. A total of 120 alternative matching specifications were made in each of 44 REDD+ sites, giving a total of 5280 matching sets; of these 1142 met all three quality control criteria. The specifications included: a) four choice of covariates combinations; base covariates only (i.e. elevation, slope, accessibility, distance to degradation, country and biome), base covariates plus forest cover at the time of project implementation, base covariates plus forest cover in 1990, and base covariates + forest cover in 1990 and implementation year; b) choice of matching algorithm between Mahalanobis (MHN), propensity score (PSM), and random forest (RFM); c) choice of filtering regime (caliper sizes 0.1, 0.3, 0.5, 0.7 and 0.9 SD); and d) choice of whether to allow matching with replacement or not. Revised estimates of effectiveness of REDD+ in reducing deforestation Our analyses highlights the variability in the avoided deforestation delivered by REDD+ projects over their first five years of operation, both in terms of relative reductions in forest loss ( Fig. 5a ), and absolute deforestation avoided ( Fig. 5b ). While some projects have delivered substantial avoided deforestation (those to the left hand side of the Figure), the majority have effect sizes close to zero. Our project-level difference-in-differences estimates of robust matched sets were well correlated with the estimates produced by pixel-matching in Guizar-Coutiño et al. 11 . The correlation is 0.72 when measured in annual percent change, and 0.9 when measured in total area change. Download figure Open in new tab Figure 5. Annual differences in (a) deforestation rates for the 44 REDD+ projects included in this study and (b) total area loss using all matched models, with estimates below the zero-line indicating avoided deforestation. Dots indicate difference-in-differences estimates coloured according to whether the matching met our quality control criteria (red=robustly matched, grey=rejected). There are no valid models for project 1201 in Sierra Leone. For projects included in 11 , estimates from that analysis are indicated with a ‘+’ symbol. Table S2 presents the bootstrapped estimates of the deforestation avoided by each project (both in terms of percentage per year and hectares per year) with 95% confidence intervals. For 30 of the 43 projects, our best estimate of avoided deforestation as a percentage and total hectares is negative and does not overlap zero (Table S2). However, for many of those projects the reduction in deforestation is modest. Combining the 43 site-level estimates, we observe a mean reduction of 0.22% yr - 1 (95% CI: 0.13–0.36) in the rates of forest loss compared to controls, which amounted to an approximate area of avoided deforestation of 64903 Ha. This is essentially unchanged from the figures presented in Guizar-Coutiño et al. 11 , which similarly calculated a mean reduction of 0.22%/year (95% CI: 0.13–0.34) amounting to approximately 66754 ha of avoided deforestation. D iscussion Given heated debate about the extent to which REDD+ projects have effectively avoided deforestation 4 , 7 , 10 , 12 , 35 , the inevitable uncertainty in estimates using a single matching approach 22 , 24 , and concerns about analyzing deforestation as a binary process 16 , revisiting and updating the estimates of avoided deforestation provided in Guizar-Coutiño et al. 11 is valuable. We find that most projects reduce deforestation. However, effectiveness is highly variable; for many projects the reductions in deforestation are modest and for some they are non-existent. These revised estimates add to the growing evidence base exploring the impacts of site-based REDD+ project effectiveness using ex post counterfactuals 10 – 12 , 36 . Our previous study 11 used ∼30m pixels to sample deforestation. Our new approach, using 7.1-ha plots all separated by at least 1 km, has the advantage of reducing the spatial dependence of samples 28 . By using plots as our sampling unit (within which deforestation is a continuous variable) rather than pixels (where deforestation is a binary outcome), we were also able to select controls with similar trends in deforestation. There are inevitably a range of somewhat arbitrary choices which need to be made when selecting matching approaches 21 , 22 . By trialing 120 combinations of matching options and estimating the treatment effect from the full set which meet quality-control criteria, we get a valuable estimate of the uncertainty in estimates of ex post estimates of avoided deforestation. This work explores sensitivity to how matching on observed confounders was carried out. However, there is a risk that important confounders remain which cannot be accounted for 37 . Despite the importance of testing for sensitivity of results to unobserved confounding, this is not yet standard practice in conservation impact evaluation 38 . In the context of REDD+, hidden confounders might include institutional factors such as tenure security and presence of local conservation initiatives 39 – 41 or local drivers of agricultural activities 29 . Some of the critiques of the recent published estimates of the avoided deforestation from REDD+ projects 7 , 42 , 43 suggest that such large-scale analysis (covering multiple REDD+ projects in multiple sites using the same methodology) inevitably fails to account for key local-level drivers of deforestation. Further work exploring the sensitivity of these results to hidden confounders would be valuable. Climate change is advancing faster than many climate scientists had predicted. The target of keeping warming within 1.5 degrees of pre-industrial levels has already been missed 44 , or will be soon 45 . Given the contribution of tropical forests to both sequestering and storing carbon 46 , 47 , it is vital that the global community find ways to stop further loss of tropical forests 4 . The voluntary carbon market is not the only way that forest conservation can be funded. International policy is focused on developing a UN-governed schemed under article 6 of the Paris Climate Agreement which will focus on lowering deforestation over entire jurisdictions 40 rather than in specific REDD+ project locations. There are also many bilateral agreements where countries support other countries to lower their deforestation-related emissions. However, selling emissions reductions from avoiding deforestation on the voluntary carbon market is generating substantial, and hard-to-replace, finance for forest conservation and will remain important at least into the medium term. The lack of credibility of the credits being issued poses a threat to this important revenue stream 35 , 48 . There is growing recognition that conservative credits issuance 49 can help address issues such as impermanence 50 , and leakage 51 . At time when confidence in the voluntary carbon markets is low, we hope our results provide reassurance that ex-post counterfactual estimates of avoided deforestation can deliver relatively consistent estimates of additionality, helping accelerate their widespread adoption and rebuild trust in nature-based climate solutions. Data and code Data summaries and code to reproduce the main figures provided at: https://github.com/guizar/redd-sens SUPPLEMENTARY INFORMATION SUPPLEMENTARY FIGURES Download figure Open in new tab Figure S1. Patterns of deforestation, before and after project implementation, for the Ankeniheny-Zahamena (CAZ) Protected Area project in Madagascar (VCS 1311). Plot-level mean estimates of forest loss (Ha) in treated (blue) and control (red) samples. 95% confidence intervals (grey) generated by bootstrapping. This particular time-series comes from a matched set that meets all quality criteria and was used to assess the sensitivity of estimates to unobserved confounders. Download figure Open in new tab Figure S2 The relative importance of the three validation criteria as the reasons matched sets were rejected. The three panels show differences in deforestation rates for the 44 REDD+ projects (% yr - 1 ) during the first five years of project implementation, arranged by project-level average differences (x-axis). Grey indicates that the run failed to meet the criteria covered by the panel, colour shows that it met the criteria but failed to meet other criteria, black indicates that all criteria were met. SUPPLEMENTARY TABLES View this table: View inline View popup Table S1: Covariates selected for matching. All the covariates listed in the table are included as part of our core set of matching variables, except for ’Temporal estimates of forest cover’ variables, which we 460 used to iterate in the matching runs (shown in bold). View this table: View inline View popup Table S2: Project-level treatment effect estimates (bootstrapped robustly matched sets only) for the REDD+ projects for which valid matched sets are available. Estimated treated effects are averaged over the first 5 years. Projects classified as “Avoided Planned Deforestation and forest Degradation” indicated with asterisks. Acknowledgments A.G.C. received a scholarship from Consejo Nacional de Ciencia y Tecnología México, CONACYT, and the Frank Jackson Foundation through a grant to Wolfson College, as well as funding from the Tezos Foundation gift to the Cambridge Centre for Carbon Credits (4C). J.P.G.J. was an Oxford Martin School visiting fellow and short-term visiting fellow at Jesus College Oxford when working on this paper. Footnotes Introduces minor proofreading corrections and repairs a broken URL to access the article's code and data repository. Literature cited 1. ↵ Griscom , B. W. et al. Natural climate solutions . Proc Natl Acad Sci U S A 114 , 11645 – 11650 ( 2017 ). OpenUrl Abstract / FREE Full Text 2. ↵ Donofrio , S. , Patrick–Daley , C. & Ciro–Lin , K . The art of integrity: State of the voluntary carbon markets 2022 Q3. Forest Trends’ Ecosystem Marketplace ( 2022 ). 3. ↵ TSVCM . Taskforce on Scaling Voluntary Carbon Markets . www.iif.com/Portals/1/Files/TSVCM_Report.pdf ( 2021 ). 4. ↵ Jones , J. P. G. & Lewis , S. L . Forest carbon offsets are failing . Science ( 1979 ) 381 , 830 – 831 (2023). OpenUrl 5. Greenfield , P . Revealed: more than 90% of rainforest carbon offsets by biggest certifier are worthless, analysis shows . The Guardian ( 2023 ). 6. Jones , J. P. G . Scandal in the voluntary carbon market must not impede tropical forest conservation . Nat Ecol Evol in press, ( 2024 ). 7. ↵ Mitchard , E. et al. Serious errors impair an assessment of forest carbon projects: A rebuttal of West et al. (2023). SSRN Electronic Journal ( 2023 ) doi: 10.2139/SSRN.4661873 . OpenUrl CrossRef 8. ↵ Delacote , P. , Le Velly , G. & Simonet , G . Revisiting the location bias and additionality of REDD+ projects: the role of project proponents status and certification . Resour Energy Econ 67 , 101277 ( 2022 ). 9. ↵ West , T. A. P. , Bomfim , B. & Haya , B. K . Baselines. in Quality assessment of REDD+ carbon credit projects 28 – 50 (Berkley Carbon Trading Project, 2023 ). 10. ↵ West , T. A. P. et al. Action needed to make carbon offsets from forest conservation work for climate change mitigation . Science (1979) 381 , 873 – 877 ( 2023 ). OpenUrl CrossRef 11. ↵ Guizar-Coutiño , A. , Jones , J. P. G. , Balmford , A. , Carmenta , R. & Coomes , D. A . A global evaluation of the effectiveness of voluntary REDD+ projects at reducing deforestation and degradation in the moist tropics . Conservation Biology n/a , e13970 ( 2022 ). 12. ↵ West , T. A. P. , Börner , J. , Sills , E. O. & Kontoleon , A . Overstated carbon emission reductions from voluntary REDD+ projects in the Brazilian Amazon . Proc Natl Acad Sci U S A 117 , 24188 – 24194 ( 2020 ). OpenUrl Abstract / FREE Full Text 13. ↵ Baehr , C. , Benyishay , A. & Parks , B . Linking Local Infrastructure Development and Deforestation: Evidence from Satellite and Administrative Data . doi: 10.1086/712800 8 , 375 – 409 ( 2021 ). OpenUrl CrossRef 14. Anderson , C. M. , Asner , G. P. , Llactayo , W. & Lambin , E. F . Overlapping land allocations reduce deforestation in Peru . Land use policy 79 , 174 – 178 ( 2018 ). OpenUrl 15. ↵ Alix-Garcia , J. & Gibbs , H. K . Forest conservation effects of Brazil’s zero deforestation cattle agreements undermined by leakage . Global Environmental Change 47 , 201 – 217 ( 2017 ). OpenUrl 16. ↵ Garcia , A. & Heilmayr , R . Impact evaluation with nonrepeatable outcomes: The case of forest conservation . J Environ Econ Manage 125 , 102971 ( 2024 ). 17. ↵ Carlson , K. M. et al. Effect of oil palm sustainability certification on deforestation and fire in Indonesia . Proc Natl Acad Sci U S A 115 , 121 – 126 ( 2018 ). OpenUrl Abstract / FREE Full Text 18. ↵ Tabor , K. et al. Evaluating the effectiveness of conservation and development investments in reducing deforestation and fires in Ankeniheny-Zahemena Corridor, Madagascar . PLoS One 12 , e0190119 ( 2017 ). OpenUrl 19. ↵ Koch , N. , Zu Ermgassen , E. K. H. J. , Wehkamp , J. , Oliveira Filho , F. J. B. & Schwerhoff , G . Agricultural Productivity and Forest Conservation: Evidence from the Brazilian Amazon . Am J Agric Econ 101 , 919 – 940 ( 2019 ). OpenUrl 20. ↵ Börner , J. , Schulz , D. , Wunder , S. & Pfaff , A . The effectiveness of forest conservation policies and programs . 12 , 45 – 64 ( 2020 ). 21. ↵ Schleicher , J. et al. Statistical matching for conservation science . Conservation Biology 34 , 538 – 549 ( 2019 ). OpenUrl 22. ↵ Stuart , E. A . Matching Methods for Causal Inference: A Review and a Look Forward . Statistical Science 25 , 1 – 21 ( 2010 ). OpenUrl CrossRef PubMed 23. ↵ Devenish , K. , Desbureaux , S. , Willcock , S. & Jones , J. P. G . On track to achieve no net loss of forest at Madagascar’s biggest mine . Nature Sustainability 2022 5 : 6 5, 498 – 508 ( 2022 ). OpenUrl 24. ↵ Desbureaux , S . Subjective modeling choices and the robustness of impact evaluations in conservation science . Conservation Biology 35 , 1615 – 1626 ( 2021 ). OpenUrl 25. ↵ Vancutsem , C. et al. Long-term (1990–2019) monitoring of forest cover changes in the humid tropics . Sci Adv 7 , eabe1603 ( 2021 ). OpenUrl FREE Full Text 26. ↵ Verified Carbon Standard (VCS). VCS Project Database . http://www.vcsprojectdatabase.org/ ( 2019 ). 27. ↵ Avelino , A. F. T. , Baylis , K. & Honey-Rosés , J . Goldilocks and the Raster Grid: Selecting Scale when Evaluating Conservation Programs . PLoS One 11 , e0167945 ( 2016 ). OpenUrl 28. ↵ Robalino , J. A. & Pfaff , A . Contagious development: Neighbor interactions in deforestation . J Dev Econ 97 , 427 – 436 ( 2012 ). OpenUrl CrossRef 29. ↵ Busch , J. & Ferretti-Gallon , K . What Drives and Stops Deforestation, Reforestation, and Forest Degradation? An Updated Meta-analysis. Rev Environ Econ Policy 0 ( 2023 ) doi: 10.1086/725051 . OpenUrl CrossRef 30. ↵ Lin , L. , Pattanayak , S. K. , Sills , E. O. & Sunderlin , W. D . Site selection for forest carbon projects . In ;;Analysing REDD+: Challenges and choices 209 – 230 (CIFOR, Bogor, Indonesia, 2012 ). 31. ↵ Sunderlin , W. D. & Sills , E. O . REDD+ projects as a hybrid of old and new forest conservation approaches. in Analysing REDD+: Challenges and choices 177 – 191 (CIFOR, Bogor, Indonesia , 2012 ). 32. ↵ Jarvis , A. , Reuter , H. I. , Nelson , A. & Guevara , E . Hole-Filled SRTM for the Globe. V4 . http://srtm.csi.cgiar.org ( 2008 ). 33. ↵ Weiss , D. J. et al. A global map of travel time to cities to assess inequalities in accessibility in 2015 . Nature 553 , 333 – 336 ( 2018 ). OpenUrl CrossRef PubMed 34. ↵ Angrist , J. D. & Pischke , J.-S . Mostly Harmless Econometrics: An Empiricist’s Companion . ( Princeton University Press , 2008 ). 35. ↵ Balmford , A. et al. Credit credibility threatens forests . Science ( 1979 ) 380 , 466 – 467 (2023). OpenUrl 36. ↵ Correa , J. et al. Evaluating REDD+ at subnational level: Amazon fund impacts in Alta Floresta, Brazil . For Policy Econ 116 , 102178 ( 2020 ). 37. ↵ Cinelli , C. & Hazlett , C . Making sense of sensitivity: Extending omitted variable bias . J R Stat Soc Series B Stat Methodol 82 , 39 – 67 ( 2020 ). OpenUrl 38. ↵ Jones , J. P. G. et al. Quantifying uncertainty about how interventions are assigned would improve impact evaluation in conservation: Reply to Rasolofoson 2022 . Conservation Biology n/a , e14007 ( 2022 ). 39. ↵ Lin , L. , Pattanayak , S. K. , Sills , E. O. & Sunderlin , W. D . Site selection for forest carbon projects . In Analysing REDD+: Challenges and choices 209 – 230 (CIFOR, Bogor, Indonesia, 2012 ). 40. ↵ Wunder , S. et al. REDD+ in Theory and Practice: How Lessons From Local Projects Can Inform Jurisdictional Approaches . 3 , ( 2020 ). 41. ↵ Usmani , F. , Jeuland , M. & Pattanayak , S. K . NGOs and the Effectiveness of Interventions . Rev Econ Stat 1 – 45 ( 2022 ) doi: 10.1162/rest_a_01217 . OpenUrl CrossRef 42. ↵ VERRA. Technical Review of West et al. 2020 and 2023, Guizar-Coutiño 2022, and Coverage in Britain’s Guardian. Verra Preprint at https://verra.org/technical-review-of-west-et-al-2020-and-2023-guizar-coutino-2022-and-coverage-in-britains-guardian/ ( 2023 ). 43. ↵ VERRA . non-dogmatic critique of West, et al. ( 2023 ), “ Action needed to make carbon offsets from forest conservation work for climate change mitigation ,” Science 381 : 873 – 877 . (2023). OpenUrl CrossRef 44. ↵ McCulloch , M. T. , Winter , A. , Sherman , C. E. & Trotter , J. A . 300 years of sclerosponge thermometry shows global warming has exceeded 1.51°C . Nature Climate Change 2024 14 : 2 14, 171 – 177 ( 2024 ). OpenUrl 45. ↵ Diffenbaugh , N. S. & Barnes , E. A . Data-driven predictions of the time remaining until critical global warming thresholds are reached . Proc Natl Acad Sci U S A 120 , e2207183120 ( 2023 ). OpenUrl 46. ↵ Hubau , W. et al. Asynchronous carbon sink saturation in African and Amazonian tropical forests . Nature 579 , 80 – 87 ( 2020 ). OpenUrl CrossRef PubMed 47. ↵ IPCC AR6 Working Group 1: Summary for Policymakers | Climate Change 2021 : The Physical Science Basis. https://www.ipcc.ch/report/ar6/wg1/chapter/summary-for-policymakers/ . 48. ↵ Delacote , P. et al. Strong transparency required for carbon credit mechanisms . Nature Sustainability 2024 1 – 8 ( 2024 ) doi: 10.1038/s41893-024-01310-0 . OpenUrl CrossRef 49. ↵ Swinfield , T. et al. Nature-based credit markets at a crossroads . ( 2023 ) doi: 10.33774/COE-2023-PL9XV-V2 . OpenUrl CrossRef 50. ↵ Balmford , A. et al. Realizing the social value of impermanent carbon credits . Nature Climate Change 2023 13 : 11 13, 1172 – 1178 ( 2023 ). OpenUrl 51. ↵ Filewod , B. & McCarney , G . Avoiding carbon leakage from nature-based offsets by design . One Earth 6 , 790 – 802 ( 2023 ). OpenUrl View the discussion thread. Back to top Previous Next Posted May 29, 2024. Download PDF Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Sensitivity of estimates of the effectiveness of REDD+ projects to matching specifications and moving from pixels to polygons as the unit of analysis Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Sensitivity of estimates of the effectiveness of REDD+ projects to matching specifications and moving from pixels to polygons as the unit of analysis Alejandro Guizar-Coutiño , David Coomes , Tom Swinfield , Julia P G Jones bioRxiv 2024.05.22.595326; doi: https://doi.org/10.1101/2024.05.22.595326 Share This Article: Copy Citation Tools Sensitivity of estimates of the effectiveness of REDD+ projects to matching specifications and moving from pixels to polygons as the unit of analysis Alejandro Guizar-Coutiño , David Coomes , Tom Swinfield , Julia P G Jones bioRxiv 2024.05.22.595326; doi: https://doi.org/10.1101/2024.05.22.595326 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Ecology Subject Areas All Articles Animal Behavior and Cognition (7644) Biochemistry (17726) Bioengineering (13916) Bioinformatics (42033) Biophysics (21486) Cancer Biology (18635) Cell Biology (25549) Clinical Trials (138) Developmental Biology (13397) Ecology (19940) Epidemiology (2067) Evolutionary Biology (24361) Genetics (15620) Genomics (22541) Immunology (17763) Microbiology (40468) Molecular Biology (17207) Neuroscience (88739) Paleontology (667) Pathology (2842) Pharmacology and Toxicology (4834) Physiology (7659) Plant Biology (15175) Scientific Communication and Education (2047) Synthetic Biology (4304) Systems Biology (9834) Zoology (2272)