Searching for non-English literature may be unnecessary for German HTA Reports

Searching for non-English literature may be... | F1000Research "use strict";function _typeof(t){return(_typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}!function(){var t=function(){var t,e,o=[],n=window,r=n;for(;r;){try{if(r.frames.__tcfapiLocator){t=r;break}}catch(t){}if(r===n.top)break;r=r.parent}t||(!function t(){var e=n.document,o=!!n.frames.__tcfapiLocator;if(!o)if(e.body){var r=e.createElement("iframe");r.style.cssText="display:none",r.name="__tcfapiLocator",e.body.appendChild(r)}else setTimeout(t,5);return!o}(),n.__tcfapi=function(){for(var t=arguments.length,n=new Array(t),r=0;r 3&&2===parseInt(n[1],10)&&"boolean"==typeof n[3]&&(e=n[3],"function"==typeof n[2]&&n[2]("set",!0)):"ping"===n[0]?"function"==typeof n[2]&&n[2]({gdprApplies:e,cmpLoaded:!1,cmpStatus:"stub"}):o.push(n)},n.addEventListener("message",(function(t){var e="string"==typeof t.data,o={};if(e)try{o=JSON.parse(t.data)}catch(t){}else o=t.data;var n="object"===_typeof(o)&&null!==o?o.__tcfapiCall:null;n&&window.__tcfapi(n.command,n.version,(function(o,r){var a={__tcfapiReturn:{returnValue:o,success:r,callId:n.callId}};t&&t.source&&t.source.postMessage&&t.source.postMessage(e?JSON.stringify(a):a,"*")}),n.parameter)}),!1))};"undefined"!=typeof module?module.exports=t:t()}(); dataLayer = dataLayer || []; // Standard GTM initialization - Google Consent Mode handles consent automatically (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl+ '>m_auth=hzk0Vc3qFsQYhCrIoHz68A>m_preview=env-1>m_cookies_win=x';f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-MWFK8L5J'); ;window.NREUM||(NREUM={});NREUM.init={distributed_tracing:{enabled:true},privacy:{cookies_enabled:true},ajax:{deny_list:["bam.nr-data.net"]}}; ;NREUM.loader_config={accountID:"438030",trustKey:"438030",agentID:"772317073",licenseKey:"97f8f67f26",applicationID:"772317073"} ;NREUM.info={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net",licenseKey:"97f8f67f26",applicationID:"772317073",sa:1} ;/*! For license information please see nr-loader-spa-1.236.0.min.js.LICENSE.txt */ (()=>{"use strict";var e,t,r={5763:(e,t,r)=>{r.d(t,{P_:()=>l,Mt:()=>g,C5:()=>s,DL:()=>v,OP:()=>T,lF:()=>D,Yu:()=>y,Dg:()=>h,CX:()=>c,GE:()=>b,sU:()=>_});var n=r(8632),i=r(9567);const o={beacon:n.ce.beacon,errorBeacon:n.ce.errorBeacon,licenseKey:void 0,applicationID:void 0,sa:void 0,queueTime:void 0,applicationTime:void 0,ttGuid:void 0,user:void 0,account:void 0,product:void 0,extra:void 0,jsAttributes:{},userAttributes:void 0,atts:void 0,transactionName:void 0,tNamePlain:void 0},a={};function s(e){if(!e)throw new Error("All info objects require an agent identifier!");if(!a[e])throw new Error("Info for ".concat(e," was never set"));return a[e]}function c(e,t){if(!e)throw new Error("All info objects require an agent identifier!");a[e]=(0,i.D)(t,o),(0,n.Qy)(e,a[e],"info")}var u=r(7056);const d=()=>{const e={blockSelector:"[data-nr-block]",maskInputOptions:{password:!0}};return{allow_bfcache:!0,privacy:{cookies_enabled:!0},ajax:{deny_list:void 0,enabled:!0,harvestTimeSeconds:10},distributed_tracing:{enabled:void 0,exclude_newrelic_header:void 0,cors_use_newrelic_header:void 0,cors_use_tracecontext_headers:void 0,allowed_origins:void 0},session:{domain:void 0,expiresMs:u.oD,inactiveMs:u.Hb},ssl:void 0,obfuscate:void 0,jserrors:{enabled:!0,harvestTimeSeconds:10},metrics:{enabled:!0},page_action:{enabled:!0,harvestTimeSeconds:30},page_view_event:{enabled:!0},page_view_timing:{enabled:!0,harvestTimeSeconds:30,long_task:!1},session_trace:{enabled:!0,harvestTimeSeconds:10},harvest:{tooManyRequestsDelay:60},session_replay:{enabled:!1,harvestTimeSeconds:60,sampleRate:.1,errorSampleRate:.1,maskTextSelector:"*",maskAllInputs:!0,get blockClass(){return"nr-block"},get ignoreClass(){return"nr-ignore"},get maskTextClass(){return"nr-mask"},get blockSelector(){return e.blockSelector},set blockSelector(t){e.blockSelector+=",".concat(t)},get maskInputOptions(){return e.maskInputOptions},set maskInputOptions(t){e.maskInputOptions={...t,password:!0}}},spa:{enabled:!0,harvestTimeSeconds:10}}},f={};function l(e){if(!e)throw new Error("All configuration objects require an agent identifier!");if(!f[e])throw new Error("Configuration for ".concat(e," was never set"));return f[e]}function h(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");f[e]=(0,i.D)(t,d()),(0,n.Qy)(e,f[e],"config")}function g(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");var r=l(e);if(r){for(var n=t.split("."),i=0;i {r.d(t,{D:()=>i});var n=r(50);function i(e,t){try{if(!e||"object"!=typeof e)return(0,n.Z)("Setting a Configurable requires an object as input");if(!t||"object"!=typeof t)return(0,n.Z)("Setting a Configurable requires a model to set its initial properties");const r=Object.create(Object.getPrototypeOf(t),Object.getOwnPropertyDescriptors(t)),o=0===Object.keys(r).length?e:r;for(let a in o)if(void 0!==e[a])try{"object"==typeof e[a]&&"object"==typeof t[a]?r[a]=i(e[a],t[a]):r[a]=e[a]}catch(e){(0,n.Z)("An error occurred while setting a property of a Configurable",e)}return r}catch(e){(0,n.Z)("An error occured while setting a Configurable",e)}}},6818:(e,t,r)=>{r.d(t,{Re:()=>i,gF:()=>o,q4:()=>n});const n="1.236.0",i="PROD",o="CDN"},385:(e,t,r)=>{r.d(t,{FN:()=>a,IF:()=>u,Nk:()=>f,Tt:()=>s,_A:()=>o,il:()=>n,pL:()=>c,v6:()=>i,w1:()=>d});const n="undefined"!=typeof window&&!!window.document,i="undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self.navigator instanceof WorkerNavigator||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis.navigator instanceof WorkerNavigator),o=n?window:"undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis),a=""+o?.location,s=/iPad|iPhone|iPod/.test(navigator.userAgent),c=s&&"undefined"==typeof SharedWorker,u=(()=>{const e=navigator.userAgent.match(/Firefox[/\s](\d+\.\d+)/);return Array.isArray(e)&&e.length>=2?+e[1]:0})(),d=Boolean(n&&window.document.documentMode),f=!!navigator.sendBeacon},1117:(e,t,r)=>{r.d(t,{w:()=>o});var n=r(50);const i={agentIdentifier:"",ee:void 0};class o{constructor(e){try{if("object"!=typeof e)return(0,n.Z)("shared context requires an object as input");this.sharedContext={},Object.assign(this.sharedContext,i),Object.entries(e).forEach((e=>{let[t,r]=e;Object.keys(i).includes(t)&&(this.sharedContext[t]=r)}))}catch(e){(0,n.Z)("An error occured while setting SharedContext",e)}}}},8e3:(e,t,r)=>{r.d(t,{L:()=>d,R:()=>c});var n=r(2177),i=r(1284),o=r(4322),a=r(3325);const s={};function c(e,t){const r={staged:!1,priority:a.p[t]||0};u(e),s[e].get(t)||s[e].set(t,r)}function u(e){e&&(s[e]||(s[e]=new Map))}function d(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"",t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"feature";if(u(e),!e||!s[e].get(t))return a(t);s[e].get(t).staged=!0;const r=[...s[e]];function a(t){const r=e?n.ee.get(e):n.ee,a=o.X.handlers;if(r.backlog&&a){var s=r.backlog[t],c=a[t];if(c){for(var u=0;s&&u {let[t,r]=e;return r.staged}))&&(r.sort(((e,t)=>e[1].priority-t[1].priority)),r.forEach((e=>{let[t]=e;a(t)})))}function f(e,t){var r=e[1];(0,i.D)(t[r],(function(t,r){var n=e[0];if(r[0]===n){var i=r[1],o=e[3],a=e[2];i.apply(o,a)}}))}},2177:(e,t,r)=>{r.d(t,{c:()=>f,ee:()=>u});var n=r(8632),i=r(2210),o=r(1284),a=r(5763),s="nr@context";let c=(0,n.fP)();var u;function d(){}function f(e){return(0,i.X)(e,s,l)}function l(){return new d}function h(){u.aborted=!0,u.backlog={}}c.ee?u=c.ee:(u=function e(t,r){var n={},c={},f={},g=!1;try{g=16===r.length&&(0,a.OP)(r).isolatedBacklog}catch(e){}var p={on:b,addEventListener:b,removeEventListener:y,emit:v,get:x,listeners:w,context:m,buffer:A,abort:h,aborted:!1,isBuffering:E,debugId:r,backlog:g?{}:t&&"object"==typeof t.backlog?t.backlog:{}};return p;function m(e){return e&&e instanceof d?e:e?(0,i.X)(e,s,l):l()}function v(e,r,n,i,o){if(!1!==o&&(o=!0),!u.aborted||i){t&&o&&t.emit(e,r,n);for(var a=m(n),s=w(e),d=s.length,f=0;fn,p:()=>i});var n=r(2177).ee.get("handle");function i(e,t,r,i,o){o?(o.buffer([e],i),o.emit(e,t,r)):(n.buffer([e],i),n.emit(e,t,r))}},4322:(e,t,r)=>{r.d(t,{X:()=>o});var n=r(5546);o.on=a;var i=o.handlers={};function o(e,t,r,o){a(o||n.E,i,e,t,r)}function a(e,t,r,i,o){o||(o="feature"),e||(e=n.E);var a=t[o]=t[o]||{};(a[r]=a[r]||[]).push([e,i])}},3239:(e,t,r)=>{r.d(t,{bP:()=>s,iz:()=>c,m$:()=>a});var n=r(385);let i=!1,o=!1;try{const e={get passive(){return i=!0,!1},get signal(){return o=!0,!1}};n._A.addEventListener("test",null,e),n._A.removeEventListener("test",null,e)}catch(e){}function a(e,t){return i||o?{capture:!!e,passive:i,signal:t}:!!e}function s(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;window.addEventListener(e,t,a(r,n))}function c(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;document.addEventListener(e,t,a(r,n))}},4402:(e,t,r)=>{r.d(t,{Ht:()=>u,M:()=>c,Rl:()=>a,ky:()=>s});var n=r(385);const i="xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx";function o(e,t){return e?15&e[t]:16*Math.random()|0}function a(){const e=n._A?.crypto||n._A?.msCrypto;let t,r=0;return e&&e.getRandomValues&&(t=e.getRandomValues(new Uint8Array(31))),i.split("").map((e=>"x"===e?o(t,++r).toString(16):"y"===e?(3&o()|8).toString(16):e)).join("")}function s(e){const t=n._A?.crypto||n._A?.msCrypto;let r,i=0;t&&t.getRandomValues&&(r=t.getRandomValues(new Uint8Array(31)));const a=[];for(var s=0;s {r.d(t,{Bq:()=>n,Hb:()=>o,oD:()=>i});const n="NRBA",i=144e5,o=18e5},7894:(e,t,r)=>{function n(){return Math.round(performance.now())}r.d(t,{z:()=>n})},7243:(e,t,r)=>{r.d(t,{e:()=>o});var n=r(385),i={};function o(e){if(e in i)return i[e];if(0===(e||"").indexOf("data:"))return{protocol:"data"};let t;var r=n._A?.location,o={};if(n.il)t=document.createElement("a"),t.href=e;else try{t=new URL(e,r.href)}catch(e){return o}o.port=t.port;var a=t.href.split("://");!o.port&&a[1]&&(o.port=a[1].split("/")[0].split("@").pop().split(":")[1]),o.port&&"0"!==o.port||(o.port="https"===a[0]?"443":"80"),o.hostname=t.hostname||r.hostname,o.pathname=t.pathname,o.protocol=a[0],"/"!==o.pathname.charAt(0)&&(o.pathname="/"+o.pathname);var s=!t.protocol||":"===t.protocol||t.protocol===r.protocol,c=t.hostname===r.hostname&&t.port===r.port;return o.sameOrigin=s&&(!t.hostname||c),"/"===o.pathname&&(i[e]=o),o}},50:(e,t,r)=>{function n(e,t){"function"==typeof console.warn&&(console.warn("New Relic: ".concat(e)),t&&console.warn(t))}r.d(t,{Z:()=>n})},2587:(e,t,r)=>{r.d(t,{N:()=>c,T:()=>u});var n=r(2177),i=r(5546),o=r(8e3),a=r(3325);const s={stn:[a.D.sessionTrace],err:[a.D.jserrors,a.D.metrics],ins:[a.D.pageAction],spa:[a.D.spa],sr:[a.D.sessionReplay,a.D.sessionTrace]};function c(e,t){const r=n.ee.get(t);e&&"object"==typeof e&&(Object.entries(e).forEach((e=>{let[t,n]=e;void 0===u[t]&&(s[t]?s[t].forEach((e=>{n?(0,i.p)("feat-"+t,[],void 0,e,r):(0,i.p)("block-"+t,[],void 0,e,r),(0,i.p)("rumresp-"+t,[Boolean(n)],void 0,e,r)})):n&&(0,i.p)("feat-"+t,[],void 0,void 0,r),u[t]=Boolean(n))})),Object.keys(s).forEach((e=>{void 0===u[e]&&(s[e]?.forEach((t=>(0,i.p)("rumresp-"+e,[!1],void 0,t,r))),u[e]=!1)})),(0,o.L)(t,a.D.pageViewEvent))}const u={}},2210:(e,t,r)=>{r.d(t,{X:()=>i});var n=Object.prototype.hasOwnProperty;function i(e,t,r){if(n.call(e,t))return e[t];var i=r();if(Object.defineProperty&&Object.keys)try{return Object.defineProperty(e,t,{value:i,writable:!0,enumerable:!1}),i}catch(e){}return e[t]=i,i}},1284:(e,t,r)=>{r.d(t,{D:()=>n});const n=(e,t)=>Object.entries(e||{}).map((e=>{let[r,n]=e;return t(r,n)}))},4351:(e,t,r)=>{r.d(t,{P:()=>o});var n=r(2177);const i=()=>{const e=new WeakSet;return(t,r)=>{if("object"==typeof r&&null!==r){if(e.has(r))return;e.add(r)}return r}};function o(e){try{return JSON.stringify(e,i())}catch(e){try{n.ee.emit("internal-error",[e])}catch(e){}}}},3960:(e,t,r)=>{r.d(t,{K:()=>a,b:()=>o});var n=r(3239);function i(){return"undefined"==typeof document||"complete"===document.readyState}function o(e,t){if(i())return e();(0,n.bP)("load",e,t)}function a(e){if(i())return e();(0,n.iz)("DOMContentLoaded",e)}},8632:(e,t,r)=>{r.d(t,{EZ:()=>u,Qy:()=>c,ce:()=>o,fP:()=>a,gG:()=>d,mF:()=>s});var n=r(7894),i=r(385);const o={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net"};function a(){return i._A.NREUM||(i._A.NREUM={}),void 0===i._A.newrelic&&(i._A.newrelic=i._A.NREUM),i._A.NREUM}function s(){let e=a();return e.o||(e.o={ST:i._A.setTimeout,SI:i._A.setImmediate,CT:i._A.clearTimeout,XHR:i._A.XMLHttpRequest,REQ:i._A.Request,EV:i._A.Event,PR:i._A.Promise,MO:i._A.MutationObserver,FETCH:i._A.fetch}),e}function c(e,t,r){let i=a();const o=i.initializedAgents||{},s=o[e]||{};return Object.keys(s).length||(s.initializedAt={ms:(0,n.z)(),date:new Date}),i.initializedAgents={...o,[e]:{...s,[r]:t}},i}function u(e,t){a()[e]=t}function d(){return function(){let e=a();const t=e.info||{};e.info={beacon:o.beacon,errorBeacon:o.errorBeacon,...t}}(),function(){let e=a();const t=e.init||{};e.init={...t}}(),s(),function(){let e=a();const t=e.loader_config||{};e.loader_config={...t}}(),a()}},7956:(e,t,r)=>{r.d(t,{N:()=>i});var n=r(3239);function i(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],r=arguments.length>2?arguments[2]:void 0,i=arguments.length>3?arguments[3]:void 0;return void(0,n.iz)("visibilitychange",(function(){if(t)return void("hidden"==document.visibilityState&&e());e(document.visibilityState)}),r,i)}},1214:(e,t,r)=>{r.d(t,{em:()=>v,u5:()=>N,QU:()=>S,_L:()=>I,Gm:()=>L,Lg:()=>M,gy:()=>U,BV:()=>Q,Kf:()=>ee});var n=r(2177);const i="nr@original";var o=Object.prototype.hasOwnProperty,a=!1;function s(e,t){return e||(e=n.ee),r.inPlace=function(e,t,n,i,o){n||(n="");var a,s,c,u="-"===n.charAt(0);for(c=0;c 2?n-2:0),o=2;o {r(A[T],e,w),r(E[T],e,w)})),r(l._A,"fetch",y),t.on(y+"end",(function(e,r){var n=this;if(r){var i=r.headers.get("content-length");null!==i&&(n.rxSize=i),t.emit(y+"done",[null,r],n)}else t.emit(y+"done",[e],n)})),t}const O={},j=["pushState","replaceState"];function S(e){const t=function(e){return(e||n.ee).get("history")}(e);return!l.il||O[t.debugId]++||(O[t.debugId]=1,s(t).inPlace(window.history,j,"-")),t}var P=r(3239);const C={},R=["appendChild","insertBefore","replaceChild"];function I(e){const t=function(e){return(e||n.ee).get("jsonp")}(e);if(!l.il||C[t.debugId])return t;C[t.debugId]=!0;var r=s(t),i=/[?&](?:callback|cb)=([^&#]+)/,o=/(.*)\.([^.]+)/,a=/^(\w+)(\.|$)(.*)$/;function c(e,t){var r=e.match(a),n=r[1],i=r[3];return i?c(i,t[n]):t[n]}return r.inPlace(Node.prototype,R,"dom-"),t.on("dom-start",(function(e){!function(e){if(!e||"string"!=typeof e.nodeName||"script"!==e.nodeName.toLowerCase())return;if("function"!=typeof e.addEventListener)return;var n=(a=e.src,s=a.match(i),s?s[1]:null);var a,s;if(!n)return;var u=function(e){var t=e.match(o);if(t&&t.length>=3)return{key:t[2],parent:c(t[1],window)};return{key:e,parent:window}}(n);if("function"!=typeof u.parent[u.key])return;var d={};function f(){t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}function l(){t.emit("jsonp-error",[],d),t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}r.inPlace(u.parent,[u.key],"cb-",d),e.addEventListener("load",f,(0,P.m$)(!1)),e.addEventListener("error",l,(0,P.m$)(!1)),t.emit("new-jsonp",[e.src],d)}(e[0])})),t}var k=r(5763);const H={};function L(e){const t=function(e){return(e||n.ee).get("mutation")}(e);if(!l.il||H[t.debugId])return t;H[t.debugId]=!0;var r=s(t),i=k.Yu.MO;return i&&(window.MutationObserver=function(e){return this instanceof i?new i(r(e,"fn-")):i.apply(this,arguments)},MutationObserver.prototype=i.prototype),t}const z={};function M(e){const t=function(e){return(e||n.ee).get("promise")}(e);if(z[t.debugId])return t;z[t.debugId]=!0;var r=n.c,o=s(t),a=k.Yu.PR;return a&&function(){function e(r){var n=t.context(),i=o(r,"executor-",n,null,!1);const s=Reflect.construct(a,[i],e);return t.context(s).getCtx=function(){return n},s}l._A.Promise=e,Object.defineProperty(e,"name",{value:"Promise"}),e.toString=function(){return a.toString()},Object.setPrototypeOf(e,a),["all","race"].forEach((function(r){const n=a[r];e[r]=function(e){let i=!1;[...e||[]].forEach((e=>{this.resolve(e).then(a("all"===r),a(!1))}));const o=n.apply(this,arguments);return o;function a(e){return function(){t.emit("propagate",[null,!i],o,!1,!1),i=i||!e}}}})),["resolve","reject"].forEach((function(r){const n=a[r];e[r]=function(e){const r=n.apply(this,arguments);return e!==r&&t.emit("propagate",[e,!0],r,!1,!1),r}})),e.prototype=a.prototype;const n=a.prototype.then;a.prototype.then=function(){var e=this,i=r(e);i.promise=e;for(var a=arguments.length,s=new Array(a),c=0;c e())),t};function m(e,t){i.inPlace(t,["onreadystatechange"],"fn-",E)}function b(){var e=this,t=r.context(e);e.readyState>3&&!t.resolved&&(t.resolved=!0,r.emit("xhr-resolved",[],e)),i.inPlace(e,f,"fn-",E)}if(function(e,t){for(var r in e)t[r]=e[r]}(o,p),p.prototype=o.prototype,i.inPlace(p.prototype,J,"-xhr-",E),r.on("send-xhr-start",(function(e,t){m(e,t),function(e){h.push(e),a&&(y?y.then(A):u?u(A):(w=-w,x.data=w))}(t)})),r.on("open-xhr-start",m),a){var y=c&&c.resolve();if(!u&&!c){var w=1,x=document.createTextNode(w);new a(A).observe(x,{characterData:!0})}}else t.on("fn-end",(function(e){e[0]&&e[0].type===d||A()}));function A(){for(var e=0;e {r.d(t,{t:()=>n});const n=r(3325).D.ajax},6660:(e,t,r)=>{r.d(t,{A:()=>i,t:()=>n});const n=r(3325).D.jserrors,i="nr@seenError"},3081:(e,t,r)=>{r.d(t,{gF:()=>o,mY:()=>i,t9:()=>n,vz:()=>s,xS:()=>a});const n=r(3325).D.metrics,i="sm",o="cm",a="storeSupportabilityMetrics",s="storeEventMetrics"},4649:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageAction},7633:(e,t,r)=>{r.d(t,{Dz:()=>i,OJ:()=>a,qw:()=>o,t9:()=>n});const n=r(3325).D.pageViewEvent,i="firstbyte",o="domcontent",a="windowload"},9251:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageViewTiming},3614:(e,t,r)=>{r.d(t,{BST_RESOURCE:()=>i,END:()=>s,FEATURE_NAME:()=>n,FN_END:()=>u,FN_START:()=>c,PUSH_STATE:()=>d,RESOURCE:()=>o,START:()=>a});const n=r(3325).D.sessionTrace,i="bstResource",o="resource",a="-start",s="-end",c="fn"+a,u="fn"+s,d="pushState"},7836:(e,t,r)=>{r.d(t,{BODY:()=>A,CB_END:()=>E,CB_START:()=>u,END:()=>x,FEATURE_NAME:()=>i,FETCH:()=>_,FETCH_BODY:()=>v,FETCH_DONE:()=>m,FETCH_START:()=>p,FN_END:()=>c,FN_START:()=>s,INTERACTION:()=>l,INTERACTION_API:()=>d,INTERACTION_EVENTS:()=>o,JSONP_END:()=>b,JSONP_NODE:()=>g,JS_TIME:()=>T,MAX_TIMER_BUDGET:()=>a,REMAINING:()=>f,SPA_NODE:()=>h,START:()=>w,originalSetTimeout:()=>y});var n=r(5763);const i=r(3325).D.spa,o=["click","submit","keypress","keydown","keyup","change"],a=999,s="fn-start",c="fn-end",u="cb-start",d="api-ixn-",f="remaining",l="interaction",h="spaNode",g="jsonpNode",p="fetch-start",m="fetch-done",v="fetch-body-",b="jsonp-end",y=n.Yu.ST,w="-start",x="-end",A="-body",E="cb"+x,T="jsTime",_="fetch"},5938:(e,t,r)=>{r.d(t,{W:()=>o});var n=r(5763),i=r(2177);class o{constructor(e,t,r){this.agentIdentifier=e,this.aggregator=t,this.ee=i.ee.get(e,(0,n.OP)(this.agentIdentifier).isolatedBacklog),this.featureName=r,this.blocked=!1}}},9144:(e,t,r)=>{r.d(t,{j:()=>m});var n=r(3325),i=r(5763),o=r(5546),a=r(2177),s=r(7894),c=r(8e3),u=r(3960),d=r(385),f=r(50),l=r(3081),h=r(8632);function g(){const e=(0,h.gG)();["setErrorHandler","finished","addToTrace","inlineHit","addRelease","addPageAction","setCurrentRouteName","setPageViewName","setCustomAttribute","interaction","noticeError","setUserId"].forEach((t=>{e[t]=function(){for(var r=arguments.length,n=new Array(r),i=0;i 1?r-1:0),i=1;i {e.exposed&&e.api[t]&&o.push(e.api[t](...n))})),o.length>1?o:o[0]}(t,...n)}}))}var p=r(2587);function m(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},m=arguments.length>2?arguments[2]:void 0,v=arguments.length>3?arguments[3]:void 0,{init:b,info:y,loader_config:w,runtime:x={loaderType:m},exposed:A=!0}=t;const E=(0,h.gG)();y||(b=E.init,y=E.info,w=E.loader_config),(0,i.Dg)(e,b||{}),(0,i.GE)(e,w||{}),(0,i.sU)(e,x),y.jsAttributes??={},d.v6&&(y.jsAttributes.isWorker=!0),(0,i.CX)(e,y),g();const T=function(e,t){t||(0,c.R)(e,"api");const h={};var g=a.ee.get(e),p=g.get("tracer"),m="api-",v=m+"ixn-";function b(t,r,n,o){const a=(0,i.C5)(e);return null===r?delete a.jsAttributes[t]:(0,i.CX)(e,{...a,jsAttributes:{...a.jsAttributes,[t]:r}}),x(m,n,!0,o||null===r?"session":void 0)(t,r)}function y(){}["setErrorHandler","finished","addToTrace","inlineHit","addRelease"].forEach((e=>h[e]=x(m,e,!0,"api"))),h.addPageAction=x(m,"addPageAction",!0,n.D.pageAction),h.setCurrentRouteName=x(m,"routeName",!0,n.D.spa),h.setPageViewName=function(t,r){if("string"==typeof t)return"/"!==t.charAt(0)&&(t="/"+t),(0,i.OP)(e).customTransaction=(r||"http://custom.transaction")+t,x(m,"setPageViewName",!0)()},h.setCustomAttribute=function(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2];if("string"==typeof e){if(["string","number"].includes(typeof t)||null===t)return b(e,t,"setCustomAttribute",r);(0,f.Z)("Failed to execute setCustomAttribute.\nNon-null value must be a string or number type, but a type of was provided."))}else(0,f.Z)("Failed to execute setCustomAttribute.\nName must be a string type, but a type of was provided."))},h.setUserId=function(e){if("string"==typeof e||null===e)return b("enduser.id",e,"setUserId",!0);(0,f.Z)("Failed to execute setUserId.\nNon-null value must be a string type, but a type of was provided."))},h.interaction=function(){return(new y).get()};var w=y.prototype={createTracer:function(e,t){var r={},i=this,a="function"==typeof t;return(0,o.p)(v+"tracer",[(0,s.z)(),e,r],i,n.D.spa,g),function(){if(p.emit((a?"":"no-")+"fn-start",[(0,s.z)(),i,a],r),a)try{return t.apply(this,arguments)}catch(e){throw p.emit("fn-err",[arguments,this,"string"==typeof e?new Error(e):e],r),e}finally{p.emit("fn-end",[(0,s.z)()],r)}}}};function x(e,t,r,i){return function(){return(0,o.p)(l.xS,["API/"+t+"/called"],void 0,n.D.metrics,g),i&&(0,o.p)(e+t,[(0,s.z)(),...arguments],r?null:this,i,g),r?void 0:this}}function A(){r.e(439).then(r.bind(r,7438)).then((t=>{let{setAPI:r}=t;r(e),(0,c.L)(e,"api")})).catch((()=>(0,f.Z)("Downloading runtime APIs failed...")))}return["actionText","setName","setAttribute","save","ignore","onEnd","getContext","end","get"].forEach((e=>{w[e]=x(v,e,void 0,n.D.spa)})),h.noticeError=function(e,t){"string"==typeof e&&(e=new Error(e)),(0,o.p)(l.xS,["API/noticeError/called"],void 0,n.D.metrics,g),(0,o.p)("err",[e,(0,s.z)(),!1,t],void 0,n.D.jserrors,g)},d.il?(0,u.b)((()=>A()),!0):A(),h}(e,v);return(0,h.Qy)(e,T,"api"),(0,h.Qy)(e,A,"exposed"),(0,h.EZ)("activatedFeatures",p.T),T}},3325:(e,t,r)=>{r.d(t,{D:()=>n,p:()=>i});const n={ajax:"ajax",jserrors:"jserrors",metrics:"metrics",pageAction:"page_action",pageViewEvent:"page_view_event",pageViewTiming:"page_view_timing",sessionReplay:"session_replay",sessionTrace:"session_trace",spa:"spa"},i={[n.pageViewEvent]:1,[n.pageViewTiming]:2,[n.metrics]:3,[n.jserrors]:4,[n.ajax]:5,[n.sessionTrace]:6,[n.pageAction]:7,[n.spa]:8,[n.sessionReplay]:9}}},n={};function i(e){var t=n[e];if(void 0!==t)return t.exports;var o=n[e]={exports:{}};return r[e](o,o.exports,i),o.exports}i.m=r,i.d=(e,t)=>{for(var r in t)i.o(t,r)&&!i.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},i.f={},i.e=e=>Promise.all(Object.keys(i.f).reduce(((t,r)=>(i.f[r](e,t),t)),[])),i.u=e=>(({78:"page_action-aggregate",147:"metrics-aggregate",242:"session-manager",317:"jserrors-aggregate",348:"page_view_timing-aggregate",412:"lazy-feature-loader",439:"async-api",538:"recorder",590:"session_replay-aggregate",675:"compressor",733:"session_trace-aggregate",786:"page_view_event-aggregate",873:"spa-aggregate",898:"ajax-aggregate"}[e]||e)+"."+{78:"ac76d497",147:"3dc53903",148:"1a20d5fe",242:"2a64278a",317:"49e41428",348:"bd6de33a",412:"2f55ce66",439:"30bd804e",538:"1b18459f",590:"cf0efb30",675:"ae9f91a8",733:"83105561",786:"06482edd",860:"03a8b7a5",873:"e6b09d52",898:"998ef92b"}[e]+"-1.236.0.min.js"),i.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),e={},t="NRBA:",i.l=(r,n,o,a)=>{if(e[r])e[r].push(n);else{var s,c;if(void 0!==o)for(var u=document.getElementsByTagName("script"),d=0;d {s.onerror=s.onload=null,clearTimeout(h);var i=e[r];if(delete e[r],s.parentNode&&s.parentNode.removeChild(s),i&&i.forEach((e=>e(n))),t)return t(n)},h=setTimeout(l.bind(null,void 0,{type:"timeout",target:s}),12e4);s.onerror=l.bind(null,s.onerror),s.onload=l.bind(null,s.onload),c&&document.head.appendChild(s)}},i.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.j=364,i.p="https://js-agent.newrelic.com/",(()=>{var e={364:0,953:0};i.f.j=(t,r)=>{var n=i.o(e,t)?e[t]:void 0;if(0!==n)if(n)r.push(n[2]);else{var o=new Promise(((r,i)=>n=e[t]=[r,i]));r.push(n[2]=o);var a=i.p+i.u(t),s=new Error;i.l(a,(r=>{if(i.o(e,t)&&(0!==(n=e[t])&&(e[t]=void 0),n)){var o=r&&("load"===r.type?"missing":r.type),a=r&&r.target&&r.target.src;s.message="Loading chunk "+t+" failed.\n("+o+": "+a+")",s.name="ChunkLoadError",s.type=o,s.request=a,n[1](s)}}),"chunk-"+t,t)}};var t=(t,r)=>{var n,o,[a,s,c]=r,u=0;if(a.some((t=>0!==e[t]))){for(n in s)i.o(s,n)&&(i.m[n]=s[n]);if(c)c(i)}for(t&&t(r);u {i.r(o);var e=i(3325),t=i(5763);const r=Object.values(e.D);function n(e){const n={};return r.forEach((r=>{n[r]=function(e,r){return!1!==(0,t.Mt)(r,"".concat(e,".enabled"))}(r,e)})),n}var a=i(9144);var s=i(5546),c=i(385),u=i(8e3),d=i(5938),f=i(3960),l=i(50);class h extends d.W{constructor(e,t,r){let n=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];super(e,t,r),this.auto=n,this.abortHandler,this.featAggregate,this.onAggregateImported,n&&(0,u.R)(e,r)}importAggregator(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};if(this.featAggregate||!this.auto)return;const r=c.il&&!0===(0,t.Mt)(this.agentIdentifier,"privacy.cookies_enabled");let n;this.onAggregateImported=new Promise((e=>{n=e}));const o=async()=>{let t;try{if(r){const{setupAgentSession:e}=await Promise.all([i.e(860),i.e(242)]).then(i.bind(i,3228));t=e(this.agentIdentifier)}}catch(e){(0,l.Z)("A problem occurred when starting up session manager. This page will not start or extend any session.",e)}try{if(!this.shouldImportAgg(this.featureName,t))return void(0,u.L)(this.agentIdentifier,this.featureName);const{lazyFeatureLoader:r}=await i.e(412).then(i.bind(i,8582)),{Aggregate:o}=await r(this.featureName,"aggregate");this.featAggregate=new o(this.agentIdentifier,this.aggregator,e),n(!0)}catch(e){(0,l.Z)("Downloading and initializing ".concat(this.featureName," failed..."),e),this.abortHandler?.(),n(!1)}};c.il?(0,f.b)((()=>o()),!0):o()}shouldImportAgg(r,n){return r!==e.D.sessionReplay||!1!==(0,t.Mt)(this.agentIdentifier,"session_trace.enabled")&&(!!n?.isNew||!!n?.state.sessionReplay)}}var g=i(7633),p=i(7894);class m extends h{static featureName=g.t9;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];if(super(r,n,g.t9,i),("undefined"==typeof PerformanceNavigationTiming||c.Tt)&&"undefined"!=typeof PerformanceTiming){const n=(0,t.OP)(r);n[g.Dz]=Math.max(Date.now()-n.offset,0),(0,f.K)((()=>n[g.qw]=Math.max((0,p.z)()-n[g.Dz],0))),(0,f.b)((()=>{const t=(0,p.z)();n[g.OJ]=Math.max(t-n[g.Dz],0),(0,s.p)("timing",["load",t],void 0,e.D.pageViewTiming,this.ee)}))}this.importAggregator()}}var v=i(1117),b=i(1284);class y extends v.w{constructor(e){super(e),this.aggregatedData={}}store(e,t,r,n,i){var o=this.getBucket(e,t,r,i);return o.metrics=function(e,t){t||(t={count:0});return t.count+=1,(0,b.D)(e,(function(e,r){t[e]=w(r,t[e])})),t}(n,o.metrics),o}merge(e,t,r,n,i){var o=this.getBucket(e,t,n,i);if(o.metrics){var a=o.metrics;a.count+=r.count,(0,b.D)(r,(function(e,t){if("count"!==e){var n=a[e],i=r[e];i&&!i.c?a[e]=w(i.t,n):a[e]=function(e,t){if(!t)return e;t.c||(t=x(t.t));return t.min=Math.min(e.min,t.min),t.max=Math.max(e.max,t.max),t.t+=e.t,t.sos+=e.sos,t.c+=e.c,t}(i,a[e])}}))}else o.metrics=r}storeMetric(e,t,r,n){var i=this.getBucket(e,t,r);return i.stats=w(n,i.stats),i}getBucket(e,t,r,n){this.aggregatedData[e]||(this.aggregatedData[e]={});var i=this.aggregatedData[e][t];return i||(i=this.aggregatedData[e][t]={params:r||{}},n&&(i.custom=n)),i}get(e,t){return t?this.aggregatedData[e]&&this.aggregatedData[e][t]:this.aggregatedData[e]}take(e){for(var t={},r="",n=!1,i=0;i t.max&&(t.max=e),e 2&&void 0!==arguments[2])||arguments[2];super(e,r,j.t,n),c.il&&((0,t.OP)(e).initHidden=Boolean("hidden"===document.visibilityState),(0,N.N)((()=>(0,s.p)("docHidden",[(0,p.z)()],void 0,j.t,this.ee)),!0),(0,O.bP)("pagehide",(()=>(0,s.p)("winPagehide",[(0,p.z)()],void 0,j.t,this.ee))),this.importAggregator())}}var P=i(3081);class C extends h{static featureName=P.t9;constructor(e,t){let r=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(e,t,P.t9,r),this.importAggregator()}}var R,I=i(2210),k=i(1214),H=i(2177),L={};try{R=localStorage.getItem("__nr_flags").split(","),console&&"function"==typeof console.log&&(L.console=!0,-1!==R.indexOf("dev")&&(L.dev=!0),-1!==R.indexOf("nr_dev")&&(L.nrDev=!0))}catch(e){}function z(e){try{L.console&&z(e)}catch(e){}}L.nrDev&&H.ee.on("internal-error",(function(e){z(e.stack)})),L.dev&&H.ee.on("fn-err",(function(e,t,r){z(r.stack)})),L.dev&&(z("NR AGENT IN DEVELOPMENT MODE"),z("flags: "+(0,b.D)(L,(function(e,t){return e})).join(", ")));var M=i(6660);class B extends h{static featureName=M.t;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(r,n,M.t,i),this.skipNext=0;try{this.removeOnAbort=new AbortController}catch(e){}const o=this;o.ee.on("fn-start",(function(e,t,r){o.abortHandler&&(o.skipNext+=1)})),o.ee.on("fn-err",(function(t,r,n){o.abortHandler&&!n[M.A]&&((0,I.X)(n,M.A,(function(){return!0})),this.thrown=!0,(0,s.p)("err",[n,(0,p.z)()],void 0,e.D.jserrors,o.ee))})),o.ee.on("fn-end",(function(){o.abortHandler&&!this.thrown&&o.skipNext>0&&(o.skipNext-=1)})),o.ee.on("internal-error",(function(t){(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,o.ee)})),this.origOnerror=c._A.onerror,c._A.onerror=this.onerrorHandler.bind(this),c._A.addEventListener("unhandledrejection",(t=>{const r=function(e){let t="Unhandled Promise Rejection: ";if(e instanceof Error)try{return e.message=t+e.message,e}catch(t){return e}if(void 0===e)return new Error(t);try{return new Error(t+(0,D.P)(e))}catch(e){return new Error(t)}}(t.reason);(0,s.p)("err",[r,(0,p.z)(),!1,{unhandledPromiseRejection:1}],void 0,e.D.jserrors,this.ee)}),(0,O.m$)(!1,this.removeOnAbort?.signal)),(0,k.gy)(this.ee),(0,k.BV)(this.ee),(0,k.em)(this.ee),(0,t.OP)(r).xhrWrappable&&(0,k.Kf)(this.ee),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}onerrorHandler(t,r,n,i,o){"function"==typeof this.origOnerror&&this.origOnerror(...arguments);try{this.skipNext?this.skipNext-=1:(0,s.p)("err",[o||new F(t,r,n),(0,p.z)()],void 0,e.D.jserrors,this.ee)}catch(t){try{(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,this.ee)}catch(e){}}return!1}}function F(e,t,r){this.message=e||"Uncaught error with no additional information",this.sourceURL=t,this.line=r}let U=1;const q="nr@id";function G(e){const t=typeof e;return!e||"object"!==t&&"function"!==t?-1:e===c._A?0:(0,I.X)(e,q,(function(){return U++}))}function V(e){if("string"==typeof e&&e.length)return e.length;if("object"==typeof e){if("undefined"!=typeof ArrayBuffer&&e instanceof ArrayBuffer&&e.byteLength)return e.byteLength;if("undefined"!=typeof Blob&&e instanceof Blob&&e.size)return e.size;if(!("undefined"!=typeof FormData&&e instanceof FormData))try{return(0,D.P)(e).length}catch(e){return}}}var X=i(7243);class W{constructor(e){this.agentIdentifier=e,this.generateTracePayload=this.generateTracePayload.bind(this),this.shouldGenerateTrace=this.shouldGenerateTrace.bind(this)}generateTracePayload(e){if(!this.shouldGenerateTrace(e))return null;var r=(0,t.DL)(this.agentIdentifier);if(!r)return null;var n=(r.accountID||"").toString()||null,i=(r.agentID||"").toString()||null,o=(r.trustKey||"").toString()||null;if(!n||!i)return null;var a=(0,_.M)(),s=(0,_.Ht)(),c=Date.now(),u={spanId:a,traceId:s,timestamp:c};return(e.sameOrigin||this.isAllowedOrigin(e)&&this.useTraceContextHeadersForCors())&&(u.traceContextParentHeader=this.generateTraceContextParentHeader(a,s),u.traceContextStateHeader=this.generateTraceContextStateHeader(a,c,n,i,o)),(e.sameOrigin&&!this.excludeNewrelicHeader()||!e.sameOrigin&&this.isAllowedOrigin(e)&&this.useNewrelicHeaderForCors())&&(u.newrelicHeader=this.generateTraceHeader(a,s,c,n,i,o)),u}generateTraceContextParentHeader(e,t){return"00-"+t+"-"+e+"-01"}generateTraceContextStateHeader(e,t,r,n,i){return i+"@nr=0-1-"+r+"-"+n+"-"+e+"----"+t}generateTraceHeader(e,t,r,n,i,o){if(!("function"==typeof c._A?.btoa))return null;var a={v:[0,1],d:{ty:"Browser",ac:n,ap:i,id:e,tr:t,ti:r}};return o&&n!==o&&(a.d.tk=o),btoa((0,D.P)(a))}shouldGenerateTrace(e){return this.isDtEnabled()&&this.isAllowedOrigin(e)}isAllowedOrigin(e){var r=!1,n={};if((0,t.Mt)(this.agentIdentifier,"distributed_tracing")&&(n=(0,t.P_)(this.agentIdentifier).distributed_tracing),e.sameOrigin)r=!0;else if(n.allowed_origins instanceof Array)for(var i=0;i 2&&void 0!==arguments[2])||arguments[2];super(r,n,Z.t,i),(0,t.OP)(r).xhrWrappable&&(this.dt=new W(r),this.handler=(e,t,r,n)=>(0,s.p)(e,t,r,n,this.ee),(0,k.u5)(this.ee),(0,k.Kf)(this.ee),function(r,n,i,o){function a(e){var t=this;t.totalCbs=0,t.called=0,t.cbTime=0,t.end=E,t.ended=!1,t.xhrGuids={},t.lastSize=null,t.loadCaptureCalled=!1,t.params=this.params||{},t.metrics=this.metrics||{},e.addEventListener("load",(function(r){_(t,e)}),(0,O.m$)(!1)),c.IF||e.addEventListener("progress",(function(e){t.lastSize=e.loaded}),(0,O.m$)(!1))}function s(e){this.params={method:e[0]},T(this,e[1]),this.metrics={}}function u(e,n){var i=(0,t.DL)(r);i.xpid&&this.sameOrigin&&n.setRequestHeader("X-NewRelic-ID",i.xpid);var a=o.generateTracePayload(this.parsedOrigin);if(a){var s=!1;a.newrelicHeader&&(n.setRequestHeader("newrelic",a.newrelicHeader),s=!0),a.traceContextParentHeader&&(n.setRequestHeader("traceparent",a.traceContextParentHeader),a.traceContextStateHeader&&n.setRequestHeader("tracestate",a.traceContextStateHeader),s=!0),s&&(this.dt=a)}}function d(e,t){var r=this.metrics,i=e[0],o=this;if(r&&i){var a=V(i);a&&(r.txSize=a)}this.startTime=(0,p.z)(),this.listener=function(e){try{"abort"!==e.type||o.loadCaptureCalled||(o.params.aborted=!0),("load"!==e.type||o.called===o.totalCbs&&(o.onloadCalled||"function"!=typeof t.onload)&&"function"==typeof o.end)&&o.end(t)}catch(e){try{n.emit("internal-error",[e])}catch(e){}}};for(var s=0;s 1?e[1]=i:e.push(i)}else e[0]&&e[0].headers&&s(e[0].headers,n)&&(this.dt=n);function s(e,t){var r=!1;return t.newrelicHeader&&(e.set("newrelic",t.newrelicHeader),r=!0),t.traceContextParentHeader&&(e.set("traceparent",t.traceContextParentHeader),t.traceContextStateHeader&&e.set("tracestate",t.traceContextStateHeader),r=!0),r}}function x(e,t){this.params={},this.metrics={},this.startTime=(0,p.z)(),this.dt=t,e.length>=1&&(this.target=e[0]),e.length>=2&&(this.opts=e[1]);var r,n=this.opts||{},i=this.target;"string"==typeof i?r=i:"object"==typeof i&&i instanceof Y?r=i.url:c._A?.URL&&"object"==typeof i&&i instanceof URL&&(r=i.href),T(this,r);var o=(""+(i&&i instanceof Y&&i.method||n.method||"GET")).toUpperCase();this.params.method=o,this.txSize=V(n.body)||0}function A(t,r){var n;this.endTime=(0,p.z)(),this.params||(this.params={}),this.params.status=r?r.status:0,"string"==typeof this.rxSize&&this.rxSize.length>0&&(n=+this.rxSize);var o={txSize:this.txSize,rxSize:n,duration:(0,p.z)()-this.startTime};i("xhr",[this.params,o,this.startTime,this.endTime,"fetch"],this,e.D.ajax)}function E(t){var r=this.params,n=this.metrics;if(!this.ended){this.ended=!0;for(var o=0;o 2&&void 0!==arguments[2])||arguments[2];super(e,t,we.t,r),this.importAggregator()}}new class{constructor(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:(0,_.ky)(16);c._A?(this.agentIdentifier=t,this.sharedAggregator=new y({agentIdentifier:this.agentIdentifier}),this.features={},this.desiredFeatures=new Set(e.features||[]),this.desiredFeatures.add(m),Object.assign(this,(0,a.j)(this.agentIdentifier,e,e.loaderType||"agent")),this.start()):(0,l.Z)("Failed to initial the agent. Could not determine the runtime environment.")}get config(){return{info:(0,t.C5)(this.agentIdentifier),init:(0,t.P_)(this.agentIdentifier),loader_config:(0,t.DL)(this.agentIdentifier),runtime:(0,t.OP)(this.agentIdentifier)}}start(){const t="features";try{const r=n(this.agentIdentifier),i=[...this.desiredFeatures];i.sort(((t,r)=>e.p[t.featureName]-e.p[r.featureName])),i.forEach((t=>{if(r[t.featureName]||t.featureName===e.D.pageViewEvent){const n=function(t){switch(t){case e.D.ajax:return[e.D.jserrors];case e.D.sessionTrace:return[e.D.ajax,e.D.pageViewEvent];case e.D.sessionReplay:return[e.D.sessionTrace];case e.D.pageViewTiming:return[e.D.pageViewEvent];default:return[]}}(t.featureName);n.every((e=>r[e]))||(0,l.Z)("".concat(t.featureName," is enabled but one or more dependent features has been disabled (").concat((0,D.P)(n),"). This may cause unintended consequences or missing data...")),this.features[t.featureName]=new t(this.agentIdentifier,this.sharedAggregator)}})),(0,T.Qy)(this.agentIdentifier,this.features,t)}catch(e){(0,l.Z)("Failed to initialize all enabled instrument classes (agent aborted) -",e);for(const e in this.features)this.features[e].abortHandler?.();const r=(0,T.fP)();return delete r.initializedAgents[this.agentIdentifier]?.api,delete r.initializedAgents[this.agentIdentifier]?.[t],delete this.sharedAggregator,r.ee?.abort(),delete r.ee?.get(this.agentIdentifier),!1}}}({features:[J,m,S,class extends h{static featureName=oe;constructor(t,r){if(super(t,r,oe,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;const n=this.ee;let i;(0,k.QU)(n),this.eventsEE=(0,k.em)(n),this.eventsEE.on(se,(function(e,t){this.bstStart=(0,p.z)()})),this.eventsEE.on(ae,(function(t,r){(0,s.p)("bst",[t[0],r,this.bstStart,(0,p.z)()],void 0,e.D.sessionTrace,n)})),n.on(ce+ne,(function(e){this.time=(0,p.z)(),this.startPath=location.pathname+location.hash})),n.on(ce+ie,(function(t){(0,s.p)("bstHist",[location.pathname+location.hash,this.startPath,this.time],void 0,e.D.sessionTrace,n)}));try{i=new PerformanceObserver((t=>{const r=t.getEntries();(0,s.p)(te,[r],void 0,e.D.sessionTrace,n)})),i.observe({type:re,buffered:!0})}catch(e){}this.importAggregator({resourceObserver:i})}},C,xe,B,class extends h{static featureName=de;constructor(e,r){if(super(e,r,de,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;if(!(0,t.OP)(e).xhrWrappable)return;try{this.removeOnAbort=new AbortController}catch(e){}let n,i=0;const o=this.ee.get("tracer"),a=(0,k._L)(this.ee),s=(0,k.Lg)(this.ee),u=(0,k.BV)(this.ee),d=(0,k.Kf)(this.ee),f=this.ee.get("events"),l=(0,k.u5)(this.ee),h=(0,k.QU)(this.ee),g=(0,k.Gm)(this.ee);function m(e,t){h.emit("newURL",[""+window.location,t])}function v(){i++,n=window.location.hash,this[ve]=(0,p.z)()}function b(){i--,window.location.hash!==n&&m(0,!0);var e=(0,p.z)();this[pe]=~~this[pe]+e-this[ve],this[ye]=e}function y(e,t){e.on(t,(function(){this[t]=(0,p.z)()}))}this.ee.on(ve,v),s.on(be,v),a.on(be,v),this.ee.on(ye,b),s.on(ge,b),a.on(ge,b),this.ee.buffer([ve,ye,"xhr-resolved"],this.featureName),f.buffer([ve],this.featureName),u.buffer(["setTimeout"+le,"clearTimeout"+fe,ve],this.featureName),d.buffer([ve,"new-xhr","send-xhr"+fe],this.featureName),l.buffer([me+fe,me+"-done",me+he+fe,me+he+le],this.featureName),h.buffer(["newURL"],this.featureName),g.buffer([ve],this.featureName),s.buffer(["propagate",be,ge,"executor-err","resolve"+fe],this.featureName),o.buffer([ve,"no-"+ve],this.featureName),a.buffer(["new-jsonp","cb-start","jsonp-error","jsonp-end"],this.featureName),y(l,me+fe),y(l,me+"-done"),y(a,"new-jsonp"),y(a,"jsonp-end"),y(a,"cb-start"),h.on("pushState-end",m),h.on("replaceState-end",m),window.addEventListener("hashchange",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("load",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("popstate",(function(){m(0,i>1)}),(0,O.m$)(!0,this.removeOnAbort?.signal)),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}}],loaderType:"spa"})})(),window.NRBA=o})(); window.jQuery || document.write(' ') CKEDITOR_BASEPATH='https://f1000research.com/js/vendor/ckeditor/' window.reactTheme = 'research'; window.MathJax = { CommonHTML: { linebreaks: { automatic: true } }, 'HTML-CSS': { linebreaks: { automatic: true } }, SVG: { linebreaks: { automatic: true } }, AuthorInit: function() { MathJax.Hub.Register.MessageHook('End Process', function () { let timeout = false; // holder for timeout id const delay = 250; // delay after event is "complete" to run callback const reflowMath = function() { const dispFormulas = document.querySelectorAll('.disp-formula.panel'); if (!dispFormulas) { return; } for (const dispFormula of dispFormulas) { const child = dispFormula.querySelector('.MathJax_Preview').nextSibling.firstChild; const isMultiline = MathJax.Hub.getAllJax(dispFormula)[0].root.isMultiline; if (dispFormula.offsetWidth < child.offsetWidth || isMultiline) { MathJax.Hub.Queue(['Rerender', MathJax.Hub, dispFormula]); } } }; window.addEventListener('resize', function() { clearTimeout(timeout); // clear the timeout timeout = setTimeout(reflowMath, delay); // start timing for event "completion" }); }); }, }; if (window.location.hash == '#_=_'){ window.location = window.location.href.split('#')[0] } !function(f,b,e,v,n,t,s){if(f.fbq)return;n=f.fbq=function() {n.callMethod? n.callMethod.apply(n,arguments):n.queue.push(arguments)} ;if(!f._fbq)f._fbq=n; n.push=n;n.loaded=!0;n.version='2.0';n.queue=[];t=b.createElement(e);t.async=!0; t.src=v;s=b.getElementsByTagName(e)[0];s.parentNode.insertBefore(t,s)}(window, document,'script','https://connect.facebook.net/en_US/fbevents.js'); fbq('init', '1641728616063202'); fbq('track', "PixelInitialized", {}); (function(h,o,t,j,a,r){ h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)}; h._hjSettings={hjid:2318163,hjsv:6}; a=o.getElementsByTagName('head')[0]; r=o.createElement('script');r.async=1; r.src=t+h._hjSettings.hjid+j+h._hjSettings.hjsv; a.appendChild(r); })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv='); search file_upload Submit your research search menu close search Browse Gateways & Collections How to Publish Submit your Research My Submissions Article Guidelines Article Guidelines (New Versions) Open Data, Software and Code Guidelines Open Data and Accessible Source Materials Guidelines (HSS) Open Data, Software and Code Guidelines (PSE) Prepublication Checks Production Process Posters and Slides Guidelines Document Guidelines Article Processing Charges Peer Review Finding Article Reviewers About How it Works For Reviewers Our Advisors Policies Glossary FAQs For Developers Newsroom Contact My Research Submissions Content and Tracking Alerts My Details Sign In file_upload Submit your research { "@context": "https://schema.org", "@type": "ScholarlyArticle", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://f1000research.com/articles/13-1134" }, "headline": "Searching for non-English literature may be unnecessary for German HTA Reports", "datePublished": "2024-10-04T16:48:49", "dateModified": "2025-07-07T08:59:40", "author": [ { "@type": "Person", "name": "Elke Hausner" }, { "@type": "Person", "name": "Sibylle Sturtz" }, { "@type": "Person", "name": "Sandra Molnar" }, { "@type": "Person", "name": "Lisa Schell" }, { "@type": "Person", "name": "Wiebke Sieben" }, { "@type": "Person", "name": "Stefan Sauerland" } ], "publisher": { "@type": "Organization", "name": "F1000Research", "logo": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 480, "width": 60 } }, "image": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 1200, "width": 150 }, "description": " Background Health technology assessment (HTA) reports are based on comprehensive information retrieval. Current standards discourage the use of search restrictions, such as publication date and language. Given limited resources, it was unclear whether the effort invested in screening and translating studies published in languages other than English provided relevant additional information compared with the inclusion of English-language publications alone. We therefore analysed the impact of non-English publications on the conclusions of HTA reports produced by the German HTA agency, the Institute for Quality and Efficiency in Health Care (IQWiG). Methods We determined whether non-English publications were included in all German HTA reports on non-drug interventions (published by IQWiG between 06/2007 to 08/2018) and on selected drug interventions. If at least one non-English publication was included, we assessed for each endpoint whether or not the exclusion of non-English publications changed the conclusion. If a non-English publication did not contain information relevant to the HTA report, we classified the publication as “not relevant”. Results Of 70 HTA reports, 38 (54%) included 126 non-English publications. In 4 reports (6%) with 50 endpoints investigated in 39 PICO questions, the exclusion of a total of 10 non-English publications led to a change in the conclusions for 13 endpoints (8 PICO questions). This was largely due to the fact that in many cases, non-English publications were the predominant or only literature available, resulting in a lack of analysable data after their exclusion. Conclusions In general, studies only published in non-English languages have little influence on the conclusions of German HTA reports. For the vast majority of topics, a language restriction to English seems justified. Studies published in non-English languages may be useful in exceptional cases, for example when an intervention is only available in certain countries. " } { "@context": "http://schema.org", "@type": "BreadcrumbList", "itemListElement": [ { "@type": "ListItem", "position": "1", "item": { "@id": "https://f1000research.com/", "name": "Home" } }, { "@type": "ListItem", "position": "2", "item": { "@id": "https://f1000research.com/browse/articles", "name": "Browse" } }, { "@type": "ListItem", "position": "3", "item": { "@id": "https://f1000research.com/articles/13-1134", "name": "Searching for non-English literature may be unnecessary for German..." } } ] } Home Browse Searching for non-English literature may be unnecessary for German... ALL Metrics - Views Downloads Get PDF Get XML Cite How to cite this article Hausner E, Sturtz S, Molnar S et al. Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.12688/f1000research.151365.3 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. Close Copy Citation Details Export Export Citation Sciwheel EndNote Ref. Manager Bibtex ProCite Sente EXPORT Select a format first Track Share ▬ ✚ Research Article Revised Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] Elke Hausner https://orcid.org/0000-0003-2126-0904 1 , Sibylle Sturtz 2 , Sandra Molnar 3 , Lisa Schell 3 , Wiebke Sieben 2 , Stefan Sauerland 4 Elke Hausner https://orcid.org/0000-0003-2126-0904 1 , Sibylle Sturtz 2 , [...] Sandra Molnar 3 , Lisa Schell 3 , Wiebke Sieben 2 , Stefan Sauerland 4 PUBLISHED 07 Jul 2025 Author details Author details 1 Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, Siegburger Str. 237, Germany 2 Department of Medical Biometry, Institute for Quality and Efficiency in Health Care, Cologne, Siegburger Str. 237, Germany 3 Former employee: Department of Non-Drug Interventions, Institute for Quality and Efficiency in Health Care, Cologne, Germany 4 Department of Non-Drug Interventions, Institute for Quality and Efficiency in Health Care, Cologne, Germany Elke Hausner Roles: Conceptualization, Methodology, Project Administration, Writing – Original Draft Preparation Sibylle Sturtz Roles: Data Curation, Formal Analysis, Methodology, Writing – Original Draft Preparation Sandra Molnar Roles: Data Curation, Methodology, Writing – Original Draft Preparation Lisa Schell Roles: Conceptualization, Data Curation, Methodology, Project Administration, Writing – Original Draft Preparation Wiebke Sieben Roles: Data Curation, Formal Analysis, Methodology, Writing – Review & Editing Stefan Sauerland Roles: Conceptualization, Supervision, Writing – Review & Editing OPEN PEER REVIEW DETAILS REVIEWER STATUS This article is included in the Future of Research (FoR) collection. Abstract Background Health technology assessment (HTA) reports are based on comprehensive information retrieval. Current standards discourage the use of search restrictions, such as publication date and language. Given limited resources, it was unclear whether the effort invested in screening and translating studies published in languages other than English provided relevant additional information compared with the inclusion of English-language publications alone. We therefore analysed the impact of non-English publications on the conclusions of HTA reports produced by the German HTA agency, the Institute for Quality and Efficiency in Health Care (IQWiG). Methods We determined whether non-English publications were included in all German HTA reports on non-drug interventions (published by IQWiG between 06/2007 to 08/2018) and on selected drug interventions. If at least one non-English publication was included, we assessed for each endpoint whether or not the exclusion of non-English publications changed the conclusion. If a non-English publication did not contain information relevant to the HTA report, we classified the publication as “not relevant”. Results Of 70 HTA reports, 38 (54%) included 126 non-English publications. In 4 reports (6%) with 50 endpoints investigated in 39 PICO questions, the exclusion of a total of 10 non-English publications led to a change in the conclusions for 13 endpoints (8 PICO questions). This was largely due to the fact that in many cases, non-English publications were the predominant or only literature available, resulting in a lack of analysable data after their exclusion. Conclusions In general, studies only published in non-English languages have little influence on the conclusions of German HTA reports. For the vast majority of topics, a language restriction to English seems justified. Studies published in non-English languages may be useful in exceptional cases, for example when an intervention is only available in certain countries. READ ALL READ LESS Keywords Language; Publication Bias; Publications; Retrospective Studies Corresponding Author(s) Elke Hausner ( [email protected] ) Close Corresponding author: Elke Hausner Competing interests: No competing interests were disclosed. Grant information: The author(s) declared that no grants were involved in supporting this work. Copyright: © 2025 Hausner E et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. How to cite: Hausner E, Sturtz S, Molnar S et al. Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.12688/f1000research.151365.3 ) First published: 04 Oct 2024, 13 :1134 ( https://doi.org/10.12688/f1000research.151365.1 ) Latest published: 07 Jul 2025, 13 :1134 ( https://doi.org/10.12688/f1000research.151365.3 ) Revised Amendments from Version 2 In this version, the flowchart has been adapted in response to a peer reviewer's suggestion In this version, the flowchart has been adapted in response to a peer reviewer's suggestion See the authors' detailed response to the review by Tarquin Mittermayr See the authors' detailed response to the review by Sonia Garcia Gonzalez-Moral See the authors' detailed response to the review by Liz Dennett See the authors' detailed response to the review by Steve McDonald See the authors' detailed response to the review by Knut Sundell READ REVIEWER RESPONSES Introduction Health technology assessments (HTAs), which usually include systematic reviews, are based on comprehensive information retrieval requiring the use of multiple information sources. 1 , 2 Current standards discourage the use of search restrictions, such as publication date and language. 1 , 3 This is justified by the assumption that otherwise HTA conclusions could be biased, as studies with statistically significant results are more likely to be published in English-language journals, whereas non-significant results are more likely to be published in journals in languages other than English. 4 However, there is now evidence that significant results are increasingly being published in non-English journals. 5 In practice, many systematic reviewers restrict their searches to English-language articles. 6 , 7 For a long time, the reasons for this were of pragmatic nature: non-English literature is often more difficult to obtain and translation costs are high. 8 However, a recent systematic review by Dobrescu et al. 9 found out, that restricting evidence syntheses of interventions to English-language publications is a feasible methodological shortcut for most medical topics. The aim of our analysis was to examine whether the results of the systematic review are transferable to the context of HTA reports. The Institute for Quality and Efficiency in Health Care (IQWiG 10 ) is a German HTA agency. In addition to more than 100 HTAs based on dossiers submitted by drug companies, IQWiG conducts around 10-15 other HTAs per year, mostly on non-drug interventions. The HTA reports investigate patient-relevant endpoints or validated surrogate endpoints and may comprise a number of different PICO questions on population, intervention, comparator, and outcomes. If possible, results of single studies are pooled in meta-analyses. The overall certainty of conclusions is graded into 3 levels of increasing certainty (hint, indication or proof of harm or benefit) according to the amount and quality of the available evidence. A detailed overview of IQWiG’s methods is provided in its methods paper. 11 IQWiG’s reports are generally based on comprehensive information retrieval. However, it was previously unclear whether it was worthwhile searching for, screening, and translating non-English publications on primary studies, i.e. whether their inclusion influenced the conclusions of the reports. Aim The aim of this analysis was to assess the impact of non-English publications on the conclusions of German HTA reports. Methods Following an internal project outline, we screened HTA reports (all reports on non-drug interventions published by IQWiG between 06/2007 and 08/2018 and randomly selected reports on drugs published between 09/2011 and 11/2016) for the inclusion of non-English publications. This procedure was not updated to include more recent HTA reports in the present article, as from September 2018 onwards, IQWiG restricted its searches to publications with English or German full texts. We focused on non-drug interventions, as we expected a higher number of non-English publications in this area. In order to identify non-English publications, we screened all publications listed in the study pools of the HTA reports. This step was carried out via the EndNote databases of the underlying projects or, for older projects, via the reference lists in the HTA reports. In the next step, the language of the publications was identified using the entries in the bibliographic databases (MEDLINE, Embase). In a few cases this was not possible, so the language was checked using the full text or journal description. Data were extracted by one person and checked by another. Only journal publications were included in the analysis, i.e., unpublished reports, conferences abstracts, evidence syntheses, clinical study reports, or registry entries were not considered. For updates of HTA reports, only the newly identified references were taken into account. For all HTA reports that included at least one non-English publication, we analysed whether the exclusion of such publications led to a change in conclusions; this was done separately for each PICO question and, if applicable, for each endpoint using specific categories ( Table 1 ). Table 1. Classification of non-English publication. Category number Classification of non-English publication A Change in the conclusions of the HTA report B Conclusion does not change • Non-English study with low weight for PICO • All studies including non-English publication point into the same direction • Higher level of evidence compared to non-English publication available • Amount and quality of the evidence of all studies included in the HTA report was too low for reliable conclusions, regardless of inclusion of non-English publication • All studies included have unclear clinical relevance • Diagnostic study without direct relevance for the HTA report C Publication without relevance for the HTA report • Quality of study too low to change conclusion • A secondary publication in English was included • No data from non-English publication was used in the HTA report • Non-English publication was not translated Analysis Using standardized methods (IQWiG’s methods paper 11 and internal guidance), we re-assessed the conclusions for the affected endpoints and PICO questions in the HTA reports. We did not re-analyse HTA reports on diagnostic test accuracy (DTA), as no conclusion on test accuracy endpoints is drawn in such reports. For this purpose, the project manager reviewed all reports and assessed the impact of the exclusion of the non-English publication(s). This included the examination of individual studies on a specific endpoint or PICO question, qualitative re-assessments, and, in 3 cases, meta-analyses that had to be recalculated. When in doubt, the project manager consulted with another project manager or statistician. For each endpoint, we defined a change in a conclusion due to the exclusion of non-English publications as either an upgrading or downgrading of the certainty of the conclusion or a complete loss of data (no conclusion possible). The effect of the exclusion of non-English publications on conclusions was categorized for each HTA report as follows (see Table 1 ): Category A: change in the conclusion for a particular endpoint (and therefore of the HTA report); Category B: no change in the conclusion for a particular endpoint (and therefore of the HTA report); Category C: the non-English publications do not contain relevant information for the HTA report (e.g. because the quality of the study was too low to change the conclusion) and are therefore classified as “not relevant”. The categorization was carried out independently by the project manager and a statistician. Any discrepancies were resolved by consensus between the two. Results 70 eligible HTA reports including 2328 publications were identified 30 ( Figure 1 ). Due to the inclusion of only a small number of selected drug reports, the vast majority of the HTA reports (96%) were on non-drug interventions ( Table 2 ). Figure 1. Flowchart on the presence and impact of non-English publications. Table 2. Characteristics of the HTA reports. Characteristics of the HTA reports Number (percentage) or median [minimum; maximum] Type of intervention • Drug • Non-drug ○ Diagnostic ○ Screening ○ Treatment 3 (4) 67 (96) 16 (23) 16 (23) 35 (50) Number of publications included • Number per report 2328 22.5 [1; 156] Design of studies included (defined a priori) • RCTs only • RCTs + other study designs 33 (47) 37 (53) Number of bibliographic databases a 3 [3; 23] a In addition, information retrieval for the HTA reports usually included a search of study registries and a check of reference lists of systematic reviews. Furthermore, requests to authors were made, if necessary, and public hearings took place (except for rapid reports). Of the 70 HTA reports, 32 (46%) included English-language publications only ( Table 3 ). The remaining 38 reports (54%) included 126 non-English publications. These were most commonly published in German, Chinese, French and Spanish ( Table 4 ). Table 3. Number of non-English publications included per HTA report. Number of non-English publications included Number of HTA reports 0 32 1 17 2 4 3-11 17 Table 4. Languages of non-English publications included in the HTA reports. Language Number (%) a German 41 (1.8%) Chinese 30 (1.3%) French 12 (0.5%) Spanish 10 (0.4%) other 33 (1.4%) a % of all included 2328 publications. HTA reports with a change in the conclusions (Category A) The 4 reports with a change in the conclusions were (short titles): • A05-18: Tiotropium bromide for chronic obstructive pulmonary disease 12 , 13 • N14-02: Systemic psychotherapy in adults 14 , 15 • N16-01: Active knee motion devices for anterior cruciate ligament ruptures 16 , 17 • N16-03: Continuous passive motion (CPM) devices after knee or shoulder surgery 18 , 19 For further details, see Table 5 . Table 5. HTA reports with endpoints that resulted in a change in the conclusions (Category A). HTA reports Title Total number of PICO questions Conclusion changed (PICO) Studies published in non-English languages where exclusion from HTA report changed conclusions Evaluated EP per affected PICO question Conclusions changed (EP) A05-18 Tiotropium bromide in the treatment of chronic obstructive pulmonary disease 12 2 Fang 2008 (Chi), Jia 2008 (Chi) 2 2 9 1 N14-02 Systemic therapy in adults as a psychotherapeutic approach 18 4 Li 2010 (Chi), Yang 2005 (Chi), Wang 2011 (Chi), Wirsching 1989 (Ger), Cao 2007 (Chi), Zhang 2006 a (Chi) 2 2 6 1 17 1 9 2 N16-01 Active knee motion devices in the treatment of anterior cruciate ligament ruptures 2 1 Von Lübken 2006 (Ger) 2 2 N16-03 Motor-driven continuous passive motion (CPM) devices after interventions on the knee and shoulder joint 7 1 Michael 2005 (Ger) 3 2 a 2 publications are available for the Zhang 2006 study: Zhang 2006a and Zhang 2006b. These 4 reports investigated 50 endpoints in 39 PICO questions. In 3 out of the 4 reports (A05-18, N14-02, N16-01), the only available publication on a particular endpoint was a non-English publication. The exclusion of the non-English publication led to a change in the conclusion for 13 endpoints in 8 PICO questions ( Table 5 ). For 5 of these endpoints, the conclusion changed from a non-significant effect or an inconclusive result to “no conclusion possible” (due to a lack of data), and for 7 endpoints with previously statistically significant results, no conclusions could be drawn due to lack of data. For 1 endpoint, the conclusion changed from an inconclusive result to a hint of a benefit of the test intervention due to an effect becoming statistically significant (see example below). Of the 10 studies that influenced the conclusions, 7 were published in Chinese and 3 in German ( Table 4 ). Example from an HTA report Report N16-03 investigated the use of continuous passive motion (CPM) devices after knee or shoulder surgery. For the comparison of shoulder devices in combination with physical therapy versus physical therapy alone in patients with rotator cuff rupture, there were 2 studies on the endpoint “pain”, one in English (Garofalo 2010) and one in German (Michael 2005). When both studies were considered, the results for this endpoint were inconsistent (see Figure 2 ), and therefore no conclusion could be drawn. When Michael 2005 was excluded, there was a hint of a benefit for combination therapy, i.e. the exclusion of non-English literature changed the conclusion. Figure 2. Report N16-03: Forest plot for the endpoint “pain”. HTA reports with no change in the conclusions (Category B) If the exclusion of non-English publications did not change the conclusions of the HTA report, we identified the reason for this for each endpoint. In most cases, this was because there were several studies for each endpoint with results pointing in the same direction, meaning that the exclusion of a non-English publication had a negligible effect. Example from an HTA report Report N09-01 investigated different non-drug local treatments in patients with benign prostatic syndrome. For the comparison of holmium laser therapy (HoLEP) versus standard treatment, 6 studies were included for the endpoint “symptom scores at 3 months”, of which one was in Chinese (Zhang 2007). Neither the result of a single study nor the pooled effect estimate showed a statistically significant effect. Zhang 2007 contributed a weight of 16.1% to the pooled effect estimate (see Figure 3 ); its exclusion did not change the conclusion. Figure 3. Report N09-01: Forest plot for the endpoint “symptom scores at 3 months”. Non-English publications classified as not relevant (Category C) For the non-English publications assigned to Category C, the studies formally met the inclusion criteria of the HTA reports, and were therefore included in the study pool, but were not used in the actual assessment, mainly due to low quality of the study. Discussion Our analysis shows that the exclusion of studies published in non-English languages has only a minor effect on the conclusions of HTA reports. This is consistent with the results of a recent systematic review by Dobrescu et al. on the restriction of systematic reviews of diagnostic or treatment interventions to English-language publications. They found that the inclusion of solely English-language publications led to a change in statistical significance in only 23 out of 259 meta-analyses (9%) and concluded that “the impact of restricting systematic reviews to English-language publications is negligible for most conventional medicine topics”. 9 Our analysis shows that the results can also be transferred to the context of HTA reports. In our analysis, most studies with an impact on the conclusions of the HTA reports were published in Chinese. This is notable, as the volume of Chinese-language literature is growing rapidly and there have been calls for the inclusion of such publications. 20 , 21 However, their quality has been questioned because of methodological flaws and overly positive results. 5 On the basis of our analysis, it seems reasonable to conduct additional searches for non-English literature if a particular intervention is exclusively or preferentially available in a specific region. For example, the “continuous active motion device” is a German product that is hardly marketed outside Germany. Only 2 studies could be identified for the HTA; both were published by German authors, one in English and one in German. This has less to do with a specific clinical question and more to do with market availability, which should of course be taken into account in preparation for the systematic search. Language restrictions can save time. If an HTA report is being prepared under considerable time pressure, the exclusion of non-English literature should already take place at the level of the search strategy, resulting in a lower number of hits to be screened. Limitations Methods for meta-analysis and for the assessment of study quality have changed over the past decades, which may influence the conclusions of HTA reports. We therefore tried to follow the original methods of the HTA reports included. Only a small proportion of HTA reports required a recalculation of results. At IQWiG, the analysis presented here led to the restriction of information retrieval in HTA reports to German- and English-language literature from September 2018 onwards (see the Methods section for details). This change in approach is also described in IQWiG’s updated methods paper. 11 For the present article, it was therefore not meaningful to update the pool of HTA reports by including HTA reports published by IQWiG after October 2019. An evaluation of current HTA reports from other large HTA agencies shows that they generally use a language restriction in their reports. 22 – 29 We did not analyse whether the non-English study publications were of lower quality than the English-language ones. However, all 10 non-English studies with change in the conclusions of the HTA report (see Table 5 ) were classified as highly biased in the underlying HTA reports. Conclusions In general, studies only published in non-English languages have little influence on the conclusions of German HTA reports. For the vast majority of topics, a language restriction to English seems justified. Studies published in non-English languages may be useful in exceptional cases, for example when an intervention is only available in certain countries. Ethics and consent Ethics and consent are not required. Data availability Underlying data Zenodo: Searching for non-English literature may be unnecessary for HTA Reports - supplemental material (Version 3) [Data set]. https://doi.org/10.5281/zenodo.15025119 . 30 ▪ Supplementary-material_matrix_V3.xlsx (Data set of extracted references with details of the publication language) Data are available under the terms of the Creative Commons Attribution 4.0 International License (CC-BY 4.0). Extended data Zenodo: Searching for non-English literature may be unnecessary for HTA Reports - supplemental material [Data set]. https://doi.org/10.5281/zenodo.12642960 . 31 ▪ Supplementary-material_table.docx (HTA reports with endpoints that resulted in a change in the conclusion (Category A and B) Data are available under the terms of the Creative Commons Attribution 4.0 International License (CC-BY 4.0). Acknowledgments We thank Anke Schulz (calculations), Wolfram Groß (data curation), and Natalie McGauran (translation and editing). References 1. Lefebvre C, Glanville J, Briscoe S, et al. : Searching for and selecting studies [last updated September 2024]. In: Higgins JPT, Thomas J, Chandler J, et al. , editors. Cochrane Handbook for Systematic Reviews of Interventions version 6.5 [online]. 2024. [Accessed: 20.01.2025]. Reference Source 2. EUnetHTA JA3WP6B2-2 Authoring Team: Process of information retrieval for systematic reviews and health technology assessments on clinical effectiveness. Methodological Guidelines.2019. [Accessed: 06.02.2024]. Reference Source 3. Relevo R, Balshem H: Finding evidence for comparing medical interventions: AHRQ and the Effective Health Care Program. J. Clin. Epidemiol. 2011; 64 (11): 1168–1177. PubMed Abstract | Publisher Full Text 4. Egger M, Zellweger-Zahner T, Schneider M, et al. : Language bias in randomised controlled trials published in English and German. Lancet. 1997; 350 (9074): 326–329. Publisher Full Text 5. Wu T, Li Y, Bian Z, et al. : Randomized trials published in some Chinese journals: how many are randomized? Trials. 2009; 10 : 46. PubMed Abstract | Publisher Full Text | Free Full Text 6. Jackson JL, Kuriyama A: How Often Do Systematic Reviews Exclude Articles Not Published in English? J. Gen. Intern. Med. 2019; 34 (8): 1388–1389. PubMed Abstract | Publisher Full Text | Free Full Text 7. Hoffmann F, Allers K, Rombey T, et al. : Nearly 80 systematic reviews were published each day: Observational study on trends in epidemiology and reporting over the years 2000-2019. J. Clin. Epidemiol. 2021; 138 : 1–11. PubMed Abstract | Publisher Full Text 8. Morrison A, Moulton K, Clark M, et al. : English-Language Restriction When Conducting Systematic Review-based Meta-analyses: Systematic Review of Published Studies.2009. [Accessed: 06.02.2024]. Reference Source 9. Dobrescu AI, Nussbaumer-Streit B, Klerings I, et al. : Restricting evidence syntheses of interventions to English-language publications is a viable methodological shortcut for most medical topics: a systematic review. J. Clin. Epidemiol. 2021; 137 : 209–217. PubMed Abstract | Publisher Full Text 10. Institute for Quality and Efficiency for in Health Care: IQWiG.2024. [Accessed: 08.01.2024]. Reference Source 11. Institute for Quality and Efficiency for in Health Care: General Methods. [Accessed: 08.01.2024]. Reference Source 12. Institut für Qualität und Wirtschaftlichkeit im Gesundheitswesen: Tiotropiumbromid bei COPD; Abschlussbericht.2012. [Accessed: 06.02.2024]. Reference Source 13. Institute for Quality and Efficiency in Health Care: Tiotropium bromide for COPD; Executive Summary.2012. [Accessed: 06.02.2024]. Reference Source 14. Institut für Qualität und Wirtschaftlichkeit im Gesundheitswesen: Systemische Therapie bei Erwachsenen als Psychotherapieverfahren; Abschlussbericht.2017. [Accessed: 06.02.2024]. Reference Source 15. Institute for Quality and Efficiency in Health Care: Systemic therapy in adults as a psychotherapeutic approach; Extract.2017. [Accessed: 06.02.2024]. Reference Source 16. Institut für Qualität und Wirtschaftlichkeit im Gesundheitswesen: Aktive Kniebewegungsschienen in der Behandlung von Rupturen des vorderen Kreuzbands; Abschlussbericht.2017. [Accessed: 06.02.2024]. Reference Source 17. Institute for Quality and Efficiency in Health Care: Active knee motion devices in the treatment of anterior cruciate ligament ruptures; Extract.2017. [Accessed: 06.02.2024]. Reference Source 18. Institut für Qualität und Wirtschaftlichkeit im Gesundheitswesen: Einsatz von motorbetriebenen Bewegungsschienen (CPM) nach Interventionen am Knie- oder am Schultergelenk; Abschlussbericht.2018. [Accessed: 06.02.2024]. Reference Source 19. Institute for Quality and Efficiency for in Health Care: Motor-driven continuous passive motion (CPM) devices after interventions in the knee and shoulder joint; Extract.2018. [Accessed: 06.02.2024]. Reference Source 20. Xue J, Chen W, Chen L, et al. : Significant discrepancies were found in pooled estimates of searching with Chinese indexes versus searching with English indexes. J. Clin. Epidemiol. 2016; 70 : 246–253. PubMed Abstract | Publisher Full Text 21. Cohen JF, Korevaar DA, Wang J, et al. : Should we search Chinese biomedical databases when performing systematic reviews? Syst. Rev. 2015; 4 (1): 23. PubMed Abstract | Publisher Full Text | Free Full Text 22. Wilson M, Wang Z, Dobler CC, et al. : Noninvasive Positive Pressure Ventilation in the Home (with addendum).2020. [Accessed: 06.02.2024]. Reference Source 23. National Institute for Health and Care Excellence: Remsima (infliximab biosimilar) for subcutaneous injection for managing Crohn’s disease and ulcerative colitis; Evidence review.2021. [Accessed: 06.02.2024]. Reference Source 24. National Institute for Health and Care Excellence: Antimicrobial prescribing: delafloxacin for community-acquired pneumonia; Evidence review.2021. [Accessed: 06.02.2024]. Reference Source 25. Haute Autorité de santé: Suivi par télésurveillance des patients porteurs d’un moniteur cardiaque implantable.2021. [Accessed: 06.02.2024]. Reference Source 26. Haute Autorité de santé: Evaluation du parage de plaie de la pulpe par coiffage pulpaire direct.2019. [Accessed: 06.02.2024]. Reference Source 27. Farrell K, Horton J: Airway Pressure Release Ventilation for Acute Respiratory Distress Syndrome: An Update. Can. J. Health Technol. 2022; 2 (9). Publisher Full Text 28. Chou R, Cantor A, Dana T, et al. : Statin Use for the Primary Prevention of Cardiovascular Disease in Adults: A Systematic Review for the U.S. Preventive Services Task Force.2022. [Accessed: 06.02.2024]. Reference Source 29. Barbara AM, Grobelna A: Real-Time Continuous Glucose Monitoring for People Living With Type 1 Diabetes. Can. J. Health Technol. 2022; 2 (8). Publisher Full Text 30. Hausner E: Searching for non-English literature may be unnecessary for HTA Reports - supplemental material (Version v3) [Data set]. Zenodo. 2025. Publisher Full Text 31. Hausner E: Searching for non-English literature may be unnecessary for HTA Reports - supplemental material [Data set]. Zenodo. 2024. Publisher Full Text Comments on this article Comments (0) Version 3 VERSION 3 PUBLISHED 04 Oct 2024 ADD YOUR COMMENT Comment Author details Author details 1 Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, Siegburger Str. 237, Germany 2 Department of Medical Biometry, Institute for Quality and Efficiency in Health Care, Cologne, Siegburger Str. 237, Germany 3 Former employee: Department of Non-Drug Interventions, Institute for Quality and Efficiency in Health Care, Cologne, Germany 4 Department of Non-Drug Interventions, Institute for Quality and Efficiency in Health Care, Cologne, Germany Elke Hausner Roles: Conceptualization, Methodology, Project Administration, Writing – Original Draft Preparation Sibylle Sturtz Roles: Data Curation, Formal Analysis, Methodology, Writing – Original Draft Preparation Sandra Molnar Roles: Data Curation, Methodology, Writing – Original Draft Preparation Lisa Schell Roles: Conceptualization, Data Curation, Methodology, Project Administration, Writing – Original Draft Preparation Wiebke Sieben Roles: Data Curation, Formal Analysis, Methodology, Writing – Review & Editing Stefan Sauerland Roles: Conceptualization, Supervision, Writing – Review & Editing Competing interests No competing interests were disclosed. Grant information The author(s) declared that no grants were involved in supporting this work. Article Versions (3) version 3 Revised Published: 07 Jul 2025, 13:1134 https://doi.org/10.12688/f1000research.151365.3 version 2 Revised Published: 03 Apr 2025, 13:1134 https://doi.org/10.12688/f1000research.151365.2 version 1 Published: 04 Oct 2024, 13:1134 https://doi.org/10.12688/f1000research.151365.1 Copyright © 2025 Hausner E et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. Download Export To Sciwheel Bibtex EndNote ProCite Ref. Manager (RIS) Sente metrics Views Downloads F1000Research - - PubMed Central info_outline Data from PMC are received and updated monthly. - - Citations open_in_new 0 open_in_new 0 open_in_new SEE MORE DETAILS CITE how to cite this article Hausner E, Sturtz S, Molnar S et al. Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.12688/f1000research.151365.3 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS track receive updates on this article Track an article to receive email alerts on any updates to this article. TRACK THIS ARTICLE Share Open Peer Review Current Reviewer Status: ? Key to Reviewer Statuses VIEW HIDE Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Version 3 VERSION 3 PUBLISHED 07 Jul 2025 Revised Views 0 Cite How to cite this report: Mittermayr T. Reviewer Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.182757.r397138 ) The direct URL for this report is: https://f1000research.com/articles/13-1134/v3#referee-response-397138 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 18 Aug 2025 Tarquin Mittermayr , Austrian Institute for Health Technology Assessments, Wien, Austria Approved VIEWS 0 https://doi.org/10.5256/f1000research.182757.r397138 Thank you for providing me with another opportunity to review the revised versions of this paper. I am happy with the way the authors have taken into consideration the reviewers' comments and the changes they have made ... Continue reading READ ALL Thank you for providing me with another opportunity to review the revised versions of this paper. I am happy with the way the authors have taken into consideration the reviewers' comments and the changes they have made accordingly. Furthermore, they have provided comprehensible arguments where they did not see a reason for changing the text. I agree with the convincing comment of another reviewer that this paper provides sound arguments for the readers to make their own assessment of the impact that limiting searches to contain only English-language publications may have. Competing Interests: No competing interests were disclosed. Reviewer Expertise: Conducting systematic searches in the field of Evidence Based Medicine and Health Technology Assessment I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Mittermayr T. Reviewer Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.182757.r397138 ) The direct URL for this report is: https://f1000research.com/articles/13-1134/v3#referee-response-397138 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Version 2 VERSION 2 PUBLISHED 03 Apr 2025 Revised Views 0 Cite How to cite this report: Dennett L. Reviewer Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.179748.r376060 ) The direct URL for this report is: https://f1000research.com/articles/13-1134/v2#referee-response-376060 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 24 Apr 2025 Liz Dennett , University of Alberta, Edmonton, Alberta, Canada Approved VIEWS 0 https://doi.org/10.5256/f1000research.179748.r376060 This review reviewed 70 Health Technology Assessment reports published at the Institute for Quality and Efficiency in Health Care in Germany between 2006 and 2018. They chose to mainly include (96%) non-drug HTA's because they felt those would be more ... Continue reading READ ALL This review reviewed 70 Health Technology Assessment reports published at the Institute for Quality and Efficiency in Health Care in Germany between 2006 and 2018. They chose to mainly include (96%) non-drug HTA's because they felt those would be more likely to have non-english reports than the drug HTAs. Fifty four percent of these HTAs had english language studies included, but in only 4 HTA reports did the english language studies change results for any of the studies' endpoints. In the 4 HTA reports, 10 studies changed the conclusions on 13 endpoints. The changes were not dramatic ones but in the case of 5 endpoints, no conclusion was possible without the non-english studies as they were the only studies containing information on that endpoint. From these results the authors conclude: "In general, studies only published in non-English languages have little influence on the conclusions of German HTA reports. For the vast majority of topics, a language restriction to English seems justified". They continue to recommend searching for non-english studies in special cases (e.g. a device is only marketed in one country). Overall, I believe this to be a well conducted study with transparent methods and it presents similar findings to a number of other published articles (all of the ones I am aware of are included in the Dobrescu, et al., 2021 (Ref 1) paper cited in this study. I don't really have any comments that need to be addressed by the authors, but I can't help putting only a "Partly" for the question "Are the conclusions drawn adequately supported by the results? At the end of the day, you can't prove a negative, and it is impossible to definitively say, "It is completely fine to ignore non-english studies." This study does, however, add to the growing evidence that it is unlikely to matter that much. It is then up to individual teams if they are comfortable with the albeit small risk of losing important information for the feasibility trade off. When we train people to do evidence synthesis, we always remind students several times that the goal of every search is to be as comprehensive as possible. It is hard to go against that initial training, even when you come to appreciate that true comprehensiveness is rarely (if ever) possible. I also think it is important to keep in mind the caveat (that these authors include) that China's growing influence in scientific discourse, which means that we can't consider this topic completely settled. Is the work clearly and accurately presented and does it cite the current literature? Yes Is the study design appropriate and is the work technically sound? Yes Are sufficient details of methods and analysis provided to allow replication by others? Yes If applicable, is the statistical analysis and its interpretation appropriate? Yes Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly References 1. Dobrescu AI, Nussbaumer-Streit B, Klerings I, Wagner G, et al.: Restricting evidence syntheses of interventions to English-language publications is a viable methodological shortcut for most medical topics: a systematic review. J Clin Epidemiol . 2021; 137 : 209-217 PubMed Abstract | Publisher Full Text Competing Interests: No competing interests were disclosed. Reviewer Expertise: Expert searching I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Dennett L. Reviewer Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.179748.r376060 ) The direct URL for this report is: https://f1000research.com/articles/13-1134/v2#referee-response-376060 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 23 Aug 2025 Elke Hausner , Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, Germany 23 Aug 2025 Author Response Thank you for taking the time to review our article. We agree that it is impossible to definitively say, “It is completely fine to ignore non-English studies”. This should be ... Continue reading Thank you for taking the time to review our article. We agree that it is impossible to definitively say, “It is completely fine to ignore non-English studies”. This should be decided on a case-by-case basis, because, as we state in our article “Studies published in non-English languages may be useful in exceptional cases, for example when an intervention is only available in certain countries.” Thank you for taking the time to review our article. We agree that it is impossible to definitively say, “It is completely fine to ignore non-English studies”. This should be decided on a case-by-case basis, because, as we state in our article “Studies published in non-English languages may be useful in exceptional cases, for example when an intervention is only available in certain countries.” Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 23 Aug 2025 Elke Hausner , Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, Germany 23 Aug 2025 Author Response Thank you for taking the time to review our article. We agree that it is impossible to definitively say, “It is completely fine to ignore non-English studies”. This should be ... Continue reading Thank you for taking the time to review our article. We agree that it is impossible to definitively say, “It is completely fine to ignore non-English studies”. This should be decided on a case-by-case basis, because, as we state in our article “Studies published in non-English languages may be useful in exceptional cases, for example when an intervention is only available in certain countries.” Thank you for taking the time to review our article. We agree that it is impossible to definitively say, “It is completely fine to ignore non-English studies”. This should be decided on a case-by-case basis, because, as we state in our article “Studies published in non-English languages may be useful in exceptional cases, for example when an intervention is only available in certain countries.” Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Views 0 Cite How to cite this report: McDonald S. Reviewer Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.179748.r376055 ) The direct URL for this report is: https://f1000research.com/articles/13-1134/v2#referee-response-376055 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 21 Apr 2025 Steve McDonald , School of Public Health and Preventive Medicine, Monash University, Clayton, Victoria, Australia Approved VIEWS 0 https://doi.org/10.5256/f1000research.179748.r376055 Thank you for the opportunity to review this revised version. This paper is a useful contribution to the evidence base of how language bias may affect the results of evidence synthesis. The authors provide a comprehensive case study of the ... Continue reading READ ALL Thank you for the opportunity to review this revised version. This paper is a useful contribution to the evidence base of how language bias may affect the results of evidence synthesis. The authors provide a comprehensive case study of the impact of excluding non-English language publications on the conclusions of 70 HTA reports produced by IQWIG from 2007 to 2018. Previous peer reviewers raised several valid concerns and suggested amendments, notably that the conclusions of the study should be limited to the context of German HTA reports. I feel the authors have adequately addressed the concerns raised. The revised paper is well-written and clearly presented, with sufficient information provided to allow readers to make their own assessment of the value of restricting to the inclusion of English-language publications. The authors' conclusion that non-English publications "have little influence" or "minor effect" is supported by the data. Further, the authors provide a useful caveat to when searching for (and including) non-English publications should be considered. One brief comment: the analysis focuses on journal publications (which is reasonable). It is not clear if (for these 70 HTA reports) non-journal publications in languages other than English were included, and thus whether they had any impact on the the conclusions. I guess the question is to what extent the 'unnecessary' searching of non-English publications extends to non-journal article formats (conference abstracts, register entries). One minor comment: the flow diagram (second row) has 128 non-English publications, but in the Abstract and Results this is 126. Is the work clearly and accurately presented and does it cite the current literature? Yes Is the study design appropriate and is the work technically sound? Yes Are sufficient details of methods and analysis provided to allow replication by others? Yes If applicable, is the statistical analysis and its interpretation appropriate? Yes Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes References 1. Hausner E, Sturtz S, Molnar S, Schell L, et al.: Searching for non-English literature may be unnecessary for German HTA Reports. F1000Research . 2025; 13 . Publisher Full Text Competing Interests: No competing interests were disclosed. Reviewer Expertise: Information specialist with expertise in evidence synthesis. I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT McDonald S. Reviewer Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.179748.r376055 ) The direct URL for this report is: https://f1000research.com/articles/13-1134/v2#referee-response-376055 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 11 Sep 2025 Elke Hausner , Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, Germany 11 Sep 2025 Author Response Thank you for taking the time to review our article. Unfortunately, we are unable to answer your question as to whether our conclusions can also be applied to other forms ... Continue reading Thank you for taking the time to review our article. Unfortunately, we are unable to answer your question as to whether our conclusions can also be applied to other forms of publication such as conference abstracts, as this was not part of our original research question. Your reference to the numbers in the flowchart is correct. The following data must be adjusted in the flowchart: 2,328 instead of 2,339 publications, and 126 instead of 128 non-English publications. We will update the flowchart accordingly. Thank you for taking the time to review our article. Unfortunately, we are unable to answer your question as to whether our conclusions can also be applied to other forms of publication such as conference abstracts, as this was not part of our original research question. Your reference to the numbers in the flowchart is correct. The following data must be adjusted in the flowchart: 2,328 instead of 2,339 publications, and 126 instead of 128 non-English publications. We will update the flowchart accordingly. Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 11 Sep 2025 Elke Hausner , Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, Germany 11 Sep 2025 Author Response Thank you for taking the time to review our article. Unfortunately, we are unable to answer your question as to whether our conclusions can also be applied to other forms ... Continue reading Thank you for taking the time to review our article. Unfortunately, we are unable to answer your question as to whether our conclusions can also be applied to other forms of publication such as conference abstracts, as this was not part of our original research question. Your reference to the numbers in the flowchart is correct. The following data must be adjusted in the flowchart: 2,328 instead of 2,339 publications, and 126 instead of 128 non-English publications. We will update the flowchart accordingly. Thank you for taking the time to review our article. Unfortunately, we are unable to answer your question as to whether our conclusions can also be applied to other forms of publication such as conference abstracts, as this was not part of our original research question. Your reference to the numbers in the flowchart is correct. The following data must be adjusted in the flowchart: 2,328 instead of 2,339 publications, and 126 instead of 128 non-English publications. We will update the flowchart accordingly. Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Version 1 VERSION 1 PUBLISHED 04 Oct 2024 Views 0 Cite How to cite this report: Mittermayr T. Reviewer Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.166008.r344992 ) The direct URL for this report is: https://f1000research.com/articles/13-1134/v1#referee-response-344992 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 28 Dec 2024 Tarquin Mittermayr , Austrian Institute for Health Technology Assessments, Wien, Austria Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.166008.r344992 I am happy to be invited to write a review on this relevant topic. Whilst the method for assessing the impact of excluding non-english literature on the conclusion of HTA reports is generally explained in a clear and comprehensible ... Continue reading READ ALL I am happy to be invited to write a review on this relevant topic. Whilst the method for assessing the impact of excluding non-english literature on the conclusion of HTA reports is generally explained in a clear and comprehensible manner, Hausner et al fall short of explaining or referencing the internal protocol mentioned in the methods section. It would have also been helpful to either provide an explanation already in this section or refer to the appropriate page (e.g. “please find details on page 8”) in the limitations section as to why no publications after 2018 were considered for this assessment. Table 1 would benefit from clearer and more concise information: The same reasons for classifying low quality studies as category B in the classification table (“Amount and quality of the evidence…”) are listed in the explanation of the categories below the table under category C, thus giving room for ambiguous interpretations. Table 2 contains the total number of HTA reports included as well as the number per report. It is unclear if the number given for the latter (22.5) is an average, median or a percentage. In consideration of whether providing this number is meaningful, stating just the range of included publications per report might suffice and be more explanatory. The discussion section contains the confusing remark “We did not analyse whether the non-English study publications were of lower quality than the English language ones”. This appears to be contradictory to the explanations given for the classifications in Table 1. Should the contradictory nature of this statement be due to an imprecision in terminology (quality of evidence vs. quality of study publications), this study by Hausner et al would have benefited from performing a qualitative analyses of the publications themselves (in addition to analysing the containing evidence). In an article which argues that excluding non-English literature may have little impact on the overall conclusion of an HTA report, the example from Report N16-03 given in the Results section, where the exclusion of a non-English article led to a change in the conclusion, might sit uncomfortably with systematic evidence based medicine professionals. As graphical this example may be in showing the effect of this exclusion, it also appears to undermine the core argument of this article. A revision by the authors of this publication should focus on the coherence of their argumentation on the one hand, and on supporting their methods with more concise examples (e.g. Tables and Figures). Is the work clearly and accurately presented and does it cite the current literature? Partly Is the study design appropriate and is the work technically sound? Partly Are sufficient details of methods and analysis provided to allow replication by others? Partly If applicable, is the statistical analysis and its interpretation appropriate? I cannot comment. A qualified statistician is required. Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly Competing Interests: No competing interests were disclosed. Reviewer Expertise: Conducting systematic searches in my position as an Information Specialist I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Mittermayr T. Reviewer Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.166008.r344992 ) The direct URL for this report is: https://f1000research.com/articles/13-1134/v1#referee-response-344992 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 03 Apr 2025 Elke Hausner , Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, 50670, Germany 03 Apr 2025 Author Response Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The reviewer indicates that it would ... Continue reading Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The reviewer indicates that it would be helpful to explain or reference the internal protocol mentioned in the methods section. Reply: We have exchanged the term “protocol” with “project outline”. This document is a German-language 1-page text that provides an overview of the project, not a complete step-by-step protocol. It was not published and we therefore cannot reference it. Change in the text: We have amended the text “Following an internal project outline, we screened HTA reports (all reports on non-drug interventions published by IQWiG between 06/2007 and 08/2018 and randomly selected reports on drugs published between 09/2011 and 11/2016) for the inclusion of non-English publications.” 2. The reviewer states that it would have been helpful to either provide an explanation already in this section or refer to the appropriate page (e.g. “please find details on page 8”) in the limitations section as to why no publications after 2018 were considered for this assessment. Change in the text: We have amended the text ”For the present article, it was therefore not meaningful to update the pool of HTA reports by including HTA reports published by IQWiG after October 2019 (see the methods section for details).“ 3. The reviewer states that table 1 would benefit from clearer and more concise information: The same reasons for classifying low quality studies as category B in the classification table (“Amount and quality of the evidence…”) are listed in the explanation of the categories below the table under category C, thus giving room for ambiguous interpretations. Reply: We disagree that there is ambiguity, because in category B the reason is “Amount and quality of the evidence of all studies”, whereas in category C the reason is “Quality of study too low to change conclusion”. The first reason refers to the evidence in its totality, the second reason addresses the quality of only the non-English study. No change to the text 4. The reviewer points out that in Table 2 it is unclear whether the number of included publications per report (22.5) is an average, median or percentage. Reply: Numbers and percentages are indicated by round brackets, while median and ranges are reported with square brackets. The heading indicates that 22.5 [1; 156] is the median, with the range given. We believe that it is reasonable to display median and range, rather than just reporting the range. No change to the text 5. The reviewer refers to the different information from Table 1 and the section ‘Limitations’ regarding the examination of study quality. In addition, the reviewer states that the study would have benefited from performing a qualitative analysis for publications. Reply: We are uncertain what type of “qualitative analysis” could be useful to be performed in addition to analysing the available evidence. As HTA work is evidence-based, the methods and the results of the primary studies are the sole data sources. Unfortunately, a qualitative analysis was not possible due to the limited time available. However, this would be very interesting, as there are already several publications [1-3] on this topic that question the quality of non-English studies. Change in the text : We have added the assessment of the risk of bias for the non-English studies in the text. “However, all 10 non-English studies that changed the conclusions of an HTA report (see Table 5) were classified as highly biased in the underlying HTA reports.” 6. The reviewer points out that the example of N16-03 listed under ‘Results’ could lead to discomfort among experts in evidence-based medicine since it seems to undermine the core argument of this article. Reply: Admittedly, the recommendation not to search for non-English literature would have been easier if the decision to include or exclude such sources had not affected the conclusion of any HTA report. As 4 reports (6%) experienced a change in the conclusions, our conclusion is already somewhat toned down (“seems justified”). We agree with the reviewer that the decision is not straightforward. Some experts will continue to search non-English sources, because they consider the risk of overlooking a study more important than the resources required for searching for non-English publications. In our view, however, the main conclusion is correctly inferred from our results. No change to the text Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The reviewer indicates that it would be helpful to explain or reference the internal protocol mentioned in the methods section. Reply: We have exchanged the term “protocol” with “project outline”. This document is a German-language 1-page text that provides an overview of the project, not a complete step-by-step protocol. It was not published and we therefore cannot reference it. Change in the text: We have amended the text “Following an internal project outline, we screened HTA reports (all reports on non-drug interventions published by IQWiG between 06/2007 and 08/2018 and randomly selected reports on drugs published between 09/2011 and 11/2016) for the inclusion of non-English publications.” 2. The reviewer states that it would have been helpful to either provide an explanation already in this section or refer to the appropriate page (e.g. “please find details on page 8”) in the limitations section as to why no publications after 2018 were considered for this assessment. Change in the text: We have amended the text ”For the present article, it was therefore not meaningful to update the pool of HTA reports by including HTA reports published by IQWiG after October 2019 (see the methods section for details).“ 3. The reviewer states that table 1 would benefit from clearer and more concise information: The same reasons for classifying low quality studies as category B in the classification table (“Amount and quality of the evidence…”) are listed in the explanation of the categories below the table under category C, thus giving room for ambiguous interpretations. Reply: We disagree that there is ambiguity, because in category B the reason is “Amount and quality of the evidence of all studies”, whereas in category C the reason is “Quality of study too low to change conclusion”. The first reason refers to the evidence in its totality, the second reason addresses the quality of only the non-English study. No change to the text 4. The reviewer points out that in Table 2 it is unclear whether the number of included publications per report (22.5) is an average, median or percentage. Reply: Numbers and percentages are indicated by round brackets, while median and ranges are reported with square brackets. The heading indicates that 22.5 [1; 156] is the median, with the range given. We believe that it is reasonable to display median and range, rather than just reporting the range. No change to the text 5. The reviewer refers to the different information from Table 1 and the section ‘Limitations’ regarding the examination of study quality. In addition, the reviewer states that the study would have benefited from performing a qualitative analysis for publications. Reply: We are uncertain what type of “qualitative analysis” could be useful to be performed in addition to analysing the available evidence. As HTA work is evidence-based, the methods and the results of the primary studies are the sole data sources. Unfortunately, a qualitative analysis was not possible due to the limited time available. However, this would be very interesting, as there are already several publications [1-3] on this topic that question the quality of non-English studies. Change in the text : We have added the assessment of the risk of bias for the non-English studies in the text. “However, all 10 non-English studies that changed the conclusions of an HTA report (see Table 5) were classified as highly biased in the underlying HTA reports.” 6. The reviewer points out that the example of N16-03 listed under ‘Results’ could lead to discomfort among experts in evidence-based medicine since it seems to undermine the core argument of this article. Reply: Admittedly, the recommendation not to search for non-English literature would have been easier if the decision to include or exclude such sources had not affected the conclusion of any HTA report. As 4 reports (6%) experienced a change in the conclusions, our conclusion is already somewhat toned down (“seems justified”). We agree with the reviewer that the decision is not straightforward. Some experts will continue to search non-English sources, because they consider the risk of overlooking a study more important than the resources required for searching for non-English publications. In our view, however, the main conclusion is correctly inferred from our results. No change to the text Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 03 Apr 2025 Elke Hausner , Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, 50670, Germany 03 Apr 2025 Author Response Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The reviewer indicates that it would ... Continue reading Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The reviewer indicates that it would be helpful to explain or reference the internal protocol mentioned in the methods section. Reply: We have exchanged the term “protocol” with “project outline”. This document is a German-language 1-page text that provides an overview of the project, not a complete step-by-step protocol. It was not published and we therefore cannot reference it. Change in the text: We have amended the text “Following an internal project outline, we screened HTA reports (all reports on non-drug interventions published by IQWiG between 06/2007 and 08/2018 and randomly selected reports on drugs published between 09/2011 and 11/2016) for the inclusion of non-English publications.” 2. The reviewer states that it would have been helpful to either provide an explanation already in this section or refer to the appropriate page (e.g. “please find details on page 8”) in the limitations section as to why no publications after 2018 were considered for this assessment. Change in the text: We have amended the text ”For the present article, it was therefore not meaningful to update the pool of HTA reports by including HTA reports published by IQWiG after October 2019 (see the methods section for details).“ 3. The reviewer states that table 1 would benefit from clearer and more concise information: The same reasons for classifying low quality studies as category B in the classification table (“Amount and quality of the evidence…”) are listed in the explanation of the categories below the table under category C, thus giving room for ambiguous interpretations. Reply: We disagree that there is ambiguity, because in category B the reason is “Amount and quality of the evidence of all studies”, whereas in category C the reason is “Quality of study too low to change conclusion”. The first reason refers to the evidence in its totality, the second reason addresses the quality of only the non-English study. No change to the text 4. The reviewer points out that in Table 2 it is unclear whether the number of included publications per report (22.5) is an average, median or percentage. Reply: Numbers and percentages are indicated by round brackets, while median and ranges are reported with square brackets. The heading indicates that 22.5 [1; 156] is the median, with the range given. We believe that it is reasonable to display median and range, rather than just reporting the range. No change to the text 5. The reviewer refers to the different information from Table 1 and the section ‘Limitations’ regarding the examination of study quality. In addition, the reviewer states that the study would have benefited from performing a qualitative analysis for publications. Reply: We are uncertain what type of “qualitative analysis” could be useful to be performed in addition to analysing the available evidence. As HTA work is evidence-based, the methods and the results of the primary studies are the sole data sources. Unfortunately, a qualitative analysis was not possible due to the limited time available. However, this would be very interesting, as there are already several publications [1-3] on this topic that question the quality of non-English studies. Change in the text : We have added the assessment of the risk of bias for the non-English studies in the text. “However, all 10 non-English studies that changed the conclusions of an HTA report (see Table 5) were classified as highly biased in the underlying HTA reports.” 6. The reviewer points out that the example of N16-03 listed under ‘Results’ could lead to discomfort among experts in evidence-based medicine since it seems to undermine the core argument of this article. Reply: Admittedly, the recommendation not to search for non-English literature would have been easier if the decision to include or exclude such sources had not affected the conclusion of any HTA report. As 4 reports (6%) experienced a change in the conclusions, our conclusion is already somewhat toned down (“seems justified”). We agree with the reviewer that the decision is not straightforward. Some experts will continue to search non-English sources, because they consider the risk of overlooking a study more important than the resources required for searching for non-English publications. In our view, however, the main conclusion is correctly inferred from our results. No change to the text Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The reviewer indicates that it would be helpful to explain or reference the internal protocol mentioned in the methods section. Reply: We have exchanged the term “protocol” with “project outline”. This document is a German-language 1-page text that provides an overview of the project, not a complete step-by-step protocol. It was not published and we therefore cannot reference it. Change in the text: We have amended the text “Following an internal project outline, we screened HTA reports (all reports on non-drug interventions published by IQWiG between 06/2007 and 08/2018 and randomly selected reports on drugs published between 09/2011 and 11/2016) for the inclusion of non-English publications.” 2. The reviewer states that it would have been helpful to either provide an explanation already in this section or refer to the appropriate page (e.g. “please find details on page 8”) in the limitations section as to why no publications after 2018 were considered for this assessment. Change in the text: We have amended the text ”For the present article, it was therefore not meaningful to update the pool of HTA reports by including HTA reports published by IQWiG after October 2019 (see the methods section for details).“ 3. The reviewer states that table 1 would benefit from clearer and more concise information: The same reasons for classifying low quality studies as category B in the classification table (“Amount and quality of the evidence…”) are listed in the explanation of the categories below the table under category C, thus giving room for ambiguous interpretations. Reply: We disagree that there is ambiguity, because in category B the reason is “Amount and quality of the evidence of all studies”, whereas in category C the reason is “Quality of study too low to change conclusion”. The first reason refers to the evidence in its totality, the second reason addresses the quality of only the non-English study. No change to the text 4. The reviewer points out that in Table 2 it is unclear whether the number of included publications per report (22.5) is an average, median or percentage. Reply: Numbers and percentages are indicated by round brackets, while median and ranges are reported with square brackets. The heading indicates that 22.5 [1; 156] is the median, with the range given. We believe that it is reasonable to display median and range, rather than just reporting the range. No change to the text 5. The reviewer refers to the different information from Table 1 and the section ‘Limitations’ regarding the examination of study quality. In addition, the reviewer states that the study would have benefited from performing a qualitative analysis for publications. Reply: We are uncertain what type of “qualitative analysis” could be useful to be performed in addition to analysing the available evidence. As HTA work is evidence-based, the methods and the results of the primary studies are the sole data sources. Unfortunately, a qualitative analysis was not possible due to the limited time available. However, this would be very interesting, as there are already several publications [1-3] on this topic that question the quality of non-English studies. Change in the text : We have added the assessment of the risk of bias for the non-English studies in the text. “However, all 10 non-English studies that changed the conclusions of an HTA report (see Table 5) were classified as highly biased in the underlying HTA reports.” 6. The reviewer points out that the example of N16-03 listed under ‘Results’ could lead to discomfort among experts in evidence-based medicine since it seems to undermine the core argument of this article. Reply: Admittedly, the recommendation not to search for non-English literature would have been easier if the decision to include or exclude such sources had not affected the conclusion of any HTA report. As 4 reports (6%) experienced a change in the conclusions, our conclusion is already somewhat toned down (“seems justified”). We agree with the reviewer that the decision is not straightforward. Some experts will continue to search non-English sources, because they consider the risk of overlooking a study more important than the resources required for searching for non-English publications. In our view, however, the main conclusion is correctly inferred from our results. No change to the text Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Gonzalez-Moral SG. Reviewer Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.166008.r344994 ) The direct URL for this report is: https://f1000research.com/articles/13-1134/v1#referee-response-344994 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 14 Dec 2024 Sonia Garcia Gonzalez-Moral , Newcastle University, Newcastle upon Tyne, England, UK Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.166008.r344994 I welcome the opportunity to comment on this paper. With this study the authors have challenged the status quo of current systematic literature review guidelines such as those from Cochrane and health technology assessment methods recommendations for undertaking literature searches ... Continue reading READ ALL I welcome the opportunity to comment on this paper. With this study the authors have challenged the status quo of current systematic literature review guidelines such as those from Cochrane and health technology assessment methods recommendations for undertaking literature searches such as those recommended by EUneHTA. Such bold move feels justified if one understand the intensity of resource and capacity involved in HTA and so, while I support the rationale for this study and think that this is a good first attempt to breach this topic, in general I do not think that this study boasts the sufficient methodological rigor nor a big enough sample size to draw conclusions that could be generalized to HTA. To start with, the title of this manuscript ‘Searching for non-English literature may be unnecessary for HTA reports’ suggests the opposite of what the authors conclude ‘ At IQWiG, the analysis presented here led to the restriction of information retrieval in HTA reports to German- and English-language literature from September 2018 onwards’. I would therefore recommend that the title is revised, contextualizing this study to the needs and experience of the German HTA agency may increase the accuracy of their findings and conclusions. In the Introduction section the authors draw from one single study (Wu et al., 2009) to justify why non-English studies should no longer be searched for in HTA reports. Wu et al. analysed how randomised controlled trials were being reported in the Chinese literature written in Chinese. Their paper seemed to suggest that significant results were being published in languages other than English (in their case Chinese) and that RCTs were often poorly reported in Chinese. However, the authors of this manuscript later in the Discussion counterargue this point by remarking that ‘ […] as the volume of Chinese-language literature is growing rapidly and there have been calls for the inclusion of such publications. However, their quality has been questioned because of methodological flaws and overly positive results. ’ Deviating from the point of publication bias and focusing on the quality of the research as a reason for not searching non- English language publications in HTA. While the quality of research is extremely important, there are instruments that deal with that issue within the review process making ‘poor quality’ (according to Wu’s study) not a priory reason for not searching the entire scientific literature written on a particular language such as Chinese. In light of this contradiction the rationale for this study should be reformulated to accurately justify the reasons for not searching non-English papers and always placing these in the context of the German HTA agency. So far, the reasons presented here do not seem to warrant such claim. There are a number of omissions and inaccuracies through the methods section such as the search period in the abstract is not in line with the period reported in the Methods rendering this point unclear. Further, having read the full paper twice, I am still unsure of whether HTA reports of drugs were or not included in their assessment. I think the authors should provide a detailed list of the HTA reports (medicines and non-medicines) that were screened for inclusion in the methods section since these are not their results but the body of data they used for analysis. They should also justify why diagnostic HTA reports would have been included from the start if they were not to be re-analysed in the results. Furthermore, the authors seem to apply the inclusion and exclusion criteria unsystematically when they declare that they only considered journal articles cited in HTA reports but then they present as a limitation that ‘ For 2 reports (D06-01B and D06-01C), we also included evidence syntheses, although this document type was to be excluded.’ But fail to provide an explanation as to why such allowance was made. In the results section two random examples are provided of HTA report, can the authors justify why these are explained in more detail and not the others? which criteria led the authors to provide these two examples? This feels selection bias and should be avoided. In the Discussion section authors seem to contradict themselves. They start by saying that ‘the exclusion of studies published in non-English languages has only a minor effect on the conclusions of HTA reports’ At this point it is important to note that the measure of such effect (whether is big or small) is not provided throughout the manuscript nor it is explained how this ‘change in effect’ was estimated, there are no statistical tests and no p values provided making is really difficult to study whether the change, although minor, might or might not have been significant for the results of the HTA report and their future recommendations. The section follows by saying that ‘ On the basis of our analysis, it is not possible to identify topics where non-English publications might be relevant.’ This statement seems to contradict what has just been said. And finally, they acknowledge that actually ‘it seems reasonable to conduct additional searches for non-English literature if a particular intervention is exclusively or preferentially available in a specific region’. The latter being a statement which defeats the point of the entire manuscript in its current rationale and supporting evidence. Please fix reference number 1 to reflect the title (Cochrane Handbook for Systematic Reviews of Interventions) of the handbook in with Chapter 4 is included. Table 1 would benefit from including definitions for each of the categories. Figure 1 please include the distribution of the 38 reports as they are allocated to different categories. Table 5. What does EP mean? Is the work clearly and accurately presented and does it cite the current literature? No Is the study design appropriate and is the work technically sound? Partly Are sufficient details of methods and analysis provided to allow replication by others? No If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Partly Are the conclusions drawn adequately supported by the results? No Competing Interests: No competing interests were disclosed. Reviewer Expertise: Information retrieval, systematic reviews, HTA, health research methods I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Gonzalez-Moral SG. Reviewer Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.166008.r344994 ) The direct URL for this report is: https://f1000research.com/articles/13-1134/v1#referee-response-344994 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 03 Apr 2025 Elke Hausner , Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, 50670, Germany 03 Apr 2025 Author Response Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The first point raised is that ... Continue reading Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The first point raised is that the study is not suitable for making general statements about HTA reports. Reply: This point was also raised by Reviewer 1 (see above). We have taken your concerns and suggestions and adapted the text Change in the text: instead of referring to HTA reports in general, the text now refers specifically to the German HTA reports 2. It was also criticized that the authors draw from one single study (Wu et al., 2009) to justify why non-English studies should no longer be searched for in HTA reports. Reply: This is a misunderstanding, as Wu 2009 was cited primarily to show that the common assumption that foreign-language articles primarily produce negative/neutral results should be questioned. In order to strengthen the rationale for why we see our study as an important contribution, we have added a current systematic review in the background section, which we had originally only mentioned in the discussion. Change in the text: In the background section, we have added the systematic review by Dobrescu et al. as a rationale for the present analysis. Please see changes in the text on page 4. “However, a recent systematic review by Dobrescu et al. [9] found, that restricting evidence syntheses of interventions to English-language publications is a feasible methodological shortcut for most medical topics. The aim of our analysis was to examine whether the results of the systematic review are transferable to the context of HTA reports.” 3. Another comment refers to different information on the search / publication period in the abstract and the methods section. Change to the text: We have now presented the publication period in the abstract. “We determined whether non-English publications were included in all HTA reports on non-drug interventions (published by IQWiG between 06/2007 to 08/2018) and on selected drug interventions.” 4. It is also noted that it remains unclear whether HTA reports on drugs were included in the analysis. Reply: Table 2 displays the number of included drug / non-drug HTA reports (3 HTA reports on drugs, 67 on non-drugs). No change to the text 5. The reviewer recommends providing a detailed list of HTA reports (drug and non-drug) that were screened for inclusion in the methods section. Reply: As stated in the methods, we included all HTA reports on non-drug interventions (published between 06/2007 and 08/2018) and selected 3 reports on drugs. We have added a list of analysed HTA reports to the supplement. Change to the text: We have added a list of included HTA reports https://doi.org/10.5281/zenodo.12642959 6. The reviewer would like us to justify why diagnostic HTA reports were included from the beginning if they are not to be re-analysed in the results. Reply: HTA reports on diagnostic questions can contain both results on the benefit of the diagnostic measure and on the diagnostic accuracy. Publications in other languages than English only contained information on diagnostic accuracy. However, in no case was a conclusion derived from this, but only the diagnostic accuracy was reported. Our article focuses on the impact of non-English publications on the conclusions of HTA reports, therefore a comparison of conclusions based on different data sets is not possible. No change to the text 7. The reviewer is concerned that the inclusion and exclusion criteria were applied unsystematically and asks for an explanation as to why 2 HTA reports are cited under the limitations. Reply: Based on your concerns, we have revised the extraction of the included studies in the HTA reports and have deleted the entries that did not fulfil the inclusion criteria (n = 11). Change in the text: The tables and figures in the text have been adjusted accordingly. The total number of references included is now 2328, the number of non-English publications 126. Thus, we have deleted the corresponding section in the limitations. 8. In addition, the reviewer notes that there are two examples of HTA reports listed in the results section and asks for a more detailed explanation of why these were selected. Reply: We wanted to illustrate a little more clearly to the reader why a non-English language article was categorized as A or B. As these are only examples, we see no risk of bias in the selection. The full list of HTA reports with non-English articles in categories A and B can be found in the supplement (https://doi.org/10.5281/zenodo.12642959). No change to the text 9. The reviewer points out that the measure of effect is not explained and that no statistical tests and no p-values are given. In her view, this makes it difficult to analyse whether or not the change, even if small, could have been significant for the results of the HTA report and the future recommendations. Reply: The measure of effect was defined as “change in the conclusion of the HTA report” which is composed of the conclusions of the individual endpoints. An overall conclusion was not provided by the underlying HTA reports. For most HTA reports (94%) no changes in the endpoints were identified. All changes were assessed in a yes/no format. This effect measure was therefore qualitative, even it was partly based on statistical analyses and re-analyses. The reviewer rightly points out that even small changes in a meta-analysis may or may not have been significant for the results of the HTA report. For this very reason, our analyses were not based on any statistical comparison between primary analysis and re-analysis. Rather, we examined whether the re-analysis led to a change in the conclusion of the endpoint and thus the HTA report, as this determines reimbursement and availability of health care services. No change to the text 10. According to the reviewer, it is contradictory first to conclude that searching the non-English literature is generally unnecessary, but then to explain that such searches may be worthwhile for particular interventions that are available only a specific region. Reply: We agree that a single HTA report on a country-specific intervention (N16-01) is a weak basis for recommending additional searches of non-English sources under specific circumstances. However, our text only describes this option as “reasonable” – without recommending it for all clinical interventions without clear indications that this is useful in the respective case. Change in the text: We amended the text in the discussion “This has less to do with a specific clinical question and more to do with market availability, which should of course be taken into account when preparing the systematic search.” 11. The reviewer asks to fix reference number 1 to reflect the title (Cochrane Handbook for Systematic Reviews of Interventions) of the handbook in which Chapter 4 is included. Change in the text: We amended the citation 1. Lefebvre C, Glanville J, Briscoe S et al. Chapter 4: Searching for and selecting studies [last updated September 2024]. In: Higgins JPT, Thomas J, Chandler J, Cumpston M, Li T, Page MJ, Welch VA (editors). Cochrane Handbook for Systematic Reviews of Interventions version 6.5 [online]. 2024 [Accessed: 20.01.2025]. URL: https://training.cochrane.org/handbook/current/chapter-04 . 12. Table 1 would benefit from including definitions for each of the categories. You can find a description within the text (page 4): “Category A: change in the conclusion for a particular endpoint; Category B: no change in the conclusion for a particular endpoint; Category C: the non-English publications do not contain relevant information for the HTA report (e.g. because the amount or quality of the data is too low) and are therefore classified as “not relevant”.” Change in the text: We have added the reference to Table 1 to the text “The effect of the exclusion of non-English publications on conclusions was categorized for each HTA report as follows (see Table 1)” 13. The reviewer asks to include in Figure 1 the distribution of the 38 reports as they are allocated to different categories. Reply: The requested information can be seen in Figure 1. As shown in the figure, 13 HTA reports contain only category C publications, 12 HTA reports with category B and 4 HTA reports to category A. The footnote also refers to the fact that an HTA report can contain non-English publications from different categories (A, B or C). The HTA report was assigned to the category that has the greatest impact on the change in conclusion. For example, the 12 HTA reports with category B also contain non-English publications from category C, but none from category A. No change to the text 14. The reviewer asks what the abbreviation EP in Table 5 means. Reply: This is the abbreviation for “endpoint“. An overview of the abbreviations used in the table is added below the table. No change to the text Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The first point raised is that the study is not suitable for making general statements about HTA reports. Reply: This point was also raised by Reviewer 1 (see above). We have taken your concerns and suggestions and adapted the text Change in the text: instead of referring to HTA reports in general, the text now refers specifically to the German HTA reports 2. It was also criticized that the authors draw from one single study (Wu et al., 2009) to justify why non-English studies should no longer be searched for in HTA reports. Reply: This is a misunderstanding, as Wu 2009 was cited primarily to show that the common assumption that foreign-language articles primarily produce negative/neutral results should be questioned. In order to strengthen the rationale for why we see our study as an important contribution, we have added a current systematic review in the background section, which we had originally only mentioned in the discussion. Change in the text: In the background section, we have added the systematic review by Dobrescu et al. as a rationale for the present analysis. Please see changes in the text on page 4. “However, a recent systematic review by Dobrescu et al. [9] found, that restricting evidence syntheses of interventions to English-language publications is a feasible methodological shortcut for most medical topics. The aim of our analysis was to examine whether the results of the systematic review are transferable to the context of HTA reports.” 3. Another comment refers to different information on the search / publication period in the abstract and the methods section. Change to the text: We have now presented the publication period in the abstract. “We determined whether non-English publications were included in all HTA reports on non-drug interventions (published by IQWiG between 06/2007 to 08/2018) and on selected drug interventions.” 4. It is also noted that it remains unclear whether HTA reports on drugs were included in the analysis. Reply: Table 2 displays the number of included drug / non-drug HTA reports (3 HTA reports on drugs, 67 on non-drugs). No change to the text 5. The reviewer recommends providing a detailed list of HTA reports (drug and non-drug) that were screened for inclusion in the methods section. Reply: As stated in the methods, we included all HTA reports on non-drug interventions (published between 06/2007 and 08/2018) and selected 3 reports on drugs. We have added a list of analysed HTA reports to the supplement. Change to the text: We have added a list of included HTA reports https://doi.org/10.5281/zenodo.12642959 6. The reviewer would like us to justify why diagnostic HTA reports were included from the beginning if they are not to be re-analysed in the results. Reply: HTA reports on diagnostic questions can contain both results on the benefit of the diagnostic measure and on the diagnostic accuracy. Publications in other languages than English only contained information on diagnostic accuracy. However, in no case was a conclusion derived from this, but only the diagnostic accuracy was reported. Our article focuses on the impact of non-English publications on the conclusions of HTA reports, therefore a comparison of conclusions based on different data sets is not possible. No change to the text 7. The reviewer is concerned that the inclusion and exclusion criteria were applied unsystematically and asks for an explanation as to why 2 HTA reports are cited under the limitations. Reply: Based on your concerns, we have revised the extraction of the included studies in the HTA reports and have deleted the entries that did not fulfil the inclusion criteria (n = 11). Change in the text: The tables and figures in the text have been adjusted accordingly. The total number of references included is now 2328, the number of non-English publications 126. Thus, we have deleted the corresponding section in the limitations. 8. In addition, the reviewer notes that there are two examples of HTA reports listed in the results section and asks for a more detailed explanation of why these were selected. Reply: We wanted to illustrate a little more clearly to the reader why a non-English language article was categorized as A or B. As these are only examples, we see no risk of bias in the selection. The full list of HTA reports with non-English articles in categories A and B can be found in the supplement (https://doi.org/10.5281/zenodo.12642959). No change to the text 9. The reviewer points out that the measure of effect is not explained and that no statistical tests and no p-values are given. In her view, this makes it difficult to analyse whether or not the change, even if small, could have been significant for the results of the HTA report and the future recommendations. Reply: The measure of effect was defined as “change in the conclusion of the HTA report” which is composed of the conclusions of the individual endpoints. An overall conclusion was not provided by the underlying HTA reports. For most HTA reports (94%) no changes in the endpoints were identified. All changes were assessed in a yes/no format. This effect measure was therefore qualitative, even it was partly based on statistical analyses and re-analyses. The reviewer rightly points out that even small changes in a meta-analysis may or may not have been significant for the results of the HTA report. For this very reason, our analyses were not based on any statistical comparison between primary analysis and re-analysis. Rather, we examined whether the re-analysis led to a change in the conclusion of the endpoint and thus the HTA report, as this determines reimbursement and availability of health care services. No change to the text 10. According to the reviewer, it is contradictory first to conclude that searching the non-English literature is generally unnecessary, but then to explain that such searches may be worthwhile for particular interventions that are available only a specific region. Reply: We agree that a single HTA report on a country-specific intervention (N16-01) is a weak basis for recommending additional searches of non-English sources under specific circumstances. However, our text only describes this option as “reasonable” – without recommending it for all clinical interventions without clear indications that this is useful in the respective case. Change in the text: We amended the text in the discussion “This has less to do with a specific clinical question and more to do with market availability, which should of course be taken into account when preparing the systematic search.” 11. The reviewer asks to fix reference number 1 to reflect the title (Cochrane Handbook for Systematic Reviews of Interventions) of the handbook in which Chapter 4 is included. Change in the text: We amended the citation 1. Lefebvre C, Glanville J, Briscoe S et al. Chapter 4: Searching for and selecting studies [last updated September 2024]. In: Higgins JPT, Thomas J, Chandler J, Cumpston M, Li T, Page MJ, Welch VA (editors). Cochrane Handbook for Systematic Reviews of Interventions version 6.5 [online]. 2024 [Accessed: 20.01.2025]. URL: https://training.cochrane.org/handbook/current/chapter-04 . 12. Table 1 would benefit from including definitions for each of the categories. You can find a description within the text (page 4): “Category A: change in the conclusion for a particular endpoint; Category B: no change in the conclusion for a particular endpoint; Category C: the non-English publications do not contain relevant information for the HTA report (e.g. because the amount or quality of the data is too low) and are therefore classified as “not relevant”.” Change in the text: We have added the reference to Table 1 to the text “The effect of the exclusion of non-English publications on conclusions was categorized for each HTA report as follows (see Table 1)” 13. The reviewer asks to include in Figure 1 the distribution of the 38 reports as they are allocated to different categories. Reply: The requested information can be seen in Figure 1. As shown in the figure, 13 HTA reports contain only category C publications, 12 HTA reports with category B and 4 HTA reports to category A. The footnote also refers to the fact that an HTA report can contain non-English publications from different categories (A, B or C). The HTA report was assigned to the category that has the greatest impact on the change in conclusion. For example, the 12 HTA reports with category B also contain non-English publications from category C, but none from category A. No change to the text 14. The reviewer asks what the abbreviation EP in Table 5 means. Reply: This is the abbreviation for “endpoint“. An overview of the abbreviations used in the table is added below the table. No change to the text Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 03 Apr 2025 Elke Hausner , Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, 50670, Germany 03 Apr 2025 Author Response Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The first point raised is that ... Continue reading Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The first point raised is that the study is not suitable for making general statements about HTA reports. Reply: This point was also raised by Reviewer 1 (see above). We have taken your concerns and suggestions and adapted the text Change in the text: instead of referring to HTA reports in general, the text now refers specifically to the German HTA reports 2. It was also criticized that the authors draw from one single study (Wu et al., 2009) to justify why non-English studies should no longer be searched for in HTA reports. Reply: This is a misunderstanding, as Wu 2009 was cited primarily to show that the common assumption that foreign-language articles primarily produce negative/neutral results should be questioned. In order to strengthen the rationale for why we see our study as an important contribution, we have added a current systematic review in the background section, which we had originally only mentioned in the discussion. Change in the text: In the background section, we have added the systematic review by Dobrescu et al. as a rationale for the present analysis. Please see changes in the text on page 4. “However, a recent systematic review by Dobrescu et al. [9] found, that restricting evidence syntheses of interventions to English-language publications is a feasible methodological shortcut for most medical topics. The aim of our analysis was to examine whether the results of the systematic review are transferable to the context of HTA reports.” 3. Another comment refers to different information on the search / publication period in the abstract and the methods section. Change to the text: We have now presented the publication period in the abstract. “We determined whether non-English publications were included in all HTA reports on non-drug interventions (published by IQWiG between 06/2007 to 08/2018) and on selected drug interventions.” 4. It is also noted that it remains unclear whether HTA reports on drugs were included in the analysis. Reply: Table 2 displays the number of included drug / non-drug HTA reports (3 HTA reports on drugs, 67 on non-drugs). No change to the text 5. The reviewer recommends providing a detailed list of HTA reports (drug and non-drug) that were screened for inclusion in the methods section. Reply: As stated in the methods, we included all HTA reports on non-drug interventions (published between 06/2007 and 08/2018) and selected 3 reports on drugs. We have added a list of analysed HTA reports to the supplement. Change to the text: We have added a list of included HTA reports https://doi.org/10.5281/zenodo.12642959 6. The reviewer would like us to justify why diagnostic HTA reports were included from the beginning if they are not to be re-analysed in the results. Reply: HTA reports on diagnostic questions can contain both results on the benefit of the diagnostic measure and on the diagnostic accuracy. Publications in other languages than English only contained information on diagnostic accuracy. However, in no case was a conclusion derived from this, but only the diagnostic accuracy was reported. Our article focuses on the impact of non-English publications on the conclusions of HTA reports, therefore a comparison of conclusions based on different data sets is not possible. No change to the text 7. The reviewer is concerned that the inclusion and exclusion criteria were applied unsystematically and asks for an explanation as to why 2 HTA reports are cited under the limitations. Reply: Based on your concerns, we have revised the extraction of the included studies in the HTA reports and have deleted the entries that did not fulfil the inclusion criteria (n = 11). Change in the text: The tables and figures in the text have been adjusted accordingly. The total number of references included is now 2328, the number of non-English publications 126. Thus, we have deleted the corresponding section in the limitations. 8. In addition, the reviewer notes that there are two examples of HTA reports listed in the results section and asks for a more detailed explanation of why these were selected. Reply: We wanted to illustrate a little more clearly to the reader why a non-English language article was categorized as A or B. As these are only examples, we see no risk of bias in the selection. The full list of HTA reports with non-English articles in categories A and B can be found in the supplement (https://doi.org/10.5281/zenodo.12642959). No change to the text 9. The reviewer points out that the measure of effect is not explained and that no statistical tests and no p-values are given. In her view, this makes it difficult to analyse whether or not the change, even if small, could have been significant for the results of the HTA report and the future recommendations. Reply: The measure of effect was defined as “change in the conclusion of the HTA report” which is composed of the conclusions of the individual endpoints. An overall conclusion was not provided by the underlying HTA reports. For most HTA reports (94%) no changes in the endpoints were identified. All changes were assessed in a yes/no format. This effect measure was therefore qualitative, even it was partly based on statistical analyses and re-analyses. The reviewer rightly points out that even small changes in a meta-analysis may or may not have been significant for the results of the HTA report. For this very reason, our analyses were not based on any statistical comparison between primary analysis and re-analysis. Rather, we examined whether the re-analysis led to a change in the conclusion of the endpoint and thus the HTA report, as this determines reimbursement and availability of health care services. No change to the text 10. According to the reviewer, it is contradictory first to conclude that searching the non-English literature is generally unnecessary, but then to explain that such searches may be worthwhile for particular interventions that are available only a specific region. Reply: We agree that a single HTA report on a country-specific intervention (N16-01) is a weak basis for recommending additional searches of non-English sources under specific circumstances. However, our text only describes this option as “reasonable” – without recommending it for all clinical interventions without clear indications that this is useful in the respective case. Change in the text: We amended the text in the discussion “This has less to do with a specific clinical question and more to do with market availability, which should of course be taken into account when preparing the systematic search.” 11. The reviewer asks to fix reference number 1 to reflect the title (Cochrane Handbook for Systematic Reviews of Interventions) of the handbook in which Chapter 4 is included. Change in the text: We amended the citation 1. Lefebvre C, Glanville J, Briscoe S et al. Chapter 4: Searching for and selecting studies [last updated September 2024]. In: Higgins JPT, Thomas J, Chandler J, Cumpston M, Li T, Page MJ, Welch VA (editors). Cochrane Handbook for Systematic Reviews of Interventions version 6.5 [online]. 2024 [Accessed: 20.01.2025]. URL: https://training.cochrane.org/handbook/current/chapter-04 . 12. Table 1 would benefit from including definitions for each of the categories. You can find a description within the text (page 4): “Category A: change in the conclusion for a particular endpoint; Category B: no change in the conclusion for a particular endpoint; Category C: the non-English publications do not contain relevant information for the HTA report (e.g. because the amount or quality of the data is too low) and are therefore classified as “not relevant”.” Change in the text: We have added the reference to Table 1 to the text “The effect of the exclusion of non-English publications on conclusions was categorized for each HTA report as follows (see Table 1)” 13. The reviewer asks to include in Figure 1 the distribution of the 38 reports as they are allocated to different categories. Reply: The requested information can be seen in Figure 1. As shown in the figure, 13 HTA reports contain only category C publications, 12 HTA reports with category B and 4 HTA reports to category A. The footnote also refers to the fact that an HTA report can contain non-English publications from different categories (A, B or C). The HTA report was assigned to the category that has the greatest impact on the change in conclusion. For example, the 12 HTA reports with category B also contain non-English publications from category C, but none from category A. No change to the text 14. The reviewer asks what the abbreviation EP in Table 5 means. Reply: This is the abbreviation for “endpoint“. An overview of the abbreviations used in the table is added below the table. No change to the text Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The first point raised is that the study is not suitable for making general statements about HTA reports. Reply: This point was also raised by Reviewer 1 (see above). We have taken your concerns and suggestions and adapted the text Change in the text: instead of referring to HTA reports in general, the text now refers specifically to the German HTA reports 2. It was also criticized that the authors draw from one single study (Wu et al., 2009) to justify why non-English studies should no longer be searched for in HTA reports. Reply: This is a misunderstanding, as Wu 2009 was cited primarily to show that the common assumption that foreign-language articles primarily produce negative/neutral results should be questioned. In order to strengthen the rationale for why we see our study as an important contribution, we have added a current systematic review in the background section, which we had originally only mentioned in the discussion. Change in the text: In the background section, we have added the systematic review by Dobrescu et al. as a rationale for the present analysis. Please see changes in the text on page 4. “However, a recent systematic review by Dobrescu et al. [9] found, that restricting evidence syntheses of interventions to English-language publications is a feasible methodological shortcut for most medical topics. The aim of our analysis was to examine whether the results of the systematic review are transferable to the context of HTA reports.” 3. Another comment refers to different information on the search / publication period in the abstract and the methods section. Change to the text: We have now presented the publication period in the abstract. “We determined whether non-English publications were included in all HTA reports on non-drug interventions (published by IQWiG between 06/2007 to 08/2018) and on selected drug interventions.” 4. It is also noted that it remains unclear whether HTA reports on drugs were included in the analysis. Reply: Table 2 displays the number of included drug / non-drug HTA reports (3 HTA reports on drugs, 67 on non-drugs). No change to the text 5. The reviewer recommends providing a detailed list of HTA reports (drug and non-drug) that were screened for inclusion in the methods section. Reply: As stated in the methods, we included all HTA reports on non-drug interventions (published between 06/2007 and 08/2018) and selected 3 reports on drugs. We have added a list of analysed HTA reports to the supplement. Change to the text: We have added a list of included HTA reports https://doi.org/10.5281/zenodo.12642959 6. The reviewer would like us to justify why diagnostic HTA reports were included from the beginning if they are not to be re-analysed in the results. Reply: HTA reports on diagnostic questions can contain both results on the benefit of the diagnostic measure and on the diagnostic accuracy. Publications in other languages than English only contained information on diagnostic accuracy. However, in no case was a conclusion derived from this, but only the diagnostic accuracy was reported. Our article focuses on the impact of non-English publications on the conclusions of HTA reports, therefore a comparison of conclusions based on different data sets is not possible. No change to the text 7. The reviewer is concerned that the inclusion and exclusion criteria were applied unsystematically and asks for an explanation as to why 2 HTA reports are cited under the limitations. Reply: Based on your concerns, we have revised the extraction of the included studies in the HTA reports and have deleted the entries that did not fulfil the inclusion criteria (n = 11). Change in the text: The tables and figures in the text have been adjusted accordingly. The total number of references included is now 2328, the number of non-English publications 126. Thus, we have deleted the corresponding section in the limitations. 8. In addition, the reviewer notes that there are two examples of HTA reports listed in the results section and asks for a more detailed explanation of why these were selected. Reply: We wanted to illustrate a little more clearly to the reader why a non-English language article was categorized as A or B. As these are only examples, we see no risk of bias in the selection. The full list of HTA reports with non-English articles in categories A and B can be found in the supplement (https://doi.org/10.5281/zenodo.12642959). No change to the text 9. The reviewer points out that the measure of effect is not explained and that no statistical tests and no p-values are given. In her view, this makes it difficult to analyse whether or not the change, even if small, could have been significant for the results of the HTA report and the future recommendations. Reply: The measure of effect was defined as “change in the conclusion of the HTA report” which is composed of the conclusions of the individual endpoints. An overall conclusion was not provided by the underlying HTA reports. For most HTA reports (94%) no changes in the endpoints were identified. All changes were assessed in a yes/no format. This effect measure was therefore qualitative, even it was partly based on statistical analyses and re-analyses. The reviewer rightly points out that even small changes in a meta-analysis may or may not have been significant for the results of the HTA report. For this very reason, our analyses were not based on any statistical comparison between primary analysis and re-analysis. Rather, we examined whether the re-analysis led to a change in the conclusion of the endpoint and thus the HTA report, as this determines reimbursement and availability of health care services. No change to the text 10. According to the reviewer, it is contradictory first to conclude that searching the non-English literature is generally unnecessary, but then to explain that such searches may be worthwhile for particular interventions that are available only a specific region. Reply: We agree that a single HTA report on a country-specific intervention (N16-01) is a weak basis for recommending additional searches of non-English sources under specific circumstances. However, our text only describes this option as “reasonable” – without recommending it for all clinical interventions without clear indications that this is useful in the respective case. Change in the text: We amended the text in the discussion “This has less to do with a specific clinical question and more to do with market availability, which should of course be taken into account when preparing the systematic search.” 11. The reviewer asks to fix reference number 1 to reflect the title (Cochrane Handbook for Systematic Reviews of Interventions) of the handbook in which Chapter 4 is included. Change in the text: We amended the citation 1. Lefebvre C, Glanville J, Briscoe S et al. Chapter 4: Searching for and selecting studies [last updated September 2024]. In: Higgins JPT, Thomas J, Chandler J, Cumpston M, Li T, Page MJ, Welch VA (editors). Cochrane Handbook for Systematic Reviews of Interventions version 6.5 [online]. 2024 [Accessed: 20.01.2025]. URL: https://training.cochrane.org/handbook/current/chapter-04 . 12. Table 1 would benefit from including definitions for each of the categories. You can find a description within the text (page 4): “Category A: change in the conclusion for a particular endpoint; Category B: no change in the conclusion for a particular endpoint; Category C: the non-English publications do not contain relevant information for the HTA report (e.g. because the amount or quality of the data is too low) and are therefore classified as “not relevant”.” Change in the text: We have added the reference to Table 1 to the text “The effect of the exclusion of non-English publications on conclusions was categorized for each HTA report as follows (see Table 1)” 13. The reviewer asks to include in Figure 1 the distribution of the 38 reports as they are allocated to different categories. Reply: The requested information can be seen in Figure 1. As shown in the figure, 13 HTA reports contain only category C publications, 12 HTA reports with category B and 4 HTA reports to category A. The footnote also refers to the fact that an HTA report can contain non-English publications from different categories (A, B or C). The HTA report was assigned to the category that has the greatest impact on the change in conclusion. For example, the 12 HTA reports with category B also contain non-English publications from category C, but none from category A. No change to the text 14. The reviewer asks what the abbreviation EP in Table 5 means. Reply: This is the abbreviation for “endpoint“. An overview of the abbreviations used in the table is added below the table. No change to the text Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Sundell K. Reviewer Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.166008.r332664 ) The direct URL for this report is: https://f1000research.com/articles/13-1134/v1#referee-response-332664 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 18 Nov 2024 Knut Sundell , Department of social work and criminology, University of Gävle, Gävle, Sweden Not Approved VIEWS 0 https://doi.org/10.5256/f1000research.166008.r332664 Different aspects of publication bias (positive outcome bias, time-lag bias, and place-of-publication bias) constitute serious threats to the validity of research synthesis. If research syntheses are based on a biased selection of the available research, there is an increased risk ... Continue reading READ ALL Different aspects of publication bias (positive outcome bias, time-lag bias, and place-of-publication bias) constitute serious threats to the validity of research synthesis. If research syntheses are based on a biased selection of the available research, there is an increased risk of producing misleading results. Awareness of these types of bias is important for reading systematic reviews and incorporating their findings into clinical practice and decision making. The authors have screened 70 HTA reports from the German HTA agency, the Institute for Quality and Efficiency in Health Care (IQWiG) searching for reports that include non-English articles. If at least one non-English publication was included, the authors assessed whether the exclusion of non-English publications changed the conclusion. Of 70 HTA reports, 38 (54%) included non-English publications. The exclusion of non-English studies has only a minor effect on the conclusions of HTA reports The authors conclude that, (1) in general, studies published in non-English languages have little influence on the conclusions of HTA reports , and (2) For the vast majority of topics, a language restriction to English seems justified. These conclusions seem premature because of possible selection bias. We don’t know if the non-English articles in these reports constitute all relevant articles or a skewed sample (e.g., because of positive outcome biased, ). To be able to conclude that non-English articles have little influence on the report conclusions, the articles need to represent all relevant articles (i.e., no publication bias) which will require an extensive search for grey literature, including non-peer reviewed articles, the types of articles that the authors have excluded (i.e., conferences abstracts, evidence syntheses, clinical study reports, or registry entries). The best we can say about the results is that non-English articles that are included in IQWiG’s reports may have had scant influence on the HTA-reports’ conclusions. This may be a thought-provoking result but hardly one to base policy on including or excluding non-English articles. Furthermore, I suggest that the authors discuss the types of bias and their relative importance for the design of the study. The exclusion of non-English publications might be motivated, but the results might still be biased because of positive outcome bias, time-lag bias, place-of-publication bias or other types of bias? There is no motivation for only including the German HTA organization. What are the pros and cons of this choice? Is the work clearly and accurately presented and does it cite the current literature? Yes Is the study design appropriate and is the work technically sound? No Are sufficient details of methods and analysis provided to allow replication by others? Yes If applicable, is the statistical analysis and its interpretation appropriate? I cannot comment. A qualified statistician is required. Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? No Competing Interests: No competing interests were disclosed. I confirm that I have read this submission and believe that I have an appropriate level of expertise to state that I do not consider it to be of an acceptable scientific standard, for reasons outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Sundell K. Reviewer Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.166008.r332664 ) The direct URL for this report is: https://f1000research.com/articles/13-1134/v1#referee-response-332664 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 03 Apr 2025 Elke Hausner , Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, 50670, Germany 03 Apr 2025 Author Response Thank you very much for taking the time to review our paper. We address your comments below and believe that our replies resolve your concerns. 1. In your peer ... Continue reading Thank you very much for taking the time to review our paper. We address your comments below and believe that our replies resolve your concerns. 1. In your peer review you point out that “our conclusions seem premature because of possible selection bias“. It is unclear to us whether this refers to the selection of HTA reports or to the selection of primary studies into these HTA reports. Taking your argument in the first sense, we agree that our conclusions primarily refer to German HTA reports. It may well be that, due to different legal frameworks, HTA reports in other countries address other medical topics or address the same topics using other methods of information retrieval. To emphasize that these are the results of a German HTA agency, we have adapted the text accordingly. Change in the text: instead of referring to HTA reports in general, the text now refers specifically to German HTA reports 2. The issue of selection bias could also be understood in the sense of publication bias. The reviewer suspects that no difference between inclusion and exclusion of non-English sources was found, because relevant literature was not found by any of search approaches. According to the reviewer, this requires “an extensive search for grey literature, including non-peer reviewed articles, the types of articles that the authors have excluded (i.e., conferences abstracts, evidence syntheses, clinical study reports, or registry entries).“ As shown in Table 2, the information retrieval for IQWiG reports includes a number of databases (with a range of 3-23 per report). In addition, study registries were searched, reference lists screened, requests to authors were made and - for all reports but rapid reports - public hearings took place. Therefore, we assume the there is no issue with publication bias. Furthermore, the remaining risk of publication bias would be the nearly the same for both approaches, i.e. with or without non-English sources, because most conference abstracts, grey literature, and study reports will either be written in English or will be unusable for HTA purposes due to a lack of key study information. No change to the text 3. Furthermore, you suggest that the authors discuss the types of bias and their relative importance for the design of the study. From our point of view, it is highly likely that all the forms of bias mentioned had no influence on the changes to the conclusions. Especially if non-significant results tend to end up in non-English journals, this would have resulted in apparent differences between those HTA reports that included non-English sources and those that did not. No change to the text Finally, you state that there is no motivation to include only reports prepared by the German HTA organisation. We disagree, because it requires detailed data to perform the reassessments using other language selection criteria. It would have required multinational cooperation and adequate funding to scale up the study to several HTA agencies and several languages. We also see no point in expanding the sample. Especially since, as mentioned in our discussion, major HTA agencies such as AHRQ, HAS and NICE also use some sort of language restriction in their reports. No change to the text Thank you very much for taking the time to review our paper. We address your comments below and believe that our replies resolve your concerns. 1. In your peer review you point out that “our conclusions seem premature because of possible selection bias“. It is unclear to us whether this refers to the selection of HTA reports or to the selection of primary studies into these HTA reports. Taking your argument in the first sense, we agree that our conclusions primarily refer to German HTA reports. It may well be that, due to different legal frameworks, HTA reports in other countries address other medical topics or address the same topics using other methods of information retrieval. To emphasize that these are the results of a German HTA agency, we have adapted the text accordingly. Change in the text: instead of referring to HTA reports in general, the text now refers specifically to German HTA reports 2. The issue of selection bias could also be understood in the sense of publication bias. The reviewer suspects that no difference between inclusion and exclusion of non-English sources was found, because relevant literature was not found by any of search approaches. According to the reviewer, this requires “an extensive search for grey literature, including non-peer reviewed articles, the types of articles that the authors have excluded (i.e., conferences abstracts, evidence syntheses, clinical study reports, or registry entries).“ As shown in Table 2, the information retrieval for IQWiG reports includes a number of databases (with a range of 3-23 per report). In addition, study registries were searched, reference lists screened, requests to authors were made and - for all reports but rapid reports - public hearings took place. Therefore, we assume the there is no issue with publication bias. Furthermore, the remaining risk of publication bias would be the nearly the same for both approaches, i.e. with or without non-English sources, because most conference abstracts, grey literature, and study reports will either be written in English or will be unusable for HTA purposes due to a lack of key study information. No change to the text 3. Furthermore, you suggest that the authors discuss the types of bias and their relative importance for the design of the study. From our point of view, it is highly likely that all the forms of bias mentioned had no influence on the changes to the conclusions. Especially if non-significant results tend to end up in non-English journals, this would have resulted in apparent differences between those HTA reports that included non-English sources and those that did not. No change to the text Finally, you state that there is no motivation to include only reports prepared by the German HTA organisation. We disagree, because it requires detailed data to perform the reassessments using other language selection criteria. It would have required multinational cooperation and adequate funding to scale up the study to several HTA agencies and several languages. We also see no point in expanding the sample. Especially since, as mentioned in our discussion, major HTA agencies such as AHRQ, HAS and NICE also use some sort of language restriction in their reports. No change to the text Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 03 Apr 2025 Elke Hausner , Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, 50670, Germany 03 Apr 2025 Author Response Thank you very much for taking the time to review our paper. We address your comments below and believe that our replies resolve your concerns. 1. In your peer ... Continue reading Thank you very much for taking the time to review our paper. We address your comments below and believe that our replies resolve your concerns. 1. In your peer review you point out that “our conclusions seem premature because of possible selection bias“. It is unclear to us whether this refers to the selection of HTA reports or to the selection of primary studies into these HTA reports. Taking your argument in the first sense, we agree that our conclusions primarily refer to German HTA reports. It may well be that, due to different legal frameworks, HTA reports in other countries address other medical topics or address the same topics using other methods of information retrieval. To emphasize that these are the results of a German HTA agency, we have adapted the text accordingly. Change in the text: instead of referring to HTA reports in general, the text now refers specifically to German HTA reports 2. The issue of selection bias could also be understood in the sense of publication bias. The reviewer suspects that no difference between inclusion and exclusion of non-English sources was found, because relevant literature was not found by any of search approaches. According to the reviewer, this requires “an extensive search for grey literature, including non-peer reviewed articles, the types of articles that the authors have excluded (i.e., conferences abstracts, evidence syntheses, clinical study reports, or registry entries).“ As shown in Table 2, the information retrieval for IQWiG reports includes a number of databases (with a range of 3-23 per report). In addition, study registries were searched, reference lists screened, requests to authors were made and - for all reports but rapid reports - public hearings took place. Therefore, we assume the there is no issue with publication bias. Furthermore, the remaining risk of publication bias would be the nearly the same for both approaches, i.e. with or without non-English sources, because most conference abstracts, grey literature, and study reports will either be written in English or will be unusable for HTA purposes due to a lack of key study information. No change to the text 3. Furthermore, you suggest that the authors discuss the types of bias and their relative importance for the design of the study. From our point of view, it is highly likely that all the forms of bias mentioned had no influence on the changes to the conclusions. Especially if non-significant results tend to end up in non-English journals, this would have resulted in apparent differences between those HTA reports that included non-English sources and those that did not. No change to the text Finally, you state that there is no motivation to include only reports prepared by the German HTA organisation. We disagree, because it requires detailed data to perform the reassessments using other language selection criteria. It would have required multinational cooperation and adequate funding to scale up the study to several HTA agencies and several languages. We also see no point in expanding the sample. Especially since, as mentioned in our discussion, major HTA agencies such as AHRQ, HAS and NICE also use some sort of language restriction in their reports. No change to the text Thank you very much for taking the time to review our paper. We address your comments below and believe that our replies resolve your concerns. 1. In your peer review you point out that “our conclusions seem premature because of possible selection bias“. It is unclear to us whether this refers to the selection of HTA reports or to the selection of primary studies into these HTA reports. Taking your argument in the first sense, we agree that our conclusions primarily refer to German HTA reports. It may well be that, due to different legal frameworks, HTA reports in other countries address other medical topics or address the same topics using other methods of information retrieval. To emphasize that these are the results of a German HTA agency, we have adapted the text accordingly. Change in the text: instead of referring to HTA reports in general, the text now refers specifically to German HTA reports 2. The issue of selection bias could also be understood in the sense of publication bias. The reviewer suspects that no difference between inclusion and exclusion of non-English sources was found, because relevant literature was not found by any of search approaches. According to the reviewer, this requires “an extensive search for grey literature, including non-peer reviewed articles, the types of articles that the authors have excluded (i.e., conferences abstracts, evidence syntheses, clinical study reports, or registry entries).“ As shown in Table 2, the information retrieval for IQWiG reports includes a number of databases (with a range of 3-23 per report). In addition, study registries were searched, reference lists screened, requests to authors were made and - for all reports but rapid reports - public hearings took place. Therefore, we assume the there is no issue with publication bias. Furthermore, the remaining risk of publication bias would be the nearly the same for both approaches, i.e. with or without non-English sources, because most conference abstracts, grey literature, and study reports will either be written in English or will be unusable for HTA purposes due to a lack of key study information. No change to the text 3. Furthermore, you suggest that the authors discuss the types of bias and their relative importance for the design of the study. From our point of view, it is highly likely that all the forms of bias mentioned had no influence on the changes to the conclusions. Especially if non-significant results tend to end up in non-English journals, this would have resulted in apparent differences between those HTA reports that included non-English sources and those that did not. No change to the text Finally, you state that there is no motivation to include only reports prepared by the German HTA organisation. We disagree, because it requires detailed data to perform the reassessments using other language selection criteria. It would have required multinational cooperation and adequate funding to scale up the study to several HTA agencies and several languages. We also see no point in expanding the sample. Especially since, as mentioned in our discussion, major HTA agencies such as AHRQ, HAS and NICE also use some sort of language restriction in their reports. No change to the text Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Comments on this article Comments (0) Version 3 VERSION 3 PUBLISHED 04 Oct 2024 ADD YOUR COMMENT Comment keyboard_arrow_left keyboard_arrow_right Open Peer Review Reviewer Status info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Reviewer Reports Invited Reviewers 1 2 3 4 5 Version 3 (revision) 07 Jul 25 read Version 2 (revision) 03 Apr 25 read read Version 1 04 Oct 24 read read read Knut Sundell , University of Gävle, Gävle, Sweden Sonia Garcia Gonzalez-Moral , Newcastle University, Newcastle upon Tyne, UK Tarquin Mittermayr , Austrian Institute for Health Technology Assessments, Wien, Austria Steve McDonald , Monash University, Clayton, Australia Liz Dennett , University of Alberta, Edmonton, Canada Comments on this article All Comments (0) Add a comment Sign up for content alerts Sign Up You are now signed up to receive this alert Browse by related subjects keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 Mittermayr T. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 18 Aug 2025 | for Version 3 Tarquin Mittermayr , Austrian Institute for Health Technology Assessments, Wien, Austria 0 Views copyright © 2025 Mittermayr T. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Thank you for providing me with another opportunity to review the revised versions of this paper. I am happy with the way the authors have taken into consideration the reviewers' comments and the changes they have made accordingly. Furthermore, they have provided comprehensible arguments where they did not see a reason for changing the text. I agree with the convincing comment of another reviewer that this paper provides sound arguments for the readers to make their own assessment of the impact that limiting searches to contain only English-language publications may have. Competing Interests No competing interests were disclosed. Reviewer Expertise Conducting systematic searches in the field of Evidence Based Medicine and Health Technology Assessment I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) Mittermayr T. Peer Review Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.182757.r397138) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1134/v3#referee-response-397138 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 Dennett L. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 24 Apr 2025 | for Version 2 Liz Dennett , University of Alberta, Edmonton, Alberta, Canada 0 Views copyright © 2025 Dennett L. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This review reviewed 70 Health Technology Assessment reports published at the Institute for Quality and Efficiency in Health Care in Germany between 2006 and 2018. They chose to mainly include (96%) non-drug HTA's because they felt those would be more likely to have non-english reports than the drug HTAs. Fifty four percent of these HTAs had english language studies included, but in only 4 HTA reports did the english language studies change results for any of the studies' endpoints. In the 4 HTA reports, 10 studies changed the conclusions on 13 endpoints. The changes were not dramatic ones but in the case of 5 endpoints, no conclusion was possible without the non-english studies as they were the only studies containing information on that endpoint. From these results the authors conclude: "In general, studies only published in non-English languages have little influence on the conclusions of German HTA reports. For the vast majority of topics, a language restriction to English seems justified". They continue to recommend searching for non-english studies in special cases (e.g. a device is only marketed in one country). Overall, I believe this to be a well conducted study with transparent methods and it presents similar findings to a number of other published articles (all of the ones I am aware of are included in the Dobrescu, et al., 2021 (Ref 1) paper cited in this study. I don't really have any comments that need to be addressed by the authors, but I can't help putting only a "Partly" for the question "Are the conclusions drawn adequately supported by the results? At the end of the day, you can't prove a negative, and it is impossible to definitively say, "It is completely fine to ignore non-english studies." This study does, however, add to the growing evidence that it is unlikely to matter that much. It is then up to individual teams if they are comfortable with the albeit small risk of losing important information for the feasibility trade off. When we train people to do evidence synthesis, we always remind students several times that the goal of every search is to be as comprehensive as possible. It is hard to go against that initial training, even when you come to appreciate that true comprehensiveness is rarely (if ever) possible. I also think it is important to keep in mind the caveat (that these authors include) that China's growing influence in scientific discourse, which means that we can't consider this topic completely settled. Is the work clearly and accurately presented and does it cite the current literature? Yes Is the study design appropriate and is the work technically sound? Yes Are sufficient details of methods and analysis provided to allow replication by others? Yes If applicable, is the statistical analysis and its interpretation appropriate? Yes Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly References 1. Dobrescu AI, Nussbaumer-Streit B, Klerings I, Wagner G, et al.: Restricting evidence syntheses of interventions to English-language publications is a viable methodological shortcut for most medical topics: a systematic review. J Clin Epidemiol . 2021; 137 : 209-217 PubMed Abstract | Publisher Full Text Competing Interests No competing interests were disclosed. Reviewer Expertise Expert searching I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (1) Author Response 23 Aug 2025 Elke Hausner, Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, Germany Thank you for taking the time to review our article. We agree that it is impossible to definitively say, “It is completely fine to ignore non-English studies”. This should be decided on a case-by-case basis, because, as we state in our article “Studies published in non-English languages may be useful in exceptional cases, for example when an intervention is only available in certain countries.” View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Dennett L. Peer Review Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.179748.r376060) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1134/v2#referee-response-376060 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 McDonald S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 21 Apr 2025 | for Version 2 Steve McDonald , School of Public Health and Preventive Medicine, Monash University, Clayton, Victoria, Australia 0 Views copyright © 2025 McDonald S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Thank you for the opportunity to review this revised version. This paper is a useful contribution to the evidence base of how language bias may affect the results of evidence synthesis. The authors provide a comprehensive case study of the impact of excluding non-English language publications on the conclusions of 70 HTA reports produced by IQWIG from 2007 to 2018. Previous peer reviewers raised several valid concerns and suggested amendments, notably that the conclusions of the study should be limited to the context of German HTA reports. I feel the authors have adequately addressed the concerns raised. The revised paper is well-written and clearly presented, with sufficient information provided to allow readers to make their own assessment of the value of restricting to the inclusion of English-language publications. The authors' conclusion that non-English publications "have little influence" or "minor effect" is supported by the data. Further, the authors provide a useful caveat to when searching for (and including) non-English publications should be considered. One brief comment: the analysis focuses on journal publications (which is reasonable). It is not clear if (for these 70 HTA reports) non-journal publications in languages other than English were included, and thus whether they had any impact on the the conclusions. I guess the question is to what extent the 'unnecessary' searching of non-English publications extends to non-journal article formats (conference abstracts, register entries). One minor comment: the flow diagram (second row) has 128 non-English publications, but in the Abstract and Results this is 126. Is the work clearly and accurately presented and does it cite the current literature? Yes Is the study design appropriate and is the work technically sound? Yes Are sufficient details of methods and analysis provided to allow replication by others? Yes If applicable, is the statistical analysis and its interpretation appropriate? Yes Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes References 1. Hausner E, Sturtz S, Molnar S, Schell L, et al.: Searching for non-English literature may be unnecessary for German HTA Reports. F1000Research . 2025; 13 . Publisher Full Text Competing Interests No competing interests were disclosed. Reviewer Expertise Information specialist with expertise in evidence synthesis. I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (1) Author Response 11 Sep 2025 Elke Hausner, Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, Germany Thank you for taking the time to review our article. Unfortunately, we are unable to answer your question as to whether our conclusions can also be applied to other forms of publication such as conference abstracts, as this was not part of our original research question. Your reference to the numbers in the flowchart is correct. The following data must be adjusted in the flowchart: 2,328 instead of 2,339 publications, and 126 instead of 128 non-English publications. We will update the flowchart accordingly. View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern McDonald S. Peer Review Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.179748.r376055) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1134/v2#referee-response-376055 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Mittermayr T. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 28 Dec 2024 | for Version 1 Tarquin Mittermayr , Austrian Institute for Health Technology Assessments, Wien, Austria 0 Views copyright © 2024 Mittermayr T. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions I am happy to be invited to write a review on this relevant topic. Whilst the method for assessing the impact of excluding non-english literature on the conclusion of HTA reports is generally explained in a clear and comprehensible manner, Hausner et al fall short of explaining or referencing the internal protocol mentioned in the methods section. It would have also been helpful to either provide an explanation already in this section or refer to the appropriate page (e.g. “please find details on page 8”) in the limitations section as to why no publications after 2018 were considered for this assessment. Table 1 would benefit from clearer and more concise information: The same reasons for classifying low quality studies as category B in the classification table (“Amount and quality of the evidence…”) are listed in the explanation of the categories below the table under category C, thus giving room for ambiguous interpretations. Table 2 contains the total number of HTA reports included as well as the number per report. It is unclear if the number given for the latter (22.5) is an average, median or a percentage. In consideration of whether providing this number is meaningful, stating just the range of included publications per report might suffice and be more explanatory. The discussion section contains the confusing remark “We did not analyse whether the non-English study publications were of lower quality than the English language ones”. This appears to be contradictory to the explanations given for the classifications in Table 1. Should the contradictory nature of this statement be due to an imprecision in terminology (quality of evidence vs. quality of study publications), this study by Hausner et al would have benefited from performing a qualitative analyses of the publications themselves (in addition to analysing the containing evidence). In an article which argues that excluding non-English literature may have little impact on the overall conclusion of an HTA report, the example from Report N16-03 given in the Results section, where the exclusion of a non-English article led to a change in the conclusion, might sit uncomfortably with systematic evidence based medicine professionals. As graphical this example may be in showing the effect of this exclusion, it also appears to undermine the core argument of this article. A revision by the authors of this publication should focus on the coherence of their argumentation on the one hand, and on supporting their methods with more concise examples (e.g. Tables and Figures). Is the work clearly and accurately presented and does it cite the current literature? Partly Is the study design appropriate and is the work technically sound? Partly Are sufficient details of methods and analysis provided to allow replication by others? Partly If applicable, is the statistical analysis and its interpretation appropriate? I cannot comment. A qualified statistician is required. Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly Competing Interests No competing interests were disclosed. Reviewer Expertise Conducting systematic searches in my position as an Information Specialist I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (1) Author Response 03 Apr 2025 Elke Hausner, Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, 50670, Germany Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The reviewer indicates that it would be helpful to explain or reference the internal protocol mentioned in the methods section. Reply: We have exchanged the term “protocol” with “project outline”. This document is a German-language 1-page text that provides an overview of the project, not a complete step-by-step protocol. It was not published and we therefore cannot reference it. Change in the text: We have amended the text “Following an internal project outline, we screened HTA reports (all reports on non-drug interventions published by IQWiG between 06/2007 and 08/2018 and randomly selected reports on drugs published between 09/2011 and 11/2016) for the inclusion of non-English publications.” 2. The reviewer states that it would have been helpful to either provide an explanation already in this section or refer to the appropriate page (e.g. “please find details on page 8”) in the limitations section as to why no publications after 2018 were considered for this assessment. Change in the text: We have amended the text ”For the present article, it was therefore not meaningful to update the pool of HTA reports by including HTA reports published by IQWiG after October 2019 (see the methods section for details).“ 3. The reviewer states that table 1 would benefit from clearer and more concise information: The same reasons for classifying low quality studies as category B in the classification table (“Amount and quality of the evidence…”) are listed in the explanation of the categories below the table under category C, thus giving room for ambiguous interpretations. Reply: We disagree that there is ambiguity, because in category B the reason is “Amount and quality of the evidence of all studies”, whereas in category C the reason is “Quality of study too low to change conclusion”. The first reason refers to the evidence in its totality, the second reason addresses the quality of only the non-English study. No change to the text 4. The reviewer points out that in Table 2 it is unclear whether the number of included publications per report (22.5) is an average, median or percentage. Reply: Numbers and percentages are indicated by round brackets, while median and ranges are reported with square brackets. The heading indicates that 22.5 [1; 156] is the median, with the range given. We believe that it is reasonable to display median and range, rather than just reporting the range. No change to the text 5. The reviewer refers to the different information from Table 1 and the section ‘Limitations’ regarding the examination of study quality. In addition, the reviewer states that the study would have benefited from performing a qualitative analysis for publications. Reply: We are uncertain what type of “qualitative analysis” could be useful to be performed in addition to analysing the available evidence. As HTA work is evidence-based, the methods and the results of the primary studies are the sole data sources. Unfortunately, a qualitative analysis was not possible due to the limited time available. However, this would be very interesting, as there are already several publications [1-3] on this topic that question the quality of non-English studies. Change in the text : We have added the assessment of the risk of bias for the non-English studies in the text. “However, all 10 non-English studies that changed the conclusions of an HTA report (see Table 5) were classified as highly biased in the underlying HTA reports.” 6. The reviewer points out that the example of N16-03 listed under ‘Results’ could lead to discomfort among experts in evidence-based medicine since it seems to undermine the core argument of this article. Reply: Admittedly, the recommendation not to search for non-English literature would have been easier if the decision to include or exclude such sources had not affected the conclusion of any HTA report. As 4 reports (6%) experienced a change in the conclusions, our conclusion is already somewhat toned down (“seems justified”). We agree with the reviewer that the decision is not straightforward. Some experts will continue to search non-English sources, because they consider the risk of overlooking a study more important than the resources required for searching for non-English publications. In our view, however, the main conclusion is correctly inferred from our results. No change to the text View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Mittermayr T. Peer Review Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.166008.r344992) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1134/v1#referee-response-344992 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Gonzalez-Moral S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 14 Dec 2024 | for Version 1 Sonia Garcia Gonzalez-Moral , Newcastle University, Newcastle upon Tyne, England, UK 0 Views copyright © 2024 Gonzalez-Moral S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions I welcome the opportunity to comment on this paper. With this study the authors have challenged the status quo of current systematic literature review guidelines such as those from Cochrane and health technology assessment methods recommendations for undertaking literature searches such as those recommended by EUneHTA. Such bold move feels justified if one understand the intensity of resource and capacity involved in HTA and so, while I support the rationale for this study and think that this is a good first attempt to breach this topic, in general I do not think that this study boasts the sufficient methodological rigor nor a big enough sample size to draw conclusions that could be generalized to HTA. To start with, the title of this manuscript ‘Searching for non-English literature may be unnecessary for HTA reports’ suggests the opposite of what the authors conclude ‘ At IQWiG, the analysis presented here led to the restriction of information retrieval in HTA reports to German- and English-language literature from September 2018 onwards’. I would therefore recommend that the title is revised, contextualizing this study to the needs and experience of the German HTA agency may increase the accuracy of their findings and conclusions. In the Introduction section the authors draw from one single study (Wu et al., 2009) to justify why non-English studies should no longer be searched for in HTA reports. Wu et al. analysed how randomised controlled trials were being reported in the Chinese literature written in Chinese. Their paper seemed to suggest that significant results were being published in languages other than English (in their case Chinese) and that RCTs were often poorly reported in Chinese. However, the authors of this manuscript later in the Discussion counterargue this point by remarking that ‘ […] as the volume of Chinese-language literature is growing rapidly and there have been calls for the inclusion of such publications. However, their quality has been questioned because of methodological flaws and overly positive results. ’ Deviating from the point of publication bias and focusing on the quality of the research as a reason for not searching non- English language publications in HTA. While the quality of research is extremely important, there are instruments that deal with that issue within the review process making ‘poor quality’ (according to Wu’s study) not a priory reason for not searching the entire scientific literature written on a particular language such as Chinese. In light of this contradiction the rationale for this study should be reformulated to accurately justify the reasons for not searching non-English papers and always placing these in the context of the German HTA agency. So far, the reasons presented here do not seem to warrant such claim. There are a number of omissions and inaccuracies through the methods section such as the search period in the abstract is not in line with the period reported in the Methods rendering this point unclear. Further, having read the full paper twice, I am still unsure of whether HTA reports of drugs were or not included in their assessment. I think the authors should provide a detailed list of the HTA reports (medicines and non-medicines) that were screened for inclusion in the methods section since these are not their results but the body of data they used for analysis. They should also justify why diagnostic HTA reports would have been included from the start if they were not to be re-analysed in the results. Furthermore, the authors seem to apply the inclusion and exclusion criteria unsystematically when they declare that they only considered journal articles cited in HTA reports but then they present as a limitation that ‘ For 2 reports (D06-01B and D06-01C), we also included evidence syntheses, although this document type was to be excluded.’ But fail to provide an explanation as to why such allowance was made. In the results section two random examples are provided of HTA report, can the authors justify why these are explained in more detail and not the others? which criteria led the authors to provide these two examples? This feels selection bias and should be avoided. In the Discussion section authors seem to contradict themselves. They start by saying that ‘the exclusion of studies published in non-English languages has only a minor effect on the conclusions of HTA reports’ At this point it is important to note that the measure of such effect (whether is big or small) is not provided throughout the manuscript nor it is explained how this ‘change in effect’ was estimated, there are no statistical tests and no p values provided making is really difficult to study whether the change, although minor, might or might not have been significant for the results of the HTA report and their future recommendations. The section follows by saying that ‘ On the basis of our analysis, it is not possible to identify topics where non-English publications might be relevant.’ This statement seems to contradict what has just been said. And finally, they acknowledge that actually ‘it seems reasonable to conduct additional searches for non-English literature if a particular intervention is exclusively or preferentially available in a specific region’. The latter being a statement which defeats the point of the entire manuscript in its current rationale and supporting evidence. Please fix reference number 1 to reflect the title (Cochrane Handbook for Systematic Reviews of Interventions) of the handbook in with Chapter 4 is included. Table 1 would benefit from including definitions for each of the categories. Figure 1 please include the distribution of the 38 reports as they are allocated to different categories. Table 5. What does EP mean? Is the work clearly and accurately presented and does it cite the current literature? No Is the study design appropriate and is the work technically sound? Partly Are sufficient details of methods and analysis provided to allow replication by others? No If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Partly Are the conclusions drawn adequately supported by the results? No Competing Interests No competing interests were disclosed. Reviewer Expertise Information retrieval, systematic reviews, HTA, health research methods I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (1) Author Response 03 Apr 2025 Elke Hausner, Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, 50670, Germany Thank you for your very helpful comments. We address all your points below and have revised the relevant sections in the manuscript. 1. The first point raised is that the study is not suitable for making general statements about HTA reports. Reply: This point was also raised by Reviewer 1 (see above). We have taken your concerns and suggestions and adapted the text Change in the text: instead of referring to HTA reports in general, the text now refers specifically to the German HTA reports 2. It was also criticized that the authors draw from one single study (Wu et al., 2009) to justify why non-English studies should no longer be searched for in HTA reports. Reply: This is a misunderstanding, as Wu 2009 was cited primarily to show that the common assumption that foreign-language articles primarily produce negative/neutral results should be questioned. In order to strengthen the rationale for why we see our study as an important contribution, we have added a current systematic review in the background section, which we had originally only mentioned in the discussion. Change in the text: In the background section, we have added the systematic review by Dobrescu et al. as a rationale for the present analysis. Please see changes in the text on page 4. “However, a recent systematic review by Dobrescu et al. [9] found, that restricting evidence syntheses of interventions to English-language publications is a feasible methodological shortcut for most medical topics. The aim of our analysis was to examine whether the results of the systematic review are transferable to the context of HTA reports.” 3. Another comment refers to different information on the search / publication period in the abstract and the methods section. Change to the text: We have now presented the publication period in the abstract. “We determined whether non-English publications were included in all HTA reports on non-drug interventions (published by IQWiG between 06/2007 to 08/2018) and on selected drug interventions.” 4. It is also noted that it remains unclear whether HTA reports on drugs were included in the analysis. Reply: Table 2 displays the number of included drug / non-drug HTA reports (3 HTA reports on drugs, 67 on non-drugs). No change to the text 5. The reviewer recommends providing a detailed list of HTA reports (drug and non-drug) that were screened for inclusion in the methods section. Reply: As stated in the methods, we included all HTA reports on non-drug interventions (published between 06/2007 and 08/2018) and selected 3 reports on drugs. We have added a list of analysed HTA reports to the supplement. Change to the text: We have added a list of included HTA reports https://doi.org/10.5281/zenodo.12642959 6. The reviewer would like us to justify why diagnostic HTA reports were included from the beginning if they are not to be re-analysed in the results. Reply: HTA reports on diagnostic questions can contain both results on the benefit of the diagnostic measure and on the diagnostic accuracy. Publications in other languages than English only contained information on diagnostic accuracy. However, in no case was a conclusion derived from this, but only the diagnostic accuracy was reported. Our article focuses on the impact of non-English publications on the conclusions of HTA reports, therefore a comparison of conclusions based on different data sets is not possible. No change to the text 7. The reviewer is concerned that the inclusion and exclusion criteria were applied unsystematically and asks for an explanation as to why 2 HTA reports are cited under the limitations. Reply: Based on your concerns, we have revised the extraction of the included studies in the HTA reports and have deleted the entries that did not fulfil the inclusion criteria (n = 11). Change in the text: The tables and figures in the text have been adjusted accordingly. The total number of references included is now 2328, the number of non-English publications 126. Thus, we have deleted the corresponding section in the limitations. 8. In addition, the reviewer notes that there are two examples of HTA reports listed in the results section and asks for a more detailed explanation of why these were selected. Reply: We wanted to illustrate a little more clearly to the reader why a non-English language article was categorized as A or B. As these are only examples, we see no risk of bias in the selection. The full list of HTA reports with non-English articles in categories A and B can be found in the supplement (https://doi.org/10.5281/zenodo.12642959). No change to the text 9. The reviewer points out that the measure of effect is not explained and that no statistical tests and no p-values are given. In her view, this makes it difficult to analyse whether or not the change, even if small, could have been significant for the results of the HTA report and the future recommendations. Reply: The measure of effect was defined as “change in the conclusion of the HTA report” which is composed of the conclusions of the individual endpoints. An overall conclusion was not provided by the underlying HTA reports. For most HTA reports (94%) no changes in the endpoints were identified. All changes were assessed in a yes/no format. This effect measure was therefore qualitative, even it was partly based on statistical analyses and re-analyses. The reviewer rightly points out that even small changes in a meta-analysis may or may not have been significant for the results of the HTA report. For this very reason, our analyses were not based on any statistical comparison between primary analysis and re-analysis. Rather, we examined whether the re-analysis led to a change in the conclusion of the endpoint and thus the HTA report, as this determines reimbursement and availability of health care services. No change to the text 10. According to the reviewer, it is contradictory first to conclude that searching the non-English literature is generally unnecessary, but then to explain that such searches may be worthwhile for particular interventions that are available only a specific region. Reply: We agree that a single HTA report on a country-specific intervention (N16-01) is a weak basis for recommending additional searches of non-English sources under specific circumstances. However, our text only describes this option as “reasonable” – without recommending it for all clinical interventions without clear indications that this is useful in the respective case. Change in the text: We amended the text in the discussion “This has less to do with a specific clinical question and more to do with market availability, which should of course be taken into account when preparing the systematic search.” 11. The reviewer asks to fix reference number 1 to reflect the title (Cochrane Handbook for Systematic Reviews of Interventions) of the handbook in which Chapter 4 is included. Change in the text: We amended the citation 1. Lefebvre C, Glanville J, Briscoe S et al. Chapter 4: Searching for and selecting studies [last updated September 2024]. In: Higgins JPT, Thomas J, Chandler J, Cumpston M, Li T, Page MJ, Welch VA (editors). Cochrane Handbook for Systematic Reviews of Interventions version 6.5 [online]. 2024 [Accessed: 20.01.2025]. URL: https://training.cochrane.org/handbook/current/chapter-04 . 12. Table 1 would benefit from including definitions for each of the categories. You can find a description within the text (page 4): “Category A: change in the conclusion for a particular endpoint; Category B: no change in the conclusion for a particular endpoint; Category C: the non-English publications do not contain relevant information for the HTA report (e.g. because the amount or quality of the data is too low) and are therefore classified as “not relevant”.” Change in the text: We have added the reference to Table 1 to the text “The effect of the exclusion of non-English publications on conclusions was categorized for each HTA report as follows (see Table 1)” 13. The reviewer asks to include in Figure 1 the distribution of the 38 reports as they are allocated to different categories. Reply: The requested information can be seen in Figure 1. As shown in the figure, 13 HTA reports contain only category C publications, 12 HTA reports with category B and 4 HTA reports to category A. The footnote also refers to the fact that an HTA report can contain non-English publications from different categories (A, B or C). The HTA report was assigned to the category that has the greatest impact on the change in conclusion. For example, the 12 HTA reports with category B also contain non-English publications from category C, but none from category A. No change to the text 14. The reviewer asks what the abbreviation EP in Table 5 means. Reply: This is the abbreviation for “endpoint“. An overview of the abbreviations used in the table is added below the table. No change to the text View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Gonzalez-Moral SG. Peer Review Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.166008.r344994) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1134/v1#referee-response-344994 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Sundell K. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 18 Nov 2024 | for Version 1 Knut Sundell , Department of social work and criminology, University of Gävle, Gävle, Sweden 0 Views copyright © 2024 Sundell K. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Not Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Different aspects of publication bias (positive outcome bias, time-lag bias, and place-of-publication bias) constitute serious threats to the validity of research synthesis. If research syntheses are based on a biased selection of the available research, there is an increased risk of producing misleading results. Awareness of these types of bias is important for reading systematic reviews and incorporating their findings into clinical practice and decision making. The authors have screened 70 HTA reports from the German HTA agency, the Institute for Quality and Efficiency in Health Care (IQWiG) searching for reports that include non-English articles. If at least one non-English publication was included, the authors assessed whether the exclusion of non-English publications changed the conclusion. Of 70 HTA reports, 38 (54%) included non-English publications. The exclusion of non-English studies has only a minor effect on the conclusions of HTA reports The authors conclude that, (1) in general, studies published in non-English languages have little influence on the conclusions of HTA reports , and (2) For the vast majority of topics, a language restriction to English seems justified. These conclusions seem premature because of possible selection bias. We don’t know if the non-English articles in these reports constitute all relevant articles or a skewed sample (e.g., because of positive outcome biased, ). To be able to conclude that non-English articles have little influence on the report conclusions, the articles need to represent all relevant articles (i.e., no publication bias) which will require an extensive search for grey literature, including non-peer reviewed articles, the types of articles that the authors have excluded (i.e., conferences abstracts, evidence syntheses, clinical study reports, or registry entries). The best we can say about the results is that non-English articles that are included in IQWiG’s reports may have had scant influence on the HTA-reports’ conclusions. This may be a thought-provoking result but hardly one to base policy on including or excluding non-English articles. Furthermore, I suggest that the authors discuss the types of bias and their relative importance for the design of the study. The exclusion of non-English publications might be motivated, but the results might still be biased because of positive outcome bias, time-lag bias, place-of-publication bias or other types of bias? There is no motivation for only including the German HTA organization. What are the pros and cons of this choice? Is the work clearly and accurately presented and does it cite the current literature? Yes Is the study design appropriate and is the work technically sound? No Are sufficient details of methods and analysis provided to allow replication by others? Yes If applicable, is the statistical analysis and its interpretation appropriate? I cannot comment. A qualified statistician is required. Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? No Competing Interests No competing interests were disclosed. I confirm that I have read this submission and believe that I have an appropriate level of expertise to state that I do not consider it to be of an acceptable scientific standard, for reasons outlined above. reply Respond to this report Responses (1) Author Response 03 Apr 2025 Elke Hausner, Information Management Department, Institute for Quality and Efficiency in Health Care, Cologne, 50670, Germany Thank you very much for taking the time to review our paper. We address your comments below and believe that our replies resolve your concerns. 1. In your peer review you point out that “our conclusions seem premature because of possible selection bias“. It is unclear to us whether this refers to the selection of HTA reports or to the selection of primary studies into these HTA reports. Taking your argument in the first sense, we agree that our conclusions primarily refer to German HTA reports. It may well be that, due to different legal frameworks, HTA reports in other countries address other medical topics or address the same topics using other methods of information retrieval. To emphasize that these are the results of a German HTA agency, we have adapted the text accordingly. Change in the text: instead of referring to HTA reports in general, the text now refers specifically to German HTA reports 2. The issue of selection bias could also be understood in the sense of publication bias. The reviewer suspects that no difference between inclusion and exclusion of non-English sources was found, because relevant literature was not found by any of search approaches. According to the reviewer, this requires “an extensive search for grey literature, including non-peer reviewed articles, the types of articles that the authors have excluded (i.e., conferences abstracts, evidence syntheses, clinical study reports, or registry entries).“ As shown in Table 2, the information retrieval for IQWiG reports includes a number of databases (with a range of 3-23 per report). In addition, study registries were searched, reference lists screened, requests to authors were made and - for all reports but rapid reports - public hearings took place. Therefore, we assume the there is no issue with publication bias. Furthermore, the remaining risk of publication bias would be the nearly the same for both approaches, i.e. with or without non-English sources, because most conference abstracts, grey literature, and study reports will either be written in English or will be unusable for HTA purposes due to a lack of key study information. No change to the text 3. Furthermore, you suggest that the authors discuss the types of bias and their relative importance for the design of the study. From our point of view, it is highly likely that all the forms of bias mentioned had no influence on the changes to the conclusions. Especially if non-significant results tend to end up in non-English journals, this would have resulted in apparent differences between those HTA reports that included non-English sources and those that did not. No change to the text Finally, you state that there is no motivation to include only reports prepared by the German HTA organisation. We disagree, because it requires detailed data to perform the reassessments using other language selection criteria. It would have required multinational cooperation and adequate funding to scale up the study to several HTA agencies and several languages. We also see no point in expanding the sample. Especially since, as mentioned in our discussion, major HTA agencies such as AHRQ, HAS and NICE also use some sort of language restriction in their reports. No change to the text View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Sundell K. Peer Review Report For: Searching for non-English literature may be unnecessary for German HTA Reports [version 3; peer review: 3 approved, 1 approved with reservations, 1 not approved] . F1000Research 2025, 13 :1134 ( https://doi.org/10.5256/f1000research.166008.r332664) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1134/v1#referee-response-332664 Alongside their report, reviewers assign a status to the article: Approved - the paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations - A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved - fundamental flaws in the paper seriously undermine the findings and conclusions Adjust parameters to alter display View on desktop for interactive features Includes Interactive Elements View on desktop for interactive features Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Stay Updated Sign up for content alerts and receive a weekly or monthly email with all newly published articles Register with F1000Research Already registered? Sign in Not now, thanks close PLEASE NOTE If you are an AUTHOR of this article, please check that you signed in with the account associated with this article otherwise we cannot automatically identify your role as an author and your comment will be labelled as a “User Comment”. If you are a REVIEWER of this article, please check that you have signed in with the account associated with this article and then go to your account to submit your report, please do not post your review here. If you do not have access to your original account, please contact us . All commenters must hold a formal affiliation as per our Policies . The information that you give us will be displayed next to your comment. User comments must be in English, comprehensible and relevant to the article under discussion. We reserve the right to remove any comments that we consider to be inappropriate, offensive or otherwise in breach of the User Comment Terms and Conditions . Commenters must not use a comment for personal attacks. When criticisms of the article are based on unpublished data, the data should be made available. I accept the User Comment Terms and Conditions Please confirm that you accept the User Comment Terms and Conditions. Affiliation ✕ refresh Please enter your institution. Note: To add your institution or organisation, start typing the name and then select the correct name from the list. Where applicable, the name will appear in both the original language and in English. Do not paste in the name. If the name does not appear in the drop-down list, we will display the information you have entered. ✕ refresh Country/Region * USA UK Canada China France Germany Afghanistan Aland Islands Albania Algeria American Samoa Andorra Angola Anguilla Antarctica Antigua and Barbuda Argentina Armenia Aruba Australia Austria Azerbaijan Bahamas Bahrain Bangladesh Barbados Belarus Belgium Belize Benin Bermuda Bhutan Bolivia Bosnia and Herzegovina Botswana Bouvet Island Brazil British Indian Ocean Territory British Virgin Islands Brunei Bulgaria Burkina Faso Burundi Cambodia Cameroon Canada Cape Verde Cayman Islands Central African Republic Chad Chile China Christmas Island Cocos (Keeling) Islands Colombia Comoros Congo Cook Islands Costa Rica Cote d'Ivoire Croatia Cuba Cyprus Czech Republic Democratic Republic of the Congo Denmark Djibouti Dominica Dominican Republic Ecuador Egypt El Salvador Equatorial Guinea Eritrea Estonia Ethiopia Falkland Islands Faroe Islands Federated States of Micronesia Fiji Finland France French Guiana French Polynesia French Southern Territories Gabon Georgia Germany Ghana Gibraltar Greece Greenland Grenada Guadeloupe Guam Guatemala Guernsey Guinea Guinea-Bissau Guyana Haiti Heard Island and Mcdonald Islands Holy See (Vatican City State) Honduras Hong Kong Hungary Iceland India Indonesia Iran Iraq Ireland Israel Italy Jamaica Japan Jersey Jordan Kazakhstan Kenya Kiribati Kosovo (Serbia and Montenegro) Kuwait Kyrgyzstan Lao People's Democratic Republic Latvia Lebanon Lesotho Liberia Libya Liechtenstein Lithuania Luxembourg Macao Madagascar Malawi Malaysia Maldives Mali Malta Marshall Islands Martinique Mauritania Mauritius Mayotte Mexico Minor Outlying Islands of the United States Moldova Monaco Mongolia Montenegro Montserrat Morocco Mozambique Myanmar Namibia Nauru Nepal Netherlands Antilles New Caledonia New Zealand Nicaragua Niger Nigeria Niue Norfolk Island North Korea North Macedonia Northern Mariana Islands Norway Oman Pakistan Palau Palestinian Territory Panama Papua New Guinea Paraguay Peru Philippines Pitcairn Poland Portugal Puerto Rico Qatar Reunion Romania Russian Federation Rwanda Saint Helena Saint Kitts and Nevis Saint Lucia Saint Pierre and Miquelon Saint Vincent and the Grenadines Samoa San Marino Sao Tome and Principe Saudi Arabia Senegal Serbia Seychelles Sierra Leone Singapore Slovakia Slovenia Solomon Islands Somalia South Africa South Georgia and the South Sandwich Is South Korea South Sudan Spain Sri Lanka Sudan Suriname Svalbard and Jan Mayen Swaziland Sweden Switzerland Syria Taiwan Tajikistan Tanzania Thailand The Gambia The Netherlands Timor-Leste Togo Tokelau Tonga Trinidad and Tobago Tunisia Turkey Turkmenistan Turks and Caicos Islands Tuvalu UK USA Uganda Ukraine United Arab Emirates United States Virgin Islands Uruguay Uzbekistan Vanuatu Venezuela Vietnam Wallis and Futuna West Bank and Gaza Strip Western Sahara Yemen Zambia Zimbabwe Please select your country/region. You must enter a comment. Competing Interests Please disclose any competing interests that might be construed to influence your judgment of the article's or peer review report's validity or importance. Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Please state your competing interests The comment has been saved. An error has occurred. Please try again. Cancel Post var lTitle = "Searching for non-English literature may...".replace("'", ''); var linkedInUrl = "http://www.linkedin.com/shareArticle?url=https://f1000research.com/articles/13-1134/v3" + "&title=" + encodeURIComponent(lTitle) + "&summary=" + encodeURIComponent('Read the article by '); var deliciousUrl = "https://del.icio.us/post?url=https://f1000research.com/articles/13-1134/v3&title=" + encodeURIComponent(lTitle); var redditUrl = "http://reddit.com/submit?url=https://f1000research.com/articles/13-1134/v3" + "&title=" + encodeURIComponent(lTitle); linkedInUrl += encodeURIComponent('Hausner E et al.'); var offsetTop = /chrome/i.test( navigator.userAgent ) ? 4 : -10; var addthis_config = { ui_offset_top: offsetTop, services_compact : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_expanded : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_custom : [ { name: "LinkedIn", url: linkedInUrl, icon:"/img/icon/at_linkedin.svg" }, { name: "Mendeley", url: "http://www.mendeley.com/import/?url=https://f1000research.com/articles/13-1134/v3/mendeley", icon:"/img/icon/at_mendeley.svg" }, { name: "Reddit", url: redditUrl, icon:"/img/icon/at_reddit.svg" }, ] }; var addthis_share = { url: "https://f1000research.com/articles/13-1134", templates : { twitter : "Searching for non-English literature may be unnecessary for German.... Hausner E et al., published by " + "@F1000Research" + ", https://f1000research.com/articles/13-1134/v3" } }; if (typeof(addthis) != "undefined"){ addthis.addEventListener('addthis.ready', checkCount); addthis.addEventListener('addthis.menu.share', checkCount); } $(".f1r-shares-twitter").attr("href", "https://twitter.com/intent/tweet?text=" + addthis_share.templates.twitter); $(".f1r-shares-facebook").attr("href", "https://www.facebook.com/sharer/sharer.php?u=" + addthis_share.url); $(".f1r-shares-linkedin").attr("href", addthis_config.services_custom[0].url); $(".f1r-shares-reddit").attr("href", addthis_config.services_custom[2].url); $(".f1r-shares-mendelay").attr("href", addthis_config.services_custom[1].url); function checkCount(){ setTimeout(function(){ $(".addthis_button_expanded").each(function(){ var count = $(this).text(); if (count !== "" && count != "0") $(this).removeClass("is-hidden"); else $(this).addClass("is-hidden"); }); }, 1000); } close How to cite this report {{reportCitation}} Cancel Copy Citation Details $(function(){R.ui.buttonDropdowns('.dropdown-for-downloads');}); $(function(){R.ui.toolbarDropdowns('.toolbar-dropdown-for-downloads');}); $.get("/articles/acj/151365/182757") new F1000.Clipboard(); new F1000.ThesaurusTermsDisplay("articles", "article", "182757"); $(document).ready(function() { $( "#frame1" ).on('load', function() { var mydiv = $(this).contents().find("div"); var h = mydiv.height(); console.log(h) }); var tooltipLivingFigure = jQuery(".interactive-living-figure-label .icon-more-info"), titleLivingFigure = tooltipLivingFigure.attr("title"); tooltipLivingFigure.simpletip({ fixed: true, position: ["-115", "30"], baseClass: 'small-tooltip', content:titleLivingFigure + " " }); tooltipLivingFigure.removeAttr("title"); $("body").on("click", ".cite-living-figure", function(e) { e.preventDefault(); var ref = $(this).attr("data-ref"); $(this).closest(".living-figure-list-container").find("#" + ref).fadeIn(200); }); $("body").on("click", ".close-cite-living-figure", function(e) { e.preventDefault(); $(this).closest(".popup-window-wrapper").fadeOut(200); }); $(document).on("mouseup", function(e) { var metricsContainer = $(".article-metrics-popover-wrapper"); if (!metricsContainer.is(e.target) && metricsContainer.has(e.target).length === 0) { $(".article-metrics-close-button").click(); } }); var articleId = $('#articleId').val(); if($("#main-article-count-box").attachArticleMetrics) { $("#main-article-count-box").attachArticleMetrics(articleId, { articleMetricsView: true }); } }); var figshareWidget = $(".new_figshare_widget"); if (figshareWidget.length > 0) { window.figshare.load("f1000", function(Widget) { // Select a tag/tags defined in your page. In this tag we will place the widget. _.map(figshareWidget, function(el){ var widget = new Widget({ articleId: $(el).attr("figshare_articleId") //height:300 // this is the height of the viewer part. [Default: 550] }); widget.initialize(); // initialize the widget widget.mount(el); // mount it in a tag that's on your page // this will save the widget on the global scope for later use from // your JS scripts. This line is optional. //window.widget = widget; }); }); } close Error Close Add Reset F1000.MICROSERVICES.AFFILIATION = ''; $(document).ready(function () { $('.js-affiliations-form').each((index, form) => { new AffiliationForm({ formId: form.id, institutionErrorSelector: '.comment-enter-institution', departmentErrorSelector: '.comment-enter-department', placeSelector: '.js-add-comment-place', stateSelector: '.js-add-comment-state', zipCodeSelector: '.js-add-comment-zipcode', countrySelector: '.js-add-comment-country', countryErrorSelector: '.comment-enter-country', }); }); }); $(document).ready(function () { var reportIds = { "375429": 0, "375428": 0, "375427": 0, "344991": 0, "344990": 0, "344997": 0, "344996": 0, "344999": 0, "344998": 0, "344993": 0, "344992": 16, "344995": 0, "344994": 24, "340397": 0, "340399": 0, "340398": 0, "340405": 0, "340404": 0, "340406": 0, "340401": 0, "340400": 0, "340403": 0, "340402": 0, "336957": 0, "336956": 0, "336959": 0, "336958": 0, "336955": 0, "336964": 0, "336961": 0, "336960": 0, "336963": 0, "336962": 0, "397140": 0, "397138": 3, "397139": 0, "397136": 0, "397137": 0, "332661": 0, "376053": 0, "332660": 0, "376052": 0, "332663": 0, "376055": 12, "332662": 0, "376054": 0, "332659": 0, "376061": 0, "332668": 0, "376060": 23, "332665": 0, "376057": 0, "332664": 38, "376056": 0, "332667": 0, "376059": 0, "332666": 0, "376058": 0, }; $(".referee-response-container,.js-referee-report").each(function(index, el) { var reportId = $(el).attr("data-reportid"), reportCount = reportIds[reportId] || 0; $(el).find(".comments-count-container,.js-referee-report-views").html(reportCount); }); var uuidInput = $("#article_uuid"), oldUUId = uuidInput.val(), newUUId = "687d208b-3752-4345-ac4a-30f63ecba66e"; uuidInput.val(newUUId); $("a[href*='article_uuid=']").each(function(index, el) { var newHref = $(el).attr("href").replace(oldUUId, newUUId); $(el).attr("href", newHref); }); }); An innovative open access publishing platform offering rapid publication and open peer review, whilst supporting data deposition and sharing. Browse Gateways Collections How it Works Contact For Developers Cookie Notice Privacy Notice RSS Submit Your Research Follow us © 2012-2026 F1000 Research Ltd. ISSN 2046-1402 | Legal | Partner of Research4Life • CrossRef • ORCID • FAIRSharing R.templateTests.simpleTemplate = R.template(' $text $text $text $text $text '); R.templateTests.runTests(); var F1000platform = new F1000.Platform({ name: "f1000research", displayName: "F1000Research", hostName: "f1000research.com", id: "1", editorialEmail: "[email protected]", infoEmail: "[email protected]", usePmcStats: true }); $(function(){R.ui.dropdowns('.dropdown-for-authors, .dropdown-for-about, .dropdown-for-myresearch');}); // $(function(){R.ui.dropdowns('.dropdown-for-referees');}); $(document).ready(function () { if ($(".cookie-warning").is(":visible")) { $(".sticky").css("margin-bottom", "35px"); $(".devices").addClass("devices-and-cookie-warning"); } $(".cookie-warning .close-button").click(function (e) { $(".devices").removeClass("devices-and-cookie-warning"); $(".sticky").css("margin-bottom", "0"); }); $("#tweeter-feed .tweet-message").each(function (i, message) { var self = $(message); self.html(linkify(self.html())); }); $(".partner").on("mouseenter mouseleave", function() { $(this).find(".gray-scale, .colour").toggleClass("is-hidden"); }); }); Sign In Remember me Forgotten your password? Sign In Cancel Email or password not correct. Please try again Please wait... $(function(){ // Note: All the setup needs to run against a name attribute and *not* the id due the clonish // nature of facebox... $("a[id=googleSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("GOOGLE"); $("form[id=oAuthForm]").submit(); }); $("a[id=facebookSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("FACEBOOK"); $("form[id=oAuthForm]").submit(); }); $("a[id=orcidSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("ORCID"); $("form[id=oAuthForm]").submit(); }); }); If you've forgotten your password, please enter your email address below and we'll send you instructions on how to reset your password. The email address should be the one you originally registered with F1000. Email address not valid, please try again You registered with F1000 via Google, so we cannot reset your password. To sign in, please click here . If you still need help with your Google account password, please click here . You registered with F1000 via Facebook, so we cannot reset your password. To sign in, please click here . If you still need help with your Facebook account password, please click here . Code not correct, please try again Reset password Cancel Email us for further assistance. Server error, please try again. If your email address is registered with us, we will email you instructions to reset your password. If you think you should have received this email but it has not arrived, please check your spam filters and/or contact for further assistance. Please wait... Register $(document).ready(function () { signIn.createSignInAsRow($("#sign-in-form-gfb-popup")); $(".target-field").each(function () { var uris = $(this).val().split("/"); if (uris.pop() === "login") { $(this).val(uris.toString().replace(",","/")); } }); });