Training and Experience in Study Selection (TESS): study protocol for a pilot randomised trial within a systematic review

Training and Experience in Study Selection... | HRB Open Research dataLayer = dataLayer || []; // Standard GTM initialization - Google Consent Mode handles consent automatically (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl+ '>m_auth=hzk0Vc3qFsQYhCrIoHz68A>m_preview=env-1>m_cookies_win=x';f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-MWFK8L5J'); ;window.NREUM||(NREUM={});NREUM.init={distributed_tracing:{enabled:true},privacy:{cookies_enabled:true},ajax:{deny_list:["bam.nr-data.net"]}}; ;NREUM.loader_config={accountID:"438030",trustKey:"438030",agentID:"772317073",licenseKey:"97f8f67f26",applicationID:"772317073"} ;NREUM.info={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net",licenseKey:"97f8f67f26",applicationID:"772317073",sa:1} ;/*! For license information please see nr-loader-spa-1.236.0.min.js.LICENSE.txt */ (()=>{"use strict";var e,t,r={5763:(e,t,r)=>{r.d(t,{P_:()=>l,Mt:()=>g,C5:()=>s,DL:()=>v,OP:()=>T,lF:()=>D,Yu:()=>y,Dg:()=>h,CX:()=>c,GE:()=>b,sU:()=>_});var n=r(8632),i=r(9567);const o={beacon:n.ce.beacon,errorBeacon:n.ce.errorBeacon,licenseKey:void 0,applicationID:void 0,sa:void 0,queueTime:void 0,applicationTime:void 0,ttGuid:void 0,user:void 0,account:void 0,product:void 0,extra:void 0,jsAttributes:{},userAttributes:void 0,atts:void 0,transactionName:void 0,tNamePlain:void 0},a={};function s(e){if(!e)throw new Error("All info objects require an agent identifier!");if(!a[e])throw new Error("Info for ".concat(e," was never set"));return a[e]}function c(e,t){if(!e)throw new Error("All info objects require an agent identifier!");a[e]=(0,i.D)(t,o),(0,n.Qy)(e,a[e],"info")}var u=r(7056);const d=()=>{const e={blockSelector:"[data-nr-block]",maskInputOptions:{password:!0}};return{allow_bfcache:!0,privacy:{cookies_enabled:!0},ajax:{deny_list:void 0,enabled:!0,harvestTimeSeconds:10},distributed_tracing:{enabled:void 0,exclude_newrelic_header:void 0,cors_use_newrelic_header:void 0,cors_use_tracecontext_headers:void 0,allowed_origins:void 0},session:{domain:void 0,expiresMs:u.oD,inactiveMs:u.Hb},ssl:void 0,obfuscate:void 0,jserrors:{enabled:!0,harvestTimeSeconds:10},metrics:{enabled:!0},page_action:{enabled:!0,harvestTimeSeconds:30},page_view_event:{enabled:!0},page_view_timing:{enabled:!0,harvestTimeSeconds:30,long_task:!1},session_trace:{enabled:!0,harvestTimeSeconds:10},harvest:{tooManyRequestsDelay:60},session_replay:{enabled:!1,harvestTimeSeconds:60,sampleRate:.1,errorSampleRate:.1,maskTextSelector:"*",maskAllInputs:!0,get blockClass(){return"nr-block"},get ignoreClass(){return"nr-ignore"},get maskTextClass(){return"nr-mask"},get blockSelector(){return e.blockSelector},set blockSelector(t){e.blockSelector+=",".concat(t)},get maskInputOptions(){return e.maskInputOptions},set maskInputOptions(t){e.maskInputOptions={...t,password:!0}}},spa:{enabled:!0,harvestTimeSeconds:10}}},f={};function l(e){if(!e)throw new Error("All configuration objects require an agent identifier!");if(!f[e])throw new Error("Configuration for ".concat(e," was never set"));return f[e]}function h(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");f[e]=(0,i.D)(t,d()),(0,n.Qy)(e,f[e],"config")}function g(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");var r=l(e);if(r){for(var n=t.split("."),i=0;i {r.d(t,{D:()=>i});var n=r(50);function i(e,t){try{if(!e||"object"!=typeof e)return(0,n.Z)("Setting a Configurable requires an object as input");if(!t||"object"!=typeof t)return(0,n.Z)("Setting a Configurable requires a model to set its initial properties");const r=Object.create(Object.getPrototypeOf(t),Object.getOwnPropertyDescriptors(t)),o=0===Object.keys(r).length?e:r;for(let a in o)if(void 0!==e[a])try{"object"==typeof e[a]&&"object"==typeof t[a]?r[a]=i(e[a],t[a]):r[a]=e[a]}catch(e){(0,n.Z)("An error occurred while setting a property of a Configurable",e)}return r}catch(e){(0,n.Z)("An error occured while setting a Configurable",e)}}},6818:(e,t,r)=>{r.d(t,{Re:()=>i,gF:()=>o,q4:()=>n});const n="1.236.0",i="PROD",o="CDN"},385:(e,t,r)=>{r.d(t,{FN:()=>a,IF:()=>u,Nk:()=>f,Tt:()=>s,_A:()=>o,il:()=>n,pL:()=>c,v6:()=>i,w1:()=>d});const n="undefined"!=typeof window&&!!window.document,i="undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self.navigator instanceof WorkerNavigator||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis.navigator instanceof WorkerNavigator),o=n?window:"undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis),a=""+o?.location,s=/iPad|iPhone|iPod/.test(navigator.userAgent),c=s&&"undefined"==typeof SharedWorker,u=(()=>{const e=navigator.userAgent.match(/Firefox[/\s](\d+\.\d+)/);return Array.isArray(e)&&e.length>=2?+e[1]:0})(),d=Boolean(n&&window.document.documentMode),f=!!navigator.sendBeacon},1117:(e,t,r)=>{r.d(t,{w:()=>o});var n=r(50);const i={agentIdentifier:"",ee:void 0};class o{constructor(e){try{if("object"!=typeof e)return(0,n.Z)("shared context requires an object as input");this.sharedContext={},Object.assign(this.sharedContext,i),Object.entries(e).forEach((e=>{let[t,r]=e;Object.keys(i).includes(t)&&(this.sharedContext[t]=r)}))}catch(e){(0,n.Z)("An error occured while setting SharedContext",e)}}}},8e3:(e,t,r)=>{r.d(t,{L:()=>d,R:()=>c});var n=r(2177),i=r(1284),o=r(4322),a=r(3325);const s={};function c(e,t){const r={staged:!1,priority:a.p[t]||0};u(e),s[e].get(t)||s[e].set(t,r)}function u(e){e&&(s[e]||(s[e]=new Map))}function d(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"",t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"feature";if(u(e),!e||!s[e].get(t))return a(t);s[e].get(t).staged=!0;const r=[...s[e]];function a(t){const r=e?n.ee.get(e):n.ee,a=o.X.handlers;if(r.backlog&&a){var s=r.backlog[t],c=a[t];if(c){for(var u=0;s&&u {let[t,r]=e;return r.staged}))&&(r.sort(((e,t)=>e[1].priority-t[1].priority)),r.forEach((e=>{let[t]=e;a(t)})))}function f(e,t){var r=e[1];(0,i.D)(t[r],(function(t,r){var n=e[0];if(r[0]===n){var i=r[1],o=e[3],a=e[2];i.apply(o,a)}}))}},2177:(e,t,r)=>{r.d(t,{c:()=>f,ee:()=>u});var n=r(8632),i=r(2210),o=r(1284),a=r(5763),s="nr@context";let c=(0,n.fP)();var u;function d(){}function f(e){return(0,i.X)(e,s,l)}function l(){return new d}function h(){u.aborted=!0,u.backlog={}}c.ee?u=c.ee:(u=function e(t,r){var n={},c={},f={},g=!1;try{g=16===r.length&&(0,a.OP)(r).isolatedBacklog}catch(e){}var p={on:b,addEventListener:b,removeEventListener:y,emit:v,get:x,listeners:w,context:m,buffer:A,abort:h,aborted:!1,isBuffering:E,debugId:r,backlog:g?{}:t&&"object"==typeof t.backlog?t.backlog:{}};return p;function m(e){return e&&e instanceof d?e:e?(0,i.X)(e,s,l):l()}function v(e,r,n,i,o){if(!1!==o&&(o=!0),!u.aborted||i){t&&o&&t.emit(e,r,n);for(var a=m(n),s=w(e),d=s.length,f=0;fn,p:()=>i});var n=r(2177).ee.get("handle");function i(e,t,r,i,o){o?(o.buffer([e],i),o.emit(e,t,r)):(n.buffer([e],i),n.emit(e,t,r))}},4322:(e,t,r)=>{r.d(t,{X:()=>o});var n=r(5546);o.on=a;var i=o.handlers={};function o(e,t,r,o){a(o||n.E,i,e,t,r)}function a(e,t,r,i,o){o||(o="feature"),e||(e=n.E);var a=t[o]=t[o]||{};(a[r]=a[r]||[]).push([e,i])}},3239:(e,t,r)=>{r.d(t,{bP:()=>s,iz:()=>c,m$:()=>a});var n=r(385);let i=!1,o=!1;try{const e={get passive(){return i=!0,!1},get signal(){return o=!0,!1}};n._A.addEventListener("test",null,e),n._A.removeEventListener("test",null,e)}catch(e){}function a(e,t){return i||o?{capture:!!e,passive:i,signal:t}:!!e}function s(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;window.addEventListener(e,t,a(r,n))}function c(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;document.addEventListener(e,t,a(r,n))}},4402:(e,t,r)=>{r.d(t,{Ht:()=>u,M:()=>c,Rl:()=>a,ky:()=>s});var n=r(385);const i="xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx";function o(e,t){return e?15&e[t]:16*Math.random()|0}function a(){const e=n._A?.crypto||n._A?.msCrypto;let t,r=0;return e&&e.getRandomValues&&(t=e.getRandomValues(new Uint8Array(31))),i.split("").map((e=>"x"===e?o(t,++r).toString(16):"y"===e?(3&o()|8).toString(16):e)).join("")}function s(e){const t=n._A?.crypto||n._A?.msCrypto;let r,i=0;t&&t.getRandomValues&&(r=t.getRandomValues(new Uint8Array(31)));const a=[];for(var s=0;s {r.d(t,{Bq:()=>n,Hb:()=>o,oD:()=>i});const n="NRBA",i=144e5,o=18e5},7894:(e,t,r)=>{function n(){return Math.round(performance.now())}r.d(t,{z:()=>n})},7243:(e,t,r)=>{r.d(t,{e:()=>o});var n=r(385),i={};function o(e){if(e in i)return i[e];if(0===(e||"").indexOf("data:"))return{protocol:"data"};let t;var r=n._A?.location,o={};if(n.il)t=document.createElement("a"),t.href=e;else try{t=new URL(e,r.href)}catch(e){return o}o.port=t.port;var a=t.href.split("://");!o.port&&a[1]&&(o.port=a[1].split("/")[0].split("@").pop().split(":")[1]),o.port&&"0"!==o.port||(o.port="https"===a[0]?"443":"80"),o.hostname=t.hostname||r.hostname,o.pathname=t.pathname,o.protocol=a[0],"/"!==o.pathname.charAt(0)&&(o.pathname="/"+o.pathname);var s=!t.protocol||":"===t.protocol||t.protocol===r.protocol,c=t.hostname===r.hostname&&t.port===r.port;return o.sameOrigin=s&&(!t.hostname||c),"/"===o.pathname&&(i[e]=o),o}},50:(e,t,r)=>{function n(e,t){"function"==typeof console.warn&&(console.warn("New Relic: ".concat(e)),t&&console.warn(t))}r.d(t,{Z:()=>n})},2587:(e,t,r)=>{r.d(t,{N:()=>c,T:()=>u});var n=r(2177),i=r(5546),o=r(8e3),a=r(3325);const s={stn:[a.D.sessionTrace],err:[a.D.jserrors,a.D.metrics],ins:[a.D.pageAction],spa:[a.D.spa],sr:[a.D.sessionReplay,a.D.sessionTrace]};function c(e,t){const r=n.ee.get(t);e&&"object"==typeof e&&(Object.entries(e).forEach((e=>{let[t,n]=e;void 0===u[t]&&(s[t]?s[t].forEach((e=>{n?(0,i.p)("feat-"+t,[],void 0,e,r):(0,i.p)("block-"+t,[],void 0,e,r),(0,i.p)("rumresp-"+t,[Boolean(n)],void 0,e,r)})):n&&(0,i.p)("feat-"+t,[],void 0,void 0,r),u[t]=Boolean(n))})),Object.keys(s).forEach((e=>{void 0===u[e]&&(s[e]?.forEach((t=>(0,i.p)("rumresp-"+e,[!1],void 0,t,r))),u[e]=!1)})),(0,o.L)(t,a.D.pageViewEvent))}const u={}},2210:(e,t,r)=>{r.d(t,{X:()=>i});var n=Object.prototype.hasOwnProperty;function i(e,t,r){if(n.call(e,t))return e[t];var i=r();if(Object.defineProperty&&Object.keys)try{return Object.defineProperty(e,t,{value:i,writable:!0,enumerable:!1}),i}catch(e){}return e[t]=i,i}},1284:(e,t,r)=>{r.d(t,{D:()=>n});const n=(e,t)=>Object.entries(e||{}).map((e=>{let[r,n]=e;return t(r,n)}))},4351:(e,t,r)=>{r.d(t,{P:()=>o});var n=r(2177);const i=()=>{const e=new WeakSet;return(t,r)=>{if("object"==typeof r&&null!==r){if(e.has(r))return;e.add(r)}return r}};function o(e){try{return JSON.stringify(e,i())}catch(e){try{n.ee.emit("internal-error",[e])}catch(e){}}}},3960:(e,t,r)=>{r.d(t,{K:()=>a,b:()=>o});var n=r(3239);function i(){return"undefined"==typeof document||"complete"===document.readyState}function o(e,t){if(i())return e();(0,n.bP)("load",e,t)}function a(e){if(i())return e();(0,n.iz)("DOMContentLoaded",e)}},8632:(e,t,r)=>{r.d(t,{EZ:()=>u,Qy:()=>c,ce:()=>o,fP:()=>a,gG:()=>d,mF:()=>s});var n=r(7894),i=r(385);const o={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net"};function a(){return i._A.NREUM||(i._A.NREUM={}),void 0===i._A.newrelic&&(i._A.newrelic=i._A.NREUM),i._A.NREUM}function s(){let e=a();return e.o||(e.o={ST:i._A.setTimeout,SI:i._A.setImmediate,CT:i._A.clearTimeout,XHR:i._A.XMLHttpRequest,REQ:i._A.Request,EV:i._A.Event,PR:i._A.Promise,MO:i._A.MutationObserver,FETCH:i._A.fetch}),e}function c(e,t,r){let i=a();const o=i.initializedAgents||{},s=o[e]||{};return Object.keys(s).length||(s.initializedAt={ms:(0,n.z)(),date:new Date}),i.initializedAgents={...o,[e]:{...s,[r]:t}},i}function u(e,t){a()[e]=t}function d(){return function(){let e=a();const t=e.info||{};e.info={beacon:o.beacon,errorBeacon:o.errorBeacon,...t}}(),function(){let e=a();const t=e.init||{};e.init={...t}}(),s(),function(){let e=a();const t=e.loader_config||{};e.loader_config={...t}}(),a()}},7956:(e,t,r)=>{r.d(t,{N:()=>i});var n=r(3239);function i(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],r=arguments.length>2?arguments[2]:void 0,i=arguments.length>3?arguments[3]:void 0;return void(0,n.iz)("visibilitychange",(function(){if(t)return void("hidden"==document.visibilityState&&e());e(document.visibilityState)}),r,i)}},1214:(e,t,r)=>{r.d(t,{em:()=>v,u5:()=>N,QU:()=>S,_L:()=>I,Gm:()=>L,Lg:()=>M,gy:()=>U,BV:()=>Q,Kf:()=>ee});var n=r(2177);const i="nr@original";var o=Object.prototype.hasOwnProperty,a=!1;function s(e,t){return e||(e=n.ee),r.inPlace=function(e,t,n,i,o){n||(n="");var a,s,c,u="-"===n.charAt(0);for(c=0;c 2?n-2:0),o=2;o {r(A[T],e,w),r(E[T],e,w)})),r(l._A,"fetch",y),t.on(y+"end",(function(e,r){var n=this;if(r){var i=r.headers.get("content-length");null!==i&&(n.rxSize=i),t.emit(y+"done",[null,r],n)}else t.emit(y+"done",[e],n)})),t}const O={},j=["pushState","replaceState"];function S(e){const t=function(e){return(e||n.ee).get("history")}(e);return!l.il||O[t.debugId]++||(O[t.debugId]=1,s(t).inPlace(window.history,j,"-")),t}var P=r(3239);const C={},R=["appendChild","insertBefore","replaceChild"];function I(e){const t=function(e){return(e||n.ee).get("jsonp")}(e);if(!l.il||C[t.debugId])return t;C[t.debugId]=!0;var r=s(t),i=/[?&](?:callback|cb)=([^&#]+)/,o=/(.*)\.([^.]+)/,a=/^(\w+)(\.|$)(.*)$/;function c(e,t){var r=e.match(a),n=r[1],i=r[3];return i?c(i,t[n]):t[n]}return r.inPlace(Node.prototype,R,"dom-"),t.on("dom-start",(function(e){!function(e){if(!e||"string"!=typeof e.nodeName||"script"!==e.nodeName.toLowerCase())return;if("function"!=typeof e.addEventListener)return;var n=(a=e.src,s=a.match(i),s?s[1]:null);var a,s;if(!n)return;var u=function(e){var t=e.match(o);if(t&&t.length>=3)return{key:t[2],parent:c(t[1],window)};return{key:e,parent:window}}(n);if("function"!=typeof u.parent[u.key])return;var d={};function f(){t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}function l(){t.emit("jsonp-error",[],d),t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}r.inPlace(u.parent,[u.key],"cb-",d),e.addEventListener("load",f,(0,P.m$)(!1)),e.addEventListener("error",l,(0,P.m$)(!1)),t.emit("new-jsonp",[e.src],d)}(e[0])})),t}var k=r(5763);const H={};function L(e){const t=function(e){return(e||n.ee).get("mutation")}(e);if(!l.il||H[t.debugId])return t;H[t.debugId]=!0;var r=s(t),i=k.Yu.MO;return i&&(window.MutationObserver=function(e){return this instanceof i?new i(r(e,"fn-")):i.apply(this,arguments)},MutationObserver.prototype=i.prototype),t}const z={};function M(e){const t=function(e){return(e||n.ee).get("promise")}(e);if(z[t.debugId])return t;z[t.debugId]=!0;var r=n.c,o=s(t),a=k.Yu.PR;return a&&function(){function e(r){var n=t.context(),i=o(r,"executor-",n,null,!1);const s=Reflect.construct(a,[i],e);return t.context(s).getCtx=function(){return n},s}l._A.Promise=e,Object.defineProperty(e,"name",{value:"Promise"}),e.toString=function(){return a.toString()},Object.setPrototypeOf(e,a),["all","race"].forEach((function(r){const n=a[r];e[r]=function(e){let i=!1;[...e||[]].forEach((e=>{this.resolve(e).then(a("all"===r),a(!1))}));const o=n.apply(this,arguments);return o;function a(e){return function(){t.emit("propagate",[null,!i],o,!1,!1),i=i||!e}}}})),["resolve","reject"].forEach((function(r){const n=a[r];e[r]=function(e){const r=n.apply(this,arguments);return e!==r&&t.emit("propagate",[e,!0],r,!1,!1),r}})),e.prototype=a.prototype;const n=a.prototype.then;a.prototype.then=function(){var e=this,i=r(e);i.promise=e;for(var a=arguments.length,s=new Array(a),c=0;c e())),t};function m(e,t){i.inPlace(t,["onreadystatechange"],"fn-",E)}function b(){var e=this,t=r.context(e);e.readyState>3&&!t.resolved&&(t.resolved=!0,r.emit("xhr-resolved",[],e)),i.inPlace(e,f,"fn-",E)}if(function(e,t){for(var r in e)t[r]=e[r]}(o,p),p.prototype=o.prototype,i.inPlace(p.prototype,J,"-xhr-",E),r.on("send-xhr-start",(function(e,t){m(e,t),function(e){h.push(e),a&&(y?y.then(A):u?u(A):(w=-w,x.data=w))}(t)})),r.on("open-xhr-start",m),a){var y=c&&c.resolve();if(!u&&!c){var w=1,x=document.createTextNode(w);new a(A).observe(x,{characterData:!0})}}else t.on("fn-end",(function(e){e[0]&&e[0].type===d||A()}));function A(){for(var e=0;e {r.d(t,{t:()=>n});const n=r(3325).D.ajax},6660:(e,t,r)=>{r.d(t,{A:()=>i,t:()=>n});const n=r(3325).D.jserrors,i="nr@seenError"},3081:(e,t,r)=>{r.d(t,{gF:()=>o,mY:()=>i,t9:()=>n,vz:()=>s,xS:()=>a});const n=r(3325).D.metrics,i="sm",o="cm",a="storeSupportabilityMetrics",s="storeEventMetrics"},4649:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageAction},7633:(e,t,r)=>{r.d(t,{Dz:()=>i,OJ:()=>a,qw:()=>o,t9:()=>n});const n=r(3325).D.pageViewEvent,i="firstbyte",o="domcontent",a="windowload"},9251:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageViewTiming},3614:(e,t,r)=>{r.d(t,{BST_RESOURCE:()=>i,END:()=>s,FEATURE_NAME:()=>n,FN_END:()=>u,FN_START:()=>c,PUSH_STATE:()=>d,RESOURCE:()=>o,START:()=>a});const n=r(3325).D.sessionTrace,i="bstResource",o="resource",a="-start",s="-end",c="fn"+a,u="fn"+s,d="pushState"},7836:(e,t,r)=>{r.d(t,{BODY:()=>A,CB_END:()=>E,CB_START:()=>u,END:()=>x,FEATURE_NAME:()=>i,FETCH:()=>_,FETCH_BODY:()=>v,FETCH_DONE:()=>m,FETCH_START:()=>p,FN_END:()=>c,FN_START:()=>s,INTERACTION:()=>l,INTERACTION_API:()=>d,INTERACTION_EVENTS:()=>o,JSONP_END:()=>b,JSONP_NODE:()=>g,JS_TIME:()=>T,MAX_TIMER_BUDGET:()=>a,REMAINING:()=>f,SPA_NODE:()=>h,START:()=>w,originalSetTimeout:()=>y});var n=r(5763);const i=r(3325).D.spa,o=["click","submit","keypress","keydown","keyup","change"],a=999,s="fn-start",c="fn-end",u="cb-start",d="api-ixn-",f="remaining",l="interaction",h="spaNode",g="jsonpNode",p="fetch-start",m="fetch-done",v="fetch-body-",b="jsonp-end",y=n.Yu.ST,w="-start",x="-end",A="-body",E="cb"+x,T="jsTime",_="fetch"},5938:(e,t,r)=>{r.d(t,{W:()=>o});var n=r(5763),i=r(2177);class o{constructor(e,t,r){this.agentIdentifier=e,this.aggregator=t,this.ee=i.ee.get(e,(0,n.OP)(this.agentIdentifier).isolatedBacklog),this.featureName=r,this.blocked=!1}}},9144:(e,t,r)=>{r.d(t,{j:()=>m});var n=r(3325),i=r(5763),o=r(5546),a=r(2177),s=r(7894),c=r(8e3),u=r(3960),d=r(385),f=r(50),l=r(3081),h=r(8632);function g(){const e=(0,h.gG)();["setErrorHandler","finished","addToTrace","inlineHit","addRelease","addPageAction","setCurrentRouteName","setPageViewName","setCustomAttribute","interaction","noticeError","setUserId"].forEach((t=>{e[t]=function(){for(var r=arguments.length,n=new Array(r),i=0;i 1?r-1:0),i=1;i {e.exposed&&e.api[t]&&o.push(e.api[t](...n))})),o.length>1?o:o[0]}(t,...n)}}))}var p=r(2587);function m(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},m=arguments.length>2?arguments[2]:void 0,v=arguments.length>3?arguments[3]:void 0,{init:b,info:y,loader_config:w,runtime:x={loaderType:m},exposed:A=!0}=t;const E=(0,h.gG)();y||(b=E.init,y=E.info,w=E.loader_config),(0,i.Dg)(e,b||{}),(0,i.GE)(e,w||{}),(0,i.sU)(e,x),y.jsAttributes??={},d.v6&&(y.jsAttributes.isWorker=!0),(0,i.CX)(e,y),g();const T=function(e,t){t||(0,c.R)(e,"api");const h={};var g=a.ee.get(e),p=g.get("tracer"),m="api-",v=m+"ixn-";function b(t,r,n,o){const a=(0,i.C5)(e);return null===r?delete a.jsAttributes[t]:(0,i.CX)(e,{...a,jsAttributes:{...a.jsAttributes,[t]:r}}),x(m,n,!0,o||null===r?"session":void 0)(t,r)}function y(){}["setErrorHandler","finished","addToTrace","inlineHit","addRelease"].forEach((e=>h[e]=x(m,e,!0,"api"))),h.addPageAction=x(m,"addPageAction",!0,n.D.pageAction),h.setCurrentRouteName=x(m,"routeName",!0,n.D.spa),h.setPageViewName=function(t,r){if("string"==typeof t)return"/"!==t.charAt(0)&&(t="/"+t),(0,i.OP)(e).customTransaction=(r||"http://custom.transaction")+t,x(m,"setPageViewName",!0)()},h.setCustomAttribute=function(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2];if("string"==typeof e){if(["string","number"].includes(typeof t)||null===t)return b(e,t,"setCustomAttribute",r);(0,f.Z)("Failed to execute setCustomAttribute.\nNon-null value must be a string or number type, but a type of was provided."))}else(0,f.Z)("Failed to execute setCustomAttribute.\nName must be a string type, but a type of was provided."))},h.setUserId=function(e){if("string"==typeof e||null===e)return b("enduser.id",e,"setUserId",!0);(0,f.Z)("Failed to execute setUserId.\nNon-null value must be a string type, but a type of was provided."))},h.interaction=function(){return(new y).get()};var w=y.prototype={createTracer:function(e,t){var r={},i=this,a="function"==typeof t;return(0,o.p)(v+"tracer",[(0,s.z)(),e,r],i,n.D.spa,g),function(){if(p.emit((a?"":"no-")+"fn-start",[(0,s.z)(),i,a],r),a)try{return t.apply(this,arguments)}catch(e){throw p.emit("fn-err",[arguments,this,"string"==typeof e?new Error(e):e],r),e}finally{p.emit("fn-end",[(0,s.z)()],r)}}}};function x(e,t,r,i){return function(){return(0,o.p)(l.xS,["API/"+t+"/called"],void 0,n.D.metrics,g),i&&(0,o.p)(e+t,[(0,s.z)(),...arguments],r?null:this,i,g),r?void 0:this}}function A(){r.e(439).then(r.bind(r,7438)).then((t=>{let{setAPI:r}=t;r(e),(0,c.L)(e,"api")})).catch((()=>(0,f.Z)("Downloading runtime APIs failed...")))}return["actionText","setName","setAttribute","save","ignore","onEnd","getContext","end","get"].forEach((e=>{w[e]=x(v,e,void 0,n.D.spa)})),h.noticeError=function(e,t){"string"==typeof e&&(e=new Error(e)),(0,o.p)(l.xS,["API/noticeError/called"],void 0,n.D.metrics,g),(0,o.p)("err",[e,(0,s.z)(),!1,t],void 0,n.D.jserrors,g)},d.il?(0,u.b)((()=>A()),!0):A(),h}(e,v);return(0,h.Qy)(e,T,"api"),(0,h.Qy)(e,A,"exposed"),(0,h.EZ)("activatedFeatures",p.T),T}},3325:(e,t,r)=>{r.d(t,{D:()=>n,p:()=>i});const n={ajax:"ajax",jserrors:"jserrors",metrics:"metrics",pageAction:"page_action",pageViewEvent:"page_view_event",pageViewTiming:"page_view_timing",sessionReplay:"session_replay",sessionTrace:"session_trace",spa:"spa"},i={[n.pageViewEvent]:1,[n.pageViewTiming]:2,[n.metrics]:3,[n.jserrors]:4,[n.ajax]:5,[n.sessionTrace]:6,[n.pageAction]:7,[n.spa]:8,[n.sessionReplay]:9}}},n={};function i(e){var t=n[e];if(void 0!==t)return t.exports;var o=n[e]={exports:{}};return r[e](o,o.exports,i),o.exports}i.m=r,i.d=(e,t)=>{for(var r in t)i.o(t,r)&&!i.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},i.f={},i.e=e=>Promise.all(Object.keys(i.f).reduce(((t,r)=>(i.f[r](e,t),t)),[])),i.u=e=>(({78:"page_action-aggregate",147:"metrics-aggregate",242:"session-manager",317:"jserrors-aggregate",348:"page_view_timing-aggregate",412:"lazy-feature-loader",439:"async-api",538:"recorder",590:"session_replay-aggregate",675:"compressor",733:"session_trace-aggregate",786:"page_view_event-aggregate",873:"spa-aggregate",898:"ajax-aggregate"}[e]||e)+"."+{78:"ac76d497",147:"3dc53903",148:"1a20d5fe",242:"2a64278a",317:"49e41428",348:"bd6de33a",412:"2f55ce66",439:"30bd804e",538:"1b18459f",590:"cf0efb30",675:"ae9f91a8",733:"83105561",786:"06482edd",860:"03a8b7a5",873:"e6b09d52",898:"998ef92b"}[e]+"-1.236.0.min.js"),i.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),e={},t="NRBA:",i.l=(r,n,o,a)=>{if(e[r])e[r].push(n);else{var s,c;if(void 0!==o)for(var u=document.getElementsByTagName("script"),d=0;d {s.onerror=s.onload=null,clearTimeout(h);var i=e[r];if(delete e[r],s.parentNode&&s.parentNode.removeChild(s),i&&i.forEach((e=>e(n))),t)return t(n)},h=setTimeout(l.bind(null,void 0,{type:"timeout",target:s}),12e4);s.onerror=l.bind(null,s.onerror),s.onload=l.bind(null,s.onload),c&&document.head.appendChild(s)}},i.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.j=364,i.p="https://js-agent.newrelic.com/",(()=>{var e={364:0,953:0};i.f.j=(t,r)=>{var n=i.o(e,t)?e[t]:void 0;if(0!==n)if(n)r.push(n[2]);else{var o=new Promise(((r,i)=>n=e[t]=[r,i]));r.push(n[2]=o);var a=i.p+i.u(t),s=new Error;i.l(a,(r=>{if(i.o(e,t)&&(0!==(n=e[t])&&(e[t]=void 0),n)){var o=r&&("load"===r.type?"missing":r.type),a=r&&r.target&&r.target.src;s.message="Loading chunk "+t+" failed.\n("+o+": "+a+")",s.name="ChunkLoadError",s.type=o,s.request=a,n[1](s)}}),"chunk-"+t,t)}};var t=(t,r)=>{var n,o,[a,s,c]=r,u=0;if(a.some((t=>0!==e[t]))){for(n in s)i.o(s,n)&&(i.m[n]=s[n]);if(c)c(i)}for(t&&t(r);u {i.r(o);var e=i(3325),t=i(5763);const r=Object.values(e.D);function n(e){const n={};return r.forEach((r=>{n[r]=function(e,r){return!1!==(0,t.Mt)(r,"".concat(e,".enabled"))}(r,e)})),n}var a=i(9144);var s=i(5546),c=i(385),u=i(8e3),d=i(5938),f=i(3960),l=i(50);class h extends d.W{constructor(e,t,r){let n=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];super(e,t,r),this.auto=n,this.abortHandler,this.featAggregate,this.onAggregateImported,n&&(0,u.R)(e,r)}importAggregator(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};if(this.featAggregate||!this.auto)return;const r=c.il&&!0===(0,t.Mt)(this.agentIdentifier,"privacy.cookies_enabled");let n;this.onAggregateImported=new Promise((e=>{n=e}));const o=async()=>{let t;try{if(r){const{setupAgentSession:e}=await Promise.all([i.e(860),i.e(242)]).then(i.bind(i,3228));t=e(this.agentIdentifier)}}catch(e){(0,l.Z)("A problem occurred when starting up session manager. This page will not start or extend any session.",e)}try{if(!this.shouldImportAgg(this.featureName,t))return void(0,u.L)(this.agentIdentifier,this.featureName);const{lazyFeatureLoader:r}=await i.e(412).then(i.bind(i,8582)),{Aggregate:o}=await r(this.featureName,"aggregate");this.featAggregate=new o(this.agentIdentifier,this.aggregator,e),n(!0)}catch(e){(0,l.Z)("Downloading and initializing ".concat(this.featureName," failed..."),e),this.abortHandler?.(),n(!1)}};c.il?(0,f.b)((()=>o()),!0):o()}shouldImportAgg(r,n){return r!==e.D.sessionReplay||!1!==(0,t.Mt)(this.agentIdentifier,"session_trace.enabled")&&(!!n?.isNew||!!n?.state.sessionReplay)}}var g=i(7633),p=i(7894);class m extends h{static featureName=g.t9;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];if(super(r,n,g.t9,i),("undefined"==typeof PerformanceNavigationTiming||c.Tt)&&"undefined"!=typeof PerformanceTiming){const n=(0,t.OP)(r);n[g.Dz]=Math.max(Date.now()-n.offset,0),(0,f.K)((()=>n[g.qw]=Math.max((0,p.z)()-n[g.Dz],0))),(0,f.b)((()=>{const t=(0,p.z)();n[g.OJ]=Math.max(t-n[g.Dz],0),(0,s.p)("timing",["load",t],void 0,e.D.pageViewTiming,this.ee)}))}this.importAggregator()}}var v=i(1117),b=i(1284);class y extends v.w{constructor(e){super(e),this.aggregatedData={}}store(e,t,r,n,i){var o=this.getBucket(e,t,r,i);return o.metrics=function(e,t){t||(t={count:0});return t.count+=1,(0,b.D)(e,(function(e,r){t[e]=w(r,t[e])})),t}(n,o.metrics),o}merge(e,t,r,n,i){var o=this.getBucket(e,t,n,i);if(o.metrics){var a=o.metrics;a.count+=r.count,(0,b.D)(r,(function(e,t){if("count"!==e){var n=a[e],i=r[e];i&&!i.c?a[e]=w(i.t,n):a[e]=function(e,t){if(!t)return e;t.c||(t=x(t.t));return t.min=Math.min(e.min,t.min),t.max=Math.max(e.max,t.max),t.t+=e.t,t.sos+=e.sos,t.c+=e.c,t}(i,a[e])}}))}else o.metrics=r}storeMetric(e,t,r,n){var i=this.getBucket(e,t,r);return i.stats=w(n,i.stats),i}getBucket(e,t,r,n){this.aggregatedData[e]||(this.aggregatedData[e]={});var i=this.aggregatedData[e][t];return i||(i=this.aggregatedData[e][t]={params:r||{}},n&&(i.custom=n)),i}get(e,t){return t?this.aggregatedData[e]&&this.aggregatedData[e][t]:this.aggregatedData[e]}take(e){for(var t={},r="",n=!1,i=0;i t.max&&(t.max=e),e 2&&void 0!==arguments[2])||arguments[2];super(e,r,j.t,n),c.il&&((0,t.OP)(e).initHidden=Boolean("hidden"===document.visibilityState),(0,N.N)((()=>(0,s.p)("docHidden",[(0,p.z)()],void 0,j.t,this.ee)),!0),(0,O.bP)("pagehide",(()=>(0,s.p)("winPagehide",[(0,p.z)()],void 0,j.t,this.ee))),this.importAggregator())}}var P=i(3081);class C extends h{static featureName=P.t9;constructor(e,t){let r=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(e,t,P.t9,r),this.importAggregator()}}var R,I=i(2210),k=i(1214),H=i(2177),L={};try{R=localStorage.getItem("__nr_flags").split(","),console&&"function"==typeof console.log&&(L.console=!0,-1!==R.indexOf("dev")&&(L.dev=!0),-1!==R.indexOf("nr_dev")&&(L.nrDev=!0))}catch(e){}function z(e){try{L.console&&z(e)}catch(e){}}L.nrDev&&H.ee.on("internal-error",(function(e){z(e.stack)})),L.dev&&H.ee.on("fn-err",(function(e,t,r){z(r.stack)})),L.dev&&(z("NR AGENT IN DEVELOPMENT MODE"),z("flags: "+(0,b.D)(L,(function(e,t){return e})).join(", ")));var M=i(6660);class B extends h{static featureName=M.t;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(r,n,M.t,i),this.skipNext=0;try{this.removeOnAbort=new AbortController}catch(e){}const o=this;o.ee.on("fn-start",(function(e,t,r){o.abortHandler&&(o.skipNext+=1)})),o.ee.on("fn-err",(function(t,r,n){o.abortHandler&&!n[M.A]&&((0,I.X)(n,M.A,(function(){return!0})),this.thrown=!0,(0,s.p)("err",[n,(0,p.z)()],void 0,e.D.jserrors,o.ee))})),o.ee.on("fn-end",(function(){o.abortHandler&&!this.thrown&&o.skipNext>0&&(o.skipNext-=1)})),o.ee.on("internal-error",(function(t){(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,o.ee)})),this.origOnerror=c._A.onerror,c._A.onerror=this.onerrorHandler.bind(this),c._A.addEventListener("unhandledrejection",(t=>{const r=function(e){let t="Unhandled Promise Rejection: ";if(e instanceof Error)try{return e.message=t+e.message,e}catch(t){return e}if(void 0===e)return new Error(t);try{return new Error(t+(0,D.P)(e))}catch(e){return new Error(t)}}(t.reason);(0,s.p)("err",[r,(0,p.z)(),!1,{unhandledPromiseRejection:1}],void 0,e.D.jserrors,this.ee)}),(0,O.m$)(!1,this.removeOnAbort?.signal)),(0,k.gy)(this.ee),(0,k.BV)(this.ee),(0,k.em)(this.ee),(0,t.OP)(r).xhrWrappable&&(0,k.Kf)(this.ee),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}onerrorHandler(t,r,n,i,o){"function"==typeof this.origOnerror&&this.origOnerror(...arguments);try{this.skipNext?this.skipNext-=1:(0,s.p)("err",[o||new F(t,r,n),(0,p.z)()],void 0,e.D.jserrors,this.ee)}catch(t){try{(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,this.ee)}catch(e){}}return!1}}function F(e,t,r){this.message=e||"Uncaught error with no additional information",this.sourceURL=t,this.line=r}let U=1;const q="nr@id";function G(e){const t=typeof e;return!e||"object"!==t&&"function"!==t?-1:e===c._A?0:(0,I.X)(e,q,(function(){return U++}))}function V(e){if("string"==typeof e&&e.length)return e.length;if("object"==typeof e){if("undefined"!=typeof ArrayBuffer&&e instanceof ArrayBuffer&&e.byteLength)return e.byteLength;if("undefined"!=typeof Blob&&e instanceof Blob&&e.size)return e.size;if(!("undefined"!=typeof FormData&&e instanceof FormData))try{return(0,D.P)(e).length}catch(e){return}}}var X=i(7243);class W{constructor(e){this.agentIdentifier=e,this.generateTracePayload=this.generateTracePayload.bind(this),this.shouldGenerateTrace=this.shouldGenerateTrace.bind(this)}generateTracePayload(e){if(!this.shouldGenerateTrace(e))return null;var r=(0,t.DL)(this.agentIdentifier);if(!r)return null;var n=(r.accountID||"").toString()||null,i=(r.agentID||"").toString()||null,o=(r.trustKey||"").toString()||null;if(!n||!i)return null;var a=(0,_.M)(),s=(0,_.Ht)(),c=Date.now(),u={spanId:a,traceId:s,timestamp:c};return(e.sameOrigin||this.isAllowedOrigin(e)&&this.useTraceContextHeadersForCors())&&(u.traceContextParentHeader=this.generateTraceContextParentHeader(a,s),u.traceContextStateHeader=this.generateTraceContextStateHeader(a,c,n,i,o)),(e.sameOrigin&&!this.excludeNewrelicHeader()||!e.sameOrigin&&this.isAllowedOrigin(e)&&this.useNewrelicHeaderForCors())&&(u.newrelicHeader=this.generateTraceHeader(a,s,c,n,i,o)),u}generateTraceContextParentHeader(e,t){return"00-"+t+"-"+e+"-01"}generateTraceContextStateHeader(e,t,r,n,i){return i+"@nr=0-1-"+r+"-"+n+"-"+e+"----"+t}generateTraceHeader(e,t,r,n,i,o){if(!("function"==typeof c._A?.btoa))return null;var a={v:[0,1],d:{ty:"Browser",ac:n,ap:i,id:e,tr:t,ti:r}};return o&&n!==o&&(a.d.tk=o),btoa((0,D.P)(a))}shouldGenerateTrace(e){return this.isDtEnabled()&&this.isAllowedOrigin(e)}isAllowedOrigin(e){var r=!1,n={};if((0,t.Mt)(this.agentIdentifier,"distributed_tracing")&&(n=(0,t.P_)(this.agentIdentifier).distributed_tracing),e.sameOrigin)r=!0;else if(n.allowed_origins instanceof Array)for(var i=0;i 2&&void 0!==arguments[2])||arguments[2];super(r,n,Z.t,i),(0,t.OP)(r).xhrWrappable&&(this.dt=new W(r),this.handler=(e,t,r,n)=>(0,s.p)(e,t,r,n,this.ee),(0,k.u5)(this.ee),(0,k.Kf)(this.ee),function(r,n,i,o){function a(e){var t=this;t.totalCbs=0,t.called=0,t.cbTime=0,t.end=E,t.ended=!1,t.xhrGuids={},t.lastSize=null,t.loadCaptureCalled=!1,t.params=this.params||{},t.metrics=this.metrics||{},e.addEventListener("load",(function(r){_(t,e)}),(0,O.m$)(!1)),c.IF||e.addEventListener("progress",(function(e){t.lastSize=e.loaded}),(0,O.m$)(!1))}function s(e){this.params={method:e[0]},T(this,e[1]),this.metrics={}}function u(e,n){var i=(0,t.DL)(r);i.xpid&&this.sameOrigin&&n.setRequestHeader("X-NewRelic-ID",i.xpid);var a=o.generateTracePayload(this.parsedOrigin);if(a){var s=!1;a.newrelicHeader&&(n.setRequestHeader("newrelic",a.newrelicHeader),s=!0),a.traceContextParentHeader&&(n.setRequestHeader("traceparent",a.traceContextParentHeader),a.traceContextStateHeader&&n.setRequestHeader("tracestate",a.traceContextStateHeader),s=!0),s&&(this.dt=a)}}function d(e,t){var r=this.metrics,i=e[0],o=this;if(r&&i){var a=V(i);a&&(r.txSize=a)}this.startTime=(0,p.z)(),this.listener=function(e){try{"abort"!==e.type||o.loadCaptureCalled||(o.params.aborted=!0),("load"!==e.type||o.called===o.totalCbs&&(o.onloadCalled||"function"!=typeof t.onload)&&"function"==typeof o.end)&&o.end(t)}catch(e){try{n.emit("internal-error",[e])}catch(e){}}};for(var s=0;s 1?e[1]=i:e.push(i)}else e[0]&&e[0].headers&&s(e[0].headers,n)&&(this.dt=n);function s(e,t){var r=!1;return t.newrelicHeader&&(e.set("newrelic",t.newrelicHeader),r=!0),t.traceContextParentHeader&&(e.set("traceparent",t.traceContextParentHeader),t.traceContextStateHeader&&e.set("tracestate",t.traceContextStateHeader),r=!0),r}}function x(e,t){this.params={},this.metrics={},this.startTime=(0,p.z)(),this.dt=t,e.length>=1&&(this.target=e[0]),e.length>=2&&(this.opts=e[1]);var r,n=this.opts||{},i=this.target;"string"==typeof i?r=i:"object"==typeof i&&i instanceof Y?r=i.url:c._A?.URL&&"object"==typeof i&&i instanceof URL&&(r=i.href),T(this,r);var o=(""+(i&&i instanceof Y&&i.method||n.method||"GET")).toUpperCase();this.params.method=o,this.txSize=V(n.body)||0}function A(t,r){var n;this.endTime=(0,p.z)(),this.params||(this.params={}),this.params.status=r?r.status:0,"string"==typeof this.rxSize&&this.rxSize.length>0&&(n=+this.rxSize);var o={txSize:this.txSize,rxSize:n,duration:(0,p.z)()-this.startTime};i("xhr",[this.params,o,this.startTime,this.endTime,"fetch"],this,e.D.ajax)}function E(t){var r=this.params,n=this.metrics;if(!this.ended){this.ended=!0;for(var o=0;o 2&&void 0!==arguments[2])||arguments[2];super(e,t,we.t,r),this.importAggregator()}}new class{constructor(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:(0,_.ky)(16);c._A?(this.agentIdentifier=t,this.sharedAggregator=new y({agentIdentifier:this.agentIdentifier}),this.features={},this.desiredFeatures=new Set(e.features||[]),this.desiredFeatures.add(m),Object.assign(this,(0,a.j)(this.agentIdentifier,e,e.loaderType||"agent")),this.start()):(0,l.Z)("Failed to initial the agent. Could not determine the runtime environment.")}get config(){return{info:(0,t.C5)(this.agentIdentifier),init:(0,t.P_)(this.agentIdentifier),loader_config:(0,t.DL)(this.agentIdentifier),runtime:(0,t.OP)(this.agentIdentifier)}}start(){const t="features";try{const r=n(this.agentIdentifier),i=[...this.desiredFeatures];i.sort(((t,r)=>e.p[t.featureName]-e.p[r.featureName])),i.forEach((t=>{if(r[t.featureName]||t.featureName===e.D.pageViewEvent){const n=function(t){switch(t){case e.D.ajax:return[e.D.jserrors];case e.D.sessionTrace:return[e.D.ajax,e.D.pageViewEvent];case e.D.sessionReplay:return[e.D.sessionTrace];case e.D.pageViewTiming:return[e.D.pageViewEvent];default:return[]}}(t.featureName);n.every((e=>r[e]))||(0,l.Z)("".concat(t.featureName," is enabled but one or more dependent features has been disabled (").concat((0,D.P)(n),"). This may cause unintended consequences or missing data...")),this.features[t.featureName]=new t(this.agentIdentifier,this.sharedAggregator)}})),(0,T.Qy)(this.agentIdentifier,this.features,t)}catch(e){(0,l.Z)("Failed to initialize all enabled instrument classes (agent aborted) -",e);for(const e in this.features)this.features[e].abortHandler?.();const r=(0,T.fP)();return delete r.initializedAgents[this.agentIdentifier]?.api,delete r.initializedAgents[this.agentIdentifier]?.[t],delete this.sharedAggregator,r.ee?.abort(),delete r.ee?.get(this.agentIdentifier),!1}}}({features:[J,m,S,class extends h{static featureName=oe;constructor(t,r){if(super(t,r,oe,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;const n=this.ee;let i;(0,k.QU)(n),this.eventsEE=(0,k.em)(n),this.eventsEE.on(se,(function(e,t){this.bstStart=(0,p.z)()})),this.eventsEE.on(ae,(function(t,r){(0,s.p)("bst",[t[0],r,this.bstStart,(0,p.z)()],void 0,e.D.sessionTrace,n)})),n.on(ce+ne,(function(e){this.time=(0,p.z)(),this.startPath=location.pathname+location.hash})),n.on(ce+ie,(function(t){(0,s.p)("bstHist",[location.pathname+location.hash,this.startPath,this.time],void 0,e.D.sessionTrace,n)}));try{i=new PerformanceObserver((t=>{const r=t.getEntries();(0,s.p)(te,[r],void 0,e.D.sessionTrace,n)})),i.observe({type:re,buffered:!0})}catch(e){}this.importAggregator({resourceObserver:i})}},C,xe,B,class extends h{static featureName=de;constructor(e,r){if(super(e,r,de,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;if(!(0,t.OP)(e).xhrWrappable)return;try{this.removeOnAbort=new AbortController}catch(e){}let n,i=0;const o=this.ee.get("tracer"),a=(0,k._L)(this.ee),s=(0,k.Lg)(this.ee),u=(0,k.BV)(this.ee),d=(0,k.Kf)(this.ee),f=this.ee.get("events"),l=(0,k.u5)(this.ee),h=(0,k.QU)(this.ee),g=(0,k.Gm)(this.ee);function m(e,t){h.emit("newURL",[""+window.location,t])}function v(){i++,n=window.location.hash,this[ve]=(0,p.z)()}function b(){i--,window.location.hash!==n&&m(0,!0);var e=(0,p.z)();this[pe]=~~this[pe]+e-this[ve],this[ye]=e}function y(e,t){e.on(t,(function(){this[t]=(0,p.z)()}))}this.ee.on(ve,v),s.on(be,v),a.on(be,v),this.ee.on(ye,b),s.on(ge,b),a.on(ge,b),this.ee.buffer([ve,ye,"xhr-resolved"],this.featureName),f.buffer([ve],this.featureName),u.buffer(["setTimeout"+le,"clearTimeout"+fe,ve],this.featureName),d.buffer([ve,"new-xhr","send-xhr"+fe],this.featureName),l.buffer([me+fe,me+"-done",me+he+fe,me+he+le],this.featureName),h.buffer(["newURL"],this.featureName),g.buffer([ve],this.featureName),s.buffer(["propagate",be,ge,"executor-err","resolve"+fe],this.featureName),o.buffer([ve,"no-"+ve],this.featureName),a.buffer(["new-jsonp","cb-start","jsonp-error","jsonp-end"],this.featureName),y(l,me+fe),y(l,me+"-done"),y(a,"new-jsonp"),y(a,"jsonp-end"),y(a,"cb-start"),h.on("pushState-end",m),h.on("replaceState-end",m),window.addEventListener("hashchange",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("load",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("popstate",(function(){m(0,i>1)}),(0,O.m$)(!0,this.removeOnAbort?.signal)),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}}],loaderType:"spa"})})(),window.NRBA=o})(); const cookieDomain = window.location.hostname; const tld = buildTLD(); function buildTLD() { const domainParts = cookieDomain.split('.'); while (domainParts.length > 2) { domainParts.shift(); } return domainParts.join('.'); } function deleteCookie(cookieName) { if (!cookieName) { return; } const cookiePath = (cookieName.startsWith('retraction_warning_') || cookieName.startsWith('version_warning_')) ? '/articles/' + cookieName : '/'; // Yes, these three slightly different ways to try and remove cookies are necessary document.cookie = cookieName + '=; path=' + cookiePath + '; domain=' + cookieDomain + '; expires=Thu, 01 Jan 1970 00:00:01 GMT;'; document.cookie = cookieName + '=; path=' + cookiePath + '; domain=.' + tld + '; expires=Thu, 01 Jan 1970 00:00:01 GMT;'; document.cookie = cookieName + '=; path=' + cookiePath + '; expires=Thu, 01 Jan 1970 00:00:01 GMT;'; } function deleteGroupCookies(group) { if (!group) { return; } const domainData = OneTrust.GetDomainData(), cookies = domainData.Groups.filter(cookieGroup => cookieGroup.OptanonGroupId === group)[0].Cookies; cookies.forEach(cookie => deleteCookie(cookie.Name)); } function OptanonWrapper() { const cookieConsentGroups = []; for (group of OneTrust.GetDomainData().Groups) { cookieConsentGroups.push(group.OptanonGroupId); } OneTrust.OnConsentChanged(function(e) { const cookieConsentActiveGroups = OnetrustActiveGroups.split(',').filter(activeGroup => activeGroup); cookieConsentGroups.forEach(group => { if (!cookieConsentActiveGroups.includes(group)) { deleteGroupCookies(group); }; }); }); } window.jQuery || document.write(' ') CKEDITOR_BASEPATH='https://hrbopenresearch.org/js/vendor/ckeditor/' window.reactTheme = 'HRB'; window.MathJax = { CommonHTML: { linebreaks: { automatic: true } }, 'HTML-CSS': { linebreaks: { automatic: true } }, SVG: { linebreaks: { automatic: true } }, AuthorInit: function() { MathJax.Hub.Register.MessageHook('End Process', function () { let timeout = false; // holder for timeout id const delay = 250; // delay after event is "complete" to run callback const reflowMath = function() { const dispFormulas = document.querySelectorAll('.disp-formula.panel'); if (!dispFormulas) { return; } for (const dispFormula of dispFormulas) { const child = dispFormula.querySelector('.MathJax_Preview').nextSibling.firstChild; const isMultiline = MathJax.Hub.getAllJax(dispFormula)[0].root.isMultiline; if (dispFormula.offsetWidth < child.offsetWidth || isMultiline) { MathJax.Hub.Queue(['Rerender', MathJax.Hub, dispFormula]); } } }; window.addEventListener('resize', function() { clearTimeout(timeout); // clear the timeout timeout = setTimeout(reflowMath, delay); // start timing for event "completion" }); }); }, }; if (window.location.hash == '#_=_'){ window.location = window.location.href.split('#')[0] } !function(f,b,e,v,n,t,s){if(f.fbq)return;n=f.fbq=function() {n.callMethod? n.callMethod.apply(n,arguments):n.queue.push(arguments)} ;if(!f._fbq)f._fbq=n; n.push=n;n.loaded=!0;n.version='2.0';n.queue=[];t=b.createElement(e);t.async=!0; t.src=v;s=b.getElementsByTagName(e)[0];s.parentNode.insertBefore(t,s)}(window, document,'script','https://connect.facebook.net/en_US/fbevents.js'); fbq('init', '1641728616063202'); fbq('track', "PixelInitialized", {}); Skip to content HRB Open Research file_upload Submit your research search clear search menu close clear Search Browse Gateways & Collections How to Publish Submit your Research My Submissions Article Guidelines Article Guidelines (New Versions) Data Guidelines Prepublication Checks Production Process Article Processing Charges Finding Article Reviewers About How it Works For Reviewers National Steering Group Policies Glossary FAQs Contact Blog My Account Submissions Content and Tracking Alerts My Details Sign In file_upload Submit your research { "@context": "https://schema.org", "@type": "ScholarlyArticle", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://hrbopenresearch.org/articles/8-54" }, "headline": "Training and Experience in Study Selection (TESS): study protocol for a pilot randomised trial within a...", "datePublished": "2025-04-14T08:43:40", "dateModified": "2026-04-16T15:58:53", "author": [ { "@type": "Person", "name": "Elayne Ahern" }, { "@type": "Person", "name": "Temitayo Adedeji" }, { "@type": "Person", "name": "Aoife Whiston" }, { "@type": "Person", "name": "Sarah Dillon" }, { "@type": "Person", "name": "Fiona Lynn" } ], "publisher": { "@type": "Organization", "name": "HRB Open Research", "logo": { "@type": "ImageObject", "url": "https://hrbopenresearch.org/img/AMP/HRB_image.png", "height": 566, "width": 60 } }, "image": { "@type": "ImageObject", "url": "https://hrbopenresearch.org/img/AMP/HRB_image.png", "height": 1200, "width": 127 }, "description": " Background Systematic reviews can be resource-intensive and require timely completion, yet experienced reviewers may have limited availability, necessitating the inclusion of novice screeners. This Study Within A Review (SWAR) will pilot and examine the feasibility of study methods to explore whether training and level of experience within the screening pair affects the reliability of decisions made by novice screeners during study selection. Methods A 2(training: task-specific, minimal guidance) x 2(experience level of screening partner, ‘Reviewer 1’: moderate experience, minimal experience) pilot randomised trial will be conducted within a host systematic review in the topic area of depression and psychosocial functioning. Participants (N = 12), consisting of higher education students with no prior experience in evidence synthesis, will be randomised to one of the four conditions to complete a standardised study selection task at title/abstract level (k = 219 records) on Covidence screening software, blindly and independently. Total participation time is estimated at 5 hours. Screening decisions made by participants will be assessed for reliability against the consensus-based decisions by two reviewers with content and methodological expertise (expert standard), through calculation of chance-corrected Cohen’s kappa, prevalence-adjusted bias-adjusted kappa (PABAK), and percentage of agreement, then compared across the conditions. Secondary outcomes will include reliability within the screening pair (participant and allocated screening partner), validity of screening decisions (false positives, false negatives, sensitivity, specificity), feasibility measures, including time taken to complete the study selection task and success of blinding, as well as acceptability. Conclusions Findings will be used to inform the design of subsequent trial work to determine the efficacy of training and screener pairing for study selection. Ultimately, these insights will help to build capacity among novice screeners to engage with evidence synthesis and work alongside experienced review teams. Registration Northern Ireland Hub for Trials Methodology Research SWAR Registry: SWAR 38. " } { "@context": "http://schema.org", "@type": "BreadcrumbList", "itemListElement": [ { "@type": "ListItem", "position": "1", "item": { "@id": "https://hrbopenresearch.org/", "name": "Home" } }, { "@type": "ListItem", "position": "2", "item": { "@id": "https://hrbopenresearch.org/browse/articles", "name": "Browse" } }, { "@type": "ListItem", "position": "3", "item": { "@id": "https://hrbopenresearch.org/articles/8-54", "name": "Training and Experience in Study Selection (TESS): study protocol..." } } ] } Home Browse Training and Experience in Study Selection (TESS): study protocol... ALL Metrics - Views Downloads Get PDF Get XML Cite How to cite this article Ahern E, Adedeji T, Whiston A et al. Training and Experience in Study Selection (TESS): study protocol for a pilot randomised trial within a systematic review [version 3; peer review: 2 approved] . HRB Open Res 2026, 8 :54 ( https://doi.org/10.12688/hrbopenres.14129.3 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. Close Copy Citation Details Export Export Citation Sciwheel EndNote Ref. Manager Bibtex ProCite Sente EXPORT Select a format first Track Share ▬ ✚ Study Protocol Revised Training and Experience in Study Selection (TESS): study protocol for a pilot randomised trial within a systematic review [version 3; peer review: 2 approved] Elayne Ahern https://orcid.org/0000-0001-9230-6776 1,2 , Temitayo Adedeji https://orcid.org/0009-0006-8630-8066 1 , Aoife Whiston https://orcid.org/0000-0002-0300-5910 1,2 , Sarah Dillon https://orcid.org/0000-0002-6659-2606 2,3 , Fiona Lynn https://orcid.org/0000-0002-0216-643X 4 Elayne Ahern https://orcid.org/0000-0001-9230-6776 1,2 , Temitayo Adedeji https://orcid.org/0009-0006-8630-8066 1 , [...] Aoife Whiston https://orcid.org/0000-0002-0300-5910 1,2 , Sarah Dillon https://orcid.org/0000-0002-6659-2606 2,3 , Fiona Lynn https://orcid.org/0000-0002-0216-643X 4 PUBLISHED 16 Apr 2026 Author details Author details 1 Department of Psychology, University of Limerick, Castletroy, Limerick, V94 T9PX, Ireland 2 Health Research Institute, University of Limerick, Castletroy, Limerick, V94 T9PX, Ireland 3 School of Allied Health, University of Limerick, Castletroy, Limerick, V94 T9PX, Ireland 4 School of Nursing and Midwifery, Queen's University Belfast, Belfast, BT9 7BL, UK Elayne Ahern Roles: Conceptualization, Data Curation, Funding Acquisition, Methodology, Project Administration, Resources, Supervision, Visualization, Writing – Original Draft Preparation, Writing – Review & Editing Temitayo Adedeji Roles: Methodology, Project Administration, Resources, Writing – Original Draft Preparation, Writing – Review & Editing Aoife Whiston Roles: Conceptualization, Funding Acquisition, Methodology, Resources, Writing – Review & Editing Sarah Dillon Roles: Conceptualization, Funding Acquisition, Methodology, Resources, Writing – Review & Editing Fiona Lynn Roles: Conceptualization, Funding Acquisition, Methodology, Writing – Review & Editing OPEN PEER REVIEW DETAILS REVIEWER STATUS Abstract Background Systematic reviews can be resource-intensive and require timely completion, yet experienced reviewers may have limited availability, necessitating the inclusion of novice screeners. This Study Within A Review (SWAR) will pilot and examine the feasibility of study methods to explore whether training and level of experience within the screening pair affects the reliability of decisions made by novice screeners during study selection. Methods A 2(training: task-specific, minimal guidance) x 2(experience level of screening partner, ‘Reviewer 1’: moderate experience, minimal experience) pilot randomised trial will be conducted within a host systematic review in the topic area of depression and psychosocial functioning. Participants ( N = 12), consisting of higher education students with no prior experience in evidence synthesis, will be randomised to one of the four conditions to complete a standardised study selection task at title/abstract level ( k = 219 records) on Covidence screening software, blindly and independently. Total participation time is estimated at 5 hours. Screening decisions made by participants will be assessed for reliability against the consensus-based decisions by two reviewers with content and methodological expertise (expert standard), through calculation of chance-corrected Cohen’s kappa, prevalence-adjusted bias-adjusted kappa (PABAK), and percentage of agreement, then compared across the conditions. Secondary outcomes will include reliability within the screening pair (participant and allocated screening partner), validity of screening decisions (false positives, false negatives, sensitivity, specificity), feasibility measures, including time taken to complete the study selection task and success of blinding, as well as acceptability. Conclusions Findings will be used to inform the design of subsequent trial work to determine the efficacy of training and screener pairing for study selection. Ultimately, these insights will help to build capacity among novice screeners to engage with evidence synthesis and work alongside experienced review teams. Registration Northern Ireland Hub for Trials Methodology Research SWAR Registry: SWAR 38. READ ALL READ LESS Keywords higher education, novice, interrater reliability, study selection, study within a review, SWAR, systematic review, training Corresponding Author(s) Elayne Ahern ( [email protected] ) Close Corresponding author: Elayne Ahern Competing interests: No competing interests were disclosed. Grant information: This study was supported by the Health Research Board (Ireland) and the HSC Public Health Agency [ESI-2021-001] through Evidence Synthesis Ireland/Cochrane Ireland. The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript. Copyright: © 2026 Ahern E et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. How to cite: Ahern E, Adedeji T, Whiston A et al. Training and Experience in Study Selection (TESS): study protocol for a pilot randomised trial within a systematic review [version 3; peer review: 2 approved] . HRB Open Res 2026, 8 :54 ( https://doi.org/10.12688/hrbopenres.14129.3 ) First published: 14 Apr 2025, 8 :54 ( https://doi.org/10.12688/hrbopenres.14129.1 ) Latest published: 16 Apr 2026, 8 :54 ( https://doi.org/10.12688/hrbopenres.14129.3 ) Revised Amendments from Version 2 Summary of revisions made to the article following peer review (Version 2): - Clarified the purpose and planned handling of open-ended text responses in the Analysis section, specifying that the responses will be reviewed descriptively, with illustrative quotes where appropriate, to inform the refinement of the intervention and subsequent trial design, rather than undergoing formal qualitative analysis. Summary of revisions made to the article following peer review (Version 2): - Clarified the purpose and planned handling of open-ended text responses in the Analysis section, specifying that the responses will be reviewed descriptively, with illustrative quotes where appropriate, to inform the refinement of the intervention and subsequent trial design, rather than undergoing formal qualitative analysis. See the authors' detailed response to the review by Heather MacDonald See the authors' detailed response to the review by Declan Devane READ REVIEWER RESPONSES Background Systematic reviews have gained prominence with a 20-fold increase in the number of systematic reviews produced from 2009–2019 ( Hoffmann et al ., 2021 ). This type of review involves a rigorous synthesis of studies on a chosen topic, aiming to assess the current state of knowledge and identify gaps in the literature through a systematic approach from literature searching, through to study selection, data extraction, quality appraisal or assessment and, finally, synthesis ( Siddaway et al ., 2019 ). For this reason, systematic reviews are highly regarded as a basis for evidence-based policy and practice recommendations. However, systematic reviews require time and significant human resources for completion due to their comprehensive and rigorous approach to identifying and synthesising all relevant literature in a given topic area ( Borah et al ., 2017 ). This can present as a barrier to timely completion and necessitates that consideration is given towards how to improve efficiencies in the process, whilst also maintaining high quality standards. Specifically, study selection is a resource-intensive step in the process. As per international best practice recommendations ( Edwards et al ., 2002 ; Waffenschmidt et al ., 2019 ), study selection should involve at least two independent reviewers to ‘screen’ study records, and thus can be referred to as ‘screeners’ for this specific stage of the systematic review ( Stoll et al ., 2019 ). By accurately removing records at the earliest stages, study selection is key to efficiency in the process. Moreover, study selection is integral to ensuring that relevant studies proceed through to inclusion in the systematic review, supporting a complete and valid synthesis of evidence relevant to the review question ( Polanin et al ., 2019 ). The use of two screeners offers several benefits, such as ensuring consistent application of the eligibility criteria (i.e. determining which records should be included or excluded in order to address the review question) to avoid any systematic errors as well as reducing the likelihood of any random errors by one screener compromising the accurate selection of records ( Waffenschmidt et al ., 2019 ). Having at least two, independent screeners on each study record helps to ensure a transparent and reproducible method where the eligibility criteria have been applied appropriately and consistently throughout, as agreed by both screeners ( Edwards et al ., 2002 ). This can be evaluated through the calculation of interrater reliability, or a chance-corrected estimate known as Cohen’s kappa ( Cohen, 1960 ), which is more conservative and accounts for chance, spurious agreement. However, the recommendation of two or more reviewers to screen each study record is not always feasible. Researchers with content and methodological expertise ( Lasserson et al ., 2021 ) may not be available to complete vast amounts of screening for study selection within defined project periods. Furthermore, systematic reviews are often constrained by tight deadlines and limited budgets, which can hinder their completion within the required timeframe ( Waffenschmidt et al ., 2019 ). Consequently, systematic review teams may comprise of not only experienced researchers ( Lasserson et al ., 2021 ) but also more novice team members such as student supervisees who undertake research as part of their programme of study (e.g., Tendal et al ., 2009 ). Although rapid review methods present as a solution, allowing for a timely response to inform evidence-based decision making, these methods can be deemed responsive to situational constraints and the urgency for the evidence ( Moons et al ., 2021 ). For example, only one database may be searched or study selection undertaken by just one screener. Rapid reviews, although adhering to core standards of systematic review ( Garritty et al ., 2021 ; Nussbaumer-Streit et al ., 2023 ), are not a substitute for the comprehensiveness and rigour of the full systematic review process. Thus, understanding how efficiencies can be achieved in the systematic review process is still of utmost interest. Study selection methods are relatively under explored ( k = 11; Robson et al ., 2019 ) compared to other steps in the systematic review process such as data extraction and study appraisal. Current guidance primarily focuses on the study selection procedure itself (e.g., single vs. dual screening, Gartlehner et al ., 2020 ; Waffenschmidt et al ., 2019 ; titles-first vs. title and abstract, Mateen et al ., 2013 ), while practical guidelines on the composition of the screening team, particularly in terms of experience level, have received little attention. The few studies that address this, such as Cooper et al . (2006) and Ng et al . (2014) , have found that the experience level of screeners can impact performance during study selection. Cooper et al. (2006) , for example, examined how novice screeners, either professional dieticians or graduate nutrition students, compare with an expert standard (PhD trained nutritional scientists) during study selection for records regarding the diet-disease relationship. Findings indicated that interrater reliability within the pairs of professionals or pairs of students did not significantly differ, nonetheless, important differences between these pairs with the expert standard were observed. Novice screener pairs, whether professionals or students, may remove potentially eligible studies during title and abstract screening and thus impact the accuracy of study selection. This is further underscored by Ng et al . (2014) who found that study selection undertaken by medical students with no prior experience in systematic reviews varied to a large degree, with only a modest level of agreement (sensitivity) with the experienced reviewer. Further, no studies have been identified on the role of training to improve study selection outcomes among novice screeners, although training has been assessed for other steps of the systematic review process such as quality appraisal (e.g., Acosta et al ., 2020 ; da Costa et al ., 2017 ; Oremus et al ., 2012 ). For example, Acosta et al. (2020) explored the effect of a 2-hour structured training on reducing errors and bias during the quality appraisal stage of a systematic review among 14 doctoral students, with no prior experience in systematic reviews. This training included a practice activity where the novices compared their scores across the nine criteria on the Methodological Quality Questionnaire for a given study record to that of the expert, discussing hits (agreements) and misses (disagreements) for the practice activity. The student reviewers were randomly assigned one of three study records of varying quality for appraisal. Following independent completion of the study quality appraisal task, and when compared to expert raters, 43% of the novice student reviewers accurately assessed 7 out of 9 (78%) methodological quality criteria when evaluating the quality of the study record. Training-based benefits have not always been observed among reviewers with already established expertise ( Fourcade et al ., 2007 ) or even among novice student reviewers ( Oremus et al ., 2012 ) during the quality assessment phase. Nonetheless, when formats of training (e.g., intensive vs. minimal) have been investigated among novice student reviewers, findings have suggested that the content of training could influence the likelihood of positive training effects. da Costa et al . (2017) examined minimal training, consisting of an introductory 1-hour lecture on risk of bias assessment, with intensive training, which included the same lecture but with specific assessment instructions provided by an experienced reviewer (vs. reading material provided to the minimal training condition). Further, the student reviewers who received intensive training then undertook a practice assessment in a purposely selected sample of 10 records, with assessment decisions then discussed by the experienced reviewer. Overall, this produced more favourable kappa estimates for both within- and between-group interrater reliability, as novice screeners who received the intensive training were afforded an opportunity to calibrate their assessments to that of the experienced reviewer during the practice task. This underscores the value of specific training in enhancing the ability of novice reviewers to engage with systematic review processes. Alongside the role of training, there is reason to believe that the pairing of novice screeners with more experienced review team members when undertaking study selection could help improve reliability estimates. As evidenced by da Costa et al . (2017) , interrater reliability estimates can be improved when the opportunity is afforded to novice reviewers to align their decisions with that of the experienced reviewer. During study selection, the use of software such as Covidence (Veritas Health Innovation) can help to streamline the process among screeners and promptly notify of conflict (disagreement) detection between screeners, offering an opportunity for novice screeners to subsequently adjust and re-calibrate their performance. As such, the experience level of the screening partner is also of interest. To the best of our knowledge, studies that have examined novice reviewer performance in dual assessment tasks have only ever done so investigating novice-novice pairings (e.g., da Costa et al ., 2017 ; Oremus et al ., 2012 ). Although experience level has not always been associated with more valid and reliable performance during systematic review processes (e.g. Horton et al ., 2010 ; data extraction), it is plausible that novice reviewers could use conflict detection to recognise patterns of disagreements during study selection as a result of exposure to more expert decision-making and adjust their approach for subsequent study records accordingly. As per guidelines, explicit discussions should be had within the review team to resolve conflicts and consolidate the rationale behind the decision to include or exclude a study record ( Lefebvre et al ., 2024 ). Nonetheless, the active pairing of a novice reviewer with a more experienced reviewer could offer a referential standard through real-time feedback during the screening process, helping to overall improve efficiency. Real-time feedback on conflict detection during dual screening with an experienced reviewer could help to consolidate the impact of training, however, this has not yet been investigated. Aim and objectives Systematic reviews are considered the highest form of evidence, guiding policy and practice through the transparent and rigorous synthesis of research ( Haddaway & Pullin, 2014 ). The reliability of study selection is integral to this process, particularly when review teams include more novice screeners. Inconsistent selection processes can directly influence which studies are included, ultimately shaping the available evidence base upon which recommendations are developed. The aim of this pilot study is to assess the feasibility and acceptability of the proposed methods to plan for a definitive trial, with a view to exploring whether training and level of experience within the screening pair affects the reliability of decisions made by novice screeners during study selection for a systematic review. Drawing on the Study Within A Review (SWAR; Devane et al ., 2022 ) methodology, where a study is embedded within a systematic review in order to address questions of methodological uncertainty during the review process, we propose the following objectives in preparation for a future definitive trial: i). To explore whether experience level of the screening partner and the provision of training affects the reliability of screening decisions made by novice, student screeners (between-group reliability). ii). To explore whether pairing a novice, student screener with a more experienced screener (vs. novice screener), alongside the provision of task-specific training (vs. minimal guidance), influences reliability estimates within the screening pair during the study selection process (within-group reliability). iii). To explore the feasibility and acceptability of study procedures and training materials for study selection by novice, student screeners. Specifically, we anticipate that reliability in study selection will be highest among the group of novice screeners that receive task-specific training and are paired with a more experienced screener. However, any comparisons are exploratory and will be used to inform the design of parameters for future trial work rather than formally test hypotheses. Methods This pilot study will be conducted in accordance with SWAR guidance ( Devane et al ., 2022 ) and is pre-registered on the Northern Ireland Network for Trials Methodology Research SWAR Repository Store (SWAR 38; see ‘Extended Data’ section below, Ahern et al ., 2025 ), with notice of acceptance on 4 February 2025. The protocol is reported according to SPIRIT guidelines for trial protocols (Standard Protocol Items: Recommendations for Interventional Trials; Chan et al ., 2013 ), and supplemented by the CONSORT extension to pilot trials ( Eldridge et al ., 2016 ), as is recommended for the reporting of pilot and feasibility study protocols ( Thabane & Lancaster, 2019 ). See the completed SPIRIT and CONSORT extension checklists, available from the OSF link provided in the ‘Reporting Guidelines’ section below. Any modifications to the protocol will be recorded and subsequently presented in the pilot study report. As the primary aim is to assess feasibility and acceptability of the proposed methods to inform the design of a subsequent fully powered trial, pre-defined progression criteria will be used to guide decisions about whether the design is suitable to scale up for a definitive trial, or whether modifications are required before such can be undertaken. See ‘Progression criteria’ section. Study design This SWAR is a pilot study that will employ a 2x2 factorial randomised controlled trial (RCT) design. Participants will be randomised using a random number generator for simple randomisation to one of four conditions: (i) task-specific training with a novice screening partner, (ii) task-specific training with a moderately experienced screening partner, (iii) minimal guidance training with a novice screening partner, and (iv) minimal guidance training with a moderately experienced screening partner (see Figure 1 ). A member of the research team (AW) has computer-generated the randomisation sequence (conditions A, B, C, D) in advance of recruitment and will be kept blind to treatment allocation as well as data analysis. Recruitment and onboarding of participants will be separately managed by an unblinded research assistant coordinator (TA), who following participant informed consent, will contact the respective team member that handles randomisation to determine the next pre-determined allocation in the sequence. Members of the team that will be involved in data analysis (EA) will remain blinded to allocation to ensure minimal risk of bias to data analysis. Participants will remain blinded to their allocated group throughout the study. Ethical approval for this study has been obtained from the Faculty of Education and Health Sciences Research Ethics Committee at the University of Limerick (EHS Approval Number: 2024_06_24_EHS). Figure 1. Flow diagram of randomisation to the respective pilot study conditions. Figure Note. Participants ( N = 12) will be randomised to a training condition (task-specific or minimal guidance) and screening partner (novice screener or moderately experienced screener). The consensus-based decisions of the expert reviewers (E1, E2) will inform the expert reference standard. E1 = Expert reviewer 1 with content and methodological expertise; E2 = Expert reviewer 2 with content and methodological expertise. Study selection task Study records. An on-going systematic review project led by the PI (PROSPERO registration: CRD42022324367) will be used to host the SWAR. This host review addresses non-pharmacological treatments for psychosocial functioning in major depressive disorder. Please see ‘Extended Data’ ( Ahern et al ., 2025 ) for a sample database search strategy. A comprehensive search of the literature has been undertaken across 6 databases, identifying in excess of 12,500 potentially eligible study records from inception through to the most recent updated search in August 2024. Based on calculations using the kappaSize package ( Rotondi, 2018 ) and PowerBinary function on r, 219 records were determined for the study selection task in order to achieve 80% power to detect a significant difference in reliability assuming kappa = 0.51 (weak/fair agreement, McHugh, 2012 ; Orwin, 1994 ) between the expert standard and the minimal guidance training+novice screening partner condition relative to kappa = 0.71 (moderate/good agreement, McHugh, 2012 ; Orwin, 1994 ) between the expert standard and task-specific training+moderately experienced screening partner condition, with an alpha level of .05, and assuming a 20% prevalence of include at title/abstract level. Consistent with other SWAR methodologies assessing rater agreement ( Cooper et al ., 2006 ; da Costa et al ., 2017 ), power calculations are used to determine the adequate number of study records that need to be assessed to ensure the calculation of reliable kappa statistics. In accordance with power calculations, a sample of 2% ( k = 219) from the database search returns has been identified for use in the study selection task for this SWAR. Selection of these records was undertaken by the PI to ensure representation across the breadth of the host review’s eligibility criteria. Firstly, the records from the host review that were ‘includes’ following independent screening by two reviewers at the title/abstract level were reviewed. Of these, a sample of ‘includes’ were selected ( k = 44). The remaining records for the study selection task ( k = 175) were drawn from those screened as ‘exclude’ with the intention of covering the full spectrum of reasons for exclusion in the host review. This ensured participants would encounter a wide array of decisions as would be typical of any review project. Further, variation in the exclusion rationales that needed to be applied during study selection was intended to support a meaningful assessment of potential differences across conditions. Participants would need to correctly interpret and be able to apply all eligibility criteria, not just a select few, to produce reliable decisions. Prior to finalising the selection of records to be used for the screening task, a pilot sample of k = 20 records were selected for screening within the review team (EA, AW), with the rationale for inclusion/exclusion documented. Interrater reliability, that is both reviewers agreeing ‘YES’ to include or ‘NO’ to exclude to each respective study record, was 100% (κ = 1.00) suggesting perfect agreement ( Cohen, 1960 ; McHugh, 2012 ). The reference list of study records for the study selection task are available on the project OSF, see ‘Extended Data’ ( Ahern et al ., 2025 ). Expert standard. All study records have been screened independently by two members of the review team (EA, AW) with several years of methodological experience in evidence synthesis and content expertise in the area of depression ( M = 9.5, SD = 1.41 years of experience). As per the criteria proposed by Horton et al . (2010) , EA and AW would meet the standards of ‘substantial experience’ in systematic review (>7 years involved in systematic reviews, >7 reviews for which involved in at least study selection). Screening has been completed on separate Covidence project sites to ensure complete independence in decision making. Any screening discrepancies have been resolved through discussion with the consensus-based decisions then used as the expert standard. The interrater reliability was 93.15% agreement (κ = 0.77, SE = 0.06, 95% CI[0.66, 0.88]). Screening partner (‘Reviewer 1’). A novice member of the review team (TA) with ‘minimal experience’ (≤2 years involved in systematic reviews, ≤2 systematic reviews for which involved in at least study selection) and another member (SD) with ‘moderate experience’ in systematic review methodology (4–6 years involved in systematic reviews, 4–6 systematic reviews for which involved in at least study selection), as per the classifications proposed by Horton et al . (2010) , have independently completed screening for the selected records on separate Covidence project sites to avoid influencing their respective screening decisions. The screening decisions made by TA and SD will then be used for the ‘Screening Partner’ condition, corresponding to novice and moderately experienced screening partner, respectively. The screening decisions have been loaded on to Covidence project sites in advance and described as the screening decisions made by ‘Reviewer 1’. Participants will be randomly allocated to one of two screening partners in combination with a training condition (task-specific, minimal guidance; See ‘Training’ section below). Participants A total of 12 participants (N = 12) will be recruited within an anticipated 3-month timeframe. Recruitment commenced in February 2025. As this SWAR is designed as a pilot study, power analyses were not used to determine sample size and it was deemed appropriate to recruit a small number of participants per trial condition, consistent with other SWAR methodologies (e.g., Acosta et al ., 2020 ; Cooper et al ., 2006 ; da Costa et al ., 2017 ; McGuire et al ., 1985 ; Oremus et al ., 2012 ). Further, the small sample size is appropriated given that this pilot trial is not intended for hypothesis-testing but more so for exploration of hypotheses, and to assess the feasibility and acceptability of the proposed methods to inform the design of a subsequent fully powered trial. Participants will be recruited from higher education institutions with convenience sampling of health sciences students from the University of Limerick in the first instance given the small target sample size and the access afforded to the research team given roles in teaching at undergraduate and postgraduate level within the discipline. Information regarding the study will be disseminated through course site postings on the virtual learning environment, with word-of-mouth and snowball sampling methods also employed. The eligibility criteria for participation include: (i) being at least 18 years of age; (ii) being enrolled as a student in higher education; (iii) having no prior training or experience in conducting evidence synthesis; and (iv) having access to a laptop or computer with a reliable internet connection. Participants will be entered into a draw to win a monetary prize (2 x €50 vouchers) as renumeration for their participation. Training conditions Both the task-specific training and minimal guidance training conditions will receive a standardised outline of eligibility criteria for study selection (see ‘Extended Data’, Ahern et al ., 2025 ). Additionally, a recorded online training with narration, embedded video content, and slide deck will be provided, but the extent of training will differ between the conditions (task-specific vs. minimal guidance). Covidence Support (Veritas Health Innovation) provides a bank of online training videos which have served as a point of reference in developing the content for this session and instructional guide on use of the software. The content of the training has been piloted within the review team before finalisation. Both the training and control have been time-matched for an estimated completion time of 45 minutes, consisting of the following core content: an introduction to the review topic area, what is a systematic review, study selection and eligibility criteria, an introduction to Covidence systematic review software, guidelines for study selection, and instructions for the study selection task. Control (minimal guidance training). In addition to the core training content, the control comparator training session includes extended content on what is a systematic review, considering when to or when not to conduct a systematic review and instruction on some reporting guidelines, such as the PRISMA flow diagram to visualise the study selection process. Specifically, the content relevant to study selection and Covidence is generalised and not specific to the host review. Intervention (task-specific training). In addition to the core training content, an extended discussion on eligibility criteria for the host review is provided, offering clarification and recommendations when screening, followed by a demonstration of screening for a sample of 10 records in the review topic area, not included in the sample of records for the study selection task, with clear rationale provided for the screening decisions made. Similar online training approaches with a sample demonstration of records, reflecting what is likely to be encountered during screening, have been implemented for novice screeners (‘crowd,’ non-specialist) on systematic review projects ( Noel-Storr et al ., 2021 ). The purpose of this training is to refine knowledge and understanding specific to engaging with study selection for the host review. For ease of access, all details will be hosted on a virtual learning environment project site, separate for each training condition. A sample of the project site interface is available from the project OSF (see ‘Extended Data’, Ahern et al ., 2025 ). Across both the task-specific and minimal guidance conditions, participants will start at the first unit, ‘TESS Study: Introduction’, followed by ‘The Research Team’, and then proceed through Steps 1–5 accordingly: (1) ‘Demographic and Background Questionnaire’, (2) ‘Training Materials’, (3) ‘Access the Study Selection Task’, (4) ‘The Study Selection Task’, (5) ‘Post-Task Questionnaire’. Measures and outcomes Interrater reliability. All relevant performance outcome data will be recorded on Covidence systematic review screening software. Participants will be instructed to select between two options, ‘YES’ (include) and ‘NO’ (exclude). This will allow for the calculation of an unweighted, chance-corrected Cohen’s kappa ( Cohen, 1960 ) as the response decisions are dichotomous. Based on the criteria set out by McHugh (2012) , Cohen’s kappa values can be interpreted as no agreement (κ = ≤ 0.20), minimal (κ = 0.21–0.39), weak (κ = 0.40–0.59), moderate (κ = 0.60–0.79), strong (κ = 0.80–0.90), or almost perfect (κ > 0.90). Furthermore, the percentage agreement will be reported alongside Cohen’s (1960) kappa, as per recommendations ( McHugh, 2012 ) given that the kappa statistic can underestimate reliability due to the underlying assumptions made on chance rater agreement. Primary and secondary outcomes are: Primary (i) between-group reliability, defined as the reliability between the decisions of the participants and the expert standard, calculated as unweighted, chance-corrected Cohen’s kappa ( Cohen, 1960 ), the prevalence-adjusted bias-adjusted kappa (PABAK; Byrt et al. , 1993 ), and percentage agreement. Between-group Cohen’s kappa will be calculated as, κ ns, expert standard = (P o1 – P e1 ) / (1 − P e1 ), where P o1 is the proportion of records screened when the decision by the participant novice screener is in agreement with the expert standard, and P e1 is the proportion of records where the decision by the participant novice screener is in agreement with the expert standard by chance. As Cohen’s kappa is sensitive to prevalence and can underestimate reliability, such as when there are low numbers of ‘includes’ during study selection, PABAK will also be calculated to provide a more meaningful and stable estimate using the formula 2P o1 –1, where 2 is indicative of the number of response categories (i.e. ‘Yes’ or ‘No’ to include the study record; Byrt et al. , 1993 ). The associated mean between-group reliability estimate, standard error, and 95% CI will be generated for each of the four trial conditions (e.g., task-specific training and moderately experienced screening partner; minimal guidance and novice screening partner) to enable exploration of differences across the conditions. Secondary (i) within-group reliability, defined as the reliability within each pair of participant and screening partner, calculated as unweighted, chance-corrected Cohen’s kappa, PABAK, and percent agreement. Within-group Cohen’s kappa will be calculated as κ ns, screening partner = (P o2 – P e2 ) / (1− P e2 ), where P o2 is the proportion of records screened when the decision by the participant novice screener is in agreement with their allocated screening partner, and P e2 is the proportion of records where the decision by the participant novice screener is in agreement by chance. Within-group PABAK will be calculated as 2P o2 –1. The associated mean within-group reliability estimate, standard error, and 95% CI will be generated to allow for exploration of differences across conditions. (ii) the validity of screening decisions made by participants relative to the expert standard, including the total number of false positives (vote ‘YES’ by participant to include irrelevant articles), false negatives (vote ‘NO’ by participant to exclude relevant articles), sensitivity (ability of participant to vote ‘YES’ to include relevant articles), and specificity (ability of participant to vote ‘NO’ to exclude irrelevant articles). Several measures will be collected from participants through use of the online survey software tool, Qualtrics. Sociodemographic, education, and experience level. A Demographic and Background Questionnaire will be administered at baseline to collect data relevant to experience in evidence synthesis, level of education, area of study, age, gender. Feasibility and Acceptability Outcomes. Following screening completion, participants will again be asked to complete a brief Qualtrics questionnaire to collect data on our primary feasibility and acceptability outcomes, namely: Efficiency. Participant’s self-report of the time taken in hours and minutes to complete the study selection task. Success of blinding. Participants will be asked to indicate what training condition they believe they were allocated to in order to determine the success of participant blinding procedures. Task evaluation and perceived usefulness of training and screening partner decisions in study selection. The difficulty of the study selection task, perceived usefulness of the training and pairing with the screening partner in the completion of study selection, as well as broader knowledge and skill development following task completion will be assessed across 6 items on a Likert scale from 1 ( strongly disagree ) to 5 ( strongly agree ). Items include, for example, ‘ The screening decisions of ‘Screener 1’ (whether there was a conflict in decision or not) helped me to adjust my approach to the study selection process’ and ‘The training was useful in assisting me to complete study selection ’. Secondary feasibility measures include: Motivation/interest. In response to the question ‘ What was your main interest or motivation in participating in this study? ’, participants will be asked to rank the options in order of their main motivation and/or interest, including: interest in the topic area, interest in systematic review methods, the chance to get involved in research, skill development/capacity building in systematic review methods, and other (please specify). Recommendations for improvement. Participants will be provided with an open-ended response type question on ‘ What, if anything, would have improved the overall experience ?’ Procedure Individuals who express interest in participation will be provided with an information sheet and access to an online Qualtrics survey by the study research assistant (TA) where participants can record their provision of informed consent (see ‘Extended Data’, Ahern et al ., 2025 ). Consenting participants will then be enrolled in the study following self-declaration that they meet the eligibility criteria. A unique identification number will be assigned to allow for participant questionnaire responses and reliability data from the study selection task on Covidence to be linked. The study research assistant (TA) will generate a pseudonymisation key, only accessible to them during the study, and this will be stored securely and separately on the institution’s cloud-based storage. Participants will proceed to complete a brief demographic and background questionnaire on Qualtrics. Participants will then be randomised to a trial condition (for randomisation process, see ‘Study Design’) and provided with instructions on how to access the training site on the virtual learning environment and their Covidence account, which has been set up on their behalf by the study research assistant (TA) through an institutional licence. Participants will be able to select their own log-in password, with their e-mail as the username. Covidence will be set to ‘dual screener’ mode, meaning that each study record requires a decision by two independent screeners. For the purposes of this study, only the ‘YES’ and ‘NO’ screening options will be used to avoid difficulties in ascertaining the interrater reliability of responses assessed as ‘MAYBE’. As such an ‘erring on the side of inclusion’ approach will be applied, as would be typical for study selection at title/abstract level ( Teo et al ., 2023 ): higher sensitivity (i.e. voting ‘YES’ even when unsure) will be prioritised to reduce the risk of incorrectly excluding relevant studies although it increases the likelihood of including irrelevant studies, thus lowering specificity. The screening decisions of ‘Reviewer 1’ will be loaded up on the respective review page on Covidence, prior to the participant commencing (as ‘Reviewer 2’). Please refer to section ‘Screening Partner (‘Reviewer 1’)’ for further detail. For each record screened, the participant will be notified by Covidence as to whether they are in agreement or in conflict with ‘Reviewer 1’. The participant need only complete the study selection task at the title/abstract level, with no requirement to proceed to conflict resolution or full-text screening. Instructions on the study selection task will be provided in the training material (see ‘Extended Data’, Ahern et al ., 2025 ). In brief, participants will be instructed to read the title and abstract of each study record and make a decision based on the eligibility criteria using their best judgement. If there is reason to believe that a study may be eligible, but cannot confirm at the title/abstract stage, then a conservative decision should be made to retain the study record (‘YES’ vote). If the decision conflicts with Reviewer 1’s, this is a typical part of the study selection process. No further action is needed in resolving the conflicts and this will be managed by the review team. In order to maintain the integrity of the independent screening process (between Reviewer 1 and Reviewer 2), a decision should not be changed once made. Participants will also be asked not to discuss their training with others to minimise the potential risk of contamination across the trial conditions. It will be recommended to complete the study selection task in focused, distraction free blocks (e.g., 30 minutes) to avoid screening fatigue. In total, the recommendation is to complete 1-hour of screening each day, across 4 days. Queries or any technical difficulties can be raised with the study team, however, queries directly related to the study selection task cannot be answered to avoid potentially influencing the participant’s screening decision. Upon completion of the study selection task, the participant will be invited to complete a brief questionnaire on Qualtrics to self-report on feasibility outcomes such as the time taken to complete the study selection task, success of blinding, and acceptability outcomes, including perceived usefulness of the training and screening decisions of their screening partner, motivation/interest to engage in the task, and recommendations, before being debriefed on the study. A gentle e-mail reminder will be sent following 1-week of enrolment, and then every week for up to 3 weeks in the instance that the participant has not yet completed the study selection task. Total participation time across the study conditions is estimated at 5 hours. A schematic summary of the participation timeline is provided in Figure 2 . Figure 2. Schematic figure of participation tasks and anticipated timeline. Figure Note. In the instance that the study selection task has not been completed following 1-week of enrolment, a gentle e-mail reminder will be sent and then again every week for up to 3 weeks. Analysis Data will be collated from the Qualtrics survey and the respective project sites on Covidence systematic review screening software and then reviewed by EA for completeness and accuracy. Analysis will be conducted by EA who will remain blinded to participant allocation. Any queries related to interrater reliability will be directed to the study research assistant (TA) who will confirm the accuracy of the data from the respective Covidence project sites, ensuring that the integrity of blinding is upheld. The final dataset will be accessible via cloud-based sharing to members of the research team only. As this SWAR is designed as a pilot study, the focus is on descriptive statistics and estimation, not hypothesis-testing ( Lee et al ., 2014 ; Leon et al ., 2011 ). Thus, any analyses are intended as exploratory for the purposes of informing plans for a subsequent, definitive trial, where feasible. Inferential statistics will be generated for the purposes of inferring if differences exist between the trial conditions, but conclusions on the efficacy of training or experience within the screener pairing will not be drawn. Cohen’s kappa and PABAK will be calculated on r using the epiR package ( Stevenson & Sergeant, 2025 ). A mean kappa and PABAK estimate will be computed for each trial condition, along with the 95% CI and standard error. Percentage agreement in screening decisions will also be calculated as a simple percentage and interpreted alongside the respective kappa statistics. All available screening data, including partial data (i.e., screening decisions available for 1 ≤ k < 219 records) in the instance of study selection task non-completion, will be used in the calculation of interrater reliability. As interrater reliability can be calculated from the completion of just k = 1 study record, a minimum threshold of 10% record completion for the study selection task ( k = 22 records) is imposed for data to be included in the condition-level summaries (Mean, 95% CI) and comparison between conditions. The minimum 10% threshold is consistent with the Agency for Healthcare Research and Quality Methods guidance on the pilot phase of study selection to reduce errors and as a form of calibration in the interpretation of eligibility criteria between two independent reviewers, before a single reviewer subsequently proceeds to complete study selection ( McDonagh et al ., 2013 ). This criterion ensures that reported summary statistics and comparisons will be based on a meaningful amount of data while still allowing (partially) available data from this SWAR pilot study to be appropriately incorporated. To assess the potential impact of missing data, sensitivity analyses will be conducted using only complete cases (i.e., screening decisions available for k = 219 records) to determine the robustness of findings. To explore differences between the kappa estimates produced by participants (vs. expert standard) in each trial condition and within the screening pairs, bootstrapping will be conducted using the boot package ( Canty & Ripley, 2024 ; Davison & Hinkley, 1997 ) to generate bootstrapped CIs and p -values. Differences in kappa estimates will be visually presented using a forest plot. In addition, an item-level mixed-effects analysis will be conducted as a complementary analysis to explore patterns in screening decisions while taking into account clustering by participant and by item (i.e. individual study records). Each screening decision will be treated as an observation, with agreement (agree, disagree) specified as the binary outcome. Fixed effects will include training condition, screening partner experience, and their interaction while random intercepts for participant and study record will account for non-independence of decisions arising from repeated assessments by a given individual and of study records. This analysis is intended to inform the design and analytical plan for a subsequent definitive trial, rather than hypothesis-testing at this pilot stage. Measures of validity, including false positives, sensitivity, and specificity, will be calculated and presented in tabular form. Rank-based responses on motivation/interest to engage in the study and categorical responses indicating the success of blinding will be reported as frequencies and percentages. For Likert scale response outcomes on acceptability, total scores will be computed and reported using M and SD estimates (or median and interquartile ranges in the instance of non-normal distributions), with a factorial between-group ANOVA generated, where appropriate, to explore potential differences in acceptability between the groups, again for the purposes of planning for future trial work, not confirmatory. Efficiency in the study selection task (time taken) will be assessed descriptively and inferentially to explore if time taken for completion likely varies across the trial conditions. Open-ended text responses to the question on recommendations for improvement will be reviewed to identify possible areas for refinement of the intervention or study design. Given the small sample size and intention to generate brief as opposed to in-depth responses, formal qualitative analysis will not be undertaken. Instead, responses will be summarised descriptively, with illustrative quotes presented where appropriate to highlight key aspects of participants’ experiences and suggested improvements, to inform subsequent trial work. Progression criteria Drawing on the feasibility and acceptability data, progression to a subsequent fully powered trial will be considered appropriate if at least 80% of participants complete the study selection task and the associated post-task questionnaire; the median self-reported time taken to complete the selection task is consistent with the planned burden of approximately 5 hours, and acceptability ratings on the 5-point Likert items are generally favourable, indicative that the majority agreed (at least 80% scoring at least 4 ‘ somewhat agree ’ or 5 ‘ strongly agree ’ in response to an item) that the training and screening partner decisions were useful in helping them to complete study selection. If one of these criteria is not met, but feasibility remains broadly acceptable, for example, if at least 70% of participants complete the task and no substantial barriers are identified (e.g., time for completion frequently exceeds the expected 5 hours, or many participants do not agree that training materials are useful), progression may still be warranted, but with refinement to specific elements such as training materials, instructions, or participant support. Conclusions It is intended that the findings will be disseminated in a peer-reviewed publication and knowledge exchange event, with associated materials from the design and reporting of the project made available on the OSF project site. Findings are anticipated to provide insights into further efficiencies that can be achieved in the systematic review process by determining whether novice screeners can achieve a high standard of reliability during study selection following training and pairing with a screening partner. Thus, novice screeners could successfully be integrated into review teams to assist with project completion. Alongside this is the impact of these potential findings on capacity building for evidence synthesis among student or early career researchers. It is intended that findings will be applied to help inform the development of training for student researchers, which could potentially lead to a sustainable research culture where students have the opportunity to work alongside more experienced faculty on evidence synthesis projects. As this is a pilot SWAR, certain design limitations must be acknowledged. In particular, each level of screening partner experience is represented by a single individual meaning that screening partner experience is confounded with screening partner identity. Accordingly, any findings related to screening partner experience should be interpreted cautiously and considered exploratory as observed differences may reflect characteristics of the screening partner (e.g., individual screening style) rather than experience level, per se. A future trial design with multiple screening partners at each experience level would likely be required to disentangle such potential confounding effects. To this end, the findings of this SWAR pilot trial will be used to inform the design of subsequent trial work to appropriately test the efficacy of training and screener pairings for study selection. Study status At the time of protocol submission, recruitment had commenced with data collection on-going. Author contributions EA: Conceptualization; methodology; data curation; project administration; resources; supervision; writing original draft; writing review and editing; funding acquisition; visualization. TA: Methodology; resources; project administration; writing original draft; writing review and editing. AW: Conceptualization; methodology; resources; writing review and editing; funding acquisition. SD: Conceptualization; methodology; resources; writing review and editing; funding acquisition. FL: Conceptualization; methodology; writing review and editing; funding acquisition. Data availability Underlying data No data are associated with this article. Extended data Open Science Framework: Training and Experience in Study Selection - The TESS Study. https://doi.org/10.17605/OSF.IO/WKHQJ ( Ahern et al ., 2025 ). This project contains the following extended data: • Participant Information Sheet and Consent Form, including TESS Study Ethical Consent Form.pdf and TESS Study Participant Information Sheet.pdf • Project Training Site, including TESS Study_VLE project site interface_sample.pdf • Protocol Reporting Checklists, including CONSORT Extension Pilot and Feasibility Trials Checklist.pdf and SPIRIT Checklist.pdf • Study Selection Task, including TESS Study Eligibility Criteria.pdf, TESS Study Eligibility Criteria_Supporting Materials.pdf, TESS Study Instructions for Study Selection Task.pdf, TESS Study Sample Database Search Strategy.pdf, and TESS Study Selection Task References.pdf Data are available under the terms of the Creative Commons Zero “No rights reserved” data waiver (CC0 1.0 Public domain dedication). The Northern Ireland Network for Trials Methodology Research. SWAR Repository Store. SWAR registration for ‘Training and Experience in Study Selection (TESS): A Pilot Randomised Trial within a Systematic Review’. SWAR 38 . References Acosta S, Garza T, Hsu HY, et al. : Assessing quality in systematic literature reviews: a study of novice rater training. Sage Open. 2020; 10 (3): 1–11. Publisher Full Text Ahern E, Whiston A, Dillon S, et al. : Training and Experience in Study Selection - The TESS study. [Open Science Framework Project].2025; Publisher Full Text Borah R, Brown AW, Capers PL, et al. : Analysis of the time and workers needed to conduct systematic reviews of medical interventions using data from the PROSPERO registry. BMJ Open. 2017; 7 (2): e012545. Publisher Full Text Byrt T, Bishop J, Carlin JB: Bias, prevalence and kappa. J Clin Epidemiol. 1993; 46 (5): 423–429. PubMed Abstract | Publisher Full Text Canty A, Ripley BD: boot: bootstrap R (S-Plus) functions. R package version 1.3–31.2024. Chan AW, Tetzlaff JM, Altman DG, et al. : SPIRIT 2013 statement: defining standard protocol items for clinical trials. Ann Intern Med. 2013; 158 (3): 200–207. PubMed Abstract | Publisher Full Text | Free Full Text Cohen J: A coefficient of agreement for nominal scales. Educ Psychol Meas. 1960; 20 (1): 37–46. Publisher Full Text Cooper M, Ungar W, Zlotkin S: An assessment of inter-rater agreement of the literature filtering process in the development of evidence-based dietary guidelines. Public Health Nutr. 2006; 9 (4): 494–500. PubMed Abstract | Publisher Full Text da Costa BR , Beckett B, Diaz A, et al. : Effect of standardized training on the reliability of the Cochrane risk of bias assessment tool: a prospective study. Syst Rev. 2017; 6 (1): 44. PubMed Abstract | Publisher Full Text | Free Full Text Davison AC, Hinkley DV: Bootstrap methods and their application. :Cambridge University Press; 1997. Publisher Full Text Devane D, Burke NN, Treweek S, et al. : Study Within A Review (SWAR). J Evid Based Med. 2022; 15 (4): 328–332. PubMed Abstract | Publisher Full Text Edwards P, Clarke M, DiGuiseppi C, et al. : Identification of randomized controlled trials in systematic reviews: accuracy and reliability of screening records. Stat Med. 2002; 21 (11): 1635–1640. PubMed Abstract | Publisher Full Text Eldridge SM, Chan CL, Campbell MJ, et al. : CONSORT 2010 statement: extension to randomised pilot and feasibility trials. BMJ. 2016; 355 : i5239. PubMed Abstract | Publisher Full Text | Free Full Text Fourcade L, Boutron I, Moher D, et al. : Development and evaluation of a pedagogical tool to improve understanding of a quality checklist: a randomised controlled trial. PLoS Clin Trials. 2007; 2 (5): e22. PubMed Abstract | Publisher Full Text | Free Full Text Garritty C, Gartlehner G, Nussbaumer-Streit B, et al. : Cochrane rapid reviews methods group offers evidence-informed guidance to conduct rapid reviews. J Clin Epidemiol. 2021; 130 : 13–22. PubMed Abstract | Publisher Full Text Gartlehner G, Affengruber L, Titscher V, et al. : Single-reviewer abstract screening missed 13 percent of relevant studies: a crowd-based, randomized controlled trial. J Clin Epidemiol. 2020; 121 : 20–28. Publisher Full Text Haddaway NR, Pullin AS: The policy role of systematic reviews: past, present and future. Springer Sci Rev. 2014; 2 (1–2): 179–183. Publisher Full Text Hoffmann F, Allers K, Rombey T, et al. : Nearly 80 systematic reviews were published each day: observational study on trends in epidemiology and reporting over the years 2000–2019. J Clin Epidemiol. 2021; 138 : 1–11. PubMed Abstract | Publisher Full Text Horton J, Vandermeer B, Hartling L, et al. : Systematic review data extraction: cross-sectional study showed that experience did not increase accuracy. J Clin Epidemiol. 2010; 63 (3): 289–298. PubMed Abstract | Publisher Full Text Lasserson TJ, Thomas J, Higgins JPT: Chapter 1: starting a review. In: Higgins JPT, Thomas J, Chandler J, et al. , editors. Cochrane handbook for systematic reviews of interventions. version 6.5. Cochrane2024, 2021. Reference Source Lee EC, Whitehead AL, Jacques RM, et al. : The statistical interpretation of pilot trials: should significance thresholds be reconsidered? BMC Med Res Methodol. 2014; 14 (1): 41. PubMed Abstract | Publisher Full Text | Free Full Text Lefebvre C, Glanville J, Briscoe S, et al. : Chapter 4: Searching for and selecting studies. In: Higgins JPT, Thomas J, Chandler J, et al. , editors. Cochrane Handbook for Systematic Reviews of Interventions. Cochrane, version 6.5. 2024. Reference Source Leon AC, Davis LL, Kraemer HC: The role and interpretation of pilot studies in clinical research. J Psychiatr Res. 2011; 45 (5): 626–629. PubMed Abstract | Publisher Full Text | Free Full Text Mateen FJ, Oh J, Tergas AI, et al. : Titles versus titles and abstracts for initial screening of articles for systematic reviews. Clin Epidemiol. 2013; 5 : 89–95. Publisher Full Text McDonagh M, Peterson K, Raina P, et al. : Avoiding bias in selecting studies. In: Methods Guide for Comparative Effectiveness Reviews. AHRQ Publication No. 13–EHC045–EF. Agency for Healthcare Research and Quality, 2013. PubMed Abstract McGuire J, Bates GW, Dretzke BJ, et al. : Methodological quality as a component of meta-analysis. Educ Psychol. 1985; 20 (1): 1–5. Publisher Full Text McHugh MH: Interrater reliability: the kappa statistic. Biochem Med (Zagreb). 2012; 22 (3): 276–282. PubMed Abstract | Publisher Full Text | Free Full Text Moons P, Goossens E, Thompson DR: Rapid reviews: the pros and cons of an accelerated review process. Eur J Cardiovasc Nurs. 2021; 20 (5): 515–519. PubMed Abstract | Publisher Full Text Ng L, Pitt V, Huckvale K, et al. : Title and Abstract Screening and Evaluation in Systematic Reviews (TASER): a pilot randomised controlled trial of title and abstract screening by medical students. Syst Rev. 2014; 3 : 121. PubMed Abstract | Publisher Full Text | Free Full Text Noel-Storr A, Dooley G, Elliott J, et al. : An evaluation of Cochrane Crowd found that crowdsourcing produced accurate results in identifying randomized trials. J Clin Epidemiol. 2021; 133 : 130–139. PubMed Abstract | Publisher Full Text Nussbaumer-Streit B, Sommer I, Hamel C, et al. : Rapid reviews methods series: guidance on team considerations, study selection, data extraction and risk of bias assessment. BMJ Evid Based Med. 2023; 28 (6): 418–423. PubMed Abstract | Publisher Full Text Oremus M, Oremus C, Hall GB, et al. : Inter-rater and test–retest reliability of quality assessments by novice student raters using the Jadad and Newcastle–Ottawa Scales. BMJ Open. 2012; 2 (4): e001368. PubMed Abstract | Publisher Full Text | Free Full Text Orwin RG: Evaluating coding decisions. In: Cooper H, Hedges LV, editors. The handbook of research synthesis. Russell Sage Foundation, 1994; 139–162. Reference Source Polanin JR, Pigott TD, Espelage DL, et al. : Best practice guidelines for abstract screening large-evidence systematic reviews and meta-analyses. Res Synth Methods 2019; 10 (3): 330–342. Publisher Full Text Robson RC, Pham B, Hwee J, et al. : Few studies exist examining methods for selecting studies, abstracting data, and appraising quality in a systematic review. J Clin Epidemiol. 2019; 106 : 121–135. Publisher Full Text Rotondi MA: kappaSize: sample size estimation functions for studies of interobserver agreement. R package version 1.2.2018. Publisher Full Text Siddaway AP, Wood AM, Hedges LV: How to do a systematic review: a best practice guide for conducting and reporting narrative reviews, meta-analyses, and meta-syntheses. Annu Rev Psychol. 2019; 70 (1): 747–770. PubMed Abstract | Publisher Full Text Stevenson M, Sergeant E: epiR: tools for the analysis of epidemiological data. R package version 2.0.89.2025. Publisher Full Text Stoll CRT, Izadi S, Fowler S, et al. : The value of a second reviewer for study selection in systematic reviews. Res Synth Methods. 2019; 10 (4): 539–545. PubMed Abstract | Publisher Full Text Tendal B, Higgins JP, Jüni P, et al. : Disagreements in meta-analyses using outcomes measured on continuous or rating scales: observer agreement study. BMJ. 2009; 339 : b3128. PubMed Abstract | Publisher Full Text | Free Full Text Teo L, Van Elswyk ME, Lau CS, et al. : Title-plus-abstract versus title-only first-level screening approach: a case study using a systematic review of dietary patterns and sarcopenia risk to compare screening performance. Syst Rev. 2023; 12 : 211. PubMed Abstract | Publisher Full Text | Free Full Text Thabane L, Lancaster G: A guide to the reporting of protocols of pilot and feasibility trials. Pilot Feasibility Stud. 2019; 5 (1): 37. PubMed Abstract | Publisher Full Text | Free Full Text Veritas Health Innovation: Covidence systematic review software. Reference Source Waffenschmidt S, Knelangen M, Sieben W, et al. : Single screening versus conventional double screening for study selection in systematic reviews: a methodological systematic review. BMC Med Res Methodol. 2019; 19 (1): 132. PubMed Abstract | Publisher Full Text | Free Full Text Comments on this article Comments (0) Version 3 VERSION 3 PUBLISHED 14 Apr 2025 ADD YOUR COMMENT Comment Author details Author details 1 Department of Psychology, University of Limerick, Castletroy, Limerick, V94 T9PX, Ireland 2 Health Research Institute, University of Limerick, Castletroy, Limerick, V94 T9PX, Ireland 3 School of Allied Health, University of Limerick, Castletroy, Limerick, V94 T9PX, Ireland 4 School of Nursing and Midwifery, Queen's University Belfast, Belfast, BT9 7BL, UK Elayne Ahern Roles: Conceptualization, Data Curation, Funding Acquisition, Methodology, Project Administration, Resources, Supervision, Visualization, Writing – Original Draft Preparation, Writing – Review & Editing Temitayo Adedeji Roles: Methodology, Project Administration, Resources, Writing – Original Draft Preparation, Writing – Review & Editing Aoife Whiston Roles: Conceptualization, Funding Acquisition, Methodology, Resources, Writing – Review & Editing Sarah Dillon Roles: Conceptualization, Funding Acquisition, Methodology, Resources, Writing – Review & Editing Fiona Lynn Roles: Conceptualization, Funding Acquisition, Methodology, Writing – Review & Editing Competing interests No competing interests were disclosed. Grant information This study was supported by the Health Research Board (Ireland) and the HSC Public Health Agency [ESI-2021-001] through Evidence Synthesis Ireland/Cochrane Ireland. The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript. Article Versions (3) version 3 Revised Published: 16 Apr 2026, 8:54 https://doi.org/10.12688/hrbopenres.14129.3 version 2 Revised Published: 15 Jan 2026, 8:54 https://doi.org/10.12688/hrbopenres.14129.2 version 1 Published: 14 Apr 2025, 8:54 https://doi.org/10.12688/hrbopenres.14129.1 Copyright © 2026 Ahern E et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. Download Export To Sciwheel Bibtex EndNote ProCite Ref. Manager (RIS) Sente metrics VIEWS $counts.viewCount downloads Citations open_in_new 0 open_in_new 0 open_in_new SEE MORE DETAILS CITE how to cite this article Ahern E, Adedeji T, Whiston A et al. Training and Experience in Study Selection (TESS): study protocol for a pilot randomised trial within a systematic review [version 3; peer review: 2 approved] . HRB Open Res 2026, 8 :54 ( https://doi.org/10.12688/hrbopenres.14129.3 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS track receive updates on this article Track an article to receive email alerts on any updates to this article. TRACK THIS ARTICLE Share Open Peer Review Current Reviewer Status: ? Key to Reviewer Statuses VIEW HIDE Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Version 2 VERSION 2 PUBLISHED 15 Jan 2026 Revised Views 0 Cite How to cite this report: MacDonald H. Reviewer Report For: Training and Experience in Study Selection (TESS): study protocol for a pilot randomised trial within a systematic review [version 3; peer review: 2 approved] . HRB Open Res 2026, 8 :54 ( https://doi.org/10.21956/hrbopenres.15764.r53773 ) The direct URL for this report is: https://hrbopenresearch.org/articles/8-54/v2#referee-response-53773 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 12 Mar 2026 Heather MacDonald , MacOdrum Library, Carleton University, Ottawa, On, Canada Approved VIEWS 0 https://doi.org/10.21956/hrbopenres.15764.r53773 This protocol is well designed to address the study questions posed. There is currently a lack of evidence on novice screener training, and this protocol helps fill this gap. The changes from the first round of revisions to this ... Continue reading READ ALL This protocol is well designed to address the study questions posed. There is currently a lack of evidence on novice screener training, and this protocol helps fill this gap. The changes from the first round of revisions to this protocol increases its rigour and clarity. I only have one comment about the open-ended question about recommendations for improvement. There is no analysis mentioned for this. What do you hope to gain from including this question? With such a small sample size a thematic analysis is not appropriate. Will you highlight any quotes? Consider adding a sentence to address this. Is the rationale for, and objectives of, the study clearly described? Yes Is the study design appropriate for the research question? Yes Are sufficient details of the methods provided to allow replication by others? Yes Are the datasets clearly presented in a useable and accessible format? Not applicable Competing Interests: No competing interests were disclosed. Reviewer Expertise: information literacy, evidence synthesis I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT MacDonald H. Reviewer Report For: Training and Experience in Study Selection (TESS): study protocol for a pilot randomised trial within a systematic review [version 3; peer review: 2 approved] . HRB Open Res 2026, 8 :54 ( https://doi.org/10.21956/hrbopenres.15764.r53773 ) The direct URL for this report is: https://hrbopenresearch.org/articles/8-54/v2#referee-response-53773 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 16 Apr 2026 Elayne Ahern , Department of Psychology, University of Limerick, Castletroy, V94 T9PX, Ireland 16 Apr 2026 Author Response Thank you kindly for the time and attention given to reviewing our protocol. The feedback is very much appreciated and has helped to enhance the quality of our work. For ... Continue reading Thank you kindly for the time and attention given to reviewing our protocol. The feedback is very much appreciated and has helped to enhance the quality of our work. For ease of reference, we have listed the peer-review comments in bold, followed immediately by our responses, presented point-by-point. This protocol is well designed to address the study questions posed. There is currently a lack of evidence on novice screener training, and this protocol helps fill this gap. The changes from the first round of revisions to this protocol increases its rigour and clarity. Thank you for positively receiving our work and for noting the improved rigour and clarity as a result of the revisions already incorporated. I only have one comment about the open-ended question about recommendations for improvement. There is no analysis mentioned for this. What do you hope to gain from including this question? With such a small sample size a thematic analysis is not appropriate. Will you highlight any quotes? Consider adding a sentence to address this. Thank you for this helpful suggestion as we acknowledge that we lacked sufficient detail on this in our previous manuscript version. We have now clarified the handling of the open-ended responses about recommendations, which we intend to use for the purposes of refining the intervention or study design in subsequent work. Given the small sample size and the expectation that responses will be brief, rather than in-depth, we do not plan to conduct formal qualitative analysis. Instead, the responses will be summarised descriptively, with illustrative quotes included where they help to highlight aspects of participants’ experiences and suggested improvements for future trial work. Thank you kindly for the time and attention given to reviewing our protocol. The feedback is very much appreciated and has helped to enhance the quality of our work. For ease of reference, we have listed the peer-review comments in bold, followed immediately by our responses, presented point-by-point. This protocol is well designed to address the study questions posed. There is currently a lack of evidence on novice screener training, and this protocol helps fill this gap. The changes from the first round of revisions to this protocol increases its rigour and clarity. Thank you for positively receiving our work and for noting the improved rigour and clarity as a result of the revisions already incorporated. I only have one comment about the open-ended question about recommendations for improvement. There is no analysis mentioned for this. What do you hope to gain from including this question? With such a small sample size a thematic analysis is not appropriate. Will you highlight any quotes? Consider adding a sentence to address this. Thank you for this helpful suggestion as we acknowledge that we lacked sufficient detail on this in our previous manuscript version. We have now clarified the handling of the open-ended responses about recommendations, which we intend to use for the purposes of refining the intervention or study design in subsequent work. Given the small sample size and the expectation that responses will be brief, rather than in-depth, we do not plan to conduct formal qualitative analysis. Instead, the responses will be summarised descriptively, with illustrative quotes included where they help to highlight aspects of participants’ experiences and suggested improvements for future trial work. Competing Interests: N/A Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 16 Apr 2026 Elayne Ahern , Department of Psychology, University of Limerick, Castletroy, V94 T9PX, Ireland 16 Apr 2026 Author Response Thank you kindly for the time and attention given to reviewing our protocol. The feedback is very much appreciated and has helped to enhance the quality of our work. For ... Continue reading Thank you kindly for the time and attention given to reviewing our protocol. The feedback is very much appreciated and has helped to enhance the quality of our work. For ease of reference, we have listed the peer-review comments in bold, followed immediately by our responses, presented point-by-point. This protocol is well designed to address the study questions posed. There is currently a lack of evidence on novice screener training, and this protocol helps fill this gap. The changes from the first round of revisions to this protocol increases its rigour and clarity. Thank you for positively receiving our work and for noting the improved rigour and clarity as a result of the revisions already incorporated. I only have one comment about the open-ended question about recommendations for improvement. There is no analysis mentioned for this. What do you hope to gain from including this question? With such a small sample size a thematic analysis is not appropriate. Will you highlight any quotes? Consider adding a sentence to address this. Thank you for this helpful suggestion as we acknowledge that we lacked sufficient detail on this in our previous manuscript version. We have now clarified the handling of the open-ended responses about recommendations, which we intend to use for the purposes of refining the intervention or study design in subsequent work. Given the small sample size and the expectation that responses will be brief, rather than in-depth, we do not plan to conduct formal qualitative analysis. Instead, the responses will be summarised descriptively, with illustrative quotes included where they help to highlight aspects of participants’ experiences and suggested improvements for future trial work. Thank you kindly for the time and attention given to reviewing our protocol. The feedback is very much appreciated and has helped to enhance the quality of our work. For ease of reference, we have listed the peer-review comments in bold, followed immediately by our responses, presented point-by-point. This protocol is well designed to address the study questions posed. There is currently a lack of evidence on novice screener training, and this protocol helps fill this gap. The changes from the first round of revisions to this protocol increases its rigour and clarity. Thank you for positively receiving our work and for noting the improved rigour and clarity as a result of the revisions already incorporated. I only have one comment about the open-ended question about recommendations for improvement. There is no analysis mentioned for this. What do you hope to gain from including this question? With such a small sample size a thematic analysis is not appropriate. Will you highlight any quotes? Consider adding a sentence to address this. Thank you for this helpful suggestion as we acknowledge that we lacked sufficient detail on this in our previous manuscript version. We have now clarified the handling of the open-ended responses about recommendations, which we intend to use for the purposes of refining the intervention or study design in subsequent work. Given the small sample size and the expectation that responses will be brief, rather than in-depth, we do not plan to conduct formal qualitative analysis. Instead, the responses will be summarised descriptively, with illustrative quotes included where they help to highlight aspects of participants’ experiences and suggested improvements for future trial work. Competing Interests: N/A Close Report a concern COMMENT ON THIS REPORT Version 1 VERSION 1 PUBLISHED 14 Apr 2025 Views 0 Cite How to cite this report: Devane D. Reviewer Report For: Training and Experience in Study Selection (TESS): study protocol for a pilot randomised trial within a systematic review [version 3; peer review: 2 approved] . HRB Open Res 2026, 8 :54 ( https://doi.org/10.21956/hrbopenres.15528.r50011 ) The direct URL for this report is: https://hrbopenresearch.org/articles/8-54/v1#referee-response-50011 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 30 Sep 2025 Declan Devane , University of Galway, Galway, Ireland Approved VIEWS 0 https://doi.org/10.21956/hrbopenres.15528.r50011 This protocol addresses a practical and under‑studied step of evidence synthesis: how to train and pair novice screeners so that study selection remains reliable and efficient. The rationale is well supported by prior work on dual screening, training effects, ... Continue reading READ ALL This protocol addresses a practical and under‑studied step of evidence synthesis: how to train and pair novice screeners so that study selection remains reliable and efficient. The rationale is well supported by prior work on dual screening, training effects, and kappa‑based reliability. Strengths include a clear factorial design, preregistration, transparency about materials, and attention to feasibility and acceptability. Overall, the protocol is clear and useful, but several aspects would benefit from clarification or modest redesign to strengthen internal validity and the interpretability of results. Your paper already says this is a pilot that will inform a larger trial. To make that unmistakable, add a brief sentence near the start of Methods and again in Analysis that the goal is learning and planning. Then list two or three simple progression rules, for example completion rate, time to finish, and acceptability thresholds, based on the feasibility data you already collect on pages 7 to 9. The current aim reads like a test of which arm is best. A small rewording might fix that e.g., “This pilot will assess feasibility and acceptability to plan a definitive trial” and align outcomes accordingly. Readers will appreciate a brief expansion of how the 219 records will be chosen. Because Covidence shows agreement or conflict as people work, learning during the task is very plausible. Suggest that the record order be randomised for each participant if possible, and report performance by time or by item deciles to show any learning pattern. With only three participants per cell, the averages of kappa by arm will be wide regardless. You might consider keeping those and adding a complementary item-level mixed model that treats each screened record as one observation and includes effects for training, partner type, and their interaction, with random effects for participant and item. In Analysis on page 9 you plan to treat confidence intervals that cross zero as evidence of equivalence. That can be misread. Suggest you avoid equivalence language or set a small, prespecified equivalence margin for kappa differences and say any tests are exploratory. Kappa can be sensitive to prevalence. Keep kappa and percent agreement, then add one robust agreement index such as PABAK. This may make the results easier to read if the include rate is low or high. You define the partner with 4 to 6 years as moderate experience on page 6, but label the arm “experienced”. Maybe revised labelling. Because one person represents each partner type, partner identity is confounded with partner type. Flag this as a pilot limitation and say that the full trial will include multiple partners per level. The Procedure seems to ask participants to vote YES when unsure. Suggest adding a sentence that explains the consequence of this rule for sensitivity and specificity, so readers know you expect higher sensitivity and possibly lower specificity by design. Time is self-reported in your feasibility outcomes. If Covidence logs can provide start or finish times, say that you will also capture those as an objective check on efficiency. Is the rationale for, and objectives of, the study clearly described? Yes Is the study design appropriate for the research question? Yes Are sufficient details of the methods provided to allow replication by others? Yes Are the datasets clearly presented in a useable and accessible format? Not applicable Competing Interests: No competing interests were disclosed. Reviewer Expertise: Clincial trials and evidence synthesis methodology I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Devane D. Reviewer Report For: Training and Experience in Study Selection (TESS): study protocol for a pilot randomised trial within a systematic review [version 3; peer review: 2 approved] . HRB Open Res 2026, 8 :54 ( https://doi.org/10.21956/hrbopenres.15528.r50011 ) The direct URL for this report is: https://hrbopenresearch.org/articles/8-54/v1#referee-response-50011 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 31 Dec 2025 Elayne Ahern , Department of Psychology, University of Limerick, Castletroy, V94 T9PX, Ireland 31 Dec 2025 Author Response Thank you kindly for the time and consideration given to reviewing our protocol. For ease, we have provided the peer-review comments, formatted in bold, with our response then presented immediately ... Continue reading Thank you kindly for the time and consideration given to reviewing our protocol. For ease, we have provided the peer-review comments, formatted in bold, with our response then presented immediately thereafter, point-by-point. This protocol addresses a practical and under‑studied step of evidence synthesis: how to train and pair novice screeners so that study selection remains reliable and efficient. The rationale is well supported by prior work on dual screening, training effects, and kappa‑based reliability. Strengths include a clear factorial design, preregistration, transparency about materials, and attention to feasibility and acceptability. Overall, the protocol is clear and useful, but several aspects would benefit from clarification or modest redesign to strengthen internal validity and the interpretability of results. Thank you for positively receiving our work. The recommendations are much appreciated, with the validity and clarity of our protocol strengthened following these suggested revisions. Your paper already says this is a pilot that will inform a larger trial. To make that unmistakable, add a brief sentence near the start of Methods and again in Analysis that the goal is learning and planning. Then list two or three simple progression rules, for example completion rate, time to finish, and acceptability thresholds, based on the feasibility data you already collect on pages 7 to 9. Thank you for this suggestion. We have added a statement at the beginning of the Methods and again throughout the Analysis section to clarify that the primary purpose of this SWAR is to learn from pilot study findings and plan for a future definitive trial. In addition, we have now added a ‘Progression criteria’ subsection to the Methods, where we have specified criteria to guide decisions about progression and refinement of the full-scale trial based on feasibility and acceptability outcomes, including completion rate, time required of participants, and perceived usefulness of training and screener pairings when undertaking the study selection task. The current aim reads like a test of which arm is best. A small rewording might fix that e.g., “This pilot will assess feasibility and acceptability to plan a definitive trial” and align outcomes accordingly. Thank you for this helpful suggestion. We have revised the aim to more clearly reflect the pilot purpose of assessing feasibility and acceptability to inform a future definitive trial. We have also updated the objectives and language so that reliability outcomes are framed as exploratory estimates intended to guide the design of the future trial, rather than as formal comparative tests. Readers will appreciate a brief expansion of how the 219 records will be chosen. We have expanded the Methods section to clarify the process used to select the 219 records. Specifically, we now explain that the sample was purposefully drawn to reflect the breadth of eligibility criteria in the host review, including a proportion of records previously screened as “include” ( k = 44) and a diverse set of “exclude” records ( k = 175) to represent the full range of exclusion reasons. This ensures that participants encounter a variety of screening decisions and the need to apply various exclusion rationales, supporting a meaningful assessment of how well the training conditions prepare novice screeners to apply the eligibility criteria. Because Covidence shows agreement or conflict as people work, learning during the task is very plausible. Suggest that the record order be randomised for each participant if possible, and report performance by time or by item deciles to show any learning pattern. Thank you for this helpful guidance. We explored options for randomising record order and analysing learning patterns. However, as confirmed by Covidence Support, Covidence software does not currently support randomisation of the screening order. Records are displayed by default using the “most relevant” ordering, and this is how the study selection task was presented to participants. The “most relevant” ordering is underpinned by Covidence’s internal active learning algorithm, although users can switch to alternative static orderings (e.g., Title or Author) if they choose. Further, the Covidence platform does not provide time-stamped screening data (only the date the record was screened is available), which prevents analyses by time or by item deciles. We recognise that not being able to explore learning during study selection is a limitation and unfortunately cannot action this for our pilot SWAR given the constraints of the platform. With only three participants per cell, the averages of kappa by arm will be wide regardless. You might consider keeping those and adding a complementary item-level mixed model that treats each screened record as one observation and includes effects for training, partner type, and their interaction, with random effects for participant and item. Thank you for this recommended addition to our analysis plan. We agree that with three participants per condition, arm-level kappa estimates will be imprecise and should be interpreted cautiously. Consistent with the pilot and exploratory aims of this SWAR, we have added an item-level mixed-effects analysis to explore patterns in screening decisions while accounting for clustering by participant and by item (i.e. individual study records). We clarify that this analysis is exploratory and is intended to inform the design and analytical approach of a subsequent definitive trial, rather than to support confirmatory analyses. In Analysis on page 9 you plan to treat confidence intervals that cross zero as evidence of equivalence. That can be misread. Suggest you avoid equivalence language or set a small, prespecified equivalence margin for kappa differences and say any tests are exploratory. We agree. Any reference to evidence of equivalence has now been removed and we further underscore that any analyses are exploratory. Kappa can be sensitive to prevalence. Keep kappa and percent agreement, then add one robust agreement index such as PABAK. This may make the results easier to read if the include rate is low or high. Thank you for this suggestion. We have revised the protocol to include reporting of the prevalence-adjusted and bias-adjusted kappa (PABAK), alongside Cohen’s kappa and percent agreement, to support interpretation of reliability given the low number of ‘includes’ during the study selection task. You define the partner with 4 to 6 years as moderate experience on page 6, but label the arm “experienced”. Maybe revised labelling. This was an oversight on our part and we thank you for drawing this to our attention. We have now updated our figure labels to reflect ‘moderately experienced’ and likewise updated this throughout the manuscript. Because one person represents each partner type, partner identity is confounded with partner type. Flag this as a pilot limitation and say that the full trial will include multiple partners per level. Thank you for prompting consideration of this. We have now included a note on this limitation in our conclusion section with a view to addressing it in the future trial, where feasible. The Procedure seems to ask participants to vote YES when unsure. Suggest adding a sentence that explains the consequence of this rule for sensitivity and specificity, so readers know you expect higher sensitivity and possibly lower specificity by design. Many thanks for this suggestion. We acknowledge the importance of this caveat and have added a note to our methods to indicate the impact and trade-off between sensitivity and specificity by reducing decision options to just two, ‘YES’ or ‘NO’. Time is self-reported in your feasibility outcomes. If Covidence logs can provide start or finish times, say that you will also capture those as an objective check on efficiency. We have queried the availability of such logs with Covidence Support and unfortunately such data is not available. Our self-reported measure was included as a means to overcome this and have access to data on time efficiency. We nonetheless acknowledge that an objective measure would strengthen the validity and reliability of this outcome. Thank you kindly for the time and consideration given to reviewing our protocol. For ease, we have provided the peer-review comments, formatted in bold, with our response then presented immediately thereafter, point-by-point. This protocol addresses a practical and under‑studied step of evidence synthesis: how to train and pair novice screeners so that study selection remains reliable and efficient. The rationale is well supported by prior work on dual screening, training effects, and kappa‑based reliability. Strengths include a clear factorial design, preregistration, transparency about materials, and attention to feasibility and acceptability. Overall, the protocol is clear and useful, but several aspects would benefit from clarification or modest redesign to strengthen internal validity and the interpretability of results. Thank you for positively receiving our work. The recommendations are much appreciated, with the validity and clarity of our protocol strengthened following these suggested revisions. Your paper already says this is a pilot that will inform a larger trial. To make that unmistakable, add a brief sentence near the start of Methods and again in Analysis that the goal is learning and planning. Then list two or three simple progression rules, for example completion rate, time to finish, and acceptability thresholds, based on the feasibility data you already collect on pages 7 to 9. Thank you for this suggestion. We have added a statement at the beginning of the Methods and again throughout the Analysis section to clarify that the primary purpose of this SWAR is to learn from pilot study findings and plan for a future definitive trial. In addition, we have now added a ‘Progression criteria’ subsection to the Methods, where we have specified criteria to guide decisions about progression and refinement of the full-scale trial based on feasibility and acceptability outcomes, including completion rate, time required of participants, and perceived usefulness of training and screener pairings when undertaking the study selection task. The current aim reads like a test of which arm is best. A small rewording might fix that e.g., “This pilot will assess feasibility and acceptability to plan a definitive trial” and align outcomes accordingly. Thank you for this helpful suggestion. We have revised the aim to more clearly reflect the pilot purpose of assessing feasibility and acceptability to inform a future definitive trial. We have also updated the objectives and language so that reliability outcomes are framed as exploratory estimates intended to guide the design of the future trial, rather than as formal comparative tests. Readers will appreciate a brief expansion of how the 219 records will be chosen. We have expanded the Methods section to clarify the process used to select the 219 records. Specifically, we now explain that the sample was purposefully drawn to reflect the breadth of eligibility criteria in the host review, including a proportion of records previously screened as “include” ( k = 44) and a diverse set of “exclude” records ( k = 175) to represent the full range of exclusion reasons. This ensures that participants encounter a variety of screening decisions and the need to apply various exclusion rationales, supporting a meaningful assessment of how well the training conditions prepare novice screeners to apply the eligibility criteria. Because Covidence shows agreement or conflict as people work, learning during the task is very plausible. Suggest that the record order be randomised for each participant if possible, and report performance by time or by item deciles to show any learning pattern. Thank you for this helpful guidance. We explored options for randomising record order and analysing learning patterns. However, as confirmed by Covidence Support, Covidence software does not currently support randomisation of the screening order. Records are displayed by default using the “most relevant” ordering, and this is how the study selection task was presented to participants. The “most relevant” ordering is underpinned by Covidence’s internal active learning algorithm, although users can switch to alternative static orderings (e.g., Title or Author) if they choose. Further, the Covidence platform does not provide time-stamped screening data (only the date the record was screened is available), which prevents analyses by time or by item deciles. We recognise that not being able to explore learning during study selection is a limitation and unfortunately cannot action this for our pilot SWAR given the constraints of the platform. With only three participants per cell, the averages of kappa by arm will be wide regardless. You might consider keeping those and adding a complementary item-level mixed model that treats each screened record as one observation and includes effects for training, partner type, and their interaction, with random effects for participant and item. Thank you for this recommended addition to our analysis plan. We agree that with three participants per condition, arm-level kappa estimates will be imprecise and should be interpreted cautiously. Consistent with the pilot and exploratory aims of this SWAR, we have added an item-level mixed-effects analysis to explore patterns in screening decisions while accounting for clustering by participant and by item (i.e. individual study records). We clarify that this analysis is exploratory and is intended to inform the design and analytical approach of a subsequent definitive trial, rather than to support confirmatory analyses. In Analysis on page 9 you plan to treat confidence intervals that cross zero as evidence of equivalence. That can be misread. Suggest you avoid equivalence language or set a small, prespecified equivalence margin for kappa differences and say any tests are exploratory. We agree. Any reference to evidence of equivalence has now been removed and we further underscore that any analyses are exploratory. Kappa can be sensitive to prevalence. Keep kappa and percent agreement, then add one robust agreement index such as PABAK. This may make the results easier to read if the include rate is low or high. Thank you for this suggestion. We have revised the protocol to include reporting of the prevalence-adjusted and bias-adjusted kappa (PABAK), alongside Cohen’s kappa and percent agreement, to support interpretation of reliability given the low number of ‘includes’ during the study selection task. You define the partner with 4 to 6 years as moderate experience on page 6, but label the arm “experienced”. Maybe revised labelling. This was an oversight on our part and we thank you for drawing this to our attention. We have now updated our figure labels to reflect ‘moderately experienced’ and likewise updated this throughout the manuscript. Because one person represents each partner type, partner identity is confounded with partner type. Flag this as a pilot limitation and say that the full trial will include multiple partners per level. Thank you for prompting consideration of this. We have now included a note on this limitation in our conclusion section with a view to addressing it in the future trial, where feasible. The Procedure seems to ask participants to vote YES when unsure. Suggest adding a sentence that explains the consequence of this rule for sensitivity and specificity, so readers know you expect higher sensitivity and possibly lower specificity by design. Many thanks for this suggestion. We acknowledge the importance of this caveat and have added a note to our methods to indicate the impact and trade-off between sensitivity and specificity by reducing decision options to just two, ‘YES’ or ‘NO’. Time is self-reported in your feasibility outcomes. If Covidence logs can provide start or finish times, say that you will also capture those as an objective check on efficiency. We have queried the availability of such logs with Covidence Support and unfortunately such data is not available. Our self-reported measure was included as a means to overcome this and have access to data on time efficiency. We nonetheless acknowledge that an objective measure would strengthen the validity and reliability of this outcome. Competing Interests: N/A Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 31 Dec 2025 Elayne Ahern , Department of Psychology, University of Limerick, Castletroy, V94 T9PX, Ireland 31 Dec 2025 Author Response Thank you kindly for the time and consideration given to reviewing our protocol. For ease, we have provided the peer-review comments, formatted in bold, with our response then presented immediately ... Continue reading Thank you kindly for the time and consideration given to reviewing our protocol. For ease, we have provided the peer-review comments, formatted in bold, with our response then presented immediately thereafter, point-by-point. This protocol addresses a practical and under‑studied step of evidence synthesis: how to train and pair novice screeners so that study selection remains reliable and efficient. The rationale is well supported by prior work on dual screening, training effects, and kappa‑based reliability. Strengths include a clear factorial design, preregistration, transparency about materials, and attention to feasibility and acceptability. Overall, the protocol is clear and useful, but several aspects would benefit from clarification or modest redesign to strengthen internal validity and the interpretability of results. Thank you for positively receiving our work. The recommendations are much appreciated, with the validity and clarity of our protocol strengthened following these suggested revisions. Your paper already says this is a pilot that will inform a larger trial. To make that unmistakable, add a brief sentence near the start of Methods and again in Analysis that the goal is learning and planning. Then list two or three simple progression rules, for example completion rate, time to finish, and acceptability thresholds, based on the feasibility data you already collect on pages 7 to 9. Thank you for this suggestion. We have added a statement at the beginning of the Methods and again throughout the Analysis section to clarify that the primary purpose of this SWAR is to learn from pilot study findings and plan for a future definitive trial. In addition, we have now added a ‘Progression criteria’ subsection to the Methods, where we have specified criteria to guide decisions about progression and refinement of the full-scale trial based on feasibility and acceptability outcomes, including completion rate, time required of participants, and perceived usefulness of training and screener pairings when undertaking the study selection task. The current aim reads like a test of which arm is best. A small rewording might fix that e.g., “This pilot will assess feasibility and acceptability to plan a definitive trial” and align outcomes accordingly. Thank you for this helpful suggestion. We have revised the aim to more clearly reflect the pilot purpose of assessing feasibility and acceptability to inform a future definitive trial. We have also updated the objectives and language so that reliability outcomes are framed as exploratory estimates intended to guide the design of the future trial, rather than as formal comparative tests. Readers will appreciate a brief expansion of how the 219 records will be chosen. We have expanded the Methods section to clarify the process used to select the 219 records. Specifically, we now explain that the sample was purposefully drawn to reflect the breadth of eligibility criteria in the host review, including a proportion of records previously screened as “include” ( k = 44) and a diverse set of “exclude” records ( k = 175) to represent the full range of exclusion reasons. This ensures that participants encounter a variety of screening decisions and the need to apply various exclusion rationales, supporting a meaningful assessment of how well the training conditions prepare novice screeners to apply the eligibility criteria. Because Covidence shows agreement or conflict as people work, learning during the task is very plausible. Suggest that the record order be randomised for each participant if possible, and report performance by time or by item deciles to show any learning pattern. Thank you for this helpful guidance. We explored options for randomising record order and analysing learning patterns. However, as confirmed by Covidence Support, Covidence software does not currently support randomisation of the screening order. Records are displayed by default using the “most relevant” ordering, and this is how the study selection task was presented to participants. The “most relevant” ordering is underpinned by Covidence’s internal active learning algorithm, although users can switch to alternative static orderings (e.g., Title or Author) if they choose. Further, the Covidence platform does not provide time-stamped screening data (only the date the record was screened is available), which prevents analyses by time or by item deciles. We recognise that not being able to explore learning during study selection is a limitation and unfortunately cannot action this for our pilot SWAR given the constraints of the platform. With only three participants per cell, the averages of kappa by arm will be wide regardless. You might consider keeping those and adding a complementary item-level mixed model that treats each screened record as one observation and includes effects for training, partner type, and their interaction, with random effects for participant and item. Thank you for this recommended addition to our analysis plan. We agree that with three participants per condition, arm-level kappa estimates will be imprecise and should be interpreted cautiously. Consistent with the pilot and exploratory aims of this SWAR, we have added an item-level mixed-effects analysis to explore patterns in screening decisions while accounting for clustering by participant and by item (i.e. individual study records). We clarify that this analysis is exploratory and is intended to inform the design and analytical approach of a subsequent definitive trial, rather than to support confirmatory analyses. In Analysis on page 9 you plan to treat confidence intervals that cross zero as evidence of equivalence. That can be misread. Suggest you avoid equivalence language or set a small, prespecified equivalence margin for kappa differences and say any tests are exploratory. We agree. Any reference to evidence of equivalence has now been removed and we further underscore that any analyses are exploratory. Kappa can be sensitive to prevalence. Keep kappa and percent agreement, then add one robust agreement index such as PABAK. This may make the results easier to read if the include rate is low or high. Thank you for this suggestion. We have revised the protocol to include reporting of the prevalence-adjusted and bias-adjusted kappa (PABAK), alongside Cohen’s kappa and percent agreement, to support interpretation of reliability given the low number of ‘includes’ during the study selection task. You define the partner with 4 to 6 years as moderate experience on page 6, but label the arm “experienced”. Maybe revised labelling. This was an oversight on our part and we thank you for drawing this to our attention. We have now updated our figure labels to reflect ‘moderately experienced’ and likewise updated this throughout the manuscript. Because one person represents each partner type, partner identity is confounded with partner type. Flag this as a pilot limitation and say that the full trial will include multiple partners per level. Thank you for prompting consideration of this. We have now included a note on this limitation in our conclusion section with a view to addressing it in the future trial, where feasible. The Procedure seems to ask participants to vote YES when unsure. Suggest adding a sentence that explains the consequence of this rule for sensitivity and specificity, so readers know you expect higher sensitivity and possibly lower specificity by design. Many thanks for this suggestion. We acknowledge the importance of this caveat and have added a note to our methods to indicate the impact and trade-off between sensitivity and specificity by reducing decision options to just two, ‘YES’ or ‘NO’. Time is self-reported in your feasibility outcomes. If Covidence logs can provide start or finish times, say that you will also capture those as an objective check on efficiency. We have queried the availability of such logs with Covidence Support and unfortunately such data is not available. Our self-reported measure was included as a means to overcome this and have access to data on time efficiency. We nonetheless acknowledge that an objective measure would strengthen the validity and reliability of this outcome. Thank you kindly for the time and consideration given to reviewing our protocol. For ease, we have provided the peer-review comments, formatted in bold, with our response then presented immediately thereafter, point-by-point. This protocol addresses a practical and under‑studied step of evidence synthesis: how to train and pair novice screeners so that study selection remains reliable and efficient. The rationale is well supported by prior work on dual screening, training effects, and kappa‑based reliability. Strengths include a clear factorial design, preregistration, transparency about materials, and attention to feasibility and acceptability. Overall, the protocol is clear and useful, but several aspects would benefit from clarification or modest redesign to strengthen internal validity and the interpretability of results. Thank you for positively receiving our work. The recommendations are much appreciated, with the validity and clarity of our protocol strengthened following these suggested revisions. Your paper already says this is a pilot that will inform a larger trial. To make that unmistakable, add a brief sentence near the start of Methods and again in Analysis that the goal is learning and planning. Then list two or three simple progression rules, for example completion rate, time to finish, and acceptability thresholds, based on the feasibility data you already collect on pages 7 to 9. Thank you for this suggestion. We have added a statement at the beginning of the Methods and again throughout the Analysis section to clarify that the primary purpose of this SWAR is to learn from pilot study findings and plan for a future definitive trial. In addition, we have now added a ‘Progression criteria’ subsection to the Methods, where we have specified criteria to guide decisions about progression and refinement of the full-scale trial based on feasibility and acceptability outcomes, including completion rate, time required of participants, and perceived usefulness of training and screener pairings when undertaking the study selection task. The current aim reads like a test of which arm is best. A small rewording might fix that e.g., “This pilot will assess feasibility and acceptability to plan a definitive trial” and align outcomes accordingly. Thank you for this helpful suggestion. We have revised the aim to more clearly reflect the pilot purpose of assessing feasibility and acceptability to inform a future definitive trial. We have also updated the objectives and language so that reliability outcomes are framed as exploratory estimates intended to guide the design of the future trial, rather than as formal comparative tests. Readers will appreciate a brief expansion of how the 219 records will be chosen. We have expanded the Methods section to clarify the process used to select the 219 records. Specifically, we now explain that the sample was purposefully drawn to reflect the breadth of eligibility criteria in the host review, including a proportion of records previously screened as “include” ( k = 44) and a diverse set of “exclude” records ( k = 175) to represent the full range of exclusion reasons. This ensures that participants encounter a variety of screening decisions and the need to apply various exclusion rationales, supporting a meaningful assessment of how well the training conditions prepare novice screeners to apply the eligibility criteria. Because Covidence shows agreement or conflict as people work, learning during the task is very plausible. Suggest that the record order be randomised for each participant if possible, and report performance by time or by item deciles to show any learning pattern. Thank you for this helpful guidance. We explored options for randomising record order and analysing learning patterns. However, as confirmed by Covidence Support, Covidence software does not currently support randomisation of the screening order. Records are displayed by default using the “most relevant” ordering, and this is how the study selection task was presented to participants. The “most relevant” ordering is underpinned by Covidence’s internal active learning algorithm, although users can switch to alternative static orderings (e.g., Title or Author) if they choose. Further, the Covidence platform does not provide time-stamped screening data (only the date the record was screened is available), which prevents analyses by time or by item deciles. We recognise that not being able to explore learning during study selection is a limitation and unfortunately cannot action this for our pilot SWAR given the constraints of the platform. With only three participants per cell, the averages of kappa by arm will be wide regardless. You might consider keeping those and adding a complementary item-level mixed model that treats each screened record as one observation and includes effects for training, partner type, and their interaction, with random effects for participant and item. Thank you for this recommended addition to our analysis plan. We agree that with three participants per condition, arm-level kappa estimates will be imprecise and should be interpreted cautiously. Consistent with the pilot and exploratory aims of this SWAR, we have added an item-level mixed-effects analysis to explore patterns in screening decisions while accounting for clustering by participant and by item (i.e. individual study records). We clarify that this analysis is exploratory and is intended to inform the design and analytical approach of a subsequent definitive trial, rather than to support confirmatory analyses. In Analysis on page 9 you plan to treat confidence intervals that cross zero as evidence of equivalence. That can be misread. Suggest you avoid equivalence language or set a small, prespecified equivalence margin for kappa differences and say any tests are exploratory. We agree. Any reference to evidence of equivalence has now been removed and we further underscore that any analyses are exploratory. Kappa can be sensitive to prevalence. Keep kappa and percent agreement, then add one robust agreement index such as PABAK. This may make the results easier to read if the include rate is low or high. Thank you for this suggestion. We have revised the protocol to include reporting of the prevalence-adjusted and bias-adjusted kappa (PABAK), alongside Cohen’s kappa and percent agreement, to support interpretation of reliability given the low number of ‘includes’ during the study selection task. You define the partner with 4 to 6 years as moderate experience on page 6, but label the arm “experienced”. Maybe revised labelling. This was an oversight on our part and we thank you for drawing this to our attention. We have now updated our figure labels to reflect ‘moderately experienced’ and likewise updated this throughout the manuscript. Because one person represents each partner type, partner identity is confounded with partner type. Flag this as a pilot limitation and say that the full trial will include multiple partners per level. Thank you for prompting consideration of this. We have now included a note on this limitation in our conclusion section with a view to addressing it in the future trial, where feasible. The Procedure seems to ask participants to vote YES when unsure. Suggest adding a sentence that explains the consequence of this rule for sensitivity and specificity, so readers know you expect higher sensitivity and possibly lower specificity by design. Many thanks for this suggestion. We acknowledge the importance of this caveat and have added a note to our methods to indicate the impact and trade-off between sensitivity and specificity by reducing decision options to just two, ‘YES’ or ‘NO’. Time is self-reported in your feasibility outcomes. If Covidence logs can provide start or finish times, say that you will also capture those as an objective check on efficiency. We have queried the availability of such logs with Covidence Support and unfortunately such data is not available. Our self-reported measure was included as a means to overcome this and have access to data on time efficiency. We nonetheless acknowledge that an objective measure would strengthen the validity and reliability of this outcome. Competing Interests: N/A Close Report a concern COMMENT ON THIS REPORT Comments on this article Comments (0) Version 3 VERSION 3 PUBLISHED 14 Apr 2025 ADD YOUR COMMENT Comment keyboard_arrow_left keyboard_arrow_right Open Peer Review Reviewer Status info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Reviewer Reports Invited Reviewers 1 2 Version 3 (revision) 16 Apr 26 Version 2 (revision) 15 Jan 26 read Version 1 14 Apr 25 read Declan Devane , University of Galway, Galway, Ireland Heather MacDonald , Carleton University, Ottawa, Canada Comments on this article All Comments (0) Add a comment Sign up for content alerts Sign Up You are now signed up to receive this alert keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2026 MacDonald H. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 12 Mar 2026 | for Version 2 Heather MacDonald , MacOdrum Library, Carleton University, Ottawa, On, Canada 0 Views copyright © 2026 MacDonald H. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This protocol is well designed to address the study questions posed. There is currently a lack of evidence on novice screener training, and this protocol helps fill this gap. The changes from the first round of revisions to this protocol increases its rigour and clarity. I only have one comment about the open-ended question about recommendations for improvement. There is no analysis mentioned for this. What do you hope to gain from including this question? With such a small sample size a thematic analysis is not appropriate. Will you highlight any quotes? Consider adding a sentence to address this. Is the rationale for, and objectives of, the study clearly described? Yes Is the study design appropriate for the research question? Yes Are sufficient details of the methods provided to allow replication by others? Yes Are the datasets clearly presented in a useable and accessible format? Not applicable Competing Interests No competing interests were disclosed. Reviewer Expertise information literacy, evidence synthesis I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (1) Author Response 16 Apr 2026 Elayne Ahern, Department of Psychology, University of Limerick, Castletroy, V94 T9PX, Ireland Thank you kindly for the time and attention given to reviewing our protocol. The feedback is very much appreciated and has helped to enhance the quality of our work. For ease of reference, we have listed the peer-review comments in bold, followed immediately by our responses, presented point-by-point. This protocol is well designed to address the study questions posed. There is currently a lack of evidence on novice screener training, and this protocol helps fill this gap. The changes from the first round of revisions to this protocol increases its rigour and clarity. Thank you for positively receiving our work and for noting the improved rigour and clarity as a result of the revisions already incorporated. I only have one comment about the open-ended question about recommendations for improvement. There is no analysis mentioned for this. What do you hope to gain from including this question? With such a small sample size a thematic analysis is not appropriate. Will you highlight any quotes? Consider adding a sentence to address this. Thank you for this helpful suggestion as we acknowledge that we lacked sufficient detail on this in our previous manuscript version. We have now clarified the handling of the open-ended responses about recommendations, which we intend to use for the purposes of refining the intervention or study design in subsequent work. Given the small sample size and the expectation that responses will be brief, rather than in-depth, we do not plan to conduct formal qualitative analysis. Instead, the responses will be summarised descriptively, with illustrative quotes included where they help to highlight aspects of participants’ experiences and suggested improvements for future trial work. View more View less Competing Interests N/A reply Respond Report a concern MacDonald H. Peer Review Report For: Training and Experience in Study Selection (TESS): study protocol for a pilot randomised trial within a systematic review [version 3; peer review: 2 approved] . HRB Open Res 2026, 8 :54 ( https://doi.org/10.21956/hrbopenres.15764.r53773) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://hrbopenresearch.org/articles/8-54/v2#referee-response-53773 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 Devane D. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 30 Sep 2025 | for Version 1 Declan Devane , University of Galway, Galway, Ireland 0 Views copyright © 2025 Devane D. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This protocol addresses a practical and under‑studied step of evidence synthesis: how to train and pair novice screeners so that study selection remains reliable and efficient. The rationale is well supported by prior work on dual screening, training effects, and kappa‑based reliability. Strengths include a clear factorial design, preregistration, transparency about materials, and attention to feasibility and acceptability. Overall, the protocol is clear and useful, but several aspects would benefit from clarification or modest redesign to strengthen internal validity and the interpretability of results. Your paper already says this is a pilot that will inform a larger trial. To make that unmistakable, add a brief sentence near the start of Methods and again in Analysis that the goal is learning and planning. Then list two or three simple progression rules, for example completion rate, time to finish, and acceptability thresholds, based on the feasibility data you already collect on pages 7 to 9. The current aim reads like a test of which arm is best. A small rewording might fix that e.g., “This pilot will assess feasibility and acceptability to plan a definitive trial” and align outcomes accordingly. Readers will appreciate a brief expansion of how the 219 records will be chosen. Because Covidence shows agreement or conflict as people work, learning during the task is very plausible. Suggest that the record order be randomised for each participant if possible, and report performance by time or by item deciles to show any learning pattern. With only three participants per cell, the averages of kappa by arm will be wide regardless. You might consider keeping those and adding a complementary item-level mixed model that treats each screened record as one observation and includes effects for training, partner type, and their interaction, with random effects for participant and item. In Analysis on page 9 you plan to treat confidence intervals that cross zero as evidence of equivalence. That can be misread. Suggest you avoid equivalence language or set a small, prespecified equivalence margin for kappa differences and say any tests are exploratory. Kappa can be sensitive to prevalence. Keep kappa and percent agreement, then add one robust agreement index such as PABAK. This may make the results easier to read if the include rate is low or high. You define the partner with 4 to 6 years as moderate experience on page 6, but label the arm “experienced”. Maybe revised labelling. Because one person represents each partner type, partner identity is confounded with partner type. Flag this as a pilot limitation and say that the full trial will include multiple partners per level. The Procedure seems to ask participants to vote YES when unsure. Suggest adding a sentence that explains the consequence of this rule for sensitivity and specificity, so readers know you expect higher sensitivity and possibly lower specificity by design. Time is self-reported in your feasibility outcomes. If Covidence logs can provide start or finish times, say that you will also capture those as an objective check on efficiency. Is the rationale for, and objectives of, the study clearly described? Yes Is the study design appropriate for the research question? Yes Are sufficient details of the methods provided to allow replication by others? Yes Are the datasets clearly presented in a useable and accessible format? Not applicable Competing Interests No competing interests were disclosed. Reviewer Expertise Clincial trials and evidence synthesis methodology I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (1) Author Response 31 Dec 2025 Elayne Ahern, Department of Psychology, University of Limerick, Castletroy, V94 T9PX, Ireland Thank you kindly for the time and consideration given to reviewing our protocol. For ease, we have provided the peer-review comments, formatted in bold, with our response then presented immediately thereafter, point-by-point. This protocol addresses a practical and under‑studied step of evidence synthesis: how to train and pair novice screeners so that study selection remains reliable and efficient. The rationale is well supported by prior work on dual screening, training effects, and kappa‑based reliability. Strengths include a clear factorial design, preregistration, transparency about materials, and attention to feasibility and acceptability. Overall, the protocol is clear and useful, but several aspects would benefit from clarification or modest redesign to strengthen internal validity and the interpretability of results. Thank you for positively receiving our work. The recommendations are much appreciated, with the validity and clarity of our protocol strengthened following these suggested revisions. Your paper already says this is a pilot that will inform a larger trial. To make that unmistakable, add a brief sentence near the start of Methods and again in Analysis that the goal is learning and planning. Then list two or three simple progression rules, for example completion rate, time to finish, and acceptability thresholds, based on the feasibility data you already collect on pages 7 to 9. Thank you for this suggestion. We have added a statement at the beginning of the Methods and again throughout the Analysis section to clarify that the primary purpose of this SWAR is to learn from pilot study findings and plan for a future definitive trial. In addition, we have now added a ‘Progression criteria’ subsection to the Methods, where we have specified criteria to guide decisions about progression and refinement of the full-scale trial based on feasibility and acceptability outcomes, including completion rate, time required of participants, and perceived usefulness of training and screener pairings when undertaking the study selection task. The current aim reads like a test of which arm is best. A small rewording might fix that e.g., “This pilot will assess feasibility and acceptability to plan a definitive trial” and align outcomes accordingly. Thank you for this helpful suggestion. We have revised the aim to more clearly reflect the pilot purpose of assessing feasibility and acceptability to inform a future definitive trial. We have also updated the objectives and language so that reliability outcomes are framed as exploratory estimates intended to guide the design of the future trial, rather than as formal comparative tests. Readers will appreciate a brief expansion of how the 219 records will be chosen. We have expanded the Methods section to clarify the process used to select the 219 records. Specifically, we now explain that the sample was purposefully drawn to reflect the breadth of eligibility criteria in the host review, including a proportion of records previously screened as “include” ( k = 44) and a diverse set of “exclude” records ( k = 175) to represent the full range of exclusion reasons. This ensures that participants encounter a variety of screening decisions and the need to apply various exclusion rationales, supporting a meaningful assessment of how well the training conditions prepare novice screeners to apply the eligibility criteria. Because Covidence shows agreement or conflict as people work, learning during the task is very plausible. Suggest that the record order be randomised for each participant if possible, and report performance by time or by item deciles to show any learning pattern. Thank you for this helpful guidance. We explored options for randomising record order and analysing learning patterns. However, as confirmed by Covidence Support, Covidence software does not currently support randomisation of the screening order. Records are displayed by default using the “most relevant” ordering, and this is how the study selection task was presented to participants. The “most relevant” ordering is underpinned by Covidence’s internal active learning algorithm, although users can switch to alternative static orderings (e.g., Title or Author) if they choose. Further, the Covidence platform does not provide time-stamped screening data (only the date the record was screened is available), which prevents analyses by time or by item deciles. We recognise that not being able to explore learning during study selection is a limitation and unfortunately cannot action this for our pilot SWAR given the constraints of the platform. With only three participants per cell, the averages of kappa by arm will be wide regardless. You might consider keeping those and adding a complementary item-level mixed model that treats each screened record as one observation and includes effects for training, partner type, and their interaction, with random effects for participant and item. Thank you for this recommended addition to our analysis plan. We agree that with three participants per condition, arm-level kappa estimates will be imprecise and should be interpreted cautiously. Consistent with the pilot and exploratory aims of this SWAR, we have added an item-level mixed-effects analysis to explore patterns in screening decisions while accounting for clustering by participant and by item (i.e. individual study records). We clarify that this analysis is exploratory and is intended to inform the design and analytical approach of a subsequent definitive trial, rather than to support confirmatory analyses. In Analysis on page 9 you plan to treat confidence intervals that cross zero as evidence of equivalence. That can be misread. Suggest you avoid equivalence language or set a small, prespecified equivalence margin for kappa differences and say any tests are exploratory. We agree. Any reference to evidence of equivalence has now been removed and we further underscore that any analyses are exploratory. Kappa can be sensitive to prevalence. Keep kappa and percent agreement, then add one robust agreement index such as PABAK. This may make the results easier to read if the include rate is low or high. Thank you for this suggestion. We have revised the protocol to include reporting of the prevalence-adjusted and bias-adjusted kappa (PABAK), alongside Cohen’s kappa and percent agreement, to support interpretation of reliability given the low number of ‘includes’ during the study selection task. You define the partner with 4 to 6 years as moderate experience on page 6, but label the arm “experienced”. Maybe revised labelling. This was an oversight on our part and we thank you for drawing this to our attention. We have now updated our figure labels to reflect ‘moderately experienced’ and likewise updated this throughout the manuscript. Because one person represents each partner type, partner identity is confounded with partner type. Flag this as a pilot limitation and say that the full trial will include multiple partners per level. Thank you for prompting consideration of this. We have now included a note on this limitation in our conclusion section with a view to addressing it in the future trial, where feasible. The Procedure seems to ask participants to vote YES when unsure. Suggest adding a sentence that explains the consequence of this rule for sensitivity and specificity, so readers know you expect higher sensitivity and possibly lower specificity by design. Many thanks for this suggestion. We acknowledge the importance of this caveat and have added a note to our methods to indicate the impact and trade-off between sensitivity and specificity by reducing decision options to just two, ‘YES’ or ‘NO’. Time is self-reported in your feasibility outcomes. If Covidence logs can provide start or finish times, say that you will also capture those as an objective check on efficiency. We have queried the availability of such logs with Covidence Support and unfortunately such data is not available. Our self-reported measure was included as a means to overcome this and have access to data on time efficiency. We nonetheless acknowledge that an objective measure would strengthen the validity and reliability of this outcome. View more View less Competing Interests N/A reply Respond Report a concern Devane D. Peer Review Report For: Training and Experience in Study Selection (TESS): study protocol for a pilot randomised trial within a systematic review [version 3; peer review: 2 approved] . HRB Open Res 2026, 8 :54 ( https://doi.org/10.21956/hrbopenres.15528.r50011) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://hrbopenresearch.org/articles/8-54/v1#referee-response-50011 Alongside their report, reviewers assign a status to the article: Approved - the paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations - A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved - fundamental flaws in the paper seriously undermine the findings and conclusions Adjust parameters to alter display View on desktop for interactive features Includes Interactive Elements View on desktop for interactive features Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Stay Updated Sign up for content alerts and receive a weekly or monthly email with all newly published articles Register with HRB Open Research Already registered? Sign in Not now, thanks close PLEASE NOTE If you are an AUTHOR of this article, please check that you signed in with the account associated with this article otherwise we cannot automatically identify your role as an author and your comment will be labelled as a “User Comment”. If you are a REVIEWER of this article, please check that you have signed in with the account associated with this article and then go to your account to submit your report, please do not post your review here. If you do not have access to your original account, please contact us . All commenters must hold a formal affiliation as per our Policies . The information that you give us will be displayed next to your comment. User comments must be in English, comprehensible and relevant to the article under discussion. We reserve the right to remove any comments that we consider to be inappropriate, offensive or otherwise in breach of the User Comment Terms and Conditions . Commenters must not use a comment for personal attacks. When criticisms of the article are based on unpublished data, the data should be made available. I accept the User Comment Terms and Conditions Please confirm that you accept the User Comment Terms and Conditions. Affiliation ✕ refresh Please enter your institution. Note: To add your institution or organisation, start typing the name and then select the correct name from the list. Where applicable, the name will appear in both the original language and in English. Do not paste in the name. If the name does not appear in the drop-down list, we will display the information you have entered. ✕ refresh Country/Region * USA UK Canada China France Germany Afghanistan Aland Islands Albania Algeria American Samoa Andorra Angola Anguilla Antarctica Antigua and Barbuda Argentina Armenia Aruba Australia Austria Azerbaijan Bahamas Bahrain Bangladesh Barbados Belarus Belgium Belize Benin Bermuda Bhutan Bolivia Bosnia and Herzegovina Botswana Bouvet Island Brazil British Indian Ocean Territory British Virgin Islands Brunei Bulgaria Burkina Faso Burundi Cambodia Cameroon Canada Cape Verde Cayman Islands Central African Republic Chad Chile China Christmas Island Cocos (Keeling) Islands Colombia Comoros Congo Cook Islands Costa Rica Cote d'Ivoire Croatia Cuba Cyprus Czech Republic Democratic Republic of the Congo Denmark Djibouti Dominica Dominican Republic Ecuador Egypt El Salvador Equatorial Guinea Eritrea Estonia Ethiopia Falkland Islands Faroe Islands Federated States of Micronesia Fiji Finland France French Guiana French Polynesia French Southern Territories Gabon Georgia Germany Ghana Gibraltar Greece Greenland Grenada Guadeloupe Guam Guatemala Guernsey Guinea Guinea-Bissau Guyana Haiti Heard Island and Mcdonald Islands Holy See (Vatican City State) Honduras Hong Kong Hungary Iceland India Indonesia Iran Iraq Ireland Israel Italy Jamaica Japan Jersey Jordan Kazakhstan Kenya Kiribati Kosovo (Serbia and Montenegro) Kuwait Kyrgyzstan Lao People's Democratic Republic Latvia Lebanon Lesotho Liberia Libya Liechtenstein Lithuania Luxembourg Macao Madagascar Malawi Malaysia Maldives Mali Malta Marshall Islands Martinique Mauritania Mauritius Mayotte Mexico Minor Outlying Islands of the United States Moldova Monaco Mongolia Montenegro Montserrat Morocco Mozambique Myanmar Namibia Nauru Nepal Netherlands Antilles New Caledonia New Zealand Nicaragua Niger Nigeria Niue Norfolk Island North Korea North Macedonia Northern Mariana Islands Norway Oman Pakistan Palau Palestinian Territory Panama Papua New Guinea Paraguay Peru Philippines Pitcairn Poland Portugal Puerto Rico Qatar Reunion Romania Russian Federation Rwanda Saint Helena Saint Kitts and Nevis Saint Lucia Saint Pierre and Miquelon Saint Vincent and the Grenadines Samoa San Marino Sao Tome and Principe Saudi Arabia Senegal Serbia Seychelles Sierra Leone Singapore Slovakia Slovenia Solomon Islands Somalia South Africa South Georgia and the South Sandwich Is South Korea South Sudan Spain Sri Lanka Sudan Suriname Svalbard and Jan Mayen Swaziland Sweden Switzerland Syria Taiwan Tajikistan Tanzania Thailand The Gambia The Netherlands Timor-Leste Togo Tokelau Tonga Trinidad and Tobago Tunisia Turkey Turkmenistan Turks and Caicos Islands Tuvalu UK USA Uganda Ukraine United Arab Emirates United States Virgin Islands Uruguay Uzbekistan Vanuatu Venezuela Vietnam Wallis and Futuna West Bank and Gaza Strip Western Sahara Yemen Zambia Zimbabwe Please select your country/region. You must enter a comment. Competing Interests Please disclose any competing interests that might be construed to influence your judgment of the article's or peer review report's validity or importance. Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Please state your competing interests The comment has been saved. An error has occurred. Please try again. Cancel Post var lTitle = "Training and Experience in Study Selection...".replace("'", ''); var linkedInUrl = "http://www.linkedin.com/shareArticle?url=https://hrbopenresearch.org/articles/8-54/v3" + "&title=" + encodeURIComponent(lTitle) + "&summary=" + encodeURIComponent('Read the article by '); var deliciousUrl = "https://del.icio.us/post?url=https://hrbopenresearch.org/articles/8-54/v3&title=" + encodeURIComponent(lTitle); var redditUrl = "http://reddit.com/submit?url=https://hrbopenresearch.org/articles/8-54/v3" + "&title=" + encodeURIComponent(lTitle); linkedInUrl += encodeURIComponent('Ahern E et al.'); var offsetTop = /chrome/i.test( navigator.userAgent ) ? 4 : -10; var addthis_config = { ui_offset_top: offsetTop, services_compact : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_expanded : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_custom : [ { name: "LinkedIn", url: linkedInUrl, icon:"/img/icon/at_linkedin.svg" }, { name: "Mendeley", url: "http://www.mendeley.com/import/?url=https://hrbopenresearch.org/articles/8-54/v3/mendeley", icon:"/img/icon/at_mendeley.svg" }, { name: "Reddit", url: redditUrl, icon:"/img/icon/at_reddit.svg" }, ] }; var addthis_share = { url: "https://hrbopenresearch.org/articles/8-54", templates : { twitter : "Training and Experience in Study Selection (TESS): study protocol.... Ahern E et al., published by " + "@HRBOpenRes" + ", https://hrbopenresearch.org/articles/8-54/v3" } }; if (typeof(addthis) != "undefined"){ addthis.addEventListener('addthis.ready', checkCount); addthis.addEventListener('addthis.menu.share', checkCount); } $(".f1r-shares-twitter").attr("href", "https://twitter.com/intent/tweet?text=" + addthis_share.templates.twitter); $(".f1r-shares-facebook").attr("href", "https://www.facebook.com/sharer/sharer.php?u=" + addthis_share.url); $(".f1r-shares-linkedin").attr("href", addthis_config.services_custom[0].url); $(".f1r-shares-reddit").attr("href", addthis_config.services_custom[2].url); $(".f1r-shares-mendelay").attr("href", addthis_config.services_custom[1].url); function checkCount(){ setTimeout(function(){ $(".addthis_button_expanded").each(function(){ var count = $(this).text(); if (count !== "" && count != "0") $(this).removeClass("is-hidden"); else $(this).addClass("is-hidden"); }); }, 1000); } close How to cite this report {{reportCitation}} Cancel Copy Citation Details $(function(){R.ui.buttonDropdowns('.dropdown-for-downloads');}); $(function(){R.ui.toolbarDropdowns('.toolbar-dropdown-for-downloads');}); $.get("/articles/acj/14129/15892") new F1000.Clipboard(); new F1000.ThesaurusTermsDisplay("articles", "article", "15892"); $(document).ready(function() { $( "#frame1" ).on('load', function() { var mydiv = $(this).contents().find("div"); var h = mydiv.height(); console.log(h) }); var tooltipLivingFigure = jQuery(".interactive-living-figure-label .icon-more-info"), titleLivingFigure = tooltipLivingFigure.attr("title"); tooltipLivingFigure.simpletip({ fixed: true, position: ["-115", "30"], baseClass: 'small-tooltip', content:titleLivingFigure + " " }); tooltipLivingFigure.removeAttr("title"); $("body").on("click", ".cite-living-figure", function(e) { e.preventDefault(); var ref = $(this).attr("data-ref"); $(this).closest(".living-figure-list-container").find("#" + ref).fadeIn(200); }); $("body").on("click", ".close-cite-living-figure", function(e) { e.preventDefault(); $(this).closest(".popup-window-wrapper").fadeOut(200); }); $(document).on("mouseup", function(e) { var metricsContainer = $(".article-metrics-popover-wrapper"); if (!metricsContainer.is(e.target) && metricsContainer.has(e.target).length === 0) { $(".article-metrics-close-button").click(); } }); var articleId = $('#articleId').val(); if($("#main-article-count-box").attachArticleMetrics) { $("#main-article-count-box").attachArticleMetrics(articleId, { articleMetricsView: true }); } }); var figshareWidget = $(".new_figshare_widget"); if (figshareWidget.length > 0) { window.figshare.load("f1000", function(Widget) { // Select a tag/tags defined in your page. In this tag we will place the widget. _.map(figshareWidget, function(el){ var widget = new Widget({ articleId: $(el).attr("figshare_articleId") //height:300 // this is the height of the viewer part. [Default: 550] }); widget.initialize(); // initialize the widget widget.mount(el); // mount it in a tag that's on your page // this will save the widget on the global scope for later use from // your JS scripts. This line is optional. //window.widget = widget; }); }); } close Error Close Add Reset F1000.MICROSERVICES.AFFILIATION = ''; $(document).ready(function () { $('.js-affiliations-form').each((index, form) => { new AffiliationForm({ formId: form.id, institutionErrorSelector: '.comment-enter-institution', departmentErrorSelector: '.comment-enter-department', placeSelector: '.js-add-comment-place', stateSelector: '.js-add-comment-state', zipCodeSelector: '.js-add-comment-zipcode', countrySelector: '.js-add-comment-country', countryErrorSelector: '.comment-enter-country', }); }); }); $(document).ready(function () { var reportIds = { "54027": 0, "54028": 0, "54029": 0, "53773": 6, "46861": 0, "53774": 0, "54030": 0, "46862": 0, "53775": 0, "54031": 0, "46863": 0, "53776": 0, "54032": 0, "46864": 0, "53777": 0, "54033": 0, "46865": 0, "53778": 0, "54034": 0, "46866": 0, "53779": 0, "54035": 0, "46867": 0, "53780": 0, "54036": 0, "46868": 0, "53781": 0, "48661": 0, "46869": 0, "53782": 0, "48662": 0, "46870": 0, "48663": 0, "48664": 0, "48665": 0, "48666": 0, "48667": 0, "48668": 0, "48669": 0, "48670": 0, "54321": 0, "54322": 0, "54323": 0, "54324": 0, "54325": 0, "54326": 0, "54327": 0, "54328": 0, "54329": 0, "54330": 0, "47167": 0, "47168": 0, "47169": 0, "47170": 0, "47171": 0, "47172": 0, "47173": 0, "47174": 0, "47175": 0, "47176": 0, "50002": 0, "50003": 0, "50004": 0, "50005": 0, "50006": 0, "50007": 0, "50008": 0, "50009": 0, "50010": 0, "50011": 16, "53354": 0, "53355": 0, "53356": 0, "53357": 0, "53358": 0, "53359": 0, "53360": 0, "53361": 0, "53362": 0, "52889": 0, "47015": 0, "47016": 0, "47017": 0, "54186": 0, "47018": 0, "54187": 0, "47019": 0, "54188": 0, "47020": 0, "54189": 0, "47021": 0, "54190": 0, "47022": 0, "54191": 0, "47023": 0, "54192": 0, "47024": 0, "54193": 0, "54194": 0, "54195": 0, "54720": 0, "54721": 0, "47305": 0, "47306": 0, "47307": 0, "47308": 0, "47309": 0, "47310": 0, "47311": 0, "47312": 0, "47313": 0, "47314": 0, }; $(".referee-response-container,.js-referee-report").each(function(index, el) { var reportId = $(el).attr("data-reportid"), reportCount = reportIds[reportId] || 0; $(el).find(".comments-count-container,.js-referee-report-views").html(reportCount); }); var uuidInput = $("#article_uuid"), oldUUId = uuidInput.val(), newUUId = "8e5f9284-8c83-4b98-ba6b-f2924c49daff"; uuidInput.val(newUUId); $("a[href*='article_uuid=']").each(function(index, el) { var newHref = $(el).attr("href").replace(oldUUId, newUUId); $(el).attr("href", newHref); }); }); Are you a HRB-funded researcher? Submission to HRB Open Research is open to all HRB grantholders or people working on a HRB-funded/co-funded grant on or since 1 January 2017. Sign up for information about developments, publishing and publications from HRB Open Research. First Name * You must provide your first name Last Name * You must provide your last name Email * You must provide a valid email address Institution You must provide an institution. Submit Thank you! We'll keep you updated on any major new updates to HRB Open Research HRB Open Research Browse How to Publish About Contact RSS Cookie Notice Privacy Notice Legal Submit Your Research © F1000 Research Limited and its licensors ISSN 2515-4826 | Legal background var F1000platform = new F1000.Platform({ name: "hrb", displayName: "HRB Open Research", hostName: "hrbopenresearch.org", id: "5", editorialEmail: "[email protected]", infoEmail: "[email protected]" }); Sign In Remember me Forgotten your password? Sign In Cancel Email or password not correct. Please try again Please wait... $(function(){ // Note: All the setup needs to run against a name attribute and *not* the id due the clonish // nature of facebox... $("a[id=googleSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("GOOGLE"); $("form[id=oAuthForm]").submit(); }); $("a[id=facebookSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("FACEBOOK"); $("form[id=oAuthForm]").submit(); }); $("a[id=orcidSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("ORCID"); $("form[id=oAuthForm]").submit(); }); }); If you've forgotten your password, please enter your email address below and we'll send you instructions on how to reset your password. The email address should be the one you originally registered with F1000. Email address not valid, please try again You registered with F1000 via Google, so we cannot reset your password. To sign in, please click here . If you still need help with your Google account password, please click here . You registered with F1000 via Facebook, so we cannot reset your password. To sign in, please click here . If you still need help with your Facebook account password, please click here . Code not correct, please try again Reset password Cancel Email us for further assistance. Server error, please try again. If your email address is registered with us, we will email you instructions to reset your password. If you think you should have received this email but it has not arrived, please check your spam filters and/or contact for further assistance. Please wait... Register $(document).ready(function () { signIn.createSignInAsRow($("#sign-in-form-gfb-popup")); $(".target-field").each(function () { var uris = $(this).val().split("/"); if (uris.pop() === "login") { $(this).val(uris.toString().replace(",","/")); } }); }); (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9fe3e1783b6a4193',t:'MTc3OTIwMTg3Nw=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();