Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources

Teaching critical thinking about health information... | F1000Research "use strict";function _typeof(t){return(_typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}!function(){var t=function(){var t,e,o=[],n=window,r=n;for(;r;){try{if(r.frames.__tcfapiLocator){t=r;break}}catch(t){}if(r===n.top)break;r=r.parent}t||(!function t(){var e=n.document,o=!!n.frames.__tcfapiLocator;if(!o)if(e.body){var r=e.createElement("iframe");r.style.cssText="display:none",r.name="__tcfapiLocator",e.body.appendChild(r)}else setTimeout(t,5);return!o}(),n.__tcfapi=function(){for(var t=arguments.length,n=new Array(t),r=0;r 3&&2===parseInt(n[1],10)&&"boolean"==typeof n[3]&&(e=n[3],"function"==typeof n[2]&&n[2]("set",!0)):"ping"===n[0]?"function"==typeof n[2]&&n[2]({gdprApplies:e,cmpLoaded:!1,cmpStatus:"stub"}):o.push(n)},n.addEventListener("message",(function(t){var e="string"==typeof t.data,o={};if(e)try{o=JSON.parse(t.data)}catch(t){}else o=t.data;var n="object"===_typeof(o)&&null!==o?o.__tcfapiCall:null;n&&window.__tcfapi(n.command,n.version,(function(o,r){var a={__tcfapiReturn:{returnValue:o,success:r,callId:n.callId}};t&&t.source&&t.source.postMessage&&t.source.postMessage(e?JSON.stringify(a):a,"*")}),n.parameter)}),!1))};"undefined"!=typeof module?module.exports=t:t()}(); dataLayer = dataLayer || []; // Standard GTM initialization - Google Consent Mode handles consent automatically (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl+ '>m_auth=hzk0Vc3qFsQYhCrIoHz68A>m_preview=env-1>m_cookies_win=x';f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-MWFK8L5J'); ;window.NREUM||(NREUM={});NREUM.init={distributed_tracing:{enabled:true},privacy:{cookies_enabled:true},ajax:{deny_list:["bam.nr-data.net"]}}; ;NREUM.loader_config={accountID:"438030",trustKey:"438030",agentID:"772317073",licenseKey:"97f8f67f26",applicationID:"772317073"} ;NREUM.info={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net",licenseKey:"97f8f67f26",applicationID:"772317073",sa:1} ;/*! For license information please see nr-loader-spa-1.236.0.min.js.LICENSE.txt */ (()=>{"use strict";var e,t,r={5763:(e,t,r)=>{r.d(t,{P_:()=>l,Mt:()=>g,C5:()=>s,DL:()=>v,OP:()=>T,lF:()=>D,Yu:()=>y,Dg:()=>h,CX:()=>c,GE:()=>b,sU:()=>_});var n=r(8632),i=r(9567);const o={beacon:n.ce.beacon,errorBeacon:n.ce.errorBeacon,licenseKey:void 0,applicationID:void 0,sa:void 0,queueTime:void 0,applicationTime:void 0,ttGuid:void 0,user:void 0,account:void 0,product:void 0,extra:void 0,jsAttributes:{},userAttributes:void 0,atts:void 0,transactionName:void 0,tNamePlain:void 0},a={};function s(e){if(!e)throw new Error("All info objects require an agent identifier!");if(!a[e])throw new Error("Info for ".concat(e," was never set"));return a[e]}function c(e,t){if(!e)throw new Error("All info objects require an agent identifier!");a[e]=(0,i.D)(t,o),(0,n.Qy)(e,a[e],"info")}var u=r(7056);const d=()=>{const e={blockSelector:"[data-nr-block]",maskInputOptions:{password:!0}};return{allow_bfcache:!0,privacy:{cookies_enabled:!0},ajax:{deny_list:void 0,enabled:!0,harvestTimeSeconds:10},distributed_tracing:{enabled:void 0,exclude_newrelic_header:void 0,cors_use_newrelic_header:void 0,cors_use_tracecontext_headers:void 0,allowed_origins:void 0},session:{domain:void 0,expiresMs:u.oD,inactiveMs:u.Hb},ssl:void 0,obfuscate:void 0,jserrors:{enabled:!0,harvestTimeSeconds:10},metrics:{enabled:!0},page_action:{enabled:!0,harvestTimeSeconds:30},page_view_event:{enabled:!0},page_view_timing:{enabled:!0,harvestTimeSeconds:30,long_task:!1},session_trace:{enabled:!0,harvestTimeSeconds:10},harvest:{tooManyRequestsDelay:60},session_replay:{enabled:!1,harvestTimeSeconds:60,sampleRate:.1,errorSampleRate:.1,maskTextSelector:"*",maskAllInputs:!0,get blockClass(){return"nr-block"},get ignoreClass(){return"nr-ignore"},get maskTextClass(){return"nr-mask"},get blockSelector(){return e.blockSelector},set blockSelector(t){e.blockSelector+=",".concat(t)},get maskInputOptions(){return e.maskInputOptions},set maskInputOptions(t){e.maskInputOptions={...t,password:!0}}},spa:{enabled:!0,harvestTimeSeconds:10}}},f={};function l(e){if(!e)throw new Error("All configuration objects require an agent identifier!");if(!f[e])throw new Error("Configuration for ".concat(e," was never set"));return f[e]}function h(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");f[e]=(0,i.D)(t,d()),(0,n.Qy)(e,f[e],"config")}function g(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");var r=l(e);if(r){for(var n=t.split("."),i=0;i {r.d(t,{D:()=>i});var n=r(50);function i(e,t){try{if(!e||"object"!=typeof e)return(0,n.Z)("Setting a Configurable requires an object as input");if(!t||"object"!=typeof t)return(0,n.Z)("Setting a Configurable requires a model to set its initial properties");const r=Object.create(Object.getPrototypeOf(t),Object.getOwnPropertyDescriptors(t)),o=0===Object.keys(r).length?e:r;for(let a in o)if(void 0!==e[a])try{"object"==typeof e[a]&&"object"==typeof t[a]?r[a]=i(e[a],t[a]):r[a]=e[a]}catch(e){(0,n.Z)("An error occurred while setting a property of a Configurable",e)}return r}catch(e){(0,n.Z)("An error occured while setting a Configurable",e)}}},6818:(e,t,r)=>{r.d(t,{Re:()=>i,gF:()=>o,q4:()=>n});const n="1.236.0",i="PROD",o="CDN"},385:(e,t,r)=>{r.d(t,{FN:()=>a,IF:()=>u,Nk:()=>f,Tt:()=>s,_A:()=>o,il:()=>n,pL:()=>c,v6:()=>i,w1:()=>d});const n="undefined"!=typeof window&&!!window.document,i="undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self.navigator instanceof WorkerNavigator||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis.navigator instanceof WorkerNavigator),o=n?window:"undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis),a=""+o?.location,s=/iPad|iPhone|iPod/.test(navigator.userAgent),c=s&&"undefined"==typeof SharedWorker,u=(()=>{const e=navigator.userAgent.match(/Firefox[/\s](\d+\.\d+)/);return Array.isArray(e)&&e.length>=2?+e[1]:0})(),d=Boolean(n&&window.document.documentMode),f=!!navigator.sendBeacon},1117:(e,t,r)=>{r.d(t,{w:()=>o});var n=r(50);const i={agentIdentifier:"",ee:void 0};class o{constructor(e){try{if("object"!=typeof e)return(0,n.Z)("shared context requires an object as input");this.sharedContext={},Object.assign(this.sharedContext,i),Object.entries(e).forEach((e=>{let[t,r]=e;Object.keys(i).includes(t)&&(this.sharedContext[t]=r)}))}catch(e){(0,n.Z)("An error occured while setting SharedContext",e)}}}},8e3:(e,t,r)=>{r.d(t,{L:()=>d,R:()=>c});var n=r(2177),i=r(1284),o=r(4322),a=r(3325);const s={};function c(e,t){const r={staged:!1,priority:a.p[t]||0};u(e),s[e].get(t)||s[e].set(t,r)}function u(e){e&&(s[e]||(s[e]=new Map))}function d(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"",t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"feature";if(u(e),!e||!s[e].get(t))return a(t);s[e].get(t).staged=!0;const r=[...s[e]];function a(t){const r=e?n.ee.get(e):n.ee,a=o.X.handlers;if(r.backlog&&a){var s=r.backlog[t],c=a[t];if(c){for(var u=0;s&&u {let[t,r]=e;return r.staged}))&&(r.sort(((e,t)=>e[1].priority-t[1].priority)),r.forEach((e=>{let[t]=e;a(t)})))}function f(e,t){var r=e[1];(0,i.D)(t[r],(function(t,r){var n=e[0];if(r[0]===n){var i=r[1],o=e[3],a=e[2];i.apply(o,a)}}))}},2177:(e,t,r)=>{r.d(t,{c:()=>f,ee:()=>u});var n=r(8632),i=r(2210),o=r(1284),a=r(5763),s="nr@context";let c=(0,n.fP)();var u;function d(){}function f(e){return(0,i.X)(e,s,l)}function l(){return new d}function h(){u.aborted=!0,u.backlog={}}c.ee?u=c.ee:(u=function e(t,r){var n={},c={},f={},g=!1;try{g=16===r.length&&(0,a.OP)(r).isolatedBacklog}catch(e){}var p={on:b,addEventListener:b,removeEventListener:y,emit:v,get:x,listeners:w,context:m,buffer:A,abort:h,aborted:!1,isBuffering:E,debugId:r,backlog:g?{}:t&&"object"==typeof t.backlog?t.backlog:{}};return p;function m(e){return e&&e instanceof d?e:e?(0,i.X)(e,s,l):l()}function v(e,r,n,i,o){if(!1!==o&&(o=!0),!u.aborted||i){t&&o&&t.emit(e,r,n);for(var a=m(n),s=w(e),d=s.length,f=0;fn,p:()=>i});var n=r(2177).ee.get("handle");function i(e,t,r,i,o){o?(o.buffer([e],i),o.emit(e,t,r)):(n.buffer([e],i),n.emit(e,t,r))}},4322:(e,t,r)=>{r.d(t,{X:()=>o});var n=r(5546);o.on=a;var i=o.handlers={};function o(e,t,r,o){a(o||n.E,i,e,t,r)}function a(e,t,r,i,o){o||(o="feature"),e||(e=n.E);var a=t[o]=t[o]||{};(a[r]=a[r]||[]).push([e,i])}},3239:(e,t,r)=>{r.d(t,{bP:()=>s,iz:()=>c,m$:()=>a});var n=r(385);let i=!1,o=!1;try{const e={get passive(){return i=!0,!1},get signal(){return o=!0,!1}};n._A.addEventListener("test",null,e),n._A.removeEventListener("test",null,e)}catch(e){}function a(e,t){return i||o?{capture:!!e,passive:i,signal:t}:!!e}function s(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;window.addEventListener(e,t,a(r,n))}function c(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;document.addEventListener(e,t,a(r,n))}},4402:(e,t,r)=>{r.d(t,{Ht:()=>u,M:()=>c,Rl:()=>a,ky:()=>s});var n=r(385);const i="xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx";function o(e,t){return e?15&e[t]:16*Math.random()|0}function a(){const e=n._A?.crypto||n._A?.msCrypto;let t,r=0;return e&&e.getRandomValues&&(t=e.getRandomValues(new Uint8Array(31))),i.split("").map((e=>"x"===e?o(t,++r).toString(16):"y"===e?(3&o()|8).toString(16):e)).join("")}function s(e){const t=n._A?.crypto||n._A?.msCrypto;let r,i=0;t&&t.getRandomValues&&(r=t.getRandomValues(new Uint8Array(31)));const a=[];for(var s=0;s {r.d(t,{Bq:()=>n,Hb:()=>o,oD:()=>i});const n="NRBA",i=144e5,o=18e5},7894:(e,t,r)=>{function n(){return Math.round(performance.now())}r.d(t,{z:()=>n})},7243:(e,t,r)=>{r.d(t,{e:()=>o});var n=r(385),i={};function o(e){if(e in i)return i[e];if(0===(e||"").indexOf("data:"))return{protocol:"data"};let t;var r=n._A?.location,o={};if(n.il)t=document.createElement("a"),t.href=e;else try{t=new URL(e,r.href)}catch(e){return o}o.port=t.port;var a=t.href.split("://");!o.port&&a[1]&&(o.port=a[1].split("/")[0].split("@").pop().split(":")[1]),o.port&&"0"!==o.port||(o.port="https"===a[0]?"443":"80"),o.hostname=t.hostname||r.hostname,o.pathname=t.pathname,o.protocol=a[0],"/"!==o.pathname.charAt(0)&&(o.pathname="/"+o.pathname);var s=!t.protocol||":"===t.protocol||t.protocol===r.protocol,c=t.hostname===r.hostname&&t.port===r.port;return o.sameOrigin=s&&(!t.hostname||c),"/"===o.pathname&&(i[e]=o),o}},50:(e,t,r)=>{function n(e,t){"function"==typeof console.warn&&(console.warn("New Relic: ".concat(e)),t&&console.warn(t))}r.d(t,{Z:()=>n})},2587:(e,t,r)=>{r.d(t,{N:()=>c,T:()=>u});var n=r(2177),i=r(5546),o=r(8e3),a=r(3325);const s={stn:[a.D.sessionTrace],err:[a.D.jserrors,a.D.metrics],ins:[a.D.pageAction],spa:[a.D.spa],sr:[a.D.sessionReplay,a.D.sessionTrace]};function c(e,t){const r=n.ee.get(t);e&&"object"==typeof e&&(Object.entries(e).forEach((e=>{let[t,n]=e;void 0===u[t]&&(s[t]?s[t].forEach((e=>{n?(0,i.p)("feat-"+t,[],void 0,e,r):(0,i.p)("block-"+t,[],void 0,e,r),(0,i.p)("rumresp-"+t,[Boolean(n)],void 0,e,r)})):n&&(0,i.p)("feat-"+t,[],void 0,void 0,r),u[t]=Boolean(n))})),Object.keys(s).forEach((e=>{void 0===u[e]&&(s[e]?.forEach((t=>(0,i.p)("rumresp-"+e,[!1],void 0,t,r))),u[e]=!1)})),(0,o.L)(t,a.D.pageViewEvent))}const u={}},2210:(e,t,r)=>{r.d(t,{X:()=>i});var n=Object.prototype.hasOwnProperty;function i(e,t,r){if(n.call(e,t))return e[t];var i=r();if(Object.defineProperty&&Object.keys)try{return Object.defineProperty(e,t,{value:i,writable:!0,enumerable:!1}),i}catch(e){}return e[t]=i,i}},1284:(e,t,r)=>{r.d(t,{D:()=>n});const n=(e,t)=>Object.entries(e||{}).map((e=>{let[r,n]=e;return t(r,n)}))},4351:(e,t,r)=>{r.d(t,{P:()=>o});var n=r(2177);const i=()=>{const e=new WeakSet;return(t,r)=>{if("object"==typeof r&&null!==r){if(e.has(r))return;e.add(r)}return r}};function o(e){try{return JSON.stringify(e,i())}catch(e){try{n.ee.emit("internal-error",[e])}catch(e){}}}},3960:(e,t,r)=>{r.d(t,{K:()=>a,b:()=>o});var n=r(3239);function i(){return"undefined"==typeof document||"complete"===document.readyState}function o(e,t){if(i())return e();(0,n.bP)("load",e,t)}function a(e){if(i())return e();(0,n.iz)("DOMContentLoaded",e)}},8632:(e,t,r)=>{r.d(t,{EZ:()=>u,Qy:()=>c,ce:()=>o,fP:()=>a,gG:()=>d,mF:()=>s});var n=r(7894),i=r(385);const o={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net"};function a(){return i._A.NREUM||(i._A.NREUM={}),void 0===i._A.newrelic&&(i._A.newrelic=i._A.NREUM),i._A.NREUM}function s(){let e=a();return e.o||(e.o={ST:i._A.setTimeout,SI:i._A.setImmediate,CT:i._A.clearTimeout,XHR:i._A.XMLHttpRequest,REQ:i._A.Request,EV:i._A.Event,PR:i._A.Promise,MO:i._A.MutationObserver,FETCH:i._A.fetch}),e}function c(e,t,r){let i=a();const o=i.initializedAgents||{},s=o[e]||{};return Object.keys(s).length||(s.initializedAt={ms:(0,n.z)(),date:new Date}),i.initializedAgents={...o,[e]:{...s,[r]:t}},i}function u(e,t){a()[e]=t}function d(){return function(){let e=a();const t=e.info||{};e.info={beacon:o.beacon,errorBeacon:o.errorBeacon,...t}}(),function(){let e=a();const t=e.init||{};e.init={...t}}(),s(),function(){let e=a();const t=e.loader_config||{};e.loader_config={...t}}(),a()}},7956:(e,t,r)=>{r.d(t,{N:()=>i});var n=r(3239);function i(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],r=arguments.length>2?arguments[2]:void 0,i=arguments.length>3?arguments[3]:void 0;return void(0,n.iz)("visibilitychange",(function(){if(t)return void("hidden"==document.visibilityState&&e());e(document.visibilityState)}),r,i)}},1214:(e,t,r)=>{r.d(t,{em:()=>v,u5:()=>N,QU:()=>S,_L:()=>I,Gm:()=>L,Lg:()=>M,gy:()=>U,BV:()=>Q,Kf:()=>ee});var n=r(2177);const i="nr@original";var o=Object.prototype.hasOwnProperty,a=!1;function s(e,t){return e||(e=n.ee),r.inPlace=function(e,t,n,i,o){n||(n="");var a,s,c,u="-"===n.charAt(0);for(c=0;c 2?n-2:0),o=2;o {r(A[T],e,w),r(E[T],e,w)})),r(l._A,"fetch",y),t.on(y+"end",(function(e,r){var n=this;if(r){var i=r.headers.get("content-length");null!==i&&(n.rxSize=i),t.emit(y+"done",[null,r],n)}else t.emit(y+"done",[e],n)})),t}const O={},j=["pushState","replaceState"];function S(e){const t=function(e){return(e||n.ee).get("history")}(e);return!l.il||O[t.debugId]++||(O[t.debugId]=1,s(t).inPlace(window.history,j,"-")),t}var P=r(3239);const C={},R=["appendChild","insertBefore","replaceChild"];function I(e){const t=function(e){return(e||n.ee).get("jsonp")}(e);if(!l.il||C[t.debugId])return t;C[t.debugId]=!0;var r=s(t),i=/[?&](?:callback|cb)=([^&#]+)/,o=/(.*)\.([^.]+)/,a=/^(\w+)(\.|$)(.*)$/;function c(e,t){var r=e.match(a),n=r[1],i=r[3];return i?c(i,t[n]):t[n]}return r.inPlace(Node.prototype,R,"dom-"),t.on("dom-start",(function(e){!function(e){if(!e||"string"!=typeof e.nodeName||"script"!==e.nodeName.toLowerCase())return;if("function"!=typeof e.addEventListener)return;var n=(a=e.src,s=a.match(i),s?s[1]:null);var a,s;if(!n)return;var u=function(e){var t=e.match(o);if(t&&t.length>=3)return{key:t[2],parent:c(t[1],window)};return{key:e,parent:window}}(n);if("function"!=typeof u.parent[u.key])return;var d={};function f(){t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}function l(){t.emit("jsonp-error",[],d),t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}r.inPlace(u.parent,[u.key],"cb-",d),e.addEventListener("load",f,(0,P.m$)(!1)),e.addEventListener("error",l,(0,P.m$)(!1)),t.emit("new-jsonp",[e.src],d)}(e[0])})),t}var k=r(5763);const H={};function L(e){const t=function(e){return(e||n.ee).get("mutation")}(e);if(!l.il||H[t.debugId])return t;H[t.debugId]=!0;var r=s(t),i=k.Yu.MO;return i&&(window.MutationObserver=function(e){return this instanceof i?new i(r(e,"fn-")):i.apply(this,arguments)},MutationObserver.prototype=i.prototype),t}const z={};function M(e){const t=function(e){return(e||n.ee).get("promise")}(e);if(z[t.debugId])return t;z[t.debugId]=!0;var r=n.c,o=s(t),a=k.Yu.PR;return a&&function(){function e(r){var n=t.context(),i=o(r,"executor-",n,null,!1);const s=Reflect.construct(a,[i],e);return t.context(s).getCtx=function(){return n},s}l._A.Promise=e,Object.defineProperty(e,"name",{value:"Promise"}),e.toString=function(){return a.toString()},Object.setPrototypeOf(e,a),["all","race"].forEach((function(r){const n=a[r];e[r]=function(e){let i=!1;[...e||[]].forEach((e=>{this.resolve(e).then(a("all"===r),a(!1))}));const o=n.apply(this,arguments);return o;function a(e){return function(){t.emit("propagate",[null,!i],o,!1,!1),i=i||!e}}}})),["resolve","reject"].forEach((function(r){const n=a[r];e[r]=function(e){const r=n.apply(this,arguments);return e!==r&&t.emit("propagate",[e,!0],r,!1,!1),r}})),e.prototype=a.prototype;const n=a.prototype.then;a.prototype.then=function(){var e=this,i=r(e);i.promise=e;for(var a=arguments.length,s=new Array(a),c=0;c e())),t};function m(e,t){i.inPlace(t,["onreadystatechange"],"fn-",E)}function b(){var e=this,t=r.context(e);e.readyState>3&&!t.resolved&&(t.resolved=!0,r.emit("xhr-resolved",[],e)),i.inPlace(e,f,"fn-",E)}if(function(e,t){for(var r in e)t[r]=e[r]}(o,p),p.prototype=o.prototype,i.inPlace(p.prototype,J,"-xhr-",E),r.on("send-xhr-start",(function(e,t){m(e,t),function(e){h.push(e),a&&(y?y.then(A):u?u(A):(w=-w,x.data=w))}(t)})),r.on("open-xhr-start",m),a){var y=c&&c.resolve();if(!u&&!c){var w=1,x=document.createTextNode(w);new a(A).observe(x,{characterData:!0})}}else t.on("fn-end",(function(e){e[0]&&e[0].type===d||A()}));function A(){for(var e=0;e {r.d(t,{t:()=>n});const n=r(3325).D.ajax},6660:(e,t,r)=>{r.d(t,{A:()=>i,t:()=>n});const n=r(3325).D.jserrors,i="nr@seenError"},3081:(e,t,r)=>{r.d(t,{gF:()=>o,mY:()=>i,t9:()=>n,vz:()=>s,xS:()=>a});const n=r(3325).D.metrics,i="sm",o="cm",a="storeSupportabilityMetrics",s="storeEventMetrics"},4649:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageAction},7633:(e,t,r)=>{r.d(t,{Dz:()=>i,OJ:()=>a,qw:()=>o,t9:()=>n});const n=r(3325).D.pageViewEvent,i="firstbyte",o="domcontent",a="windowload"},9251:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageViewTiming},3614:(e,t,r)=>{r.d(t,{BST_RESOURCE:()=>i,END:()=>s,FEATURE_NAME:()=>n,FN_END:()=>u,FN_START:()=>c,PUSH_STATE:()=>d,RESOURCE:()=>o,START:()=>a});const n=r(3325).D.sessionTrace,i="bstResource",o="resource",a="-start",s="-end",c="fn"+a,u="fn"+s,d="pushState"},7836:(e,t,r)=>{r.d(t,{BODY:()=>A,CB_END:()=>E,CB_START:()=>u,END:()=>x,FEATURE_NAME:()=>i,FETCH:()=>_,FETCH_BODY:()=>v,FETCH_DONE:()=>m,FETCH_START:()=>p,FN_END:()=>c,FN_START:()=>s,INTERACTION:()=>l,INTERACTION_API:()=>d,INTERACTION_EVENTS:()=>o,JSONP_END:()=>b,JSONP_NODE:()=>g,JS_TIME:()=>T,MAX_TIMER_BUDGET:()=>a,REMAINING:()=>f,SPA_NODE:()=>h,START:()=>w,originalSetTimeout:()=>y});var n=r(5763);const i=r(3325).D.spa,o=["click","submit","keypress","keydown","keyup","change"],a=999,s="fn-start",c="fn-end",u="cb-start",d="api-ixn-",f="remaining",l="interaction",h="spaNode",g="jsonpNode",p="fetch-start",m="fetch-done",v="fetch-body-",b="jsonp-end",y=n.Yu.ST,w="-start",x="-end",A="-body",E="cb"+x,T="jsTime",_="fetch"},5938:(e,t,r)=>{r.d(t,{W:()=>o});var n=r(5763),i=r(2177);class o{constructor(e,t,r){this.agentIdentifier=e,this.aggregator=t,this.ee=i.ee.get(e,(0,n.OP)(this.agentIdentifier).isolatedBacklog),this.featureName=r,this.blocked=!1}}},9144:(e,t,r)=>{r.d(t,{j:()=>m});var n=r(3325),i=r(5763),o=r(5546),a=r(2177),s=r(7894),c=r(8e3),u=r(3960),d=r(385),f=r(50),l=r(3081),h=r(8632);function g(){const e=(0,h.gG)();["setErrorHandler","finished","addToTrace","inlineHit","addRelease","addPageAction","setCurrentRouteName","setPageViewName","setCustomAttribute","interaction","noticeError","setUserId"].forEach((t=>{e[t]=function(){for(var r=arguments.length,n=new Array(r),i=0;i 1?r-1:0),i=1;i {e.exposed&&e.api[t]&&o.push(e.api[t](...n))})),o.length>1?o:o[0]}(t,...n)}}))}var p=r(2587);function m(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},m=arguments.length>2?arguments[2]:void 0,v=arguments.length>3?arguments[3]:void 0,{init:b,info:y,loader_config:w,runtime:x={loaderType:m},exposed:A=!0}=t;const E=(0,h.gG)();y||(b=E.init,y=E.info,w=E.loader_config),(0,i.Dg)(e,b||{}),(0,i.GE)(e,w||{}),(0,i.sU)(e,x),y.jsAttributes??={},d.v6&&(y.jsAttributes.isWorker=!0),(0,i.CX)(e,y),g();const T=function(e,t){t||(0,c.R)(e,"api");const h={};var g=a.ee.get(e),p=g.get("tracer"),m="api-",v=m+"ixn-";function b(t,r,n,o){const a=(0,i.C5)(e);return null===r?delete a.jsAttributes[t]:(0,i.CX)(e,{...a,jsAttributes:{...a.jsAttributes,[t]:r}}),x(m,n,!0,o||null===r?"session":void 0)(t,r)}function y(){}["setErrorHandler","finished","addToTrace","inlineHit","addRelease"].forEach((e=>h[e]=x(m,e,!0,"api"))),h.addPageAction=x(m,"addPageAction",!0,n.D.pageAction),h.setCurrentRouteName=x(m,"routeName",!0,n.D.spa),h.setPageViewName=function(t,r){if("string"==typeof t)return"/"!==t.charAt(0)&&(t="/"+t),(0,i.OP)(e).customTransaction=(r||"http://custom.transaction")+t,x(m,"setPageViewName",!0)()},h.setCustomAttribute=function(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2];if("string"==typeof e){if(["string","number"].includes(typeof t)||null===t)return b(e,t,"setCustomAttribute",r);(0,f.Z)("Failed to execute setCustomAttribute.\nNon-null value must be a string or number type, but a type of was provided."))}else(0,f.Z)("Failed to execute setCustomAttribute.\nName must be a string type, but a type of was provided."))},h.setUserId=function(e){if("string"==typeof e||null===e)return b("enduser.id",e,"setUserId",!0);(0,f.Z)("Failed to execute setUserId.\nNon-null value must be a string type, but a type of was provided."))},h.interaction=function(){return(new y).get()};var w=y.prototype={createTracer:function(e,t){var r={},i=this,a="function"==typeof t;return(0,o.p)(v+"tracer",[(0,s.z)(),e,r],i,n.D.spa,g),function(){if(p.emit((a?"":"no-")+"fn-start",[(0,s.z)(),i,a],r),a)try{return t.apply(this,arguments)}catch(e){throw p.emit("fn-err",[arguments,this,"string"==typeof e?new Error(e):e],r),e}finally{p.emit("fn-end",[(0,s.z)()],r)}}}};function x(e,t,r,i){return function(){return(0,o.p)(l.xS,["API/"+t+"/called"],void 0,n.D.metrics,g),i&&(0,o.p)(e+t,[(0,s.z)(),...arguments],r?null:this,i,g),r?void 0:this}}function A(){r.e(439).then(r.bind(r,7438)).then((t=>{let{setAPI:r}=t;r(e),(0,c.L)(e,"api")})).catch((()=>(0,f.Z)("Downloading runtime APIs failed...")))}return["actionText","setName","setAttribute","save","ignore","onEnd","getContext","end","get"].forEach((e=>{w[e]=x(v,e,void 0,n.D.spa)})),h.noticeError=function(e,t){"string"==typeof e&&(e=new Error(e)),(0,o.p)(l.xS,["API/noticeError/called"],void 0,n.D.metrics,g),(0,o.p)("err",[e,(0,s.z)(),!1,t],void 0,n.D.jserrors,g)},d.il?(0,u.b)((()=>A()),!0):A(),h}(e,v);return(0,h.Qy)(e,T,"api"),(0,h.Qy)(e,A,"exposed"),(0,h.EZ)("activatedFeatures",p.T),T}},3325:(e,t,r)=>{r.d(t,{D:()=>n,p:()=>i});const n={ajax:"ajax",jserrors:"jserrors",metrics:"metrics",pageAction:"page_action",pageViewEvent:"page_view_event",pageViewTiming:"page_view_timing",sessionReplay:"session_replay",sessionTrace:"session_trace",spa:"spa"},i={[n.pageViewEvent]:1,[n.pageViewTiming]:2,[n.metrics]:3,[n.jserrors]:4,[n.ajax]:5,[n.sessionTrace]:6,[n.pageAction]:7,[n.spa]:8,[n.sessionReplay]:9}}},n={};function i(e){var t=n[e];if(void 0!==t)return t.exports;var o=n[e]={exports:{}};return r[e](o,o.exports,i),o.exports}i.m=r,i.d=(e,t)=>{for(var r in t)i.o(t,r)&&!i.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},i.f={},i.e=e=>Promise.all(Object.keys(i.f).reduce(((t,r)=>(i.f[r](e,t),t)),[])),i.u=e=>(({78:"page_action-aggregate",147:"metrics-aggregate",242:"session-manager",317:"jserrors-aggregate",348:"page_view_timing-aggregate",412:"lazy-feature-loader",439:"async-api",538:"recorder",590:"session_replay-aggregate",675:"compressor",733:"session_trace-aggregate",786:"page_view_event-aggregate",873:"spa-aggregate",898:"ajax-aggregate"}[e]||e)+"."+{78:"ac76d497",147:"3dc53903",148:"1a20d5fe",242:"2a64278a",317:"49e41428",348:"bd6de33a",412:"2f55ce66",439:"30bd804e",538:"1b18459f",590:"cf0efb30",675:"ae9f91a8",733:"83105561",786:"06482edd",860:"03a8b7a5",873:"e6b09d52",898:"998ef92b"}[e]+"-1.236.0.min.js"),i.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),e={},t="NRBA:",i.l=(r,n,o,a)=>{if(e[r])e[r].push(n);else{var s,c;if(void 0!==o)for(var u=document.getElementsByTagName("script"),d=0;d {s.onerror=s.onload=null,clearTimeout(h);var i=e[r];if(delete e[r],s.parentNode&&s.parentNode.removeChild(s),i&&i.forEach((e=>e(n))),t)return t(n)},h=setTimeout(l.bind(null,void 0,{type:"timeout",target:s}),12e4);s.onerror=l.bind(null,s.onerror),s.onload=l.bind(null,s.onload),c&&document.head.appendChild(s)}},i.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.j=364,i.p="https://js-agent.newrelic.com/",(()=>{var e={364:0,953:0};i.f.j=(t,r)=>{var n=i.o(e,t)?e[t]:void 0;if(0!==n)if(n)r.push(n[2]);else{var o=new Promise(((r,i)=>n=e[t]=[r,i]));r.push(n[2]=o);var a=i.p+i.u(t),s=new Error;i.l(a,(r=>{if(i.o(e,t)&&(0!==(n=e[t])&&(e[t]=void 0),n)){var o=r&&("load"===r.type?"missing":r.type),a=r&&r.target&&r.target.src;s.message="Loading chunk "+t+" failed.\n("+o+": "+a+")",s.name="ChunkLoadError",s.type=o,s.request=a,n[1](s)}}),"chunk-"+t,t)}};var t=(t,r)=>{var n,o,[a,s,c]=r,u=0;if(a.some((t=>0!==e[t]))){for(n in s)i.o(s,n)&&(i.m[n]=s[n]);if(c)c(i)}for(t&&t(r);u {i.r(o);var e=i(3325),t=i(5763);const r=Object.values(e.D);function n(e){const n={};return r.forEach((r=>{n[r]=function(e,r){return!1!==(0,t.Mt)(r,"".concat(e,".enabled"))}(r,e)})),n}var a=i(9144);var s=i(5546),c=i(385),u=i(8e3),d=i(5938),f=i(3960),l=i(50);class h extends d.W{constructor(e,t,r){let n=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];super(e,t,r),this.auto=n,this.abortHandler,this.featAggregate,this.onAggregateImported,n&&(0,u.R)(e,r)}importAggregator(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};if(this.featAggregate||!this.auto)return;const r=c.il&&!0===(0,t.Mt)(this.agentIdentifier,"privacy.cookies_enabled");let n;this.onAggregateImported=new Promise((e=>{n=e}));const o=async()=>{let t;try{if(r){const{setupAgentSession:e}=await Promise.all([i.e(860),i.e(242)]).then(i.bind(i,3228));t=e(this.agentIdentifier)}}catch(e){(0,l.Z)("A problem occurred when starting up session manager. This page will not start or extend any session.",e)}try{if(!this.shouldImportAgg(this.featureName,t))return void(0,u.L)(this.agentIdentifier,this.featureName);const{lazyFeatureLoader:r}=await i.e(412).then(i.bind(i,8582)),{Aggregate:o}=await r(this.featureName,"aggregate");this.featAggregate=new o(this.agentIdentifier,this.aggregator,e),n(!0)}catch(e){(0,l.Z)("Downloading and initializing ".concat(this.featureName," failed..."),e),this.abortHandler?.(),n(!1)}};c.il?(0,f.b)((()=>o()),!0):o()}shouldImportAgg(r,n){return r!==e.D.sessionReplay||!1!==(0,t.Mt)(this.agentIdentifier,"session_trace.enabled")&&(!!n?.isNew||!!n?.state.sessionReplay)}}var g=i(7633),p=i(7894);class m extends h{static featureName=g.t9;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];if(super(r,n,g.t9,i),("undefined"==typeof PerformanceNavigationTiming||c.Tt)&&"undefined"!=typeof PerformanceTiming){const n=(0,t.OP)(r);n[g.Dz]=Math.max(Date.now()-n.offset,0),(0,f.K)((()=>n[g.qw]=Math.max((0,p.z)()-n[g.Dz],0))),(0,f.b)((()=>{const t=(0,p.z)();n[g.OJ]=Math.max(t-n[g.Dz],0),(0,s.p)("timing",["load",t],void 0,e.D.pageViewTiming,this.ee)}))}this.importAggregator()}}var v=i(1117),b=i(1284);class y extends v.w{constructor(e){super(e),this.aggregatedData={}}store(e,t,r,n,i){var o=this.getBucket(e,t,r,i);return o.metrics=function(e,t){t||(t={count:0});return t.count+=1,(0,b.D)(e,(function(e,r){t[e]=w(r,t[e])})),t}(n,o.metrics),o}merge(e,t,r,n,i){var o=this.getBucket(e,t,n,i);if(o.metrics){var a=o.metrics;a.count+=r.count,(0,b.D)(r,(function(e,t){if("count"!==e){var n=a[e],i=r[e];i&&!i.c?a[e]=w(i.t,n):a[e]=function(e,t){if(!t)return e;t.c||(t=x(t.t));return t.min=Math.min(e.min,t.min),t.max=Math.max(e.max,t.max),t.t+=e.t,t.sos+=e.sos,t.c+=e.c,t}(i,a[e])}}))}else o.metrics=r}storeMetric(e,t,r,n){var i=this.getBucket(e,t,r);return i.stats=w(n,i.stats),i}getBucket(e,t,r,n){this.aggregatedData[e]||(this.aggregatedData[e]={});var i=this.aggregatedData[e][t];return i||(i=this.aggregatedData[e][t]={params:r||{}},n&&(i.custom=n)),i}get(e,t){return t?this.aggregatedData[e]&&this.aggregatedData[e][t]:this.aggregatedData[e]}take(e){for(var t={},r="",n=!1,i=0;i t.max&&(t.max=e),e 2&&void 0!==arguments[2])||arguments[2];super(e,r,j.t,n),c.il&&((0,t.OP)(e).initHidden=Boolean("hidden"===document.visibilityState),(0,N.N)((()=>(0,s.p)("docHidden",[(0,p.z)()],void 0,j.t,this.ee)),!0),(0,O.bP)("pagehide",(()=>(0,s.p)("winPagehide",[(0,p.z)()],void 0,j.t,this.ee))),this.importAggregator())}}var P=i(3081);class C extends h{static featureName=P.t9;constructor(e,t){let r=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(e,t,P.t9,r),this.importAggregator()}}var R,I=i(2210),k=i(1214),H=i(2177),L={};try{R=localStorage.getItem("__nr_flags").split(","),console&&"function"==typeof console.log&&(L.console=!0,-1!==R.indexOf("dev")&&(L.dev=!0),-1!==R.indexOf("nr_dev")&&(L.nrDev=!0))}catch(e){}function z(e){try{L.console&&z(e)}catch(e){}}L.nrDev&&H.ee.on("internal-error",(function(e){z(e.stack)})),L.dev&&H.ee.on("fn-err",(function(e,t,r){z(r.stack)})),L.dev&&(z("NR AGENT IN DEVELOPMENT MODE"),z("flags: "+(0,b.D)(L,(function(e,t){return e})).join(", ")));var M=i(6660);class B extends h{static featureName=M.t;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(r,n,M.t,i),this.skipNext=0;try{this.removeOnAbort=new AbortController}catch(e){}const o=this;o.ee.on("fn-start",(function(e,t,r){o.abortHandler&&(o.skipNext+=1)})),o.ee.on("fn-err",(function(t,r,n){o.abortHandler&&!n[M.A]&&((0,I.X)(n,M.A,(function(){return!0})),this.thrown=!0,(0,s.p)("err",[n,(0,p.z)()],void 0,e.D.jserrors,o.ee))})),o.ee.on("fn-end",(function(){o.abortHandler&&!this.thrown&&o.skipNext>0&&(o.skipNext-=1)})),o.ee.on("internal-error",(function(t){(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,o.ee)})),this.origOnerror=c._A.onerror,c._A.onerror=this.onerrorHandler.bind(this),c._A.addEventListener("unhandledrejection",(t=>{const r=function(e){let t="Unhandled Promise Rejection: ";if(e instanceof Error)try{return e.message=t+e.message,e}catch(t){return e}if(void 0===e)return new Error(t);try{return new Error(t+(0,D.P)(e))}catch(e){return new Error(t)}}(t.reason);(0,s.p)("err",[r,(0,p.z)(),!1,{unhandledPromiseRejection:1}],void 0,e.D.jserrors,this.ee)}),(0,O.m$)(!1,this.removeOnAbort?.signal)),(0,k.gy)(this.ee),(0,k.BV)(this.ee),(0,k.em)(this.ee),(0,t.OP)(r).xhrWrappable&&(0,k.Kf)(this.ee),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}onerrorHandler(t,r,n,i,o){"function"==typeof this.origOnerror&&this.origOnerror(...arguments);try{this.skipNext?this.skipNext-=1:(0,s.p)("err",[o||new F(t,r,n),(0,p.z)()],void 0,e.D.jserrors,this.ee)}catch(t){try{(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,this.ee)}catch(e){}}return!1}}function F(e,t,r){this.message=e||"Uncaught error with no additional information",this.sourceURL=t,this.line=r}let U=1;const q="nr@id";function G(e){const t=typeof e;return!e||"object"!==t&&"function"!==t?-1:e===c._A?0:(0,I.X)(e,q,(function(){return U++}))}function V(e){if("string"==typeof e&&e.length)return e.length;if("object"==typeof e){if("undefined"!=typeof ArrayBuffer&&e instanceof ArrayBuffer&&e.byteLength)return e.byteLength;if("undefined"!=typeof Blob&&e instanceof Blob&&e.size)return e.size;if(!("undefined"!=typeof FormData&&e instanceof FormData))try{return(0,D.P)(e).length}catch(e){return}}}var X=i(7243);class W{constructor(e){this.agentIdentifier=e,this.generateTracePayload=this.generateTracePayload.bind(this),this.shouldGenerateTrace=this.shouldGenerateTrace.bind(this)}generateTracePayload(e){if(!this.shouldGenerateTrace(e))return null;var r=(0,t.DL)(this.agentIdentifier);if(!r)return null;var n=(r.accountID||"").toString()||null,i=(r.agentID||"").toString()||null,o=(r.trustKey||"").toString()||null;if(!n||!i)return null;var a=(0,_.M)(),s=(0,_.Ht)(),c=Date.now(),u={spanId:a,traceId:s,timestamp:c};return(e.sameOrigin||this.isAllowedOrigin(e)&&this.useTraceContextHeadersForCors())&&(u.traceContextParentHeader=this.generateTraceContextParentHeader(a,s),u.traceContextStateHeader=this.generateTraceContextStateHeader(a,c,n,i,o)),(e.sameOrigin&&!this.excludeNewrelicHeader()||!e.sameOrigin&&this.isAllowedOrigin(e)&&this.useNewrelicHeaderForCors())&&(u.newrelicHeader=this.generateTraceHeader(a,s,c,n,i,o)),u}generateTraceContextParentHeader(e,t){return"00-"+t+"-"+e+"-01"}generateTraceContextStateHeader(e,t,r,n,i){return i+"@nr=0-1-"+r+"-"+n+"-"+e+"----"+t}generateTraceHeader(e,t,r,n,i,o){if(!("function"==typeof c._A?.btoa))return null;var a={v:[0,1],d:{ty:"Browser",ac:n,ap:i,id:e,tr:t,ti:r}};return o&&n!==o&&(a.d.tk=o),btoa((0,D.P)(a))}shouldGenerateTrace(e){return this.isDtEnabled()&&this.isAllowedOrigin(e)}isAllowedOrigin(e){var r=!1,n={};if((0,t.Mt)(this.agentIdentifier,"distributed_tracing")&&(n=(0,t.P_)(this.agentIdentifier).distributed_tracing),e.sameOrigin)r=!0;else if(n.allowed_origins instanceof Array)for(var i=0;i 2&&void 0!==arguments[2])||arguments[2];super(r,n,Z.t,i),(0,t.OP)(r).xhrWrappable&&(this.dt=new W(r),this.handler=(e,t,r,n)=>(0,s.p)(e,t,r,n,this.ee),(0,k.u5)(this.ee),(0,k.Kf)(this.ee),function(r,n,i,o){function a(e){var t=this;t.totalCbs=0,t.called=0,t.cbTime=0,t.end=E,t.ended=!1,t.xhrGuids={},t.lastSize=null,t.loadCaptureCalled=!1,t.params=this.params||{},t.metrics=this.metrics||{},e.addEventListener("load",(function(r){_(t,e)}),(0,O.m$)(!1)),c.IF||e.addEventListener("progress",(function(e){t.lastSize=e.loaded}),(0,O.m$)(!1))}function s(e){this.params={method:e[0]},T(this,e[1]),this.metrics={}}function u(e,n){var i=(0,t.DL)(r);i.xpid&&this.sameOrigin&&n.setRequestHeader("X-NewRelic-ID",i.xpid);var a=o.generateTracePayload(this.parsedOrigin);if(a){var s=!1;a.newrelicHeader&&(n.setRequestHeader("newrelic",a.newrelicHeader),s=!0),a.traceContextParentHeader&&(n.setRequestHeader("traceparent",a.traceContextParentHeader),a.traceContextStateHeader&&n.setRequestHeader("tracestate",a.traceContextStateHeader),s=!0),s&&(this.dt=a)}}function d(e,t){var r=this.metrics,i=e[0],o=this;if(r&&i){var a=V(i);a&&(r.txSize=a)}this.startTime=(0,p.z)(),this.listener=function(e){try{"abort"!==e.type||o.loadCaptureCalled||(o.params.aborted=!0),("load"!==e.type||o.called===o.totalCbs&&(o.onloadCalled||"function"!=typeof t.onload)&&"function"==typeof o.end)&&o.end(t)}catch(e){try{n.emit("internal-error",[e])}catch(e){}}};for(var s=0;s 1?e[1]=i:e.push(i)}else e[0]&&e[0].headers&&s(e[0].headers,n)&&(this.dt=n);function s(e,t){var r=!1;return t.newrelicHeader&&(e.set("newrelic",t.newrelicHeader),r=!0),t.traceContextParentHeader&&(e.set("traceparent",t.traceContextParentHeader),t.traceContextStateHeader&&e.set("tracestate",t.traceContextStateHeader),r=!0),r}}function x(e,t){this.params={},this.metrics={},this.startTime=(0,p.z)(),this.dt=t,e.length>=1&&(this.target=e[0]),e.length>=2&&(this.opts=e[1]);var r,n=this.opts||{},i=this.target;"string"==typeof i?r=i:"object"==typeof i&&i instanceof Y?r=i.url:c._A?.URL&&"object"==typeof i&&i instanceof URL&&(r=i.href),T(this,r);var o=(""+(i&&i instanceof Y&&i.method||n.method||"GET")).toUpperCase();this.params.method=o,this.txSize=V(n.body)||0}function A(t,r){var n;this.endTime=(0,p.z)(),this.params||(this.params={}),this.params.status=r?r.status:0,"string"==typeof this.rxSize&&this.rxSize.length>0&&(n=+this.rxSize);var o={txSize:this.txSize,rxSize:n,duration:(0,p.z)()-this.startTime};i("xhr",[this.params,o,this.startTime,this.endTime,"fetch"],this,e.D.ajax)}function E(t){var r=this.params,n=this.metrics;if(!this.ended){this.ended=!0;for(var o=0;o 2&&void 0!==arguments[2])||arguments[2];super(e,t,we.t,r),this.importAggregator()}}new class{constructor(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:(0,_.ky)(16);c._A?(this.agentIdentifier=t,this.sharedAggregator=new y({agentIdentifier:this.agentIdentifier}),this.features={},this.desiredFeatures=new Set(e.features||[]),this.desiredFeatures.add(m),Object.assign(this,(0,a.j)(this.agentIdentifier,e,e.loaderType||"agent")),this.start()):(0,l.Z)("Failed to initial the agent. Could not determine the runtime environment.")}get config(){return{info:(0,t.C5)(this.agentIdentifier),init:(0,t.P_)(this.agentIdentifier),loader_config:(0,t.DL)(this.agentIdentifier),runtime:(0,t.OP)(this.agentIdentifier)}}start(){const t="features";try{const r=n(this.agentIdentifier),i=[...this.desiredFeatures];i.sort(((t,r)=>e.p[t.featureName]-e.p[r.featureName])),i.forEach((t=>{if(r[t.featureName]||t.featureName===e.D.pageViewEvent){const n=function(t){switch(t){case e.D.ajax:return[e.D.jserrors];case e.D.sessionTrace:return[e.D.ajax,e.D.pageViewEvent];case e.D.sessionReplay:return[e.D.sessionTrace];case e.D.pageViewTiming:return[e.D.pageViewEvent];default:return[]}}(t.featureName);n.every((e=>r[e]))||(0,l.Z)("".concat(t.featureName," is enabled but one or more dependent features has been disabled (").concat((0,D.P)(n),"). This may cause unintended consequences or missing data...")),this.features[t.featureName]=new t(this.agentIdentifier,this.sharedAggregator)}})),(0,T.Qy)(this.agentIdentifier,this.features,t)}catch(e){(0,l.Z)("Failed to initialize all enabled instrument classes (agent aborted) -",e);for(const e in this.features)this.features[e].abortHandler?.();const r=(0,T.fP)();return delete r.initializedAgents[this.agentIdentifier]?.api,delete r.initializedAgents[this.agentIdentifier]?.[t],delete this.sharedAggregator,r.ee?.abort(),delete r.ee?.get(this.agentIdentifier),!1}}}({features:[J,m,S,class extends h{static featureName=oe;constructor(t,r){if(super(t,r,oe,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;const n=this.ee;let i;(0,k.QU)(n),this.eventsEE=(0,k.em)(n),this.eventsEE.on(se,(function(e,t){this.bstStart=(0,p.z)()})),this.eventsEE.on(ae,(function(t,r){(0,s.p)("bst",[t[0],r,this.bstStart,(0,p.z)()],void 0,e.D.sessionTrace,n)})),n.on(ce+ne,(function(e){this.time=(0,p.z)(),this.startPath=location.pathname+location.hash})),n.on(ce+ie,(function(t){(0,s.p)("bstHist",[location.pathname+location.hash,this.startPath,this.time],void 0,e.D.sessionTrace,n)}));try{i=new PerformanceObserver((t=>{const r=t.getEntries();(0,s.p)(te,[r],void 0,e.D.sessionTrace,n)})),i.observe({type:re,buffered:!0})}catch(e){}this.importAggregator({resourceObserver:i})}},C,xe,B,class extends h{static featureName=de;constructor(e,r){if(super(e,r,de,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;if(!(0,t.OP)(e).xhrWrappable)return;try{this.removeOnAbort=new AbortController}catch(e){}let n,i=0;const o=this.ee.get("tracer"),a=(0,k._L)(this.ee),s=(0,k.Lg)(this.ee),u=(0,k.BV)(this.ee),d=(0,k.Kf)(this.ee),f=this.ee.get("events"),l=(0,k.u5)(this.ee),h=(0,k.QU)(this.ee),g=(0,k.Gm)(this.ee);function m(e,t){h.emit("newURL",[""+window.location,t])}function v(){i++,n=window.location.hash,this[ve]=(0,p.z)()}function b(){i--,window.location.hash!==n&&m(0,!0);var e=(0,p.z)();this[pe]=~~this[pe]+e-this[ve],this[ye]=e}function y(e,t){e.on(t,(function(){this[t]=(0,p.z)()}))}this.ee.on(ve,v),s.on(be,v),a.on(be,v),this.ee.on(ye,b),s.on(ge,b),a.on(ge,b),this.ee.buffer([ve,ye,"xhr-resolved"],this.featureName),f.buffer([ve],this.featureName),u.buffer(["setTimeout"+le,"clearTimeout"+fe,ve],this.featureName),d.buffer([ve,"new-xhr","send-xhr"+fe],this.featureName),l.buffer([me+fe,me+"-done",me+he+fe,me+he+le],this.featureName),h.buffer(["newURL"],this.featureName),g.buffer([ve],this.featureName),s.buffer(["propagate",be,ge,"executor-err","resolve"+fe],this.featureName),o.buffer([ve,"no-"+ve],this.featureName),a.buffer(["new-jsonp","cb-start","jsonp-error","jsonp-end"],this.featureName),y(l,me+fe),y(l,me+"-done"),y(a,"new-jsonp"),y(a,"jsonp-end"),y(a,"cb-start"),h.on("pushState-end",m),h.on("replaceState-end",m),window.addEventListener("hashchange",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("load",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("popstate",(function(){m(0,i>1)}),(0,O.m$)(!0,this.removeOnAbort?.signal)),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}}],loaderType:"spa"})})(),window.NRBA=o})(); window.jQuery || document.write(' ') CKEDITOR_BASEPATH='https://f1000research.com/js/vendor/ckeditor/' window.reactTheme = 'research'; window.MathJax = { CommonHTML: { linebreaks: { automatic: true } }, 'HTML-CSS': { linebreaks: { automatic: true } }, SVG: { linebreaks: { automatic: true } }, AuthorInit: function() { MathJax.Hub.Register.MessageHook('End Process', function () { let timeout = false; // holder for timeout id const delay = 250; // delay after event is "complete" to run callback const reflowMath = function() { const dispFormulas = document.querySelectorAll('.disp-formula.panel'); if (!dispFormulas) { return; } for (const dispFormula of dispFormulas) { const child = dispFormula.querySelector('.MathJax_Preview').nextSibling.firstChild; const isMultiline = MathJax.Hub.getAllJax(dispFormula)[0].root.isMultiline; if (dispFormula.offsetWidth < child.offsetWidth || isMultiline) { MathJax.Hub.Queue(['Rerender', MathJax.Hub, dispFormula]); } } }; window.addEventListener('resize', function() { clearTimeout(timeout); // clear the timeout timeout = setTimeout(reflowMath, delay); // start timing for event "completion" }); }); }, }; if (window.location.hash == '#_=_'){ window.location = window.location.href.split('#')[0] } !function(f,b,e,v,n,t,s){if(f.fbq)return;n=f.fbq=function() {n.callMethod? n.callMethod.apply(n,arguments):n.queue.push(arguments)} ;if(!f._fbq)f._fbq=n; n.push=n;n.loaded=!0;n.version='2.0';n.queue=[];t=b.createElement(e);t.async=!0; t.src=v;s=b.getElementsByTagName(e)[0];s.parentNode.insertBefore(t,s)}(window, document,'script','https://connect.facebook.net/en_US/fbevents.js'); fbq('init', '1641728616063202'); fbq('track', "PixelInitialized", {}); (function(h,o,t,j,a,r){ h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)}; h._hjSettings={hjid:2318163,hjsv:6}; a=o.getElementsByTagName('head')[0]; r=o.createElement('script');r.async=1; r.src=t+h._hjSettings.hjid+j+h._hjSettings.hjsv; a.appendChild(r); })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv='); search file_upload Submit your research search menu close search Browse Gateways & Collections How to Publish Submit your Research My Submissions Article Guidelines Article Guidelines (New Versions) Open Data, Software and Code Guidelines Open Data and Accessible Source Materials Guidelines (HSS) Open Data, Software and Code Guidelines (PSE) Prepublication Checks Production Process Posters and Slides Guidelines Document Guidelines Article Processing Charges Peer Review Finding Article Reviewers About How it Works For Reviewers Our Advisors Policies Glossary FAQs For Developers Newsroom Contact My Research Submissions Content and Tracking Alerts My Details Sign In file_upload Submit your research { "@context": "https://schema.org", "@type": "ScholarlyArticle", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://f1000research.com/articles/12-481" }, "headline": "Teaching critical thinking about health information and choices in secondary schools: human-centred design...", "datePublished": "2023-05-11T12:26:57", "dateModified": "2024-09-04T15:58:04", "author": [ { "@type": "Person", "name": "Sarah Rosenbaum" }, { "@type": "Person", "name": "Jenny Moberg" }, { "@type": "Person", "name": "Faith Chesire" }, { "@type": "Person", "name": "Michael Mugisha" }, { "@type": "Person", "name": "Ronald Ssenyonga" }, { "@type": "Person", "name": "Marlyn A Ochieng" }, { "@type": "Person", "name": "Clarisse Marie Claudine Simbi" }, { "@type": "Person", "name": "Esther Nakyejwe" }, { "@type": "Person", "name": "Benson Ngatia" }, { "@type": "Person", "name": "Gabriel Rada" }, { "@type": "Person", "name": "Juan Vásquez-Laval" }, { "@type": "Person", "name": "José Damián Garrido" }, { "@type": "Person", "name": "Grace Baguma" }, { "@type": "Person", "name": "Sam Kuloba" }, { "@type": "Person", "name": "Edward Sebukyu" }, { "@type": "Person", "name": "Richard Kabanda" }, { "@type": "Person", "name": "Irene Mwenyango" }, { "@type": "Person", "name": "Tonny Muzaale" }, { "@type": "Person", "name": "Pamela Nandi" }, { "@type": "Person", "name": "Jane Njue" }, { "@type": "Person", "name": "Cyril Oyuga" }, { "@type": "Person", "name": "Florian Rutiyomba" }, { "@type": "Person", "name": "Felecien Rugengamanzi" }, { "@type": "Person", "name": "Joan Murungi" }, { "@type": "Person", "name": "Allen Nsangi" }, { "@type": "Person", "name": "Daniel Semakula" }, { "@type": "Person", "name": "Margaret Kaseje" }, { "@type": "Person", "name": "Nelson Sewankambo" }, { "@type": "Person", "name": "Laetitia Nyirazinyoye" }, { "@type": "Person", "name": "Simon Lewin" }, { "@type": "Person", "name": "Andrew D Oxman" }, { "@type": "Person", "name": "Matt Oxman" } ], "publisher": { "@type": "Organization", "name": "F1000Research", "logo": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 480, "width": 60 } }, "image": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 1200, "width": 150 }, "description": " Background Learning to thinking critically about health information and choices can protect people from unnecessary suffering, harm, and resource waste. Earlier work revealed that children can learn these skills, but printing costs and curricula compatibility remain important barriers to school implementation. We aimed to develop a set of digital learning resources for students to think critically about health that were suitable for use in Kenyan, Rwandan, and Ugandan secondary schools. Methods We conducted work in two phases collaborating with teachers, students, schools, and national curriculum development offices using a human-centred design approach. First, we conducted context analyses and an overview of teaching strategies, prioritised content and collected examples. Next, we developed lessons and guidance iteratively, informed by data from user-testing, individual and group interviews, and school pilots. Results Final resources include online lesson plans, teachers’ guide, and extra resources, with lesson plans in two modes, for use in a classroom equipped with a blackboard/flip-chart and a projector. The resources are accessible offline for use when electricity or Internet is lacking. Teachers preferred the projector mode, as it provided structure and a focal point for class attention. Feedback was largely positive, with teachers and students appreciating the learning and experiencing it as relevant. Four main challenges included time to teach lessons; incorrect comprehension; identifying suitable examples; and technical, logistical, and behavioural challenges with a student-computer mode that we piloted. We resolved challenges by simplifying and combining lessons; increasing opportunities for review and assessment; developing teacher training materials, creating a searchable set of examples; and deactivating the student-computer mode. Conclusion Using a human-centred design approach, we created digital resources for teaching secondary school students to think critically about health actions and for training teachers. Be smart about your health resources are open access and can be translated or adapted to other settings. " } { "@context": "http://schema.org", "@type": "BreadcrumbList", "itemListElement": [ { "@type": "ListItem", "position": "1", "item": { "@id": "https://f1000research.com/", "name": "Home" } }, { "@type": "ListItem", "position": "2", "item": { "@id": "https://f1000research.com/browse/articles", "name": "Browse" } }, { "@type": "ListItem", "position": "3", "item": { "@id": "https://f1000research.com/articles/12-481/v3", "name": "Teaching critical thinking about health information and choices in..." } } ] } Home Browse Teaching critical thinking about health information and choices in... ALL Metrics - Views Downloads Get PDF Get XML Cite How to cite this article Rosenbaum S, Moberg J, Chesire F et al. Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources [version 3; peer review: 1 approved, 2 approved with reservations] . F1000Research 2024, 12 :481 ( https://doi.org/10.12688/f1000research.132580.3 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. Close Copy Citation Details Export Export Citation Sciwheel EndNote Ref. Manager Bibtex ProCite Sente EXPORT Select a format first Track Share ▬ ✚ Research Article Revised Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources [version 3; peer review: 1 approved, 2 approved with reservations] Sarah Rosenbaum https://orcid.org/0000-0001-8334-7664 1 , Jenny Moberg 1 , Faith Chesire 2,3 , [...] Michael Mugisha 3,4 , Ronald Ssenyonga 3,5 , Marlyn A Ochieng https://orcid.org/0000-0002-0455-8517 2 , Clarisse Marie Claudine Simbi 4 , Esther Nakyejwe 5 , Benson Ngatia https://orcid.org/0009-0009-5837-2310 2 , Gabriel Rada 6 , Juan Vásquez-Laval 6 , José Damián Garrido 6 , Grace Baguma 7 , Sam Kuloba 8 , Edward Sebukyu 8 , Richard Kabanda 9,10 , Irene Mwenyango 10 , Tonny Muzaale 11 , Pamela Nandi 12 , Jane Njue 13 , Cyril Oyuga 13 , Florian Rutiyomba 14 , Felecien Rugengamanzi 14 , Joan Murungi 14 , Allen Nsangi https://orcid.org/0000-0001-8702-9217 5 , Daniel Semakula https://orcid.org/0000-0002-0806-213X 5 , Margaret Kaseje https://orcid.org/0000-0001-9789-1927 2 , Nelson Sewankambo 5 , Laetitia Nyirazinyoye 4 , Simon Lewin 1,15,16 , Andrew D Oxman 1 , Matt Oxman https://orcid.org/0000-0002-6323-9620 1,17 Sarah Rosenbaum https://orcid.org/0000-0001-8334-7664 1 , Jenny Moberg 1 , [...] Faith Chesire 2,3 , Michael Mugisha 3,4 , Ronald Ssenyonga 3,5 , Marlyn A Ochieng https://orcid.org/0000-0002-0455-8517 2 , Clarisse Marie Claudine Simbi 4 , Esther Nakyejwe 5 , Benson Ngatia https://orcid.org/0009-0009-5837-2310 2 , Gabriel Rada 6 , Juan Vásquez-Laval 6 , José Damián Garrido 6 , Grace Baguma 7 , Sam Kuloba 8 , Edward Sebukyu 8 , Richard Kabanda 9,10 , Irene Mwenyango 10 , Tonny Muzaale 11 , Pamela Nandi 12 , Jane Njue 13 , Cyril Oyuga 13 , Florian Rutiyomba 14 , Felecien Rugengamanzi 14 , Joan Murungi 14 , Allen Nsangi https://orcid.org/0000-0001-8702-9217 5 , Daniel Semakula https://orcid.org/0000-0002-0806-213X 5 , Margaret Kaseje https://orcid.org/0000-0001-9789-1927 2 , Nelson Sewankambo 5 , Laetitia Nyirazinyoye 4 , Simon Lewin 1,15,16 , Andrew D Oxman 1 , Matt Oxman https://orcid.org/0000-0002-6323-9620 1,17 PUBLISHED 04 Sep 2024 Author details Author details 1 Centre for Epidemic Interventions Research, Norwegian Institute of Public Health, Oslo, 0213, Norway 2 Tropical Institute of Community Health and Development in Africa, Kisumu, Kenya 3 Institute of Health and Society, Faculty of Medicine, Universitetet i Oslo, Oslo, Oslo, Norway 4 School of Public Health, College of Medicine and Health Sciences, University of Rwanda, Butare, Southern Province, Rwanda 5 School of Medicine, College of Health Sciences, Makerere University, Kampala, Central Region, Uganda 6 Epistemonikos Foundation, Santiago, Santiago Metropolitan Region, Chile 7 National Curriculum Development Centre, Kampala, Uganda 8 Ministry of Education and Sports, Kampala, Uganda 9 Faculty of Health Sciences, Uganda Martyrs University, Kampala, Central Region, Uganda 10 Ministry of Health, Kampala, Uganda 11 Uganda National Examinations Board, Kampala, Uganda 12 Huma Girls Secondary School, Kisumu, Kenya 13 Kenya Institute of Curriculum Development, Nairobi, Kenya 14 Rwanda Basic Education Board, Kigali, Rwanda 15 Health Systems Research Unit, South African Medical Research Council, Cape Town, South Africa 16 Department of Health Sciences, Norwegian University of Science and Technology (NTNU), Ålesund, Norway 17 Faculty of Health Sciences, Oslo Metropolitan University, Oslo, Norway Sarah Rosenbaum Roles: Conceptualization, Data Curation, Formal Analysis, Funding Acquisition, Investigation, Methodology, Supervision, Visualization, Writing – Original Draft Preparation, Writing – Review & Editing Jenny Moberg Roles: Data Curation, Formal Analysis, Investigation, Writing – Original Draft Preparation, Writing – Review & Editing Faith Chesire Roles: Data Curation, Formal Analysis, Investigation, Project Administration, Writing – Review & Editing Michael Mugisha Roles: Data Curation, Formal Analysis, Investigation, Project Administration, Writing – Review & Editing Ronald Ssenyonga Roles: Data Curation, Formal Analysis, Investigation, Project Administration, Writing – Review & Editing Marlyn A Ochieng Roles: Data Curation, Formal Analysis, Investigation, Writing – Review & Editing Clarisse Marie Claudine Simbi Roles: Data Curation, Formal Analysis, Investigation, Writing – Review & Editing Esther Nakyejwe Roles: Data Curation, Investigation, Writing – Review & Editing Benson Ngatia Roles: Data Curation, Investigation, Writing – Review & Editing Gabriel Rada Roles: Software, Supervision, Writing – Review & Editing Juan Vásquez-Laval Roles: Project Administration, Software, Writing – Review & Editing José Damián Garrido Roles: Software, Writing – Review & Editing Grace Baguma Roles: Investigation, Writing – Review & Editing Sam Kuloba Roles: Investigation, Writing – Review & Editing Edward Sebukyu Roles: Investigation, Writing – Review & Editing Richard Kabanda Roles: Investigation, Writing – Review & Editing Irene Mwenyango Roles: Investigation, Writing – Review & Editing Tonny Muzaale Roles: Investigation, Writing – Review & Editing Pamela Nandi Roles: Investigation, Writing – Review & Editing Jane Njue Roles: Investigation, Writing – Review & Editing Cyril Oyuga Roles: Investigation, Writing – Review & Editing Florian Rutiyomba Roles: Investigation, Writing – Review & Editing Felecien Rugengamanzi Roles: Investigation, Writing – Review & Editing Joan Murungi Roles: Investigation, Writing – Review & Editing Allen Nsangi Roles: Conceptualization, Project Administration, Supervision, Writing – Review & Editing Daniel Semakula Roles: Conceptualization, Project Administration, Supervision, Writing – Review & Editing Margaret Kaseje Roles: Conceptualization, Funding Acquisition, Project Administration, Supervision, Writing – Review & Editing Nelson Sewankambo Roles: Conceptualization, Funding Acquisition, Project Administration, Supervision, Writing – Review & Editing Laetitia Nyirazinyoye Roles: Conceptualization, Funding Acquisition, Project Administration, Supervision, Writing – Review & Editing Simon Lewin Roles: Supervision, Writing – Review & Editing Andrew D Oxman Roles: Conceptualization, Formal Analysis, Funding Acquisition, Project Administration, Supervision, Writing – Review & Editing Matt Oxman Roles: Conceptualization, Data Curation, Formal Analysis, Writing – Original Draft Preparation, Writing – Review & Editing OPEN PEER REVIEW DETAILS REVIEWER STATUS Abstract Background Learning to thinking critically about health information and choices can protect people from unnecessary suffering, harm, and resource waste. Earlier work revealed that children can learn these skills, but printing costs and curricula compatibility remain important barriers to school implementation. We aimed to develop a set of digital learning resources for students to think critically about health that were suitable for use in Kenyan, Rwandan, and Ugandan secondary schools. Methods We conducted work in two phases collaborating with teachers, students, schools, and national curriculum development offices using a human-centred design approach. First, we conducted context analyses and an overview of teaching strategies, prioritised content and collected examples. Next, we developed lessons and guidance iteratively, informed by data from user-testing, individual and group interviews, and school pilots. Results Final resources include online lesson plans, teachers’ guide, and extra resources, with lesson plans in two modes, for use in a classroom equipped with a blackboard/flip-chart and a projector. The resources are accessible offline for use when electricity or Internet is lacking. Teachers preferred the projector mode, as it provided structure and a focal point for class attention. Feedback was largely positive, with teachers and students appreciating the learning and experiencing it as relevant. Four main challenges included time to teach lessons; incorrect comprehension; identifying suitable examples; and technical, logistical, and behavioural challenges with a student-computer mode that we piloted. We resolved challenges by simplifying and combining lessons; increasing opportunities for review and assessment; developing teacher training materials, creating a searchable set of examples; and deactivating the student-computer mode. Conclusion Using a human-centred design approach, we created digital resources for teaching secondary school students to think critically about health actions and for training teachers. Be smart about your health resources are open access and can be translated or adapted to other settings. READ ALL READ LESS Keywords critical thinking, critical health literacy, informed decision making, infodemic, health education, secondary school, educational design, human centred design Corresponding Author(s) Sarah Rosenbaum ( [email protected] ) Close Corresponding author: Sarah Rosenbaum Competing interests: No competing interests were disclosed. Grant information: Grant information: This work was supported by the Research Council of Norway Project number 284683, grant no: 69006, awarded to ADO. The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript. Copyright: © 2024 Rosenbaum S et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. How to cite: Rosenbaum S, Moberg J, Chesire F et al. Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources [version 3; peer review: 1 approved, 2 approved with reservations] . F1000Research 2024, 12 :481 ( https://doi.org/10.12688/f1000research.132580.3 ) First published: 11 May 2023, 12 :481 ( https://doi.org/10.12688/f1000research.132580.1 ) Latest published: 04 Sep 2024, 12 :481 ( https://doi.org/10.12688/f1000research.132580.3 ) Revised Amendments from Version 2 In response to peer review, we have made the three types of findings more explicit and easier to distinguish from each other: the final version of the designed output, findings that emerged from data analyses, and our corresponding design decisions. In the discussion section, we also added a description of why we used a human-centred design approach, its' strengths and limitations, and reflections on learning theory as related to this work. In response to peer review, we have made the three types of findings more explicit and easier to distinguish from each other: the final version of the designed output, findings that emerged from data analyses, and our corresponding design decisions. In the discussion section, we also added a description of why we used a human-centred design approach, its' strengths and limitations, and reflections on learning theory as related to this work. See the authors' detailed response to the review by Cynthia Baur See the authors' detailed response to the review by Anke Steckelberg READ REVIEWER RESPONSES Introduction Claims about how to care for our health are everywhere, spread by friends, family, news media, industry, healthcare professionals, policymakers, researchers, and others. Many of these claims unreliable, 1 but people often lack the skills needed to assess them. 2 , 3 When we believe unreliable claims, we might take ineffective or harmful actions, or fail to take helpful actions. The Covid-19 pandemic showed how easily unreliable claims 4 – 6 and research 7 – 9 spread, impacting public trust and protective behaviours. 10 , 11 Publicly debunking untrustworthy information has value, but the effectiveness of this retroactive strategy might be limited. Misinformation, once spread, can be resistant to correction; an alternative narrative may not exist (for instance, there may not be a safe or proven alternative treatment); and individuals may reject the scientific process or the source altogether rather than change their established views about some topics. 12 Pre-emptively ‘inoculating’ people against misinformation 13 , 14 by teaching them to think critically about claims of what works, and how to make informed decisions, has the potential to provide broader, more long-lasting protection. In earlier work developing and evaluating Informed Health Choices (IHC) resources for primary school students, we have shown that it is feasible to teach critical thinking skills about health actions to children as young as 10-years old. 15 – 20 In that project, we created a framework of principles that are important for people to understand when assessing the reliability of healthcare claims and making informed choices: the IHC Key Concepts. 21 , 22 This framework is a starting point for designing curricula, learning resources, and evaluation tools. Focusing on a selection of the IHC Key Concepts, we then developed printed learning resources for primary school children (age 10-12 years) and their teachers 15 and a podcast for parents. 16 To evaluate the effect of these resources, we conducted two randomised trials in Uganda, one including 120 schools and more than 10,000 children, and another that included 675 parents and guardians of primary school children. 23 The primary outcome measure was responses to multiple-choice questions that measure respondents’ ability to apply IHC Key Concepts: the Claim Evaluation Tools. 17 , 18 The primary school trial demonstrated that use of the resources led to a large improvement in the ability of children and teachers to apply IHC Key Concepts to hypothetical scenarios. 17 The podcast trial showed a smaller, similar effect among parents. 18 Follow-up data showed that children retained their learning for at least one year, 19 while the performance of the parents declined. 20 The primary school resources have been translated into 12 languages and adapted for use in other countries. 24 Alongside the trials, we undertook process evaluations to explore barriers and facilitators for scaling up use of the learning resources, potential adverse effects, and potential additional benefits. 25 , 26 We found that children, teachers, parents, and district education officers valued the IHC primary school resources. The human-centred design approach we employed - iteratively addressing user and stakeholder concerns prior to and during development - was likely an important contributing factor for this positive reception. However, the study pointed to two important implementation barriers in Uganda: printing costs and lack of time in the school schedule. Informed by these findings, we began the current five-year project in 2019 to develop and evaluate resources for lower secondary schools (age 13-16) in Kenya, Rwanda, and Uganda. Since drafting this article, we have evaluated the resources in three parallel randomised trials, 27 – 29 are carrying out process evaluations, 30 – 32 and will conduct one-year follow-up trials in each of the three countries. This article describes the development of these resources that took place prior to evaluation during the first 2.5 years of the project in two phases. We began development with this set of objectives (see protocol 33 ): To explore how we might develop resources that are • Digital (avoiding printing costs) • Suitable for use with schools’ available digital technology and infrastructure • Compatible with national curricula • Based on evidence about effective strategies for teaching critical thinking • Experienced as accessible, useful, usable, understandable, credible, and desirable by students and teachers, and well-suited to use in their schools • Easily translatable and adaptable to other contexts • Sustainable (i.e., not dependent on our team for rolling out at scale) We organised the work in two phases: 1) a set of preliminary studies (involving data documented and discussed elsewhere) and 2) an iterative development phase with data collection and analysis reported in this article. Although the preliminary studies have already been published elsewhere, findings from those studies resulted in reframing of some of our objectives and provided the basis for content and design decisions made in the second phase. Therefore, we describe methods and results from both phases (with references for more detail about phase one). In the discussion, we describe how this study may inform development of similar educational resources for teaching critical thinking. Methods We employed perspectives and methods from human-centred design. This is an iterative approach to creating products, systems and services that places users’ and other stakeholders’ needs and experiences at the centre of the design process. 34 , 35 Early and continued engagement with stakeholder groups and multidisciplinary collaboration are central components of a human-centred design approach. 36 , 37 We employed qualitative data collection and analysis methods to explore the user experiences of multiple types of stakeholders for the purpose of informing the development process. We established a core team with backgrounds in health systems and public health research, design, journalism, education, social science, statistics, and information and communication technology (ICT). Research leadership was shared among senior team members in East Africa and Norway. Three PhD fellows (one female and two male) based in Kisumu (Kenya), Kigali (Rwanda) and Kampala (Uganda), engaged with stakeholders and collected and analysed data, supported by their local teams including researchers with experience developing and evaluating the IHC primary school resources. All teams contributed to design and content development, led by the team based in Norway who also analysed data. The ICT team, based in Chile, developed the technical solutions. We conducted this work in two phases: 1) Groundwork and 2) Development cycles ( Figure 1 ). The first phase began in August 2019, the second phase began January 2020 and ended in April 2022. Figure 1. Phases of the work. Identifying and recruiting participating stakeholders We defined stakeholders as people or organisations that have a vested interest in the process or results of the study. Intended users – secondary school teachers and students – were the key stakeholders. Additional stakeholders included curriculum developers and other education policymakers, school administrators, parents, educational researchers, researchers in public or clinical health, and health professionals. For the purpose of providing continuous input, we established groups of stakeholders in each country early in the project. We formed teacher networks, recruiting via formal invitation letters to teachers who worked in a mix of government funded and private schools, with varied ICT resources (Uganda), and in rural, semi-rural, and urban environments (Kenya and Rwanda). We formed student networks by asking members of the teacher networks to suggest lower secondary school students who were interested, likely to contribute, and able to participate without adversely affecting their schoolwork, sending them formal invitations and assent forms, and consent forms to parents. Due to Covid-related school closures in parts of Development Cycle 1, we also established “home networks” of students that we had access to when schools were closed. Home networks included both students that were already recruited to the student network and additional students whom team members could easily reach from nearby communities. We employed convenience sampling and used the same assent forms and consent forms to parents as for the student networks. We engaged with the national curriculum development offices to establish channels of contact and collaboration. We formed national advisory panels of policy makers at the ministerial and regional and district levels, school directors, head teachers, leaders of teacher unions, and representatives of parents’ groups and civil society. We created an international advisory panel of people from 18 countries who had expertise in education, education policy, and relevant areas of research, such as health literacy, evidence-informed decision making, science communication and ICT. Additionally, during the final cycle of development, we recruited seven schools across the three countries to pilot use of the resources in one or more classes over a school semester. Apart from people in the international advisory panel, many of whom were a part of our existing professional networks, and some of the students in home networks, we did not have established relationships with the stakeholders prior to the onset of this project. Details of how we selected and recruited each stakeholder group and characteristics of participants and schools are described in Tables 1 - 3 . More information about the methods, degrees, and nature of engagement with different stakeholders, can be found in a protocol for evaluating stakeholder engagement. 38 Table 1. Participant selection and recruitment methods. Stakeholder group Country What/who we sought Sampling/selection Method of approach Teacher networks Kenya, Rwanda, Uganda Schools: mix of government-funded/private and rural/semi-urban/urban Teachers: diverse (gender, level of education, years of experience, subject areas) Stratified sampling used to select schools, based on school ownership and ICT resources. We contacted head teachers and asked them to suggest one or more teachers to be a part of the network We sent formal invitation letters to teachers Student networks (including home networks) Kenya, Rwanda, Uganda Lower secondary school students who were: ‐ interested ‐ likely to contribute ‐ able to participate without adversely affecting their schoolwork We asked each member of the teacher networks to suggest one or more students to join student networks in each country, and aimed for a mix of top, average and low academic performers, as well as a mix of gender and age We sent formal invitations to the students and their parents, with consent forms for the parents and assent forms for the students to sign. In some cases, we visited parents at the homes to deliver the letters and explain the study National curriculum developers Members of the national curriculum committees We invited them via email, telephone, or face-to-face visits National advisory panels Kenya, Rwanda, Uganda Policymakers at the ministerial and regional and district levels including curriculum developers, school directors, head teachers, leaders of teacher unions, and representatives of parents’ groups and civil society We identified potential members by consulting informal contacts, the teacher networks, and initial invitees (snowballing) We invited them via email, telephone, or face-to-face visits International advisory panel Australia, Brazil, Canada, China, Croatia, Iran, Ireland, Italy, Kenya, Mexico, Norway, Palestine, Rwanda, South Africa, Spain, Uganda, UK, USA, 3ie (international) People from low-, middle-, and high-income countries with expertise in education and education policy, relevant areas of research (e.g., education, health literacy, science communication, evidence-based practice, critical thinking), design, ICT, and learning games We identified these through existing contacts, publications, and snowballing We sent invitations via email Pilot schools Kenya Mix of schools by ownership and geographical: two semi urban public schools one rural public school Stratified sampling to select schools by ownership and location. We contacted head teachers and asked them to suggest one or more teachers to be a part of the study We formally introduced the study objectives to ministry of education officials at national, regional and county in Kenya. Together with the officials selected based on selected by ownership and geographical location and contacted the head teachers via formal invitation letters asking their schools to participate in the study Rwanda Two Schools varied from location (urban/rural), had ICT facilities and were able to dedicate their time in the pilot of learning resources Purposively selected them with recommendation from Rwanda Education Board (REB) We sent a request to REB and REB requested two schools in a letter addressed to school directors Uganda Two privately owned schools (rural/urban) Selected schools in proximity to the research teams. We contacted head teachers and asked them to suggest one or more teachers to participate in the pilot. Then we enrolled the teacher after consenting them We sent formal invitation letters to schools’ administration (head teachers) Table 2. Participant characteristics. Stakeholder group Country # people More detail Teacher networks Kenya 22 teachers Schools: 4 urban, 6 semi-urban, 12 rural Subjects taught: Business studies, Chemistry, History, Biology, Kiswahili, French, Mathematics, Home-science, CRE, English Gender: 12 female, 11 male Rwanda 19 teachers Schools: 10 schools (1 private, 5 public and 4 government aided schools). Teaching experience: 4 to 10 years. Subjects taught: 13 Teachers teach biology. Others teach Chemistry, Physics, Math, French and English. Gender: 7 female, 12 male Uganda 22 teachers Schools: 5 public schools with at least a projector, 5 private schools with at least a projector, 6 public schools with no projector and 6 private schools with no projector. Gender: 7 female, 15 male Teaching experience: range was 2-34 years, average of 12 years Subjects taught : 7 biology & Chemistry, 7 Maths & Physics, 3 geography & History, 4 English & ICT, 1 Home science Student networks Kenya 24 students Schools: 1 public urban, 1 public rural and 1 private urban. Students: 15–18-year-olds, (18 female, 6 male) Rwanda 19 students Schools: 5 public schools, 4 private schools. Students: 10–16-year-olds Gender: 10 female, 9 male Uganda 21 students Schools: 1 public with at least a projector, 1 private with at least a projector and 1 private with no projector. Ages: 14–19-year-olds Gender: 13 female, 8 male National curriculum developer office Kenya 1 institute Kenya Institute of Curriculum Development (KICD) Rwanda 1 institute Rwanda Basic Education Board (REB) Uganda 1 institute National Curriculum Development Centre (NCDC) National advisory groups Kenya 12 Researchers and practitioners from various fields (education, health, design/IT), curriculum specialists and teachers) Rwanda 11 1 curriculum developer, 1 ICT specialist, 3 from health sector (adolescent, community health and health services), 1 educational researcher and 5 educational development partners working in Rwanda Uganda 14 Curriculum developers, Policy makers (commissioners, district education officers) from government, school head teachers, teachers, education researchers International advisory group 18 countries: Australia, Brazil, Canada, China, Croatia, Iran, Ireland, Italy, Kenya, Mexico, Norway, Palestine, Rwanda, South Africa, Spain, Uganda, UK, USA 56 people Researchers and practitioners from various fields: education, public and clinical health, health literacy, design/IT/games, methodologists, and funders/international organisations Table 3. Characteristics of schools participating in pilot of ten lessons. Country Number and type of schools # of classes/groups Class level Total # of students Kenya 2 public semi urban and 1 public- rural 5 classes Form one (Students that are equivalent in age to Rwanda and Uganda “senior two in ordinary level”, approx. 13-15 years old) Approximately 200 students Rwanda 1 public/rural/and day school and 1 government aided/urban/and day school 2 classes Senior two in ordinary level 84 students Uganda 2 private schools (one with a projector and the other without) 2 groups (conducted on school premises during a period of school closure due to Covid) Senior two in ordinary level 23 students Phase 1: Groundwork The groundwork phase consisted of a series of preliminary studies. More detail about the methods and results can be found in separate publications (see overview in Table 4 ). Table 4. Overview of preliminary studies. Study aim Methods/Study design Data sources More detail Analysing contexts in Kenya, Rwanda, and Uganda Individual and group interviews, document analysis, non-participatory observation Interviews: Teachers, students, curriculum developers, ICT support staff Document analysis: learning materials for health and science subjects, curriculum documents, ICT policy documents Observation : School classes 39 – 41 Prioritising IHC Key Concepts (learning goals) Iterative, structured consensus process Curriculum specialists, teachers, and researchers 55 Identifying teaching strategies for critical thinking Overview of systematic reviews Systematic reviews of the effects of teaching strategies for critical thinking 44 Collecting examples Individual and group interviews Secondary school students in Kenya, Rwanda, and Uganda 45 Analysing contexts To inform the development of the resources, as well as their potential implementation in Kenya, Rwanda and Uganda lower secondary schools, we explored if there was demand for learning resources to teach critical thinking; if such resources were in use; how the content maps onto existing curricula; what administrative approval was necessary; what ICT infrastructure is available in schools and how it is used in schools. We analysed the three country contexts. Detailed methods are reported in a separate publication. 39 – 41 To synthesize findings from these studies in a way that was useful for developing educational resources, we drew on the behaviour change wheel. 42 This is a framework for characterising and designing interventions for behaviour change, built around three essential conditions for change: Capability, Opportunity, and Motivation (the COM-B system). We organised findings from the three context analyses according to these three themes. The synthesized findings from these three context analyses supplemented our original set of development aims and provided a much more detailed understanding of contextual challenges before we started the development phase. Prioritising IHC Key Concepts Due to the scope of our research funding, as well as the pandemic and school closures, we could not pilot and evaluate resources that took longer to use than one school semester. To prioritise which of the 49 IHC Key Concepts to include in the learning goals for the resources, we organised a structured, iterative consensus process with curriculum specialists, teachers, and researchers. Detailed methods are reported in a separate publication. 43 Identifying teaching strategies To identify effective, relevant teaching strategies to inform the resources, we conducted an overview of systematic reviews of strategies for teaching critical thinking. Detailed methods are reported in a separate protocol. 44 Collecting examples To identify relevant and engaging examples for the resources, we interviewed Kenyan, Rwandan, and Ugandan secondary school students (aged 14 to 16) from the student networks, via WhatsApp. We collected examples of health actions, their claimed and actual effects, and information sources. The students rated their interest in a broad range of health actions and conditions, and we used those with higher scores as examples in the resources. Detailed methods are reported in an online report. 45 Phase 2: Development cycles Starting with what we learned and developed in the preliminary studies, we developed resources in three iterative cycles of content creation and feedback. Each development cycle ( Figure 2 ) included: - Generating ideas - Prototyping - Pilot testing prototypes and collecting data - Analysing problems - Checking our analysis of the most important problems and solution ideas with key stakeholders, drawing on the concept of “member checking” 46 Figure 2. Single development cycle. We also sought ad hoc feedback from the student and teacher networks as needed. Generating ideas We generated content and design ideas in two ways: brainstorming 33 internally, within the team; and identifying ideas suggested in external feedback, from participating stakeholders. 47 We selected ideas based on considerations such as consistency with what teachers and students said they valued, and ease of implementation. Prototyping A prototype is a sketch or model. We developed increasingly refined prototypes in each cycle. We used the platforms Google Drive (including Google Docs and Google Slides ) and Adobe XD for drafting, editing, and sketching. We drafted and edited text using Google Docs, exported early prototypes to PDF format, and used Google Slides and Adobe XD later to create interactive sketches. Final versions were published as web pages , using Google Drive as a content editing platform. Testing prototypes and collecting data The three PhD fellows conducted interviews in their respective settings, supported by their research assistants. They had university degrees (Bachelor of Statistics, Master of Epidemiology & Biostatistics, Master of Public Health, and Master of Community Health and Development) and have attended PhD-level courses in qualitative study methods. Individual interviews employing user testing methods Using semi-structured guides 48 that were pilot tested, the three PhD fellows conducted user test with individual teachers and students, to explore how they experienced prototypes and how we might make improvements. Data were collected face to face or online when covid measures restricted face to face meetings. When possible, user testing immediately followed a pilot lesson, so the participant had their experiences from the lesson fresh in mind. Interview duration was approximately one hour. Occasionally, teams printed the digital prototypes on paper to collect feedback where there was no access to a device or internet connection. Each user testing session had three main parts. First, interviewers introduced themselves and the reasons for the interview. Next, they employed a think-aloud approach: a method of qualitative data collection that encourages the participant to articulate their thoughts while performing a task, 49 – 51 in this case reviewing prototypes of the resources. The last part was conducted as a semi-structured interview. We captured data as observers’ notes and audio recordings. We designed the interview guides to explore different facets of the user experience, including usefulness, understandability, usability, credibility, desirability, and identification, based on a revised version of Morville’s framework of user experience ( Table 5 ). 34 Table 5. Revised version of Morville’s framework of user experience. Facet Description Usefulness Does this product have practical value for the user? Usability How easy and satisfying is this product to use? Understandability Does the user recognize what the product is, and do they understand the content? (Subjective experience of understanding) Credibility Is the content trustworthy? Desirability Is this product something the user wants – has a positive emotional response to? Identification Does the user feel the product is for “someone like me” or is it alienating/foreign-feeling? (e.g., age, gender, culture–appropriate) Group interviews The PhD fellows conducted semi-structured interviews with groups of students or teachers to explore their experiences of the prototypes, using pilot-tested interview guides. 48 When possible, interviews were scheduled immediately after a pilot lesson when the participants had their experiences fresh in mind. Sessions lasted approximately one hour. We chose group interviews, rather than user testing or individual interviews, when we anticipated that the former would improve the quality of the data, for instance by increasing students’ confidence in speaking with researchers. Group interviews took place at schools or other locations determined to be practical for the participants. After introducing themselves and explaining reasons for the interview, one researcher moderated and one or more researchers observed and took notes. Sometimes, teachers were present during student interviews. With written informed consent (from parents and teachers) and assent (from students), we audio-recorded sessions. Piloting and observation We facilitated pilots of the prototypes to explore how teachers and students used and experienced them in as natural settings as possible. Early pilots involved single lessons, with one of the PhD fellows sometimes assuming the role of the teacher. These took place both at schools and in other settings, such as students’ neighbourhoods, when schools were closed due to the pandemic. In the final pilot, teachers taught all the lessons over a school term, with access to a complete prototype of the resources including the teacher’s guide. (In Uganda, although schools were still closed, we got permission for students from the student network and teachers to meet at the schools for the purpose of conducting pilot lessons.) PhD fellows or research assistants observed and took notes using a structured guide, 48 without intervening. We followed up pilots with either individual or group interviews. “Critical Thinking about Health” Test After piloting a full set of lessons, students took the “Critical Thinking about Health” Test. 52 Administering the test had two purposes: validating the items included in the test and giving us a sense of whether the prototypes had the intended effects. We report the validation in detail in a separate article. 52 Consulting the advisory groups Twice annually, we emailed the international advisory group, to update them on the project, and ask for feedback on specific parts or prototypes. We entered their feedback in a spreadsheet, 47 familiarised ourselves with those data, and discussed and agreed on how to deal with any specific suggestions. We held face-to-face meetings with the national advisory groups to keep them updated and seek feedback related to ensuring the sustainability and future scaling up of resources, if shown to be effective. Data analysis In Uganda and Kenya, all interviews were carried out in English and transcribed. In Rwanda, interviews were carried out in Kinyarwanda and transcribed in English. There, the PhD fellow and research assistant relied on notes they made during interview sessions, and selected some recordings to review when further clarification of their notes was needed. The three PhD fellows and their three research assistants reviewed transcriptions, their own interview and observation notes, and recordings, and extracted data about negative and positive experiences. They entered the data into Excel spreadsheets, 47 supplemented with quotations where relevant, and tagged the data using pre-determined codes related to the nature of an experience (e.g., “negative”); the “location” of the experience, i.e., the relevant part of the resources (e.g., “illustration/graphics”), and suggested implication (e.g., “consider changes”). We did not anticipate differences in results between male and female students, so coding did not include gender identification. The prototyping team reviewed the coded data, flagged data entries that they did not understand, suggested changes to codes, and discussed the suggestions with the PhD fellows and their research assistants. After agreeing with the original coders on the final codes, the three-person prototyping team sorted data entries according to the nature, implication and location, and re-reviewed them, focusing on data tagged with the implication “showstopper” or “consider changes”. New topic codes emerged during this process and were added as thematic labels of issues that needed addressing ( e.g., “Time”, “Conceptual misunderstanding”, “student-computer mode”). This coding scheme evolved throughout. For an example from the dataset, 47 see the “Topics” column in the Development Cycle 2 – codes and response options . When data analysis raised important unanswered questions or resulted disagreement about interpretation, the team either contacted teacher or student networks for additional input, or adjusted interview guides to include these issues in the next development cycle. The prototyping team drafted a description and assessment of the most important problems with the prototypes, with an initial set of ideas for solutions. The whole project team discussed these drafts, reached a consensus about which were the most important problems and brainstormed additional ideas for solutions. Based on this input, the prototyping team made the final decision about which solutions to implement. Checking with stakeholders Drawing on the method “member checking”, 46 we carried out “stakeholder checking”, checking our analysis and ideas with the teacher networks and curriculum developers. At the end of each development cycle, we presented descriptions of what we considered the most important problems and our proposed solutions. We asked about the accuracy of the analysis, their reactions to the proposed solutions, and any suggestions of their own. We modified solutions based on this input. Ad hoc feedback At different points within each development cycle, we also contacted people in the teacher and student networks for quick feedback on specific issues, for instance what to call the resources. Figure 3 and Table 6 provide an overview of what we did in the three development cycles. Figure 3. Overview of the three development cycles. Table 6. Scope of teacher and student data collection (interviews, lesson pilots, tests). Method Development cycle 1 Development cycle 2 Student interviews 18 interviews (11 individual user tests and 7 group interviews) 17 interviews (2 individual user tests and 15 group interviews) Teacher interviews 23 interviews (19 individual user tests of which 8 took place after teachers taught a lesson, and 4 group interviews of which 4 took place after teachers taught a lesson) 35 interviews (33 individual user test interviews of which 9 took place after teachers taught a lesson, and 2 group interviews of which 2 took place after teachers taught a lesson) Piloting lessons and test Our team observed 4 pilot lessons 9 classes piloted the full set of 10 lessons Our team observed 37 pilot lessons 259 students took “Critical Thinking about Health” test Ethics approval We obtained ethics approval for the entire project from local institutional review boards and national research governing councils in Kenya, Rwanda, and Uganda: the Rwanda National Ethics Committee (approval number 916/RNEC/2019); Masinde Muliro University of Science and Technology Institutional Ethics Review Committee and the Kenyan National Commission for Science, Technology and Innovation (approval number NACOSTI/P119/1986); and Makerere University School of Medicine Research Ethics Committee (REC REF 2020-139) and Uganda National Council of Science and Technology (HS916ES). Results Drawing on a guideline for reporting design in the context of research, 53 we present three different types of results that emerged from this work: designed output (the final version of the resources), descriptions of the main findings from data analyses, and the design decisions we made corresponding to those findings. We report the designed output first, to give the reader an overview of the resource features and components we refer to in the rest of the findings. Designed output: the final version of “Be smart about your health” resources The final set of resources, Be smart about your health , is an open access website for teachers comprised of ten lesson plans, a teachers’ guide, and extra resources including materials for teacher training. 54 Lesson plans are designed in two modes: for use in classrooms with a blackboard/flipchart (optimized for smartphone) and for classrooms with a projector with lessons formatted as downloadable Google Slides presentations. See Table 7 and Figures 4 – 9 . Table 7. Final list of ten 40-minute lesson plans. Lesson title Description Lesson 1: Health actions Health actions, and why it is important to think carefully about them Lesson 2: Health claims Recognising claims about the effects of health actions Lesson 3: Unreliable claims Identifying some common types of unreliable claims Lesson 4: Reliable claims Identifying more reliable claims by considering whether the claim is based on a comparison of health actions Lesson 5: Using what we learned A review of the first four lessons, applying what has been learned to daily life, and recognising its limits Lesson 6: Randomly-created groups Assessing the reliability of comparisons by considering one important feature of reliable comparisons: randomly-created groups Lesson 7: Large-enough groups Continuing to assess the reliability of comparisons by considering whether the groups are large enough Lesson 8: Personal choices Thinking critically to make smart personal choices about health actions Lesson 9: Community choices Thinking critically to make smart community health choices Lesson 10: Using what we learned A review of all nine lessons, applying what has been learned to daily life, and recognising its limits Figure 4. Final version of the “Be smart about your health” resources. Contents include Teachers’ guide , Blackboard lesson plans, Projector lesson plans , and Extra resources. Figure 5. Lessons designed in two modes: Blackboard mode for use in classrooms equipped with a blackboard or a flipboard, and Projector mode for use in classrooms with a projector. Figure 6. Lesson plan content. Each lesson plan includes an Overview , Lesson , and Background section for teachers. Each Lesson has three parts: Introduction, Activity, and Wrap-up. Figure 7. Teachers’ guide includes: Introduction, Overview of the lessons, Using the resources, Background for teachers, Development and evaluation, and Other relevant resources. Figure 8. Teacher training materials. Figure 9. The examples collection provides alternative health actions or health problems/goals for each lesson or Key Concept in the resources. Lessons plans: - Content: Set of ten lesson plans, two of which are for review and applying what students learned to their daily lives ( Table 7 ) - Structure: Each Lesson Plan has an Overview, Lesson, and Background section. Each Lesson is designed to be taught in 40 minutes, and has three parts: Introduction, Activity, and Wrap-up ( Figure 5 ) - Format: To accommodate for varied ICT infrastructure in schools, we created two modes of resources for teachers to deliver lessons: Blackboard lesson plans and Projector lesson plans. Blackboard lesson plans are optimised for teachers on mobile devices and work also offline. These can also serve as a back-up in the event of electricity outages. Projector mode is for use in classrooms that have access to a projector. We also created sets of optional, downloadable printouts for teachers to use as a paper back-up ( Figure 6 ) Teachers’ guide : This includes an introduction to why the learning goals are important, a description of the content and how it ties to the curriculum in Kenya, Rwanda and Uganda, how to navigate the resources, how they were developed, and where to find other relevant resources ( Figure 7 ). Extra resources : - Teacher training materials, designed to be taught in workshops by teachers who participated in pilots ( Figure 8 ) - Glossary - Examples collection (alternative examples of health actions for each lesson) ( Figure 9 ) - Teaching strategies for teaching critical thinking - Underlying principles Findings from Phase 1: Groundwork Below we describe output from the preliminary studies in Phase 1. As explained earlier, data from these studies have already been reported and discussed in other publications. However, for the reader to understand the full development process, and how these results influenced our objectives, content, and design, they are summarised below. Context analysis findings and corresponding design decisions We organised key findings from the context analyses in four themes: motivation to teach or learn the content; capability to teach the content; capability to use digital resources; and opportunity to teach the content within existing curricula using digital resources. 39 – 42 Based on these findings, we identified opportunities and challenges, and made decisions about how to resolve those. Stakeholders considered it to be important for students to learn to think critically about health, and the curricula included related learning goals. But teachers lacked capabilities and opportunities to teach both critical thinking in general, and about health choices more specifically. Additionally, schools’ digital infrastructures ranged from fully equipped with student computer labs to schools with few devices and no internet. Some of the strategies we decided to use included creating a teacher’s guide with in-depth background content and providing extra information about teaching strategies for critical thinking. We created lessons in different modes, with offline functionality to accommodate for variation in access to computers or digital infrastructure. See Table 8 for more detail about these findings and our solutions. Table 8. Synthesis from context analyses and solutions. Theme Opportunities and challenges Design decisions Motivation to teach or learn the content Curricula in Kenya, Rwanda and Uganda included learning goals related to ‘critical thinking’ as a generic competence and to ‘health’, but none related specifically to ‘critical thinking about health’. Curriculum developers, teachers, and students said it was important for students to learn to think critically about health information and choices. However, teaching was largely exam-focused, so teachers and students were unlikely to prioritise content not included in exams. In the teachers’ guide, we included descriptions of how the content mapped onto each country’s national curriculum. We communicated regularly with curriculum development offices to facilitate alignment with curricula, ownership of the resources, and future uptake of them. Capability to teach the content Teachers lacked prior knowledge of the IHC Key Concepts. We created a teachers’ guide with an introduction and more detail about the IHC Key Concepts. In each lesson plan, we created a detailed background section describing the respective IHC Key Concepts for that lesson. Teacher said they lacked experience teaching and evaluating critical thinking in general. We did not identify any existing resources in use for learning or teaching critical thinking. We created lessons drawing on teaching strategies for critical thinking that we identified in our overview of systematic reviews and included descriptions of these in lesson plans and Extra resources. Capability to use digital resources Teachers had varied experience using ICT for teaching. Many lacked ICT training. To facilitate ease of access we developed open-access web-based resources. The design is responsive, therefore suitable for any screen size. To increase ease of use, we dropped log-in functionality and simplified the interface as much as possible. We used large font sizes, consistent formatting, and minimized amount of texts to facilitate ease of use during teaching. In the teachers’ guide, we included a help section explaining the navigation and technical features of the site. We created sets of optional, downloadable printouts for each lesson for teachers who preferred or had the opportunity of making paper copies. Opportunity to teach the content within existing curricula using digital resources We identified subjects in the Kenyan, Rwandan, and Ugandan curricula where the IHC Key Concepts could fit in the event of future uptake. Curriculum development offices would need to approve the use of any new teaching resource. Resources must be possible to download, adapt and republish on national platforms. To facilitate tailored implementation, we created an adaptable, translatable solution using Google drive as an editing platform. Curriculum development offices can install their own versions of the resources for future translations or adaptations. Schools had very different levels of access to ICT infrastructure for teaching and learning, ranging from almost no access to well-equipped computer labs. About half of Rwandan secondary schools have “smart classrooms”: computer labs with laptops and Internet access. Poor Internet connectivity and unstable electricity were persistent barriers to use of digital resources in many schools, in all three countries. To accommodate for varied access to ICT across schools, we created resources for delivering lessons in three different modes: Blackboard, Projector, and Student-computer modes. Blackboard lesson plans were optimised for teachers on mobile devices. These could also serve as a back-up in the event of electricity outages. Projector lesson plans included downloadable lessons formatted as Google Slides presentations. We also created sets of optional, downloadable printouts. Student-computer mode were designed for computer lab classrooms, with interactive lessons for students to use individually or in groups, in a class led by a teacher. (We later deactivated this mode, see results from development cycles below.) To accommodate users with a poor Internet connection, we built in offline functionality: when the user visits a page online, an automatic download begins, with messages and icons alerting the user. We also created a cached version that loads faster, for slow Internet connections. Stakeholders chose nine IHC Key Concepts to form the basis for the lessons Twelve curriculum specialists, teachers, and researchers, in Kenya, Rwanda, and Uganda prioritised nine of the IHC Key Concepts ( Box 1 ) as suitable, relevant, and important for lower-secondary school students. 55 These nine concepts formed the basis for the learning goals and subsequent content development. Box 1. Nine prioritised IHC Key Concepts. • Treatments can cause harms as well as benefits. • Widely used treatments or those that have been used for a long time are not necessarily beneficial or safe. • Treatments that are new or technologically impressive may not be better than available alternatives. • Personal experiences or anecdotes alone are an unreliable basis for most claims. • Identifying effects of treatments depends on making comparisons. • Comparison groups should be as similar as possible. • Small studies may be misleading. • Large, dramatic effects are rare. • Weigh the benefits and savings against the harms and costs of acting or not. We embedded teaching strategies for critical thinking in the lessons In an overview of systematic reviews, we identified teaching strategies for helping primary and secondary school students learn to think critically. 44 We experimented with using different strategies in each lesson, but found it added too much to teachers’ and students’ procedural cognitive load. Therefore, we chose a limited set to minimize variation and used most strategies across all lessons ( Box 2 ). Additionally, we prepared summaries of all the teaching strategies we considered using. Box 2. Teaching strategies embedded in lessons. Strategies used across all lessons: ‐ Guided notetaking ‐ Small group work ‐ Response cards ‐ Homework – collecting claims and choices about health actions ‐ Standard lesson structure ‐ Setting objectives and providing feedback ‐ Multimedia design Strategies used in individual lessons: ‐ Concept mapping ‐ Concept cartoons ‐ Inquiry-based instruction ‐ Quiz ‐ Role play For more detail, see also descriptions of teaching strategies we considered including (located under the Extra resources menu of the Besmarthealth.org website). We compiled a collection of student-relevant examples To illustrate concepts, and make lessons more engaging, we used a broad range of examples in the lessons involving conditions and health actions that students said they found interesting. 45 We also created a searchable collection of relevant health actions and health problems or goals that can be used to illustrate key concepts, with plain language summaries of the evidence for each example ( Figure 9 ). The examples collection is part of the “Extra resources”. Findings from Phase 2: Development cycles Using our findings from the Groundwork phase as a starting point, we conducted three development cycles of idea generation, prototyping, data collection and analysis, and stakeholder checking, which form the basis of this paper. The feedback was generally consistent regarding what people experienced as positive or problematic. Although there were far more positive findings, our main focus in collecting and reporting feedback was to identify problems that would require changes for the next iteration of the resources. We present the main findings thematically below, together with the corresponding design decisions that we made based on those findings. Positive, constructive findings Many teachers and students appreciated the learning in these lessons and experienced it as relevant both for other subjects and for their daily lives. Many students participated actively in class, including students who normally were less active. Many teachers felt the Projector mode of lessons worked best, providing structure and visual support to the lesson and helping to focus class attention. The blackboard mode and print-outs work satisfactorily for classes without projectors, or as a back-up in the event of power failure. Table 9 provides a detailed description of positive or constructive findings. Table 9. Positive, constructive findings. Theme Description Illustrative quotes or design decisions Relevance to daily life Many teachers and students said the lessons were relevant to their daily lives and enjoyable. "I liked the idea that most of the lesson examples were daily life, we could easily relate." (Student, group interview, Uganda) “The lessons enabled the students to think of a personal choice and the advantages and disadvantages of making a certain decision” (Teacher, Kenya Group Interview). Engaging and confidence-building for students Most students found the lessons engaging and that they increased their confidence. In the pilot lessons, students participated actively in quizzes and activities, including some who normally were quiet. “I am shy, but group work made me stand to present and this will give me confidence.” (Student, group interview, Uganda) Structure and design of the resources For the most part, the structure of the resources and the lessons was appreciated. “I am impressed of how you present the content in a clear and easy-to-follow manner. I also like how you organize the concepts in main groups and subgroups, it gives an immediate understanding of the focus areas.” (International advisory group member) In Rwanda, stakeholders favored a lesson structure that encourages “discovery”, where the teacher does not provide explanations before an activity. However, in Kenya, stakeholders favored a lesson structure where the teacher provides definitions and explanations were upfront, which are then reinforced through learning activities. Design decision: We added a “discovery” question at the start of each lesson and provided definitions and explanations before the activity. Usefulness for thinking critically about health actions Many teachers and students said the resources were useful for thinking critically about health actions and in general. “It was nice and educative. It will help us not to just follow band wagon. Like we buy certain toothpaste but just because it is many friends use it.” (Student, group interview, Uganda) Satisfaction with projector and blackboard modes The projector mode worked best, with the illustrated slides helping focus class attention, and aiding understanding. The blackboard mode also seemed to work satisfactorily. “Projector based lesson plans helped both teacher and students to focus on one activity than the computer-based version. There wasn't any interruption during the lesson.” (Observation pilot lesson, Rwanda) Useful lesson printouts Some teachers found the printouts a useful way of continuing lessons even when there was a power outage preventing them from using a projector. “After some few minutes, the Power went off and the projector stopped showing the lesson. The teacher immediately gives the printouts to students to follow the lesson. After 3 minutes, the power returned.” (Observation pilot lesson, Rwanda) Negative findings and corresponding design decisions We resolved problems as they appeared and changed the design of the prototypes dramatically from the first to the final development cycle. However, four main thematic challenges persisted throughout development: issues regarding the student-computer mode of lesson plans, Time, Comprehension, and Examples. • Computers in classrooms took time to set up; equipment problems were common; students were often distracted by other online content; teachers found class discussions difficult to organise when students were using computers. Since teachers who had tried both the student-computer and Projector modes preferred the latter, we agreed with curriculum developers to drop the student-computer mode. • School schedules lacked time to teach lessons that were not in the curriculum, and teachers struggled to complete lessons within the allotted 40 minutes. We simplified and shortened lesson content as much as possible, and developed modules for teacher training workshops to increase teachers’ confidence and capability to teach the lessons. • Students had varying degrees of difficulty understanding lesson content, sometimes displaying a misunderstanding that was exactly the opposite of what they were intended to learn. We added “Common misunderstandings” to the lesson background, and designed lessons to include more review opportunities and informal assessment questions. We are exploring experiences and views of potential adverse effects in separate process evaluations 30 , 31 , 56 and in a qualitative evidence synthesis. 57 • We struggled finding appropriate examples of reliable and unreliable claims, as students often thought the lesson was about the example rather than the underlying Key Concept. Our solutions included finding a balance between real and fictional, well-labeled examples, guidance and prompts for teachers (including suggestions to find their own examples), and developing an example collection for teachers to find alternatives. Tables 10 - 13 provide a more detailed description of challenges and corresponding design decisions. Table 10. Difficulties using the Student-computer mode. Description Illustrative quotes Design decisions Some curriculum developers wanted more media-rich, interactive resources that students could use for self-study in computer labs. However, we found technical, logistical, and behavioural problems with the student-computer mode: • Valuable teacher time used for setting up and charging the computers beforehand. • Broken computers. • Necessity for students to sometimes share a computer. • Students needing help finding browsers and reopening the resources after power outages. • Students cheating on quizzes or avoiding discussing things by opening the teacher resources or jumping ahead in the lessons to find the correct answers. • Students visiting other websites during the class. • Group discussions, one of the teaching strategies we had prioritized, not working well. Teachers who had tried both the student-computer and projector modes preferred the latter. “It is not easy for a teacher to handle technical issues while teaching. It disturbs her and the students also lose the focus. Most technical issues they experience, is to reopen the resources, to restart the computer, to help the students find the pages of where the teacher is focusing on, and the power issues.” (Observation pilot lesson, Rwanda) “The students were told to discuss in groups the activity part 1,2 and 3 but the students did not focus on doing these activities, they were busy exploring how the laptops work, visiting other websites, and others were writing the notes instead of discussing in groups.” (Observation pilot lesson, Rwanda) After the first few pilot lessons, in consultation with the curriculum developers, we decided to drop the student-computer mode. Table 11. Time limitations. Description Illustrative quotes Design decisions With early prototypes, lessons often took more than the intended 40 minutes because: • The content, including key terms, was often unfamiliar to students and teachers. • Some of the teaching strategies, such as getting students to take structured notes, and organising small group discussions, were time-consuming. In pilots, teachers tried to save time by skipping parts of, or entire, lessons or by falling back on didactic teaching strategies. Participating schools squeezed pilot lessons into their schedules at a time when schedules were already under extra pressure due to pandemic-related school closures. Classes were often held before or after regular school hours, when both teachers and students were tired. "I think the teacher should give us more time to think because you could just start thinking about an answer and the teacher says the 3 minutes are done" (Student, group interview, Uganda) We suggested reducing the number of IHC Key Concepts included. Due to resistance to this in stakeholder checking, we rearranged content and combined some lessons. We made a number of other timesaving changes: • removed quizzes and shortened reviews in most lessons, • limited activities to one per lesson, • used buzz-groups (2-3 students who are sitting next to each other) or whole-class discussions instead of small-group work, • developed printable student handouts and classroom posters to reduce students’ need to take notes. Table 12. Problems with understanding. Description Illustrative quotes Design decisions Some students did not understand some of the content. When asked to describe take-home messages from a lesson, students sometimes displayed a misunderstanding that was exactly the opposite of what they were intended to learn. Many students performed poorly on the test at the end of the pilot. Low scores in all three countries were likely in part linked to teachers’ lack of familiarity with the content lessons—especially concepts about research evidence. Results can be found in the dataset:“Pilot data and test results - analysis of problems and proposed solutions.pdf”. Teachers emphasized the need for checking students’ comprehension from lesson to lesson. “Whenever I go to buy a medicine, I need to check how long it has been in use and how many have used it” (Rwanda, student interview) “Personal experience can help to have information about any disease if you have experienced it” (Rwanda, student interview) To support students’ and teachers’ understanding: • We iteratively simplified the lesson structure, language, and content. • We added “Common misunderstandings” to each lesson Background. • We added review questions and short key messages at the end of lessons, which are repeated at the start of the next lesson. We developed printouts with definitions of key terms. To support teachers’ evaluation of students understanding and progress, we incorporated two review lessons (lessons 5 and 10) with quizzes. To support teachers’ understanding of the content and confidence in using the resources, we added a teacher training component to the intervention: a two-day workshop with presentations and activity materials, introducing teachers to the resources, teaching strategies, and content of each lesson, as well as providing teaching and preparation tips. We designed these training materials to be used by teachers who were already familiar with the resources (e.g., those who participated in pilots), so training would not be dependent on our team’s participation and would be scalable. Table 13. Unsuitable or distracting examples illustrating the Key Concepts. Description Illustrative quotes Design decisions Familiar examples engaged students, but often led to class discussions focused on the specific example rather than the underlying, generic concept that it illustrated. Some students thought that the lesson was about the information in the example, and not about learning to think critically about health information. On the other hand, fictional or unfamiliar examples appeared less engaging, because students could not relate them to their own lives or found them confusing. Many students did not readily question the reliability of claims about Covid-19 measures that were already widely implemented. Some teachers proposed that we add examples that were more familiar to students. In many cases, teachers supplemented with their own examples. “They take examples as key messages of the lesson they learnt.” (Rwanda, teacher group interview) “Student were not able to give examples of other possible options that could be made if someone doesn’t choose to cool a burn with running water. This example was not familiar to them.” (Rwanda, observation pilot lesson) “Some examples of Covid-19 used in these lessons limit students to think big. For example, if you are asking the students if wearing a mask is effective in reducing the spread of Covid-19, they will all say yes, because they are all using them, without even thinking if they are good to use or not.” (Rwanda, teacher group interview) “She just suggested adding more information about adolescent health and more examples of claims that are often told to the youth such as believing that sexual intercourse wipes away the pimples and lessens the menstrual cramps during the period.” (Rwanda, teacher interview) “The teacher used other examples that students seemed to relate with: use of marijuana and its effect on school dropout, anti-retroviral drugs and death rates, circumcision and HIV, and a community with drunkenness and community development.” (Kenya, observation pilot lesson) As far as possible, we used real, familiar, and relevant examples, drawing on examples collected from students. Where we were unable to identify a real example that seemed appropriate, we used fictional examples and tried to clearly indicate that they were fictional. In the teachers’ guide, we suggested teachers could use their own examples or find new ones in the searchable collection of health actions in the extra resources. In the teachers’ guide and the teacher training materials, we emphasized that the examples must illustrate the generic concepts, not just draw attention to specific health conditions, actions, or claims. In the background section of each lesson plan, we summarised research evidence about the used in that lesson. We also provided summaries of the evidence for the effects of all the health actions used in the resources in the searchable collection of health actions. We did this to ensure that the teachers were familiar with the examples, felt comfortable using them, and were prepared to prevent misunderstandings about the reliability of claims or evidence relating to them. Other noteworthy design decisions In addition to the main “design decisions” presented in Tables 8 - 13 above, we also made larger, more sweeping decisions that involved a reframing of the problem we were trying to solve. For example, our original project aim was to design digital learning resources for students to interact with. But over time we gradually increased our focus on designing resources that met teachers’ needs, as we became aware of the challenges they experienced teaching this content within the constraints of their skillsets, curricula, class sizes, and varying quality of ICT infrastructure. Ultimately we eliminated the student computer mode of the resources altogether, even for schools with computer labs, as a consequence of the scope of technical, logistical and behavioral challenges we observed in school pilots. We reflect on this decision in the discussion, in particular as it relates to considerations for teaching critical thinking. Covid-19 consequences The COVID-19 pandemic disrupted our work in many ways. At times, social-distancing and travel restrictions, as well as school closures, prohibited in-person meetings and data collection and piloting. When schools reopened, teachers were pressed to make up for lost time, sometimes rushing to finish a pilot lesson or not completing it. On the other hand, the abundance of unreliable claims about Covid-19 provided timely examples and motivation. “The lesson itself was very easy to understand because the students have already experienced what was used as example in the content (closing schools to reduce the spread of covid-19 infections).” (Rwanda, Observation pilot lesson) Workarounds to challenges included communicating with stakeholders individually via audio or video call, or chat, and conducting some of interviews with home networks. Discussion Using a human-centred design approach, we developed adaptable, digital resources for teaching critical thinking about health in secondary schools in Kenya, Rwanda, and Uganda. In Phase 1 of the design, we gained insight into user and stakeholder needs, and the wider educational contexts. In Phase 2, we iteratively developed prototypes by generating ideas, prototyping, collecting and analysing data, and checking our ideas and analysis with stakeholders. The result was a final set of educational resources, descriptions of findings from data analyses, and corresponding design decisions we made. Why a human-centred design approach? Human-centred design is a creative approach to problem solving (e.g. developing artifacts, services, systems) that is characterised by early engagement with multiple stakeholders and iterative development. 58 It has been used across a number of different areas of research and development in fields not traditionally associated with design, 59 – 62 and is particularly well suited to addressing complex problems. 63 The problem we set out to address was complex in many ways, with many unknowns: - We knew little about whether the attitudes and perspectives of key stakeholders were aligned with the overall goals of this project - We knew little about where in the curricula this new content might fit in or overlap - We knew very little about about the context of use (e.g. what digital infrastructure was available in secondary schools across the three countries, and how it was being used in teaching) - We knew very little about the characteristics and needs of end users - We knew very little about the desired attributes of the solution, especially given the complexity of teaching critical thinking. Tiruneh et al. argue that “Critical thinking requires complex learning … learning environments that may be successful for simple learning outcomes may not work well for the development of complex learning” 64 - We needed to learn what factors regarding the solution could support not just use, but future implementation For these reasons, we needed an approach that was not merely task-focused or “user-centered” but that included exploration of the learning context, collaboration with multiple key stakeholders, and that allowed us to learn about and reframe and resolve problems as we worked. Reflections about learning theory We did not start this work with an explicit theory of learning. Our intention was to understand the existing curricula and teaching practices in the three countries, then develop resources that could fit these (or be adapted to them) rather than imposing a new theoretical perspective from the outside. Some of the curriculum development offices have made their learning theories explicit. For instance, the Republic of Kenya Basic Education curriculum Framework is clearly founded on a constructivist approach to learning. 65 In retrospect, our approach to lesson structure resembles Merill’s First Principles of Instruction. 66 BeSmart lessons are taught in the context of health conditions, treatments and claims that learners are familiar with and interested in; lessons begin by engaging learners’ prior knowledge and experiences; teachers provide a brief introduction to new content and demonstrate how it is used; learners apply the new knowledge on new examples; peer-to-peer and class discussions provide opportunities for reflection on what is learned and how it is relevant to the learners’ own lives. More generally, we are influenced by Bruner’s concept of a “spiral curriculum” when conceptualizing how resources to teach IHC Key Concepts might be best integrated into a curriculum: introducing concepts at a young age and revisiting them repeatedly with increasing degrees of difficulty and reinforcement of previous learning. 67 Design and technology implications for teaching and learning critical thinking In many countries, curriculum developers are moving away from traditional knowledge-based curricula to competency-based curricula, with critical thinking among the core competencies, 68 including in Kenya, Rwanda, and Uganda. 39 – 41 However, our context analyses showed that teachers lacked training and resources to teach critical thinking. 39 – 41 In their systematic review of strategies for teaching students to think critically, Abrami et al. found that three strategies were most effective, especially when combined: “the opportunity for dialogue, the exposure of students to authentic or situated problems and examples, and mentoring”. 69 Based on those findings, and findings from our overview of systematic reviews, 44 we focused on using authentic examples and discussion activities in our secondary school resources. A review of education technology in low-income countries distinguishes between two categories of digital interventions: computer aided learning (CAL) and computer aided instruction (CAI). 70 Broadly speaking, CAL interventions are designed for the student-as-user, and either complement or supplement classroom instruction. CAI interventions, on the other hand, are designed for the teacher-as-user, and aim to enhance the quality of instruction. We developed and piloted resources for both CAL (the student-computer mode of lessons) and CAI (the blackboard and projector modes of lessons). We found a variety of problems with the student-computer (CAL) resources. Many of the problems were logistical or technical, and not specific to our project, such as time needed to set up and charge computers, faulty or missing equipment, and unstable electricity. In addition, some curriculum developers (where student-computer resources were piloted) said that the resources lacked media-richness (such as animations or videos) as well as interactivity that we could not feasibly develop in this project and that teachers and students were possibly not experienced enough to use. Regardless of the level of ICT preparedness, it might be that self-led study strategies and CAL interventions are not the best starting point for developing critical thinking skills, since it is more challenging to creating opportunities for dialogue when students are alone behind a screen, working at different paces from each other. In our case, developing functionality for dialogue to take place on the computer was not an option, due to technical barriers in the settings, such as lack or unreliable internet access. Computer-based mentoring 71 – 74 would require much more sophisticated programming than we had resources for. Moreover, less literate and less computer-literate students would be at a disadvantage. In their review, Abrami and colleagues’ found that discussion was especially effective when the teacher posed questions and there were whole-class, teacher-led discussions, or teacher-led group discussions. 69 Rwandan teachers who piloted both the student-computer and projector resources preferred the latter, with the blackboard and printed resources as a backup for when electricity failed. Besides being technically less demanding, the slides seemed to better activate students, support small and large-group discussion, and focus collective attention. Moreover, teachers can download and tailor slides. The lower-tech CAI modes require much less investment in equipment than high-tech CAL modes and could dramatically lower the threshold for bringing digital learning resources that support critical thinking into low-resource classrooms. Strengths and limitations Strengths of this study include comprehensive groundwork; several development cycles; including pilots of all lessons in three countries; input from a wide variety of stakeholders and participants; systematic methods used to prioritise content and identify effective teaching strategies; data collection using complementary, mixed methods; independent initial coding and analysis by more than one researcher; and stakeholder checking to triangulate analyses. A limitation is that the project team developed the prototypes and collected and coded data. This might have biased participants to give positive feedback, or introduced bias in the analysis, leading to an exaggeration of positive feedback or lack of attention to negative feedback. We took measures to mitigate this: the interview guide included questions that explicitly encouraged participants to provide negative feedback; data was collected in three countries and coding in each country was reviewed by two researchers in order to reduce potential for bias; we introduced “stakeholder checking” to triangulate our data analyses and check that the solutions we developed were appropriate. It is possible that the amount of data we collected placed an undue burden on participants and other stakeholders. We are exploring this in a separate stakeholder evaluation. 38 It is also possible that the prototypes caused harm to some students who misunderstood concepts or examples. We are exploring potential adverse effects of the final resources in process evaluations 30 , 31 , 56 and a qualitative evidence synthesis. 57 Strengths and limitations of the human-centred design approach The main strengths of this approach are ensuring we are solving the right set of problems and building ownership to the output by engaging early with a wide range of stakeholders; working iteratively with low-fidelity prototypes before committing to a final solution; and taking a holistic approach that includes early consideration of implementation during development. 63 , 75 , 76 However, this approach to developing educational resources likely takes much more time to practice than less comprehensive methods. 75 When practiced outside of research projects, human-centred design involves processes that resemble qualitative research methods, but with less emphasis on creating an audit trail that can be replicated by others. 77 When these design processes are embedded in and framed as research, any point of contact with stakeholders (e.g. observation, feedback, or collaboration) becomes a potential data collection event. The large number of interactions we had with stakeholders resulted in a large amount of data that was at times difficult to analyse, a problem that can also occur qualitative research. 78 We also chose to report entire 2.5 years of development in this one article, possibly making it difficult to follow the data trail. An alternative article strategy may have been to write up each design iteration as a separate manuscript, then linking them together as reports summarized in one article. 79 Reflexivity considerations The core project team all have backgrounds as public health researchers, while the stakeholders recruited for the purposes of collaborating and providing feedback have mostly backgrounds in education with no previous engagement in this research topic. The core project team members are from many different settings, while all stakeholders are from East Africa with the exception of the international advisory group. The PhD fellows and several other core team members were working full-on this project, and therefore highly invested in a positive outcome, while stakeholders were participating voluntarily, in addition to their other full-time work or study commitments. We have tried to be mindful of our differing agendas, cultural perspectives, and the ways in which the core team’s ambitions might influence the outcome of the work or place undue burden on the participants. We discussed these topics frequently during weekly core team meetings and have also explored some of these questions using more structured methods, designed as separate studies. The context analyses we conducted at the beginning of this work helped us view our research agenda from the perspective of teachers and curriculum developers and understand whether and how our research might better align with existing objectives within the three educational systems. The stakeholder engagement assessment (manuscript under development) provided stakeholders with opportunities to give us feedback about how they experienced their participation while it was ongoing. The qualitative evidence synthesis of post-trial process evaluations (manuscript under development) will help us understand if there have been any adverse effects of the resources, seen from the point of view of the participants. Relationships between the core team and recruited stakeholders developed over time, and some people who were initially recruited as participants became engaged to the degree that they fulfilled requirements for co-authorship of this study. Conclusion Using a human-centred design approach, we created adaptable, digital open access resources for teaching lower secondary school students to think critically about health actions that teachers and students in Kenya, Rwanda and Uganda found relevant and useful. The human-centred design approach provided a helpful framework to incrementally develop solutions that served the needs of teachers, students, and schools in both high- and low-resource contexts. Designed to accommodate use in classrooms with differing digital infrastructure, such as lack of internet connectivity or unstable electricity, they include materials for ten 40-minute lessons in two modes: for use in classrooms equipped with blackboards or projectors . In addition to the lessons, resources include a teachers’ guide , glossary , a searchable set of alternative examples of health actions , summaries of teaching strategies for critical thinking , and teacher training materials . Teachers with access to a projector appeared to prefer the projector resources. Persistent challenges during development included issues regarding the modules for student computers, finding sufficient time in the school schedule for content not included in the curriculum, misunderstanding of some of the concepts, and problems related to the use of examples. We provide details of how we approached emerging challenges with corresponding design decisions. The Covid-19 pandemic disrupted the design of the resources, but also provided timely examples and motivation to learn. A Prospective meta-analysis synthesizing the results of the three randomised trials has shown that the resources led to a large improvement in the ability of students and teachers to think critically about health choices, though only 42% of students achieved a passing score. 80 Next research steps are well underway: process evaluations of the three trials, reporting about potential adverse effects, and one-year follow up studies. Data availability statement Underlying data Zenodo. Dataset for “Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources”, https://doi.org/10.5281/zenodo.7695782 . 47 This project contains the following underlying data: • DevelopmentCycle1__data feedback and observations.csv (Coded datapoints extracted in Development cycle 1 from individual/group interviews with teachers/students and observations of lesson pilots in Rwanda, Kenya and Uganda) • DevelopmentCycle1_internal team feedback.csv (Feedback, comments, suggestions from our research team during Development cycle 1) • DevelopmentCycle1_interntl-advisory-grp-dec-jan-2020.csv (First round of feedback from international advisory group, via email) • DevelopmentCycle1_interntl-advisory-grp-jun-aug-2020.csv (Second round of feedback from international advisory group, via email) • DevelopmentCycle1_interntl-advisory-grp-proto1.2.csv (Third round of feedback from international advisory group, via email) • DevelopmentCycle1_legend.csv (Description of implications codes we used to assess the importance of observations and feedback for the next iteration of the resources, e.g., “Problem/Showstopper/A problem that we should address”) in Development Cycle 2 • DevelopmentCycle2__data feedback and observations data.csv (Coded datapoints extracted in Development cycle 2 from individual/group interviews with teachers/students and observations of lesson pilots in Rwanda, Kenya and Uganda,) • DevelopmentCycle2_codes and response options.csv (List of codes/response options used in Development cycle 2 to code or describe datapoints) • DevelopmentCycle2_definitions.csv (Description of “nature”, “implications” and “Action for resolving” codes used in Development cycle 2) • DevelopmentCycle2_internal team feedback.csv (Feedback, comments, suggestions from our research team during Development cycle 2) • Pilot data and test results - analysis of problems and proposed solutions.pdf (Analysis of problems and proposed solutions after lesson pilots at the end of Development cycle 2 that we presented to curriculum developers for stakeholder checking and input) Extended data Zenodo. Extended dataset and Coreq Checklist for “Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources”, https://doi.org/10.5281/zenodo.7806139 . 48 This project contains the following extended data: • COREQ_checklist New.pdf (COREQ checklist for this article) • Design reporting checklist.pdf • Group Interview guide Post-Pilot - Students Group interview guide.docx (Guide used as a basis for interviewing groups of students after they had participated in piloting lessons at the end of Development cycle 2) • Group Interview guide Post-Pilot - Teachers.docx (Guide used as a basis for interviewing teachers after they had piloted lessons at the end of Development cycle 2) • Interveiw guide Early - Student network.docx (Guide used as a basis for interviewing students in Development cycle 1) • Interview guide Early - Curriculum develop Questionnaire.docx (Guide used as a basis for interviewing curriculum developers in Development cycle 1) • Interview guide Pilot - Teachers User testing interview guide.docx (Guide used as a basis for user test interviews with participating teachers in pilot lessons at the end of Development cycle 2) • Interview Guide Pre-pilot - Teacher-Student - User test.docx (Guide used as a basis for user test interviews of students and teachers before they participated in pilot lessons at the end of Development Cycle 2) • Observation form Pilot - Observers.docx (Form used by observers to structure observation data of pilot lessons) Data are available under the terms of the Creative Commons Attribution 4.0 International Public License. Acknowledgments We would like to thank all of the students, teachers, and schools who generously gave of their time to participate in networks and pilots. In particular, we would like to thank Besweri Wandera, Joseph Jude Agaba, Enock Kiyemba, Deogratious Ssekyole, Dan Bubaale, Fredrick Kiyingi Kinobe, Grace Namakula, Peter Kizza, Edward Kanoonya, Enock Kiyemba, Judith Nantongo, Patrick Ssendujja, Steven Musisi, John Bosco Kalangwa, Paul Sendi, Grace Namakula, Bridgious Kato, Charles Musabi, Ann Murangira, Stanely Shitanda, lynnet Juma, Pamela Nandi, Robert Onyango, Brian Okoth, Emmanuel Otieno, Rose Omondi, Bridgit Abayo, Fredick Okoth, John Ouko, Lawrence Ariang’a, Innocent Uwimana, Paul Nkundimana, Aloys Kayinamura, Janvier Ntibizerwa, Aaron Habarurema, Donat Nkurunziza, Chui Hsia, and Christine Holst. We also thank the Kenya Institute of Curriculum Development (KICD), Rwanda Basic Education Board (REB), and National Curriculum Development Centre (NCDC) in Uganda for their ongoing encouragement and collaboration. References 1. Oxman M, Larun L, Gaxiola GP, et al. : Quality of information in news media reports about the effects of health interventions: systematic review and meta-analyses. F1000Res. 2021; 10 : 10. Publisher Full Text 2. Boutron I, Haneef R, Yavchitz A, et al. : Three randomized controlled trials evaluating the impact of “spin” in health news stories reporting studies of pharmacologic treatments on patients’/caregivers’ interpretation of treatment benefit. BMC Med. 2019; 17 (1): 1–10. 3. Dahlgren A, Furuseth-Olsen K, Rose CJ, et al. : The Norwegian public’s ability to assess treatment claims: results of a cross-sectional study of critical health literacy. F1000Res. 2021; 9 (179): 179. Publisher Full Text 4. Li HO-Y, Bailey A, Huynh D, et al. : YouTube as a source of information on COVID-19: a pandemic of misinformation? BMJ Glob. Health. 2020; 5 (5): e002604. PubMed Abstract | Publisher Full Text | Free Full Text 5. Sharp MK, Forde Z, McGeown C, et al. : Irish media coverage of COVID-19 evidence-based research reports from one national agency. Int. J. Health Policy Manag. 2021; 11 : 2464–2475. PubMed Abstract | Publisher Full Text | Free Full Text 6. Wagner DN, Marcon AR, Caulfield T: “Immune Boosting” in the time of COVID: selling immunity on Instagram. Allergy, Asthma Clin. Immunol. 2020; 16 (1): 1–5. 7. Janiaud P, Axfors C, van't Hooft J , et al. : The worldwide clinical trial research response to the COVID-19 pandemic - the first 100 days [version 2; peer review: 2 approved]. F1000Res. 2020; 9 (1193). Publisher Full Text 8. Jones CW, Woodford AL, Platts-Mills TF: Characteristics of COVID-19 clinical trials registered with ClinicalTrials. gov: cross-sectional analysis. BMJ Open. 2020; 10 (9): e041276. Publisher Full Text 9. Glasziou PP, Sanders S, Hoffmann T: Waste in covid-19 research. BMJ. 2020; 369 : m1847. Publisher Full Text 10. Pian W, Chi J, Ma F: The causes, impacts and countermeasures of COVID-19 “Infodemic”: A systematic review using narrative synthesis. Inf. Process. Manag. 2021; 58 (6): 102713. PubMed Abstract | Publisher Full Text | Free Full Text 11. Borges do Nascimento IJ, Pizarro AB, Almeida JM, et al. : Infodemics and health misinformation: a systematic review of reviews. Bull. World Health Organ. 2022; 100 (9): 544–561. PubMed Abstract | Publisher Full Text | Free Full Text 12. Chou W-YS, Gaysynsky A, Vanderpool RC: The COVID-19 Misinfodemic: moving beyond fact-checking. Health Educ. Behav. 2021; 48 (1): 9–13. PubMed Abstract | Publisher Full Text | Free Full Text 13. Cusack L, Del Mar CB, Chalmers I, et al. : Educational interventions to improve people’s understanding of key concepts in assessing the effects of health interventions: a systematic review. Syst. Rev. 2018; 7 (1): 68. PubMed Abstract | Publisher Full Text | Free Full Text 14. Roozenbeek J, van der Linden S , Goldberg B, et al. : Psychological inoculation improves resilience against misinformation on social media. Sci. Adv. 2022; 8 (34): eabo6254. Publisher Full Text 15. Nsangi A, Semakula D, Rosenbaum SE, et al. : Development of the informed health choices resources in four countries to teach primary school children to assess claims about treatment effects: a qualitative study employing a user-centred approach. Pilot Feasibility Stud. 2020; 6 (18): 18. PubMed Abstract | Publisher Full Text | Free Full Text 16. Semakula D, Nsangi A, Oxman M, et al. : Development of mass media resources to improve the ability of parents of primary school children in Uganda to assess the trustworthiness of claims about the effects of treatments: a human-centred design approach. Pilot Feasibility Stud. 2019; 5 (1): 1–17. Publisher Full Text 17. Nsangi A, Semakula D, Oxman AD, et al. : Effects of the Informed Health Choices primary school intervention on the ability of children in Uganda to assess the reliability of claims about treatment effects: a cluster-randomised controlled trial. Lancet. 2017; 390 (10092): 374–388. PubMed Abstract | Publisher Full Text 18. Semakula D, Nsangi A, Oxman AD, et al. : Effects of the Informed Health Choices podcast on the ability of parents of primary school children in Uganda to assess claims about treatment effects: a randomised controlled trial. Lancet. 2017; 390 (10092): 389–398. PubMed Abstract | Publisher Full Text 19. Nsangi A, Semakula D, Oxman AD, et al. : Effects of the Informed Health Choices primary school intervention on the ability of children in Uganda to assess the reliability of claims about treatment effects, 1-year follow-up: a cluster-randomised trial. Trials. 2020; 21 (1): 1–22. Publisher Full Text 20. Semakula D, Nsangi A, Oxman AD, et al. : Effects of the Informed Health Choices podcast on the ability of parents of primary school children in Uganda to assess the trustworthiness of claims about treatment effects: one-year follow up of a randomised trial. Trials. 2020; 21 (1). Publisher Full Text 21. Austvoll-Dahlgren A, Oxman AD, Chalmers I, et al. : Key concepts that people need to understand to assess claims about treatment effects. J. Evid. Based Med. 2015; 8 (3): 112–125. PubMed Abstract | Publisher Full Text 22. Chalmers I, Oxman AD, Austvoll-Dahlgren A, et al. : Key concepts for informed health choices: a framework for helping people learn how to assess treatment claims and make informed choices. BMJ Evid. Based Med. 2018; 23 (1): 29–33. PubMed Abstract | Publisher Full Text 23. Austvoll-Dahlgren A, Semakula D, Nsangi A, et al. : Measuring ability to assess claims about treatment effects: the development of the 'Claim Evaluation Tools'. BMJ Open. 2017; 7 (5): e013184. PubMed Abstract | Publisher Full Text | Free Full Text 24. Informed Health Choices: Informed Health Choices. n.d. Reference Source 25. Nsangi A, Semakula D, Glenton C, et al. : Informed health choices intervention to teach primary school children in low-income countries to assess claims about treatment effects: process evaluation. BMJ Open. 2019; 9 (9): e030787. PubMed Abstract | Publisher Full Text | Free Full Text 26. Semakula D, Nsangi A, Oxman A, et al. : Informed Health Choices media intervention for improving people’s ability to critically appraise the trustworthiness of claims about treatment effects: a mixed-methods process evaluation of a randomised trial in Uganda. BMJ Open. 2019; 9 (12): e031510. PubMed Abstract | Publisher Full Text | Free Full Text 27. Effects of the Informed Health Choices secondary school intervention on the ability of lower secondary students in Kenya to think critically about health information and choices: Protocol for a cluster-randomized trial. Zenodo. 2022 [cited November, 2022]. 28. Effects of using the Informed Health Choices digital secondary school resources on the ability of Rwandan students to think critically about health: protocol for a cluster-randomised trial. Zenodo. 2022 [cited November, 2022]. 29. Does the use of the Informed Health Choices teaching resources improve the secondary students' ability to critically think about health in Uganda? A cluster randomised trial protocol.2022 [cited November 2022]. 30. Ssenyonga R, Lewin S, Nakyejwe E, et al. : Informed heath choices intervention to teach secondary school adolescents in Uganda to assess claims about treatment effects: a process evaluation protocol.2022. Publisher Full Text 31. Mugisha M, Nyirazinyoye L, Oxman AD, et al. : Use of the Informed Health Choices digital resources for teaching lower secondary school students in Rwanda to think critically about health: protocol for a process evaluation.2022. 32. Chesire F, Kaseje M, Ochieng M, et al. : Effect of the Informed Health Choices digital secondary school resources on the ability of lower secondary students in Kenya to critically appraise health claims: protocol for a process evaluation.2022 Feb 27, 2023. Reference Source 33. Rosenbaum S, Oxman M, Oxman AD, et al. : Human-centred design development of Informed Health Choices (IHC) learning resources for secondary school students: Protocol. Zenodo. 2019 2019/12. Reference Source 34. Rosenbaum SE: Improving the user experience of evidence: A design approach to evidence-informed health care. Oslo: Arkitektur- og designhøgskolen i Oslo; 2010. 35. Giacomin J: What is Human Centred Design? Des. J. 2014; 17 (4,2014): 606–623. Publisher Full Text 36. Melles M, Albayrak A, Goossens R: Innovating health care: key characteristics of human-centered design. Int. J. Qual. Health Care. 2021; 33 (Supplement_1): 37–44. PubMed Abstract | Publisher Full Text | Free Full Text 37. ISO: Ergonomics of human-system interaction — Part 210: Human-centred design for interactive systems. Geneva: 2019. 38. Nsangi A, Oxman AD, Oxman M, et al. : Protocol for assessing stakeholder engagement in the development and evaluation of the Informed Health Choices resources teaching secondary school students to think critically about health claims and choices. PLoS One. 2020; 15 (10): e0239985. PubMed Abstract | Publisher Full Text | Free Full Text 39. Chesire F, Ochieng M, Mugisha M, et al. : Contextualizing critical thinking about health using digital technology in secondary schools in Kenya: a qualitative analysis. Pilot Feasibility Stud. 2022; 8 (1): 227. PubMed Abstract | Publisher Full Text | Free Full Text 40. Mugisha M, Uwitonze AM, Chesire F, et al. : Teaching critical thinking about health using digital technology in lower secondary schools in Rwanda: A qualitative context analysis. PLoS One. 2021; 16 (3): e0248773. PubMed Abstract | Publisher Full Text | Free Full Text 41. Ssenyonga R, Sewankambo NK, Mugagga SK, et al. : Learning to think critically about health using digital technology in Ugandan lower secondary schools: A contextual analysis. PLoS One. 2022; 17 (2): e0260367. PubMed Abstract | Publisher Full Text | Free Full Text 42. Michie S, Van Stralen MM, West R: The behaviour change wheel: a new method for characterising and designing behaviour change interventions. Implement. Sci. 2011; 6 (1): 1–12. 43. Agaba JJ, Chesire F, Mugisha M, et al. : Prioritisation of Informed Health Choices (IHC) key concepts to be included in lower secondary school resources: A consensus study. PLoS One. 2023; 18 (4): e0267422. PubMed Abstract | Publisher Full Text | Free Full Text 44. Oxman AD, Dahlgren A, García LM, et al. : The effects of teaching strategies on learning to think critically in primary and secondary schools: protocol for an overview of systematic reviews. Norwegian Institute of Public Health; 2019. 45. Moberg J: Creating an examples collection for the “Be Smart About Your Health” resources: part of the CHOICE project.2023. Publisher Full Text 46. Birt L, Scott S, Cavers D, et al. : Member checking: a tool to enhance trustworthiness or merely a nod to validation? Qual. Health Res. 2016; 26 (13): 1802–1811. Publisher Full Text 47. Rosenbaum S, et al. : Dataset for “Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources”. [Data set]. Zenodo. 2023. Publisher Full Text 48. Rosenbaum S, et al. : Extended dataset and Coreq Checklist for "Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources". Zenodo. 2023. Publisher Full Text 49. Charters E: The use of think-aloud methods in qualitative research an introduction to think-aloud methods. Brock Educ. J. 2003; 12 (2). Publisher Full Text 50. Ericsson K, Simon HA: Verbal reports as data. Psychol. Rev. 1980; 87 (3): 215–251. Publisher Full Text 51. Kuniavsky M: Observing the User Experience: A Practitioner's Guide to User Research. 1 ed.Kaufmann M, editor. Morgan Kaufmann; 2003; 575 p. 52. Dahlgren A, Semakula D, Chesire F, et al. : Critical thinking about treatment effects in Eastern Africa: development and Rasch analysis of an assessment tool. F1000Research. 2023; 12 : 887. 53. Bazzano AN, Yan SD, Martin J, et al. : Improving the reporting of health research involving design: a proposed guideline. BMJ Glob. Health. 2020; 5 (2): e002248. PubMed Abstract | Publisher Full Text | Free Full Text 54. Informed Health Choices project: Be Smart About Your Health 2022 [Educational resources]. Reference Source 55. Agaba JJ, Chesire F, Michael M, et al. : Prioritisation of Informed Health Choices (IHC) Key Concepts to be included in lower-secondary school resources: a consensus study.2022. 56. Chesire F, Kaseje M, Ochieng M, et al. : Effect of the Informed Health Choices digital secondary school resources on the ability of lower secondary students in Kenya to critically appraise health claims: protocol for a process evaluation.2022. 57. Oxman M, Oxman AD, Fretheim A, et al. : Participants' and investigators' experiences and views of potential adverse effects of an educational intervention: Protocol for a qualitative evidence synthesis (Version 1).2023. Publisher Full Text 58. Rodriguez NM, Burleson G, Linnes JC, et al. : Thinking beyond the device: an overview of human-and equity-centered approaches for health technology design. Annu. Rev. Biomed. Eng. 2023; 25 (1):257–280. PubMed Abstract | Publisher Full Text | Free Full Text 59. Chen E, Leos C, Kowitt SD, et al. : Enhancing community-based participatory research through human-centered design strategies. Health Promot. Pract. 2020; 21 (1):37–48. PubMed Abstract | Publisher Full Text 60. Leary M, Cacchione PZ, Demiris G, et al. , editors: An integrative review of human-centered design and design thinking for the creation of health interventions.Nursing Forum.2022: Wiley Online Library. 61. Bazzano AN, Martin J, Hicks E, et al. : Human-centred design in global health: a scoping review of applications and contexts. PLoS One. 2017; 12 (11): e0186744. PubMed Abstract | Publisher Full Text | Free Full Text 62. Lyon AR, Brewer SK, Areán PA: Leveraging human-centered design to implement modern psychological science: Return on an early investment. Am. Psychol. 2020; 75 (8):1067. 63. Rosinsky K, Murray D, Nagle K, et al. : A review of human-centered design in human services. OPRE Report 2022-78. Washington, DC: Office of Planning. Research, and Evaluation, Administration for Children and Families, US Department of Health and Human Services. 2022. 64. Tiruneh DT, De Cock M, Spector JM, et al. : Toward a systematic and model-based approach to design learning environments for critical thinking. Learning, design, and technology: An International compendium of theory, research, practice, and policy. Springer; 2023; pp. 955–991. 65. Kenya Institute of Curriculum Development. Basic Education Curriculum Framework; 2017. 66. Merrill MD: First principles of instruction. Educ. Technol. Res. Dev. 2002; 50 :43–59. Publisher Full Text 67. Bruner JS: The process of education.Harvard University Press; 2009. Publisher Full Text 68. Voogt J, Roblin NP: A comparative analysis of international frameworks for 21st century competences: Implications for national curriculum policies. J. Curric. Stud. 2012; 44 (3): 299–321. Publisher Full Text 69. Abrami PC, Bernard RM, Borokhovski E, et al. : Strategies for teaching students to think critically: A meta-analysis. Rev. Educ. Res. 2015; 85 (2): 275–314. Publisher Full Text 70. Rodriguez-Segura D: Educational technology in developing countries: A systematic review. University of Virginia EdPolicy Works Working Papers.2020. Reference Source 71. Gerard L, Matuk C, McElhaney K, et al. : Automated, adaptive guidance for K-12 education. Educ. Res. Rev. 2015; 15 : 41–58. Publisher Full Text 72. Xu Z, Wijekumar K, Ramirez G, et al. : The effectiveness of intelligent tutoring systems on K-12 students' reading comprehension: A meta-analysis. Br. J. Educ. Technol. 2019; 50 (6): 3119–3137. Publisher Full Text 73. Ma W, Adesope OO, Nesbit JC, et al. : Intelligent tutoring systems and learning outcomes: A meta-analysis. J. Educ. Psychol. 2014; 106 (4): 901–918. Publisher Full Text 74. Steenbergen-Hu S, Cooper H: A meta-analysis of the effectiveness of intelligent tutoring systems on K–12 students’ mathematical learning. J. Educ. Psychol. 2013; 105 (4): 970–987. Publisher Full Text 75. Melles M, Albayrak A, Goossens R: Innovating health care: key characteristics of human-centered design.Int. J. Qual. Health Care.2021; 33 (Supplement_1): 37–44. 76. Brest P, Roumani N, Bade J: Problem Solving, Human-Centered Design, and Strategic Processes.Standford PACS Center on Philanthropy and Civil Society; 2015. 77. Tolley E: Traditional Socio-Behavioral Research and Human Centered Design: Similarities, Unique Contributions, and Synergies. Report prepared for CAMI Health/IMPT Secretariat Washington, DC: Public Health Institute; 2017. 78. Glenton C, Carlsen B, Lewin S, et al. : Applying GRADE-CERQual to qualitative evidence synthesis findings—paper 5: how to assess adequacy of data. Implement. Sci. 2018; 13 (1):14. PubMed Abstract | Publisher Full Text | Free Full Text 79. Göttgens I, Oertelt-Prigione S: The application of human-centered design approaches in health research and innovation: a narrative review of current practices. JMIR Mhealth Uhealth. 2021; 9 (12): e28102. PubMed Abstract | Publisher Full Text | Free Full Text 80. Chesire F, Mugisha M, Ssenyonga R, et al. : Effects of the Informed Health Choices secondary school intervention: A prospective meta-analysis. J. Evid. Based Med. 2023; 16 (3): 321–331. PubMed Abstract | Publisher Full Text Comments on this article Comments (0) Version 3 VERSION 3 PUBLISHED 11 May 2023 ADD YOUR COMMENT Comment Author details Author details 1 Centre for Epidemic Interventions Research, Norwegian Institute of Public Health, Oslo, 0213, Norway 2 Tropical Institute of Community Health and Development in Africa, Kisumu, Kenya 3 Institute of Health and Society, Faculty of Medicine, Universitetet i Oslo, Oslo, Oslo, Norway 4 School of Public Health, College of Medicine and Health Sciences, University of Rwanda, Butare, Southern Province, Rwanda 5 School of Medicine, College of Health Sciences, Makerere University, Kampala, Central Region, Uganda 6 Epistemonikos Foundation, Santiago, Santiago Metropolitan Region, Chile 7 National Curriculum Development Centre, Kampala, Uganda 8 Ministry of Education and Sports, Kampala, Uganda 9 Faculty of Health Sciences, Uganda Martyrs University, Kampala, Central Region, Uganda 10 Ministry of Health, Kampala, Uganda 11 Uganda National Examinations Board, Kampala, Uganda 12 Huma Girls Secondary School, Kisumu, Kenya 13 Kenya Institute of Curriculum Development, Nairobi, Kenya 14 Rwanda Basic Education Board, Kigali, Rwanda 15 Health Systems Research Unit, South African Medical Research Council, Cape Town, South Africa 16 Department of Health Sciences, Norwegian University of Science and Technology (NTNU), Ålesund, Norway 17 Faculty of Health Sciences, Oslo Metropolitan University, Oslo, Norway Sarah Rosenbaum Roles: Conceptualization, Data Curation, Formal Analysis, Funding Acquisition, Investigation, Methodology, Supervision, Visualization, Writing – Original Draft Preparation, Writing – Review & Editing Jenny Moberg Roles: Data Curation, Formal Analysis, Investigation, Writing – Original Draft Preparation, Writing – Review & Editing Faith Chesire Roles: Data Curation, Formal Analysis, Investigation, Project Administration, Writing – Review & Editing Michael Mugisha Roles: Data Curation, Formal Analysis, Investigation, Project Administration, Writing – Review & Editing Ronald Ssenyonga Roles: Data Curation, Formal Analysis, Investigation, Project Administration, Writing – Review & Editing Marlyn A Ochieng Roles: Data Curation, Formal Analysis, Investigation, Writing – Review & Editing Clarisse Marie Claudine Simbi Roles: Data Curation, Formal Analysis, Investigation, Writing – Review & Editing Esther Nakyejwe Roles: Data Curation, Investigation, Writing – Review & Editing Benson Ngatia Roles: Data Curation, Investigation, Writing – Review & Editing Gabriel Rada Roles: Software, Supervision, Writing – Review & Editing Juan Vásquez-Laval Roles: Project Administration, Software, Writing – Review & Editing José Damián Garrido Roles: Software, Writing – Review & Editing Grace Baguma Roles: Investigation, Writing – Review & Editing Sam Kuloba Roles: Investigation, Writing – Review & Editing Edward Sebukyu Roles: Investigation, Writing – Review & Editing Richard Kabanda Roles: Investigation, Writing – Review & Editing Irene Mwenyango Roles: Investigation, Writing – Review & Editing Tonny Muzaale Roles: Investigation, Writing – Review & Editing Pamela Nandi Roles: Investigation, Writing – Review & Editing Jane Njue Roles: Investigation, Writing – Review & Editing Cyril Oyuga Roles: Investigation, Writing – Review & Editing Florian Rutiyomba Roles: Investigation, Writing – Review & Editing Felecien Rugengamanzi Roles: Investigation, Writing – Review & Editing Joan Murungi Roles: Investigation, Writing – Review & Editing Allen Nsangi Roles: Conceptualization, Project Administration, Supervision, Writing – Review & Editing Daniel Semakula Roles: Conceptualization, Project Administration, Supervision, Writing – Review & Editing Margaret Kaseje Roles: Conceptualization, Funding Acquisition, Project Administration, Supervision, Writing – Review & Editing Nelson Sewankambo Roles: Conceptualization, Funding Acquisition, Project Administration, Supervision, Writing – Review & Editing Laetitia Nyirazinyoye Roles: Conceptualization, Funding Acquisition, Project Administration, Supervision, Writing – Review & Editing Simon Lewin Roles: Supervision, Writing – Review & Editing Andrew D Oxman Roles: Conceptualization, Formal Analysis, Funding Acquisition, Project Administration, Supervision, Writing – Review & Editing Matt Oxman Roles: Conceptualization, Data Curation, Formal Analysis, Writing – Original Draft Preparation, Writing – Review & Editing Competing interests No competing interests were disclosed. Grant information Grant information: This work was supported by the Research Council of Norway Project number 284683, grant no: 69006, awarded to ADO. The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript. Article Versions (3) version 3 Revised Published: 04 Sep 2024, 12:481 https://doi.org/10.12688/f1000research.132580.3 version 2 Revised Published: 11 Mar 2024, 12:481 https://doi.org/10.12688/f1000research.132580.2 version 1 Published: 11 May 2023, 12:481 https://doi.org/10.12688/f1000research.132580.1 Copyright © 2024 Rosenbaum S et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. Download Export To Sciwheel Bibtex EndNote ProCite Ref. Manager (RIS) Sente metrics Views Downloads F1000Research - - PubMed Central info_outline Data from PMC are received and updated monthly. - - Citations open_in_new 0 open_in_new 0 open_in_new SEE MORE DETAILS CITE how to cite this article Rosenbaum S, Moberg J, Chesire F et al. Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources [version 3; peer review: 1 approved, 2 approved with reservations] . F1000Research 2024, 12 :481 ( https://doi.org/10.12688/f1000research.132580.3 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS track receive updates on this article Track an article to receive email alerts on any updates to this article. TRACK THIS ARTICLE Share Open Peer Review Current Reviewer Status: ? Key to Reviewer Statuses VIEW HIDE Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Version 2 VERSION 2 PUBLISHED 11 Mar 2024 Revised Views 0 Cite How to cite this report: Baur C. Reviewer Report For: Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources [version 3; peer review: 1 approved, 2 approved with reservations] . F1000Research 2024, 12 :481 ( https://doi.org/10.5256/f1000research.163450.r250510 ) The direct URL for this report is: https://f1000research.com/articles/12-481/v2#referee-response-250510 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 14 May 2024 Cynthia Baur , University of Maryland at College Park, College Park, Maryland, USA Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.163450.r250510 The authors provide a detailed description of a multi-phase and multi-step process to develop and test learning modules about health information and health choices for children aged 13-16 in Kenya, Rwanda, and Uganda. The project was developed and tested in ... Continue reading READ ALL The authors provide a detailed description of a multi-phase and multi-step process to develop and test learning modules about health information and health choices for children aged 13-16 in Kenya, Rwanda, and Uganda. The project was developed and tested in partnership with many stakeholders in the three countries. The Introduction opens with a brief discussion of the many challenges of identifying accurate information but ends with a purpose statement that the article will describe the development of the learning modules. The rest of the paper is about the module development process with Methods and Results being about the modules and learning techniques. Although the paper includes a discussion of teaching strategies in the Results section, what is missing is a theory of learning and expected results that would help explain why the team chose the human-centered design approach that it did, along with some literature justifying this choice. The paper provides a lot of detail about how a human-centered design approach collects feedback and iterates prototypes, but I am unclear why a human-centered design approach was necessary to produce this set of modules. I am also unclear about the lessons the authors want to impart to readers. Should we always use a human-centered design approach for health curriculum for children of these ages? Under what conditions or purposes might a different approach be better? Instead of naming the products developed as the primary results, I expected to read about how well or not a human-centered approach had worked to produce these products. I also lost the thread of what happened to all the interview, group discussion and stakeholder data. The authors provide a rich description of their development process, but I'm not sure what they intend for readers to take away from this report of their process and products. Is the work clearly and accurately presented and does it cite the current literature? Partly Is the study design appropriate and is the work technically sound? Yes Are sufficient details of methods and analysis provided to allow replication by others? Partly If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly Competing Interests: No competing interests were disclosed. Reviewer Expertise: Health literacy, health communication, risk communication I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Baur C. Reviewer Report For: Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources [version 3; peer review: 1 approved, 2 approved with reservations] . F1000Research 2024, 12 :481 ( https://doi.org/10.5256/f1000research.163450.r250510 ) The direct URL for this report is: https://f1000research.com/articles/12-481/v2#referee-response-250510 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 04 Sep 2024 Sarah Rosenbaum , Centre for Epidemic Interventions Research, Norwegian Institute of Public Health, Oslo, 0213, Norway 04 Sep 2024 Author Response Peer review 2 and our corresponding responses: 1) PEER REVIEW: The Introduction opens with a brief discussion of the many challenges of identifying accurate information but ends with a ... Continue reading Peer review 2 and our corresponding responses: 1) PEER REVIEW: The Introduction opens with a brief discussion of the many challenges of identifying accurate information but ends with a purpose statement that the article will describe the development of the learning modules. Although the paper includes a discussion of teaching strategies in the Results section, what is missing is a theory of learning and expected results that would help explain why the team chose the human-centered design approach that it did, along with some literature justifying this choice. The paper provides a lot of detail about how a human-centered design approach collects feedback and iterates prototypes, but I am unclear why a human-centered design approach was necessary to produce this set of modules. RESPONSE: I agree that addressing this would improve the manuscript, and have added three new sections to the Discussion: “Reflections about learning theory”, “Why a human-centred design approach”, and “Strengths and limitations of the human-centred design approach”, as well as a number of new references. 2) PEER REVIEW: I am also unclear about the lessons the authors want to impart to readers. The authors provide a rich description of their development process, but I'm not sure what they intend for readers to take away from this report of their process and products. RESPONSE: See response #4 below about results. 3) PEER REVIEW: Should we always use a human-centered design approach for health curriculum for children of these ages? Under what conditions or purposes might a different approach be better? RESPONSE: Educating young people in low-resource school settings to think critically about health choices is a complex problem. Developing instructional material for simpler and more well-defined learning tasks might be approached by less resource-intensive methods. I have added a text elaborating on this issue under Discussion: “Why a human-centred design approach”. 4) PEER REVIEW: Instead of naming the products developed as the primary results, I expected to read about how well or not a human-centered approach had worked to produce these products. RESPONSE: “Results” are not only what we found when collecting and analysing data, but also the final solution we created and the noteworthy design decisions we made along the way. These three types of findings can be of interest to a range of readers: people who are designing educational resources based on IHC Key Concepts, people who are designing related educational resources, such as for teaching critical thinking or for use in low-resource learning environments, or people interested in the design approach more generally within the context of a research inquiry. However, I agree that we could have also included more explicit findings related to the human-centred design approach. Bazzano et al (2020) proposed a reporting guideline when reporting health research involving design that suggests reporting the following from design activities in the results section: What was designed Summary of major insights or reflections from the design process Description of decisions made during the design process The guidance also suggests reflecting on the application of design to the work, including strengths and limitations. I have added a completed guidance checklist to the Extended data set, which indicates where in the manuscript each item can be found. Additionally, I am making the following changes to the manuscript: "Results" section changes: Made changes to the 1st paragraph in Results section to more clearly indicate the type of results we are reporting Added heading and text: “Other noteworthy design decisions” Made changes to some subheadings in the Results sections Made changes to some column headings in Tables 9-13, to clarify where results about "design decisions" could be found "Discussion" section changes: Added “Strengths and limitations of the human-centred design approach for this work” "Conclusion" section changes Adjusted text to differentiate more clearly between the three types of results 5) PEER REVIEW: I also lost the thread of what happened to all the interview, group discussion and stakeholder data. RESPONSE: Our data is published on Zenodo. See list of data “Underlying data”, reference 47. However, I acknowledge that the documentation of a project of this nature can be difficult to follow. Therefore I’ve added content to the Discussion addressing this under the heading: “Strengths and limitations of the human-centred design approach” Peer review 2 and our corresponding responses: 1) PEER REVIEW: The Introduction opens with a brief discussion of the many challenges of identifying accurate information but ends with a purpose statement that the article will describe the development of the learning modules. Although the paper includes a discussion of teaching strategies in the Results section, what is missing is a theory of learning and expected results that would help explain why the team chose the human-centered design approach that it did, along with some literature justifying this choice. The paper provides a lot of detail about how a human-centered design approach collects feedback and iterates prototypes, but I am unclear why a human-centered design approach was necessary to produce this set of modules. RESPONSE: I agree that addressing this would improve the manuscript, and have added three new sections to the Discussion: “Reflections about learning theory”, “Why a human-centred design approach”, and “Strengths and limitations of the human-centred design approach”, as well as a number of new references. 2) PEER REVIEW: I am also unclear about the lessons the authors want to impart to readers. The authors provide a rich description of their development process, but I'm not sure what they intend for readers to take away from this report of their process and products. RESPONSE: See response #4 below about results. 3) PEER REVIEW: Should we always use a human-centered design approach for health curriculum for children of these ages? Under what conditions or purposes might a different approach be better? RESPONSE: Educating young people in low-resource school settings to think critically about health choices is a complex problem. Developing instructional material for simpler and more well-defined learning tasks might be approached by less resource-intensive methods. I have added a text elaborating on this issue under Discussion: “Why a human-centred design approach”. 4) PEER REVIEW: Instead of naming the products developed as the primary results, I expected to read about how well or not a human-centered approach had worked to produce these products. RESPONSE: “Results” are not only what we found when collecting and analysing data, but also the final solution we created and the noteworthy design decisions we made along the way. These three types of findings can be of interest to a range of readers: people who are designing educational resources based on IHC Key Concepts, people who are designing related educational resources, such as for teaching critical thinking or for use in low-resource learning environments, or people interested in the design approach more generally within the context of a research inquiry. However, I agree that we could have also included more explicit findings related to the human-centred design approach. Bazzano et al (2020) proposed a reporting guideline when reporting health research involving design that suggests reporting the following from design activities in the results section: What was designed Summary of major insights or reflections from the design process Description of decisions made during the design process The guidance also suggests reflecting on the application of design to the work, including strengths and limitations. I have added a completed guidance checklist to the Extended data set, which indicates where in the manuscript each item can be found. Additionally, I am making the following changes to the manuscript: "Results" section changes: Made changes to the 1st paragraph in Results section to more clearly indicate the type of results we are reporting Added heading and text: “Other noteworthy design decisions” Made changes to some subheadings in the Results sections Made changes to some column headings in Tables 9-13, to clarify where results about "design decisions" could be found "Discussion" section changes: Added “Strengths and limitations of the human-centred design approach for this work” "Conclusion" section changes Adjusted text to differentiate more clearly between the three types of results 5) PEER REVIEW: I also lost the thread of what happened to all the interview, group discussion and stakeholder data. RESPONSE: Our data is published on Zenodo. See list of data “Underlying data”, reference 47. However, I acknowledge that the documentation of a project of this nature can be difficult to follow. Therefore I’ve added content to the Discussion addressing this under the heading: “Strengths and limitations of the human-centred design approach” Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 04 Sep 2024 Sarah Rosenbaum , Centre for Epidemic Interventions Research, Norwegian Institute of Public Health, Oslo, 0213, Norway 04 Sep 2024 Author Response Peer review 2 and our corresponding responses: 1) PEER REVIEW: The Introduction opens with a brief discussion of the many challenges of identifying accurate information but ends with a ... Continue reading Peer review 2 and our corresponding responses: 1) PEER REVIEW: The Introduction opens with a brief discussion of the many challenges of identifying accurate information but ends with a purpose statement that the article will describe the development of the learning modules. Although the paper includes a discussion of teaching strategies in the Results section, what is missing is a theory of learning and expected results that would help explain why the team chose the human-centered design approach that it did, along with some literature justifying this choice. The paper provides a lot of detail about how a human-centered design approach collects feedback and iterates prototypes, but I am unclear why a human-centered design approach was necessary to produce this set of modules. RESPONSE: I agree that addressing this would improve the manuscript, and have added three new sections to the Discussion: “Reflections about learning theory”, “Why a human-centred design approach”, and “Strengths and limitations of the human-centred design approach”, as well as a number of new references. 2) PEER REVIEW: I am also unclear about the lessons the authors want to impart to readers. The authors provide a rich description of their development process, but I'm not sure what they intend for readers to take away from this report of their process and products. RESPONSE: See response #4 below about results. 3) PEER REVIEW: Should we always use a human-centered design approach for health curriculum for children of these ages? Under what conditions or purposes might a different approach be better? RESPONSE: Educating young people in low-resource school settings to think critically about health choices is a complex problem. Developing instructional material for simpler and more well-defined learning tasks might be approached by less resource-intensive methods. I have added a text elaborating on this issue under Discussion: “Why a human-centred design approach”. 4) PEER REVIEW: Instead of naming the products developed as the primary results, I expected to read about how well or not a human-centered approach had worked to produce these products. RESPONSE: “Results” are not only what we found when collecting and analysing data, but also the final solution we created and the noteworthy design decisions we made along the way. These three types of findings can be of interest to a range of readers: people who are designing educational resources based on IHC Key Concepts, people who are designing related educational resources, such as for teaching critical thinking or for use in low-resource learning environments, or people interested in the design approach more generally within the context of a research inquiry. However, I agree that we could have also included more explicit findings related to the human-centred design approach. Bazzano et al (2020) proposed a reporting guideline when reporting health research involving design that suggests reporting the following from design activities in the results section: What was designed Summary of major insights or reflections from the design process Description of decisions made during the design process The guidance also suggests reflecting on the application of design to the work, including strengths and limitations. I have added a completed guidance checklist to the Extended data set, which indicates where in the manuscript each item can be found. Additionally, I am making the following changes to the manuscript: "Results" section changes: Made changes to the 1st paragraph in Results section to more clearly indicate the type of results we are reporting Added heading and text: “Other noteworthy design decisions” Made changes to some subheadings in the Results sections Made changes to some column headings in Tables 9-13, to clarify where results about "design decisions" could be found "Discussion" section changes: Added “Strengths and limitations of the human-centred design approach for this work” "Conclusion" section changes Adjusted text to differentiate more clearly between the three types of results 5) PEER REVIEW: I also lost the thread of what happened to all the interview, group discussion and stakeholder data. RESPONSE: Our data is published on Zenodo. See list of data “Underlying data”, reference 47. However, I acknowledge that the documentation of a project of this nature can be difficult to follow. Therefore I’ve added content to the Discussion addressing this under the heading: “Strengths and limitations of the human-centred design approach” Peer review 2 and our corresponding responses: 1) PEER REVIEW: The Introduction opens with a brief discussion of the many challenges of identifying accurate information but ends with a purpose statement that the article will describe the development of the learning modules. Although the paper includes a discussion of teaching strategies in the Results section, what is missing is a theory of learning and expected results that would help explain why the team chose the human-centered design approach that it did, along with some literature justifying this choice. The paper provides a lot of detail about how a human-centered design approach collects feedback and iterates prototypes, but I am unclear why a human-centered design approach was necessary to produce this set of modules. RESPONSE: I agree that addressing this would improve the manuscript, and have added three new sections to the Discussion: “Reflections about learning theory”, “Why a human-centred design approach”, and “Strengths and limitations of the human-centred design approach”, as well as a number of new references. 2) PEER REVIEW: I am also unclear about the lessons the authors want to impart to readers. The authors provide a rich description of their development process, but I'm not sure what they intend for readers to take away from this report of their process and products. RESPONSE: See response #4 below about results. 3) PEER REVIEW: Should we always use a human-centered design approach for health curriculum for children of these ages? Under what conditions or purposes might a different approach be better? RESPONSE: Educating young people in low-resource school settings to think critically about health choices is a complex problem. Developing instructional material for simpler and more well-defined learning tasks might be approached by less resource-intensive methods. I have added a text elaborating on this issue under Discussion: “Why a human-centred design approach”. 4) PEER REVIEW: Instead of naming the products developed as the primary results, I expected to read about how well or not a human-centered approach had worked to produce these products. RESPONSE: “Results” are not only what we found when collecting and analysing data, but also the final solution we created and the noteworthy design decisions we made along the way. These three types of findings can be of interest to a range of readers: people who are designing educational resources based on IHC Key Concepts, people who are designing related educational resources, such as for teaching critical thinking or for use in low-resource learning environments, or people interested in the design approach more generally within the context of a research inquiry. However, I agree that we could have also included more explicit findings related to the human-centred design approach. Bazzano et al (2020) proposed a reporting guideline when reporting health research involving design that suggests reporting the following from design activities in the results section: What was designed Summary of major insights or reflections from the design process Description of decisions made during the design process The guidance also suggests reflecting on the application of design to the work, including strengths and limitations. I have added a completed guidance checklist to the Extended data set, which indicates where in the manuscript each item can be found. Additionally, I am making the following changes to the manuscript: "Results" section changes: Made changes to the 1st paragraph in Results section to more clearly indicate the type of results we are reporting Added heading and text: “Other noteworthy design decisions” Made changes to some subheadings in the Results sections Made changes to some column headings in Tables 9-13, to clarify where results about "design decisions" could be found "Discussion" section changes: Added “Strengths and limitations of the human-centred design approach for this work” "Conclusion" section changes Adjusted text to differentiate more clearly between the three types of results 5) PEER REVIEW: I also lost the thread of what happened to all the interview, group discussion and stakeholder data. RESPONSE: Our data is published on Zenodo. See list of data “Underlying data”, reference 47. However, I acknowledge that the documentation of a project of this nature can be difficult to follow. Therefore I’ve added content to the Discussion addressing this under the heading: “Strengths and limitations of the human-centred design approach” Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Version 1 VERSION 1 PUBLISHED 11 May 2023 Views 0 Cite How to cite this report: Mathews C. Reviewer Report For: Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources [version 3; peer review: 1 approved, 2 approved with reservations] . F1000Research 2024, 12 :481 ( https://doi.org/10.5256/f1000research.145511.r250515 ) The direct URL for this report is: https://f1000research.com/articles/12-481/v1#referee-response-250515 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 05 Sep 2024 Catherine Mathews , South African Medical Research Council, Cape Town, South Africa Approved VIEWS 0 https://doi.org/10.5256/f1000research.145511.r250515 This manuscript describes the processes undertaken to develop a set of digital learning resources to teach students in Kenyan, Rwandan and Ugandan schools to think critically about health and health care, and to develop skills to assess the reliability of ... Continue reading READ ALL This manuscript describes the processes undertaken to develop a set of digital learning resources to teach students in Kenyan, Rwandan and Ugandan schools to think critically about health and health care, and to develop skills to assess the reliability of claims about interventions to improve health. Interventions to teach critical thinking are seldom included in the curricula of schools in the countries in which this research was conducted, despite that critical thinking is recognized as a valuable skill for young people to acquire, which will help them make decisions through the life course. In the countries in which the research was conducted, and in many other countries, teachers lack skills and resources to enable them to teach critical thinking. Both the intervention and the measure selected to evaluate the effects of the intervention are based on a large body of sound research which is referenced in the manuscript. The manuscript provides valuable information about how to use a human-centered design approach to adapt a program that promotes critical thinking about health and to adapt measures to evaluate the program. Stakeholders contributed to the intervention development process is described in sufficient detail and is appropriate. The changes made to the interventions in response to stakeholders’ input are detailed. The problems identified during the design process, and the actions taken to address such problems indicate how feedback from stakeholders was used to inform the development cycles of the intervention. The description of the set of lessons is clear and valuable and together with information about the key concepts that were taught in a sample lesson helps the reader gain more information about the intervention. The manuscript is written clearly, and the research processes are described comprehensively. It is rare to read a manuscript in which the intervention development process is set out systematically and clearly. The manuscript is excellent, and I do not have any suggestions for improvement. Major points None Minor points None Is the work clearly and accurately presented and does it cite the current literature? Yes Is the study design appropriate and is the work technically sound? Yes Are sufficient details of methods and analysis provided to allow replication by others? Yes If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Competing Interests: No competing interests were disclosed. Reviewer Expertise: Sexual and reproductive health I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Mathews C. Reviewer Report For: Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources [version 3; peer review: 1 approved, 2 approved with reservations] . F1000Research 2024, 12 :481 ( https://doi.org/10.5256/f1000research.145511.r250515 ) The direct URL for this report is: https://f1000research.com/articles/12-481/v1#referee-response-250515 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Steckelberg A. Reviewer Report For: Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources [version 3; peer review: 1 approved, 2 approved with reservations] . F1000Research 2024, 12 :481 ( https://doi.org/10.5256/f1000research.145511.r173757 ) The direct URL for this report is: https://f1000research.com/articles/12-481/v1#referee-response-173757 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 01 Jun 2023 Anke Steckelberg , Interdisciplinary Center for Health Sciences, Institute of Health and Nursing Science, Martin-Luther-Universitat Halle-Wittenberg, Halle (Saale), Saxony-Anhalt, Germany Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.145511.r173757 The authors reported the development of digital resources to teach secondary school students applying a human-centred design. This project is part of a larger project comprising extensive exploration of stakeholders perspectives and contexts and also systematic reviews of teaching methods ... Continue reading READ ALL The authors reported the development of digital resources to teach secondary school students applying a human-centred design. This project is part of a larger project comprising extensive exploration of stakeholders perspectives and contexts and also systematic reviews of teaching methods in the beginning of the project to inform this present study which will then be evaluated in randomized controlled trials in three countries in Africa. In order to enhance readability and help readers to understand the complexity of the whole project, authors reported details that had already been published elsewhere. This study is highly relevant and it was great to have access to the website. The manuscript is well written. However, authors should rather refer to already available publications and instead should provide further information on the following aspects: The conducted “Individual interviews employing user testing methods” comprised three parts. Some teachers were interviewed right after a pilot lesson, so they had their experiences from the lesson fresh in mind. Please add information how many of them already taught a pilot lesson before the interview and whether these teachers also had access to the teachers guide. Piloting and observation: The authors reported that early pilots involved single lessons. Were all lessons piloted as single lessons? The authors translated several of the audio recordings from individual and group interviews. Please add information on the selection of the recordings and why it was done. The authors provide a teacher`s guide and also reported that lack of competences was a relevant issue. Please clarify whether participation in the workshop for teachers was optional or mandatory and whether teachers competences were assessed. Is the work clearly and accurately presented and does it cite the current literature? Yes Is the study design appropriate and is the work technically sound? Yes Are sufficient details of methods and analysis provided to allow replication by others? Partly If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Competing Interests: No competing interests were disclosed. Reviewer Expertise: Critical health literacy, evidence-based health information, informed shared decision making I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Steckelberg A. Reviewer Report For: Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources [version 3; peer review: 1 approved, 2 approved with reservations] . F1000Research 2024, 12 :481 ( https://doi.org/10.5256/f1000research.145511.r173757 ) The direct URL for this report is: https://f1000research.com/articles/12-481/v1#referee-response-173757 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 13 Apr 2024 Sarah Rosenbaum , Centre for Epidemic Interventions Research, Norwegian Institute of Public Health, Oslo, 0213, Norway 13 Apr 2024 Author Response Response to Peer reviewer #1 Thank you very much for your time reviewing this article and your constructive questions. I am answering them below and updating the manuscript at the ... Continue reading Response to Peer reviewer #1 Thank you very much for your time reviewing this article and your constructive questions. I am answering them below and updating the manuscript at the same time. Question 1: The conducted “Individual interviews employing user testing methods” comprised three parts. Some teachers were interviewed right after a pilot lesson, so they had their experiences from the lesson fresh in mind. Please add information how many of them already taught a pilot lesson before the interview and whether these teachers also had access to the teachers guide. Reply 1a & b: 1a: How many taught a pilot lesson before the interview: I’ve made changes to Table 6 so that this information is added there. I did this by rechecking with the PhD fellows about the participant data, and we found small errors in the original table, so these are also fixed. 1b: Whether teachers had access to the teachers guide: In the first development phase, we had not begun work on the teacher's guide, so no teachers had access to this in the early interviews or pilots. Regarding the second development phase, I’ve edited the text in the paragraph “Piloting and observation” so that the following information is provided: “In the final pilot, teachers taught all the lessons over a school term, with access to a complete prototype of the resources including the teacher’s guide.” Question 2: Piloting and observation: The authors reported that early pilots involved single lessons. Were all lessons piloted as single lessons? Reply 2: No. I have added the following clarifying text under “Piloting and observation”: “Initially we sought to try out different approaches to overarching questions that were not lesson specific (e.g. how much content, what kind of lesson structure, what kind of teaching strategies to build on, how to format the materials). As we made formatting and content decisions, we gradually developed a full set of lessons, not all of which were piloted individually in the the first development phase." Question 3: The authors translated several of the audio recordings from individual and group interviews. Please add information on the selection of the recordings and why it was done. Reply 3: I changed the text in the first paragraph under “Data analysis” to reflect more accurately what was done in each country: “In Uganda and Kenya, all interviews were carried out in English and transcribed. In Rwanda, interviews were carried out in Kinyarwanda and transcribed in English. There, the PhD fellow and research assistant relied on notes they made during interview sessions, and selected some recordings to review when further clarification of their notes was needed.” Question 4: The authors provide a teacher`s guide and also reported that lack of competences was a relevant issue. Please clarify whether participation in the workshop for teachers was optional or mandatory and whether teachers competences were assessed. Reply: The workshop was a part of the intervention that was tested in three randomised trials evaluating the effect of these resources, and reported in those studies. Participation in the workshop was not mandatory. Teachers’ competences were not assessed at the end of the workshop, but they were assessed in the trials at the end of all the lessons. You can find the trial results for both students and teachers in Table 3 of this article: Effects of the Informed Health Choices secondary school intervention: A prospective meta-analysis: https://onlinelibrary.wiley.com/doi/10.1111/jebm.12552 I have added the following short summary of the results of this prospective meta-analysis in the Conclusions section, along with the corresponding citation reference: “A Prospective meta-analysis synthesizing the results of the three randomised trials has shown that the resources led to a large improvement in the ability of students and teachers to think critically about health choices, though only 42% of students achieved a passing score[64]. Next research steps are well underway: process evaluations of the three trials and one-year follow up studies.” Other updates to reflect referenced studies that are now published: Reference #52 has been updated as this article has now been published. Reference #64 in the Conclusion is new, as the trial results are now published. Response to Peer reviewer #1 Thank you very much for your time reviewing this article and your constructive questions. I am answering them below and updating the manuscript at the same time. Question 1: The conducted “Individual interviews employing user testing methods” comprised three parts. Some teachers were interviewed right after a pilot lesson, so they had their experiences from the lesson fresh in mind. Please add information how many of them already taught a pilot lesson before the interview and whether these teachers also had access to the teachers guide. Reply 1a & b: 1a: How many taught a pilot lesson before the interview: I’ve made changes to Table 6 so that this information is added there. I did this by rechecking with the PhD fellows about the participant data, and we found small errors in the original table, so these are also fixed. 1b: Whether teachers had access to the teachers guide: In the first development phase, we had not begun work on the teacher's guide, so no teachers had access to this in the early interviews or pilots. Regarding the second development phase, I’ve edited the text in the paragraph “Piloting and observation” so that the following information is provided: “In the final pilot, teachers taught all the lessons over a school term, with access to a complete prototype of the resources including the teacher’s guide.” Question 2: Piloting and observation: The authors reported that early pilots involved single lessons. Were all lessons piloted as single lessons? Reply 2: No. I have added the following clarifying text under “Piloting and observation”: “Initially we sought to try out different approaches to overarching questions that were not lesson specific (e.g. how much content, what kind of lesson structure, what kind of teaching strategies to build on, how to format the materials). As we made formatting and content decisions, we gradually developed a full set of lessons, not all of which were piloted individually in the the first development phase." Question 3: The authors translated several of the audio recordings from individual and group interviews. Please add information on the selection of the recordings and why it was done. Reply 3: I changed the text in the first paragraph under “Data analysis” to reflect more accurately what was done in each country: “In Uganda and Kenya, all interviews were carried out in English and transcribed. In Rwanda, interviews were carried out in Kinyarwanda and transcribed in English. There, the PhD fellow and research assistant relied on notes they made during interview sessions, and selected some recordings to review when further clarification of their notes was needed.” Question 4: The authors provide a teacher`s guide and also reported that lack of competences was a relevant issue. Please clarify whether participation in the workshop for teachers was optional or mandatory and whether teachers competences were assessed. Reply: The workshop was a part of the intervention that was tested in three randomised trials evaluating the effect of these resources, and reported in those studies. Participation in the workshop was not mandatory. Teachers’ competences were not assessed at the end of the workshop, but they were assessed in the trials at the end of all the lessons. You can find the trial results for both students and teachers in Table 3 of this article: Effects of the Informed Health Choices secondary school intervention: A prospective meta-analysis: https://onlinelibrary.wiley.com/doi/10.1111/jebm.12552 I have added the following short summary of the results of this prospective meta-analysis in the Conclusions section, along with the corresponding citation reference: “A Prospective meta-analysis synthesizing the results of the three randomised trials has shown that the resources led to a large improvement in the ability of students and teachers to think critically about health choices, though only 42% of students achieved a passing score[64]. Next research steps are well underway: process evaluations of the three trials and one-year follow up studies.” Other updates to reflect referenced studies that are now published: Reference #52 has been updated as this article has now been published. Reference #64 in the Conclusion is new, as the trial results are now published. Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 13 Apr 2024 Sarah Rosenbaum , Centre for Epidemic Interventions Research, Norwegian Institute of Public Health, Oslo, 0213, Norway 13 Apr 2024 Author Response Response to Peer reviewer #1 Thank you very much for your time reviewing this article and your constructive questions. I am answering them below and updating the manuscript at the ... Continue reading Response to Peer reviewer #1 Thank you very much for your time reviewing this article and your constructive questions. I am answering them below and updating the manuscript at the same time. Question 1: The conducted “Individual interviews employing user testing methods” comprised three parts. Some teachers were interviewed right after a pilot lesson, so they had their experiences from the lesson fresh in mind. Please add information how many of them already taught a pilot lesson before the interview and whether these teachers also had access to the teachers guide. Reply 1a & b: 1a: How many taught a pilot lesson before the interview: I’ve made changes to Table 6 so that this information is added there. I did this by rechecking with the PhD fellows about the participant data, and we found small errors in the original table, so these are also fixed. 1b: Whether teachers had access to the teachers guide: In the first development phase, we had not begun work on the teacher's guide, so no teachers had access to this in the early interviews or pilots. Regarding the second development phase, I’ve edited the text in the paragraph “Piloting and observation” so that the following information is provided: “In the final pilot, teachers taught all the lessons over a school term, with access to a complete prototype of the resources including the teacher’s guide.” Question 2: Piloting and observation: The authors reported that early pilots involved single lessons. Were all lessons piloted as single lessons? Reply 2: No. I have added the following clarifying text under “Piloting and observation”: “Initially we sought to try out different approaches to overarching questions that were not lesson specific (e.g. how much content, what kind of lesson structure, what kind of teaching strategies to build on, how to format the materials). As we made formatting and content decisions, we gradually developed a full set of lessons, not all of which were piloted individually in the the first development phase." Question 3: The authors translated several of the audio recordings from individual and group interviews. Please add information on the selection of the recordings and why it was done. Reply 3: I changed the text in the first paragraph under “Data analysis” to reflect more accurately what was done in each country: “In Uganda and Kenya, all interviews were carried out in English and transcribed. In Rwanda, interviews were carried out in Kinyarwanda and transcribed in English. There, the PhD fellow and research assistant relied on notes they made during interview sessions, and selected some recordings to review when further clarification of their notes was needed.” Question 4: The authors provide a teacher`s guide and also reported that lack of competences was a relevant issue. Please clarify whether participation in the workshop for teachers was optional or mandatory and whether teachers competences were assessed. Reply: The workshop was a part of the intervention that was tested in three randomised trials evaluating the effect of these resources, and reported in those studies. Participation in the workshop was not mandatory. Teachers’ competences were not assessed at the end of the workshop, but they were assessed in the trials at the end of all the lessons. You can find the trial results for both students and teachers in Table 3 of this article: Effects of the Informed Health Choices secondary school intervention: A prospective meta-analysis: https://onlinelibrary.wiley.com/doi/10.1111/jebm.12552 I have added the following short summary of the results of this prospective meta-analysis in the Conclusions section, along with the corresponding citation reference: “A Prospective meta-analysis synthesizing the results of the three randomised trials has shown that the resources led to a large improvement in the ability of students and teachers to think critically about health choices, though only 42% of students achieved a passing score[64]. Next research steps are well underway: process evaluations of the three trials and one-year follow up studies.” Other updates to reflect referenced studies that are now published: Reference #52 has been updated as this article has now been published. Reference #64 in the Conclusion is new, as the trial results are now published. Response to Peer reviewer #1 Thank you very much for your time reviewing this article and your constructive questions. I am answering them below and updating the manuscript at the same time. Question 1: The conducted “Individual interviews employing user testing methods” comprised three parts. Some teachers were interviewed right after a pilot lesson, so they had their experiences from the lesson fresh in mind. Please add information how many of them already taught a pilot lesson before the interview and whether these teachers also had access to the teachers guide. Reply 1a & b: 1a: How many taught a pilot lesson before the interview: I’ve made changes to Table 6 so that this information is added there. I did this by rechecking with the PhD fellows about the participant data, and we found small errors in the original table, so these are also fixed. 1b: Whether teachers had access to the teachers guide: In the first development phase, we had not begun work on the teacher's guide, so no teachers had access to this in the early interviews or pilots. Regarding the second development phase, I’ve edited the text in the paragraph “Piloting and observation” so that the following information is provided: “In the final pilot, teachers taught all the lessons over a school term, with access to a complete prototype of the resources including the teacher’s guide.” Question 2: Piloting and observation: The authors reported that early pilots involved single lessons. Were all lessons piloted as single lessons? Reply 2: No. I have added the following clarifying text under “Piloting and observation”: “Initially we sought to try out different approaches to overarching questions that were not lesson specific (e.g. how much content, what kind of lesson structure, what kind of teaching strategies to build on, how to format the materials). As we made formatting and content decisions, we gradually developed a full set of lessons, not all of which were piloted individually in the the first development phase." Question 3: The authors translated several of the audio recordings from individual and group interviews. Please add information on the selection of the recordings and why it was done. Reply 3: I changed the text in the first paragraph under “Data analysis” to reflect more accurately what was done in each country: “In Uganda and Kenya, all interviews were carried out in English and transcribed. In Rwanda, interviews were carried out in Kinyarwanda and transcribed in English. There, the PhD fellow and research assistant relied on notes they made during interview sessions, and selected some recordings to review when further clarification of their notes was needed.” Question 4: The authors provide a teacher`s guide and also reported that lack of competences was a relevant issue. Please clarify whether participation in the workshop for teachers was optional or mandatory and whether teachers competences were assessed. Reply: The workshop was a part of the intervention that was tested in three randomised trials evaluating the effect of these resources, and reported in those studies. Participation in the workshop was not mandatory. Teachers’ competences were not assessed at the end of the workshop, but they were assessed in the trials at the end of all the lessons. You can find the trial results for both students and teachers in Table 3 of this article: Effects of the Informed Health Choices secondary school intervention: A prospective meta-analysis: https://onlinelibrary.wiley.com/doi/10.1111/jebm.12552 I have added the following short summary of the results of this prospective meta-analysis in the Conclusions section, along with the corresponding citation reference: “A Prospective meta-analysis synthesizing the results of the three randomised trials has shown that the resources led to a large improvement in the ability of students and teachers to think critically about health choices, though only 42% of students achieved a passing score[64]. Next research steps are well underway: process evaluations of the three trials and one-year follow up studies.” Other updates to reflect referenced studies that are now published: Reference #52 has been updated as this article has now been published. Reference #64 in the Conclusion is new, as the trial results are now published. Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Comments on this article Comments (0) Version 3 VERSION 3 PUBLISHED 11 May 2023 ADD YOUR COMMENT Comment keyboard_arrow_left keyboard_arrow_right Open Peer Review Reviewer Status info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Reviewer Reports Invited Reviewers 1 2 3 Version 3 (revision) 04 Sep 24 Version 2 (revision) 11 Mar 24 read Version 1 11 May 23 read read Anke Steckelberg , Martin-Luther-Universitat Halle-Wittenberg, Halle (Saale), Germany Cynthia Baur , University of Maryland at College Park, College Park, USA Catherine Mathews , South African Medical Research Council, Cape Town, South Africa Comments on this article All Comments (0) Add a comment Sign up for content alerts Sign Up You are now signed up to receive this alert Browse by related subjects keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Baur C. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 14 May 2024 | for Version 2 Cynthia Baur , University of Maryland at College Park, College Park, Maryland, USA 0 Views copyright © 2024 Baur C. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions The authors provide a detailed description of a multi-phase and multi-step process to develop and test learning modules about health information and health choices for children aged 13-16 in Kenya, Rwanda, and Uganda. The project was developed and tested in partnership with many stakeholders in the three countries. The Introduction opens with a brief discussion of the many challenges of identifying accurate information but ends with a purpose statement that the article will describe the development of the learning modules. The rest of the paper is about the module development process with Methods and Results being about the modules and learning techniques. Although the paper includes a discussion of teaching strategies in the Results section, what is missing is a theory of learning and expected results that would help explain why the team chose the human-centered design approach that it did, along with some literature justifying this choice. The paper provides a lot of detail about how a human-centered design approach collects feedback and iterates prototypes, but I am unclear why a human-centered design approach was necessary to produce this set of modules. I am also unclear about the lessons the authors want to impart to readers. Should we always use a human-centered design approach for health curriculum for children of these ages? Under what conditions or purposes might a different approach be better? Instead of naming the products developed as the primary results, I expected to read about how well or not a human-centered approach had worked to produce these products. I also lost the thread of what happened to all the interview, group discussion and stakeholder data. The authors provide a rich description of their development process, but I'm not sure what they intend for readers to take away from this report of their process and products. Is the work clearly and accurately presented and does it cite the current literature? Partly Is the study design appropriate and is the work technically sound? Yes Are sufficient details of methods and analysis provided to allow replication by others? Partly If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly Competing Interests No competing interests were disclosed. Reviewer Expertise Health literacy, health communication, risk communication I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (1) Author Response 04 Sep 2024 Sarah Rosenbaum, Centre for Epidemic Interventions Research, Norwegian Institute of Public Health, Oslo, 0213, Norway Peer review 2 and our corresponding responses: 1) PEER REVIEW: The Introduction opens with a brief discussion of the many challenges of identifying accurate information but ends with a purpose statement that the article will describe the development of the learning modules. Although the paper includes a discussion of teaching strategies in the Results section, what is missing is a theory of learning and expected results that would help explain why the team chose the human-centered design approach that it did, along with some literature justifying this choice. The paper provides a lot of detail about how a human-centered design approach collects feedback and iterates prototypes, but I am unclear why a human-centered design approach was necessary to produce this set of modules. RESPONSE: I agree that addressing this would improve the manuscript, and have added three new sections to the Discussion: “Reflections about learning theory”, “Why a human-centred design approach”, and “Strengths and limitations of the human-centred design approach”, as well as a number of new references. 2) PEER REVIEW: I am also unclear about the lessons the authors want to impart to readers. The authors provide a rich description of their development process, but I'm not sure what they intend for readers to take away from this report of their process and products. RESPONSE: See response #4 below about results. 3) PEER REVIEW: Should we always use a human-centered design approach for health curriculum for children of these ages? Under what conditions or purposes might a different approach be better? RESPONSE: Educating young people in low-resource school settings to think critically about health choices is a complex problem. Developing instructional material for simpler and more well-defined learning tasks might be approached by less resource-intensive methods. I have added a text elaborating on this issue under Discussion: “Why a human-centred design approach”. 4) PEER REVIEW: Instead of naming the products developed as the primary results, I expected to read about how well or not a human-centered approach had worked to produce these products. RESPONSE: “Results” are not only what we found when collecting and analysing data, but also the final solution we created and the noteworthy design decisions we made along the way. These three types of findings can be of interest to a range of readers: people who are designing educational resources based on IHC Key Concepts, people who are designing related educational resources, such as for teaching critical thinking or for use in low-resource learning environments, or people interested in the design approach more generally within the context of a research inquiry. However, I agree that we could have also included more explicit findings related to the human-centred design approach. Bazzano et al (2020) proposed a reporting guideline when reporting health research involving design that suggests reporting the following from design activities in the results section: What was designed Summary of major insights or reflections from the design process Description of decisions made during the design process The guidance also suggests reflecting on the application of design to the work, including strengths and limitations. I have added a completed guidance checklist to the Extended data set, which indicates where in the manuscript each item can be found. Additionally, I am making the following changes to the manuscript: "Results" section changes: Made changes to the 1st paragraph in Results section to more clearly indicate the type of results we are reporting Added heading and text: “Other noteworthy design decisions” Made changes to some subheadings in the Results sections Made changes to some column headings in Tables 9-13, to clarify where results about "design decisions" could be found "Discussion" section changes: Added “Strengths and limitations of the human-centred design approach for this work” "Conclusion" section changes Adjusted text to differentiate more clearly between the three types of results 5) PEER REVIEW: I also lost the thread of what happened to all the interview, group discussion and stakeholder data. RESPONSE: Our data is published on Zenodo. See list of data “Underlying data”, reference 47. However, I acknowledge that the documentation of a project of this nature can be difficult to follow. Therefore I’ve added content to the Discussion addressing this under the heading: “Strengths and limitations of the human-centred design approach” View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Baur C. Peer Review Report For: Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources [version 3; peer review: 1 approved, 2 approved with reservations] . F1000Research 2024, 12 :481 ( https://doi.org/10.5256/f1000research.163450.r250510) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/12-481/v2#referee-response-250510 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Mathews C. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 05 Sep 2024 | for Version 1 Catherine Mathews , South African Medical Research Council, Cape Town, South Africa 0 Views copyright © 2024 Mathews C. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This manuscript describes the processes undertaken to develop a set of digital learning resources to teach students in Kenyan, Rwandan and Ugandan schools to think critically about health and health care, and to develop skills to assess the reliability of claims about interventions to improve health. Interventions to teach critical thinking are seldom included in the curricula of schools in the countries in which this research was conducted, despite that critical thinking is recognized as a valuable skill for young people to acquire, which will help them make decisions through the life course. In the countries in which the research was conducted, and in many other countries, teachers lack skills and resources to enable them to teach critical thinking. Both the intervention and the measure selected to evaluate the effects of the intervention are based on a large body of sound research which is referenced in the manuscript. The manuscript provides valuable information about how to use a human-centered design approach to adapt a program that promotes critical thinking about health and to adapt measures to evaluate the program. Stakeholders contributed to the intervention development process is described in sufficient detail and is appropriate. The changes made to the interventions in response to stakeholders’ input are detailed. The problems identified during the design process, and the actions taken to address such problems indicate how feedback from stakeholders was used to inform the development cycles of the intervention. The description of the set of lessons is clear and valuable and together with information about the key concepts that were taught in a sample lesson helps the reader gain more information about the intervention. The manuscript is written clearly, and the research processes are described comprehensively. It is rare to read a manuscript in which the intervention development process is set out systematically and clearly. The manuscript is excellent, and I do not have any suggestions for improvement. Major points None Minor points None Is the work clearly and accurately presented and does it cite the current literature? Yes Is the study design appropriate and is the work technically sound? Yes Are sufficient details of methods and analysis provided to allow replication by others? Yes If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Competing Interests No competing interests were disclosed. Reviewer Expertise Sexual and reproductive health I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) Mathews C. Peer Review Report For: Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources [version 3; peer review: 1 approved, 2 approved with reservations] . F1000Research 2024, 12 :481 ( https://doi.org/10.5256/f1000research.145511.r250515) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/12-481/v1#referee-response-250515 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2023 Steckelberg A. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 01 Jun 2023 | for Version 1 Anke Steckelberg , Interdisciplinary Center for Health Sciences, Institute of Health and Nursing Science, Martin-Luther-Universitat Halle-Wittenberg, Halle (Saale), Saxony-Anhalt, Germany 0 Views copyright © 2023 Steckelberg A. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions The authors reported the development of digital resources to teach secondary school students applying a human-centred design. This project is part of a larger project comprising extensive exploration of stakeholders perspectives and contexts and also systematic reviews of teaching methods in the beginning of the project to inform this present study which will then be evaluated in randomized controlled trials in three countries in Africa. In order to enhance readability and help readers to understand the complexity of the whole project, authors reported details that had already been published elsewhere. This study is highly relevant and it was great to have access to the website. The manuscript is well written. However, authors should rather refer to already available publications and instead should provide further information on the following aspects: The conducted “Individual interviews employing user testing methods” comprised three parts. Some teachers were interviewed right after a pilot lesson, so they had their experiences from the lesson fresh in mind. Please add information how many of them already taught a pilot lesson before the interview and whether these teachers also had access to the teachers guide. Piloting and observation: The authors reported that early pilots involved single lessons. Were all lessons piloted as single lessons? The authors translated several of the audio recordings from individual and group interviews. Please add information on the selection of the recordings and why it was done. The authors provide a teacher`s guide and also reported that lack of competences was a relevant issue. Please clarify whether participation in the workshop for teachers was optional or mandatory and whether teachers competences were assessed. Is the work clearly and accurately presented and does it cite the current literature? Yes Is the study design appropriate and is the work technically sound? Yes Are sufficient details of methods and analysis provided to allow replication by others? Partly If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Competing Interests No competing interests were disclosed. Reviewer Expertise Critical health literacy, evidence-based health information, informed shared decision making I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (1) Author Response 13 Apr 2024 Sarah Rosenbaum, Centre for Epidemic Interventions Research, Norwegian Institute of Public Health, Oslo, 0213, Norway Response to Peer reviewer #1 Thank you very much for your time reviewing this article and your constructive questions. I am answering them below and updating the manuscript at the same time. Question 1: The conducted “Individual interviews employing user testing methods” comprised three parts. Some teachers were interviewed right after a pilot lesson, so they had their experiences from the lesson fresh in mind. Please add information how many of them already taught a pilot lesson before the interview and whether these teachers also had access to the teachers guide. Reply 1a & b: 1a: How many taught a pilot lesson before the interview: I’ve made changes to Table 6 so that this information is added there. I did this by rechecking with the PhD fellows about the participant data, and we found small errors in the original table, so these are also fixed. 1b: Whether teachers had access to the teachers guide: In the first development phase, we had not begun work on the teacher's guide, so no teachers had access to this in the early interviews or pilots. Regarding the second development phase, I’ve edited the text in the paragraph “Piloting and observation” so that the following information is provided: “In the final pilot, teachers taught all the lessons over a school term, with access to a complete prototype of the resources including the teacher’s guide.” Question 2: Piloting and observation: The authors reported that early pilots involved single lessons. Were all lessons piloted as single lessons? Reply 2: No. I have added the following clarifying text under “Piloting and observation”: “Initially we sought to try out different approaches to overarching questions that were not lesson specific (e.g. how much content, what kind of lesson structure, what kind of teaching strategies to build on, how to format the materials). As we made formatting and content decisions, we gradually developed a full set of lessons, not all of which were piloted individually in the the first development phase." Question 3: The authors translated several of the audio recordings from individual and group interviews. Please add information on the selection of the recordings and why it was done. Reply 3: I changed the text in the first paragraph under “Data analysis” to reflect more accurately what was done in each country: “In Uganda and Kenya, all interviews were carried out in English and transcribed. In Rwanda, interviews were carried out in Kinyarwanda and transcribed in English. There, the PhD fellow and research assistant relied on notes they made during interview sessions, and selected some recordings to review when further clarification of their notes was needed.” Question 4: The authors provide a teacher`s guide and also reported that lack of competences was a relevant issue. Please clarify whether participation in the workshop for teachers was optional or mandatory and whether teachers competences were assessed. Reply: The workshop was a part of the intervention that was tested in three randomised trials evaluating the effect of these resources, and reported in those studies. Participation in the workshop was not mandatory. Teachers’ competences were not assessed at the end of the workshop, but they were assessed in the trials at the end of all the lessons. You can find the trial results for both students and teachers in Table 3 of this article: Effects of the Informed Health Choices secondary school intervention: A prospective meta-analysis: https://onlinelibrary.wiley.com/doi/10.1111/jebm.12552 I have added the following short summary of the results of this prospective meta-analysis in the Conclusions section, along with the corresponding citation reference: “A Prospective meta-analysis synthesizing the results of the three randomised trials has shown that the resources led to a large improvement in the ability of students and teachers to think critically about health choices, though only 42% of students achieved a passing score[64]. Next research steps are well underway: process evaluations of the three trials and one-year follow up studies.” Other updates to reflect referenced studies that are now published: Reference #52 has been updated as this article has now been published. Reference #64 in the Conclusion is new, as the trial results are now published. View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Steckelberg A. Peer Review Report For: Teaching critical thinking about health information and choices in secondary schools: human-centred design of digital resources [version 3; peer review: 1 approved, 2 approved with reservations] . F1000Research 2024, 12 :481 ( https://doi.org/10.5256/f1000research.145511.r173757) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/12-481/v1#referee-response-173757 Alongside their report, reviewers assign a status to the article: Approved - the paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations - A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved - fundamental flaws in the paper seriously undermine the findings and conclusions Adjust parameters to alter display View on desktop for interactive features Includes Interactive Elements View on desktop for interactive features Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Stay Updated Sign up for content alerts and receive a weekly or monthly email with all newly published articles Register with F1000Research Already registered? Sign in Not now, thanks close PLEASE NOTE If you are an AUTHOR of this article, please check that you signed in with the account associated with this article otherwise we cannot automatically identify your role as an author and your comment will be labelled as a “User Comment”. If you are a REVIEWER of this article, please check that you have signed in with the account associated with this article and then go to your account to submit your report, please do not post your review here. If you do not have access to your original account, please contact us . All commenters must hold a formal affiliation as per our Policies . The information that you give us will be displayed next to your comment. User comments must be in English, comprehensible and relevant to the article under discussion. We reserve the right to remove any comments that we consider to be inappropriate, offensive or otherwise in breach of the User Comment Terms and Conditions . Commenters must not use a comment for personal attacks. When criticisms of the article are based on unpublished data, the data should be made available. I accept the User Comment Terms and Conditions Please confirm that you accept the User Comment Terms and Conditions. Affiliation ✕ refresh Please enter your institution. Note: To add your institution or organisation, start typing the name and then select the correct name from the list. Where applicable, the name will appear in both the original language and in English. Do not paste in the name. If the name does not appear in the drop-down list, we will display the information you have entered. ✕ refresh Country/Region * USA UK Canada China France Germany Afghanistan Aland Islands Albania Algeria American Samoa Andorra Angola Anguilla Antarctica Antigua and Barbuda Argentina Armenia Aruba Australia Austria Azerbaijan Bahamas Bahrain Bangladesh Barbados Belarus Belgium Belize Benin Bermuda Bhutan Bolivia Bosnia and Herzegovina Botswana Bouvet Island Brazil British Indian Ocean Territory British Virgin Islands Brunei Bulgaria Burkina Faso Burundi Cambodia Cameroon Canada Cape Verde Cayman Islands Central African Republic Chad Chile China Christmas Island Cocos (Keeling) Islands Colombia Comoros Congo Cook Islands Costa Rica Cote d'Ivoire Croatia Cuba Cyprus Czech Republic Democratic Republic of the Congo Denmark Djibouti Dominica Dominican Republic Ecuador Egypt El Salvador Equatorial Guinea Eritrea Estonia Ethiopia Falkland Islands Faroe Islands Federated States of Micronesia Fiji Finland France French Guiana French Polynesia French Southern Territories Gabon Georgia Germany Ghana Gibraltar Greece Greenland Grenada Guadeloupe Guam Guatemala Guernsey Guinea Guinea-Bissau Guyana Haiti Heard Island and Mcdonald Islands Holy See (Vatican City State) Honduras Hong Kong Hungary Iceland India Indonesia Iran Iraq Ireland Israel Italy Jamaica Japan Jersey Jordan Kazakhstan Kenya Kiribati Kosovo (Serbia and Montenegro) Kuwait Kyrgyzstan Lao People's Democratic Republic Latvia Lebanon Lesotho Liberia Libya Liechtenstein Lithuania Luxembourg Macao Madagascar Malawi Malaysia Maldives Mali Malta Marshall Islands Martinique Mauritania Mauritius Mayotte Mexico Minor Outlying Islands of the United States Moldova Monaco Mongolia Montenegro Montserrat Morocco Mozambique Myanmar Namibia Nauru Nepal Netherlands Antilles New Caledonia New Zealand Nicaragua Niger Nigeria Niue Norfolk Island North Korea North Macedonia Northern Mariana Islands Norway Oman Pakistan Palau Palestinian Territory Panama Papua New Guinea Paraguay Peru Philippines Pitcairn Poland Portugal Puerto Rico Qatar Reunion Romania Russian Federation Rwanda Saint Helena Saint Kitts and Nevis Saint Lucia Saint Pierre and Miquelon Saint Vincent and the Grenadines Samoa San Marino Sao Tome and Principe Saudi Arabia Senegal Serbia Seychelles Sierra Leone Singapore Slovakia Slovenia Solomon Islands Somalia South Africa South Georgia and the South Sandwich Is South Korea South Sudan Spain Sri Lanka Sudan Suriname Svalbard and Jan Mayen Swaziland Sweden Switzerland Syria Taiwan Tajikistan Tanzania Thailand The Gambia The Netherlands Timor-Leste Togo Tokelau Tonga Trinidad and Tobago Tunisia Turkey Turkmenistan Turks and Caicos Islands Tuvalu UK USA Uganda Ukraine United Arab Emirates United States Virgin Islands Uruguay Uzbekistan Vanuatu Venezuela Vietnam Wallis and Futuna West Bank and Gaza Strip Western Sahara Yemen Zambia Zimbabwe Please select your country/region. You must enter a comment. Competing Interests Please disclose any competing interests that might be construed to influence your judgment of the article's or peer review report's validity or importance. Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Please state your competing interests The comment has been saved. An error has occurred. Please try again. Cancel Post var lTitle = "Teaching critical thinking about health information...".replace("'", ''); var linkedInUrl = "http://www.linkedin.com/shareArticle?url=https://f1000research.com/articles/12-481/v3" + "&title=" + encodeURIComponent(lTitle) + "&summary=" + encodeURIComponent('Read the article by '); var deliciousUrl = "https://del.icio.us/post?url=https://f1000research.com/articles/12-481/v3&title=" + encodeURIComponent(lTitle); var redditUrl = "http://reddit.com/submit?url=https://f1000research.com/articles/12-481/v3" + "&title=" + encodeURIComponent(lTitle); linkedInUrl += encodeURIComponent('Rosenbaum S et al.'); var offsetTop = /chrome/i.test( navigator.userAgent ) ? 4 : -10; var addthis_config = { ui_offset_top: offsetTop, services_compact : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_expanded : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_custom : [ { name: "LinkedIn", url: linkedInUrl, icon:"/img/icon/at_linkedin.svg" }, { name: "Mendeley", url: "http://www.mendeley.com/import/?url=https://f1000research.com/articles/12-481/v3/mendeley", icon:"/img/icon/at_mendeley.svg" }, { name: "Reddit", url: redditUrl, icon:"/img/icon/at_reddit.svg" }, ] }; var addthis_share = { url: "https://f1000research.com/articles/12-481", templates : { twitter : "Teaching critical thinking about health information and choices.... Rosenbaum S et al., published by " + "@F1000Research" + ", https://f1000research.com/articles/12-481/v3" } }; if (typeof(addthis) != "undefined"){ addthis.addEventListener('addthis.ready', checkCount); addthis.addEventListener('addthis.menu.share', checkCount); } $(".f1r-shares-twitter").attr("href", "https://twitter.com/intent/tweet?text=" + addthis_share.templates.twitter); $(".f1r-shares-facebook").attr("href", "https://www.facebook.com/sharer/sharer.php?u=" + addthis_share.url); $(".f1r-shares-linkedin").attr("href", addthis_config.services_custom[0].url); $(".f1r-shares-reddit").attr("href", addthis_config.services_custom[2].url); $(".f1r-shares-mendelay").attr("href", addthis_config.services_custom[1].url); function checkCount(){ setTimeout(function(){ $(".addthis_button_expanded").each(function(){ var count = $(this).text(); if (count !== "" && count != "0") $(this).removeClass("is-hidden"); else $(this).addClass("is-hidden"); }); }, 1000); } close How to cite this report {{reportCitation}} Cancel Copy Citation Details $(function(){R.ui.buttonDropdowns('.dropdown-for-downloads');}); $(function(){R.ui.toolbarDropdowns('.toolbar-dropdown-for-downloads');}); $.get("/articles/acj/132580/171072") new F1000.Clipboard(); new F1000.ThesaurusTermsDisplay("articles", "article", "171072"); $(document).ready(function() { $( "#frame1" ).on('load', function() { var mydiv = $(this).contents().find("div"); var h = mydiv.height(); console.log(h) }); var tooltipLivingFigure = jQuery(".interactive-living-figure-label .icon-more-info"), titleLivingFigure = tooltipLivingFigure.attr("title"); tooltipLivingFigure.simpletip({ fixed: true, position: ["-115", "30"], baseClass: 'small-tooltip', content:titleLivingFigure + " " }); tooltipLivingFigure.removeAttr("title"); $("body").on("click", ".cite-living-figure", function(e) { e.preventDefault(); var ref = $(this).attr("data-ref"); $(this).closest(".living-figure-list-container").find("#" + ref).fadeIn(200); }); $("body").on("click", ".close-cite-living-figure", function(e) { e.preventDefault(); $(this).closest(".popup-window-wrapper").fadeOut(200); }); $(document).on("mouseup", function(e) { var metricsContainer = $(".article-metrics-popover-wrapper"); if (!metricsContainer.is(e.target) && metricsContainer.has(e.target).length === 0) { $(".article-metrics-close-button").click(); } }); var articleId = $('#articleId').val(); if($("#main-article-count-box").attachArticleMetrics) { $("#main-article-count-box").attachArticleMetrics(articleId, { articleMetricsView: true }); } }); var figshareWidget = $(".new_figshare_widget"); if (figshareWidget.length > 0) { window.figshare.load("f1000", function(Widget) { // Select a tag/tags defined in your page. In this tag we will place the widget. _.map(figshareWidget, function(el){ var widget = new Widget({ articleId: $(el).attr("figshare_articleId") //height:300 // this is the height of the viewer part. [Default: 550] }); widget.initialize(); // initialize the widget widget.mount(el); // mount it in a tag that's on your page // this will save the widget on the global scope for later use from // your JS scripts. This line is optional. //window.widget = widget; }); }); } close Error Close Add Reset F1000.MICROSERVICES.AFFILIATION = ''; $(document).ready(function () { $('.js-affiliations-form').each((index, form) => { new AffiliationForm({ formId: form.id, institutionErrorSelector: '.comment-enter-institution', departmentErrorSelector: '.comment-enter-department', placeSelector: '.js-add-comment-place', stateSelector: '.js-add-comment-state', zipCodeSelector: '.js-add-comment-zipcode', countrySelector: '.js-add-comment-country', countryErrorSelector: '.comment-enter-country', }); }); }); $(document).ready(function () { var reportIds = { "361479": 0, "361478": 0, "359425": 0, "359424": 0, "359427": 0, "359426": 0, "361485": 0, "361484": 0, "361487": 0, "361486": 0, "361481": 0, "176142": 0, "361480": 0, "176143": 0, "361483": 0, "176140": 0, "361482": 0, "176141": 0, "176144": 0, "290327": 0, "176145": 0, "290328": 0, "290329": 0, "320039": 0, "320040": 0, "320808": 0, "320041": 0, "320042": 0, "320043": 0, "318772": 0, "254517": 0, "318771": 0, "318784": 0, "319052": 0, "363341": 0, "363340": 0, "363343": 0, "363342": 0, "363337": 0, "363336": 0, "363339": 0, "363338": 0, "363345": 0, "363344": 0, "279132": 0, "281183": 0, "281196": 0, "281195": 0, "187762": 0, "187763": 0, "187766": 0, "187767": 0, "187764": 0, "187765": 0, "187770": 0, "187771": 0, "187768": 0, "187769": 0, "223103": 0, "260739": 0, "223107": 0, "223106": 0, "223105": 0, "320391": 0, "223104": 0, "260743": 0, "223111": 0, "260742": 0, "223110": 0, "223109": 0, "223108": 0, "260747": 0, "250507": 0, "250506": 0, "260745": 0, "250511": 0, "250510": 22, "260749": 0, "278410": 0, "250509": 0, "250508": 0, "260755": 0, "278164": 0, "250515": 9, "250514": 0, "285078": 0, "250513": 0, "260752": 0, "285079": 0, "250512": 0, "260757": 0, "260756": 0, "285080": 0, "285081": 0, "285082": 0, "279723": 0, "173755": 0, "173758": 0, "193470": 0, "173759": 0, "193471": 0, "173756": 0, "193468": 0, "173757": 31, "193469": 0, "193474": 0, "193475": 0, "193472": 0, "193473": 0, "193476": 0, "193477": 0, "207059": 0, "207058": 0, "207063": 0, "207062": 0, "207061": 0, "207060": 0, "207065": 0, "320735": 0, "207064": 0, "181983": 0, "207069": 0, "181986": 0, "181987": 0, "181984": 0, "181985": 0, "320736": 0, "181990": 0, "181988": 0, "181989": 0, "357103": 0, "357097": 0, "357109": 0, "357108": 0, "357111": 0, "357110": 0, "357105": 0, "359408": 0, "357104": 0, "357107": 0, "357106": 0, "359421": 0, "359423": 0, "359422": 0, "359417": 0, "359419": 0, }; $(".referee-response-container,.js-referee-report").each(function(index, el) { var reportId = $(el).attr("data-reportid"), reportCount = reportIds[reportId] || 0; $(el).find(".comments-count-container,.js-referee-report-views").html(reportCount); }); var uuidInput = $("#article_uuid"), oldUUId = uuidInput.val(), newUUId = "1e6f5f82-6be1-4597-af02-408589089501"; uuidInput.val(newUUId); $("a[href*='article_uuid=']").each(function(index, el) { var newHref = $(el).attr("href").replace(oldUUId, newUUId); $(el).attr("href", newHref); }); }); An innovative open access publishing platform offering rapid publication and open peer review, whilst supporting data deposition and sharing. Browse Gateways Collections How it Works Contact For Developers Cookie Notice Privacy Notice RSS Submit Your Research Follow us © 2012-2026 F1000 Research Ltd. ISSN 2046-1402 | Legal | Partner of Research4Life • CrossRef • ORCID • FAIRSharing R.templateTests.simpleTemplate = R.template(' $text $text $text $text $text '); R.templateTests.runTests(); var F1000platform = new F1000.Platform({ name: "f1000research", displayName: "F1000Research", hostName: "f1000research.com", id: "1", editorialEmail: "[email protected]", infoEmail: "[email protected]", usePmcStats: true }); $(function(){R.ui.dropdowns('.dropdown-for-authors, .dropdown-for-about, .dropdown-for-myresearch');}); // $(function(){R.ui.dropdowns('.dropdown-for-referees');}); $(document).ready(function () { if ($(".cookie-warning").is(":visible")) { $(".sticky").css("margin-bottom", "35px"); $(".devices").addClass("devices-and-cookie-warning"); } $(".cookie-warning .close-button").click(function (e) { $(".devices").removeClass("devices-and-cookie-warning"); $(".sticky").css("margin-bottom", "0"); }); $("#tweeter-feed .tweet-message").each(function (i, message) { var self = $(message); self.html(linkify(self.html())); }); $(".partner").on("mouseenter mouseleave", function() { $(this).find(".gray-scale, .colour").toggleClass("is-hidden"); }); }); Sign In Remember me Forgotten your password? Sign In Cancel Email or password not correct. Please try again Please wait... $(function(){ // Note: All the setup needs to run against a name attribute and *not* the id due the clonish // nature of facebox... $("a[id=googleSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("GOOGLE"); $("form[id=oAuthForm]").submit(); }); $("a[id=facebookSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("FACEBOOK"); $("form[id=oAuthForm]").submit(); }); $("a[id=orcidSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("ORCID"); $("form[id=oAuthForm]").submit(); }); }); If you've forgotten your password, please enter your email address below and we'll send you instructions on how to reset your password. The email address should be the one you originally registered with F1000. Email address not valid, please try again You registered with F1000 via Google, so we cannot reset your password. To sign in, please click here . If you still need help with your Google account password, please click here . You registered with F1000 via Facebook, so we cannot reset your password. To sign in, please click here . If you still need help with your Facebook account password, please click here . Code not correct, please try again Reset password Cancel Email us for further assistance. Server error, please try again. If your email address is registered with us, we will email you instructions to reset your password. If you think you should have received this email but it has not arrived, please check your spam filters and/or contact for further assistance. Please wait... Register $(document).ready(function () { signIn.createSignInAsRow($("#sign-in-form-gfb-popup")); $(".target-field").each(function () { var uris = $(this).val().split("/"); if (uris.pop() === "login") { $(this).val(uris.toString().replace(",","/")); } }); });