Monitoring Open Science as transformative change: Towards a systemic framework

Monitoring Open Science as transformative change:... | F1000Research "use strict";function _typeof(t){return(_typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}!function(){var t=function(){var t,e,o=[],n=window,r=n;for(;r;){try{if(r.frames.__tcfapiLocator){t=r;break}}catch(t){}if(r===n.top)break;r=r.parent}t||(!function t(){var e=n.document,o=!!n.frames.__tcfapiLocator;if(!o)if(e.body){var r=e.createElement("iframe");r.style.cssText="display:none",r.name="__tcfapiLocator",e.body.appendChild(r)}else setTimeout(t,5);return!o}(),n.__tcfapi=function(){for(var t=arguments.length,n=new Array(t),r=0;r 3&&2===parseInt(n[1],10)&&"boolean"==typeof n[3]&&(e=n[3],"function"==typeof n[2]&&n[2]("set",!0)):"ping"===n[0]?"function"==typeof n[2]&&n[2]({gdprApplies:e,cmpLoaded:!1,cmpStatus:"stub"}):o.push(n)},n.addEventListener("message",(function(t){var e="string"==typeof t.data,o={};if(e)try{o=JSON.parse(t.data)}catch(t){}else o=t.data;var n="object"===_typeof(o)&&null!==o?o.__tcfapiCall:null;n&&window.__tcfapi(n.command,n.version,(function(o,r){var a={__tcfapiReturn:{returnValue:o,success:r,callId:n.callId}};t&&t.source&&t.source.postMessage&&t.source.postMessage(e?JSON.stringify(a):a,"*")}),n.parameter)}),!1))};"undefined"!=typeof module?module.exports=t:t()}(); dataLayer = dataLayer || []; // Standard GTM initialization - Google Consent Mode handles consent automatically (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl+ '>m_auth=hzk0Vc3qFsQYhCrIoHz68A>m_preview=env-1>m_cookies_win=x';f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-MWFK8L5J'); ;window.NREUM||(NREUM={});NREUM.init={distributed_tracing:{enabled:true},privacy:{cookies_enabled:true},ajax:{deny_list:["bam.nr-data.net"]}}; ;NREUM.loader_config={accountID:"438030",trustKey:"438030",agentID:"772317073",licenseKey:"97f8f67f26",applicationID:"772317073"} ;NREUM.info={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net",licenseKey:"97f8f67f26",applicationID:"772317073",sa:1} ;/*! For license information please see nr-loader-spa-1.236.0.min.js.LICENSE.txt */ (()=>{"use strict";var e,t,r={5763:(e,t,r)=>{r.d(t,{P_:()=>l,Mt:()=>g,C5:()=>s,DL:()=>v,OP:()=>T,lF:()=>D,Yu:()=>y,Dg:()=>h,CX:()=>c,GE:()=>b,sU:()=>_});var n=r(8632),i=r(9567);const o={beacon:n.ce.beacon,errorBeacon:n.ce.errorBeacon,licenseKey:void 0,applicationID:void 0,sa:void 0,queueTime:void 0,applicationTime:void 0,ttGuid:void 0,user:void 0,account:void 0,product:void 0,extra:void 0,jsAttributes:{},userAttributes:void 0,atts:void 0,transactionName:void 0,tNamePlain:void 0},a={};function s(e){if(!e)throw new Error("All info objects require an agent identifier!");if(!a[e])throw new Error("Info for ".concat(e," was never set"));return a[e]}function c(e,t){if(!e)throw new Error("All info objects require an agent identifier!");a[e]=(0,i.D)(t,o),(0,n.Qy)(e,a[e],"info")}var u=r(7056);const d=()=>{const e={blockSelector:"[data-nr-block]",maskInputOptions:{password:!0}};return{allow_bfcache:!0,privacy:{cookies_enabled:!0},ajax:{deny_list:void 0,enabled:!0,harvestTimeSeconds:10},distributed_tracing:{enabled:void 0,exclude_newrelic_header:void 0,cors_use_newrelic_header:void 0,cors_use_tracecontext_headers:void 0,allowed_origins:void 0},session:{domain:void 0,expiresMs:u.oD,inactiveMs:u.Hb},ssl:void 0,obfuscate:void 0,jserrors:{enabled:!0,harvestTimeSeconds:10},metrics:{enabled:!0},page_action:{enabled:!0,harvestTimeSeconds:30},page_view_event:{enabled:!0},page_view_timing:{enabled:!0,harvestTimeSeconds:30,long_task:!1},session_trace:{enabled:!0,harvestTimeSeconds:10},harvest:{tooManyRequestsDelay:60},session_replay:{enabled:!1,harvestTimeSeconds:60,sampleRate:.1,errorSampleRate:.1,maskTextSelector:"*",maskAllInputs:!0,get blockClass(){return"nr-block"},get ignoreClass(){return"nr-ignore"},get maskTextClass(){return"nr-mask"},get blockSelector(){return e.blockSelector},set blockSelector(t){e.blockSelector+=",".concat(t)},get maskInputOptions(){return e.maskInputOptions},set maskInputOptions(t){e.maskInputOptions={...t,password:!0}}},spa:{enabled:!0,harvestTimeSeconds:10}}},f={};function l(e){if(!e)throw new Error("All configuration objects require an agent identifier!");if(!f[e])throw new Error("Configuration for ".concat(e," was never set"));return f[e]}function h(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");f[e]=(0,i.D)(t,d()),(0,n.Qy)(e,f[e],"config")}function g(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");var r=l(e);if(r){for(var n=t.split("."),i=0;i {r.d(t,{D:()=>i});var n=r(50);function i(e,t){try{if(!e||"object"!=typeof e)return(0,n.Z)("Setting a Configurable requires an object as input");if(!t||"object"!=typeof t)return(0,n.Z)("Setting a Configurable requires a model to set its initial properties");const r=Object.create(Object.getPrototypeOf(t),Object.getOwnPropertyDescriptors(t)),o=0===Object.keys(r).length?e:r;for(let a in o)if(void 0!==e[a])try{"object"==typeof e[a]&&"object"==typeof t[a]?r[a]=i(e[a],t[a]):r[a]=e[a]}catch(e){(0,n.Z)("An error occurred while setting a property of a Configurable",e)}return r}catch(e){(0,n.Z)("An error occured while setting a Configurable",e)}}},6818:(e,t,r)=>{r.d(t,{Re:()=>i,gF:()=>o,q4:()=>n});const n="1.236.0",i="PROD",o="CDN"},385:(e,t,r)=>{r.d(t,{FN:()=>a,IF:()=>u,Nk:()=>f,Tt:()=>s,_A:()=>o,il:()=>n,pL:()=>c,v6:()=>i,w1:()=>d});const n="undefined"!=typeof window&&!!window.document,i="undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self.navigator instanceof WorkerNavigator||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis.navigator instanceof WorkerNavigator),o=n?window:"undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis),a=""+o?.location,s=/iPad|iPhone|iPod/.test(navigator.userAgent),c=s&&"undefined"==typeof SharedWorker,u=(()=>{const e=navigator.userAgent.match(/Firefox[/\s](\d+\.\d+)/);return Array.isArray(e)&&e.length>=2?+e[1]:0})(),d=Boolean(n&&window.document.documentMode),f=!!navigator.sendBeacon},1117:(e,t,r)=>{r.d(t,{w:()=>o});var n=r(50);const i={agentIdentifier:"",ee:void 0};class o{constructor(e){try{if("object"!=typeof e)return(0,n.Z)("shared context requires an object as input");this.sharedContext={},Object.assign(this.sharedContext,i),Object.entries(e).forEach((e=>{let[t,r]=e;Object.keys(i).includes(t)&&(this.sharedContext[t]=r)}))}catch(e){(0,n.Z)("An error occured while setting SharedContext",e)}}}},8e3:(e,t,r)=>{r.d(t,{L:()=>d,R:()=>c});var n=r(2177),i=r(1284),o=r(4322),a=r(3325);const s={};function c(e,t){const r={staged:!1,priority:a.p[t]||0};u(e),s[e].get(t)||s[e].set(t,r)}function u(e){e&&(s[e]||(s[e]=new Map))}function d(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"",t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"feature";if(u(e),!e||!s[e].get(t))return a(t);s[e].get(t).staged=!0;const r=[...s[e]];function a(t){const r=e?n.ee.get(e):n.ee,a=o.X.handlers;if(r.backlog&&a){var s=r.backlog[t],c=a[t];if(c){for(var u=0;s&&u {let[t,r]=e;return r.staged}))&&(r.sort(((e,t)=>e[1].priority-t[1].priority)),r.forEach((e=>{let[t]=e;a(t)})))}function f(e,t){var r=e[1];(0,i.D)(t[r],(function(t,r){var n=e[0];if(r[0]===n){var i=r[1],o=e[3],a=e[2];i.apply(o,a)}}))}},2177:(e,t,r)=>{r.d(t,{c:()=>f,ee:()=>u});var n=r(8632),i=r(2210),o=r(1284),a=r(5763),s="nr@context";let c=(0,n.fP)();var u;function d(){}function f(e){return(0,i.X)(e,s,l)}function l(){return new d}function h(){u.aborted=!0,u.backlog={}}c.ee?u=c.ee:(u=function e(t,r){var n={},c={},f={},g=!1;try{g=16===r.length&&(0,a.OP)(r).isolatedBacklog}catch(e){}var p={on:b,addEventListener:b,removeEventListener:y,emit:v,get:x,listeners:w,context:m,buffer:A,abort:h,aborted:!1,isBuffering:E,debugId:r,backlog:g?{}:t&&"object"==typeof t.backlog?t.backlog:{}};return p;function m(e){return e&&e instanceof d?e:e?(0,i.X)(e,s,l):l()}function v(e,r,n,i,o){if(!1!==o&&(o=!0),!u.aborted||i){t&&o&&t.emit(e,r,n);for(var a=m(n),s=w(e),d=s.length,f=0;fn,p:()=>i});var n=r(2177).ee.get("handle");function i(e,t,r,i,o){o?(o.buffer([e],i),o.emit(e,t,r)):(n.buffer([e],i),n.emit(e,t,r))}},4322:(e,t,r)=>{r.d(t,{X:()=>o});var n=r(5546);o.on=a;var i=o.handlers={};function o(e,t,r,o){a(o||n.E,i,e,t,r)}function a(e,t,r,i,o){o||(o="feature"),e||(e=n.E);var a=t[o]=t[o]||{};(a[r]=a[r]||[]).push([e,i])}},3239:(e,t,r)=>{r.d(t,{bP:()=>s,iz:()=>c,m$:()=>a});var n=r(385);let i=!1,o=!1;try{const e={get passive(){return i=!0,!1},get signal(){return o=!0,!1}};n._A.addEventListener("test",null,e),n._A.removeEventListener("test",null,e)}catch(e){}function a(e,t){return i||o?{capture:!!e,passive:i,signal:t}:!!e}function s(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;window.addEventListener(e,t,a(r,n))}function c(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;document.addEventListener(e,t,a(r,n))}},4402:(e,t,r)=>{r.d(t,{Ht:()=>u,M:()=>c,Rl:()=>a,ky:()=>s});var n=r(385);const i="xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx";function o(e,t){return e?15&e[t]:16*Math.random()|0}function a(){const e=n._A?.crypto||n._A?.msCrypto;let t,r=0;return e&&e.getRandomValues&&(t=e.getRandomValues(new Uint8Array(31))),i.split("").map((e=>"x"===e?o(t,++r).toString(16):"y"===e?(3&o()|8).toString(16):e)).join("")}function s(e){const t=n._A?.crypto||n._A?.msCrypto;let r,i=0;t&&t.getRandomValues&&(r=t.getRandomValues(new Uint8Array(31)));const a=[];for(var s=0;s {r.d(t,{Bq:()=>n,Hb:()=>o,oD:()=>i});const n="NRBA",i=144e5,o=18e5},7894:(e,t,r)=>{function n(){return Math.round(performance.now())}r.d(t,{z:()=>n})},7243:(e,t,r)=>{r.d(t,{e:()=>o});var n=r(385),i={};function o(e){if(e in i)return i[e];if(0===(e||"").indexOf("data:"))return{protocol:"data"};let t;var r=n._A?.location,o={};if(n.il)t=document.createElement("a"),t.href=e;else try{t=new URL(e,r.href)}catch(e){return o}o.port=t.port;var a=t.href.split("://");!o.port&&a[1]&&(o.port=a[1].split("/")[0].split("@").pop().split(":")[1]),o.port&&"0"!==o.port||(o.port="https"===a[0]?"443":"80"),o.hostname=t.hostname||r.hostname,o.pathname=t.pathname,o.protocol=a[0],"/"!==o.pathname.charAt(0)&&(o.pathname="/"+o.pathname);var s=!t.protocol||":"===t.protocol||t.protocol===r.protocol,c=t.hostname===r.hostname&&t.port===r.port;return o.sameOrigin=s&&(!t.hostname||c),"/"===o.pathname&&(i[e]=o),o}},50:(e,t,r)=>{function n(e,t){"function"==typeof console.warn&&(console.warn("New Relic: ".concat(e)),t&&console.warn(t))}r.d(t,{Z:()=>n})},2587:(e,t,r)=>{r.d(t,{N:()=>c,T:()=>u});var n=r(2177),i=r(5546),o=r(8e3),a=r(3325);const s={stn:[a.D.sessionTrace],err:[a.D.jserrors,a.D.metrics],ins:[a.D.pageAction],spa:[a.D.spa],sr:[a.D.sessionReplay,a.D.sessionTrace]};function c(e,t){const r=n.ee.get(t);e&&"object"==typeof e&&(Object.entries(e).forEach((e=>{let[t,n]=e;void 0===u[t]&&(s[t]?s[t].forEach((e=>{n?(0,i.p)("feat-"+t,[],void 0,e,r):(0,i.p)("block-"+t,[],void 0,e,r),(0,i.p)("rumresp-"+t,[Boolean(n)],void 0,e,r)})):n&&(0,i.p)("feat-"+t,[],void 0,void 0,r),u[t]=Boolean(n))})),Object.keys(s).forEach((e=>{void 0===u[e]&&(s[e]?.forEach((t=>(0,i.p)("rumresp-"+e,[!1],void 0,t,r))),u[e]=!1)})),(0,o.L)(t,a.D.pageViewEvent))}const u={}},2210:(e,t,r)=>{r.d(t,{X:()=>i});var n=Object.prototype.hasOwnProperty;function i(e,t,r){if(n.call(e,t))return e[t];var i=r();if(Object.defineProperty&&Object.keys)try{return Object.defineProperty(e,t,{value:i,writable:!0,enumerable:!1}),i}catch(e){}return e[t]=i,i}},1284:(e,t,r)=>{r.d(t,{D:()=>n});const n=(e,t)=>Object.entries(e||{}).map((e=>{let[r,n]=e;return t(r,n)}))},4351:(e,t,r)=>{r.d(t,{P:()=>o});var n=r(2177);const i=()=>{const e=new WeakSet;return(t,r)=>{if("object"==typeof r&&null!==r){if(e.has(r))return;e.add(r)}return r}};function o(e){try{return JSON.stringify(e,i())}catch(e){try{n.ee.emit("internal-error",[e])}catch(e){}}}},3960:(e,t,r)=>{r.d(t,{K:()=>a,b:()=>o});var n=r(3239);function i(){return"undefined"==typeof document||"complete"===document.readyState}function o(e,t){if(i())return e();(0,n.bP)("load",e,t)}function a(e){if(i())return e();(0,n.iz)("DOMContentLoaded",e)}},8632:(e,t,r)=>{r.d(t,{EZ:()=>u,Qy:()=>c,ce:()=>o,fP:()=>a,gG:()=>d,mF:()=>s});var n=r(7894),i=r(385);const o={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net"};function a(){return i._A.NREUM||(i._A.NREUM={}),void 0===i._A.newrelic&&(i._A.newrelic=i._A.NREUM),i._A.NREUM}function s(){let e=a();return e.o||(e.o={ST:i._A.setTimeout,SI:i._A.setImmediate,CT:i._A.clearTimeout,XHR:i._A.XMLHttpRequest,REQ:i._A.Request,EV:i._A.Event,PR:i._A.Promise,MO:i._A.MutationObserver,FETCH:i._A.fetch}),e}function c(e,t,r){let i=a();const o=i.initializedAgents||{},s=o[e]||{};return Object.keys(s).length||(s.initializedAt={ms:(0,n.z)(),date:new Date}),i.initializedAgents={...o,[e]:{...s,[r]:t}},i}function u(e,t){a()[e]=t}function d(){return function(){let e=a();const t=e.info||{};e.info={beacon:o.beacon,errorBeacon:o.errorBeacon,...t}}(),function(){let e=a();const t=e.init||{};e.init={...t}}(),s(),function(){let e=a();const t=e.loader_config||{};e.loader_config={...t}}(),a()}},7956:(e,t,r)=>{r.d(t,{N:()=>i});var n=r(3239);function i(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],r=arguments.length>2?arguments[2]:void 0,i=arguments.length>3?arguments[3]:void 0;return void(0,n.iz)("visibilitychange",(function(){if(t)return void("hidden"==document.visibilityState&&e());e(document.visibilityState)}),r,i)}},1214:(e,t,r)=>{r.d(t,{em:()=>v,u5:()=>N,QU:()=>S,_L:()=>I,Gm:()=>L,Lg:()=>M,gy:()=>U,BV:()=>Q,Kf:()=>ee});var n=r(2177);const i="nr@original";var o=Object.prototype.hasOwnProperty,a=!1;function s(e,t){return e||(e=n.ee),r.inPlace=function(e,t,n,i,o){n||(n="");var a,s,c,u="-"===n.charAt(0);for(c=0;c 2?n-2:0),o=2;o {r(A[T],e,w),r(E[T],e,w)})),r(l._A,"fetch",y),t.on(y+"end",(function(e,r){var n=this;if(r){var i=r.headers.get("content-length");null!==i&&(n.rxSize=i),t.emit(y+"done",[null,r],n)}else t.emit(y+"done",[e],n)})),t}const O={},j=["pushState","replaceState"];function S(e){const t=function(e){return(e||n.ee).get("history")}(e);return!l.il||O[t.debugId]++||(O[t.debugId]=1,s(t).inPlace(window.history,j,"-")),t}var P=r(3239);const C={},R=["appendChild","insertBefore","replaceChild"];function I(e){const t=function(e){return(e||n.ee).get("jsonp")}(e);if(!l.il||C[t.debugId])return t;C[t.debugId]=!0;var r=s(t),i=/[?&](?:callback|cb)=([^&#]+)/,o=/(.*)\.([^.]+)/,a=/^(\w+)(\.|$)(.*)$/;function c(e,t){var r=e.match(a),n=r[1],i=r[3];return i?c(i,t[n]):t[n]}return r.inPlace(Node.prototype,R,"dom-"),t.on("dom-start",(function(e){!function(e){if(!e||"string"!=typeof e.nodeName||"script"!==e.nodeName.toLowerCase())return;if("function"!=typeof e.addEventListener)return;var n=(a=e.src,s=a.match(i),s?s[1]:null);var a,s;if(!n)return;var u=function(e){var t=e.match(o);if(t&&t.length>=3)return{key:t[2],parent:c(t[1],window)};return{key:e,parent:window}}(n);if("function"!=typeof u.parent[u.key])return;var d={};function f(){t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}function l(){t.emit("jsonp-error",[],d),t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}r.inPlace(u.parent,[u.key],"cb-",d),e.addEventListener("load",f,(0,P.m$)(!1)),e.addEventListener("error",l,(0,P.m$)(!1)),t.emit("new-jsonp",[e.src],d)}(e[0])})),t}var k=r(5763);const H={};function L(e){const t=function(e){return(e||n.ee).get("mutation")}(e);if(!l.il||H[t.debugId])return t;H[t.debugId]=!0;var r=s(t),i=k.Yu.MO;return i&&(window.MutationObserver=function(e){return this instanceof i?new i(r(e,"fn-")):i.apply(this,arguments)},MutationObserver.prototype=i.prototype),t}const z={};function M(e){const t=function(e){return(e||n.ee).get("promise")}(e);if(z[t.debugId])return t;z[t.debugId]=!0;var r=n.c,o=s(t),a=k.Yu.PR;return a&&function(){function e(r){var n=t.context(),i=o(r,"executor-",n,null,!1);const s=Reflect.construct(a,[i],e);return t.context(s).getCtx=function(){return n},s}l._A.Promise=e,Object.defineProperty(e,"name",{value:"Promise"}),e.toString=function(){return a.toString()},Object.setPrototypeOf(e,a),["all","race"].forEach((function(r){const n=a[r];e[r]=function(e){let i=!1;[...e||[]].forEach((e=>{this.resolve(e).then(a("all"===r),a(!1))}));const o=n.apply(this,arguments);return o;function a(e){return function(){t.emit("propagate",[null,!i],o,!1,!1),i=i||!e}}}})),["resolve","reject"].forEach((function(r){const n=a[r];e[r]=function(e){const r=n.apply(this,arguments);return e!==r&&t.emit("propagate",[e,!0],r,!1,!1),r}})),e.prototype=a.prototype;const n=a.prototype.then;a.prototype.then=function(){var e=this,i=r(e);i.promise=e;for(var a=arguments.length,s=new Array(a),c=0;c e())),t};function m(e,t){i.inPlace(t,["onreadystatechange"],"fn-",E)}function b(){var e=this,t=r.context(e);e.readyState>3&&!t.resolved&&(t.resolved=!0,r.emit("xhr-resolved",[],e)),i.inPlace(e,f,"fn-",E)}if(function(e,t){for(var r in e)t[r]=e[r]}(o,p),p.prototype=o.prototype,i.inPlace(p.prototype,J,"-xhr-",E),r.on("send-xhr-start",(function(e,t){m(e,t),function(e){h.push(e),a&&(y?y.then(A):u?u(A):(w=-w,x.data=w))}(t)})),r.on("open-xhr-start",m),a){var y=c&&c.resolve();if(!u&&!c){var w=1,x=document.createTextNode(w);new a(A).observe(x,{characterData:!0})}}else t.on("fn-end",(function(e){e[0]&&e[0].type===d||A()}));function A(){for(var e=0;e {r.d(t,{t:()=>n});const n=r(3325).D.ajax},6660:(e,t,r)=>{r.d(t,{A:()=>i,t:()=>n});const n=r(3325).D.jserrors,i="nr@seenError"},3081:(e,t,r)=>{r.d(t,{gF:()=>o,mY:()=>i,t9:()=>n,vz:()=>s,xS:()=>a});const n=r(3325).D.metrics,i="sm",o="cm",a="storeSupportabilityMetrics",s="storeEventMetrics"},4649:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageAction},7633:(e,t,r)=>{r.d(t,{Dz:()=>i,OJ:()=>a,qw:()=>o,t9:()=>n});const n=r(3325).D.pageViewEvent,i="firstbyte",o="domcontent",a="windowload"},9251:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageViewTiming},3614:(e,t,r)=>{r.d(t,{BST_RESOURCE:()=>i,END:()=>s,FEATURE_NAME:()=>n,FN_END:()=>u,FN_START:()=>c,PUSH_STATE:()=>d,RESOURCE:()=>o,START:()=>a});const n=r(3325).D.sessionTrace,i="bstResource",o="resource",a="-start",s="-end",c="fn"+a,u="fn"+s,d="pushState"},7836:(e,t,r)=>{r.d(t,{BODY:()=>A,CB_END:()=>E,CB_START:()=>u,END:()=>x,FEATURE_NAME:()=>i,FETCH:()=>_,FETCH_BODY:()=>v,FETCH_DONE:()=>m,FETCH_START:()=>p,FN_END:()=>c,FN_START:()=>s,INTERACTION:()=>l,INTERACTION_API:()=>d,INTERACTION_EVENTS:()=>o,JSONP_END:()=>b,JSONP_NODE:()=>g,JS_TIME:()=>T,MAX_TIMER_BUDGET:()=>a,REMAINING:()=>f,SPA_NODE:()=>h,START:()=>w,originalSetTimeout:()=>y});var n=r(5763);const i=r(3325).D.spa,o=["click","submit","keypress","keydown","keyup","change"],a=999,s="fn-start",c="fn-end",u="cb-start",d="api-ixn-",f="remaining",l="interaction",h="spaNode",g="jsonpNode",p="fetch-start",m="fetch-done",v="fetch-body-",b="jsonp-end",y=n.Yu.ST,w="-start",x="-end",A="-body",E="cb"+x,T="jsTime",_="fetch"},5938:(e,t,r)=>{r.d(t,{W:()=>o});var n=r(5763),i=r(2177);class o{constructor(e,t,r){this.agentIdentifier=e,this.aggregator=t,this.ee=i.ee.get(e,(0,n.OP)(this.agentIdentifier).isolatedBacklog),this.featureName=r,this.blocked=!1}}},9144:(e,t,r)=>{r.d(t,{j:()=>m});var n=r(3325),i=r(5763),o=r(5546),a=r(2177),s=r(7894),c=r(8e3),u=r(3960),d=r(385),f=r(50),l=r(3081),h=r(8632);function g(){const e=(0,h.gG)();["setErrorHandler","finished","addToTrace","inlineHit","addRelease","addPageAction","setCurrentRouteName","setPageViewName","setCustomAttribute","interaction","noticeError","setUserId"].forEach((t=>{e[t]=function(){for(var r=arguments.length,n=new Array(r),i=0;i 1?r-1:0),i=1;i {e.exposed&&e.api[t]&&o.push(e.api[t](...n))})),o.length>1?o:o[0]}(t,...n)}}))}var p=r(2587);function m(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},m=arguments.length>2?arguments[2]:void 0,v=arguments.length>3?arguments[3]:void 0,{init:b,info:y,loader_config:w,runtime:x={loaderType:m},exposed:A=!0}=t;const E=(0,h.gG)();y||(b=E.init,y=E.info,w=E.loader_config),(0,i.Dg)(e,b||{}),(0,i.GE)(e,w||{}),(0,i.sU)(e,x),y.jsAttributes??={},d.v6&&(y.jsAttributes.isWorker=!0),(0,i.CX)(e,y),g();const T=function(e,t){t||(0,c.R)(e,"api");const h={};var g=a.ee.get(e),p=g.get("tracer"),m="api-",v=m+"ixn-";function b(t,r,n,o){const a=(0,i.C5)(e);return null===r?delete a.jsAttributes[t]:(0,i.CX)(e,{...a,jsAttributes:{...a.jsAttributes,[t]:r}}),x(m,n,!0,o||null===r?"session":void 0)(t,r)}function y(){}["setErrorHandler","finished","addToTrace","inlineHit","addRelease"].forEach((e=>h[e]=x(m,e,!0,"api"))),h.addPageAction=x(m,"addPageAction",!0,n.D.pageAction),h.setCurrentRouteName=x(m,"routeName",!0,n.D.spa),h.setPageViewName=function(t,r){if("string"==typeof t)return"/"!==t.charAt(0)&&(t="/"+t),(0,i.OP)(e).customTransaction=(r||"http://custom.transaction")+t,x(m,"setPageViewName",!0)()},h.setCustomAttribute=function(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2];if("string"==typeof e){if(["string","number"].includes(typeof t)||null===t)return b(e,t,"setCustomAttribute",r);(0,f.Z)("Failed to execute setCustomAttribute.\nNon-null value must be a string or number type, but a type of was provided."))}else(0,f.Z)("Failed to execute setCustomAttribute.\nName must be a string type, but a type of was provided."))},h.setUserId=function(e){if("string"==typeof e||null===e)return b("enduser.id",e,"setUserId",!0);(0,f.Z)("Failed to execute setUserId.\nNon-null value must be a string type, but a type of was provided."))},h.interaction=function(){return(new y).get()};var w=y.prototype={createTracer:function(e,t){var r={},i=this,a="function"==typeof t;return(0,o.p)(v+"tracer",[(0,s.z)(),e,r],i,n.D.spa,g),function(){if(p.emit((a?"":"no-")+"fn-start",[(0,s.z)(),i,a],r),a)try{return t.apply(this,arguments)}catch(e){throw p.emit("fn-err",[arguments,this,"string"==typeof e?new Error(e):e],r),e}finally{p.emit("fn-end",[(0,s.z)()],r)}}}};function x(e,t,r,i){return function(){return(0,o.p)(l.xS,["API/"+t+"/called"],void 0,n.D.metrics,g),i&&(0,o.p)(e+t,[(0,s.z)(),...arguments],r?null:this,i,g),r?void 0:this}}function A(){r.e(439).then(r.bind(r,7438)).then((t=>{let{setAPI:r}=t;r(e),(0,c.L)(e,"api")})).catch((()=>(0,f.Z)("Downloading runtime APIs failed...")))}return["actionText","setName","setAttribute","save","ignore","onEnd","getContext","end","get"].forEach((e=>{w[e]=x(v,e,void 0,n.D.spa)})),h.noticeError=function(e,t){"string"==typeof e&&(e=new Error(e)),(0,o.p)(l.xS,["API/noticeError/called"],void 0,n.D.metrics,g),(0,o.p)("err",[e,(0,s.z)(),!1,t],void 0,n.D.jserrors,g)},d.il?(0,u.b)((()=>A()),!0):A(),h}(e,v);return(0,h.Qy)(e,T,"api"),(0,h.Qy)(e,A,"exposed"),(0,h.EZ)("activatedFeatures",p.T),T}},3325:(e,t,r)=>{r.d(t,{D:()=>n,p:()=>i});const n={ajax:"ajax",jserrors:"jserrors",metrics:"metrics",pageAction:"page_action",pageViewEvent:"page_view_event",pageViewTiming:"page_view_timing",sessionReplay:"session_replay",sessionTrace:"session_trace",spa:"spa"},i={[n.pageViewEvent]:1,[n.pageViewTiming]:2,[n.metrics]:3,[n.jserrors]:4,[n.ajax]:5,[n.sessionTrace]:6,[n.pageAction]:7,[n.spa]:8,[n.sessionReplay]:9}}},n={};function i(e){var t=n[e];if(void 0!==t)return t.exports;var o=n[e]={exports:{}};return r[e](o,o.exports,i),o.exports}i.m=r,i.d=(e,t)=>{for(var r in t)i.o(t,r)&&!i.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},i.f={},i.e=e=>Promise.all(Object.keys(i.f).reduce(((t,r)=>(i.f[r](e,t),t)),[])),i.u=e=>(({78:"page_action-aggregate",147:"metrics-aggregate",242:"session-manager",317:"jserrors-aggregate",348:"page_view_timing-aggregate",412:"lazy-feature-loader",439:"async-api",538:"recorder",590:"session_replay-aggregate",675:"compressor",733:"session_trace-aggregate",786:"page_view_event-aggregate",873:"spa-aggregate",898:"ajax-aggregate"}[e]||e)+"."+{78:"ac76d497",147:"3dc53903",148:"1a20d5fe",242:"2a64278a",317:"49e41428",348:"bd6de33a",412:"2f55ce66",439:"30bd804e",538:"1b18459f",590:"cf0efb30",675:"ae9f91a8",733:"83105561",786:"06482edd",860:"03a8b7a5",873:"e6b09d52",898:"998ef92b"}[e]+"-1.236.0.min.js"),i.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),e={},t="NRBA:",i.l=(r,n,o,a)=>{if(e[r])e[r].push(n);else{var s,c;if(void 0!==o)for(var u=document.getElementsByTagName("script"),d=0;d {s.onerror=s.onload=null,clearTimeout(h);var i=e[r];if(delete e[r],s.parentNode&&s.parentNode.removeChild(s),i&&i.forEach((e=>e(n))),t)return t(n)},h=setTimeout(l.bind(null,void 0,{type:"timeout",target:s}),12e4);s.onerror=l.bind(null,s.onerror),s.onload=l.bind(null,s.onload),c&&document.head.appendChild(s)}},i.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.j=364,i.p="https://js-agent.newrelic.com/",(()=>{var e={364:0,953:0};i.f.j=(t,r)=>{var n=i.o(e,t)?e[t]:void 0;if(0!==n)if(n)r.push(n[2]);else{var o=new Promise(((r,i)=>n=e[t]=[r,i]));r.push(n[2]=o);var a=i.p+i.u(t),s=new Error;i.l(a,(r=>{if(i.o(e,t)&&(0!==(n=e[t])&&(e[t]=void 0),n)){var o=r&&("load"===r.type?"missing":r.type),a=r&&r.target&&r.target.src;s.message="Loading chunk "+t+" failed.\n("+o+": "+a+")",s.name="ChunkLoadError",s.type=o,s.request=a,n[1](s)}}),"chunk-"+t,t)}};var t=(t,r)=>{var n,o,[a,s,c]=r,u=0;if(a.some((t=>0!==e[t]))){for(n in s)i.o(s,n)&&(i.m[n]=s[n]);if(c)c(i)}for(t&&t(r);u {i.r(o);var e=i(3325),t=i(5763);const r=Object.values(e.D);function n(e){const n={};return r.forEach((r=>{n[r]=function(e,r){return!1!==(0,t.Mt)(r,"".concat(e,".enabled"))}(r,e)})),n}var a=i(9144);var s=i(5546),c=i(385),u=i(8e3),d=i(5938),f=i(3960),l=i(50);class h extends d.W{constructor(e,t,r){let n=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];super(e,t,r),this.auto=n,this.abortHandler,this.featAggregate,this.onAggregateImported,n&&(0,u.R)(e,r)}importAggregator(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};if(this.featAggregate||!this.auto)return;const r=c.il&&!0===(0,t.Mt)(this.agentIdentifier,"privacy.cookies_enabled");let n;this.onAggregateImported=new Promise((e=>{n=e}));const o=async()=>{let t;try{if(r){const{setupAgentSession:e}=await Promise.all([i.e(860),i.e(242)]).then(i.bind(i,3228));t=e(this.agentIdentifier)}}catch(e){(0,l.Z)("A problem occurred when starting up session manager. This page will not start or extend any session.",e)}try{if(!this.shouldImportAgg(this.featureName,t))return void(0,u.L)(this.agentIdentifier,this.featureName);const{lazyFeatureLoader:r}=await i.e(412).then(i.bind(i,8582)),{Aggregate:o}=await r(this.featureName,"aggregate");this.featAggregate=new o(this.agentIdentifier,this.aggregator,e),n(!0)}catch(e){(0,l.Z)("Downloading and initializing ".concat(this.featureName," failed..."),e),this.abortHandler?.(),n(!1)}};c.il?(0,f.b)((()=>o()),!0):o()}shouldImportAgg(r,n){return r!==e.D.sessionReplay||!1!==(0,t.Mt)(this.agentIdentifier,"session_trace.enabled")&&(!!n?.isNew||!!n?.state.sessionReplay)}}var g=i(7633),p=i(7894);class m extends h{static featureName=g.t9;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];if(super(r,n,g.t9,i),("undefined"==typeof PerformanceNavigationTiming||c.Tt)&&"undefined"!=typeof PerformanceTiming){const n=(0,t.OP)(r);n[g.Dz]=Math.max(Date.now()-n.offset,0),(0,f.K)((()=>n[g.qw]=Math.max((0,p.z)()-n[g.Dz],0))),(0,f.b)((()=>{const t=(0,p.z)();n[g.OJ]=Math.max(t-n[g.Dz],0),(0,s.p)("timing",["load",t],void 0,e.D.pageViewTiming,this.ee)}))}this.importAggregator()}}var v=i(1117),b=i(1284);class y extends v.w{constructor(e){super(e),this.aggregatedData={}}store(e,t,r,n,i){var o=this.getBucket(e,t,r,i);return o.metrics=function(e,t){t||(t={count:0});return t.count+=1,(0,b.D)(e,(function(e,r){t[e]=w(r,t[e])})),t}(n,o.metrics),o}merge(e,t,r,n,i){var o=this.getBucket(e,t,n,i);if(o.metrics){var a=o.metrics;a.count+=r.count,(0,b.D)(r,(function(e,t){if("count"!==e){var n=a[e],i=r[e];i&&!i.c?a[e]=w(i.t,n):a[e]=function(e,t){if(!t)return e;t.c||(t=x(t.t));return t.min=Math.min(e.min,t.min),t.max=Math.max(e.max,t.max),t.t+=e.t,t.sos+=e.sos,t.c+=e.c,t}(i,a[e])}}))}else o.metrics=r}storeMetric(e,t,r,n){var i=this.getBucket(e,t,r);return i.stats=w(n,i.stats),i}getBucket(e,t,r,n){this.aggregatedData[e]||(this.aggregatedData[e]={});var i=this.aggregatedData[e][t];return i||(i=this.aggregatedData[e][t]={params:r||{}},n&&(i.custom=n)),i}get(e,t){return t?this.aggregatedData[e]&&this.aggregatedData[e][t]:this.aggregatedData[e]}take(e){for(var t={},r="",n=!1,i=0;i t.max&&(t.max=e),e 2&&void 0!==arguments[2])||arguments[2];super(e,r,j.t,n),c.il&&((0,t.OP)(e).initHidden=Boolean("hidden"===document.visibilityState),(0,N.N)((()=>(0,s.p)("docHidden",[(0,p.z)()],void 0,j.t,this.ee)),!0),(0,O.bP)("pagehide",(()=>(0,s.p)("winPagehide",[(0,p.z)()],void 0,j.t,this.ee))),this.importAggregator())}}var P=i(3081);class C extends h{static featureName=P.t9;constructor(e,t){let r=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(e,t,P.t9,r),this.importAggregator()}}var R,I=i(2210),k=i(1214),H=i(2177),L={};try{R=localStorage.getItem("__nr_flags").split(","),console&&"function"==typeof console.log&&(L.console=!0,-1!==R.indexOf("dev")&&(L.dev=!0),-1!==R.indexOf("nr_dev")&&(L.nrDev=!0))}catch(e){}function z(e){try{L.console&&z(e)}catch(e){}}L.nrDev&&H.ee.on("internal-error",(function(e){z(e.stack)})),L.dev&&H.ee.on("fn-err",(function(e,t,r){z(r.stack)})),L.dev&&(z("NR AGENT IN DEVELOPMENT MODE"),z("flags: "+(0,b.D)(L,(function(e,t){return e})).join(", ")));var M=i(6660);class B extends h{static featureName=M.t;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(r,n,M.t,i),this.skipNext=0;try{this.removeOnAbort=new AbortController}catch(e){}const o=this;o.ee.on("fn-start",(function(e,t,r){o.abortHandler&&(o.skipNext+=1)})),o.ee.on("fn-err",(function(t,r,n){o.abortHandler&&!n[M.A]&&((0,I.X)(n,M.A,(function(){return!0})),this.thrown=!0,(0,s.p)("err",[n,(0,p.z)()],void 0,e.D.jserrors,o.ee))})),o.ee.on("fn-end",(function(){o.abortHandler&&!this.thrown&&o.skipNext>0&&(o.skipNext-=1)})),o.ee.on("internal-error",(function(t){(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,o.ee)})),this.origOnerror=c._A.onerror,c._A.onerror=this.onerrorHandler.bind(this),c._A.addEventListener("unhandledrejection",(t=>{const r=function(e){let t="Unhandled Promise Rejection: ";if(e instanceof Error)try{return e.message=t+e.message,e}catch(t){return e}if(void 0===e)return new Error(t);try{return new Error(t+(0,D.P)(e))}catch(e){return new Error(t)}}(t.reason);(0,s.p)("err",[r,(0,p.z)(),!1,{unhandledPromiseRejection:1}],void 0,e.D.jserrors,this.ee)}),(0,O.m$)(!1,this.removeOnAbort?.signal)),(0,k.gy)(this.ee),(0,k.BV)(this.ee),(0,k.em)(this.ee),(0,t.OP)(r).xhrWrappable&&(0,k.Kf)(this.ee),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}onerrorHandler(t,r,n,i,o){"function"==typeof this.origOnerror&&this.origOnerror(...arguments);try{this.skipNext?this.skipNext-=1:(0,s.p)("err",[o||new F(t,r,n),(0,p.z)()],void 0,e.D.jserrors,this.ee)}catch(t){try{(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,this.ee)}catch(e){}}return!1}}function F(e,t,r){this.message=e||"Uncaught error with no additional information",this.sourceURL=t,this.line=r}let U=1;const q="nr@id";function G(e){const t=typeof e;return!e||"object"!==t&&"function"!==t?-1:e===c._A?0:(0,I.X)(e,q,(function(){return U++}))}function V(e){if("string"==typeof e&&e.length)return e.length;if("object"==typeof e){if("undefined"!=typeof ArrayBuffer&&e instanceof ArrayBuffer&&e.byteLength)return e.byteLength;if("undefined"!=typeof Blob&&e instanceof Blob&&e.size)return e.size;if(!("undefined"!=typeof FormData&&e instanceof FormData))try{return(0,D.P)(e).length}catch(e){return}}}var X=i(7243);class W{constructor(e){this.agentIdentifier=e,this.generateTracePayload=this.generateTracePayload.bind(this),this.shouldGenerateTrace=this.shouldGenerateTrace.bind(this)}generateTracePayload(e){if(!this.shouldGenerateTrace(e))return null;var r=(0,t.DL)(this.agentIdentifier);if(!r)return null;var n=(r.accountID||"").toString()||null,i=(r.agentID||"").toString()||null,o=(r.trustKey||"").toString()||null;if(!n||!i)return null;var a=(0,_.M)(),s=(0,_.Ht)(),c=Date.now(),u={spanId:a,traceId:s,timestamp:c};return(e.sameOrigin||this.isAllowedOrigin(e)&&this.useTraceContextHeadersForCors())&&(u.traceContextParentHeader=this.generateTraceContextParentHeader(a,s),u.traceContextStateHeader=this.generateTraceContextStateHeader(a,c,n,i,o)),(e.sameOrigin&&!this.excludeNewrelicHeader()||!e.sameOrigin&&this.isAllowedOrigin(e)&&this.useNewrelicHeaderForCors())&&(u.newrelicHeader=this.generateTraceHeader(a,s,c,n,i,o)),u}generateTraceContextParentHeader(e,t){return"00-"+t+"-"+e+"-01"}generateTraceContextStateHeader(e,t,r,n,i){return i+"@nr=0-1-"+r+"-"+n+"-"+e+"----"+t}generateTraceHeader(e,t,r,n,i,o){if(!("function"==typeof c._A?.btoa))return null;var a={v:[0,1],d:{ty:"Browser",ac:n,ap:i,id:e,tr:t,ti:r}};return o&&n!==o&&(a.d.tk=o),btoa((0,D.P)(a))}shouldGenerateTrace(e){return this.isDtEnabled()&&this.isAllowedOrigin(e)}isAllowedOrigin(e){var r=!1,n={};if((0,t.Mt)(this.agentIdentifier,"distributed_tracing")&&(n=(0,t.P_)(this.agentIdentifier).distributed_tracing),e.sameOrigin)r=!0;else if(n.allowed_origins instanceof Array)for(var i=0;i 2&&void 0!==arguments[2])||arguments[2];super(r,n,Z.t,i),(0,t.OP)(r).xhrWrappable&&(this.dt=new W(r),this.handler=(e,t,r,n)=>(0,s.p)(e,t,r,n,this.ee),(0,k.u5)(this.ee),(0,k.Kf)(this.ee),function(r,n,i,o){function a(e){var t=this;t.totalCbs=0,t.called=0,t.cbTime=0,t.end=E,t.ended=!1,t.xhrGuids={},t.lastSize=null,t.loadCaptureCalled=!1,t.params=this.params||{},t.metrics=this.metrics||{},e.addEventListener("load",(function(r){_(t,e)}),(0,O.m$)(!1)),c.IF||e.addEventListener("progress",(function(e){t.lastSize=e.loaded}),(0,O.m$)(!1))}function s(e){this.params={method:e[0]},T(this,e[1]),this.metrics={}}function u(e,n){var i=(0,t.DL)(r);i.xpid&&this.sameOrigin&&n.setRequestHeader("X-NewRelic-ID",i.xpid);var a=o.generateTracePayload(this.parsedOrigin);if(a){var s=!1;a.newrelicHeader&&(n.setRequestHeader("newrelic",a.newrelicHeader),s=!0),a.traceContextParentHeader&&(n.setRequestHeader("traceparent",a.traceContextParentHeader),a.traceContextStateHeader&&n.setRequestHeader("tracestate",a.traceContextStateHeader),s=!0),s&&(this.dt=a)}}function d(e,t){var r=this.metrics,i=e[0],o=this;if(r&&i){var a=V(i);a&&(r.txSize=a)}this.startTime=(0,p.z)(),this.listener=function(e){try{"abort"!==e.type||o.loadCaptureCalled||(o.params.aborted=!0),("load"!==e.type||o.called===o.totalCbs&&(o.onloadCalled||"function"!=typeof t.onload)&&"function"==typeof o.end)&&o.end(t)}catch(e){try{n.emit("internal-error",[e])}catch(e){}}};for(var s=0;s 1?e[1]=i:e.push(i)}else e[0]&&e[0].headers&&s(e[0].headers,n)&&(this.dt=n);function s(e,t){var r=!1;return t.newrelicHeader&&(e.set("newrelic",t.newrelicHeader),r=!0),t.traceContextParentHeader&&(e.set("traceparent",t.traceContextParentHeader),t.traceContextStateHeader&&e.set("tracestate",t.traceContextStateHeader),r=!0),r}}function x(e,t){this.params={},this.metrics={},this.startTime=(0,p.z)(),this.dt=t,e.length>=1&&(this.target=e[0]),e.length>=2&&(this.opts=e[1]);var r,n=this.opts||{},i=this.target;"string"==typeof i?r=i:"object"==typeof i&&i instanceof Y?r=i.url:c._A?.URL&&"object"==typeof i&&i instanceof URL&&(r=i.href),T(this,r);var o=(""+(i&&i instanceof Y&&i.method||n.method||"GET")).toUpperCase();this.params.method=o,this.txSize=V(n.body)||0}function A(t,r){var n;this.endTime=(0,p.z)(),this.params||(this.params={}),this.params.status=r?r.status:0,"string"==typeof this.rxSize&&this.rxSize.length>0&&(n=+this.rxSize);var o={txSize:this.txSize,rxSize:n,duration:(0,p.z)()-this.startTime};i("xhr",[this.params,o,this.startTime,this.endTime,"fetch"],this,e.D.ajax)}function E(t){var r=this.params,n=this.metrics;if(!this.ended){this.ended=!0;for(var o=0;o 2&&void 0!==arguments[2])||arguments[2];super(e,t,we.t,r),this.importAggregator()}}new class{constructor(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:(0,_.ky)(16);c._A?(this.agentIdentifier=t,this.sharedAggregator=new y({agentIdentifier:this.agentIdentifier}),this.features={},this.desiredFeatures=new Set(e.features||[]),this.desiredFeatures.add(m),Object.assign(this,(0,a.j)(this.agentIdentifier,e,e.loaderType||"agent")),this.start()):(0,l.Z)("Failed to initial the agent. Could not determine the runtime environment.")}get config(){return{info:(0,t.C5)(this.agentIdentifier),init:(0,t.P_)(this.agentIdentifier),loader_config:(0,t.DL)(this.agentIdentifier),runtime:(0,t.OP)(this.agentIdentifier)}}start(){const t="features";try{const r=n(this.agentIdentifier),i=[...this.desiredFeatures];i.sort(((t,r)=>e.p[t.featureName]-e.p[r.featureName])),i.forEach((t=>{if(r[t.featureName]||t.featureName===e.D.pageViewEvent){const n=function(t){switch(t){case e.D.ajax:return[e.D.jserrors];case e.D.sessionTrace:return[e.D.ajax,e.D.pageViewEvent];case e.D.sessionReplay:return[e.D.sessionTrace];case e.D.pageViewTiming:return[e.D.pageViewEvent];default:return[]}}(t.featureName);n.every((e=>r[e]))||(0,l.Z)("".concat(t.featureName," is enabled but one or more dependent features has been disabled (").concat((0,D.P)(n),"). This may cause unintended consequences or missing data...")),this.features[t.featureName]=new t(this.agentIdentifier,this.sharedAggregator)}})),(0,T.Qy)(this.agentIdentifier,this.features,t)}catch(e){(0,l.Z)("Failed to initialize all enabled instrument classes (agent aborted) -",e);for(const e in this.features)this.features[e].abortHandler?.();const r=(0,T.fP)();return delete r.initializedAgents[this.agentIdentifier]?.api,delete r.initializedAgents[this.agentIdentifier]?.[t],delete this.sharedAggregator,r.ee?.abort(),delete r.ee?.get(this.agentIdentifier),!1}}}({features:[J,m,S,class extends h{static featureName=oe;constructor(t,r){if(super(t,r,oe,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;const n=this.ee;let i;(0,k.QU)(n),this.eventsEE=(0,k.em)(n),this.eventsEE.on(se,(function(e,t){this.bstStart=(0,p.z)()})),this.eventsEE.on(ae,(function(t,r){(0,s.p)("bst",[t[0],r,this.bstStart,(0,p.z)()],void 0,e.D.sessionTrace,n)})),n.on(ce+ne,(function(e){this.time=(0,p.z)(),this.startPath=location.pathname+location.hash})),n.on(ce+ie,(function(t){(0,s.p)("bstHist",[location.pathname+location.hash,this.startPath,this.time],void 0,e.D.sessionTrace,n)}));try{i=new PerformanceObserver((t=>{const r=t.getEntries();(0,s.p)(te,[r],void 0,e.D.sessionTrace,n)})),i.observe({type:re,buffered:!0})}catch(e){}this.importAggregator({resourceObserver:i})}},C,xe,B,class extends h{static featureName=de;constructor(e,r){if(super(e,r,de,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;if(!(0,t.OP)(e).xhrWrappable)return;try{this.removeOnAbort=new AbortController}catch(e){}let n,i=0;const o=this.ee.get("tracer"),a=(0,k._L)(this.ee),s=(0,k.Lg)(this.ee),u=(0,k.BV)(this.ee),d=(0,k.Kf)(this.ee),f=this.ee.get("events"),l=(0,k.u5)(this.ee),h=(0,k.QU)(this.ee),g=(0,k.Gm)(this.ee);function m(e,t){h.emit("newURL",[""+window.location,t])}function v(){i++,n=window.location.hash,this[ve]=(0,p.z)()}function b(){i--,window.location.hash!==n&&m(0,!0);var e=(0,p.z)();this[pe]=~~this[pe]+e-this[ve],this[ye]=e}function y(e,t){e.on(t,(function(){this[t]=(0,p.z)()}))}this.ee.on(ve,v),s.on(be,v),a.on(be,v),this.ee.on(ye,b),s.on(ge,b),a.on(ge,b),this.ee.buffer([ve,ye,"xhr-resolved"],this.featureName),f.buffer([ve],this.featureName),u.buffer(["setTimeout"+le,"clearTimeout"+fe,ve],this.featureName),d.buffer([ve,"new-xhr","send-xhr"+fe],this.featureName),l.buffer([me+fe,me+"-done",me+he+fe,me+he+le],this.featureName),h.buffer(["newURL"],this.featureName),g.buffer([ve],this.featureName),s.buffer(["propagate",be,ge,"executor-err","resolve"+fe],this.featureName),o.buffer([ve,"no-"+ve],this.featureName),a.buffer(["new-jsonp","cb-start","jsonp-error","jsonp-end"],this.featureName),y(l,me+fe),y(l,me+"-done"),y(a,"new-jsonp"),y(a,"jsonp-end"),y(a,"cb-start"),h.on("pushState-end",m),h.on("replaceState-end",m),window.addEventListener("hashchange",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("load",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("popstate",(function(){m(0,i>1)}),(0,O.m$)(!0,this.removeOnAbort?.signal)),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}}],loaderType:"spa"})})(),window.NRBA=o})(); window.jQuery || document.write(' ') CKEDITOR_BASEPATH='https://f1000research.com/js/vendor/ckeditor/' window.reactTheme = 'research'; window.MathJax = { CommonHTML: { linebreaks: { automatic: true } }, 'HTML-CSS': { linebreaks: { automatic: true } }, SVG: { linebreaks: { automatic: true } }, AuthorInit: function() { MathJax.Hub.Register.MessageHook('End Process', function () { let timeout = false; // holder for timeout id const delay = 250; // delay after event is "complete" to run callback const reflowMath = function() { const dispFormulas = document.querySelectorAll('.disp-formula.panel'); if (!dispFormulas) { return; } for (const dispFormula of dispFormulas) { const child = dispFormula.querySelector('.MathJax_Preview').nextSibling.firstChild; const isMultiline = MathJax.Hub.getAllJax(dispFormula)[0].root.isMultiline; if (dispFormula.offsetWidth < child.offsetWidth || isMultiline) { MathJax.Hub.Queue(['Rerender', MathJax.Hub, dispFormula]); } } }; window.addEventListener('resize', function() { clearTimeout(timeout); // clear the timeout timeout = setTimeout(reflowMath, delay); // start timing for event "completion" }); }); }, }; if (window.location.hash == '#_=_'){ window.location = window.location.href.split('#')[0] } !function(f,b,e,v,n,t,s){if(f.fbq)return;n=f.fbq=function() {n.callMethod? n.callMethod.apply(n,arguments):n.queue.push(arguments)} ;if(!f._fbq)f._fbq=n; n.push=n;n.loaded=!0;n.version='2.0';n.queue=[];t=b.createElement(e);t.async=!0; t.src=v;s=b.getElementsByTagName(e)[0];s.parentNode.insertBefore(t,s)}(window, document,'script','https://connect.facebook.net/en_US/fbevents.js'); fbq('init', '1641728616063202'); fbq('track', "PixelInitialized", {}); (function(h,o,t,j,a,r){ h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)}; h._hjSettings={hjid:2318163,hjsv:6}; a=o.getElementsByTagName('head')[0]; r=o.createElement('script');r.async=1; r.src=t+h._hjSettings.hjid+j+h._hjSettings.hjsv; a.appendChild(r); })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv='); search file_upload Submit your research search menu close search Browse Gateways & Collections How to Publish Submit your Research My Submissions Article Guidelines Article Guidelines (New Versions) Open Data, Software and Code Guidelines Open Data and Accessible Source Materials Guidelines (HSS) Open Data, Software and Code Guidelines (PSE) Prepublication Checks Production Process Posters and Slides Guidelines Document Guidelines Article Processing Charges Peer Review Finding Article Reviewers About How it Works For Reviewers Our Advisors Policies Glossary FAQs For Developers Newsroom Contact My Research Submissions Content and Tracking Alerts My Details Sign In file_upload Submit your research { "@context": "https://schema.org", "@type": "ScholarlyArticle", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://f1000research.com/articles/13-320" }, "headline": "Monitoring Open Science as transformative change: Towards a systemic framework", "datePublished": "2024-04-23T17:26:04", "dateModified": "2024-04-23T17:26:04", "author": [ { "@type": "Person", "name": "Ismael Rafols" }, { "@type": "Person", "name": "Ingeborg Meijer" }, { "@type": "Person", "name": "Jordi Molas-Gallart" } ], "publisher": { "@type": "Organization", "name": "F1000Research", "logo": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 480, "width": 60 } }, "image": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 1200, "width": 150 }, "description": " Background Following a flurry of policies for Open Science (OS), there is now a wave of initiatives to monitor its adoption. However, the great diversity of understandings and activities related to Open Science makes monitoring very challenging. There is a danger that by focusing on what can be readily observed (e.g. publications) many other OS activities are overlooked (e.g. participation), with a potential narrowing of OS scope, streetlight effects, and deviation from the values of OS. Since Open Science can be understood as a systemic transformation of the research system, we have borrowed concepts from Transformative Innovation Policies frameworks which aim at evaluating socio-technical transitions. In accordance with this view of OS as a systemic transformation, we propose that the new monitoring efforts should shift towards: (i) systemic perspectives which considers the various actions related to OS, including policies and outputs (e.g. datasets) but also processes (e.g. participatory events), outcomes (e.g. citizen interest in science) and expected impacts (e.g. better scientific contributions to addressing societal problems); (ii) implementation of monitoring as reflexive learning (rather than accountability or benchmarking); (iii) mapping the directionality of the activities and the values associated with the choices in directions. In summary, a monitoring framework for OS requires a profound change in conventional monitoring practices. The scope should broaden from current focus on outputs (such as publications) towards the processes of connection that make science ‘open’ (usage, co-creation and dialogue), as well as towards outcomes (changes in practices) and the longer-term impacts that reflect the values and normative commitments of OS. " } { "@context": "http://schema.org", "@type": "BreadcrumbList", "itemListElement": [ { "@type": "ListItem", "position": "1", "item": { "@id": "https://f1000research.com/", "name": "Home" } }, { "@type": "ListItem", "position": "2", "item": { "@id": "https://f1000research.com/browse/articles", "name": "Browse" } }, { "@type": "ListItem", "position": "3", "item": { "@id": "https://f1000research.com/articles/13-320", "name": "Monitoring Open Science as transformative change: Towards a systemic..." } } ] } Home Browse Monitoring Open Science as transformative change: Towards a systemic... ALL Metrics - Views Downloads Get PDF Get XML Cite How to cite this article Rafols I, Meijer I and Molas-Gallart J. Monitoring Open Science as transformative change: Towards a systemic framework [version 1; peer review: 2 approved] . F1000Research 2024, 13 :320 ( https://doi.org/10.12688/f1000research.148290.1 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. Close Copy Citation Details Export Export Citation Sciwheel EndNote Ref. Manager Bibtex ProCite Sente EXPORT Select a format first Track Share ▬ ✚ Opinion Article Monitoring Open Science as transformative change: Towards a systemic framework [version 1; peer review: 2 approved] Ismael Rafols https://orcid.org/0000-0002-6527-7778 1 , Ingeborg Meijer 1 , Jordi Molas-Gallart https://orcid.org/0000-0002-8320-2681 2 Ismael Rafols https://orcid.org/0000-0002-6527-7778 1 , Ingeborg Meijer 1 , Jordi Molas-Gallart https://orcid.org/0000-0002-8320-2681 2 PUBLISHED 23 Apr 2024 Author details Author details 1 Centre for Science and Technology Studies, Leiden University, Leiden, The Netherlands 2 INGENIO (CSIC-UPV), Universitat Politecnica de Valencia, Valencia, Spain Ismael Rafols Roles: Conceptualization, Visualization, Writing – Original Draft Preparation, Writing – Review & Editing Ingeborg Meijer Roles: Conceptualization, Writing – Review & Editing Jordi Molas-Gallart Roles: Conceptualization, Writing – Review & Editing OPEN PEER REVIEW DETAILS REVIEWER STATUS This article is included in the Research on Research, Policy & Culture gateway. Abstract Background Following a flurry of policies for Open Science (OS), there is now a wave of initiatives to monitor its adoption. However, the great diversity of understandings and activities related to Open Science makes monitoring very challenging. There is a danger that by focusing on what can be readily observed (e.g. publications) many other OS activities are overlooked (e.g. participation), with a potential narrowing of OS scope, streetlight effects, and deviation from the values of OS. Since Open Science can be understood as a systemic transformation of the research system, we have borrowed concepts from Transformative Innovation Policies frameworks which aim at evaluating socio-technical transitions. In accordance with this view of OS as a systemic transformation, we propose that the new monitoring efforts should shift towards: (i) systemic perspectives which considers the various actions related to OS, including policies and outputs (e.g. datasets) but also processes (e.g. participatory events), outcomes (e.g. citizen interest in science) and expected impacts (e.g. better scientific contributions to addressing societal problems); (ii) implementation of monitoring as reflexive learning (rather than accountability or benchmarking); (iii) mapping the directionality of the activities and the values associated with the choices in directions. In summary, a monitoring framework for OS requires a profound change in conventional monitoring practices. The scope should broaden from current focus on outputs (such as publications) towards the processes of connection that make science ‘open’ (usage, co-creation and dialogue), as well as towards outcomes (changes in practices) and the longer-term impacts that reflect the values and normative commitments of OS. READ ALL READ LESS Keywords Open Science, Monitoring, Evaluation, Open Access, Open Data Corresponding Author(s) Ismael Rafols ( [email protected] ) Close Corresponding author: Ismael Rafols Competing interests: IR is the Chairholder of the UNESCO Chair of Diversity and Inclusion in Global Science and was involved in advisory meetings of the UNESCO Working Group on Open Science Monitoring Framework, as detailed in the UNESCO OS Implementation webpage. Grant information: The author(s) declared that no grants were involved in supporting this work. Copyright: © 2024 Rafols I et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. How to cite: Rafols I, Meijer I and Molas-Gallart J. Monitoring Open Science as transformative change: Towards a systemic framework [version 1; peer review: 2 approved] . F1000Research 2024, 13 :320 ( https://doi.org/10.12688/f1000research.148290.1 ) First published: 23 Apr 2024, 13 :320 ( https://doi.org/10.12688/f1000research.148290.1 ) Latest published: 23 Apr 2024, 13 :320 ( https://doi.org/10.12688/f1000research.148290.1 ) 1. Introduction In the wake of a flurry of policies and investments in Open Science (OS), there is now a wave of efforts to monitor its promotion and adoption. Institutions such as the European Commission 1 , the French 2 and Finish 3 governments, and the European Open Science Cloud (EOSC) 4 are gathering statistics on OS outputs, policies and practices and making them available in dedicated websites. The European Commission has funded a string of projects aimed at capturing and understanding uptake and effects of Open Science, for example through the development of indicators for research assessment ( Opus and GraspOS projects) and on the impacts of OS ( PathOS project). The UNESCO has a Working Group to monitor the implementation of its 2021 Recommendations on OS ( UNESCO, 2021 , p. 33) for the 193 countries that signed it 5 and has published its first outlook of OS activities in December 2023 ( UNESCO, 2023 ). Monitoring Open Science poses major challenges. Different stakeholders hold disparate understandings of OS, often associated with different conceptual models and interests about what science is and aims to accomplish. Most national and institutional policies on OS are recent and thus changes are incipient, with a degree of engagement that varies largely across geographies, disciplines and topics. Some developments are hotly contested, since they run against some of the principles of equity and inclusion espoused by OS, such as open access of publication via Article Processing Charges (APCs) ( Ross-Hellauer et al., 2022 ). While the dimensions of open access (OS) publications and open data (OD) sets are covered, many other relevant activities, such as engagement with societal actors, are seldom covered and others, like open educational resources, are just beginning to be scanned in a few pioneering countries. There is the danger that this lack of monitoring attention has a ‘street-light effect’ against the adoption of these activities ( UNESCO, 2023 ). Yet in our view, the main challenge is that ‘monitoring OS’ also requires a transformation of what ‘monitoring science’ means, since the shift towards OS implies a deep transformation of science itself. In this paper, we propose that this change in the understanding of monitoring cannot be a replacement of old output indicators with new output indicators, as it is often assumed. Instead, new conceptual frameworks and practices for monitoring are required that align with the conceptual shifts associated with OS. We suggest that conceptual frameworks from recent work on evaluation of Transformative Innovation Policies can be useful for thinking how to monitor OS ( Haddad & Bergek, 2023 ; Molas-Gallart et al., 2021 ). These frameworks propose to shift efforts towards understanding monitoring as learning, as awareness of values and directionality, with more focus on processes and outcomes. This study is prompted by our engagement with the UNESCO Working Group on Monitoring OS ( UNESCO, 2023 ). Therefore, it aims to gain insights on the overall progress of OS across the world, which results from a large variety of national and institutional policies and social developments, rather than assessing specific initiatives. Given that UNESCO’s OS framework is broader and more explicitly normative than many previous OS policies, we bring to the fore the need to monitor OS of transformative change at a global scale. A number of OS monitors are already in place. The most relevant feature is that they tend to focus on the outputs of research (publications, data, software, etc.) that are accessible from digital platforms. While this is important, we propose that monitors should embrace an understanding of OS as a transformation of the research systems, and therefore encompass a broader range of dimensions, which include: • policies for OS. • OS outputs such as publications and datasets, • OS processes such as increasing collaborative practices (including public engagement) and evaluation approaches in line with OS, • OS outcomes such as increased attention by researchers to societal problems, or evaluations that don’t discriminate by gender or social group, • impacts such as the values highlighted in the list of values of UNESCO OS Recommendation: integrity, inclusion, epistemic diversity, or identifiable contributions for addressing of societal problems. We first discuss the diversity of activities and the plural (and sometimes conflicting) understandings of OS. Second, we make the case that OS is a systemic transformation of research. Third, we review how models of science are associated with particular monitoring frameworks. Fourth, we describe concepts that can be useful for monitoring OS as a transformation. In particular, we propose: • to broaden out the monitoring OS from a focus on scientific outputs (e.g. datasets), to processes (e.g. participatory events), outcomes (e.g. better public engagement) and impacts (e.g. contributions to the achievement of SDGs); • to adopt a learning approach to monitoring , aimed at informing future-oriented strategies, i.e. helping navigate and make choices among options within OS; • to consider directionality, i.e. to open up the monitoring so as to map the different trajectories within each OS dimension (e.g. different colours of the routes to OS), as well as their potential effects and associated values. In the conclusions we discuss the implications for this perspective for current OS policies. 2. What is Open Science? A plurality of views on a diversity of activities A first difficulty in monitoring Open Science is the vast diversity of activities that it encompasses, from Open Access (OA) publishing to transparency and citizen engagement 6 . It is often said that OS is an umbrella term or better, a mushroom (as illustrated in Figure 1 ), in the sense that it consists of many visible practical activities (often digital) and that it also concerns, below its surface, many underlying institutional issues such as integrity, infrastructure and evaluation. Figure 1. Open Science mushroom, illustrating the diversity of activities and issues involved. Legend: This figure was drawn by Judit Eva’s Fazekas-Paragh based on Eva Méndez (2021, p. 5) . Source: https://www.openaire.eu/blogs/hungary-on-the-move-1 . The programmatic definition of the European Commission (2016 , p. 33), highlighted two aspects: first, the mainstream focus on (digital) access and collaborative activities and, second, the expectation that it would transform relations between science and society: ‘Open Science represents a new approach to the scientific process based on cooperative work and new ways of diffusing knowledge by using digital technologies and new collaborative tools. (…) allowing end users to be producers of ideas, relations and services and in doing so enabling new working models, new social relationships and leading to a new modus operandi for science.’ (see ( Leonelli, 2023 , p. 18). A definition of OS with these two sides is often found: on the one hand sharing or collaborative (digital) practices; on the other hand an explicit reference to the societal benefits in terms of participation and inclusion ( Arza & Fressoli, 2017 ). Normative commitments to equity and inclusion have been common in the OS movement as launched in 2001 in the Budapest Open Science Initiative ( Ross-Hellauer et al., 2022 , p. 3). Nevertheless, definitions and policies have been diverse and ambiguous, depending on the underlying political programmes. For example, whereas many organisations in Latin America highlighted the role of OS for advancing science as a public good ( Babini, 2019 ), policy adoption in the European context initially viewed the benefits of OS in terms of improvement in quality, efficiency, optimization, integration and potential of science ( Shelley-Egan et al., 2020 , p. 11), in a way that emphasized ‘market principles of competition foregrounding its economic role in training the workforce and fostering new products and services’ ( Ross-Hellauer et al., 2022 , p. 18). To put some light in this diversity, Fecher and Friesike (2014) proposed five main ‘schools of thought’ (narratives) depending on the focus and goals of OS: infrastructure (platforms), pragmatic (efficiency), public (access), measurement (evaluation) and democratic (free knowledge) schools, as shown in Figure 2 . 7 Figure 2. Five schools of thought on Open Science, according to ( Fecher & Friesike, 2014 ). There is a strong contrast between the initial aspirations towards global equity of the OS discourse, and the narrower policies on access. Ironically, these policies have actually resulted in greater inequalities in some dimensions, in particular in terms of OA and APCs. This dissonance has led some OS advocates to state that, for all the claimed benefits, the current OS model has not made science more inclusive. Instead, it is still keeping many scientists underrepresented; new technologies exclude those with limited digital rights; and citizens seldom can shape research agendas (see review by Ross-Hellauer et al., 2022 ). The Manifesto of the Open and Collaborative Network (OCSDNet) ( Albornoz et al., 2020 ) is an example of this perspective. In recent years, one can observe policy trends towards a broadening of the OS agenda from more technical aspects of accessibility (e.g. OA and OD) towards more process-oriented aspects (e.g. citizen science and evaluation) and values (diversity and inclusion, the distribution of research benefits). The International Science Council (ISC) has now defined OS as ‘best characterised as the necessary transformation of scientific practice to adapt to the changes, challenges and opportunities of the 21st century digital era to advance knowledge and to improve our world. (…) One of the purposes of Open Science viewed as a call for transformation, is to ensure that ‘no-one is left behind” ( ISC, 2020 , p. 2). Also the European Commission states in its Open Science Portal that ‘when partners from across academia, industry, public authorities and citizen groups are invited to participate in the research and innovation process, creativity and trust in science increases.’ 8 Some critical voices stress this shift from an understanding of OS as the ‘possibility of access’ towards an understanding of OS as the ‘process of connection’ in accordance with values related to integrity and epistemic justice. Chan proposes that ‘the ability to participate, to connect, and to co-produce knowledge with others who share common concerns is far more important than simply access to content or resources’ ( Chan et al., 2020 , p. 2). Leonelli argues that ‘… from being solely a question of sharing resources, openness is thereby conceptualized as the opportunity to make and maintain connections among relevant stakeholders (…) in ways that help to develop ever more relevant forms of interaction with the world’ ( Leonelli, 2023 , p. 66). Ross-Hellauer has stressed that enabling access is not enough to foster equity in science in the face of huge disparities in resources ( Ross-Hellauer et al., 2022 ). The UNESCO recommendation on OS is the first internationally agreed definition of OS ( UNESCO, 2021 ). As a consensus document it exemplifies the breadth, ambiguity and transformative nature of OS discourses. This document made explicit two further steps. First, as illustrated in Figure 3 , it highlighted the diversity of activities related to OS, particularly by giving more prominence to issues such as open engagement and dialogue with non-academic and marginalised actors (process-oriented practices), as already highlighted by the OECD ( Dai et al., 2018 ). Second, as shown in Figure 4 , it made explicit the values and principles associated with OS. Some of these values and principles are related to rigor in scientific practices (quality, integrity, transparency, reproducibility, etc.). Others are more related to notions of justice within science (equity and fairness), and some more associated with the properties of the social contributions (diversity, inclusiveness, responsibility, participation) and their impacts (collective benefits). Many of these values are shared by other high level STI policy formulations, even by organisation’s which had traditionally focused on economic impacts, as illustrated in the OECD’s recent report ( OECD, 2023 , pp. 87–121). Figure 3. Illustration of the areas of concern mobilised by the UNESCO Recommendation on Open Science. OS is seen as having four pillars: 1) Scientific Knowledge (publications, data, educational resources, software, hardware); 2) Infrastructure (virtual and physical); 3) Engagement of societal actors (citizen and participatory science, volunteering, crowdsourcing, crowdfunding); and 4) Dialogue with other knowledge systems (indigenous peoples, marginalised scholars and local communities). Source: UNESCO (2021) . Figure 4. Values and principles mobilised by the UNESCO recommendation on Open Science. Source: UNESCO (2021) . Both OECD and UNESCO’s comprehensive perspectives allow different stakeholders to highlight those activities and values which are most precious to them. However, this colourful palette also reveals that stakeholders hold a plurality of views on OS, with a battery of disparate activities. In consequence, its monitoring framework has to embrace this diversity of activities in a pluralistic way for its application in different contexts. 3. Open Science as a transformation of the research system While stakeholders may not agree on the full range of activities or values of OS, a recurrent theme in OS discourse is that Open Science is about a transition or transformation of the research system ( Shelley-Egan et al., 2020 ). The transformation is seen as a more collaborative way of doing research, partly driven by the application in science of digitalisation and associated information and communication technologies ( Nielsen, 2011 ). However, it is also related to the perception of a major change in the contract between science and society, as discussed by various science policy discourses, including Mode-2 and transdisciplinary research ( Gibbons, 1999 ; Nowotny et al., 2003 ), Responsible Research and Innovation (RRI) ( Owen et al., 2012 ; Shelley-Egan et al., 2020 ), mission-oriented research ( Mazzucato, 2018 ), Transformative Innovation Policies ( Loorbach & Wittmayer, 2024 ; Schot & Steinmueller, 2018 ) and science for SDGs ( Ciarli, 2022 ). In all these discourses, there is a call for transformative changes that are related to the changing relationship between science and society ( Shelley-Egan et al., 2020 , p. 11). Let us then take OS as transformation of the research system in all its dimensions: social, epistemic, institutional, organisational – even geographical 9 . Then, OS would entail the evolution of the current research system towards a future research system in which free information flows and collaborative activities would play a more prominent role than they play nowadays. However, if we think of science as an evolutionary system with many-fold potential futures there is not one, but many potential OS futures, depending on the direction of change. This plurality of potential futures reflects the ambiguity of the concept and implies that the transformation it entails is open-ended, with different potential but uncertain futures ( Stirling, 2011 ). Thus, the direction of this transformation will be given by the relative importance or the balance of efforts put into the different OS activities. A concentration of efforts around certain activities reflects a particular school of thought, and a particular model of what OS should be. From this evolutionary perspective, the key question is not if there is more or less OS, but which type of OS progresses, i.e. towards which type of (OS) practices the research system is being pushed and it evolves. If one accepts that OS is a transformation of the science system, then the monitoring should aim at capturing the directions of transformative change. In the next section, we examine how changes in the models of science have been historically associated with reformulation of monitoring systems. 4. New models of science require new monitoring frameworks Monitoring has the aim to check, show and justify that policy interventions, activities or organisations deliver the desired outcomes and impacts. Explicitly or not, monitoring and evaluation systems, as well as associated indicators, are derived from conceptual models on how some policies and activities lead to change towards the expected outcomes and impacts. These models guide the search of the properties that should be monitored and where possible measured ( Smith, 2004 ). If the models and systems under observation seriously evolve, the tools and methods of description also need to undergo a transformation. Yet monitoring tools and indicators are applied through existing (and thus old) institutional channels. As a result, efforts to capture new systemic changes are always difficult to adopt as they pose conceptual, technical, organisational and institutional challenges: institutions have a strong inertia to monitor according to past frameworks. However, at some point monitoring and classification systems evolve in accordance with social transformations. For example, the logics of wine classification in bottle labels shifted from specific territory ( terroir ) (as traditionally done in France) to wine grape varieties ( cépage) as a strategy by Californian companies in a context of globalisation in which the drinkers know little about local wine regions. Or dictionaries of commerce radically changed their classification of trade activities from the 18 th to the 19 th century, when the industrial revolution reconfigured the understandings of manufacture ( Douglas, 1986 , pp. 91–109). Let’s apply this idea of changes in monitoring to changing models of science. Schot and Steinmueller (2018) have proposed that there have been three main conceptual models about policies for science, technology and innovation (STI): the linear model (frame 1), the innovation systems model (frame 2), and more recently a model of transformative research and innovation policies (cf. Haddad & Bergek, 2023 ). The linear (‘science-push’) model assumes an unproblematic flow from R&D investments to scientific knowledge, to technological inventions, and to market innovations that will eventually result in economic growth and increased well-being. Therefore, monitoring in policy frame 1 is conducted focusing on R&D inputs (investments) and S&T outputs. The OECD’s statistical manual for R&D data collection, known as the Frascati Manual ( OECD, 2015 ) is the main instrument to collect R&D investments. Scholarly publications are conventionally used as proxies for scientific output (the supply side), and patents as proxies for technological output 10 . The linear model became increasingly questioned when it was realised that some countries (such as Japan in the 1980s) and organisations were very successful at innovation without being leaders in formal R&D. Publications and patents allow fine-grained analyses of the S&T contents, but they are poor in terms of tracing how and where this knowledge is used. The perspective of the innovation systems model (frame 2) shifts the analysis towards the interactions between actors and the process of innovation in the firm ( Kline & Rosenberg, 1986 ; Smith, 2004 ). Here, the assumption is that innovation is best achieved when knowledge flows across academic organisations and industry. Since these processes of knowledge exchange were not captured in formal R&D measured by frame 1 statistics, the so-called Community Innovation Surveys (CIS) were developed to gather information directly from companies on how they create and introduce innovations. This effort consolidated in 1992 in the OECD’s Oslo Manual for innovation data in a long process that involved testing, trials-and-errors and learning through the engagement of policymakers and researchers ( Arundel & Smith, 2013 ; OECD & Eurostat, 2018 ). The European Innovation Scoreboard of the European Commission is another example of a monitoring framework based on the innovation system perspective. In this case, the approach takes a broad battery of about 25-30 indicators that describe different aspects of an innovation system, including education of human resources, attractiveness of research systems, R&D investments, innovation data (from CIS), firm and academic collaborations, and employment and sales impacts, among others. We can take several lessons from these previous monitoring frameworks. First, their development takes time and is achieved through the creation of communities of practice at the interface of policy and research ( Godin, 2004 ). Second, monitoring frameworks are adjusted to adapt to evolving conceptualisations 11 . Third, the focus of monitoring changes in accordance with the logics at the heart of each model: from inputs and outputs of formal research activities (frame 1) to the network of knowledge activities and introduction of market innovations (frame 2). In consequence of this drift in location, the new frameworks do not substitute but rather complement previous ones. In recent decades, the idea that more innovation would always lead to more societal benefits has become increasingly questioned. It is now widely accepted that innovation in itself can lead to both desirable and undesirable societal impacts for example in terms of either lack of attention or negative impacts on health or the environment ( OECD, 2023 , p. 87). As a response, a third model of innovation policy has been gradually emerging, with some STI policies explicitly directed at supporting transformative change towards desired goals in specific sectors such as food, health or mobility systems. This frame 3, known as Transformative Innovation Policies, shifts the focus towards innovation for transformative change in socio-technical systems such as those of energy, mobility or health, etc. particularly to address societal challenges such as climate change, urban sustainability, aging society ( Schot & Steinmueller, 2018 ; Weber & Rohracher, 2012 ). In frame 3, co-creation and transdisciplinary approaches are crucial to align research with societal needs. Let us now take science as a socio-technical system in itself, the one concerned with the production of knowledge. If we then think of OS as a transformative agenda of this socio-technical system, as argued in the previous section, then OS policies can be understood as transformative policies for science. We can then apply the theoretical developments on Transformative Innovation Policies to OS. It might be argued that the research system differs from health, mobility or food systems in that it is not a tangible societal need. Still, the research system produces ‘knowledge’ for which there is a social demand, and then OS initiatives can be seen as innovations in the (socio-technical) system of knowledge production ( Loorbach & Wittmayer, 2024 ). Let us then examine the recent insights on STI evaluation and monitoring from a frame 3 perspective, following the recent publication of a string of contributions ( Haddad & Bergek, 2023 ; Janssen et al., 2022 ; Kofler & Wieser, 2023 ; Molas-Gallart et al., 2021 ; Rohracher et al., 2023 ). In summary, we find that these studies (actually more focused on evaluation than on monitoring) suggest a shift in focus towards: 1. A systemic perspective of the various activities related to OS, including policies and outputs (e.g. software) but also processes (e.g. new evaluation methods in line with OS), outcomes (more inclusive recruitment of researchers) and expected impacts (e.g. scientific contributions to societal problems so far non-addressed); 2. Learning with regards to strategic choices and reflexive evaluation, with participatory processes and mixed methods if possible; 3. Directionality of the activities and the values associated with the choices in directions. Since the purpose of monitoring in this frame 3 is not accountability or benchmarking, but learning (i.e. informing future strategy for specific different contexts), the methods and tools for conducting the monitoring do not need to be unidimensional indicators. Instead, new methodological tools can be created, for example showing collaborative networks, science maps or other visualisations that ‘open up’ reflexive thinking on the OS trajectories to be pursued ( Barre, 2010 ; Ràfols, 2019 ) 12 . In the next three sections we explore how these concepts could be applied in the context of a general OS monitoring, this is, a monitoring which aims to promote OS by capturing overall trends affected by a variety of (OS) policies, rather than assessing specific transformative programmes or policies. 5. Towards a systemic monitoring: broadening the palette towards processes and outcomes Most initial initiatives towards monitoring OS have focused on describing whether research outputs such as publications, datasets or software are publicly accessible. Here we argue that a broader palette is needed for OS monitoring, for two reasons. First, because OS is defined as a broader set of activities than just outputs (which are the first pillar in Figure 3 ). Second, because public access to outputs in itself does not mean that the information or knowledge can be mobilised. As Michel Callon argued, science is not a conventional ‘public good’ because to make fruitful use of it, the potential beneficiaries need to invest or have invested previously in developing capabilities ( Callon, 1994 ). This is why firms invest in ‘absorptive capacity’, to ‘be able to recognize the value of new, external information, assimilate it and apply it’ ( Cohen & Levinthal, 1990 , p. 128). By focusing on accessibility without considering the capabilities needed to use information, there is the danger that outputs will not be utilized. Thus, policies have been developed to foster knowledge exchange, both within science (interdisciplinarity), and with specialised stakeholders (translational efforts, transdisciplinary projects and transfer offices) ( Molas-Gallart et al., 2016 ). Since open outputs will mainly benefit those organisations or territories with absorptive capacities, generic open outputs from science may sometimes reinforce inequalities (see discussion on OD in the Global South ( Bezuidenhout & Chakauya, 2018 ; Bezuidenhout et al., 2017 )). Development and innovation studies have documented that in many middle and low- income regions, science has not generated local benefits due to lack of local learning capabilities or absorptive capacity ( Arocena & Sutz, 2010 ; Bell & Pavitt, 1995 ). Yet, as Leonelli argues, genuine research openness is not about accessibility but about developing relevant interactions with the world ( Leonelli, 2023 , p. 66). An open publication (or dataset or software) may be formally accessible but created and communicated in such a way that it does not reach its potential beneficiaries. Either within or beyond academia, explicit efforts have to be made to search for users, to learn about their needs (participatory agenda setting), to engage them (co-creation), and to communicate to them the knowledge in meaningful ways (outreach). Therefore, a systemic perspective of OS also needs to track in what ways knowledge is created and used: whether it is generated and used through plural collaborations (processes) involving learning to and from stakeholder engagement (outcomes) and leading eventually to some social contributions (impacts). Therefore, we propose that a systemic perspective would include a description of OS policies (enabling conditions) and OS outputs (e.g. publications), but crucially it should also include a monitoring of OS processes (e.g. public engagement, dialogue), outcomes (e.g. policy uptake) and, where possible, broader impacts (e.g. contributions to societal goals). This broadening of the monitoring to the full palette of scientific activities should allow to see changes towards more inclusive research processes, and whether the accessible scientific outputs are actually accessed and put to social use. Such a breadth is consistent with the ‘opening of monitoring’ and the ‘people-centred’ (process) framework suggested by the UNESCO OS Outlook ( UNESCO, 2023 , pp. 27, 30). And this is important to avoid a streetlight effect ( Davies et al., 2021 ; Molas-Gallart & Rafols, 2018 ). Figure 5 and Table 1 aim to summarise our proposal for a monitoring framework that is comprehensive and systemic, including the various aspects of OS: policies, outputs, processes, outcomes, and impacts. It should be noted that the distinction between processes, outcomes and impacts depends on the underlying evaluation theory, i.e. the specific theory of change on how the policy intervention achieves its goals. Since we are discussing OS policies in general, there is a degree of ambiguity regarding which actions can be considered processes, outcomes and/or impacts. Figure 5. Different aspects of OS to be monitored: policies, outputs, processes, outcomes and impacts. Table 1. Main aspects of OS monitoring, with examples of dimensions and monitoring efforts already conducted. Aspect of OS Examples of dimensions Example of monitoring Policies … on OA publications EOSC Observatory , national … on incentives rewards for OS EOSC Observatory , national … on engagement EOSC Observatory , national Institutional repositories OpenDOAR , national Outputs Publications in OA COKI Open Access Dashboard PhD theses French OS Monitor Open datasets DataCite Open software French OS Monitor , Extracted from ORCID * Processes Open peer review Graz U. survey ( Ross-Hellauer et al., 2017 ) Public engagement SuperMoRRI, Bath survey ( Lawson et al., 2019 ) Sharing of research materials Tokyo U. survey ( Shibayama et al., 2012 ) Integrity practices Aarhus U. survey ( Schneider et al., 2023 ) Outcomes Policy utilization Graz U. survey ( Cole et al., 2023 ) Fair evaluation in gender See review in Sugimoto & Larivière (2023) Impacts Contributions of research to SDGs Sussex U. analysis ( Ciarli, 2022 ; Purnell, 2022 ) Territorial distribution of research Banaras Hindu U. ( Singh et al., 2021 ) Language diversity COKI analysis Ethnic and gender distribution in the scientific workforce U. Luxemburg ( Kozlowski et al., 2022 ) * https://datacite.org/ see UNESCO Outlook ( UNESCO, 2023 , p. 40). For example, stakeholder engagement can be interpreted as a research practice (a process in itself), a change in a practice (and thus an outcome of the OS policy), and it may translate as goal in itself (an impact towards inclusion). We don’t think this ambiguity is problematic, as far as each dimension is transparently defined in a specific monitoring context. Thus, from a more nuanced perspective, the specific instances of monitoring can contextualise the aspect of stakeholder engagement captured, making the distinction between counting a given participatory event (a process), reporting of stakeholder learning during participation (an outcome), or reporting of inclusion of marginalised communities (an impact). 5.1 Current focus of monitoring: policies and outputs The monitoring of enabling conditions refers to the description of dimensions that will facilitate or hinder the progress of OS. This comprises policies and institutional initiatives aimed at fostering OS, including specific OS action plans (e.g. infrastructure investment, training for FAIRisation of data) as well as transversal policies with major effects in OS (e.g. evaluation reform). The EOSC Observatory provides an example of policy monitoring at the EU level. The Finish OS monitor has designed a survey to institutions about their practices to support OS. The first step in UNESCO OS monitoring has been the development of a survey for the Member States to report their policies for promoting OS. 13 These surveys provide a solid ground for exploring institutional policies and commitments to OS. The tracking of OS outputs in the form of OA publications or Open datasets is the most established monitoring practice. Bibliometric databases allow to conduct fine-grained analyses of the growth in the accessibility of the scientific literature. A problem is the uneven coverage of most commercial databases of across countries, disciplines and language, which marginalises knowledge relevant some world regions and communities ( Mongeon & Paul-Hus, 2016 ). Initiatives are under way across continents to address this issue ( Maricato et al., 2023 ). The French OS Monitor offers an example of the feasibility of rigorous analysis with highly comprehensive coverage using open scholarly infrastructure. Initiatives that make this tracking possible deserve to be valued and supported: for example, National CRIS systems like the Norwegian ( CRISTIN ), regional bibliometric databases like SciELO (Latin America) and J-Stage (Japan), or portal for local OA scholarly journals (such as Catalonia’s raco.cat ). The tracking of open research data, open educational resources, open software, and open hardware is patchy but illustrative of growing efforts ( UNESCO, 2023 , pp. 40–42), although the estimates of numbers of datasets, scripts or courses as general indicators is problematic given the large difference in size and efforts invested per unit. 5.2 Underdeveloped dimensions of monitoring: processes and outcomes Broadening out the palette of OS activities While counting outputs is a key monitoring tool, we should bear in mind that open outputs do not always lead to more OS. First, because, as argued above, since openness is situated and the capabilities of beneficiaries are unequal, potential access does not necessarily translate into the possibility of reaching, understanding and mobilising the knowledge ( Callon, 1994 ; Chan et al., 2020 ). Also, because there can be instances in which the conventional ‘open’ routes of the outputs may not be associated with an increased flow of knowledge. For example, for a political scientist in Sweden publishing an opinion piece in Sweden in a newspaper behind paywall or in the local grey literature may achieve more readers and influence (in this sense more academic ‘openness’) than publishing an article in an OA academic journal in the US ( Salö et al., 2023 ). Or for an agricultural engineer in Colombia, publishing in Spanish in non-OA professional newsletter may be more important to reach her stakeholders (farmers) than publishing in English in a European OA academic journal ( Chavarro et al., 2017 ). For the particular case of policy influence of OS practices, Cole et al. (2023 , p.1) study found that: ‘…there is little evidence that Open Research products, namely Open Access and open data (…) are useful in integrating science into policy-making. Instead, we found that the cognitive accessibility of research outputs is more important than their physical accessibility, and that inclusive and collaborative Open Research processes, like upstream engagement, co-creation and Citizen Science, are most effective at doing so.’ Processes of interactions such as collaborations and engagement are seen as separate dimensions to accessibility, and leading to different types of benefits ( Arza et al., 2017 ). This is why, in the UNESCO framing, as depicted in Figure 3 , OS involves not only outputs (in green (1) pilar), but also research processes: in terms of collaboration in infrastructures (pink (2) pilar), stakeholder and citizen participation (orange (3) pilar), and dialogue with other knowledge systems (blue (4) pilar). These processes should also be monitored to assess progress in OS. Unfortunately, according to the recent UNESCO OS Outlook, they are poorly captured by OS reports at the moment: new methods need to be developed ( UNESCO, 2023 ). Having made the argument that OS is a transformation of the research system, it is also important to monitor the changes in practices and behaviours induced by OS, i.e. the outcomes. Indeed, proposals for monitoring Transformative Innovation Policies agree that monitoring should focus on outcomes; that is, on assessing the ‘changes in the behaviour, relationships, activities, or actions of the people, groups, and organizations (…)’ ( Molas-Gallart et al., 2021 , p. 435). Monitoring outcomes concerns questions such as how Open Data is re-used and by whom, and how public engagement influences research agendas. One first approach to monitoring processes and outcomes is to approximate them through some relevant characteristics of the scientific outputs; for example, estimating frequency and type of exchanges with policy, industrial and civic organisations through formal co-authorship in publications ( Tijssen, 2006 ), or through the mention of scientific publications in social media or policy documents ( Díaz-Faes et al., 2019 ). But whereas many studies have looked at whether OA leads to more citations, let us emphasise here that for the benefits do not just lie on the amount of interaction and usage, but on the distributions across epistemic and geographical spaces. For example a recent study has reported that OA publications (even more if green OA) are cited in more diverse ways: they garner citations from more territories (countries and regions) and more disciplines ( Huang et al., 2024 ). This is a good example of an outcome (a change induced by OA: broader reach) aligned with the value of making science more diverse. The need to gather new data sources and surveys While analyses of usage of outputs are worth conducting and provide some information, qualitative studies have concluded that interactions between social actors and academics are extremely diverse ( Molas-Gallart et al., 2002 ). As a result, traces of interaction through some channels tend to be specific (biased) to certain socio-economic sectors and issues. For example, with some scientific issues (e.g. food consumption) are often mentioned in social media, while other (e.g. food production) receive more mentions in policy documents, and still others (e.g. agricultural methods) are only shared across professional information channels ( Noyons & Rafols, 2018 ). Moreover, many exchanges between academic knowledge and professions are not conducted through scientific publications, but through direct interactions and grey literature reports, often in the local languages ( Salö et al., 2023 ; Thune et al., 2023 ). In conclusion, it is generally not possible to capture knowledge exchange trends reliably by using only traces left by conventional scientific outputs in digital platforms. In order to know how researchers share and use materials and data, how they interact with non-academics, etc., it is necessary to collect new data. Surveys are an obvious source of information. Some of the OS pilot monitors have started to conduct surveys. For example, the Finish monitor has proposed to conduct biannually a survey especially on ‘the development of services, policy documents, research assessment and culture of open scholarship’. There is a long research experience of conducting surveys in order to understand the knowledge exchange practices in relation to a variety of collaborative practices ( Díaz-Faes et al., 2023 ), modes of knowledge exchange with industry ( D’Este & Patel, 2007 ), exchange with policy ( Thune et al., 2023 ), innovation practices in industry through the Oslo Manual ( Bloch, 2007 ; Gault, 2020 ; OECD & Eurostat, 2018 ), innovation in the public sector ( Arundel et al., 2019 ), and practices of Responsible Research and Innovation (RRI) ( Holtrop et al., 2022 ). Some studies aimed at monitoring OS processes also relied on surveys: for example on sharing of research materials ( Shibayama et al., 2012 ), open peer review ( Ross-Hellauer et al., 2017 ), policy use ( Cole et al., 2023 ), engagement with non-academics ( Lawson et al., 2019 ), integrity ( Schneider et al., 2023 ), or broader perceptions and habits of OS ( Ollé et al., 2023 ). To our knowledge, Arza et al. (2017) conducted the only survey looking into the multiple dimensions of OS, showing differences by field and region. 5.3 Monitoring potential impacts Finally, there is the monitoring of the benefits of IS. It is common in evaluation theory to distinguish between ‘outcomes’ (changes in behaviours, practices, organisations directly induced by a policy intervention, e.g. more stakeholder participation), and ‘impact’ (referring to the ultimate changes that the intervention pursues such as societal benefits derived from scientific activity or gender balance) ( Molas-Gallart et al., 2021 ). Notice that in the UNESCO framing, the impacts refer the values expressed in Figure 4 , which can be considered goals in themselves: more collective benefits, more equal, diverse and inclusive science. 14 We argue that the ways in which outputs, processes, outcomes and final impacts are related to each other should be mapped. In this way, monitoring may capture whether the evolution of the research system leads towards desired directions. This is important, because in a complex system one cannot assume that achieving the desired outputs (e.g. more open datasets) will lead to the desired outcomes (e.g. broader societal participation in specific research area) and desired impacts (the achievement of the collective benefits). Moreover, since the impacts are multifactorial, even if OS is pushing in a desired direction, other larger factors may counter the positive effects – and complementary policies may need to be considered. For example, evaluation reform may be needed to stimulate data sharing, because good open data repositories on their own may not be enough to trigger such behavioural change. As potential dimensions of impact of OS, we propose those dimensions in the values of the UNESCO OS Recommendation, such as contributions to SDGs (collective benefits), distribution of research across countries (geographical diversity), distribution of research across languages (linguistic and cultural diversity), and balance in gender and ethnic group representation in scientific institutions (social inclusion). 6. Learning: reflexive monitoring for supporting strategic decision-making Various studies have proposed that the learning function of evaluation should be emphasised in Transformative Innovation Policies ( Janssen et al., 2022 ; Molas-Gallart et al., 2021 ; Rohracher et al., 2023 ). ‘Reflexive monitoring and evaluation’ has emerged as a specific approach that distinguishes itself from more common ‘result-oriented’ evaluations by considering learning how to contribute to system innovation the central goal of evaluation. ‘Result-oriented approaches’ focus on accountability and steering, and on a set of predefined objectives, while ‘reflexive monitoring and evaluation’ put ‘the prevailing values and institutional settings up for discussion’ ( Van Mierlo et al., 2010 , p. 36). Kofler and Wieser (2023) have suggested the metaphor of ‘reflexive navigation system’ to convey the idea of monitoring for learning through engagement with stakeholders. The monitoring thus designed ‘organises information and knowledge flows, offering stakeholders an overview of achievements and challenges’, and this is used in expert panels in ‘regular exchanges and inclusive conversations with decision-makers’. The idea is that the main use of monitoring should shift from accountability towards fostering thinking over strategic choices. In sociotechnical transformations there is great uncertainty regarding which of the many potential futures will be reached. This means, for example, that there is not one single trajectory or pathway to monitor, but a gamut of possible trajectories of which only one will be realised ( Stirling, 2011 ). Thus, in a transition towards a low-carbon energy technologies, a range of disparate energy portfolios is possible in a given territory or community (some dominated by wind-turbines, others by photovoltaics), with different environmental, security and economic consequences. Lessons learned from monitoring can facilitate debates towards re-balancing the portfolio in strategic ways, as exemplified in the RIPEET project . Similarly, OS monitoring can facilitate this type of reflexive learning. For example, a given organisation has the potential to invest relatively more or less efforts across the activities showed in Figure 3 , e.g. on how much resource goes to open data vs. engagement or collaborative infrastructure. OS monitoring of institutions can show that they have strengths and weakness over a portfolio of OS activities – and thus help reflection on developing strategies towards addressing these. The question is not whether the organisation has more or less OS (an accountability logic), but what types of OS it has and with what consequences in relation to its goals (a learning logic). When monitoring aims to have a learning component, it should be comprehensive and contextual about the activities carried out (or not) (as discussed in the previous sections) and allow to reflect on the various available options. This type of information then empowers stakeholders to express preferences and make relevant normative choices. As put by the in the UNESCO OS Outlook, ‘[c] onsidering science as a global public good, it is important not to reduce open science to a few standardized metrics monitored in a top-down approach. Masking variations, particularly those between and within countries, may undermine the transition to a genuinely open science, accessible to all and with benefits for all’ ( UNESCO, 2023a , p. 18). Given the current policy inclination for narrow monitoring focused on accountability or efficiency, re-purposing monitoring to support learning will require new monitoring designs and training of users. This means in particular, applying monitoring methods in such a way that they facilitate the use of indicators as ‘debatable devices, enabling collective learning’ through contextualisation and stakeholder engagement ( Barre, 2010 ; Ràfols, 2019 ). To do this, monitoring can be enriched by ‘broadening out’ the range of issues considered in a given dimension (vertical axis in Figure 6 ) and by allowing users to ‘open up’ the indicators to scrutiny under various conditional perspectives (vertical axis in Figure 6 ) ( Leach et al., 2010 ; Stirling, 2008 ). Ideally, these ‘indicators frameworks’ would need to change according to different monitoring purposes and contexts ( Wouters et al., 2019 ). Figure 6. ‘Broadening out’ (vertical) vs ‘opening up’ (horizontal) in monitoring for the case of Open Access. Legend: Framework of ‘broadening out’ (vertical axis: including more information in the analysis) and ‘opening up’ (horizontal: presenting an analysis allowing choices), based on Leach et al. (2010) . Source of graph with trends: UNESCO (2023 , p. 32). For example, for Open Access monitoring an initial analysis might just focus into the growth of the number of OA publications (top left of Figure 6 ) and conclude that the trends are positive. By ‘broadening out’ the type of issues to type of OA, the analysis opens up a new perspective by visualising the types of OA which are growing fastest (gold and hybrid) – thus raising critical questions. A further step would be to consider the geographical distribution of publications. In contrast to rigid ranking lists which fostered narrow interpretations, the development in recent years of monitoring systems based on interactive visualisation platforms with filtering possibilities, has greatly facilitated ‘unpacking’ of indicators, mappings or collaborative networks ( Rafols & Stirling, 2021 ). 7. Directionality: visualising normative choices over trajectories of science Directionality refers to the notion that science, technology and innovation (STI) systems follow certain trajectories (or pathways) rather than others. Evolutionary understandings have shown that STI systems develop along sociotechnical trajectories with path dependency ( Stirling, 2011 ). Different organisations or countries may follow alternative pathways. For example, Denmark committed to wind energy whilst France to nuclear energy as a result of divergent social and political commitments. Therefore, STI trajectories do not evolve over one dimension, but across a multidimensional space with a plurality of potential directions. There is a diversity of optional STI futures and different actors may favour different choices depending on their values and interests. A key insight from frame 3 of innovation policy ( Section 3 ) is that a specific STI system (e.g. mobility or energy) are locked-in in problematic pathways (e.g. due to pollution and CO 2 emissions) and changes need to be supported towards alternative sustainable trajectories (e.g. wind-turbines, or nuclear). ‘Directionality means that innovation policy will not just stimulate specific technological options, but will look into the social and environmental drivers and consequences of each option, then aim for a deliberation on desirable pol-icy directions and eventually foster some desired directions for innovation, while blocking undesirable ones.’ ( Molas-Gallart et al. 2021 ) To sum up, in any given transformation, the socio-technical trajectory is also associated with social commitments related to certain values. Monitoring should help in visualising alternative trajectories (wind vs. nuclear; green, diamond vs. gold OA). Where possible, the monitoring framework would inform of the broader social implications of each trajectory in terms of values, e.g. in alignment with SDGs, inclusion, distribution of benefits across stakeholders, etc. In the case of OS, the notion of directionality means that OS will follow different trajectories, depending on the choices made regarding the adoption (or not) of particular commitments (e.g. FAIR, CARE datasets). A frame 3 perspective helps understand why monitoring the directions is so important: these choices define the OS future against alternative (not adopted) trajectories, often with broad, and sometimes unexpected, societal consequences. Let us look for example, on how values are associated with different trajectories of Open Access in Figure 6 . One of the key rationales for supporting an OA trajectory was to foster inclusion of researchers from Global South in transnational research communities through access to all journals. However, in the case of the gold and hybrid OA trajectories, access to reading is being (partly) achieved through the use of Article Processing Charges (APCs). Since these APCs are not affordable to many researchers in the Global South, these trajectories become a major barrier for researchers without resources to publish in highly visible journals or to publish in OA in hybrid journals, thus enhancing inequity. In terms of outputs, these trajectories seem successful (they have higher % of Open Access articles), but the overall outcome goes against the value of inclusion. Olejniczak & Wilson (2020) show that men, older faculty, private and elite universities are more likely to publish in gold OA. This is a Lampedusian transition 15 in the sense that publishing practices change, but the incumbent actors (the large publishing corporations) preserve their interests and dominant position by using existing path dependency to shape the OS transformation towards those trajectories that favour them ( Larivière et al., 2015 ; Stirling, 2019 ). A monitoring strategy that is sensitive to value commitments, should not just look at the percentage of OA, but at the distribution of OA modes (green, gold, diamond, etc.) and its consequences in terms of equity, as illustrated in Figure 6 . Similarly, the apparently technical choices made on the implementation of Open Data (OD) can be analysed as representing different data sharing trajectories, with different ethical and social implications. On the one hand, data sharing is discussed according to the relative value of OD under different curation and infrastructure situations. This led to the proposal of the FAIR principles (Findable, Accessible, Interoperable and Reusable), with attributes that have implications in terms of the value of efficiency rather than fairness, in spite of its name ( Wilkinson et al., 2016 ). However, other framings like the CARE Principles for Indigenous Data Governance (Collective Benefit, Authority to Control, Responsibility, and Ethics) ( Carroll et al., 2020 ) put the values of equity and collective benefits at the centre, highlighting that there are major challenges on the usability of data by social actors ( Dosemagen & Williams, 2022 ), on data control and equity, and on the distribution of the benefits across Northern vs. Southern researchers ( Bezuidenhout et al., 2017 ) and stakeholders (e.g. communities endowed with genetic or natural resources). Given the implications of the different data sharing modes on OS values (as suggested by UNESCO), it seems relevant to include compliance or not with FAIR and CARE principles in a monitoring exercise. This means that, in terms of monitoring OD, the information should be provided with the particular OD trajectory (FAIR, CARE, both or other). A third example is the choice regarding the governance data infrastructures in terms of centralised vs decentralised/federated architectures, which have implications on the geographical balance of power across countries and organisations, data control or accessibility ( Bezuidenhout & Havemann, 2020 ). In summary, for each dimension of OS, there are alternative trajectories with different social implications. 8. Conclusions: ‘Opening up’ the monitoring of Open Science As we are writing this paper in early 2024, many governments and scientific institutions have adopted OS policies and are developing methods to monitor progress toward the goals of OS. We observe that there is a tendency to focus the analysis on growth along those dimensions of OS that can be more easily measured, generally via the analysis of publications. However, these early measurements show already that ‘progress’ (adoption of OA) is not necessarily aligned with some of the OS policy goals (increased equity and inclusion in science) ( UNESCO, 2023 ). Such choices have practical implications. Monitoring tools created by national governments and supranational organisations such as the UNESCO may appear to be far away from most OS research practices. However, one should not underestimate the influence of monitoring systems in shaping practices. Norms on how to develop and govern science are strongly influenced by (inter-)governmental institutions ( Finnemore, 1993 ) and practices on quantitative monitoring and evaluation constitute a key technology for governance ( Rottenburg & Merry, 2015 ). Traditional monitoring approaches have often suppressed diversity in research and its interactions with society in ways that run against the values of OS ( Rafols et al., 2012 ; Sugimoto & Larivière, 2023 ). In this article, we propose an approach to monitoring that aims to engage with the complexity and ambiguity of OS ( Fecher & Friesike, 2014 ), and is aware of the potential suppression diversity associated with streetlight effects. Our point of departure is the assumption that OS is a transformation of the research system. Therefore its monitoring will require to map dimensions of research that have not been considered relevant in previous science models. Building on insights from the evaluation literature of transformative innovation policies, we have proposed strategies for monitoring OS (systemic view, learning and directionality), as summarised in Table 2 (compare with Leonelli (2023 , p. 64, Table 3)). Table 2. Summary of the shift proposed in monitoring strategies for Open Science. Mainstream OS monitoring Proposed OS monitoring Evidence of OS transformation More open outputs, i.e. the products of research can be easily accessed Reconfiguration of research system, in particular, changes in processes and outcomes likely to lead to improved social contributions Systemic change Few dimensions Transformation in multiple dimensions Main OS concept OS as accessibility OS as relevant connections Focus of analysis Science supply side (objects): publications, datasets, software, educational resources Knowledge exchange processes and user/demand side (subjects): collaborations, engagement and dialogue events, usage of outputs Main purpose of monitoring Accountability and benchmarking Learning and strategy Notion of directionality Absent Directions as trajectories associated with normative preferences (i.e. choices in types of openness, e.g. green vs. gold OA), for each dimension Relationship to OS values and principles Absent or implicit Making explicit the relations between monitored variables and desired impacts as related to collective benefits, diversity, inclusion, equity and integrity First, we propose to ‘broaden out’ the dimensions to be monitored, adopting a systemic perspective which considers the various actions related to OS, including processes (e.g. participatory events), outcomes (e.g. use of knowledge in policy) and expected impacts (e.g. increased contribution to societal problems). This means going beyond the current focus on the science supply, on monitoring policies and outputs (e.g. datasets). Such a pluralistic perspective is warranted by an ongoing shift in the conceptualisation of OS from ‘accessibility to research outputs’ towards ‘judicious interactions’ ( Leonelli, 2023 ) both in the scientific practices and the societal usages of research. The type of information needed cannot be garnered only from research output data in digital platforms. Therefore existing data sources on outputs (objects) will have to be complemented with new data sources on researchers’ OS activities, especially interactive processes. Given that many collaborative practices and science-society exchanges are invisible in outputs or digital platforms, we believe that direct inquiries are needed, via surveys or qualitative approaches, to understand OS dynamics. Second, given the uncertainty associated with sociotechnical transformations, and the unexpected outcomes of transformative policies (as illustrated by increased publishing inequity generated by OA), monitoring should be oriented towards supporting reflective learning – rather than a focus on accountability and benchmarking. Monitoring should be understood as a navigation instrument that helps to set sails towards the desired goals in the face of hidden currents and shifting winds. Third, monitoring should be ‘opened up’ by making directionality visible. Since there a many potential futures in a transformation and contending visions, monitoring OS should not be only about tracking more or less OS, but about what trajectories of OS are developed and with what consequences. Different trajectories (e.g. green vs. gold OA; FAIR vs. CARE OD) are associated with different interests and values – and therefore lead to different OS futures. In order to map choices, it is necessary to adopt multidimensional monitoring tools and visualisations. A final consideration is the diversity of contexts of science and society. This means that flexibility (an OS principle in the UNESCO Recommendation) will be required to apply monitoring in different locations or organisations ( Wouters et al., 2019 ). The strength of a monitoring framework is its capacity to summarise a diverse world. But this can become a liability without a good dose of humility and reflexivity ( Stirling, 2023 , p. 5). As Leslie Chan reminds us: ‘… there is no single or universal concept of Open Science that is sufficient to encompass the diversity of knowledge traditions and practices from around the world. Hence the term Open Science and the notion of “openness” is highly situated, constantly subjected to negotiation according to local contexts and historical contingencies. Our collective observations therefore challenge the tendency to define Open Science as a set of technical infrastructure, workflow, protocols, and licensing conditions that can be universally applied regardless of context, history, and human agency.’ ( Chan et al., 2020 , p. 17) Just like the wealth of ecosystems in the Earth cannot be described by simply counting the number of individuals of each species, the transformation towards OS cannot be described by counting open outputs. Instead, the evolution of ecosystems is understood through the changes in the network of interactions among species in each ecosystem. Similarly, the evolution of OS needs to be monitored through the trajectories in the exchanges of scientists and citizens – learning and adapting dynamically in situated knowledge ecosystems. Data and software availability No data are associated with this article. Acknowledgements Earlier versions of this work were presented in the 2023 Eu-SPRI conference, the 2023 STI Indicators Conference, and as an LSE Blog post . We are grateful for fruitful exchanges with colleagues in these events, in particular in discussions in the UNESCO Working Group on Monitoring OS, with special thanks to UNESCO OS team (Ana Peršić, Fereshteh Rafieian and Tiffany Straza). We also thank Valeria Arza, Anestis Amanatidis, André Brasil, Louise Bezuidenhout, Diana Hicks, Thed van Leeuwen, Matías Milia, Alex Rushforth, Jonas Torrens, Vincent Traag, Harald Wieser, Iris Wanzenböck and Ludo Waltman for fruitful discussions. References Albornoz D, Hillyer B, Posada A, et al. : Principles for an Inclusive Open Science: The OCSDNet Manifesto.Chan IL, Okune A, Hillyer B, et al. , editors. Contextualizing Openness: Situating Open Science. University of Ottawa Press; 2020; pp. 23–50. Arocena R, Sutz J: Weak knowledge demand in the South: Learning divides and innovation policies. Sci. Public Policy. 2010; 37 (8): 571–582. Publisher Full Text Arundel A, Bloch C, Ferguson B: Advancing innovation in the public sector: Aligning innovation measurement with policy goals. Res. Policy. 2019; 48 (3): 789–798. Publisher Full Text Arundel A, Smith K: History of the community innovation survey.Gault F, editor. Handbook of innovation indicators and measurement. Edward Elgar Publishing; 2013; pp. 60–87. Arza V, Fressoli M: Benefits of Open Science: An Analytical Framework Illustrated with Case Study Evidence from Argentina. Expanding Perspectives on Open Science: Communities, Cultures and Diversity in Concepts and Practices. Proceedings of the 21st International Conference on Electronic Publishing. IOS Press; 2017. Arza V, Fressoli M, Lopez E: Ciencia abierta en Argentina: Un mapeo de experiencias actuales. Ciencia, Docencia y Tecnología. 2017; 55 : 78–114. Babini D: La comunicación científica en América Latina es abierta, colaborativa y no comercial. Desafíos para las revistas. Palabra Clave (La Plata). 2019; 8 (2): e065. Publisher Full Text Barre R: Towards socially robust ST indicators: Indicators as debatable devices, enabling collective learning. Res. Eval. 2010; 19 (3): 227–231. Publisher Full Text Bell M, Pavitt K: The development of technological capabilities. Trade, Technology and International Competitiveness. 1995; 22 (4831): 69–101. Bezuidenhout L, Chakauya E: Hidden concerns of sharing research data by low/middle-income country scientists. Glob. Bioeth. 2018; 29 (1): 39–54. PubMed Abstract | Publisher Full Text | Free Full Text Bezuidenhout L, Havemann J: The varying openness of digital open science tools. F1000Res. 2020; 9 : 1292. Publisher Full Text Bezuidenhout LM, Leonelli S, Kelly AH, et al. : Beyond the digital divide: Towards a situated approach to open data. Sci. Public Policy. 2017; 44 (4): 464–475. Publisher Full Text Bloch C: Assessing recent developments in innovation measurement: The third edition of the Oslo Manual . Sci. Public Policy. 2007; 34 (1): 23–34. Publisher Full Text Callon M: Is Science a Public Good? Fifth Mullins Lecture, Virginia Polytechnic Institute, 23 March 1993. Sci. Technol. Hum. Values. 1994; 19 (4): 395–424. Publisher Full Text Carroll SR, Garba I, Figueroa-Rodríguez OL, et al. : The CARE Principles for Indigenous Data Governance. Data Sci. J. 2020; 19 : 43. Publisher Full Text Chan L, Okune A, Hillyer B, et al. : Contextualizing Openness: Situating Open Science. University of Ottawa Press; 2020. Chavarro D, Tang P, Ràfols I: Why researchers publish in non-mainstream journals: Training, knowledge bridging, and gap filling. Res. Policy. 2017; 46 (9): 1666–1680. Publisher Full Text Ciarli T: Changing Directions: Steering science, technology and innovation towards the Sustainable Development Goals. University of Sussex; 2022. Publisher Full Text Cohen WM, Levinthal DA: Absorptive Capacity: A New Perspective on Learning and Innovation. Adm. Sci. Q. 1990; 35 (1): 128–152. Publisher Full Text Cole NL, Reichmann S, Ross-Hellauer T: The potential of inclusive and collaborative Open Research processes at the science-policy interface [Preprint]. SocArXiv.2023. Publisher Full Text Dai Q, Shin E, Smith C: Open and inclusive collaboration in science: A framework (OECD Science, Technology and Industry Working Papers 2018/07; OECD Science, Technology and Industry Working Papers.2018; Vol. 2018/07 . Publisher Full Text Davies SW, Putnam HM, Ainsworth T, et al. : Promoting inclusive metrics of success and impact to dismantle a discriminatory reward system in science. PLoS Biol. 2021; 19 (6): e3001282. PubMed Abstract | Publisher Full Text | Free Full Text D’Este P, Patel P: University–industry linkages in the UK: What are the factors underlying the variety of interactions with industry? Res. Policy. 2007; 36 (9): 1295–1313. Publisher Full Text Díaz-Faes AA, Bowman TD, Costas R: Towards a second generation of ‘social media metrics’: Characterizing Twitter communities of attention around science. PLoS One. 2019; 14 (5): e0216408. PubMed Abstract | Publisher Full Text | Free Full Text Díaz-Faes AA, Llopis O, D’Este P, et al. : Assessing the variety of collaborative practices in translational research: An analysis of scientists’ ego-networks. Res. Eval. 2023; 32 (2): 426–440. Publisher Full Text Dosemagen S, Williams E: Data Usability: The Forgotten Segment of Environmental Data Workflows. Front. Clim. 2022; 4 : 785269. Publisher Full Text Douglas M: How institutions think. Syracuse University Press; 1986. European Commission: Open Innovation, Open Science, Open to the World: A Vision for Europe. European Commission; 2016. Reference Source Fecher B, Friesike S: Open Science: One term, five schools of thought.Bartling S, Friesike S, editors. Opening Science. Springer International Publishing; 2014. Publisher Full Text Finnemore M: International organizations as teachers of norms: The United Nations Educational, Scientific, and Cutural Organization and science policy. Int. Organ. 1993; 47 (4): 565–597. Publisher Full Text Gault F: Measuring Innovation Everywhere. Edward Elgar Publishing; 2020. Publisher Full Text Gibbons M: Science’s new social contract with society. Nature. 1999; 402 : c81–c84. Publisher Full Text Godin B: Measurement and Statistics on Science and Technology. Routledge; 0th ed2004. Publisher Full Text Gault F: The Oslo Manual.Gault F, editor. Handbook of innovation indicators and measurement. Edward Elgar Publishing; 2013; pp. 41–59. Haddad CR, Bergek A: Towards an integrated framework for evaluating transformative innovation policy. Res. Policy. 2023; 52 (2): 104676. Publisher Full Text Holtrop T, Meijer I, Otero-Hermida P, et al. : Translating between diverse stakeholders in regional RRI projects. Fteval Journal. 2022; 53 : 77–84. Publisher Full Text Huang C-K, Neylon C, Montgomery L, et al. : Open access research outputs receive more diverse citations. Scientometrics. 2024; 129 : 825–845. Publisher Full Text ISC: Open Science for a Global Transformation. Committe on Data (CODATA), International Science Council; 2020. Janssen MJ, Bergek A, Wesseling JH: Evaluating systemic innovation and transition programmes: Towards a culture of learning. PLOS Sustainability and Transformation. 2022; 1 (3): e0000008. Publisher Full Text Kline SJ, Rosenberg N: An overview of innovation.Edquist C, McKelvey M, editors. Systems of innovation: Growth, competitiveness and employment. Edward Elgar; 1986; pp. 3–33. Kofler J, Wieser H: Navigating transitions: A ‘reflexive navigation system’ for monitoring and learning in mission-oriented innovation policies. Eu-SPRI Conference. Brighton, UK: 2023. Kozlowski D, Larivière V, Sugimoto CR, et al. : Intersectional inequalities in science. Proc. Natl. Acad. Sci. 2022; 119 (2): e2113067119. PubMed Abstract | Publisher Full Text | Free Full Text Larivière V, Haustein S, Mongeon P: The Oligopoly of Academic Publishers in the Digital Era. PLoS One. 2015; 10 (6): e0127502. PubMed Abstract | Publisher Full Text | Free Full Text Lawson C, Salter A, Hughes A, et al. : Citizens of somewhere: Examining the geography of foreign and native-born academics’ engagement with external actors. Res. Policy. 2019; 48 (3): 759–774. PubMed Abstract | Publisher Full Text | Free Full Text Leach M, Scoones I, Stirling A: Dynamic sustainabilities: Technology, environment, social justice. Earthscan; 2010. Leonelli S: Philosophy of Open Science. 1st ed.Cambridge University Press; 2023. Publisher Full Text Loorbach DA, Wittmayer J: Transforming universities: Mobilizing research and education for sustainability transitions at Erasmus University Rotterdam, The Netherlands. Sustain. Sci. 2024; 19 (1): 19–33. Publisher Full Text Maricato J d M, Mazoni A, Mugnaini R, et al. : SciELO as an open scientometric research infrastructure: General discussion of coverage in OpenAlex, WoS, Scopus and Dimensions. S&T Indicators Conference. Mazzucato M: Mission-oriented innovation policies: Challenges and opportunities. Ind. Corp. Chang. 2018; 27 (5): 803–815. Publisher Full Text Méndez E: Open Science por defecto. La nueva normalidad para la investigación. Arbor. 2021; 197 (799): a587. Publisher Full Text Molas-Gallart J, Boni A, Giachi S, et al. : A formative approach to the evaluation of Transformative Innovation Policies. Res. Eval. 2021; rvab016. Publisher Full Text Molas-Gallart J, D’Este P, Llopis O, et al. : Towards an alternative framework for the evaluation of translational research initiatives. Res. Eval. 2016; 25 (3): 235–243. Publisher Full Text Molas-Gallart J, Rafols I: Why bibliometric indicators break down: Unstable parameters, incorrect models and irrelevant prope. BiD: Textos Universitaris de Biblioteconomia i Documentació. 2018; 2018 : 40. Publisher Full Text Molas-Gallart J, Salter A, Patel P, et al. : Measuring third stream activities. Final Report to the Russell Group of Universities. Brighton: SPRU, University of Sussex; 2002. Mongeon P, Paul-Hus A: The journal coverage of Web of Science and Scopus: A comparative analysis. Scientometrics. 2016; 106 (1): 213–228. Publisher Full Text Nielsen M: Reinventing Discovery: The New Era of Networked Science. Princeton University Press; 2011. Nowotny H, Scott P, Gibbons M: Introduction. ‘Mode 2’ Revisited: The New Production of Knowledge. Minerva. 2003; 41 (3): 179–194. Publisher Full Text Noyons E, Rafols I: Can bibliometrics help in assessing societal contributions of agricultural research? Exploring societal interactions across research areas. Proceedings of the 23rd International Conference on Science and Technology Indicators. 2018; pp. 1049–1057. OECD: Frascati Manual 2015: Guidelines for Collecting and Reporting Data on Research and Experimental Development. OECD; 2015. Publisher Full Text OECD: OECD Science, Technology and Innovation Outlook 2023: Enabling Transitions in Times of Disruption. OECD; 2023. Publisher Full Text OECD & Eurostat: Oslo Manual 2018: Guidelines for Collecting, Reporting and Using Data on Innovation. 4th Edition.OECD; 2018. Publisher Full Text Olejniczak AJ, Wilson MJ: Who’s writing open access (OA) articles? Characteristics of OA authors at Ph.D.-granting institutions in the United States. Quant. Sci. Stud. 2020; 1 (4): 1429–1450. Publisher Full Text Ollé C, López-Borrull A, Melero R, et al. : Habits and perceptions regarding open science by researchers from Spanish institutions. PLoS One. 2023; 18 (7): e0288313. PubMed Abstract | Publisher Full Text | Free Full Text Owen R, Macnaghten P, Stilgoe J: Responsible research and innovation: From science in society to science for society, with society. Sci. Public Policy. 2012; 39 (6): 751–760. Publisher Full Text Purnell PJ: A comparison of different methods of identifying publications related to the United Nations Sustainable Development Goals: Case study of SDG 13—Climate Action. Quant. Sci. Stud. 2022; 3 (4): 976–1002. Publisher Full Text Ràfols I: S&T indicators in the wild: Contextualization and participation for responsible metrics. Res. Eval. 2019; 28 (1): 7–22. Publisher Full Text Rafols I, Leydesdorff L, O’Hare A, et al. : How journal rankings can suppress interdisciplinary research: A comparison between Innovation Studies and Business & Management. Res. Policy. 2012; 41 (7): 1262–1282. Publisher Full Text Rafols I, Stirling A: Designing indicators for opening up evaluation: Insights from research assessment.Dahler-Larsen P, editors. A research agenda for evaluation. Edward Elgar; 2021. Publisher Full Text Reference Source Rohracher H, Coenen L, Kordas O: Mission incomplete: Layered practices of monitoring and evaluation in Swedish transformative innovation policy. Sci. Public Policy. 2023; 50 : 336–349. Publisher Full Text Ross-Hellauer T, Deppe A, Schmidt B: Survey on open peer review: Attitudes and experience amongst editors, authors and reviewers. PLoS One. 2017; 12 (12): e0189311. PubMed Abstract | Publisher Full Text | Free Full Text Ross-Hellauer T, Reichmann S, Cole NL, et al. : Dynamics of cumulative advantage and threats to equity in open science: A scoping review. R. Soc. Open Sci. 2022; 9 : 211032. PubMed Abstract | Publisher Full Text | Free Full Text Rottenburg R, Merry SE: A World of Indicators. The Making of Governmental Knowledge through Indicators. The world of indicators: The making of governmental knowledge through quantification. Cambridge University Press; 2015; pp. 1–33. Salö L, Hammarfelt B, Nelhans G: Sources of policy: Knowledge brokering in governmental reports.Mattsson IP, Perez Vico E, Salö L, editors. Making universities matter: Collaboration, engagement, impact. Springer Nature; 2023; pp. 185–210. Publisher Full Text Schneider JW, Allum N, Andersen JP, et al. : Is something rotten in the state of Denmark? Cross-national evidence for widespread involvement but not systematic use of questionable research practices across all fields of research. [Preprint]. MetaArXiv. 2023. Publisher Full Text Schot J, Steinmueller WE: Three frames for innovation policy: R&D, systems of innovation and transformative change. Res. Policy. 2018; 47 (9): 1554–1567. Publisher Full Text Shelley-Egan C, Gjefsen MD, Nydal R: Consolidating RRI and Open Science: Understanding the potential for transformative change. Life Sci. Soc. Policy. 2020; 16 (1): 7. PubMed Abstract | Publisher Full Text | Free Full Text Shibayama S, Walsh JP, Baba Y: Academic Entrepreneurship and Exchange of Scientific Resources: Material Transfer in Life and Materials Sciences in Japanese Universities. Am. Sociol. Rev. 2012; 77 (5): 804–830. Publisher Full Text Singh VK, Singh P, Karmakar M, et al. : The journal coverage of Web of Science, Scopus and Dimensions: A comparative analysis. Scientometrics. 2021; 126 (6): 5113–5142. Publisher Full Text Smith K: Measuring innovation.Fagerberg J, Mowery DC, Nelson RR, editors. The Oxford handbook of innovation. Oxford University Press; 2004; pp. 148–177. Stirling A: ‘Opening Up’ and ‘Closing Down’: Power, Participation, and Pluralism in the Social Appraisal of Technology. Sci. Technol. Hum. Values. 2008; 33 (2): 262–294. Publisher Full Text Stirling A: Pluralising progress: From integrative transitions to transformative diversity. Environ. Innov. Soc. Trans. 2011; 1 (1): 82–88. Publisher Full Text Stirling A: How deep is incumbency? A ‘configuring fields’ approach to redistributing and reorienting power in socio-material change. Energy Res. Soc. Sci. 2019; 58 : 101239. Publisher Full Text Stirling A: Against misleading technocratic precision in research evaluation and wider policy – A response to Franzoni and Stephan (2023), ‘uncertainty and risk-taking in science’. Res. Policy. 2023; 52 (3): 104709. Publisher Full Text Sugimoto CR, Larivière V: Equity for Women in Science: Dismantling Systemic Barriers to Advancement. Harvard University Press; 2023. Thune T, Reymert I, Gulbrandsen M, et al. : Populating the science-policy co-production space: Academic and policymaker perspectives on knowledge exchange. Stud. High. Educ. 2023; 48 (5): 733–746. Publisher Full Text Tijssen RJW: Universities and industrially relevant science: Towards measurement models and indicators of entrepreneurial orientation. Res. Policy. 2006; 35 (10): 1569–1585. Publisher Full Text UNESCO: UNESCO Recommendation on Open Science. UNESCO; 2021. Publisher Full Text UNESCO: Open science outlook 1: Status and trends around the world. UNESCO; 2023. Publisher Full Text Van Mierlo B, Arkesteijn M, Leeuwis C: Enhancing the Reflexivity of System Innovation Projects With System Analyses. Am. J. Eval. 2010; 31 (2): 143–161. Publisher Full Text Weber KM, Rohracher H: Legitimizing research, technology and innovation policies for transformative change. Res. Policy. 2012; 41 (6): 1037–1047. Publisher Full Text Wilkinson MD, Dumontier M, Aalbersberg IJ, et al. : The FAIR Guiding Principles for scientific data management and stewardship. Sci. Data. 2016; 3 (1): 160018. PubMed Abstract | Publisher Full Text | Free Full Text Wouters P, Rafols I, Oancea A, et al. : Indicator frameworks for fostering open knowledge practices in science and scholarship.European Commission Directorate General for Research and Innovation. Publications Office; 2019. Publisher Full Text Footnotes 1 https://research-and-innovation.ec.europa.eu/strategy/strategy-2020-2024/our-digital-future/open-science/open-science-monitor_en 2 https://frenchopensciencemonitor.esr.gouv.fr/software/general 3 https://avointiede.fi/en/policies-materials/monitoring 4 https://eoscobservatory.eosc-portal.eu/home 5 https://unesdoc.unesco.org/ark:/48223/pf0000383805 6 It should be noted that the breadth of the concept of science is ambiguous. In accordance with most science policy, here we take a broad definition: we use it for all scholarly disciplines including social science and humanities, as well as research activities outside of academia. We would prefer using the terms research and open research , but we keep the term science in order to align with current policy discourse on OS. 7 While their framing in terms the five schools is possibly outdated, it illustrates that very disparate understandings on OS have long existed. 8 The recent addition of ‘trust in science’ shows again the political nature of the definition of OS, now a major issue following the battle for public confidence in COVID-19 responses. 9 A more inclusive science should be more less concentrated in rich nations and rich cities ( Ciarli, 2022 ; UNESCO, 2023 , p. 36). 10 See for example, the description of ‘U.S. and Global Science and Technology Capabilities’ in the Science and Engineering 2022 report . 11 The Oslo manual (as the Frascati) has undergone various revisions. For example the definition of innovation has broadened from only technological product and process innovation in the first edition (1992), to include also non-technological products and processes, new marketing methods, or a new organization method in business practices, workplace organization or external relations, in the third edition in 2005. Also, an annex was added to the Oslo Manual in 2005 for its use in developing countries, after the experience provided by the Ibero-American Network of Indicators of S&T (RICYT) in Bogotá Manual ( Gault, 2013 , p. 51). In recent years, proposal have been made to include public sector innovations (which are not counted currently since they are not market-oriented), social and household innovations and innovations in the informal sector ( Gault, 2020 ). Similarly, the battery of indicators of the Innovation Scoreboard has evolved over time – for example, with the introduction of indicators of environmental sustainability in recent years (which is a hint of a shift towards frame 3). 12 We thank Anestis Amanatidis for suggesting this point. 13 https://unesdoc.unesco.org/ark:/48223/pf0000386862 14 Here it should be noted that, if inclusiveness or integrity are considered to be instrumental properties (rather than a value in itself ), then they will be considered a change in practice (an outcome) that is expected to contribute to higher order values (for instance by contributing to the achievement of social goals). 15 After Tommaso Di Lampedusa, the author of the novel The Leopard which describes how the old Sicilian aristocracy preserved their power through the unification of Italy and the ascent of capitalism: ‘Everything must change for everything to remain the same’. 16 https://datacite.org/ see UNESCO Outlook ( UNESCO, 2023a , p. 40). Comments on this article Comments (0) Version 1 VERSION 1 PUBLISHED 23 Apr 2024 ADD YOUR COMMENT Comment Author details Author details 1 Centre for Science and Technology Studies, Leiden University, Leiden, The Netherlands 2 INGENIO (CSIC-UPV), Universitat Politecnica de Valencia, Valencia, Spain Ismael Rafols Roles: Conceptualization, Visualization, Writing – Original Draft Preparation, Writing – Review & Editing Ingeborg Meijer Roles: Conceptualization, Writing – Review & Editing Jordi Molas-Gallart Roles: Conceptualization, Writing – Review & Editing Competing interests IR is the Chairholder of the UNESCO Chair of Diversity and Inclusion in Global Science and was involved in advisory meetings of the UNESCO Working Group on Open Science Monitoring Framework, as detailed in the UNESCO OS Implementation webpage. Grant information The author(s) declared that no grants were involved in supporting this work. Article Versions (1) version 1 Published: 23 Apr 2024, 13:320 https://doi.org/10.12688/f1000research.148290.1 Copyright © 2024 Rafols I et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. Download Export To Sciwheel Bibtex EndNote ProCite Ref. Manager (RIS) Sente metrics Views Downloads F1000Research - - PubMed Central info_outline Data from PMC are received and updated monthly. - - Citations open_in_new 0 open_in_new 0 open_in_new SEE MORE DETAILS CITE how to cite this article Rafols I, Meijer I and Molas-Gallart J. Monitoring Open Science as transformative change: Towards a systemic framework [version 1; peer review: 2 approved] . F1000Research 2024, 13 :320 ( https://doi.org/10.12688/f1000research.148290.1 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS track receive updates on this article Track an article to receive email alerts on any updates to this article. TRACK THIS ARTICLE Share Open Peer Review Current Reviewer Status: ? Key to Reviewer Statuses VIEW HIDE Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Version 1 VERSION 1 PUBLISHED 23 Apr 2024 Views 0 Cite How to cite this report: Di Donato F. Reviewer Report For: Monitoring Open Science as transformative change: Towards a systemic framework [version 1; peer review: 2 approved] . F1000Research 2024, 13 :320 ( https://doi.org/10.5256/f1000research.162580.r282405 ) The direct URL for this report is: https://f1000research.com/articles/13-320/v1#referee-response-282405 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 06 Jun 2024 Francesca Di Donato , Consiglio Nazionale delle Ricerche (CNR-ILC), Pisa, Italy Approved VIEWS 0 https://doi.org/10.5256/f1000research.162580.r282405 The article first discusses the plural understandings of Open Science (OS), focusing on the OS definition provided by the UNESCO Recommendation of 2021, endorsed by 193 countries, which provides the most complex yet inclusive definition available. This definition implies a ... Continue reading READ ALL The article first discusses the plural understandings of Open Science (OS), focusing on the OS definition provided by the UNESCO Recommendation of 2021, endorsed by 193 countries, which provides the most complex yet inclusive definition available. This definition implies a shift from an understanding of OS as the ‘possibility of access’ towards an understanding as the ‘process of connection’ in accordance with values related to integrity and epistemic justice. Based on this premise, the article makes the case that OS is a systemic transformation of research in all its dimensions (social, epistemic, institutional, organisational and geographical). It then reviews how models of science are associated with particular monitoring frameworks, considering several lessons learnt from the existing ones and how they complement each other, and it specifically focuses on the frame known as Transformative Innovation Policy (TIP), which shifts the focus towards innovation for transformative change in socio-technical systems such as those of energy, mobility or health, particularly to address societal challenges. Furthermore, the paper describes concepts that can be useful for monitoring OS as a transformation, suggesting a shift in monitoring frameworks for OS towards three directions, aimed at adopting: (i) a systemic perspective, switching from an outputs oriented perspective towards a wider one, focusing on policies and outputs, processes, outcomes and impacts; (ii) a learning approach to monitoring, i.e. helping taking decision considering diverse options within OS; (ii) and a directional perspective, which considers the different trajectories within OS dimensions and the associated values and potential effects. The opinion article is based on a very accurate analysis of the current literature, including a documented literature analysis which supports the developed arguments; in particular, all factual statements are adequately supported by citations, and the conclusions are on these basis fully justified. Is the topic of the opinion article discussed accurately in the context of the current literature? Yes Are all factual statements correct and adequately supported by citations? Yes Are arguments sufficiently supported by evidence from the published literature? Yes Are the conclusions drawn balanced and justified on the basis of the presented arguments? Yes Competing Interests: No competing interests were disclosed. Reviewer Expertise: Political philosophy, Open science, Research on research I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Di Donato F. Reviewer Report For: Monitoring Open Science as transformative change: Towards a systemic framework [version 1; peer review: 2 approved] . F1000Research 2024, 13 :320 ( https://doi.org/10.5256/f1000research.162580.r282405 ) The direct URL for this report is: https://f1000research.com/articles/13-320/v1#referee-response-282405 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Views 0 Cite How to cite this report: De Jonge H. Reviewer Report For: Monitoring Open Science as transformative change: Towards a systemic framework [version 1; peer review: 2 approved] . F1000Research 2024, 13 :320 ( https://doi.org/10.5256/f1000research.162580.r282406 ) The direct URL for this report is: https://f1000research.com/articles/13-320/v1#referee-response-282406 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 05 Jun 2024 Hans De Jonge , Dutch Research Council, The Hague, The Netherlands Approved VIEWS 0 https://doi.org/10.5256/f1000research.162580.r282406 In this opinion article the authors argue that the way open science is being monitored is in need of a fundamental revision. Traditional open science monitors have very much focused on outputs using quantitative approaches: analyzing the number of open ... Continue reading READ ALL In this opinion article the authors argue that the way open science is being monitored is in need of a fundamental revision. Traditional open science monitors have very much focused on outputs using quantitative approaches: analyzing the number of open access publications over time. More recently, these reports have been complemented with attempts to monitor open availability of research data and code / software. The authors – in my view, rightly – argue that these narrow approaches can lead to streetlight effects (counting what can be counted) and misrepresentation of the much broader open science agenda as systematic transformation of the research system. Informed by the literature about Transformative Innovation Policies the authors call for a shift along three lines: The dimension of monitoring should be broadened as to include not only (more) outputs but also processes, outcomes and impacts. Open Science monitoring exercises should be more directed towards learning instead of only accountability and benchmarking. Monitoring should open up by making visible the ‘directionality’, especially the problematic sometimes unintended consequences of certain choices. Given the enormous proliferation OS monitoring worldwide I think the authors view is very valuable and topical. The article is well structured and overall, well written (see editorial notes below) and is very well supported by the most recent literature on the topic. I have only a few comments and leave it to the authors to take them into consideration or discard them. Directionality. I struggled with the concept of directionality. It is only very briefly introduced in the abstract and the introduction (p. 4) and again on p. 10. but for me not to an extend that I understood what was meant by it. I didn’t find the examples (“e.g. different colour of the routes to OS”) very helpful. Only on p. 16 the authors explain that directionality means that in the adoption of certain OS practices different choices (trajectories) can be made, that some of these trajectories can have unintended consequences or even outcomes that are outright in conflict with the values of open science (three examples are given that conflict with the equity value). I completely concur with the authors that being open to negative, unintended or conflicting consequences need to part of a more comprehensive open science monitoring but I think some readers might need some help in understanding this point throughout the article. Schools of thought I am very happy to see that the authors take the effort of making clear that Open Science is a very broad concept which encompasses a multitude of activities, practices and agendas. In that context reference is being made to the seminal work of Fecher and Friesike who have argued that under the broader umbrella term of Open Science five schools of thought can be distinguished. Each school has its own focus, depending on what each school thinks is the most pressing problem in the current science system. While I applaud the authors for emphasizing the plurality of the Open Science movement, I cannot completely escape the impression that they themselves feel more closely associated with one particular school (the public school) and less with others, leading to – in my view - a somewhat narrow definition of what Open Science is aiming for. Two examples to make this clear. First, when arguing that monitoring should not only focus on policies and outputs but also on processes and outcomes, this is consistently illustrated by the authors with examples from the intended ‘societal’ benefits of OS: OS leading to more citizen engagement, researchers more focusing on societal problems, more participatory events, open engagement and dialogue with non-academic actors. (Also in figure 5 and table 1 outcomes and impacts of OS are only defined in terms of societal benefits). While I agree that these are very important objectives of OS, I do not think they are the only ones. Much of what is being done in Open Science is – I would argue – to improve to research process itself. To make science more transparent, more reproducible, more efficient, more interoperable, the dissemination more quick and efficient. All to ensure the quality of science increases and trust in science (by scientists and society) increases. This notion that one of the intended and hoped for outcomes of Open Science is increased quality and integrity is markedly missing from the narrative of the authors, although quality and integrity is the first of the 4 values mentioned by UNESCO and the authors stress the importance to focus monitoring OS on these UNESCO values. A dimension that is – in my view - also somewhat under represented is that one of the ‘collective benefits’ of Open Science is its potential economic impact. Although I agree, that many of the initial aspirations (for instance the Budapest Declaration) were to do with global equity and fair participation of underrepresented parts of the world in the research system, one cannot deny, that a lot of the policy debates around open science are motivated by considerations around unlocking publicly funded research for innovation, knowledge transfer and economic growth. Indeed, much of the early government support for Open Access in countries like the NLs and the UK were motivated by Open Science as potential driver for economic impact. Apart from the quality/trust motivation, this economic dimension is also absent from the authors narrative. I would encourage the authors to think of their own position in the ‘five schools of thought’ and consider whether in presenting examples of how to monitor OS outcomes and OS impacts they have struck the right balance. Do these examples cover the broadness of the Open Science agenda and the plurality of activities, practices and motivations that are currently covered by Open Science? UNESCO survey The work of this opinion article is being done as the result of the authors’ engagement with the UNESCO working group on Open Science monitoring. The authors present a very interesting framework for future Open Science monitoring which is much more comprehensive and systematic in nature than much of the older monitoring exercises. For the moment, however, it seems that UNESCO has not chosen to adopt this model, which – to me at least – feels like a missed opportunity. Instead, UNESCO will put out a survey which – at this stage at least – will focus on supporting actions taken by member states to foster open science. I would be interested to learn the authors view on this decision and since this is an opinion article encourage to take a more critical stance, maybe as part of footnote 13. Minor issue: Since the writing of this article an important initiative has been launched to support the move towards open bibliometric metadata: the Barcelona Declaration. This is at least partly motivated by the increase in open science monitoring activities and the importance to base these monitors on open instead of proprietary data. The authors might consider referring to this initiave as part of paragraph 5.1 (p.12). Editorial: Change ‘gathering statistics’ to ‘collecting statistics’ p. 3 Change ‘this study is prompted’ by ‘this article is prompted’ p. 3 Change ‘for addressing of societal problems’ to ‘for addressing societal problems’ Change ‘the one concerned with’ to ‘one concerned with’ p. 9 Change ‘FAIRisation’ to ‘FAIRification’ p.12 Change ‘most commercial databased of across countries’ to ‘most commercial databased across countries’ Elaborate this statement: ‘The tracking of open research data, open educational resources, open software, and open hardware is patchy but illustrative of growing efforts (UNESCO, 2023, pp. 40–42), although the estimates of numbers of datasets, scripts or courses as general indicators is problematic given the large difference in size and efforts invested per unit’. I don’t understand the problem. Table 1 p. 11 Change ‘Extracted from ORCID’ – I am not sure what is meant. The French OS monitor can be found online. Tabel 1 p. 11 ‘Integrity practices’ – consider ‘Open methods’ as an umbrella term for all methods like preregistration, open reporting, registered reports, etc. Change ‘(in the sense more’ to ‘(in the sense of more’ p. 13 Change ‘publishing in Spanish in non-OA professional newsletter’ to ‘publishing in Spanish in a non-OA professional newsletter’ p. 13 Review this sentence: ‘let us emphasize here that for the benefits do not just lie on the amount of interaction and usage, but on the distributions across epistemic and geographical spaces’ p. 13. > ‘for’ seems to be not right here? Rewrite ‘even more if green OA’ > too informal? Change ‘IS’ to ‘ OS’ p. 14 Change ‘impacts refer the values’ to ‘impacts refer to the values’ p. 14 Consider to further support the suggestion: ‘As potential dimensions of impact of OS, we propose those dimensions in the values of the UNESCO OS Recommendation,’ p. 14 Review quotation ‘to contribute to system innovation the central goal’ p. 14 > word seems missing. Innovation ‘as’? Change ‘As put by the in the UNESCO OS Outlook’ to ‘ As put in the UNESCO OS Outlook’ or even ‘As argued by UNESCO OS Outlook’ p.15 Change ‘By ‘broadening out’ the type of issues to type of OA’ to ‘By ‘broadening out’ to type of OA’ p. 15 Change ‘pol-icy’ to ‘policy’ p.16 Explain ‘A frame 3 perspective’ p.16 Is the topic of the opinion article discussed accurately in the context of the current literature? Yes Are all factual statements correct and adequately supported by citations? Yes Are arguments sufficiently supported by evidence from the published literature? Yes Are the conclusions drawn balanced and justified on the basis of the presented arguments? Yes Competing Interests: No competing interests were disclosed. Reviewer Expertise: Open Science, monitoring I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT De Jonge H. Reviewer Report For: Monitoring Open Science as transformative change: Towards a systemic framework [version 1; peer review: 2 approved] . F1000Research 2024, 13 :320 ( https://doi.org/10.5256/f1000research.162580.r282406 ) The direct URL for this report is: https://f1000research.com/articles/13-320/v1#referee-response-282406 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Comments on this article Comments (0) Version 1 VERSION 1 PUBLISHED 23 Apr 2024 ADD YOUR COMMENT Comment keyboard_arrow_left keyboard_arrow_right Open Peer Review Reviewer Status info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Reviewer Reports Invited Reviewers 1 2 Version 1 23 Apr 24 read read Hans De Jonge , Dutch Research Council, The Hague, The Netherlands Francesca Di Donato , Consiglio Nazionale delle Ricerche (CNR-ILC), Pisa, Italy Comments on this article All Comments (0) Add a comment Sign up for content alerts Sign Up You are now signed up to receive this alert Browse by related subjects keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Di Donato F. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 06 Jun 2024 | for Version 1 Francesca Di Donato , Consiglio Nazionale delle Ricerche (CNR-ILC), Pisa, Italy 0 Views copyright © 2024 Di Donato F. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions The article first discusses the plural understandings of Open Science (OS), focusing on the OS definition provided by the UNESCO Recommendation of 2021, endorsed by 193 countries, which provides the most complex yet inclusive definition available. This definition implies a shift from an understanding of OS as the ‘possibility of access’ towards an understanding as the ‘process of connection’ in accordance with values related to integrity and epistemic justice. Based on this premise, the article makes the case that OS is a systemic transformation of research in all its dimensions (social, epistemic, institutional, organisational and geographical). It then reviews how models of science are associated with particular monitoring frameworks, considering several lessons learnt from the existing ones and how they complement each other, and it specifically focuses on the frame known as Transformative Innovation Policy (TIP), which shifts the focus towards innovation for transformative change in socio-technical systems such as those of energy, mobility or health, particularly to address societal challenges. Furthermore, the paper describes concepts that can be useful for monitoring OS as a transformation, suggesting a shift in monitoring frameworks for OS towards three directions, aimed at adopting: (i) a systemic perspective, switching from an outputs oriented perspective towards a wider one, focusing on policies and outputs, processes, outcomes and impacts; (ii) a learning approach to monitoring, i.e. helping taking decision considering diverse options within OS; (ii) and a directional perspective, which considers the different trajectories within OS dimensions and the associated values and potential effects. The opinion article is based on a very accurate analysis of the current literature, including a documented literature analysis which supports the developed arguments; in particular, all factual statements are adequately supported by citations, and the conclusions are on these basis fully justified. Is the topic of the opinion article discussed accurately in the context of the current literature? Yes Are all factual statements correct and adequately supported by citations? Yes Are arguments sufficiently supported by evidence from the published literature? Yes Are the conclusions drawn balanced and justified on the basis of the presented arguments? Yes Competing Interests No competing interests were disclosed. Reviewer Expertise Political philosophy, Open science, Research on research I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) Di Donato F. Peer Review Report For: Monitoring Open Science as transformative change: Towards a systemic framework [version 1; peer review: 2 approved] . F1000Research 2024, 13 :320 ( https://doi.org/10.5256/f1000research.162580.r282405) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-320/v1#referee-response-282405 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 De Jonge H. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 05 Jun 2024 | for Version 1 Hans De Jonge , Dutch Research Council, The Hague, The Netherlands 0 Views copyright © 2024 De Jonge H. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions In this opinion article the authors argue that the way open science is being monitored is in need of a fundamental revision. Traditional open science monitors have very much focused on outputs using quantitative approaches: analyzing the number of open access publications over time. More recently, these reports have been complemented with attempts to monitor open availability of research data and code / software. The authors – in my view, rightly – argue that these narrow approaches can lead to streetlight effects (counting what can be counted) and misrepresentation of the much broader open science agenda as systematic transformation of the research system. Informed by the literature about Transformative Innovation Policies the authors call for a shift along three lines: The dimension of monitoring should be broadened as to include not only (more) outputs but also processes, outcomes and impacts. Open Science monitoring exercises should be more directed towards learning instead of only accountability and benchmarking. Monitoring should open up by making visible the ‘directionality’, especially the problematic sometimes unintended consequences of certain choices. Given the enormous proliferation OS monitoring worldwide I think the authors view is very valuable and topical. The article is well structured and overall, well written (see editorial notes below) and is very well supported by the most recent literature on the topic. I have only a few comments and leave it to the authors to take them into consideration or discard them. Directionality. I struggled with the concept of directionality. It is only very briefly introduced in the abstract and the introduction (p. 4) and again on p. 10. but for me not to an extend that I understood what was meant by it. I didn’t find the examples (“e.g. different colour of the routes to OS”) very helpful. Only on p. 16 the authors explain that directionality means that in the adoption of certain OS practices different choices (trajectories) can be made, that some of these trajectories can have unintended consequences or even outcomes that are outright in conflict with the values of open science (three examples are given that conflict with the equity value). I completely concur with the authors that being open to negative, unintended or conflicting consequences need to part of a more comprehensive open science monitoring but I think some readers might need some help in understanding this point throughout the article. Schools of thought I am very happy to see that the authors take the effort of making clear that Open Science is a very broad concept which encompasses a multitude of activities, practices and agendas. In that context reference is being made to the seminal work of Fecher and Friesike who have argued that under the broader umbrella term of Open Science five schools of thought can be distinguished. Each school has its own focus, depending on what each school thinks is the most pressing problem in the current science system. While I applaud the authors for emphasizing the plurality of the Open Science movement, I cannot completely escape the impression that they themselves feel more closely associated with one particular school (the public school) and less with others, leading to – in my view - a somewhat narrow definition of what Open Science is aiming for. Two examples to make this clear. First, when arguing that monitoring should not only focus on policies and outputs but also on processes and outcomes, this is consistently illustrated by the authors with examples from the intended ‘societal’ benefits of OS: OS leading to more citizen engagement, researchers more focusing on societal problems, more participatory events, open engagement and dialogue with non-academic actors. (Also in figure 5 and table 1 outcomes and impacts of OS are only defined in terms of societal benefits). While I agree that these are very important objectives of OS, I do not think they are the only ones. Much of what is being done in Open Science is – I would argue – to improve to research process itself. To make science more transparent, more reproducible, more efficient, more interoperable, the dissemination more quick and efficient. All to ensure the quality of science increases and trust in science (by scientists and society) increases. This notion that one of the intended and hoped for outcomes of Open Science is increased quality and integrity is markedly missing from the narrative of the authors, although quality and integrity is the first of the 4 values mentioned by UNESCO and the authors stress the importance to focus monitoring OS on these UNESCO values. A dimension that is – in my view - also somewhat under represented is that one of the ‘collective benefits’ of Open Science is its potential economic impact. Although I agree, that many of the initial aspirations (for instance the Budapest Declaration) were to do with global equity and fair participation of underrepresented parts of the world in the research system, one cannot deny, that a lot of the policy debates around open science are motivated by considerations around unlocking publicly funded research for innovation, knowledge transfer and economic growth. Indeed, much of the early government support for Open Access in countries like the NLs and the UK were motivated by Open Science as potential driver for economic impact. Apart from the quality/trust motivation, this economic dimension is also absent from the authors narrative. I would encourage the authors to think of their own position in the ‘five schools of thought’ and consider whether in presenting examples of how to monitor OS outcomes and OS impacts they have struck the right balance. Do these examples cover the broadness of the Open Science agenda and the plurality of activities, practices and motivations that are currently covered by Open Science? UNESCO survey The work of this opinion article is being done as the result of the authors’ engagement with the UNESCO working group on Open Science monitoring. The authors present a very interesting framework for future Open Science monitoring which is much more comprehensive and systematic in nature than much of the older monitoring exercises. For the moment, however, it seems that UNESCO has not chosen to adopt this model, which – to me at least – feels like a missed opportunity. Instead, UNESCO will put out a survey which – at this stage at least – will focus on supporting actions taken by member states to foster open science. I would be interested to learn the authors view on this decision and since this is an opinion article encourage to take a more critical stance, maybe as part of footnote 13. Minor issue: Since the writing of this article an important initiative has been launched to support the move towards open bibliometric metadata: the Barcelona Declaration. This is at least partly motivated by the increase in open science monitoring activities and the importance to base these monitors on open instead of proprietary data. The authors might consider referring to this initiave as part of paragraph 5.1 (p.12). Editorial: Change ‘gathering statistics’ to ‘collecting statistics’ p. 3 Change ‘this study is prompted’ by ‘this article is prompted’ p. 3 Change ‘for addressing of societal problems’ to ‘for addressing societal problems’ Change ‘the one concerned with’ to ‘one concerned with’ p. 9 Change ‘FAIRisation’ to ‘FAIRification’ p.12 Change ‘most commercial databased of across countries’ to ‘most commercial databased across countries’ Elaborate this statement: ‘The tracking of open research data, open educational resources, open software, and open hardware is patchy but illustrative of growing efforts (UNESCO, 2023, pp. 40–42), although the estimates of numbers of datasets, scripts or courses as general indicators is problematic given the large difference in size and efforts invested per unit’. I don’t understand the problem. Table 1 p. 11 Change ‘Extracted from ORCID’ – I am not sure what is meant. The French OS monitor can be found online. Tabel 1 p. 11 ‘Integrity practices’ – consider ‘Open methods’ as an umbrella term for all methods like preregistration, open reporting, registered reports, etc. Change ‘(in the sense more’ to ‘(in the sense of more’ p. 13 Change ‘publishing in Spanish in non-OA professional newsletter’ to ‘publishing in Spanish in a non-OA professional newsletter’ p. 13 Review this sentence: ‘let us emphasize here that for the benefits do not just lie on the amount of interaction and usage, but on the distributions across epistemic and geographical spaces’ p. 13. > ‘for’ seems to be not right here? Rewrite ‘even more if green OA’ > too informal? Change ‘IS’ to ‘ OS’ p. 14 Change ‘impacts refer the values’ to ‘impacts refer to the values’ p. 14 Consider to further support the suggestion: ‘As potential dimensions of impact of OS, we propose those dimensions in the values of the UNESCO OS Recommendation,’ p. 14 Review quotation ‘to contribute to system innovation the central goal’ p. 14 > word seems missing. Innovation ‘as’? Change ‘As put by the in the UNESCO OS Outlook’ to ‘ As put in the UNESCO OS Outlook’ or even ‘As argued by UNESCO OS Outlook’ p.15 Change ‘By ‘broadening out’ the type of issues to type of OA’ to ‘By ‘broadening out’ to type of OA’ p. 15 Change ‘pol-icy’ to ‘policy’ p.16 Explain ‘A frame 3 perspective’ p.16 Is the topic of the opinion article discussed accurately in the context of the current literature? Yes Are all factual statements correct and adequately supported by citations? Yes Are arguments sufficiently supported by evidence from the published literature? Yes Are the conclusions drawn balanced and justified on the basis of the presented arguments? Yes Competing Interests No competing interests were disclosed. Reviewer Expertise Open Science, monitoring I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) De Jonge H. Peer Review Report For: Monitoring Open Science as transformative change: Towards a systemic framework [version 1; peer review: 2 approved] . F1000Research 2024, 13 :320 ( https://doi.org/10.5256/f1000research.162580.r282406) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-320/v1#referee-response-282406 Alongside their report, reviewers assign a status to the article: Approved - the paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations - A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved - fundamental flaws in the paper seriously undermine the findings and conclusions Adjust parameters to alter display View on desktop for interactive features Includes Interactive Elements View on desktop for interactive features Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Stay Updated Sign up for content alerts and receive a weekly or monthly email with all newly published articles Register with F1000Research Already registered? Sign in Not now, thanks close PLEASE NOTE If you are an AUTHOR of this article, please check that you signed in with the account associated with this article otherwise we cannot automatically identify your role as an author and your comment will be labelled as a “User Comment”. If you are a REVIEWER of this article, please check that you have signed in with the account associated with this article and then go to your account to submit your report, please do not post your review here. If you do not have access to your original account, please contact us . All commenters must hold a formal affiliation as per our Policies . The information that you give us will be displayed next to your comment. User comments must be in English, comprehensible and relevant to the article under discussion. We reserve the right to remove any comments that we consider to be inappropriate, offensive or otherwise in breach of the User Comment Terms and Conditions . Commenters must not use a comment for personal attacks. When criticisms of the article are based on unpublished data, the data should be made available. I accept the User Comment Terms and Conditions Please confirm that you accept the User Comment Terms and Conditions. Affiliation ✕ refresh Please enter your institution. Note: To add your institution or organisation, start typing the name and then select the correct name from the list. Where applicable, the name will appear in both the original language and in English. Do not paste in the name. If the name does not appear in the drop-down list, we will display the information you have entered. ✕ refresh Country/Region * USA UK Canada China France Germany Afghanistan Aland Islands Albania Algeria American Samoa Andorra Angola Anguilla Antarctica Antigua and Barbuda Argentina Armenia Aruba Australia Austria Azerbaijan Bahamas Bahrain Bangladesh Barbados Belarus Belgium Belize Benin Bermuda Bhutan Bolivia Bosnia and Herzegovina Botswana Bouvet Island Brazil British Indian Ocean Territory British Virgin Islands Brunei Bulgaria Burkina Faso Burundi Cambodia Cameroon Canada Cape Verde Cayman Islands Central African Republic Chad Chile China Christmas Island Cocos (Keeling) Islands Colombia Comoros Congo Cook Islands Costa Rica Cote d'Ivoire Croatia Cuba Cyprus Czech Republic Democratic Republic of the Congo Denmark Djibouti Dominica Dominican Republic Ecuador Egypt El Salvador Equatorial Guinea Eritrea Estonia Ethiopia Falkland Islands Faroe Islands Federated States of Micronesia Fiji Finland France French Guiana French Polynesia French Southern Territories Gabon Georgia Germany Ghana Gibraltar Greece Greenland Grenada Guadeloupe Guam Guatemala Guernsey Guinea Guinea-Bissau Guyana Haiti Heard Island and Mcdonald Islands Holy See (Vatican City State) Honduras Hong Kong Hungary Iceland India Indonesia Iran Iraq Ireland Israel Italy Jamaica Japan Jersey Jordan Kazakhstan Kenya Kiribati Kosovo (Serbia and Montenegro) Kuwait Kyrgyzstan Lao People's Democratic Republic Latvia Lebanon Lesotho Liberia Libya Liechtenstein Lithuania Luxembourg Macao Madagascar Malawi Malaysia Maldives Mali Malta Marshall Islands Martinique Mauritania Mauritius Mayotte Mexico Minor Outlying Islands of the United States Moldova Monaco Mongolia Montenegro Montserrat Morocco Mozambique Myanmar Namibia Nauru Nepal Netherlands Antilles New Caledonia New Zealand Nicaragua Niger Nigeria Niue Norfolk Island North Korea North Macedonia Northern Mariana Islands Norway Oman Pakistan Palau Palestinian Territory Panama Papua New Guinea Paraguay Peru Philippines Pitcairn Poland Portugal Puerto Rico Qatar Reunion Romania Russian Federation Rwanda Saint Helena Saint Kitts and Nevis Saint Lucia Saint Pierre and Miquelon Saint Vincent and the Grenadines Samoa San Marino Sao Tome and Principe Saudi Arabia Senegal Serbia Seychelles Sierra Leone Singapore Slovakia Slovenia Solomon Islands Somalia South Africa South Georgia and the South Sandwich Is South Korea South Sudan Spain Sri Lanka Sudan Suriname Svalbard and Jan Mayen Swaziland Sweden Switzerland Syria Taiwan Tajikistan Tanzania Thailand The Gambia The Netherlands Timor-Leste Togo Tokelau Tonga Trinidad and Tobago Tunisia Turkey Turkmenistan Turks and Caicos Islands Tuvalu UK USA Uganda Ukraine United Arab Emirates United States Virgin Islands Uruguay Uzbekistan Vanuatu Venezuela Vietnam Wallis and Futuna West Bank and Gaza Strip Western Sahara Yemen Zambia Zimbabwe Please select your country/region. You must enter a comment. Competing Interests Please disclose any competing interests that might be construed to influence your judgment of the article's or peer review report's validity or importance. Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Please state your competing interests The comment has been saved. An error has occurred. Please try again. Cancel Post var lTitle = "Monitoring Open Science as transformative...".replace("'", ''); var linkedInUrl = "http://www.linkedin.com/shareArticle?url=https://f1000research.com/articles/13-320/v1" + "&title=" + encodeURIComponent(lTitle) + "&summary=" + encodeURIComponent('Read the article by '); var deliciousUrl = "https://del.icio.us/post?url=https://f1000research.com/articles/13-320/v1&title=" + encodeURIComponent(lTitle); var redditUrl = "http://reddit.com/submit?url=https://f1000research.com/articles/13-320/v1" + "&title=" + encodeURIComponent(lTitle); linkedInUrl += encodeURIComponent('Rafols I et al.'); var offsetTop = /chrome/i.test( navigator.userAgent ) ? 4 : -10; var addthis_config = { ui_offset_top: offsetTop, services_compact : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_expanded : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_custom : [ { name: "LinkedIn", url: linkedInUrl, icon:"/img/icon/at_linkedin.svg" }, { name: "Mendeley", url: "http://www.mendeley.com/import/?url=https://f1000research.com/articles/13-320/v1/mendeley", icon:"/img/icon/at_mendeley.svg" }, { name: "Reddit", url: redditUrl, icon:"/img/icon/at_reddit.svg" }, ] }; var addthis_share = { url: "https://f1000research.com/articles/13-320", templates : { twitter : "Monitoring Open Science as transformative change: Towards a systemic.... Rafols I et al., published by " + "@F1000Research" + ", https://f1000research.com/articles/13-320/v1" } }; if (typeof(addthis) != "undefined"){ addthis.addEventListener('addthis.ready', checkCount); addthis.addEventListener('addthis.menu.share', checkCount); } $(".f1r-shares-twitter").attr("href", "https://twitter.com/intent/tweet?text=" + addthis_share.templates.twitter); $(".f1r-shares-facebook").attr("href", "https://www.facebook.com/sharer/sharer.php?u=" + addthis_share.url); $(".f1r-shares-linkedin").attr("href", addthis_config.services_custom[0].url); $(".f1r-shares-reddit").attr("href", addthis_config.services_custom[2].url); $(".f1r-shares-mendelay").attr("href", addthis_config.services_custom[1].url); function checkCount(){ setTimeout(function(){ $(".addthis_button_expanded").each(function(){ var count = $(this).text(); if (count !== "" && count != "0") $(this).removeClass("is-hidden"); else $(this).addClass("is-hidden"); }); }, 1000); } close How to cite this report {{reportCitation}} Cancel Copy Citation Details $(function(){R.ui.buttonDropdowns('.dropdown-for-downloads');}); $(function(){R.ui.toolbarDropdowns('.toolbar-dropdown-for-downloads');}); $.get("/articles/acj/148290/162580") new F1000.Clipboard(); new F1000.ThesaurusTermsDisplay("articles", "article", "162580"); $(document).ready(function() { $( "#frame1" ).on('load', function() { var mydiv = $(this).contents().find("div"); var h = mydiv.height(); console.log(h) }); var tooltipLivingFigure = jQuery(".interactive-living-figure-label .icon-more-info"), titleLivingFigure = tooltipLivingFigure.attr("title"); tooltipLivingFigure.simpletip({ fixed: true, position: ["-115", "30"], baseClass: 'small-tooltip', content:titleLivingFigure + " " }); tooltipLivingFigure.removeAttr("title"); $("body").on("click", ".cite-living-figure", function(e) { e.preventDefault(); var ref = $(this).attr("data-ref"); $(this).closest(".living-figure-list-container").find("#" + ref).fadeIn(200); }); $("body").on("click", ".close-cite-living-figure", function(e) { e.preventDefault(); $(this).closest(".popup-window-wrapper").fadeOut(200); }); $(document).on("mouseup", function(e) { var metricsContainer = $(".article-metrics-popover-wrapper"); if (!metricsContainer.is(e.target) && metricsContainer.has(e.target).length === 0) { $(".article-metrics-close-button").click(); } }); var articleId = $('#articleId').val(); if($("#main-article-count-box").attachArticleMetrics) { $("#main-article-count-box").attachArticleMetrics(articleId, { articleMetricsView: true }); } }); var figshareWidget = $(".new_figshare_widget"); if (figshareWidget.length > 0) { window.figshare.load("f1000", function(Widget) { // Select a tag/tags defined in your page. In this tag we will place the widget. _.map(figshareWidget, function(el){ var widget = new Widget({ articleId: $(el).attr("figshare_articleId") //height:300 // this is the height of the viewer part. [Default: 550] }); widget.initialize(); // initialize the widget widget.mount(el); // mount it in a tag that's on your page // this will save the widget on the global scope for later use from // your JS scripts. This line is optional. //window.widget = widget; }); }); } close Error Close Add Reset F1000.MICROSERVICES.AFFILIATION = ''; $(document).ready(function () { $('.js-affiliations-form').each((index, form) => { new AffiliationForm({ formId: form.id, institutionErrorSelector: '.comment-enter-institution', departmentErrorSelector: '.comment-enter-department', placeSelector: '.js-add-comment-place', stateSelector: '.js-add-comment-state', zipCodeSelector: '.js-add-comment-zipcode', countrySelector: '.js-add-comment-country', countryErrorSelector: '.comment-enter-country', }); }); }); $(document).ready(function () { var reportIds = { "282404": 0, "282405": 24, "282406": 43, "282407": 0, "282400": 0, "282401": 0, "282402": 0, "282403": 0, "282408": 0, "282409": 0, }; $(".referee-response-container,.js-referee-report").each(function(index, el) { var reportId = $(el).attr("data-reportid"), reportCount = reportIds[reportId] || 0; $(el).find(".comments-count-container,.js-referee-report-views").html(reportCount); }); var uuidInput = $("#article_uuid"), oldUUId = uuidInput.val(), newUUId = "0a98ea65-0806-4465-9e5b-6ebab724b209"; uuidInput.val(newUUId); $("a[href*='article_uuid=']").each(function(index, el) { var newHref = $(el).attr("href").replace(oldUUId, newUUId); $(el).attr("href", newHref); }); }); An innovative open access publishing platform offering rapid publication and open peer review, whilst supporting data deposition and sharing. Browse Gateways Collections How it Works Contact For Developers Cookie Notice Privacy Notice RSS Submit Your Research Follow us © 2012-2026 F1000 Research Ltd. ISSN 2046-1402 | Legal | Partner of Research4Life • CrossRef • ORCID • FAIRSharing R.templateTests.simpleTemplate = R.template(' $text $text $text $text $text '); R.templateTests.runTests(); var F1000platform = new F1000.Platform({ name: "f1000research", displayName: "F1000Research", hostName: "f1000research.com", id: "1", editorialEmail: "[email protected]", infoEmail: "[email protected]", usePmcStats: true }); $(function(){R.ui.dropdowns('.dropdown-for-authors, .dropdown-for-about, .dropdown-for-myresearch');}); // $(function(){R.ui.dropdowns('.dropdown-for-referees');}); $(document).ready(function () { if ($(".cookie-warning").is(":visible")) { $(".sticky").css("margin-bottom", "35px"); $(".devices").addClass("devices-and-cookie-warning"); } $(".cookie-warning .close-button").click(function (e) { $(".devices").removeClass("devices-and-cookie-warning"); $(".sticky").css("margin-bottom", "0"); }); $("#tweeter-feed .tweet-message").each(function (i, message) { var self = $(message); self.html(linkify(self.html())); }); $(".partner").on("mouseenter mouseleave", function() { $(this).find(".gray-scale, .colour").toggleClass("is-hidden"); }); }); Sign In Remember me Forgotten your password? Sign In Cancel Email or password not correct. Please try again Please wait... $(function(){ // Note: All the setup needs to run against a name attribute and *not* the id due the clonish // nature of facebox... $("a[id=googleSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("GOOGLE"); $("form[id=oAuthForm]").submit(); }); $("a[id=facebookSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("FACEBOOK"); $("form[id=oAuthForm]").submit(); }); $("a[id=orcidSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("ORCID"); $("form[id=oAuthForm]").submit(); }); }); If you've forgotten your password, please enter your email address below and we'll send you instructions on how to reset your password. The email address should be the one you originally registered with F1000. Email address not valid, please try again You registered with F1000 via Google, so we cannot reset your password. To sign in, please click here . If you still need help with your Google account password, please click here . You registered with F1000 via Facebook, so we cannot reset your password. To sign in, please click here . If you still need help with your Facebook account password, please click here . Code not correct, please try again Reset password Cancel Email us for further assistance. Server error, please try again. If your email address is registered with us, we will email you instructions to reset your password. If you think you should have received this email but it has not arrived, please check your spam filters and/or contact for further assistance. Please wait... Register $(document).ready(function () { signIn.createSignInAsRow($("#sign-in-form-gfb-popup")); $(".target-field").each(function () { var uris = $(this).val().split("/"); if (uris.pop() === "login") { $(this).val(uris.toString().replace(",","/")); } }); });