{"paper_id":"39229f3b-ac6f-4ad4-8587-5f80c2b2f73b","body_text":"Using text-mining to measure the scientific... | F1000Research <!-- --> <!-- --> <!-- This is commented out to fix display problems on mobile devices. We may use it again once we implement a responsive design that supports native device resolutions. --> \"use strict\";function _typeof(t){return(_typeof=\"function\"==typeof Symbol&&\"symbol\"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&\"function\"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?\"symbol\":typeof t})(t)}!function(){var t=function(){var t,e,o=[],n=window,r=n;for(;r;){try{if(r.frames.__tcfapiLocator){t=r;break}}catch(t){}if(r===n.top)break;r=r.parent}t||(!function t(){var e=n.document,o=!!n.frames.__tcfapiLocator;if(!o)if(e.body){var r=e.createElement(\"iframe\");r.style.cssText=\"display:none\",r.name=\"__tcfapiLocator\",e.body.appendChild(r)}else setTimeout(t,5);return!o}(),n.__tcfapi=function(){for(var t=arguments.length,n=new Array(t),r=0;r 3&&2===parseInt(n[1],10)&&\"boolean\"==typeof n[3]&&(e=n[3],\"function\"==typeof n[2]&&n[2](\"set\",!0)):\"ping\"===n[0]?\"function\"==typeof n[2]&&n[2]({gdprApplies:e,cmpLoaded:!1,cmpStatus:\"stub\"}):o.push(n)},n.addEventListener(\"message\",(function(t){var e=\"string\"==typeof t.data,o={};if(e)try{o=JSON.parse(t.data)}catch(t){}else o=t.data;var n=\"object\"===_typeof(o)&&null!==o?o.__tcfapiCall:null;n&&window.__tcfapi(n.command,n.version,(function(o,r){var a={__tcfapiReturn:{returnValue:o,success:r,callId:n.callId}};t&&t.source&&t.source.postMessage&&t.source.postMessage(e?JSON.stringify(a):a,\"*\")}),n.parameter)}),!1))};\"undefined\"!=typeof module?module.exports=t:t()}(); dataLayer = dataLayer || []; // Standard GTM initialization - Google Consent Mode handles consent automatically (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl+ '>m_auth=hzk0Vc3qFsQYhCrIoHz68A>m_preview=env-1>m_cookies_win=x';f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-MWFK8L5J'); ;window.NREUM||(NREUM={});NREUM.init={distributed_tracing:{enabled:true},privacy:{cookies_enabled:true},ajax:{deny_list:[\"bam.nr-data.net\"]}}; ;NREUM.loader_config={accountID:\"438030\",trustKey:\"438030\",agentID:\"772317073\",licenseKey:\"97f8f67f26\",applicationID:\"772317073\"} ;NREUM.info={beacon:\"bam.nr-data.net\",errorBeacon:\"bam.nr-data.net\",licenseKey:\"97f8f67f26\",applicationID:\"772317073\",sa:1} ;/*! For license information please see nr-loader-spa-1.236.0.min.js.LICENSE.txt */ (()=>{\"use strict\";var e,t,r={5763:(e,t,r)=>{r.d(t,{P_:()=>l,Mt:()=>g,C5:()=>s,DL:()=>v,OP:()=>T,lF:()=>D,Yu:()=>y,Dg:()=>h,CX:()=>c,GE:()=>b,sU:()=>_});var n=r(8632),i=r(9567);const o={beacon:n.ce.beacon,errorBeacon:n.ce.errorBeacon,licenseKey:void 0,applicationID:void 0,sa:void 0,queueTime:void 0,applicationTime:void 0,ttGuid:void 0,user:void 0,account:void 0,product:void 0,extra:void 0,jsAttributes:{},userAttributes:void 0,atts:void 0,transactionName:void 0,tNamePlain:void 0},a={};function s(e){if(!e)throw new Error(\"All info objects require an agent identifier!\");if(!a[e])throw new Error(\"Info for \".concat(e,\" was never set\"));return a[e]}function c(e,t){if(!e)throw new Error(\"All info objects require an agent identifier!\");a[e]=(0,i.D)(t,o),(0,n.Qy)(e,a[e],\"info\")}var u=r(7056);const d=()=>{const e={blockSelector:\"[data-nr-block]\",maskInputOptions:{password:!0}};return{allow_bfcache:!0,privacy:{cookies_enabled:!0},ajax:{deny_list:void 0,enabled:!0,harvestTimeSeconds:10},distributed_tracing:{enabled:void 0,exclude_newrelic_header:void 0,cors_use_newrelic_header:void 0,cors_use_tracecontext_headers:void 0,allowed_origins:void 0},session:{domain:void 0,expiresMs:u.oD,inactiveMs:u.Hb},ssl:void 0,obfuscate:void 0,jserrors:{enabled:!0,harvestTimeSeconds:10},metrics:{enabled:!0},page_action:{enabled:!0,harvestTimeSeconds:30},page_view_event:{enabled:!0},page_view_timing:{enabled:!0,harvestTimeSeconds:30,long_task:!1},session_trace:{enabled:!0,harvestTimeSeconds:10},harvest:{tooManyRequestsDelay:60},session_replay:{enabled:!1,harvestTimeSeconds:60,sampleRate:.1,errorSampleRate:.1,maskTextSelector:\"*\",maskAllInputs:!0,get blockClass(){return\"nr-block\"},get ignoreClass(){return\"nr-ignore\"},get maskTextClass(){return\"nr-mask\"},get blockSelector(){return e.blockSelector},set blockSelector(t){e.blockSelector+=\",\".concat(t)},get maskInputOptions(){return e.maskInputOptions},set maskInputOptions(t){e.maskInputOptions={...t,password:!0}}},spa:{enabled:!0,harvestTimeSeconds:10}}},f={};function l(e){if(!e)throw new Error(\"All configuration objects require an agent identifier!\");if(!f[e])throw new Error(\"Configuration for \".concat(e,\" was never set\"));return f[e]}function h(e,t){if(!e)throw new Error(\"All configuration objects require an agent identifier!\");f[e]=(0,i.D)(t,d()),(0,n.Qy)(e,f[e],\"config\")}function g(e,t){if(!e)throw new Error(\"All configuration objects require an agent identifier!\");var r=l(e);if(r){for(var n=t.split(\".\"),i=0;i {r.d(t,{D:()=>i});var n=r(50);function i(e,t){try{if(!e||\"object\"!=typeof e)return(0,n.Z)(\"Setting a Configurable requires an object as input\");if(!t||\"object\"!=typeof t)return(0,n.Z)(\"Setting a Configurable requires a model to set its initial properties\");const r=Object.create(Object.getPrototypeOf(t),Object.getOwnPropertyDescriptors(t)),o=0===Object.keys(r).length?e:r;for(let a in o)if(void 0!==e[a])try{\"object\"==typeof e[a]&&\"object\"==typeof t[a]?r[a]=i(e[a],t[a]):r[a]=e[a]}catch(e){(0,n.Z)(\"An error occurred while setting a property of a Configurable\",e)}return r}catch(e){(0,n.Z)(\"An error occured while setting a Configurable\",e)}}},6818:(e,t,r)=>{r.d(t,{Re:()=>i,gF:()=>o,q4:()=>n});const n=\"1.236.0\",i=\"PROD\",o=\"CDN\"},385:(e,t,r)=>{r.d(t,{FN:()=>a,IF:()=>u,Nk:()=>f,Tt:()=>s,_A:()=>o,il:()=>n,pL:()=>c,v6:()=>i,w1:()=>d});const n=\"undefined\"!=typeof window&&!!window.document,i=\"undefined\"!=typeof WorkerGlobalScope&&(\"undefined\"!=typeof self&&self instanceof WorkerGlobalScope&&self.navigator instanceof WorkerNavigator||\"undefined\"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis.navigator instanceof WorkerNavigator),o=n?window:\"undefined\"!=typeof WorkerGlobalScope&&(\"undefined\"!=typeof self&&self instanceof WorkerGlobalScope&&self||\"undefined\"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis),a=\"\"+o?.location,s=/iPad|iPhone|iPod/.test(navigator.userAgent),c=s&&\"undefined\"==typeof SharedWorker,u=(()=>{const e=navigator.userAgent.match(/Firefox[/\\s](\\d+\\.\\d+)/);return Array.isArray(e)&&e.length>=2?+e[1]:0})(),d=Boolean(n&&window.document.documentMode),f=!!navigator.sendBeacon},1117:(e,t,r)=>{r.d(t,{w:()=>o});var n=r(50);const i={agentIdentifier:\"\",ee:void 0};class o{constructor(e){try{if(\"object\"!=typeof e)return(0,n.Z)(\"shared context requires an object as input\");this.sharedContext={},Object.assign(this.sharedContext,i),Object.entries(e).forEach((e=>{let[t,r]=e;Object.keys(i).includes(t)&&(this.sharedContext[t]=r)}))}catch(e){(0,n.Z)(\"An error occured while setting SharedContext\",e)}}}},8e3:(e,t,r)=>{r.d(t,{L:()=>d,R:()=>c});var n=r(2177),i=r(1284),o=r(4322),a=r(3325);const s={};function c(e,t){const r={staged:!1,priority:a.p[t]||0};u(e),s[e].get(t)||s[e].set(t,r)}function u(e){e&&(s[e]||(s[e]=new Map))}function d(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:\"\",t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:\"feature\";if(u(e),!e||!s[e].get(t))return a(t);s[e].get(t).staged=!0;const r=[...s[e]];function a(t){const r=e?n.ee.get(e):n.ee,a=o.X.handlers;if(r.backlog&&a){var s=r.backlog[t],c=a[t];if(c){for(var u=0;s&&u {let[t,r]=e;return r.staged}))&&(r.sort(((e,t)=>e[1].priority-t[1].priority)),r.forEach((e=>{let[t]=e;a(t)})))}function f(e,t){var r=e[1];(0,i.D)(t[r],(function(t,r){var n=e[0];if(r[0]===n){var i=r[1],o=e[3],a=e[2];i.apply(o,a)}}))}},2177:(e,t,r)=>{r.d(t,{c:()=>f,ee:()=>u});var n=r(8632),i=r(2210),o=r(1284),a=r(5763),s=\"nr@context\";let c=(0,n.fP)();var u;function d(){}function f(e){return(0,i.X)(e,s,l)}function l(){return new d}function h(){u.aborted=!0,u.backlog={}}c.ee?u=c.ee:(u=function e(t,r){var n={},c={},f={},g=!1;try{g=16===r.length&&(0,a.OP)(r).isolatedBacklog}catch(e){}var p={on:b,addEventListener:b,removeEventListener:y,emit:v,get:x,listeners:w,context:m,buffer:A,abort:h,aborted:!1,isBuffering:E,debugId:r,backlog:g?{}:t&&\"object\"==typeof t.backlog?t.backlog:{}};return p;function m(e){return e&&e instanceof d?e:e?(0,i.X)(e,s,l):l()}function v(e,r,n,i,o){if(!1!==o&&(o=!0),!u.aborted||i){t&&o&&t.emit(e,r,n);for(var a=m(n),s=w(e),d=s.length,f=0;f<d;f++)s[f].apply(a,r);var l=T()[c[e]];return l&&l.push([p,e,r,a]),a}}function b(e,t){n[e]=w(e).concat(t)}function y(e,t){var r=n[e];if(r)for(var i=0;i {r.d(t,{E:()=>n,p:()=>i});var n=r(2177).ee.get(\"handle\");function i(e,t,r,i,o){o?(o.buffer([e],i),o.emit(e,t,r)):(n.buffer([e],i),n.emit(e,t,r))}},4322:(e,t,r)=>{r.d(t,{X:()=>o});var n=r(5546);o.on=a;var i=o.handlers={};function o(e,t,r,o){a(o||n.E,i,e,t,r)}function a(e,t,r,i,o){o||(o=\"feature\"),e||(e=n.E);var a=t[o]=t[o]||{};(a[r]=a[r]||[]).push([e,i])}},3239:(e,t,r)=>{r.d(t,{bP:()=>s,iz:()=>c,m$:()=>a});var n=r(385);let i=!1,o=!1;try{const e={get passive(){return i=!0,!1},get signal(){return o=!0,!1}};n._A.addEventListener(\"test\",null,e),n._A.removeEventListener(\"test\",null,e)}catch(e){}function a(e,t){return i||o?{capture:!!e,passive:i,signal:t}:!!e}function s(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;window.addEventListener(e,t,a(r,n))}function c(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;document.addEventListener(e,t,a(r,n))}},4402:(e,t,r)=>{r.d(t,{Ht:()=>u,M:()=>c,Rl:()=>a,ky:()=>s});var n=r(385);const i=\"xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx\";function o(e,t){return e?15&e[t]:16*Math.random()|0}function a(){const e=n._A?.crypto||n._A?.msCrypto;let t,r=0;return e&&e.getRandomValues&&(t=e.getRandomValues(new Uint8Array(31))),i.split(\"\").map((e=>\"x\"===e?o(t,++r).toString(16):\"y\"===e?(3&o()|8).toString(16):e)).join(\"\")}function s(e){const t=n._A?.crypto||n._A?.msCrypto;let r,i=0;t&&t.getRandomValues&&(r=t.getRandomValues(new Uint8Array(31)));const a=[];for(var s=0;s {r.d(t,{Bq:()=>n,Hb:()=>o,oD:()=>i});const n=\"NRBA\",i=144e5,o=18e5},7894:(e,t,r)=>{function n(){return Math.round(performance.now())}r.d(t,{z:()=>n})},7243:(e,t,r)=>{r.d(t,{e:()=>o});var n=r(385),i={};function o(e){if(e in i)return i[e];if(0===(e||\"\").indexOf(\"data:\"))return{protocol:\"data\"};let t;var r=n._A?.location,o={};if(n.il)t=document.createElement(\"a\"),t.href=e;else try{t=new URL(e,r.href)}catch(e){return o}o.port=t.port;var a=t.href.split(\"://\");!o.port&&a[1]&&(o.port=a[1].split(\"/\")[0].split(\"@\").pop().split(\":\")[1]),o.port&&\"0\"!==o.port||(o.port=\"https\"===a[0]?\"443\":\"80\"),o.hostname=t.hostname||r.hostname,o.pathname=t.pathname,o.protocol=a[0],\"/\"!==o.pathname.charAt(0)&&(o.pathname=\"/\"+o.pathname);var s=!t.protocol||\":\"===t.protocol||t.protocol===r.protocol,c=t.hostname===r.hostname&&t.port===r.port;return o.sameOrigin=s&&(!t.hostname||c),\"/\"===o.pathname&&(i[e]=o),o}},50:(e,t,r)=>{function n(e,t){\"function\"==typeof console.warn&&(console.warn(\"New Relic: \".concat(e)),t&&console.warn(t))}r.d(t,{Z:()=>n})},2587:(e,t,r)=>{r.d(t,{N:()=>c,T:()=>u});var n=r(2177),i=r(5546),o=r(8e3),a=r(3325);const s={stn:[a.D.sessionTrace],err:[a.D.jserrors,a.D.metrics],ins:[a.D.pageAction],spa:[a.D.spa],sr:[a.D.sessionReplay,a.D.sessionTrace]};function c(e,t){const r=n.ee.get(t);e&&\"object\"==typeof e&&(Object.entries(e).forEach((e=>{let[t,n]=e;void 0===u[t]&&(s[t]?s[t].forEach((e=>{n?(0,i.p)(\"feat-\"+t,[],void 0,e,r):(0,i.p)(\"block-\"+t,[],void 0,e,r),(0,i.p)(\"rumresp-\"+t,[Boolean(n)],void 0,e,r)})):n&&(0,i.p)(\"feat-\"+t,[],void 0,void 0,r),u[t]=Boolean(n))})),Object.keys(s).forEach((e=>{void 0===u[e]&&(s[e]?.forEach((t=>(0,i.p)(\"rumresp-\"+e,[!1],void 0,t,r))),u[e]=!1)})),(0,o.L)(t,a.D.pageViewEvent))}const u={}},2210:(e,t,r)=>{r.d(t,{X:()=>i});var n=Object.prototype.hasOwnProperty;function i(e,t,r){if(n.call(e,t))return e[t];var i=r();if(Object.defineProperty&&Object.keys)try{return Object.defineProperty(e,t,{value:i,writable:!0,enumerable:!1}),i}catch(e){}return e[t]=i,i}},1284:(e,t,r)=>{r.d(t,{D:()=>n});const n=(e,t)=>Object.entries(e||{}).map((e=>{let[r,n]=e;return t(r,n)}))},4351:(e,t,r)=>{r.d(t,{P:()=>o});var n=r(2177);const i=()=>{const e=new WeakSet;return(t,r)=>{if(\"object\"==typeof r&&null!==r){if(e.has(r))return;e.add(r)}return r}};function o(e){try{return JSON.stringify(e,i())}catch(e){try{n.ee.emit(\"internal-error\",[e])}catch(e){}}}},3960:(e,t,r)=>{r.d(t,{K:()=>a,b:()=>o});var n=r(3239);function i(){return\"undefined\"==typeof document||\"complete\"===document.readyState}function o(e,t){if(i())return e();(0,n.bP)(\"load\",e,t)}function a(e){if(i())return e();(0,n.iz)(\"DOMContentLoaded\",e)}},8632:(e,t,r)=>{r.d(t,{EZ:()=>u,Qy:()=>c,ce:()=>o,fP:()=>a,gG:()=>d,mF:()=>s});var n=r(7894),i=r(385);const o={beacon:\"bam.nr-data.net\",errorBeacon:\"bam.nr-data.net\"};function a(){return i._A.NREUM||(i._A.NREUM={}),void 0===i._A.newrelic&&(i._A.newrelic=i._A.NREUM),i._A.NREUM}function s(){let e=a();return e.o||(e.o={ST:i._A.setTimeout,SI:i._A.setImmediate,CT:i._A.clearTimeout,XHR:i._A.XMLHttpRequest,REQ:i._A.Request,EV:i._A.Event,PR:i._A.Promise,MO:i._A.MutationObserver,FETCH:i._A.fetch}),e}function c(e,t,r){let i=a();const o=i.initializedAgents||{},s=o[e]||{};return Object.keys(s).length||(s.initializedAt={ms:(0,n.z)(),date:new Date}),i.initializedAgents={...o,[e]:{...s,[r]:t}},i}function u(e,t){a()[e]=t}function d(){return function(){let e=a();const t=e.info||{};e.info={beacon:o.beacon,errorBeacon:o.errorBeacon,...t}}(),function(){let e=a();const t=e.init||{};e.init={...t}}(),s(),function(){let e=a();const t=e.loader_config||{};e.loader_config={...t}}(),a()}},7956:(e,t,r)=>{r.d(t,{N:()=>i});var n=r(3239);function i(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],r=arguments.length>2?arguments[2]:void 0,i=arguments.length>3?arguments[3]:void 0;return void(0,n.iz)(\"visibilitychange\",(function(){if(t)return void(\"hidden\"==document.visibilityState&&e());e(document.visibilityState)}),r,i)}},1214:(e,t,r)=>{r.d(t,{em:()=>v,u5:()=>N,QU:()=>S,_L:()=>I,Gm:()=>L,Lg:()=>M,gy:()=>U,BV:()=>Q,Kf:()=>ee});var n=r(2177);const i=\"nr@original\";var o=Object.prototype.hasOwnProperty,a=!1;function s(e,t){return e||(e=n.ee),r.inPlace=function(e,t,n,i,o){n||(n=\"\");var a,s,c,u=\"-\"===n.charAt(0);for(c=0;c 2?n-2:0),o=2;o {r(A[T],e,w),r(E[T],e,w)})),r(l._A,\"fetch\",y),t.on(y+\"end\",(function(e,r){var n=this;if(r){var i=r.headers.get(\"content-length\");null!==i&&(n.rxSize=i),t.emit(y+\"done\",[null,r],n)}else t.emit(y+\"done\",[e],n)})),t}const O={},j=[\"pushState\",\"replaceState\"];function S(e){const t=function(e){return(e||n.ee).get(\"history\")}(e);return!l.il||O[t.debugId]++||(O[t.debugId]=1,s(t).inPlace(window.history,j,\"-\")),t}var P=r(3239);const C={},R=[\"appendChild\",\"insertBefore\",\"replaceChild\"];function I(e){const t=function(e){return(e||n.ee).get(\"jsonp\")}(e);if(!l.il||C[t.debugId])return t;C[t.debugId]=!0;var r=s(t),i=/[?&](?:callback|cb)=([^&#]+)/,o=/(.*)\\.([^.]+)/,a=/^(\\w+)(\\.|$)(.*)$/;function c(e,t){var r=e.match(a),n=r[1],i=r[3];return i?c(i,t[n]):t[n]}return r.inPlace(Node.prototype,R,\"dom-\"),t.on(\"dom-start\",(function(e){!function(e){if(!e||\"string\"!=typeof e.nodeName||\"script\"!==e.nodeName.toLowerCase())return;if(\"function\"!=typeof e.addEventListener)return;var n=(a=e.src,s=a.match(i),s?s[1]:null);var a,s;if(!n)return;var u=function(e){var t=e.match(o);if(t&&t.length>=3)return{key:t[2],parent:c(t[1],window)};return{key:e,parent:window}}(n);if(\"function\"!=typeof u.parent[u.key])return;var d={};function f(){t.emit(\"jsonp-end\",[],d),e.removeEventListener(\"load\",f,(0,P.m$)(!1)),e.removeEventListener(\"error\",l,(0,P.m$)(!1))}function l(){t.emit(\"jsonp-error\",[],d),t.emit(\"jsonp-end\",[],d),e.removeEventListener(\"load\",f,(0,P.m$)(!1)),e.removeEventListener(\"error\",l,(0,P.m$)(!1))}r.inPlace(u.parent,[u.key],\"cb-\",d),e.addEventListener(\"load\",f,(0,P.m$)(!1)),e.addEventListener(\"error\",l,(0,P.m$)(!1)),t.emit(\"new-jsonp\",[e.src],d)}(e[0])})),t}var k=r(5763);const H={};function L(e){const t=function(e){return(e||n.ee).get(\"mutation\")}(e);if(!l.il||H[t.debugId])return t;H[t.debugId]=!0;var r=s(t),i=k.Yu.MO;return i&&(window.MutationObserver=function(e){return this instanceof i?new i(r(e,\"fn-\")):i.apply(this,arguments)},MutationObserver.prototype=i.prototype),t}const z={};function M(e){const t=function(e){return(e||n.ee).get(\"promise\")}(e);if(z[t.debugId])return t;z[t.debugId]=!0;var r=n.c,o=s(t),a=k.Yu.PR;return a&&function(){function e(r){var n=t.context(),i=o(r,\"executor-\",n,null,!1);const s=Reflect.construct(a,[i],e);return t.context(s).getCtx=function(){return n},s}l._A.Promise=e,Object.defineProperty(e,\"name\",{value:\"Promise\"}),e.toString=function(){return a.toString()},Object.setPrototypeOf(e,a),[\"all\",\"race\"].forEach((function(r){const n=a[r];e[r]=function(e){let i=!1;[...e||[]].forEach((e=>{this.resolve(e).then(a(\"all\"===r),a(!1))}));const o=n.apply(this,arguments);return o;function a(e){return function(){t.emit(\"propagate\",[null,!i],o,!1,!1),i=i||!e}}}})),[\"resolve\",\"reject\"].forEach((function(r){const n=a[r];e[r]=function(e){const r=n.apply(this,arguments);return e!==r&&t.emit(\"propagate\",[e,!0],r,!1,!1),r}})),e.prototype=a.prototype;const n=a.prototype.then;a.prototype.then=function(){var e=this,i=r(e);i.promise=e;for(var a=arguments.length,s=new Array(a),c=0;c e())),t};function m(e,t){i.inPlace(t,[\"onreadystatechange\"],\"fn-\",E)}function b(){var e=this,t=r.context(e);e.readyState>3&&!t.resolved&&(t.resolved=!0,r.emit(\"xhr-resolved\",[],e)),i.inPlace(e,f,\"fn-\",E)}if(function(e,t){for(var r in e)t[r]=e[r]}(o,p),p.prototype=o.prototype,i.inPlace(p.prototype,J,\"-xhr-\",E),r.on(\"send-xhr-start\",(function(e,t){m(e,t),function(e){h.push(e),a&&(y?y.then(A):u?u(A):(w=-w,x.data=w))}(t)})),r.on(\"open-xhr-start\",m),a){var y=c&&c.resolve();if(!u&&!c){var w=1,x=document.createTextNode(w);new a(A).observe(x,{characterData:!0})}}else t.on(\"fn-end\",(function(e){e[0]&&e[0].type===d||A()}));function A(){for(var e=0;e {r.d(t,{t:()=>n});const n=r(3325).D.ajax},6660:(e,t,r)=>{r.d(t,{A:()=>i,t:()=>n});const n=r(3325).D.jserrors,i=\"nr@seenError\"},3081:(e,t,r)=>{r.d(t,{gF:()=>o,mY:()=>i,t9:()=>n,vz:()=>s,xS:()=>a});const n=r(3325).D.metrics,i=\"sm\",o=\"cm\",a=\"storeSupportabilityMetrics\",s=\"storeEventMetrics\"},4649:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageAction},7633:(e,t,r)=>{r.d(t,{Dz:()=>i,OJ:()=>a,qw:()=>o,t9:()=>n});const n=r(3325).D.pageViewEvent,i=\"firstbyte\",o=\"domcontent\",a=\"windowload\"},9251:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageViewTiming},3614:(e,t,r)=>{r.d(t,{BST_RESOURCE:()=>i,END:()=>s,FEATURE_NAME:()=>n,FN_END:()=>u,FN_START:()=>c,PUSH_STATE:()=>d,RESOURCE:()=>o,START:()=>a});const n=r(3325).D.sessionTrace,i=\"bstResource\",o=\"resource\",a=\"-start\",s=\"-end\",c=\"fn\"+a,u=\"fn\"+s,d=\"pushState\"},7836:(e,t,r)=>{r.d(t,{BODY:()=>A,CB_END:()=>E,CB_START:()=>u,END:()=>x,FEATURE_NAME:()=>i,FETCH:()=>_,FETCH_BODY:()=>v,FETCH_DONE:()=>m,FETCH_START:()=>p,FN_END:()=>c,FN_START:()=>s,INTERACTION:()=>l,INTERACTION_API:()=>d,INTERACTION_EVENTS:()=>o,JSONP_END:()=>b,JSONP_NODE:()=>g,JS_TIME:()=>T,MAX_TIMER_BUDGET:()=>a,REMAINING:()=>f,SPA_NODE:()=>h,START:()=>w,originalSetTimeout:()=>y});var n=r(5763);const i=r(3325).D.spa,o=[\"click\",\"submit\",\"keypress\",\"keydown\",\"keyup\",\"change\"],a=999,s=\"fn-start\",c=\"fn-end\",u=\"cb-start\",d=\"api-ixn-\",f=\"remaining\",l=\"interaction\",h=\"spaNode\",g=\"jsonpNode\",p=\"fetch-start\",m=\"fetch-done\",v=\"fetch-body-\",b=\"jsonp-end\",y=n.Yu.ST,w=\"-start\",x=\"-end\",A=\"-body\",E=\"cb\"+x,T=\"jsTime\",_=\"fetch\"},5938:(e,t,r)=>{r.d(t,{W:()=>o});var n=r(5763),i=r(2177);class o{constructor(e,t,r){this.agentIdentifier=e,this.aggregator=t,this.ee=i.ee.get(e,(0,n.OP)(this.agentIdentifier).isolatedBacklog),this.featureName=r,this.blocked=!1}}},9144:(e,t,r)=>{r.d(t,{j:()=>m});var n=r(3325),i=r(5763),o=r(5546),a=r(2177),s=r(7894),c=r(8e3),u=r(3960),d=r(385),f=r(50),l=r(3081),h=r(8632);function g(){const e=(0,h.gG)();[\"setErrorHandler\",\"finished\",\"addToTrace\",\"inlineHit\",\"addRelease\",\"addPageAction\",\"setCurrentRouteName\",\"setPageViewName\",\"setCustomAttribute\",\"interaction\",\"noticeError\",\"setUserId\"].forEach((t=>{e[t]=function(){for(var r=arguments.length,n=new Array(r),i=0;i 1?r-1:0),i=1;i {e.exposed&&e.api[t]&&o.push(e.api[t](...n))})),o.length>1?o:o[0]}(t,...n)}}))}var p=r(2587);function m(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},m=arguments.length>2?arguments[2]:void 0,v=arguments.length>3?arguments[3]:void 0,{init:b,info:y,loader_config:w,runtime:x={loaderType:m},exposed:A=!0}=t;const E=(0,h.gG)();y||(b=E.init,y=E.info,w=E.loader_config),(0,i.Dg)(e,b||{}),(0,i.GE)(e,w||{}),(0,i.sU)(e,x),y.jsAttributes??={},d.v6&&(y.jsAttributes.isWorker=!0),(0,i.CX)(e,y),g();const T=function(e,t){t||(0,c.R)(e,\"api\");const h={};var g=a.ee.get(e),p=g.get(\"tracer\"),m=\"api-\",v=m+\"ixn-\";function b(t,r,n,o){const a=(0,i.C5)(e);return null===r?delete a.jsAttributes[t]:(0,i.CX)(e,{...a,jsAttributes:{...a.jsAttributes,[t]:r}}),x(m,n,!0,o||null===r?\"session\":void 0)(t,r)}function y(){}[\"setErrorHandler\",\"finished\",\"addToTrace\",\"inlineHit\",\"addRelease\"].forEach((e=>h[e]=x(m,e,!0,\"api\"))),h.addPageAction=x(m,\"addPageAction\",!0,n.D.pageAction),h.setCurrentRouteName=x(m,\"routeName\",!0,n.D.spa),h.setPageViewName=function(t,r){if(\"string\"==typeof t)return\"/\"!==t.charAt(0)&&(t=\"/\"+t),(0,i.OP)(e).customTransaction=(r||\"http://custom.transaction\")+t,x(m,\"setPageViewName\",!0)()},h.setCustomAttribute=function(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2];if(\"string\"==typeof e){if([\"string\",\"number\"].includes(typeof t)||null===t)return b(e,t,\"setCustomAttribute\",r);(0,f.Z)(\"Failed to execute setCustomAttribute.\\nNon-null value must be a string or number type, but a type of was provided.\"))}else(0,f.Z)(\"Failed to execute setCustomAttribute.\\nName must be a string type, but a type of was provided.\"))},h.setUserId=function(e){if(\"string\"==typeof e||null===e)return b(\"enduser.id\",e,\"setUserId\",!0);(0,f.Z)(\"Failed to execute setUserId.\\nNon-null value must be a string type, but a type of was provided.\"))},h.interaction=function(){return(new y).get()};var w=y.prototype={createTracer:function(e,t){var r={},i=this,a=\"function\"==typeof t;return(0,o.p)(v+\"tracer\",[(0,s.z)(),e,r],i,n.D.spa,g),function(){if(p.emit((a?\"\":\"no-\")+\"fn-start\",[(0,s.z)(),i,a],r),a)try{return t.apply(this,arguments)}catch(e){throw p.emit(\"fn-err\",[arguments,this,\"string\"==typeof e?new Error(e):e],r),e}finally{p.emit(\"fn-end\",[(0,s.z)()],r)}}}};function x(e,t,r,i){return function(){return(0,o.p)(l.xS,[\"API/\"+t+\"/called\"],void 0,n.D.metrics,g),i&&(0,o.p)(e+t,[(0,s.z)(),...arguments],r?null:this,i,g),r?void 0:this}}function A(){r.e(439).then(r.bind(r,7438)).then((t=>{let{setAPI:r}=t;r(e),(0,c.L)(e,\"api\")})).catch((()=>(0,f.Z)(\"Downloading runtime APIs failed...\")))}return[\"actionText\",\"setName\",\"setAttribute\",\"save\",\"ignore\",\"onEnd\",\"getContext\",\"end\",\"get\"].forEach((e=>{w[e]=x(v,e,void 0,n.D.spa)})),h.noticeError=function(e,t){\"string\"==typeof e&&(e=new Error(e)),(0,o.p)(l.xS,[\"API/noticeError/called\"],void 0,n.D.metrics,g),(0,o.p)(\"err\",[e,(0,s.z)(),!1,t],void 0,n.D.jserrors,g)},d.il?(0,u.b)((()=>A()),!0):A(),h}(e,v);return(0,h.Qy)(e,T,\"api\"),(0,h.Qy)(e,A,\"exposed\"),(0,h.EZ)(\"activatedFeatures\",p.T),T}},3325:(e,t,r)=>{r.d(t,{D:()=>n,p:()=>i});const n={ajax:\"ajax\",jserrors:\"jserrors\",metrics:\"metrics\",pageAction:\"page_action\",pageViewEvent:\"page_view_event\",pageViewTiming:\"page_view_timing\",sessionReplay:\"session_replay\",sessionTrace:\"session_trace\",spa:\"spa\"},i={[n.pageViewEvent]:1,[n.pageViewTiming]:2,[n.metrics]:3,[n.jserrors]:4,[n.ajax]:5,[n.sessionTrace]:6,[n.pageAction]:7,[n.spa]:8,[n.sessionReplay]:9}}},n={};function i(e){var t=n[e];if(void 0!==t)return t.exports;var o=n[e]={exports:{}};return r[e](o,o.exports,i),o.exports}i.m=r,i.d=(e,t)=>{for(var r in t)i.o(t,r)&&!i.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},i.f={},i.e=e=>Promise.all(Object.keys(i.f).reduce(((t,r)=>(i.f[r](e,t),t)),[])),i.u=e=>(({78:\"page_action-aggregate\",147:\"metrics-aggregate\",242:\"session-manager\",317:\"jserrors-aggregate\",348:\"page_view_timing-aggregate\",412:\"lazy-feature-loader\",439:\"async-api\",538:\"recorder\",590:\"session_replay-aggregate\",675:\"compressor\",733:\"session_trace-aggregate\",786:\"page_view_event-aggregate\",873:\"spa-aggregate\",898:\"ajax-aggregate\"}[e]||e)+\".\"+{78:\"ac76d497\",147:\"3dc53903\",148:\"1a20d5fe\",242:\"2a64278a\",317:\"49e41428\",348:\"bd6de33a\",412:\"2f55ce66\",439:\"30bd804e\",538:\"1b18459f\",590:\"cf0efb30\",675:\"ae9f91a8\",733:\"83105561\",786:\"06482edd\",860:\"03a8b7a5\",873:\"e6b09d52\",898:\"998ef92b\"}[e]+\"-1.236.0.min.js\"),i.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),e={},t=\"NRBA:\",i.l=(r,n,o,a)=>{if(e[r])e[r].push(n);else{var s,c;if(void 0!==o)for(var u=document.getElementsByTagName(\"script\"),d=0;d {s.onerror=s.onload=null,clearTimeout(h);var i=e[r];if(delete e[r],s.parentNode&&s.parentNode.removeChild(s),i&&i.forEach((e=>e(n))),t)return t(n)},h=setTimeout(l.bind(null,void 0,{type:\"timeout\",target:s}),12e4);s.onerror=l.bind(null,s.onerror),s.onload=l.bind(null,s.onload),c&&document.head.appendChild(s)}},i.r=e=>{\"undefined\"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:\"Module\"}),Object.defineProperty(e,\"__esModule\",{value:!0})},i.j=364,i.p=\"https://js-agent.newrelic.com/\",(()=>{var e={364:0,953:0};i.f.j=(t,r)=>{var n=i.o(e,t)?e[t]:void 0;if(0!==n)if(n)r.push(n[2]);else{var o=new Promise(((r,i)=>n=e[t]=[r,i]));r.push(n[2]=o);var a=i.p+i.u(t),s=new Error;i.l(a,(r=>{if(i.o(e,t)&&(0!==(n=e[t])&&(e[t]=void 0),n)){var o=r&&(\"load\"===r.type?\"missing\":r.type),a=r&&r.target&&r.target.src;s.message=\"Loading chunk \"+t+\" failed.\\n(\"+o+\": \"+a+\")\",s.name=\"ChunkLoadError\",s.type=o,s.request=a,n[1](s)}}),\"chunk-\"+t,t)}};var t=(t,r)=>{var n,o,[a,s,c]=r,u=0;if(a.some((t=>0!==e[t]))){for(n in s)i.o(s,n)&&(i.m[n]=s[n]);if(c)c(i)}for(t&&t(r);u {i.r(o);var e=i(3325),t=i(5763);const r=Object.values(e.D);function n(e){const n={};return r.forEach((r=>{n[r]=function(e,r){return!1!==(0,t.Mt)(r,\"\".concat(e,\".enabled\"))}(r,e)})),n}var a=i(9144);var s=i(5546),c=i(385),u=i(8e3),d=i(5938),f=i(3960),l=i(50);class h extends d.W{constructor(e,t,r){let n=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];super(e,t,r),this.auto=n,this.abortHandler,this.featAggregate,this.onAggregateImported,n&&(0,u.R)(e,r)}importAggregator(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};if(this.featAggregate||!this.auto)return;const r=c.il&&!0===(0,t.Mt)(this.agentIdentifier,\"privacy.cookies_enabled\");let n;this.onAggregateImported=new Promise((e=>{n=e}));const o=async()=>{let t;try{if(r){const{setupAgentSession:e}=await Promise.all([i.e(860),i.e(242)]).then(i.bind(i,3228));t=e(this.agentIdentifier)}}catch(e){(0,l.Z)(\"A problem occurred when starting up session manager. This page will not start or extend any session.\",e)}try{if(!this.shouldImportAgg(this.featureName,t))return void(0,u.L)(this.agentIdentifier,this.featureName);const{lazyFeatureLoader:r}=await i.e(412).then(i.bind(i,8582)),{Aggregate:o}=await r(this.featureName,\"aggregate\");this.featAggregate=new o(this.agentIdentifier,this.aggregator,e),n(!0)}catch(e){(0,l.Z)(\"Downloading and initializing \".concat(this.featureName,\" failed...\"),e),this.abortHandler?.(),n(!1)}};c.il?(0,f.b)((()=>o()),!0):o()}shouldImportAgg(r,n){return r!==e.D.sessionReplay||!1!==(0,t.Mt)(this.agentIdentifier,\"session_trace.enabled\")&&(!!n?.isNew||!!n?.state.sessionReplay)}}var g=i(7633),p=i(7894);class m extends h{static featureName=g.t9;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];if(super(r,n,g.t9,i),(\"undefined\"==typeof PerformanceNavigationTiming||c.Tt)&&\"undefined\"!=typeof PerformanceTiming){const n=(0,t.OP)(r);n[g.Dz]=Math.max(Date.now()-n.offset,0),(0,f.K)((()=>n[g.qw]=Math.max((0,p.z)()-n[g.Dz],0))),(0,f.b)((()=>{const t=(0,p.z)();n[g.OJ]=Math.max(t-n[g.Dz],0),(0,s.p)(\"timing\",[\"load\",t],void 0,e.D.pageViewTiming,this.ee)}))}this.importAggregator()}}var v=i(1117),b=i(1284);class y extends v.w{constructor(e){super(e),this.aggregatedData={}}store(e,t,r,n,i){var o=this.getBucket(e,t,r,i);return o.metrics=function(e,t){t||(t={count:0});return t.count+=1,(0,b.D)(e,(function(e,r){t[e]=w(r,t[e])})),t}(n,o.metrics),o}merge(e,t,r,n,i){var o=this.getBucket(e,t,n,i);if(o.metrics){var a=o.metrics;a.count+=r.count,(0,b.D)(r,(function(e,t){if(\"count\"!==e){var n=a[e],i=r[e];i&&!i.c?a[e]=w(i.t,n):a[e]=function(e,t){if(!t)return e;t.c||(t=x(t.t));return t.min=Math.min(e.min,t.min),t.max=Math.max(e.max,t.max),t.t+=e.t,t.sos+=e.sos,t.c+=e.c,t}(i,a[e])}}))}else o.metrics=r}storeMetric(e,t,r,n){var i=this.getBucket(e,t,r);return i.stats=w(n,i.stats),i}getBucket(e,t,r,n){this.aggregatedData[e]||(this.aggregatedData[e]={});var i=this.aggregatedData[e][t];return i||(i=this.aggregatedData[e][t]={params:r||{}},n&&(i.custom=n)),i}get(e,t){return t?this.aggregatedData[e]&&this.aggregatedData[e][t]:this.aggregatedData[e]}take(e){for(var t={},r=\"\",n=!1,i=0;i t.max&&(t.max=e),e 2&&void 0!==arguments[2])||arguments[2];super(e,r,j.t,n),c.il&&((0,t.OP)(e).initHidden=Boolean(\"hidden\"===document.visibilityState),(0,N.N)((()=>(0,s.p)(\"docHidden\",[(0,p.z)()],void 0,j.t,this.ee)),!0),(0,O.bP)(\"pagehide\",(()=>(0,s.p)(\"winPagehide\",[(0,p.z)()],void 0,j.t,this.ee))),this.importAggregator())}}var P=i(3081);class C extends h{static featureName=P.t9;constructor(e,t){let r=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(e,t,P.t9,r),this.importAggregator()}}var R,I=i(2210),k=i(1214),H=i(2177),L={};try{R=localStorage.getItem(\"__nr_flags\").split(\",\"),console&&\"function\"==typeof console.log&&(L.console=!0,-1!==R.indexOf(\"dev\")&&(L.dev=!0),-1!==R.indexOf(\"nr_dev\")&&(L.nrDev=!0))}catch(e){}function z(e){try{L.console&&z(e)}catch(e){}}L.nrDev&&H.ee.on(\"internal-error\",(function(e){z(e.stack)})),L.dev&&H.ee.on(\"fn-err\",(function(e,t,r){z(r.stack)})),L.dev&&(z(\"NR AGENT IN DEVELOPMENT MODE\"),z(\"flags: \"+(0,b.D)(L,(function(e,t){return e})).join(\", \")));var M=i(6660);class B extends h{static featureName=M.t;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(r,n,M.t,i),this.skipNext=0;try{this.removeOnAbort=new AbortController}catch(e){}const o=this;o.ee.on(\"fn-start\",(function(e,t,r){o.abortHandler&&(o.skipNext+=1)})),o.ee.on(\"fn-err\",(function(t,r,n){o.abortHandler&&!n[M.A]&&((0,I.X)(n,M.A,(function(){return!0})),this.thrown=!0,(0,s.p)(\"err\",[n,(0,p.z)()],void 0,e.D.jserrors,o.ee))})),o.ee.on(\"fn-end\",(function(){o.abortHandler&&!this.thrown&&o.skipNext>0&&(o.skipNext-=1)})),o.ee.on(\"internal-error\",(function(t){(0,s.p)(\"ierr\",[t,(0,p.z)(),!0],void 0,e.D.jserrors,o.ee)})),this.origOnerror=c._A.onerror,c._A.onerror=this.onerrorHandler.bind(this),c._A.addEventListener(\"unhandledrejection\",(t=>{const r=function(e){let t=\"Unhandled Promise Rejection: \";if(e instanceof Error)try{return e.message=t+e.message,e}catch(t){return e}if(void 0===e)return new Error(t);try{return new Error(t+(0,D.P)(e))}catch(e){return new Error(t)}}(t.reason);(0,s.p)(\"err\",[r,(0,p.z)(),!1,{unhandledPromiseRejection:1}],void 0,e.D.jserrors,this.ee)}),(0,O.m$)(!1,this.removeOnAbort?.signal)),(0,k.gy)(this.ee),(0,k.BV)(this.ee),(0,k.em)(this.ee),(0,t.OP)(r).xhrWrappable&&(0,k.Kf)(this.ee),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}onerrorHandler(t,r,n,i,o){\"function\"==typeof this.origOnerror&&this.origOnerror(...arguments);try{this.skipNext?this.skipNext-=1:(0,s.p)(\"err\",[o||new F(t,r,n),(0,p.z)()],void 0,e.D.jserrors,this.ee)}catch(t){try{(0,s.p)(\"ierr\",[t,(0,p.z)(),!0],void 0,e.D.jserrors,this.ee)}catch(e){}}return!1}}function F(e,t,r){this.message=e||\"Uncaught error with no additional information\",this.sourceURL=t,this.line=r}let U=1;const q=\"nr@id\";function G(e){const t=typeof e;return!e||\"object\"!==t&&\"function\"!==t?-1:e===c._A?0:(0,I.X)(e,q,(function(){return U++}))}function V(e){if(\"string\"==typeof e&&e.length)return e.length;if(\"object\"==typeof e){if(\"undefined\"!=typeof ArrayBuffer&&e instanceof ArrayBuffer&&e.byteLength)return e.byteLength;if(\"undefined\"!=typeof Blob&&e instanceof Blob&&e.size)return e.size;if(!(\"undefined\"!=typeof FormData&&e instanceof FormData))try{return(0,D.P)(e).length}catch(e){return}}}var X=i(7243);class W{constructor(e){this.agentIdentifier=e,this.generateTracePayload=this.generateTracePayload.bind(this),this.shouldGenerateTrace=this.shouldGenerateTrace.bind(this)}generateTracePayload(e){if(!this.shouldGenerateTrace(e))return null;var r=(0,t.DL)(this.agentIdentifier);if(!r)return null;var n=(r.accountID||\"\").toString()||null,i=(r.agentID||\"\").toString()||null,o=(r.trustKey||\"\").toString()||null;if(!n||!i)return null;var a=(0,_.M)(),s=(0,_.Ht)(),c=Date.now(),u={spanId:a,traceId:s,timestamp:c};return(e.sameOrigin||this.isAllowedOrigin(e)&&this.useTraceContextHeadersForCors())&&(u.traceContextParentHeader=this.generateTraceContextParentHeader(a,s),u.traceContextStateHeader=this.generateTraceContextStateHeader(a,c,n,i,o)),(e.sameOrigin&&!this.excludeNewrelicHeader()||!e.sameOrigin&&this.isAllowedOrigin(e)&&this.useNewrelicHeaderForCors())&&(u.newrelicHeader=this.generateTraceHeader(a,s,c,n,i,o)),u}generateTraceContextParentHeader(e,t){return\"00-\"+t+\"-\"+e+\"-01\"}generateTraceContextStateHeader(e,t,r,n,i){return i+\"@nr=0-1-\"+r+\"-\"+n+\"-\"+e+\"----\"+t}generateTraceHeader(e,t,r,n,i,o){if(!(\"function\"==typeof c._A?.btoa))return null;var a={v:[0,1],d:{ty:\"Browser\",ac:n,ap:i,id:e,tr:t,ti:r}};return o&&n!==o&&(a.d.tk=o),btoa((0,D.P)(a))}shouldGenerateTrace(e){return this.isDtEnabled()&&this.isAllowedOrigin(e)}isAllowedOrigin(e){var r=!1,n={};if((0,t.Mt)(this.agentIdentifier,\"distributed_tracing\")&&(n=(0,t.P_)(this.agentIdentifier).distributed_tracing),e.sameOrigin)r=!0;else if(n.allowed_origins instanceof Array)for(var i=0;i 2&&void 0!==arguments[2])||arguments[2];super(r,n,Z.t,i),(0,t.OP)(r).xhrWrappable&&(this.dt=new W(r),this.handler=(e,t,r,n)=>(0,s.p)(e,t,r,n,this.ee),(0,k.u5)(this.ee),(0,k.Kf)(this.ee),function(r,n,i,o){function a(e){var t=this;t.totalCbs=0,t.called=0,t.cbTime=0,t.end=E,t.ended=!1,t.xhrGuids={},t.lastSize=null,t.loadCaptureCalled=!1,t.params=this.params||{},t.metrics=this.metrics||{},e.addEventListener(\"load\",(function(r){_(t,e)}),(0,O.m$)(!1)),c.IF||e.addEventListener(\"progress\",(function(e){t.lastSize=e.loaded}),(0,O.m$)(!1))}function s(e){this.params={method:e[0]},T(this,e[1]),this.metrics={}}function u(e,n){var i=(0,t.DL)(r);i.xpid&&this.sameOrigin&&n.setRequestHeader(\"X-NewRelic-ID\",i.xpid);var a=o.generateTracePayload(this.parsedOrigin);if(a){var s=!1;a.newrelicHeader&&(n.setRequestHeader(\"newrelic\",a.newrelicHeader),s=!0),a.traceContextParentHeader&&(n.setRequestHeader(\"traceparent\",a.traceContextParentHeader),a.traceContextStateHeader&&n.setRequestHeader(\"tracestate\",a.traceContextStateHeader),s=!0),s&&(this.dt=a)}}function d(e,t){var r=this.metrics,i=e[0],o=this;if(r&&i){var a=V(i);a&&(r.txSize=a)}this.startTime=(0,p.z)(),this.listener=function(e){try{\"abort\"!==e.type||o.loadCaptureCalled||(o.params.aborted=!0),(\"load\"!==e.type||o.called===o.totalCbs&&(o.onloadCalled||\"function\"!=typeof t.onload)&&\"function\"==typeof o.end)&&o.end(t)}catch(e){try{n.emit(\"internal-error\",[e])}catch(e){}}};for(var s=0;s 1?e[1]=i:e.push(i)}else e[0]&&e[0].headers&&s(e[0].headers,n)&&(this.dt=n);function s(e,t){var r=!1;return t.newrelicHeader&&(e.set(\"newrelic\",t.newrelicHeader),r=!0),t.traceContextParentHeader&&(e.set(\"traceparent\",t.traceContextParentHeader),t.traceContextStateHeader&&e.set(\"tracestate\",t.traceContextStateHeader),r=!0),r}}function x(e,t){this.params={},this.metrics={},this.startTime=(0,p.z)(),this.dt=t,e.length>=1&&(this.target=e[0]),e.length>=2&&(this.opts=e[1]);var r,n=this.opts||{},i=this.target;\"string\"==typeof i?r=i:\"object\"==typeof i&&i instanceof Y?r=i.url:c._A?.URL&&\"object\"==typeof i&&i instanceof URL&&(r=i.href),T(this,r);var o=(\"\"+(i&&i instanceof Y&&i.method||n.method||\"GET\")).toUpperCase();this.params.method=o,this.txSize=V(n.body)||0}function A(t,r){var n;this.endTime=(0,p.z)(),this.params||(this.params={}),this.params.status=r?r.status:0,\"string\"==typeof this.rxSize&&this.rxSize.length>0&&(n=+this.rxSize);var o={txSize:this.txSize,rxSize:n,duration:(0,p.z)()-this.startTime};i(\"xhr\",[this.params,o,this.startTime,this.endTime,\"fetch\"],this,e.D.ajax)}function E(t){var r=this.params,n=this.metrics;if(!this.ended){this.ended=!0;for(var o=0;o 2&&void 0!==arguments[2])||arguments[2];super(e,t,we.t,r),this.importAggregator()}}new class{constructor(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:(0,_.ky)(16);c._A?(this.agentIdentifier=t,this.sharedAggregator=new y({agentIdentifier:this.agentIdentifier}),this.features={},this.desiredFeatures=new Set(e.features||[]),this.desiredFeatures.add(m),Object.assign(this,(0,a.j)(this.agentIdentifier,e,e.loaderType||\"agent\")),this.start()):(0,l.Z)(\"Failed to initial the agent. Could not determine the runtime environment.\")}get config(){return{info:(0,t.C5)(this.agentIdentifier),init:(0,t.P_)(this.agentIdentifier),loader_config:(0,t.DL)(this.agentIdentifier),runtime:(0,t.OP)(this.agentIdentifier)}}start(){const t=\"features\";try{const r=n(this.agentIdentifier),i=[...this.desiredFeatures];i.sort(((t,r)=>e.p[t.featureName]-e.p[r.featureName])),i.forEach((t=>{if(r[t.featureName]||t.featureName===e.D.pageViewEvent){const n=function(t){switch(t){case e.D.ajax:return[e.D.jserrors];case e.D.sessionTrace:return[e.D.ajax,e.D.pageViewEvent];case e.D.sessionReplay:return[e.D.sessionTrace];case e.D.pageViewTiming:return[e.D.pageViewEvent];default:return[]}}(t.featureName);n.every((e=>r[e]))||(0,l.Z)(\"\".concat(t.featureName,\" is enabled but one or more dependent features has been disabled (\").concat((0,D.P)(n),\"). This may cause unintended consequences or missing data...\")),this.features[t.featureName]=new t(this.agentIdentifier,this.sharedAggregator)}})),(0,T.Qy)(this.agentIdentifier,this.features,t)}catch(e){(0,l.Z)(\"Failed to initialize all enabled instrument classes (agent aborted) -\",e);for(const e in this.features)this.features[e].abortHandler?.();const r=(0,T.fP)();return delete r.initializedAgents[this.agentIdentifier]?.api,delete r.initializedAgents[this.agentIdentifier]?.[t],delete this.sharedAggregator,r.ee?.abort(),delete r.ee?.get(this.agentIdentifier),!1}}}({features:[J,m,S,class extends h{static featureName=oe;constructor(t,r){if(super(t,r,oe,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;const n=this.ee;let i;(0,k.QU)(n),this.eventsEE=(0,k.em)(n),this.eventsEE.on(se,(function(e,t){this.bstStart=(0,p.z)()})),this.eventsEE.on(ae,(function(t,r){(0,s.p)(\"bst\",[t[0],r,this.bstStart,(0,p.z)()],void 0,e.D.sessionTrace,n)})),n.on(ce+ne,(function(e){this.time=(0,p.z)(),this.startPath=location.pathname+location.hash})),n.on(ce+ie,(function(t){(0,s.p)(\"bstHist\",[location.pathname+location.hash,this.startPath,this.time],void 0,e.D.sessionTrace,n)}));try{i=new PerformanceObserver((t=>{const r=t.getEntries();(0,s.p)(te,[r],void 0,e.D.sessionTrace,n)})),i.observe({type:re,buffered:!0})}catch(e){}this.importAggregator({resourceObserver:i})}},C,xe,B,class extends h{static featureName=de;constructor(e,r){if(super(e,r,de,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;if(!(0,t.OP)(e).xhrWrappable)return;try{this.removeOnAbort=new AbortController}catch(e){}let n,i=0;const o=this.ee.get(\"tracer\"),a=(0,k._L)(this.ee),s=(0,k.Lg)(this.ee),u=(0,k.BV)(this.ee),d=(0,k.Kf)(this.ee),f=this.ee.get(\"events\"),l=(0,k.u5)(this.ee),h=(0,k.QU)(this.ee),g=(0,k.Gm)(this.ee);function m(e,t){h.emit(\"newURL\",[\"\"+window.location,t])}function v(){i++,n=window.location.hash,this[ve]=(0,p.z)()}function b(){i--,window.location.hash!==n&&m(0,!0);var e=(0,p.z)();this[pe]=~~this[pe]+e-this[ve],this[ye]=e}function y(e,t){e.on(t,(function(){this[t]=(0,p.z)()}))}this.ee.on(ve,v),s.on(be,v),a.on(be,v),this.ee.on(ye,b),s.on(ge,b),a.on(ge,b),this.ee.buffer([ve,ye,\"xhr-resolved\"],this.featureName),f.buffer([ve],this.featureName),u.buffer([\"setTimeout\"+le,\"clearTimeout\"+fe,ve],this.featureName),d.buffer([ve,\"new-xhr\",\"send-xhr\"+fe],this.featureName),l.buffer([me+fe,me+\"-done\",me+he+fe,me+he+le],this.featureName),h.buffer([\"newURL\"],this.featureName),g.buffer([ve],this.featureName),s.buffer([\"propagate\",be,ge,\"executor-err\",\"resolve\"+fe],this.featureName),o.buffer([ve,\"no-\"+ve],this.featureName),a.buffer([\"new-jsonp\",\"cb-start\",\"jsonp-error\",\"jsonp-end\"],this.featureName),y(l,me+fe),y(l,me+\"-done\"),y(a,\"new-jsonp\"),y(a,\"jsonp-end\"),y(a,\"cb-start\"),h.on(\"pushState-end\",m),h.on(\"replaceState-end\",m),window.addEventListener(\"hashchange\",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener(\"load\",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener(\"popstate\",(function(){m(0,i>1)}),(0,O.m$)(!0,this.removeOnAbort?.signal)),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}}],loaderType:\"spa\"})})(),window.NRBA=o})(); window.jQuery || document.write(' ') CKEDITOR_BASEPATH='https://f1000research.com/js/vendor/ckeditor/' window.reactTheme = 'research'; window.MathJax = { CommonHTML: { linebreaks: { automatic: true } }, 'HTML-CSS': { linebreaks: { automatic: true } }, SVG: { linebreaks: { automatic: true } }, AuthorInit: function() { MathJax.Hub.Register.MessageHook('End Process', function () { let timeout = false; // holder for timeout id const delay = 250; // delay after event is \"complete\" to run callback const reflowMath = function() { const dispFormulas = document.querySelectorAll('.disp-formula.panel'); if (!dispFormulas) { return; } for (const dispFormula of dispFormulas) { const child = dispFormula.querySelector('.MathJax_Preview').nextSibling.firstChild; const isMultiline = MathJax.Hub.getAllJax(dispFormula)[0].root.isMultiline; if (dispFormula.offsetWidth < child.offsetWidth || isMultiline) { MathJax.Hub.Queue(['Rerender', MathJax.Hub, dispFormula]); } } }; window.addEventListener('resize', function() { clearTimeout(timeout); // clear the timeout timeout = setTimeout(reflowMath, delay); // start timing for event \"completion\" }); }); }, }; if (window.location.hash == '#_=_'){ window.location = window.location.href.split('#')[0] } !function(f,b,e,v,n,t,s){if(f.fbq)return;n=f.fbq=function() {n.callMethod? n.callMethod.apply(n,arguments):n.queue.push(arguments)} ;if(!f._fbq)f._fbq=n; n.push=n;n.loaded=!0;n.version='2.0';n.queue=[];t=b.createElement(e);t.async=!0; t.src=v;s=b.getElementsByTagName(e)[0];s.parentNode.insertBefore(t,s)}(window, document,'script','https://connect.facebook.net/en_US/fbevents.js'); fbq('init', '1641728616063202'); fbq('track', \"PixelInitialized\", {}); (function(h,o,t,j,a,r){ h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)}; h._hjSettings={hjid:2318163,hjsv:6}; a=o.getElementsByTagName('head')[0]; r=o.createElement('script');r.async=1; r.src=t+h._hjSettings.hjid+j+h._hjSettings.hjsv; a.appendChild(r); })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv='); search file_upload Submit your research search menu close search Browse Gateways & Collections How to Publish Submit your Research My Submissions Article Guidelines Article Guidelines (New Versions) Open Data, Software and Code Guidelines Open Data and Accessible Source Materials Guidelines (HSS) Open Data, Software and Code Guidelines (PSE) Prepublication Checks Production Process Posters and Slides Guidelines Document Guidelines Article Processing Charges Peer Review Finding Article Reviewers About How it Works For Reviewers Our Advisors Policies Glossary FAQs For Developers Newsroom Contact My Research Submissions Content and Tracking Alerts My Details Sign In file_upload Submit your research { \"@context\": \"https://schema.org\", \"@type\": \"ScholarlyArticle\", \"mainEntityOfPage\": { \"@type\": \"WebPage\", \"@id\": \"https://f1000research.com/articles/13-1547\" }, \"headline\": \"Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure...\", \"datePublished\": \"2024-12-20T12:41:39\", \"dateModified\": \"2025-04-22T10:19:36\", \"author\": [ { \"@type\": \"Person\", \"name\": \"Francesca De Leo\" }, { \"@type\": \"Person\", \"name\": \"Erika Balsyte\" }, { \"@type\": \"Person\", \"name\": \"Robert Petryszak\" }, { \"@type\": \"Person\", \"name\": \"Marilena D’Ambrosio\" }, { \"@type\": \"Person\", \"name\": \"Chiara Bruno\" }, { \"@type\": \"Person\", \"name\": \"Martin Cook\" }, { \"@type\": \"Person\", \"name\": \"Ivan Mičetić\" }, { \"@type\": \"Person\", \"name\": \"Corinne S. Martin\" } ], \"publisher\": { \"@type\": \"Organization\", \"name\": \"F1000Research\", \"logo\": { \"@type\": \"ImageObject\", \"url\": \"https://f1000research.com/img/AMP/F1000Research_image.png\", \"height\": 480, \"width\": 60 } }, \"image\": { \"@type\": \"ImageObject\", \"url\": \"https://f1000research.com/img/AMP/F1000Research_image.png\", \"height\": 1200, \"width\": 150 }, \"description\": \" Background ELIXIR is a pan-European public-funded research infrastructure dedicated to life science data. As such, it must demonstrate public value to its funders and stakeholders. We present methods to inventory research publications linked to ELIXIR that have received funding and support, as well as related citation metrics, used as performance metrics for these audiences. Methods To overcome challenges inherent in ELIXIR&rsquo;s distributed structure, and the fact that those publishing ELIXIR-supported work are typically working part-time on ELIXIR matters, a semi-automated approach, consisting of text-mining followed by manual curation, is presented. A country-level case study (ELIXIR Italy) refines and expands the methods, notably by introducing more granularity in the curation process (e.g. considering all national-level grants, examining affiliations to report publication per institute) and by additionally looking at the scientific impact of the resources developed and operated by the Italian Node of ELIXIR. Results Overall, the methods described in this article have shown to: (1) be repeatable with acceptable levels of accuracy and consistency (notably across curators); (2) require reasonable effort in terms of curation of monthly &lsquo;harvests&rsquo; of publications (obtained by text-mining); and (3) to be well-adapted to ELIXIR&rsquo;s distributed nature. Conclusions Concrete examples are provided of downstream uses of the inventoried publications and their citations, both for ELIXIR as a whole and for the Italian case study. Limitations of the methods are discussed, particularly the challenges associated with using an &lsquo;Open literature&rsquo; database (Europe PMC) for the text-mining, and the constraints related to curation capacity. The methods, along with the valuable lessons learned during their development, are sufficiently generic and pragmatic enough to be readily adapted by other similar research infrastructures. \" } { \"@context\": \"http://schema.org\", \"@type\": \"BreadcrumbList\", \"itemListElement\": [ { \"@type\": \"ListItem\", \"position\": \"1\", \"item\": { \"@id\": \"https://f1000research.com/\", \"name\": \"Home\" } }, { \"@type\": \"ListItem\", \"position\": \"2\", \"item\": { \"@id\": \"https://f1000research.com/browse/articles\", \"name\": \"Browse\" } }, { \"@type\": \"ListItem\", \"position\": \"3\", \"item\": { \"@id\": \"https://f1000research.com/articles/13-1547/v1\", \"name\": \"Using text-mining to measure the scientific impact and legacy of ELIXIR,...\" } } ] } Home Browse Using text-mining to measure the scientific impact and legacy of ELIXIR,... ALL Metrics - Views Downloads Get PDF Get XML Cite How to cite this article De Leo F, Balsyte E, Petryszak R et al. Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure for life science data [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 (ELIXIR):1547 ( https://doi.org/10.12688/f1000research.158264.1 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. Close Copy Citation Details Export Export Citation Sciwheel EndNote Ref. Manager Bibtex ProCite Sente EXPORT Select a format first Track Share ▬ ✚ Method Article Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure for life science data [version 1; peer review: 1 approved, 1 approved with reservations] Francesca De Leo https://orcid.org/0000-0003-0421-7699 1 , Erika Balsyte 2 , Robert Petryszak 3 , [...] Marilena D’Ambrosio https://orcid.org/0000-0001-7939-0504 1 , Chiara Bruno 1 , Martin Cook 2 , Ivan Mičetić https://orcid.org/0000-0003-1691-8425 4 , Corinne S. Martin https://orcid.org/0000-0002-5428-2766 2 Francesca De Leo https://orcid.org/0000-0003-0421-7699 1 , Erika Balsyte 2 , [...] Robert Petryszak 3 , Marilena D’Ambrosio https://orcid.org/0000-0001-7939-0504 1 , Chiara Bruno 1 , Martin Cook 2 , Ivan Mičetić https://orcid.org/0000-0003-1691-8425 4 , Corinne S. Martin https://orcid.org/0000-0002-5428-2766 2 PUBLISHED 20 Dec 2024 Author details Author details 1 CNR-IBIOM, Istituto di Biomembrane Bioenergetica e Biotecnologie Molecolari Consiglio Nazionale delle Ricerche, Bari, Italy, 70126, Italy 2 ELIXIR Hub, Wellcome Genome Campus, Hinxton, England, CB10 1SD, UK 3 Datasome, Cambridge, UK 4 Universita degli Studi di Padova Dipartimento di Scienze Biomediche, Padova, I-35131, Italy Francesca De Leo Roles: Conceptualization, Data Curation, Formal Analysis, Funding Acquisition, Methodology, Project Administration, Validation, Visualization, Writing – Original Draft Preparation, Writing – Review & Editing Erika Balsyte Roles: Data Curation, Formal Analysis, Methodology, Project Administration, Validation, Visualization, Writing – Original Draft Preparation Robert Petryszak Roles: Conceptualization, Data Curation, Formal Analysis, Methodology, Software, Validation, Visualization, Writing – Original Draft Preparation Marilena D’Ambrosio Roles: Data Curation, Formal Analysis, Methodology, Visualization, Writing – Original Draft Preparation Chiara Bruno Roles: Data Curation, Formal Analysis, Methodology, Visualization, Writing – Original Draft Preparation Martin Cook Roles: Visualization, Writing – Review & Editing Ivan Mičetić Roles: Data Curation, Visualization, Writing – Review & Editing Corinne S. Martin Roles: Conceptualization, Data Curation, Formal Analysis, Methodology, Project Administration, Supervision, Validation, Visualization, Writing – Original Draft Preparation OPEN PEER REVIEW DETAILS REVIEWER STATUS This article is included in the ELIXIR gateway. Abstract Background ELIXIR is a pan-European public-funded research infrastructure dedicated to life science data. As such, it must demonstrate public value to its funders and stakeholders. We present methods to inventory research publications linked to ELIXIR that have received funding and support, as well as related citation metrics, used as performance metrics for these audiences. Methods To overcome challenges inherent in ELIXIR’s distributed structure, and the fact that those publishing ELIXIR-supported work are typically working part-time on ELIXIR matters, a semi-automated approach, consisting of text-mining followed by manual curation, is presented. A country-level case study (ELIXIR Italy) refines and expands the methods, notably by introducing more granularity in the curation process (e.g. considering all national-level grants, examining affiliations to report publication per institute) and by additionally looking at the scientific impact of the resources developed and operated by the Italian Node of ELIXIR. Results Overall, the methods described in this article have shown to: (1) be repeatable with acceptable levels of accuracy and consistency (notably across curators); (2) require reasonable effort in terms of curation of monthly ‘harvests’ of publications (obtained by text-mining); and (3) to be well-adapted to ELIXIR’s distributed nature. Conclusions Concrete examples are provided of downstream uses of the inventoried publications and their citations, both for ELIXIR as a whole and for the Italian case study. Limitations of the methods are discussed, particularly the challenges associated with using an ‘Open literature’ database (Europe PMC) for the text-mining, and the constraints related to curation capacity. The methods, along with the valuable lessons learned during their development, are sufficiently generic and pragmatic enough to be readily adapted by other similar research infrastructures. READ ALL READ LESS Keywords KPI, metric, funder, literature, bioinformatics, database, resource, performance Corresponding Author(s) Francesca De Leo ( [email protected] ) Close Corresponding author: Francesca De Leo Competing interests: No competing interests were disclosed. Grant information: Italian Ministry for Universities and Research (MUR; National Recovery and Resilience Plan PNRR M4C2): ELIXIRxNextGenIT (IR0000010); ELIXIR, the research infrastructure for life science data: ELIXIR Strategic Implementation Study “Impact evaluation at Node-level - getting it done”, ELIXIR Commissioned Service “People Pulse”; European Union (Horizon Programmes): ELIXIR-CONVERGE (grant agreement number 871075), RI-PATHS (grant agreement number 777563), PathOS (grant agreement number 101058728). The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript. Copyright: © 2024 De Leo F et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. How to cite: De Leo F, Balsyte E, Petryszak R et al. Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure for life science data [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 (ELIXIR):1547 ( https://doi.org/10.12688/f1000research.158264.1 ) First published: 20 Dec 2024, 13 (ELIXIR):1547 ( https://doi.org/10.12688/f1000research.158264.1 ) Latest published: 22 Apr 2025, 13 (ELIXIR):1547 ( https://doi.org/10.12688/f1000research.158264.3 )  There is a newer version of this article available. Suppress this message for one day. Introduction ELIXIR is a pan-European distributed research infrastructure dedicated to life science data, formally founded in 2013 and currently implementing its third five-year scientific programme ( ELIXIR, 2023 ). In 2016, it was recognised by the European Strategy Forum for Research Infrastructures (ESFRI), a forum for government officials focused on research infrastructure, as a ‘Landmark’ ( ESFRI, 2016 ). Landmarks are reference research infrastructures considered pillars in the European Research Area landscape, offering not only services/resources to academic research, but also supporting development and innovation. ELIXIR currently has 21 member countries, three observer countries, plus one intergovernmental organisation, the European Molecular Biology Laboratory (EMBL), which has its own country membership (overlapping in parts with that of ELIXIR). Each ELIXIR member forms a country ‘Node’, with the European Bioinformatics Institute (EMBL-EBI) serving as the ‘European’ Node of ELIXIR. Having the Open Science principle at its heart, ELIXIR comprises over 240 research institutions across Europe , and coordinates the provision , development and discoverability of more than 500 bioinformatics resources covering databases, tools, software, interoperability standards, training, and cloud computing. These resources are financed through public funds ( Smith et al., 2020 ) and are typically free at the point of use, for academic and industry users alike, who can hence focus on their research rather than on the development and operation of the underlying infrastructure and its resources. ELIXIR also works to streamline and integrate national-level bioinformatics infrastructures, to strengthen national training programmes, and to bring together bioinformatics experts to create guidelines and best practices , towards ensuring the long-term sustainability of resources and infrastructure. ELIXIR must continually demonstrate public value to its national and international funders and stakeholders more broadly to ensure their ongoing support and hence the continuity of infrastructure and resource development and operation ( Martin et al., 2021 ). Inventories of research publications arising from the use of the infrastructure (by others than those developing/operating it) are a common source of performance metrics requested by funders and related stakeholders ( ESFRI, 2019 , OECD, 2019 , European Commission, 2020 ). A subset of ELIXIR databases, the ELIXIR Core Data Resources ( Durinx et al., 2017 ), are indeed widely cited and acknowledged in thousands of research publications by life scientists across the globe ( Drysdale et al., 2020 ). Around 700 bioinformaticians and related professionals across Europe form the developers and operators of the ELIXIR infrastructure and resources, and are typically working part-time on ELIXIR matters, often alongside more conventional research activities. These professionals usually have research profiles and, through their daily work on ELIXIR matters, actively contribute to the generation and dissemination of new technical and scientific knowledge linked to the development and operation of the ELIXIR infrastructure and its resources. These efforts are enabled (in part or fully) by a range of funding sources , such as ELIXIR’s internally-funded projects (also referred to as ‘Commissioned Services’), national-level financing, and grant funding from the European Union and other international funders awarded in the name of ELIXIR. In line with the Open Science principles, this new knowledge is shared openly via ELIXIR-supported research publications (peer-reviewed, preprints), which collectively represent an additional source of performance metrics of ELIXIR’s scientific legacy as a research infrastructure. The purpose of this article is to present the methods that have been developed and implemented to inventory research publications that have received funding and support linked to ELIXIR, as well as related citation metrics. The methods are presented for ELIXIR as a whole (lightweight, considering all Nodes) and for a particular country Node, ELIXIR Italy, itself also of distributed nature . For the latter, the methods were refined thanks to greater curation capacity at the level of the Node, and also expanded to include the inventorying of research publications citing resources developed and operated by the Node. This exercise is very informative of the Node’s scientific impact and legacy, yet it would not easily scale for the whole distributed infrastructure (in terms of curation capacity at the ELIXIR Hub, the infrastructure’s coordinating secretariat) since the infrastructure operates hundreds of resources under the ELIXIR umbrella. A semi-automated approach, consisting of text-mining followed by manual curation, is presented, with the Italian case study going further so as to better reflect the needs and expectations of national-level funders and other (internal and external) stakeholders of the Node. Of note, the guiding principle in developing these methods was that they would not require extensive technical nor programming knowledge to be applied once in place - as a result, the methods do not involve any modelling (e.g. with training and test sets), nor do they claim to be rigorously accurate. It is hoped that the steps taken, and lessons learned (both ELIXIR-wide and at the scale of a country Node), in arriving at operational methods in the context of distributed organisational settings will be useful to other such research infrastructures and organisations, in the life sciences and other disciplines. Methods Lightweight methods for the ELIXIR infrastructure as a whole (all Nodes) Main challenges Two intrinsic challenges that the lightweight methods needed to overcome were (1) that those publishing ELIXIR-supported work are typically working part-time on ELIXIR matters, and (2) the scale of ELIXIR’s “people infrastructure” (more than 700 specialists across Europe, showing a level of staff turnover). The latter itself represented a significant barrier to manual reporting of publications. As for the former, traditional bibliometrics/scientometrics approaches centred on researchers’ unique identification numbers (e.g., ORCID, ResearcherID/Publons), and/or institutional affiliations, were deemed inadequate for identifying research works (funded and/or) supported by ELIXIR, as these approaches would result in significant amounts of ‘false positives’, i.e. publications not supported by ELIXIR. To overcome these two challenges, the lightweight methods were designed to (1) detect, through text-mining within an Open research literature database ( EuropePMC , Rosonovski et al., 2024 ), publications (peer-reviewed and preprints) in which there were acknowledgments of the support and/or funding received from ELIXIR, (2) followed by manual curation for increased accuracy. Sources of ELIXIR support For the ELIXIR infrastructure as a whole (all Nodes), two main streams of funding and/or support were considered: (1) ELIXIR’s internally-funded projects , i.e. Commissioned Services comprising Implementation Studies, Staff Exchange, Knowledge Exchanges and Industry days, and Travel Grants, where funding eligibility is restricted to institutes part of the ELIXIR membership, and (2) grant funding from international funders such as the European Union (notably under the Research Infrastructures Work Programmes), in which several ELIXIR institutes are beneficiaries and the purpose of the grant is to develop and/or operate the ELIXIR infrastructure. Searching for relevant publications using text-mining EuropePMC’s Application Programming Interface (API) was used to search for relevant publications (technical documentation), as well as their citations in the Open literature (technical documentation), considering all years for which ELIXIR existed, even as a concept ( circa 2011), with a cut-off date of December 2023 (for the purpose of this article). Search configuration parameters ( Table 1 ), i.e. ‘search terms’ for short, comprised unique grant identification numbers and names, names and short codes of countries that are members of ELIXIR (e.g. ELIXIR Norway, ELIXIR NO), as well as “boiler-plate” text strings relating to ELIXIR support and/or funding including for ‘signature’ events (e.g. the ELIXIR-convened Biohackathon Europe) and technical workshops/meetings. For use in EuropePMC’s API, the search terms (query parameters) were first URL encoded using the online utility URL Encode/Decode , provided free, courtesy of Dan’s Tools : for example, the EU funded project ELIXIR-CONVERGE would be encoded as “ELIXIR%2DCONVERGE”. Of note, EuropePMC is case insensitive, meaning that it is not necessary to provide search terms in both lower/upper cases. For the lightweight methods, acknowledgement of the use of the infrastructure and its resources was deemed beyond scope due to the mismatch between curation capacity at the ELIXIR Hub and the several hundreds of resources operated under the ELIXIR umbrella. Table 1. Search configuration for lightweight methods across the ELIXIR infrastructure as a whole. Type Search terms (examples) Unique grant identification numbers 730941; 654248; 825575; 676559; 824087; 739563; etc Grant names ELIXIR-EXCELERATE; ELIXIR-CONVERGE; EOSC life; FAIRplus; etc Names and short codes of countries that are members of ELIXIR ELIXIR BE; ELIXIR Belgium; Belgian ELIXIR Node; ELIXIR GR; ELIXIR Greece; Greek ELIXIR Node; Hellenic ELIXIR Node; EMBL-EBI; etc “Boiler-plate” text strings This work was funded by ELIXIR, the research infrastructure for life science data; European life sciences Infrastructure; ELIXIR Implementation Study; ELIXIR Implementation Studies; ELIXIR Commissioned Service; ELIXIR Travel Grant; ELIXIR Staff Exchange; ELIXIR BioHackathon; Biohackathon Europe Generic search terms ELIXIR Europe; ELIXIR Node; funded by ELIXIR; funded partially by ELIXIR; supported by ELIXIR; ELIXIR Core funding; funding from ELIXIR An initial set of search terms were entered in a Google Sheet, and retrieved from there via a specific Google Sheets API . This allowed for the subsequent dynamic configuration of the search queries when new grants were awarded to ELIXIR. The Google Sheet interacted with the EuropePMC API based on the retrieved search configuration, and the text-mining process resulted in the generation of tab-delimited text files listing potentially relevant publications comprising a mix of ELIXIR-related works (‘true positives’) and false positives. Manual curation and search optimisation Manual identification of false positives was carried out using the tab-delimited text files, one column of which collated the triggers relevant to each returned publication. Triggers represented the search terms that had been detected in the publications and were found to guide curation very effectively. False positives were blacklisted for future searches. To ensure consistency, reproducibility and the lowest subjectivity possible in curation decisions, curation efforts were carried out by a small team of two ELIXIR Hub staff who kept a record of difficult and/or unclear cases, and of the decisions taken. They were supported on an ad-hoc basis by additional ELIXIR Hub staff who had more extensive knowledge and understanding of ELIXIR’s activities in certain thematic and technical areas. During the initial development of the methods, iterative searches and manual curation rounds were undertaken to test and refine the set of search terms, and to identify which sections of EuropePMC entries (for each publication) were the most informative to target by text-mining - these were found to be the title, first author, funding statement, and acknowledgments. The initial text-mining runs naturally returned large backlogs of potentially relevant publications since they covered the years 2011 to 2019: although this made the curation effort significant at the time, it also provided a rich list of results to optimise the searches, notably to identify which sections ( Table 2 ) of the publications ought to be extracted to the tab-delimited text files used during manual curation and any downstream visualisations. Table 2. Extracted publication sections from EuropePMC, used for manual curation and downstream visualisations. Sections of the publications Harvest month Triggers identified Unique EuropePMC identification number Digital Object Identifier (DOI) Title Journal name Publication year First author name First author affiliation Last author name Last author affiliation Funding statement Acknowledgements A set of known ELIXIR-supported publications was useful for fine-tuning the approach, as they would be expected to be returned by the searches. If this was not the case, underlying reasons would be investigated and, where possible, solutions found - if no solution was found, these publications were added (via their unique EuropePMC identification number) to a whitelist that would be pulled to the tab-delimited text file, regardless. Later on, once the methods were operational ( circa early 2020), the text-mining “harvests” (i.e. searches) followed by curation ended up being carried out on a monthly basis, to identify recently published and/or recently indexed publications, based on the latest set of search terms. Search terms returning very high numbers of false positives for very few true positives were gradually excluded from searches. This was for instance the case of “ELIXIR” as a standalone term (misunderstood for the word “elixir”), “implementation study” (rather than the more precise “ELIXIR implementation study”), and the project name “BY-COVID” (incorrectly read as the very common text string “[infected] by COVID -19”). This was to strike a balance between accuracy (i.e. not missing any relevant publication) and overwhelming curators. Rather, it was deemed more useful to arrive at manageable monthly text-mining harvests and curation workloads moving forwards (approximately 3 to 4 hours per monthly harvest), since the intention was to arrive at operational performance metrics that could be monitored in the long-term, rather than a very accurate one-off exercise. Case study: refined and expanded methods focusing on a single Node (ELIXIR Italy) Unless otherwise specified, the methods for the Italian case study are identical to the lightweight one described above (for ELIXIR as a whole). Sources of ELIXIR Italy support Building on the experience of the ELIXIR Hub effort, the Italian case study began at the end of 2022 and was focused on the period January 2015 (the year when Italy became an ELIXIR member country) to December 2023 (cut-off date for the purpose of this article). The main three sources of funding and/or support considered in the Italian case study were (1) those ELIXIR internally funded projects in which one or more of the institutes of the Italian Node were involved, (2) ditto for grant funding from international funders, and (3) national-level grants awarded to the Node to develop and/or operate its infrastructure and resources. Searching for relevant publications using text-mining For the Italian case study, the ‘ELIXIR as a whole’ set of search terms was refined to remove terms not relevant to the Node, e.g. if the Italian Node was not a beneficiary of a given grant. Additional search terms were introduced (e.g. national-level grants) along with other terms (e.g. Italian Node of ELIXIR, often used instead of ELIXIR-IT or ELIXIR IIB) that were known to be mentioned in publications acknowledging support and/or funding by ELIXIR Italy ( Table 3 ). Table 3. Search configuration for methods focused on a single country Node (Italy). Type Search terms (all) Unique grant identification numbers 951724; 101046203; PIR01_00017; 871075; IR0000010; 824087; 857650; 825575; 101057388; 676559; 101003551; 101017549; 653549; 956137; 862658; 101016167; 778247; 823886; 654008; 634486; 952334; 101081813 Grant names BEYOND 1M GENOME; BY-COVID; CnrBiomics; ELIXIR-Converge; ELIXIRNextGenIT; EOSC-Life; EOSC-Pillar; European Join Programme On Rare Diseases; EuroScienceGateway; EXCELERATE; Exscalate4Cov; Genomed4All; INDIGO DataCloud; IDPfun; NewTechAqua Generic search terms ELIXIR IIB; ELIXIR-IIB; ELIXIR ITA; elixir iib; elixir-iib; Italian Node of ELIXIR; ELIXIR-IT The major expansion of the methods followed Drysdale et al. (2020) and their focus on citations of ‘key articles’ describing the Node’s resources. This additional search effort was approached by using the Digital Object Identifier (DOI) of these ‘key articles’ to collate citation data and use this as proxy for the scientific impact of the Node’s resources. For example, the resource PatSearch was first described in Pesole et al. (2000) and an improved version of it was described in Grillo et al. (2003) - these are considered the resource’s ‘key articles’. The list of ELIXIR Italy resources was based on the 2023 version of the Node’s Service Delivery Plan under the ELIXIR umbrella, and the DOIs were sourced from the bio.tools registry ( Ison et al., 2019 ). For the 65 resources developed and operated by ELIXIR Italy in 2023, a total of 124 DOIs were hence collated, representing the ‘key articles’ describing initial and improved versions of these resources. Manual curation and search optimisation Curation in the context of the case study was a very similar process to that described above but carried out by ELIXIR Italy staff with knowledge of the Node’s activities. Author affiliations were additionally scrutinised and encoded to obtain a more granular view of dissemination efforts and scientific impact at the level of the Node’s institutes (30 in total at the time of writing). Sources of false positives were like that described above, e.g. project names such as “ORCHESTRA” (understood as the word “orchestra” or other projects called as such). As above, a balance was struck between accuracy and curation capacity by excluding problematic search terms (those returning high levels of false positives). Results ELIXIR infrastructure as a whole (all Nodes) For the period 2011 to 2023, 972 publications supported by ELIXIR (as a whole) were identified, and these were collectively cited over 31,000 times ( Figure 1 ). The curated inventory of publications and their citations in the Open literature were used to create visualisations in Tableau software, which were themselves featured on ELIXIR’s impact dashboard , among other indicators relating to ELIXIR. On this dashboard, the page showing ELIXIR’s scientific legacy as a research infrastructure received nearly 600 unique visitors from over 40 countries in just 12 months (December 2022 to November 2023). Figure 1. Publications and citations supported by ELIXIR (2011–2023) in Open literature (EuropePMC). Case study: ELIXIR Italy For the period 2015 to 2023, 408 publications supported by ELIXIR Italy were identified, which were collectively cited almost 5,000 times ( Figure 2 ). The distribution of publications across Node institutes ( Figure 3 ) highlights the disproportionate contribution of the National Research Council (NRC), though unsurprising given its role as the coordinating institute of ELIXIR Italy. Associated visualisations can be accessed on ELIXIR Italy’s impact dashboard which, for a 24 month period (January 2022 to December 2023), received over 5,000 unique visitors from 10 countries. In terms of the impact of resources developed and operated by ELIXIR Italy ( Table 4 ), citations of their ‘key articles’ amounted to 8,700, across the five ELIXIR Platforms , which are the technical domains of implementation of the infrastructure. Figure 2. Publications and citations supported by ELIXIR Italy (2011–2023) in Open literature (EuropePMC). Figure 3. Distribution of ELIXIR Italy publications across its Node institutes. Acronyms: see https://elixir-italy.org/about/members/ . Table 4. Citations of ELIXIR Italy's resources via DOIs of their ‘key (descriptive) articles’. Year 2015 2016 2017 2018 2019 2020 2021 2022 2023 Tools Platform 327 424 476 450 456 571 713 805 703 Data Platform 304 311 289 342 339 398 557 570 461 Compute Platform 14 12 10 4 15 15 26 19 19 Interoperability Platform 0 0 0 1 2 0 6 32 29 Total citations per year 645 747 775 797 812 984 1,302 1,426 1,212 Total citations 8,700 Discussion In almost four years of implementation, the methods have been found to be adequately performant at identifying publications supported by ELIXIR, if the relevant triggers were present (and correctly spelled) in the publications. Since the inception (in 2020) of a dedicated “How to acknowledge ELIXIR funding and support” page on the ELIXIR website (with the analogous one on the ELIXIR Italy site , bottom part), curators have noticed clear improvements in how ELIXIR’s contribution is acknowledged. The upward trend in citation numbers for both the lightweight and case study methods suggested long-term scientific impact of these publications, a reminder that impact is slow to develop. It is not just citations that take time to appear: large-scale projects such as ELIXIR-EXCELERATE which involved 47 partners from 17 countries ( Harrow et al., 2021 ; €19 million, 2014-2019) were found to be still acknowledged in publications years after their completions, This is not solely due to delayed indexing but also to the enduring legacy of their support for research activities within teams connected to the project. Overall, the methods described in this article have shown to: (1) be repeatable with acceptable levels of accuracy and consistency (notably across curators); (2) require reasonable effort in terms of curation of monthly harvests (in contrast to running an ELIXIR wide manual reporting process); and (3) to be well-adapted to ELIXIR’s distributed nature (including at Node-level in the Italian case study). While the text-mining component can be operated with minimal ELIXIR expertise, it is true that the curation component requires a deeper awareness and understanding of the organisation’s activities. One example is when ELIXIR was involved in large “cluster” grants (e.g. EOSC-life ) in which many life science research infrastructures were also funded, meaning that not all publications acknowledging EOSC-life’s grant would be relevant to ELIXIR, despite ELIXIR coordinating the project. The Italian case study highlighted the usefulness of national-level granularity in the search terms. Whilst it is true that the lightweight ‘ELIXIR as a whole’ methods do include a few national-level grants in its search terms, this is far from being fully implemented. Their inclusion was exploratory at best and, based on this experience, there is insufficient appetite at the ELIXIR Hub to fully implement national-level search terms due to capacity constraints in terms of curation and limited knowledge of ELIXIR activities at Node level. Limitations of the methods linked to using an ‘Open literature’ database (EuropePMC) One main limitation of the methods is linked to EuropePMC being used as the central source of publication data - only the publications indexed in this literature database, focused on life sciences, can be returned by the searches. At the time of writing, specialised journals such as Data Science Journal and Data Intelligence were not yet indexed due to perceived low demand of these in the user base of EuropePMC. However, based on user feedback, EuropePMC started indexing BioHackrXiv in 2021, especially as it is the recommended preprint server for outcomes of the ELIXIR-convened Biohackathon Europe series ( Castro et al., 2021 ). Although research funders are increasingly requesting that the work they fund is openly accessible, it was found that certain ELIXIR-relevant publications were behind paywalls. This meant that EuropePMC was not able to fully “ingest” the text of certain sections of the source text, e.g. funding statement, which made text-mining efforts fail, despite the publication having a presence in EuropePMC and the right trigger being present. This was manually solved on an ad-hoc basis by whitelisting these publications using their unique EuropePMC identification number. Additionally, this method can effectively identify ELIXIR-related publications that are non-compliant with open access policies and funder requirements. Looking forward, EuropePMC continues to work closely with scientific publishers to agree and implement standards so that ingestion of information from the publishers’ sites to EuropePMC is more effective. This is an on-going and quite a considerable task, and additional sources of text-mining failures included conflicts in metadata, embargoed works in paywalled journals, as well as citations being limited to the Open literature, the latter leading to underestimated scientific impact. Uses of the curated inventories of publications as source of performance metrics The intention behind this work was to arrive at operational performance metrics that could be easily monitored in the long-term, in contrast to a rigorous and hence heavy-duty process requiring significant resources, likely to be carried out once or twice at best. The curated inventories of publications, including their citations, have so far been used as input into ELIXIR’s annual reports , ELIXIR newsletters and social media posts (under the heading “Recommended reading”), technical reporting to funders, as well as ELIXIR’s monitoring report to ESFRI and a growing number of requests for information from national funders, such as Sweden ( Swedish Research Council, 2021 ), Finland ( Academy of Finland, 2023 ) and the Netherlands ( NWO, 2023 ). Similar to ELIXIR, ELIXIR Italy has used the curated inventories of publications and their citations for a range of internal and external monitoring activities, as required by various stakeholders, for example periodic Node reviews and presentations for ELIXIR’s Board of funders , Scientific Advisory Board and Industry Advisory Committee . The information has also been used to report to ELIXIR Italy’s own Scientific Advisory Board, as well as for the Node’s general assembly and monthly newsletters. Finally, the information has been used to report to ELIXIR’s Italy’s main funder, the Italian Ministry for Universities and Research (MUR)), notably as part of evaluation processes and of national-level grant funding for research infrastructures. Conclusions As illustrated by the Italian case study, these methods represent the first step of potentially many exciting downstream analyses using the curated inventories as starting materials. Their citations would shed light on the impact of these works, through their use by others. Yet these inventories alone have proved to be useful already, in terms of communicating performance and impact of both ELIXIR activities (as a whole and for a country Node) and the use of a Node’s resources. The Italian case study also demonstrated the value of knowledge sharing within an organisation, in the spirit of Open Science, with the country Node leveraging the experience of the ELIXIR Hub, building on and enhancing it. The methods’ main strength lies in its relative ease of implementation, especially in the context of a distributed life science research infrastructure staffed with highly qualified and research-active personnel: many will have the programming skills required to operationalise the methods, whilst no such technical skill is required for the curation part of the methods. If not, external assistance can be sought in the form of consulting, to get the approach off the ground. Another strength of the methods is its versatility as new funding streams become available and resources are further developed. The methods, along with the valuable lessons learned during their development, are generic and pragmatic enough to be readily adapted by other similar research infrastructures. This extends its potential application beyond life sciences to encompass physical sciences, social sciences, and the humanities. Ethical considerations Ethical approval and consent were not required. Data availability Underlying data EuropePMC ( Rosonovski et al., 2024 ; https://europepmc.org/ ), an Open research literature database, was used as the source for the underlying data (publications in the life sciences) used in the methods described in the article. Extended data The extended data created as a result of applying the methods are curated inventories of publications that have received funding and support from ELIXIR. They have been deposited in Zenodo ( Martin et al., 2024 ; https://doi.org/10.5281/zenodo.14136249 ) under a Creative Commons Attribution 4.0 International license, so that others can access them for illustrative and verification purposes. Reporting guidelines N/A Acknowledgements The authors express their appreciation to Andrew Smith and Niklas Blomberg for initiating this effort, unlocking the necessary internal resources, and providing institutional support throughout. The authors would like to thank the following ELIXIR Hub colleagues who took part in the May 2019 workshop where the scope and methods were first discussed: Susanna Repo, John Hancock, David Lloyd, and Jerry Lanfear. Additional thanks go to the following ELIXIR Hub colleagues who helped with unclear cases during curation efforts: Giselle Kerry, Gary Saunders, Rachel Drysdale, Katharina Lauer, Serena Scollen, Jonathan Tedds, and Jen Harrow. Finally, the authors would like to express their gratitude to Graziano Pesole (Head of the Italian Node of ELIXIR) and Federico Zambelli (Technical Coordinator of the Italian Node of ELIXIR) for their institutional support throughout the project and their collaboration in defining the methodology and identifying needs. References Academy of Finland: Assessment of benefits of international Research Infrastructure memberships for Finland 2020-2021. Questionnaire results. 2023. Reference Source Castro LJ, Martin C, Lazarov G, et al. : Measuring outcome and impacts from the BioHackathon Europe.2021. Publisher Full Text Reference Source Drysdale R, Cook CE, Petryszak R, et al. : The ELIXIR Core Data Resources: fundamental infrastructure for the life sciences. Bioinformatics. 2020; 36 (8): 2636–2642. PubMed Abstract | Publisher Full Text | Free Full Text Durinx C, McEntyre J, Appel R, et al. : Identifying ELIXIR core data resources. F1000Res. 2017; 5 : 2422. PubMed Abstract | Publisher Full Text | Free Full Text ELIXIR: ELIXIR Scientific Programme, 2024-2028.2023. Reference Source ESFRI: Monitoring of research infrastructures performance.2019. Reference Source ESFRI: Strategy Report on Research Infrastructures.2016. Reference Source European Commission: Supporting the transformative impact of research infrastructures on European research: report of the High-Level Expert Group to assess the progress of ESFRI and other world class research infrastructures towards implementation and long-term sustainability. Director General for Research and Innovation, Publications Office of the European Union; 2020. Reference Source Grillo G, Licciulli F, Liuni S, et al. : PatSearch: a program for the detection of patterns and structural motifs in nucleotide sequences. Nucleic Acids Res. 2003; 31 (13): 3608–3612. PubMed Abstract | Publisher Full Text | Free Full Text Harrow J, Hancock J, Blomberg N, et al. : ELIXIR-EXCELERATE: establishing Europe’s data infrastructure for the life science research of the future. EMBO J. 2021; 40 (6): e107409. PubMed Abstract | Publisher Full Text | Free Full Text Ison J, Ienasescu H, Chmura P, et al. : The bio.tools registry of software tools and data resources for the life sciences. Genome Biol. 2019; 20 (1): 164. PubMed Abstract | Publisher Full Text | Free Full Text Martin C, Repo S, Márquez JA, et al. : Demonstrating public value to funders and other stakeholders—the journey of ELIXIR, a virtual and distributed research infrastructure for life science data. Ann. Public Coop. Econ. 2021; 92 (3): 497–510. Publisher Full Text Martin C, De Leo F, D’Ambrosio M, et al. : Publications Supported by ELIXIR and ELIXIR Italy (during the Periods 2011-2023 and 2015-2023). Zenodo. 2024; 13 November 2024. Publisher Full Text NWO: Analysis of the Dutch participation in international research infrastructures.2023. Reference Source OECD: Reference framework for assessing the scientific and socio-economic impact of research infrastructures. OECD Science, Technology and Industry Policy Papers. 2019. Publisher Full Text Pesole G, Liuni S, D’Souza M: PatSearch: a pattern matcher software that finds functional elements in nucleotide and protein sequences and assesses their statistical significance. Bioinformatics. 2000; 16 (5): 439–450. PubMed Abstract | Publisher Full Text Rosonovski S, Levchenko M, Bhatnagar R, et al. : Europe PMC in 2023. Nucleic Acids Res. 2024; 52 (D1): D1668–D1676. PubMed Abstract | Publisher Full Text | Free Full Text Smith A, Martin C, Blomberg N: ELIXIR position paper on suitable business models for open data. F1000Res. 2020; 9 : 65. Publisher Full Text Swedish Research Council: National benefits from Swedish membership of international research infrastructures 2016-2019.2021. Reference Source Comments on this article Comments (0) Version 3 VERSION 3 PUBLISHED 20 Dec 2024 ADD YOUR COMMENT Comment Author details Author details 1 CNR-IBIOM, Istituto di Biomembrane Bioenergetica e Biotecnologie Molecolari Consiglio Nazionale delle Ricerche, Bari, Italy, 70126, Italy 2 ELIXIR Hub, Wellcome Genome Campus, Hinxton, England, CB10 1SD, UK 3 Datasome, Cambridge, UK 4 Universita degli Studi di Padova Dipartimento di Scienze Biomediche, Padova, I-35131, Italy Francesca De Leo Roles: Conceptualization, Data Curation, Formal Analysis, Funding Acquisition, Methodology, Project Administration, Validation, Visualization, Writing – Original Draft Preparation, Writing – Review & Editing Erika Balsyte Roles: Data Curation, Formal Analysis, Methodology, Project Administration, Validation, Visualization, Writing – Original Draft Preparation Robert Petryszak Roles: Conceptualization, Data Curation, Formal Analysis, Methodology, Software, Validation, Visualization, Writing – Original Draft Preparation Marilena D’Ambrosio Roles: Data Curation, Formal Analysis, Methodology, Visualization, Writing – Original Draft Preparation Chiara Bruno Roles: Data Curation, Formal Analysis, Methodology, Visualization, Writing – Original Draft Preparation Martin Cook Roles: Visualization, Writing – Review & Editing Ivan Mičetić Roles: Data Curation, Visualization, Writing – Review & Editing Corinne S. Martin Roles: Conceptualization, Data Curation, Formal Analysis, Methodology, Project Administration, Supervision, Validation, Visualization, Writing – Original Draft Preparation Competing interests No competing interests were disclosed. Grant information Italian Ministry for Universities and Research (MUR; National Recovery and Resilience Plan PNRR M4C2): ELIXIRxNextGenIT (IR0000010); ELIXIR, the research infrastructure for life science data: ELIXIR Strategic Implementation Study “Impact evaluation at Node-level - getting it done”, ELIXIR Commissioned Service “People Pulse”; European Union (Horizon Programmes): ELIXIR-CONVERGE (grant agreement number 871075), RI-PATHS (grant agreement number 777563), PathOS (grant agreement number 101058728). The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript. Article Versions (3) version 3 Revised Published: 22 Apr 2025, 13:1547 https://doi.org/10.12688/f1000research.158264.3 version 2 Revised Published: 03 Apr 2025, 13:1547 https://doi.org/10.12688/f1000research.158264.2 version 1 Published: 20 Dec 2024, 13:1547 https://doi.org/10.12688/f1000research.158264.1 Copyright © 2024 De Leo F et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. Download Export To Sciwheel Bibtex EndNote ProCite Ref. Manager (RIS) Sente metrics Views Downloads F1000Research - - PubMed Central info_outline Data from PMC are received and updated monthly. - - Citations open_in_new 0 open_in_new 0 open_in_new SEE MORE DETAILS CITE how to cite this article De Leo F, Balsyte E, Petryszak R et al. Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure for life science data [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 (ELIXIR):1547 ( https://doi.org/10.12688/f1000research.158264.1 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS track receive updates on this article Track an article to receive email alerts on any updates to this article. TRACK THIS ARTICLE Share Open Peer Review Current Reviewer Status: ? Key to Reviewer Statuses VIEW HIDE Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Version 1 VERSION 1 PUBLISHED 20 Dec 2024 Views 0 Cite How to cite this report: Kawashima S. Reviewer Report For: Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure for life science data [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 (ELIXIR):1547 ( https://doi.org/10.5256/f1000research.173833.r354532 ) The direct URL for this report is: https://f1000research.com/articles/13-1547/v1#referee-response-354532 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 27 Feb 2025 Shuichi Kawashima , Database Center for Life Science, Chiba, Japan Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.173833.r354532 This manuscript presents an approach to identifying and measuring the scientific impact of ELIXIR-supported publications, employing text-mining of EuropePMC alongside manual curation. It offers valuable insights into the output and influence of a distributed research infrastructure, with an in-depth case ... Continue reading READ ALL This manuscript presents an approach to identifying and measuring the scientific impact of ELIXIR-supported publications, employing text-mining of EuropePMC alongside manual curation. It offers valuable insights into the output and influence of a distributed research infrastructure, with an in-depth case study of ELIXIR Italy. The methodology is generally sound and relevant; however, several clarifications are needed to ensure transparency and reproducibility. Major Concerns: * The authors should include concrete examples (e.g., sample URLs or code snippets) illustrating how EuropePMC’s API was queried and how retrieved data were processed. This addition would strengthen reproducibility. * A subset of known ELIXIR-supported articles had to be manually whitelisted, suggesting certain limitations in the automated text-mining. The authors are encouraged to explain why these articles failed to appear in the initial searches (e.g., unusual acknowledgments or missing metadata), as such detail would be instructive for others employing similar methods. Minor concerns: * The figure 2 indicates ~6,721 citations, whereas the text mentions 5,000. This discrepancy should be resolved for clarity. * According to the text, 408 unique publications were supported by ELIXIR Italy from 2015 to 2023. Figure 3 shows the distribution of these publications across the Node’s institutes. If one sums the values for all institutes in Figure 3, the total appears to be 772, which does not match 408. The authors should clarify whether Figure 3 double-counts papers co-authored by multiple institutes or if other factors account for the difference. The manuscript is well-structured, and the research offers a promising approach for tracking the impact of distributed infrastructures. Addressing the concerns above will strengthen the paper’s clarity and transparency. I recommend minor revision prior to acceptance. Is the rationale for developing the new method (or application) clearly explained? Yes Is the description of the method technically sound? Yes Are sufficient details provided to allow replication of the method development and its use by others? Partly If any results are presented, are all the source data underlying the results available to ensure full reproducibility? Partly Are the conclusions about the method and its performance adequately supported by the findings presented in the article? Yes Competing Interests: No competing interests were disclosed. Reviewer Expertise: bioinformatics, semantic web, database I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Kawashima S. Reviewer Report For: Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure for life science data [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 (ELIXIR):1547 ( https://doi.org/10.5256/f1000research.173833.r354532 ) The direct URL for this report is: https://f1000research.com/articles/13-1547/v1#referee-response-354532 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Berman HM. Reviewer Report For: Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure for life science data [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 (ELIXIR):1547 ( https://doi.org/10.5256/f1000research.173833.r354530 ) The direct URL for this report is: https://f1000research.com/articles/13-1547/v1#referee-response-354530 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 06 Jan 2025 Helen M. Berman , Chemistry and Chemical Biology, Rutgers, The State University of New Jersey, Piscataway, NJ, USA Approved VIEWS 0 https://doi.org/10.5256/f1000research.173833.r354530 This article describes a method to assess the impact of ELIXIR supported research. This was done by using text mining to search for relevant articles contained in EuropePMC. This procedure was used for all ELIXIR nodes and for ELIXIR Italy. ... Continue reading READ ALL This article describes a method to assess the impact of ELIXIR supported research. This was done by using text mining to search for relevant articles contained in EuropePMC. This procedure was used for all ELIXIR nodes and for ELIXIR Italy. In the latter case the search terms were modified to be appropriate for that node. In the first step specific search terms were used to search sections of articles. This automated procedure was followed by manual curation. It is important to have quantitative methods to analyze impact of the ELIXIR infrastructure and this is a good first step. The paper is well written, and the limitations of the approach are adequately described. For example, the EuropePMC does not index all the journals that describe projects funded under the ELIXIR umbrella. It is also disturbing that some articles are published in journals that are behind paywalls. Although the curated datasets are put into Zenodo, the authors might consider making a website that would allow authors to submit and curate their ELIXIR related articles. Is the rationale for developing the new method (or application) clearly explained? Yes Is the description of the method technically sound? Yes Are sufficient details provided to allow replication of the method development and its use by others? Partly If any results are presented, are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions about the method and its performance adequately supported by the findings presented in the article? Yes Competing Interests: No competing interests were disclosed. Reviewer Expertise: Structural bioinformatics, structural biology I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Berman HM. Reviewer Report For: Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure for life science data [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 (ELIXIR):1547 ( https://doi.org/10.5256/f1000research.173833.r354530 ) The direct URL for this report is: https://f1000research.com/articles/13-1547/v1#referee-response-354530 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Comments on this article Comments (0) Version 3 VERSION 3 PUBLISHED 20 Dec 2024 ADD YOUR COMMENT Comment keyboard_arrow_left keyboard_arrow_right Open Peer Review Reviewer Status info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Reviewer Reports Invited Reviewers 1 2 3 Version 3 (revision) 22 Apr 25 read read Version 2 (revision) 03 Apr 25 read Version 1 20 Dec 24 read read Helen M. Berman , Rutgers, The State University of New Jersey, Piscataway, USA Shuichi Kawashima , Database Center for Life Science, Chiba, Japan Alfonso Valencia , Barcelona Supercomputing Center., Barcelona, Spain Comments on this article All Comments (0) Add a comment Sign up for content alerts Sign Up You are now signed up to receive this alert Browse by related subjects keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 Valencia A. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 29 May 2025 | for Version 3 Alfonso Valencia , Barcelona Supercomputing Center., Barcelona, Spain 0 Views copyright © 2025 Valencia A. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions First, let me congratulate the authors for addressing this challenging and to some extend nasty task full of details. I have a number of concerns related with the technical part, the presentation of the results and the discussion. Overall, my assessment is that the authors have an interesting collection of results that are only partially exposed and very superficially analysed. Given that the hard work has been done it is a pity not to complete it with a relevant analysis answering what the title announces: measuring the impact and legacy of ELIXIR (for example, after reading the paper, I still do not know if the answer is a lot or little impact or where has ELIXIR made more impact) 1.- The usual - and recommended - practice is to develop a handbook with the details steps for the curation (Annotation Guideline). A handbook describing in detail the steps for the curation of the documents and the construction of the corpus, that annotators have to follow. The annotation guidelines are necessary for the alignment of the annotators, to decrease the inter-annotator error, for the validation of the procedures and are useful for other to be able to do a similar exercise in the future. An example (in spanish: https://zenodo.org/records/6477407). In many senses, they are the most important legacy of any annotation process. This publication comes halfway to provide the information for the annotation handbook. 2.- Regarding manual curation. where are the numbers of raw identifications and the number of articles rejected and for what reasons? This information will be very interesting to better understand how effective the text mining was and what are the main source of problems. 3.- The authors discuss the limitations of the study including the access to non-open literature. This is true, but it is also true that the grants are quoted, at least in part, in the Medline entries. Is this information used? or only from full open paper in EuroPMC? and if not, why not? 4.- Table 2 with the sources of publications is important, but it does not give the numerical information on how many articles were extracted with each query and what was the overlap ? This information will be important to understand the influence of the different sources and to plan future experiments. 5.- I find difficult to understand the section dedicated to the retrieval of \"key articles\" and how articles quoting an ELIXIR resource were processed to identify contribution of ELIXIR-IT. Where the articles quoting key resources considered part of the ELIXIR-IT contribution even if they did not acknowledge a specific grant? 6.- Related with the previous point. The case study is interesting but I find the analysis of the data insufficient. Apparently, a quarter of the ELIXIR citations are from publications \"solely\" by ELIXIR.-IT? this seems to be too many, and likely many of the ELIXIR publications are derived from collaborations between ELIXIR-IT and other ELIXIR nodes. What is the overlap? how many are unique ELIXIR-IT publications? and to what do they correspond (areas of elixir? Core tools produce in Italy - detailing what is given in Fig 5?) 7.-The problem with the umbrella institutions is well known. In other cases, efforts have been made to disambiguate the affiliations to increase granularity by, for example, using the other addressed in the same paper. 8.- Fig 2. legend is publications and citations but only citations are shown. 9.- I do not understand how table 5 was generated. HOw the articles (how many?) dedicated to the core resources are assigned to the platforms? How many papers and citations have each one of them? 10.- Table 5. Will be correct to say that the Italian core resources were quoted 1426 times in 2022 and that in total publications of ELIXIR-IT were quoted 1910 times this year?I If this is the case, it will be important to discuss this point: the main source of impact of ELIXIR are the core resources and not the work in the grants (75% of the citations) right? Is this the case of ELIXIR or only for ELIXIR-IT? 11.- After making a selection of papers by grants, and even if not all the papers quote the corresponding grant, it will be very interesting to provide and discuss what grants have been quoted more often- and in this sense have been more influential. 12.- Overall, It is. a pity that the paper provides such a shallow level of analysis. For example, the overall number of papers and citations, does not really help to understand the impact of ELIXIR or ELIXIR-IT. Providing number without a context does not really advance the analysis of impact and legacy. Is the rationale for developing the new method (or application) clearly explained? Yes Is the description of the method technically sound? Partly Are sufficient details provided to allow replication of the method development and its use by others? No If any results are presented, are all the source data underlying the results available to ensure full reproducibility? Partly Are the conclusions about the method and its performance adequately supported by the findings presented in the article? Partly Competing Interests No competing interests were disclosed. Reviewer Expertise Regarding this article: NLP and text mining and active participant in ELIXIR. I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (0) Valencia A. Peer Review Report For: Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure for life science data [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 (ELIXIR):1547 ( https://doi.org/10.5256/f1000research.180668.r380723) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1547/v3#referee-response-380723 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 Kawashima S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 10 May 2025 | for Version 3 Shuichi Kawashima , Database Center for Life Science, Chiba, Japan 0 Views copyright © 2025 Kawashima S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions I thank the authors for their thorough revisions and clarifications. All of my previous concerns, including the issue regarding the citation count discrepancy, have now been addressed. I have no further comments. I recommend acceptance of the manuscript. Competing Interests No competing interests were disclosed. Reviewer Expertise bioinformatics, semantic web, database I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) Kawashima S. Peer Review Report For: Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure for life science data [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 (ELIXIR):1547 ( https://doi.org/10.5256/f1000research.180668.r379750) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1547/v3#referee-response-379750 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 Kawashima S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 14 Apr 2025 | for Version 2 Shuichi Kawashima , Database Center for Life Science, Chiba, Japan 0 Views copyright © 2025 Kawashima S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions I appreciate the authors' efforts to revise the manuscript in response to my previous comments. Most of my concerns have been appropriately addressed, and the manuscript is significantly improved in terms of clarity and reproducibility. Remaining Issue – Citation Count Discrepancy (Figure 2 vs. Text): The authors state that they have resolved the discrepancy between Figure 2 and the corresponding citation count in the text. However, upon reviewing the current version of the manuscript and figure, I was unable to identify any changes from the originally submitted version. Both the text and Figure 2 appear unchanged, and the inconsistency (i.e., the text referring to ~5,000 citations vs. the figure showing ~6,721) remains unresolved. I would appreciate clarification on this point—specifically, whether an update was made but not reflected in the posted version, or whether the discrepancy remains due to another reason. Competing Interests No competing interests were disclosed. Reviewer Expertise bioinformatics, semantic web, database I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (0) Kawashima S. Peer Review Report For: Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure for life science data [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 (ELIXIR):1547 ( https://doi.org/10.5256/f1000research.178964.r375436) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1547/v2#referee-response-375436 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 Kawashima S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 27 Feb 2025 | for Version 1 Shuichi Kawashima , Database Center for Life Science, Chiba, Japan 0 Views copyright © 2025 Kawashima S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This manuscript presents an approach to identifying and measuring the scientific impact of ELIXIR-supported publications, employing text-mining of EuropePMC alongside manual curation. It offers valuable insights into the output and influence of a distributed research infrastructure, with an in-depth case study of ELIXIR Italy. The methodology is generally sound and relevant; however, several clarifications are needed to ensure transparency and reproducibility. Major Concerns: * The authors should include concrete examples (e.g., sample URLs or code snippets) illustrating how EuropePMC’s API was queried and how retrieved data were processed. This addition would strengthen reproducibility. * A subset of known ELIXIR-supported articles had to be manually whitelisted, suggesting certain limitations in the automated text-mining. The authors are encouraged to explain why these articles failed to appear in the initial searches (e.g., unusual acknowledgments or missing metadata), as such detail would be instructive for others employing similar methods. Minor concerns: * The figure 2 indicates ~6,721 citations, whereas the text mentions 5,000. This discrepancy should be resolved for clarity. * According to the text, 408 unique publications were supported by ELIXIR Italy from 2015 to 2023. Figure 3 shows the distribution of these publications across the Node’s institutes. If one sums the values for all institutes in Figure 3, the total appears to be 772, which does not match 408. The authors should clarify whether Figure 3 double-counts papers co-authored by multiple institutes or if other factors account for the difference. The manuscript is well-structured, and the research offers a promising approach for tracking the impact of distributed infrastructures. Addressing the concerns above will strengthen the paper’s clarity and transparency. I recommend minor revision prior to acceptance. Is the rationale for developing the new method (or application) clearly explained? Yes Is the description of the method technically sound? Yes Are sufficient details provided to allow replication of the method development and its use by others? Partly If any results are presented, are all the source data underlying the results available to ensure full reproducibility? Partly Are the conclusions about the method and its performance adequately supported by the findings presented in the article? Yes Competing Interests No competing interests were disclosed. Reviewer Expertise bioinformatics, semantic web, database I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (0) Kawashima S. Peer Review Report For: Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure for life science data [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 (ELIXIR):1547 ( https://doi.org/10.5256/f1000research.173833.r354532) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1547/v1#referee-response-354532 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 Berman H. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 06 Jan 2025 | for Version 1 Helen M. Berman , Chemistry and Chemical Biology, Rutgers, The State University of New Jersey, Piscataway, NJ, USA 0 Views copyright © 2025 Berman H. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This article describes a method to assess the impact of ELIXIR supported research. This was done by using text mining to search for relevant articles contained in EuropePMC. This procedure was used for all ELIXIR nodes and for ELIXIR Italy. In the latter case the search terms were modified to be appropriate for that node. In the first step specific search terms were used to search sections of articles. This automated procedure was followed by manual curation. It is important to have quantitative methods to analyze impact of the ELIXIR infrastructure and this is a good first step. The paper is well written, and the limitations of the approach are adequately described. For example, the EuropePMC does not index all the journals that describe projects funded under the ELIXIR umbrella. It is also disturbing that some articles are published in journals that are behind paywalls. Although the curated datasets are put into Zenodo, the authors might consider making a website that would allow authors to submit and curate their ELIXIR related articles. Is the rationale for developing the new method (or application) clearly explained? Yes Is the description of the method technically sound? Yes Are sufficient details provided to allow replication of the method development and its use by others? Partly If any results are presented, are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions about the method and its performance adequately supported by the findings presented in the article? Yes Competing Interests No competing interests were disclosed. Reviewer Expertise Structural bioinformatics, structural biology I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) Berman HM. Peer Review Report For: Using text-mining to measure the scientific impact and legacy of ELIXIR, a distributed research infrastructure for life science data [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 (ELIXIR):1547 ( https://doi.org/10.5256/f1000research.173833.r354530) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1547/v1#referee-response-354530 Alongside their report, reviewers assign a status to the article: Approved - the paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations - A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved - fundamental flaws in the paper seriously undermine the findings and conclusions Adjust parameters to alter display View on desktop for interactive features Includes Interactive Elements View on desktop for interactive features Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Stay Updated Sign up for content alerts and receive a weekly or monthly email with all newly published articles Register with F1000Research Already registered? Sign in Not now, thanks close PLEASE NOTE If you are an AUTHOR of this article, please check that you signed in with the account associated with this article otherwise we cannot automatically identify your role as an author and your comment will be labelled as a “User Comment”. If you are a REVIEWER of this article, please check that you have signed in with the account associated with this article and then go to your account to submit your report, please do not post your review here. If you do not have access to your original account, please contact us . All commenters must hold a formal affiliation as per our Policies . The information that you give us will be displayed next to your comment. User comments must be in English, comprehensible and relevant to the article under discussion. We reserve the right to remove any comments that we consider to be inappropriate, offensive or otherwise in breach of the User Comment Terms and Conditions . Commenters must not use a comment for personal attacks. When criticisms of the article are based on unpublished data, the data should be made available. I accept the User Comment Terms and Conditions Please confirm that you accept the User Comment Terms and Conditions. Affiliation ✕ refresh Please enter your institution. Note: To add your institution or organisation, start typing the name and then select the correct name from the list. Where applicable, the name will appear in both the original language and in English. Do not paste in the name. If the name does not appear in the drop-down list, we will display the information you have entered. ✕ refresh Country/Region * USA UK Canada China France Germany Afghanistan Aland Islands Albania Algeria American Samoa Andorra Angola Anguilla Antarctica Antigua and Barbuda Argentina Armenia Aruba Australia Austria Azerbaijan Bahamas Bahrain Bangladesh Barbados Belarus Belgium Belize Benin Bermuda Bhutan Bolivia Bosnia and Herzegovina Botswana Bouvet Island Brazil British Indian Ocean Territory British Virgin Islands Brunei Bulgaria Burkina Faso Burundi Cambodia Cameroon Canada Cape Verde Cayman Islands Central African Republic Chad Chile China Christmas Island Cocos (Keeling) Islands Colombia Comoros Congo Cook Islands Costa Rica Cote d'Ivoire Croatia Cuba Cyprus Czech Republic Democratic Republic of the Congo Denmark Djibouti Dominica Dominican Republic Ecuador Egypt El Salvador Equatorial Guinea Eritrea Estonia Ethiopia Falkland Islands Faroe Islands Federated States of Micronesia Fiji Finland France French Guiana French Polynesia French Southern Territories Gabon Georgia Germany Ghana Gibraltar Greece Greenland Grenada Guadeloupe Guam Guatemala Guernsey Guinea Guinea-Bissau Guyana Haiti Heard Island and Mcdonald Islands Holy See (Vatican City State) Honduras Hong Kong Hungary Iceland India Indonesia Iran Iraq Ireland Israel Italy Jamaica Japan Jersey Jordan Kazakhstan Kenya Kiribati Kosovo (Serbia and Montenegro) Kuwait Kyrgyzstan Lao People's Democratic Republic Latvia Lebanon Lesotho Liberia Libya Liechtenstein Lithuania Luxembourg Macao Madagascar Malawi Malaysia Maldives Mali Malta Marshall Islands Martinique Mauritania Mauritius Mayotte Mexico Minor Outlying Islands of the United States Moldova Monaco Mongolia Montenegro Montserrat Morocco Mozambique Myanmar Namibia Nauru Nepal Netherlands Antilles New Caledonia New Zealand Nicaragua Niger Nigeria Niue Norfolk Island North Korea North Macedonia Northern Mariana Islands Norway Oman Pakistan Palau Palestinian Territory Panama Papua New Guinea Paraguay Peru Philippines Pitcairn Poland Portugal Puerto Rico Qatar Reunion Romania Russian Federation Rwanda Saint Helena Saint Kitts and Nevis Saint Lucia Saint Pierre and Miquelon Saint Vincent and the Grenadines Samoa San Marino Sao Tome and Principe Saudi Arabia Senegal Serbia Seychelles Sierra Leone Singapore Slovakia Slovenia Solomon Islands Somalia South Africa South Georgia and the South Sandwich Is South Korea South Sudan Spain Sri Lanka Sudan Suriname Svalbard and Jan Mayen Swaziland Sweden Switzerland Syria Taiwan Tajikistan Tanzania Thailand The Gambia The Netherlands Timor-Leste Togo Tokelau Tonga Trinidad and Tobago Tunisia Turkey Turkmenistan Turks and Caicos Islands Tuvalu UK USA Uganda Ukraine United Arab Emirates United States Virgin Islands Uruguay Uzbekistan Vanuatu Venezuela Vietnam Wallis and Futuna West Bank and Gaza Strip Western Sahara Yemen Zambia Zimbabwe Please select your country/region. You must enter a comment. Competing Interests Please disclose any competing interests that might be construed to influence your judgment of the article's or peer review report's validity or importance. Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Please state your competing interests The comment has been saved. An error has occurred. Please try again. Cancel Post var lTitle = \"Using text-mining to measure the scientific...\".replace(\"'\", ''); var linkedInUrl = \"http://www.linkedin.com/shareArticle?url=https://f1000research.com/articles/13-1547/v1\" + \"&title=\" + encodeURIComponent(lTitle) + \"&summary=\" + encodeURIComponent('Read the article by '); var deliciousUrl = \"https://del.icio.us/post?url=https://f1000research.com/articles/13-1547/v1&title=\" + encodeURIComponent(lTitle); var redditUrl = \"http://reddit.com/submit?url=https://f1000research.com/articles/13-1547/v1\" + \"&title=\" + encodeURIComponent(lTitle); linkedInUrl += encodeURIComponent('De Leo F et al.'); var offsetTop = /chrome/i.test( navigator.userAgent ) ? 4 : -10; var addthis_config = { ui_offset_top: offsetTop, services_compact : \"facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com\", services_expanded : \"facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com\", services_custom : [ { name: \"LinkedIn\", url: linkedInUrl, icon:\"/img/icon/at_linkedin.svg\" }, { name: \"Mendeley\", url: \"http://www.mendeley.com/import/?url=https://f1000research.com/articles/13-1547/v1/mendeley\", icon:\"/img/icon/at_mendeley.svg\" }, { name: \"Reddit\", url: redditUrl, icon:\"/img/icon/at_reddit.svg\" }, ] }; var addthis_share = { url: \"https://f1000research.com/articles/13-1547\", templates : { twitter : \"Using text-mining to measure the scientific impact and legacy.... De Leo F et al., published by \" + \"@F1000Research\" + \", https://f1000research.com/articles/13-1547/v1\" } }; if (typeof(addthis) != \"undefined\"){ addthis.addEventListener('addthis.ready', checkCount); addthis.addEventListener('addthis.menu.share', checkCount); } $(\".f1r-shares-twitter\").attr(\"href\", \"https://twitter.com/intent/tweet?text=\" + addthis_share.templates.twitter); $(\".f1r-shares-facebook\").attr(\"href\", \"https://www.facebook.com/sharer/sharer.php?u=\" + addthis_share.url); $(\".f1r-shares-linkedin\").attr(\"href\", addthis_config.services_custom[0].url); $(\".f1r-shares-reddit\").attr(\"href\", addthis_config.services_custom[2].url); $(\".f1r-shares-mendelay\").attr(\"href\", addthis_config.services_custom[1].url); function checkCount(){ setTimeout(function(){ $(\".addthis_button_expanded\").each(function(){ var count = $(this).text(); if (count !== \"\" && count != \"0\") $(this).removeClass(\"is-hidden\"); else $(this).addClass(\"is-hidden\"); }); }, 1000); } close How to cite this report {{reportCitation}} Cancel Copy Citation Details $(function(){R.ui.buttonDropdowns('.dropdown-for-downloads');}); $(function(){R.ui.toolbarDropdowns('.toolbar-dropdown-for-downloads');}); $.get(\"/articles/acj/158264/173833\") new F1000.Clipboard(); new F1000.ThesaurusTermsDisplay(\"articles\", \"article\", \"173833\"); $(document).ready(function() { $( \"#frame1\" ).on('load', function() { var mydiv = $(this).contents().find(\"div\"); var h = mydiv.height(); console.log(h) }); var tooltipLivingFigure = jQuery(\".interactive-living-figure-label .icon-more-info\"), titleLivingFigure = tooltipLivingFigure.attr(\"title\"); tooltipLivingFigure.simpletip({ fixed: true, position: [\"-115\", \"30\"], baseClass: 'small-tooltip', content:titleLivingFigure + \" \" }); tooltipLivingFigure.removeAttr(\"title\"); $(\"body\").on(\"click\", \".cite-living-figure\", function(e) { e.preventDefault(); var ref = $(this).attr(\"data-ref\"); $(this).closest(\".living-figure-list-container\").find(\"#\" + ref).fadeIn(200); }); $(\"body\").on(\"click\", \".close-cite-living-figure\", function(e) { e.preventDefault(); $(this).closest(\".popup-window-wrapper\").fadeOut(200); }); $(document).on(\"mouseup\", function(e) { var metricsContainer = $(\".article-metrics-popover-wrapper\"); if (!metricsContainer.is(e.target) && metricsContainer.has(e.target).length === 0) { $(\".article-metrics-close-button\").click(); } }); var articleId = $('#articleId').val(); if($(\"#main-article-count-box\").attachArticleMetrics) { $(\"#main-article-count-box\").attachArticleMetrics(articleId, { articleMetricsView: true }); } }); var figshareWidget = $(\".new_figshare_widget\"); if (figshareWidget.length > 0) { window.figshare.load(\"f1000\", function(Widget) { // Select a tag/tags defined in your page. In this tag we will place the widget. _.map(figshareWidget, function(el){ var widget = new Widget({ articleId: $(el).attr(\"figshare_articleId\") //height:300 // this is the height of the viewer part. [Default: 550] }); widget.initialize(); // initialize the widget widget.mount(el); // mount it in a tag that's on your page // this will save the widget on the global scope for later use from // your JS scripts. This line is optional. //window.widget = widget; }); }); } close Error Close Add Reset F1000.MICROSERVICES.AFFILIATION = ''; $(document).ready(function () { $('.js-affiliations-form').each((index, form) => { new AffiliationForm({ formId: form.id, institutionErrorSelector: '.comment-enter-institution', departmentErrorSelector: '.comment-enter-department', placeSelector: '.js-add-comment-place', stateSelector: '.js-add-comment-state', zipCodeSelector: '.js-add-comment-zipcode', countrySelector: '.js-add-comment-country', countryErrorSelector: '.comment-enter-country', }); }); }); $(document).ready(function () { var reportIds = { \"375809\": 0, \"375808\": 0, \"375811\": 0, \"375810\": 0, \"375437\": 0, \"375436\": 10, \"380717\": 0, \"380716\": 0, \"380719\": 0, \"380718\": 0, \"380715\": 0, \"380714\": 0, \"380721\": 0, \"380720\": 0, \"380723\": 13, \"380722\": 0, \"351805\": 0, \"377919\": 0, \"351807\": 0, \"351806\": 0, \"377925\": 0, \"351813\": 0, \"377924\": 0, \"351812\": 0, \"351814\": 0, \"377921\": 0, \"351809\": 0, \"377920\": 0, \"351808\": 0, \"377923\": 0, \"351811\": 0, \"377922\": 0, \"351810\": 0, \"368093\": 0, \"368092\": 0, \"368095\": 0, \"368094\": 0, \"379749\": 0, \"354533\": 0, \"368101\": 0, \"368100\": 0, \"354532\": 18, \"354535\": 0, \"379750\": 16, \"354534\": 0, \"354529\": 0, \"368097\": 0, \"368096\": 0, \"354531\": 0, \"368099\": 0, \"368098\": 0, \"354530\": 15, \"354537\": 0, \"354536\": 0, \"354538\": 0, \"375805\": 0, \"375804\": 0, \"375807\": 0, \"375806\": 0, \"375803\": 0, \"375802\": 0, }; $(\".referee-response-container,.js-referee-report\").each(function(index, el) { var reportId = $(el).attr(\"data-reportid\"), reportCount = reportIds[reportId] || 0; $(el).find(\".comments-count-container,.js-referee-report-views\").html(reportCount); }); var uuidInput = $(\"#article_uuid\"), oldUUId = uuidInput.val(), newUUId = \"e1685c3d-61ad-4de2-83f7-ca401feed6e4\"; uuidInput.val(newUUId); $(\"a[href*='article_uuid=']\").each(function(index, el) { var newHref = $(el).attr(\"href\").replace(oldUUId, newUUId); $(el).attr(\"href\", newHref); }); }); An innovative open access publishing platform offering rapid publication and open peer review, whilst supporting data deposition and sharing. Browse Gateways Collections How it Works Contact For Developers Cookie Notice Privacy Notice RSS Submit Your Research Follow us © 2012-2026 F1000 Research Ltd. ISSN 2046-1402 | Legal | Partner of Research4Life • CrossRef • ORCID • FAIRSharing R.templateTests.simpleTemplate = R.template(' $text $text $text $text $text '); R.templateTests.runTests(); var F1000platform = new F1000.Platform({ name: \"f1000research\", displayName: \"F1000Research\", hostName: \"f1000research.com\", id: \"1\", editorialEmail: \"research@f1000.com\", infoEmail: \"info@f1000.com\", usePmcStats: true }); $(function(){R.ui.dropdowns('.dropdown-for-authors, .dropdown-for-about, .dropdown-for-myresearch');}); // $(function(){R.ui.dropdowns('.dropdown-for-referees');}); $(document).ready(function () { if ($(\".cookie-warning\").is(\":visible\")) { $(\".sticky\").css(\"margin-bottom\", \"35px\"); $(\".devices\").addClass(\"devices-and-cookie-warning\"); } $(\".cookie-warning .close-button\").click(function (e) { $(\".devices\").removeClass(\"devices-and-cookie-warning\"); $(\".sticky\").css(\"margin-bottom\", \"0\"); }); $(\"#tweeter-feed .tweet-message\").each(function (i, message) { var self = $(message); self.html(linkify(self.html())); }); $(\".partner\").on(\"mouseenter mouseleave\", function() { $(this).find(\".gray-scale, .colour\").toggleClass(\"is-hidden\"); }); }); <!-- Sign in --> Sign In Remember me Forgotten your password? Sign In Cancel Email or password not correct. Please try again Please wait... $(function(){ // Note: All the setup needs to run against a name attribute and *not* the id due the clonish // nature of facebox... $(\"a[id=googleSignInButton]\").click(function(event){ event.preventDefault(); $(\"input[id=oAuthSystem]\").val(\"GOOGLE\"); $(\"form[id=oAuthForm]\").submit(); }); $(\"a[id=facebookSignInButton]\").click(function(event){ event.preventDefault(); $(\"input[id=oAuthSystem]\").val(\"FACEBOOK\"); $(\"form[id=oAuthForm]\").submit(); }); $(\"a[id=orcidSignInButton]\").click(function(event){ event.preventDefault(); $(\"input[id=oAuthSystem]\").val(\"ORCID\"); $(\"form[id=oAuthForm]\").submit(); }); }); If you've forgotten your password, please enter your email address below and we'll send you instructions on how to reset your password. The email address should be the one you originally registered with F1000. Email address not valid, please try again You registered with F1000 via Google, so we cannot reset your password. To sign in, please click here . If you still need help with your Google account password, please click here . You registered with F1000 via Facebook, so we cannot reset your password. To sign in, please click here . If you still need help with your Facebook account password, please click here . Code not correct, please try again Reset password Cancel Email us for further assistance. Server error, please try again. If your email address is registered with us, we will email you instructions to reset your password. If you think you should have received this email but it has not arrived, please check your spam filters and/or contact for further assistance. Please wait... Register $(document).ready(function () { signIn.createSignInAsRow($(\"#sign-in-form-gfb-popup\")); $(\".target-field\").each(function () { var uris = $(this).val().split(\"/\"); if (uris.pop() === \"login\") { $(this).val(uris.toString().replace(\",\",\"/\")); } }); });","source_license":"CC-BY-4.0","license_restricted":false}