Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index

Does the impact of medical publications vary by... | F1000Research "use strict";function _typeof(t){return(_typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}!function(){var t=function(){var t,e,o=[],n=window,r=n;for(;r;){try{if(r.frames.__tcfapiLocator){t=r;break}}catch(t){}if(r===n.top)break;r=r.parent}t||(!function t(){var e=n.document,o=!!n.frames.__tcfapiLocator;if(!o)if(e.body){var r=e.createElement("iframe");r.style.cssText="display:none",r.name="__tcfapiLocator",e.body.appendChild(r)}else setTimeout(t,5);return!o}(),n.__tcfapi=function(){for(var t=arguments.length,n=new Array(t),r=0;r 3&&2===parseInt(n[1],10)&&"boolean"==typeof n[3]&&(e=n[3],"function"==typeof n[2]&&n[2]("set",!0)):"ping"===n[0]?"function"==typeof n[2]&&n[2]({gdprApplies:e,cmpLoaded:!1,cmpStatus:"stub"}):o.push(n)},n.addEventListener("message",(function(t){var e="string"==typeof t.data,o={};if(e)try{o=JSON.parse(t.data)}catch(t){}else o=t.data;var n="object"===_typeof(o)&&null!==o?o.__tcfapiCall:null;n&&window.__tcfapi(n.command,n.version,(function(o,r){var a={__tcfapiReturn:{returnValue:o,success:r,callId:n.callId}};t&&t.source&&t.source.postMessage&&t.source.postMessage(e?JSON.stringify(a):a,"*")}),n.parameter)}),!1))};"undefined"!=typeof module?module.exports=t:t()}(); dataLayer = dataLayer || []; // Standard GTM initialization - Google Consent Mode handles consent automatically (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl+ '>m_auth=hzk0Vc3qFsQYhCrIoHz68A>m_preview=env-1>m_cookies_win=x';f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-MWFK8L5J'); ;window.NREUM||(NREUM={});NREUM.init={distributed_tracing:{enabled:true},privacy:{cookies_enabled:true},ajax:{deny_list:["bam.nr-data.net"]}}; ;NREUM.loader_config={accountID:"438030",trustKey:"438030",agentID:"772317073",licenseKey:"97f8f67f26",applicationID:"772317073"} ;NREUM.info={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net",licenseKey:"97f8f67f26",applicationID:"772317073",sa:1} ;/*! For license information please see nr-loader-spa-1.236.0.min.js.LICENSE.txt */ (()=>{"use strict";var e,t,r={5763:(e,t,r)=>{r.d(t,{P_:()=>l,Mt:()=>g,C5:()=>s,DL:()=>v,OP:()=>T,lF:()=>D,Yu:()=>y,Dg:()=>h,CX:()=>c,GE:()=>b,sU:()=>_});var n=r(8632),i=r(9567);const o={beacon:n.ce.beacon,errorBeacon:n.ce.errorBeacon,licenseKey:void 0,applicationID:void 0,sa:void 0,queueTime:void 0,applicationTime:void 0,ttGuid:void 0,user:void 0,account:void 0,product:void 0,extra:void 0,jsAttributes:{},userAttributes:void 0,atts:void 0,transactionName:void 0,tNamePlain:void 0},a={};function s(e){if(!e)throw new Error("All info objects require an agent identifier!");if(!a[e])throw new Error("Info for ".concat(e," was never set"));return a[e]}function c(e,t){if(!e)throw new Error("All info objects require an agent identifier!");a[e]=(0,i.D)(t,o),(0,n.Qy)(e,a[e],"info")}var u=r(7056);const d=()=>{const e={blockSelector:"[data-nr-block]",maskInputOptions:{password:!0}};return{allow_bfcache:!0,privacy:{cookies_enabled:!0},ajax:{deny_list:void 0,enabled:!0,harvestTimeSeconds:10},distributed_tracing:{enabled:void 0,exclude_newrelic_header:void 0,cors_use_newrelic_header:void 0,cors_use_tracecontext_headers:void 0,allowed_origins:void 0},session:{domain:void 0,expiresMs:u.oD,inactiveMs:u.Hb},ssl:void 0,obfuscate:void 0,jserrors:{enabled:!0,harvestTimeSeconds:10},metrics:{enabled:!0},page_action:{enabled:!0,harvestTimeSeconds:30},page_view_event:{enabled:!0},page_view_timing:{enabled:!0,harvestTimeSeconds:30,long_task:!1},session_trace:{enabled:!0,harvestTimeSeconds:10},harvest:{tooManyRequestsDelay:60},session_replay:{enabled:!1,harvestTimeSeconds:60,sampleRate:.1,errorSampleRate:.1,maskTextSelector:"*",maskAllInputs:!0,get blockClass(){return"nr-block"},get ignoreClass(){return"nr-ignore"},get maskTextClass(){return"nr-mask"},get blockSelector(){return e.blockSelector},set blockSelector(t){e.blockSelector+=",".concat(t)},get maskInputOptions(){return e.maskInputOptions},set maskInputOptions(t){e.maskInputOptions={...t,password:!0}}},spa:{enabled:!0,harvestTimeSeconds:10}}},f={};function l(e){if(!e)throw new Error("All configuration objects require an agent identifier!");if(!f[e])throw new Error("Configuration for ".concat(e," was never set"));return f[e]}function h(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");f[e]=(0,i.D)(t,d()),(0,n.Qy)(e,f[e],"config")}function g(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");var r=l(e);if(r){for(var n=t.split("."),i=0;i {r.d(t,{D:()=>i});var n=r(50);function i(e,t){try{if(!e||"object"!=typeof e)return(0,n.Z)("Setting a Configurable requires an object as input");if(!t||"object"!=typeof t)return(0,n.Z)("Setting a Configurable requires a model to set its initial properties");const r=Object.create(Object.getPrototypeOf(t),Object.getOwnPropertyDescriptors(t)),o=0===Object.keys(r).length?e:r;for(let a in o)if(void 0!==e[a])try{"object"==typeof e[a]&&"object"==typeof t[a]?r[a]=i(e[a],t[a]):r[a]=e[a]}catch(e){(0,n.Z)("An error occurred while setting a property of a Configurable",e)}return r}catch(e){(0,n.Z)("An error occured while setting a Configurable",e)}}},6818:(e,t,r)=>{r.d(t,{Re:()=>i,gF:()=>o,q4:()=>n});const n="1.236.0",i="PROD",o="CDN"},385:(e,t,r)=>{r.d(t,{FN:()=>a,IF:()=>u,Nk:()=>f,Tt:()=>s,_A:()=>o,il:()=>n,pL:()=>c,v6:()=>i,w1:()=>d});const n="undefined"!=typeof window&&!!window.document,i="undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self.navigator instanceof WorkerNavigator||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis.navigator instanceof WorkerNavigator),o=n?window:"undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis),a=""+o?.location,s=/iPad|iPhone|iPod/.test(navigator.userAgent),c=s&&"undefined"==typeof SharedWorker,u=(()=>{const e=navigator.userAgent.match(/Firefox[/\s](\d+\.\d+)/);return Array.isArray(e)&&e.length>=2?+e[1]:0})(),d=Boolean(n&&window.document.documentMode),f=!!navigator.sendBeacon},1117:(e,t,r)=>{r.d(t,{w:()=>o});var n=r(50);const i={agentIdentifier:"",ee:void 0};class o{constructor(e){try{if("object"!=typeof e)return(0,n.Z)("shared context requires an object as input");this.sharedContext={},Object.assign(this.sharedContext,i),Object.entries(e).forEach((e=>{let[t,r]=e;Object.keys(i).includes(t)&&(this.sharedContext[t]=r)}))}catch(e){(0,n.Z)("An error occured while setting SharedContext",e)}}}},8e3:(e,t,r)=>{r.d(t,{L:()=>d,R:()=>c});var n=r(2177),i=r(1284),o=r(4322),a=r(3325);const s={};function c(e,t){const r={staged:!1,priority:a.p[t]||0};u(e),s[e].get(t)||s[e].set(t,r)}function u(e){e&&(s[e]||(s[e]=new Map))}function d(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"",t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"feature";if(u(e),!e||!s[e].get(t))return a(t);s[e].get(t).staged=!0;const r=[...s[e]];function a(t){const r=e?n.ee.get(e):n.ee,a=o.X.handlers;if(r.backlog&&a){var s=r.backlog[t],c=a[t];if(c){for(var u=0;s&&u {let[t,r]=e;return r.staged}))&&(r.sort(((e,t)=>e[1].priority-t[1].priority)),r.forEach((e=>{let[t]=e;a(t)})))}function f(e,t){var r=e[1];(0,i.D)(t[r],(function(t,r){var n=e[0];if(r[0]===n){var i=r[1],o=e[3],a=e[2];i.apply(o,a)}}))}},2177:(e,t,r)=>{r.d(t,{c:()=>f,ee:()=>u});var n=r(8632),i=r(2210),o=r(1284),a=r(5763),s="nr@context";let c=(0,n.fP)();var u;function d(){}function f(e){return(0,i.X)(e,s,l)}function l(){return new d}function h(){u.aborted=!0,u.backlog={}}c.ee?u=c.ee:(u=function e(t,r){var n={},c={},f={},g=!1;try{g=16===r.length&&(0,a.OP)(r).isolatedBacklog}catch(e){}var p={on:b,addEventListener:b,removeEventListener:y,emit:v,get:x,listeners:w,context:m,buffer:A,abort:h,aborted:!1,isBuffering:E,debugId:r,backlog:g?{}:t&&"object"==typeof t.backlog?t.backlog:{}};return p;function m(e){return e&&e instanceof d?e:e?(0,i.X)(e,s,l):l()}function v(e,r,n,i,o){if(!1!==o&&(o=!0),!u.aborted||i){t&&o&&t.emit(e,r,n);for(var a=m(n),s=w(e),d=s.length,f=0;fn,p:()=>i});var n=r(2177).ee.get("handle");function i(e,t,r,i,o){o?(o.buffer([e],i),o.emit(e,t,r)):(n.buffer([e],i),n.emit(e,t,r))}},4322:(e,t,r)=>{r.d(t,{X:()=>o});var n=r(5546);o.on=a;var i=o.handlers={};function o(e,t,r,o){a(o||n.E,i,e,t,r)}function a(e,t,r,i,o){o||(o="feature"),e||(e=n.E);var a=t[o]=t[o]||{};(a[r]=a[r]||[]).push([e,i])}},3239:(e,t,r)=>{r.d(t,{bP:()=>s,iz:()=>c,m$:()=>a});var n=r(385);let i=!1,o=!1;try{const e={get passive(){return i=!0,!1},get signal(){return o=!0,!1}};n._A.addEventListener("test",null,e),n._A.removeEventListener("test",null,e)}catch(e){}function a(e,t){return i||o?{capture:!!e,passive:i,signal:t}:!!e}function s(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;window.addEventListener(e,t,a(r,n))}function c(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;document.addEventListener(e,t,a(r,n))}},4402:(e,t,r)=>{r.d(t,{Ht:()=>u,M:()=>c,Rl:()=>a,ky:()=>s});var n=r(385);const i="xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx";function o(e,t){return e?15&e[t]:16*Math.random()|0}function a(){const e=n._A?.crypto||n._A?.msCrypto;let t,r=0;return e&&e.getRandomValues&&(t=e.getRandomValues(new Uint8Array(31))),i.split("").map((e=>"x"===e?o(t,++r).toString(16):"y"===e?(3&o()|8).toString(16):e)).join("")}function s(e){const t=n._A?.crypto||n._A?.msCrypto;let r,i=0;t&&t.getRandomValues&&(r=t.getRandomValues(new Uint8Array(31)));const a=[];for(var s=0;s {r.d(t,{Bq:()=>n,Hb:()=>o,oD:()=>i});const n="NRBA",i=144e5,o=18e5},7894:(e,t,r)=>{function n(){return Math.round(performance.now())}r.d(t,{z:()=>n})},7243:(e,t,r)=>{r.d(t,{e:()=>o});var n=r(385),i={};function o(e){if(e in i)return i[e];if(0===(e||"").indexOf("data:"))return{protocol:"data"};let t;var r=n._A?.location,o={};if(n.il)t=document.createElement("a"),t.href=e;else try{t=new URL(e,r.href)}catch(e){return o}o.port=t.port;var a=t.href.split("://");!o.port&&a[1]&&(o.port=a[1].split("/")[0].split("@").pop().split(":")[1]),o.port&&"0"!==o.port||(o.port="https"===a[0]?"443":"80"),o.hostname=t.hostname||r.hostname,o.pathname=t.pathname,o.protocol=a[0],"/"!==o.pathname.charAt(0)&&(o.pathname="/"+o.pathname);var s=!t.protocol||":"===t.protocol||t.protocol===r.protocol,c=t.hostname===r.hostname&&t.port===r.port;return o.sameOrigin=s&&(!t.hostname||c),"/"===o.pathname&&(i[e]=o),o}},50:(e,t,r)=>{function n(e,t){"function"==typeof console.warn&&(console.warn("New Relic: ".concat(e)),t&&console.warn(t))}r.d(t,{Z:()=>n})},2587:(e,t,r)=>{r.d(t,{N:()=>c,T:()=>u});var n=r(2177),i=r(5546),o=r(8e3),a=r(3325);const s={stn:[a.D.sessionTrace],err:[a.D.jserrors,a.D.metrics],ins:[a.D.pageAction],spa:[a.D.spa],sr:[a.D.sessionReplay,a.D.sessionTrace]};function c(e,t){const r=n.ee.get(t);e&&"object"==typeof e&&(Object.entries(e).forEach((e=>{let[t,n]=e;void 0===u[t]&&(s[t]?s[t].forEach((e=>{n?(0,i.p)("feat-"+t,[],void 0,e,r):(0,i.p)("block-"+t,[],void 0,e,r),(0,i.p)("rumresp-"+t,[Boolean(n)],void 0,e,r)})):n&&(0,i.p)("feat-"+t,[],void 0,void 0,r),u[t]=Boolean(n))})),Object.keys(s).forEach((e=>{void 0===u[e]&&(s[e]?.forEach((t=>(0,i.p)("rumresp-"+e,[!1],void 0,t,r))),u[e]=!1)})),(0,o.L)(t,a.D.pageViewEvent))}const u={}},2210:(e,t,r)=>{r.d(t,{X:()=>i});var n=Object.prototype.hasOwnProperty;function i(e,t,r){if(n.call(e,t))return e[t];var i=r();if(Object.defineProperty&&Object.keys)try{return Object.defineProperty(e,t,{value:i,writable:!0,enumerable:!1}),i}catch(e){}return e[t]=i,i}},1284:(e,t,r)=>{r.d(t,{D:()=>n});const n=(e,t)=>Object.entries(e||{}).map((e=>{let[r,n]=e;return t(r,n)}))},4351:(e,t,r)=>{r.d(t,{P:()=>o});var n=r(2177);const i=()=>{const e=new WeakSet;return(t,r)=>{if("object"==typeof r&&null!==r){if(e.has(r))return;e.add(r)}return r}};function o(e){try{return JSON.stringify(e,i())}catch(e){try{n.ee.emit("internal-error",[e])}catch(e){}}}},3960:(e,t,r)=>{r.d(t,{K:()=>a,b:()=>o});var n=r(3239);function i(){return"undefined"==typeof document||"complete"===document.readyState}function o(e,t){if(i())return e();(0,n.bP)("load",e,t)}function a(e){if(i())return e();(0,n.iz)("DOMContentLoaded",e)}},8632:(e,t,r)=>{r.d(t,{EZ:()=>u,Qy:()=>c,ce:()=>o,fP:()=>a,gG:()=>d,mF:()=>s});var n=r(7894),i=r(385);const o={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net"};function a(){return i._A.NREUM||(i._A.NREUM={}),void 0===i._A.newrelic&&(i._A.newrelic=i._A.NREUM),i._A.NREUM}function s(){let e=a();return e.o||(e.o={ST:i._A.setTimeout,SI:i._A.setImmediate,CT:i._A.clearTimeout,XHR:i._A.XMLHttpRequest,REQ:i._A.Request,EV:i._A.Event,PR:i._A.Promise,MO:i._A.MutationObserver,FETCH:i._A.fetch}),e}function c(e,t,r){let i=a();const o=i.initializedAgents||{},s=o[e]||{};return Object.keys(s).length||(s.initializedAt={ms:(0,n.z)(),date:new Date}),i.initializedAgents={...o,[e]:{...s,[r]:t}},i}function u(e,t){a()[e]=t}function d(){return function(){let e=a();const t=e.info||{};e.info={beacon:o.beacon,errorBeacon:o.errorBeacon,...t}}(),function(){let e=a();const t=e.init||{};e.init={...t}}(),s(),function(){let e=a();const t=e.loader_config||{};e.loader_config={...t}}(),a()}},7956:(e,t,r)=>{r.d(t,{N:()=>i});var n=r(3239);function i(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],r=arguments.length>2?arguments[2]:void 0,i=arguments.length>3?arguments[3]:void 0;return void(0,n.iz)("visibilitychange",(function(){if(t)return void("hidden"==document.visibilityState&&e());e(document.visibilityState)}),r,i)}},1214:(e,t,r)=>{r.d(t,{em:()=>v,u5:()=>N,QU:()=>S,_L:()=>I,Gm:()=>L,Lg:()=>M,gy:()=>U,BV:()=>Q,Kf:()=>ee});var n=r(2177);const i="nr@original";var o=Object.prototype.hasOwnProperty,a=!1;function s(e,t){return e||(e=n.ee),r.inPlace=function(e,t,n,i,o){n||(n="");var a,s,c,u="-"===n.charAt(0);for(c=0;c 2?n-2:0),o=2;o {r(A[T],e,w),r(E[T],e,w)})),r(l._A,"fetch",y),t.on(y+"end",(function(e,r){var n=this;if(r){var i=r.headers.get("content-length");null!==i&&(n.rxSize=i),t.emit(y+"done",[null,r],n)}else t.emit(y+"done",[e],n)})),t}const O={},j=["pushState","replaceState"];function S(e){const t=function(e){return(e||n.ee).get("history")}(e);return!l.il||O[t.debugId]++||(O[t.debugId]=1,s(t).inPlace(window.history,j,"-")),t}var P=r(3239);const C={},R=["appendChild","insertBefore","replaceChild"];function I(e){const t=function(e){return(e||n.ee).get("jsonp")}(e);if(!l.il||C[t.debugId])return t;C[t.debugId]=!0;var r=s(t),i=/[?&](?:callback|cb)=([^&#]+)/,o=/(.*)\.([^.]+)/,a=/^(\w+)(\.|$)(.*)$/;function c(e,t){var r=e.match(a),n=r[1],i=r[3];return i?c(i,t[n]):t[n]}return r.inPlace(Node.prototype,R,"dom-"),t.on("dom-start",(function(e){!function(e){if(!e||"string"!=typeof e.nodeName||"script"!==e.nodeName.toLowerCase())return;if("function"!=typeof e.addEventListener)return;var n=(a=e.src,s=a.match(i),s?s[1]:null);var a,s;if(!n)return;var u=function(e){var t=e.match(o);if(t&&t.length>=3)return{key:t[2],parent:c(t[1],window)};return{key:e,parent:window}}(n);if("function"!=typeof u.parent[u.key])return;var d={};function f(){t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}function l(){t.emit("jsonp-error",[],d),t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}r.inPlace(u.parent,[u.key],"cb-",d),e.addEventListener("load",f,(0,P.m$)(!1)),e.addEventListener("error",l,(0,P.m$)(!1)),t.emit("new-jsonp",[e.src],d)}(e[0])})),t}var k=r(5763);const H={};function L(e){const t=function(e){return(e||n.ee).get("mutation")}(e);if(!l.il||H[t.debugId])return t;H[t.debugId]=!0;var r=s(t),i=k.Yu.MO;return i&&(window.MutationObserver=function(e){return this instanceof i?new i(r(e,"fn-")):i.apply(this,arguments)},MutationObserver.prototype=i.prototype),t}const z={};function M(e){const t=function(e){return(e||n.ee).get("promise")}(e);if(z[t.debugId])return t;z[t.debugId]=!0;var r=n.c,o=s(t),a=k.Yu.PR;return a&&function(){function e(r){var n=t.context(),i=o(r,"executor-",n,null,!1);const s=Reflect.construct(a,[i],e);return t.context(s).getCtx=function(){return n},s}l._A.Promise=e,Object.defineProperty(e,"name",{value:"Promise"}),e.toString=function(){return a.toString()},Object.setPrototypeOf(e,a),["all","race"].forEach((function(r){const n=a[r];e[r]=function(e){let i=!1;[...e||[]].forEach((e=>{this.resolve(e).then(a("all"===r),a(!1))}));const o=n.apply(this,arguments);return o;function a(e){return function(){t.emit("propagate",[null,!i],o,!1,!1),i=i||!e}}}})),["resolve","reject"].forEach((function(r){const n=a[r];e[r]=function(e){const r=n.apply(this,arguments);return e!==r&&t.emit("propagate",[e,!0],r,!1,!1),r}})),e.prototype=a.prototype;const n=a.prototype.then;a.prototype.then=function(){var e=this,i=r(e);i.promise=e;for(var a=arguments.length,s=new Array(a),c=0;c e())),t};function m(e,t){i.inPlace(t,["onreadystatechange"],"fn-",E)}function b(){var e=this,t=r.context(e);e.readyState>3&&!t.resolved&&(t.resolved=!0,r.emit("xhr-resolved",[],e)),i.inPlace(e,f,"fn-",E)}if(function(e,t){for(var r in e)t[r]=e[r]}(o,p),p.prototype=o.prototype,i.inPlace(p.prototype,J,"-xhr-",E),r.on("send-xhr-start",(function(e,t){m(e,t),function(e){h.push(e),a&&(y?y.then(A):u?u(A):(w=-w,x.data=w))}(t)})),r.on("open-xhr-start",m),a){var y=c&&c.resolve();if(!u&&!c){var w=1,x=document.createTextNode(w);new a(A).observe(x,{characterData:!0})}}else t.on("fn-end",(function(e){e[0]&&e[0].type===d||A()}));function A(){for(var e=0;e {r.d(t,{t:()=>n});const n=r(3325).D.ajax},6660:(e,t,r)=>{r.d(t,{A:()=>i,t:()=>n});const n=r(3325).D.jserrors,i="nr@seenError"},3081:(e,t,r)=>{r.d(t,{gF:()=>o,mY:()=>i,t9:()=>n,vz:()=>s,xS:()=>a});const n=r(3325).D.metrics,i="sm",o="cm",a="storeSupportabilityMetrics",s="storeEventMetrics"},4649:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageAction},7633:(e,t,r)=>{r.d(t,{Dz:()=>i,OJ:()=>a,qw:()=>o,t9:()=>n});const n=r(3325).D.pageViewEvent,i="firstbyte",o="domcontent",a="windowload"},9251:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageViewTiming},3614:(e,t,r)=>{r.d(t,{BST_RESOURCE:()=>i,END:()=>s,FEATURE_NAME:()=>n,FN_END:()=>u,FN_START:()=>c,PUSH_STATE:()=>d,RESOURCE:()=>o,START:()=>a});const n=r(3325).D.sessionTrace,i="bstResource",o="resource",a="-start",s="-end",c="fn"+a,u="fn"+s,d="pushState"},7836:(e,t,r)=>{r.d(t,{BODY:()=>A,CB_END:()=>E,CB_START:()=>u,END:()=>x,FEATURE_NAME:()=>i,FETCH:()=>_,FETCH_BODY:()=>v,FETCH_DONE:()=>m,FETCH_START:()=>p,FN_END:()=>c,FN_START:()=>s,INTERACTION:()=>l,INTERACTION_API:()=>d,INTERACTION_EVENTS:()=>o,JSONP_END:()=>b,JSONP_NODE:()=>g,JS_TIME:()=>T,MAX_TIMER_BUDGET:()=>a,REMAINING:()=>f,SPA_NODE:()=>h,START:()=>w,originalSetTimeout:()=>y});var n=r(5763);const i=r(3325).D.spa,o=["click","submit","keypress","keydown","keyup","change"],a=999,s="fn-start",c="fn-end",u="cb-start",d="api-ixn-",f="remaining",l="interaction",h="spaNode",g="jsonpNode",p="fetch-start",m="fetch-done",v="fetch-body-",b="jsonp-end",y=n.Yu.ST,w="-start",x="-end",A="-body",E="cb"+x,T="jsTime",_="fetch"},5938:(e,t,r)=>{r.d(t,{W:()=>o});var n=r(5763),i=r(2177);class o{constructor(e,t,r){this.agentIdentifier=e,this.aggregator=t,this.ee=i.ee.get(e,(0,n.OP)(this.agentIdentifier).isolatedBacklog),this.featureName=r,this.blocked=!1}}},9144:(e,t,r)=>{r.d(t,{j:()=>m});var n=r(3325),i=r(5763),o=r(5546),a=r(2177),s=r(7894),c=r(8e3),u=r(3960),d=r(385),f=r(50),l=r(3081),h=r(8632);function g(){const e=(0,h.gG)();["setErrorHandler","finished","addToTrace","inlineHit","addRelease","addPageAction","setCurrentRouteName","setPageViewName","setCustomAttribute","interaction","noticeError","setUserId"].forEach((t=>{e[t]=function(){for(var r=arguments.length,n=new Array(r),i=0;i 1?r-1:0),i=1;i {e.exposed&&e.api[t]&&o.push(e.api[t](...n))})),o.length>1?o:o[0]}(t,...n)}}))}var p=r(2587);function m(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},m=arguments.length>2?arguments[2]:void 0,v=arguments.length>3?arguments[3]:void 0,{init:b,info:y,loader_config:w,runtime:x={loaderType:m},exposed:A=!0}=t;const E=(0,h.gG)();y||(b=E.init,y=E.info,w=E.loader_config),(0,i.Dg)(e,b||{}),(0,i.GE)(e,w||{}),(0,i.sU)(e,x),y.jsAttributes??={},d.v6&&(y.jsAttributes.isWorker=!0),(0,i.CX)(e,y),g();const T=function(e,t){t||(0,c.R)(e,"api");const h={};var g=a.ee.get(e),p=g.get("tracer"),m="api-",v=m+"ixn-";function b(t,r,n,o){const a=(0,i.C5)(e);return null===r?delete a.jsAttributes[t]:(0,i.CX)(e,{...a,jsAttributes:{...a.jsAttributes,[t]:r}}),x(m,n,!0,o||null===r?"session":void 0)(t,r)}function y(){}["setErrorHandler","finished","addToTrace","inlineHit","addRelease"].forEach((e=>h[e]=x(m,e,!0,"api"))),h.addPageAction=x(m,"addPageAction",!0,n.D.pageAction),h.setCurrentRouteName=x(m,"routeName",!0,n.D.spa),h.setPageViewName=function(t,r){if("string"==typeof t)return"/"!==t.charAt(0)&&(t="/"+t),(0,i.OP)(e).customTransaction=(r||"http://custom.transaction")+t,x(m,"setPageViewName",!0)()},h.setCustomAttribute=function(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2];if("string"==typeof e){if(["string","number"].includes(typeof t)||null===t)return b(e,t,"setCustomAttribute",r);(0,f.Z)("Failed to execute setCustomAttribute.\nNon-null value must be a string or number type, but a type of was provided."))}else(0,f.Z)("Failed to execute setCustomAttribute.\nName must be a string type, but a type of was provided."))},h.setUserId=function(e){if("string"==typeof e||null===e)return b("enduser.id",e,"setUserId",!0);(0,f.Z)("Failed to execute setUserId.\nNon-null value must be a string type, but a type of was provided."))},h.interaction=function(){return(new y).get()};var w=y.prototype={createTracer:function(e,t){var r={},i=this,a="function"==typeof t;return(0,o.p)(v+"tracer",[(0,s.z)(),e,r],i,n.D.spa,g),function(){if(p.emit((a?"":"no-")+"fn-start",[(0,s.z)(),i,a],r),a)try{return t.apply(this,arguments)}catch(e){throw p.emit("fn-err",[arguments,this,"string"==typeof e?new Error(e):e],r),e}finally{p.emit("fn-end",[(0,s.z)()],r)}}}};function x(e,t,r,i){return function(){return(0,o.p)(l.xS,["API/"+t+"/called"],void 0,n.D.metrics,g),i&&(0,o.p)(e+t,[(0,s.z)(),...arguments],r?null:this,i,g),r?void 0:this}}function A(){r.e(439).then(r.bind(r,7438)).then((t=>{let{setAPI:r}=t;r(e),(0,c.L)(e,"api")})).catch((()=>(0,f.Z)("Downloading runtime APIs failed...")))}return["actionText","setName","setAttribute","save","ignore","onEnd","getContext","end","get"].forEach((e=>{w[e]=x(v,e,void 0,n.D.spa)})),h.noticeError=function(e,t){"string"==typeof e&&(e=new Error(e)),(0,o.p)(l.xS,["API/noticeError/called"],void 0,n.D.metrics,g),(0,o.p)("err",[e,(0,s.z)(),!1,t],void 0,n.D.jserrors,g)},d.il?(0,u.b)((()=>A()),!0):A(),h}(e,v);return(0,h.Qy)(e,T,"api"),(0,h.Qy)(e,A,"exposed"),(0,h.EZ)("activatedFeatures",p.T),T}},3325:(e,t,r)=>{r.d(t,{D:()=>n,p:()=>i});const n={ajax:"ajax",jserrors:"jserrors",metrics:"metrics",pageAction:"page_action",pageViewEvent:"page_view_event",pageViewTiming:"page_view_timing",sessionReplay:"session_replay",sessionTrace:"session_trace",spa:"spa"},i={[n.pageViewEvent]:1,[n.pageViewTiming]:2,[n.metrics]:3,[n.jserrors]:4,[n.ajax]:5,[n.sessionTrace]:6,[n.pageAction]:7,[n.spa]:8,[n.sessionReplay]:9}}},n={};function i(e){var t=n[e];if(void 0!==t)return t.exports;var o=n[e]={exports:{}};return r[e](o,o.exports,i),o.exports}i.m=r,i.d=(e,t)=>{for(var r in t)i.o(t,r)&&!i.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},i.f={},i.e=e=>Promise.all(Object.keys(i.f).reduce(((t,r)=>(i.f[r](e,t),t)),[])),i.u=e=>(({78:"page_action-aggregate",147:"metrics-aggregate",242:"session-manager",317:"jserrors-aggregate",348:"page_view_timing-aggregate",412:"lazy-feature-loader",439:"async-api",538:"recorder",590:"session_replay-aggregate",675:"compressor",733:"session_trace-aggregate",786:"page_view_event-aggregate",873:"spa-aggregate",898:"ajax-aggregate"}[e]||e)+"."+{78:"ac76d497",147:"3dc53903",148:"1a20d5fe",242:"2a64278a",317:"49e41428",348:"bd6de33a",412:"2f55ce66",439:"30bd804e",538:"1b18459f",590:"cf0efb30",675:"ae9f91a8",733:"83105561",786:"06482edd",860:"03a8b7a5",873:"e6b09d52",898:"998ef92b"}[e]+"-1.236.0.min.js"),i.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),e={},t="NRBA:",i.l=(r,n,o,a)=>{if(e[r])e[r].push(n);else{var s,c;if(void 0!==o)for(var u=document.getElementsByTagName("script"),d=0;d {s.onerror=s.onload=null,clearTimeout(h);var i=e[r];if(delete e[r],s.parentNode&&s.parentNode.removeChild(s),i&&i.forEach((e=>e(n))),t)return t(n)},h=setTimeout(l.bind(null,void 0,{type:"timeout",target:s}),12e4);s.onerror=l.bind(null,s.onerror),s.onload=l.bind(null,s.onload),c&&document.head.appendChild(s)}},i.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.j=364,i.p="https://js-agent.newrelic.com/",(()=>{var e={364:0,953:0};i.f.j=(t,r)=>{var n=i.o(e,t)?e[t]:void 0;if(0!==n)if(n)r.push(n[2]);else{var o=new Promise(((r,i)=>n=e[t]=[r,i]));r.push(n[2]=o);var a=i.p+i.u(t),s=new Error;i.l(a,(r=>{if(i.o(e,t)&&(0!==(n=e[t])&&(e[t]=void 0),n)){var o=r&&("load"===r.type?"missing":r.type),a=r&&r.target&&r.target.src;s.message="Loading chunk "+t+" failed.\n("+o+": "+a+")",s.name="ChunkLoadError",s.type=o,s.request=a,n[1](s)}}),"chunk-"+t,t)}};var t=(t,r)=>{var n,o,[a,s,c]=r,u=0;if(a.some((t=>0!==e[t]))){for(n in s)i.o(s,n)&&(i.m[n]=s[n]);if(c)c(i)}for(t&&t(r);u {i.r(o);var e=i(3325),t=i(5763);const r=Object.values(e.D);function n(e){const n={};return r.forEach((r=>{n[r]=function(e,r){return!1!==(0,t.Mt)(r,"".concat(e,".enabled"))}(r,e)})),n}var a=i(9144);var s=i(5546),c=i(385),u=i(8e3),d=i(5938),f=i(3960),l=i(50);class h extends d.W{constructor(e,t,r){let n=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];super(e,t,r),this.auto=n,this.abortHandler,this.featAggregate,this.onAggregateImported,n&&(0,u.R)(e,r)}importAggregator(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};if(this.featAggregate||!this.auto)return;const r=c.il&&!0===(0,t.Mt)(this.agentIdentifier,"privacy.cookies_enabled");let n;this.onAggregateImported=new Promise((e=>{n=e}));const o=async()=>{let t;try{if(r){const{setupAgentSession:e}=await Promise.all([i.e(860),i.e(242)]).then(i.bind(i,3228));t=e(this.agentIdentifier)}}catch(e){(0,l.Z)("A problem occurred when starting up session manager. This page will not start or extend any session.",e)}try{if(!this.shouldImportAgg(this.featureName,t))return void(0,u.L)(this.agentIdentifier,this.featureName);const{lazyFeatureLoader:r}=await i.e(412).then(i.bind(i,8582)),{Aggregate:o}=await r(this.featureName,"aggregate");this.featAggregate=new o(this.agentIdentifier,this.aggregator,e),n(!0)}catch(e){(0,l.Z)("Downloading and initializing ".concat(this.featureName," failed..."),e),this.abortHandler?.(),n(!1)}};c.il?(0,f.b)((()=>o()),!0):o()}shouldImportAgg(r,n){return r!==e.D.sessionReplay||!1!==(0,t.Mt)(this.agentIdentifier,"session_trace.enabled")&&(!!n?.isNew||!!n?.state.sessionReplay)}}var g=i(7633),p=i(7894);class m extends h{static featureName=g.t9;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];if(super(r,n,g.t9,i),("undefined"==typeof PerformanceNavigationTiming||c.Tt)&&"undefined"!=typeof PerformanceTiming){const n=(0,t.OP)(r);n[g.Dz]=Math.max(Date.now()-n.offset,0),(0,f.K)((()=>n[g.qw]=Math.max((0,p.z)()-n[g.Dz],0))),(0,f.b)((()=>{const t=(0,p.z)();n[g.OJ]=Math.max(t-n[g.Dz],0),(0,s.p)("timing",["load",t],void 0,e.D.pageViewTiming,this.ee)}))}this.importAggregator()}}var v=i(1117),b=i(1284);class y extends v.w{constructor(e){super(e),this.aggregatedData={}}store(e,t,r,n,i){var o=this.getBucket(e,t,r,i);return o.metrics=function(e,t){t||(t={count:0});return t.count+=1,(0,b.D)(e,(function(e,r){t[e]=w(r,t[e])})),t}(n,o.metrics),o}merge(e,t,r,n,i){var o=this.getBucket(e,t,n,i);if(o.metrics){var a=o.metrics;a.count+=r.count,(0,b.D)(r,(function(e,t){if("count"!==e){var n=a[e],i=r[e];i&&!i.c?a[e]=w(i.t,n):a[e]=function(e,t){if(!t)return e;t.c||(t=x(t.t));return t.min=Math.min(e.min,t.min),t.max=Math.max(e.max,t.max),t.t+=e.t,t.sos+=e.sos,t.c+=e.c,t}(i,a[e])}}))}else o.metrics=r}storeMetric(e,t,r,n){var i=this.getBucket(e,t,r);return i.stats=w(n,i.stats),i}getBucket(e,t,r,n){this.aggregatedData[e]||(this.aggregatedData[e]={});var i=this.aggregatedData[e][t];return i||(i=this.aggregatedData[e][t]={params:r||{}},n&&(i.custom=n)),i}get(e,t){return t?this.aggregatedData[e]&&this.aggregatedData[e][t]:this.aggregatedData[e]}take(e){for(var t={},r="",n=!1,i=0;i t.max&&(t.max=e),e 2&&void 0!==arguments[2])||arguments[2];super(e,r,j.t,n),c.il&&((0,t.OP)(e).initHidden=Boolean("hidden"===document.visibilityState),(0,N.N)((()=>(0,s.p)("docHidden",[(0,p.z)()],void 0,j.t,this.ee)),!0),(0,O.bP)("pagehide",(()=>(0,s.p)("winPagehide",[(0,p.z)()],void 0,j.t,this.ee))),this.importAggregator())}}var P=i(3081);class C extends h{static featureName=P.t9;constructor(e,t){let r=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(e,t,P.t9,r),this.importAggregator()}}var R,I=i(2210),k=i(1214),H=i(2177),L={};try{R=localStorage.getItem("__nr_flags").split(","),console&&"function"==typeof console.log&&(L.console=!0,-1!==R.indexOf("dev")&&(L.dev=!0),-1!==R.indexOf("nr_dev")&&(L.nrDev=!0))}catch(e){}function z(e){try{L.console&&z(e)}catch(e){}}L.nrDev&&H.ee.on("internal-error",(function(e){z(e.stack)})),L.dev&&H.ee.on("fn-err",(function(e,t,r){z(r.stack)})),L.dev&&(z("NR AGENT IN DEVELOPMENT MODE"),z("flags: "+(0,b.D)(L,(function(e,t){return e})).join(", ")));var M=i(6660);class B extends h{static featureName=M.t;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(r,n,M.t,i),this.skipNext=0;try{this.removeOnAbort=new AbortController}catch(e){}const o=this;o.ee.on("fn-start",(function(e,t,r){o.abortHandler&&(o.skipNext+=1)})),o.ee.on("fn-err",(function(t,r,n){o.abortHandler&&!n[M.A]&&((0,I.X)(n,M.A,(function(){return!0})),this.thrown=!0,(0,s.p)("err",[n,(0,p.z)()],void 0,e.D.jserrors,o.ee))})),o.ee.on("fn-end",(function(){o.abortHandler&&!this.thrown&&o.skipNext>0&&(o.skipNext-=1)})),o.ee.on("internal-error",(function(t){(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,o.ee)})),this.origOnerror=c._A.onerror,c._A.onerror=this.onerrorHandler.bind(this),c._A.addEventListener("unhandledrejection",(t=>{const r=function(e){let t="Unhandled Promise Rejection: ";if(e instanceof Error)try{return e.message=t+e.message,e}catch(t){return e}if(void 0===e)return new Error(t);try{return new Error(t+(0,D.P)(e))}catch(e){return new Error(t)}}(t.reason);(0,s.p)("err",[r,(0,p.z)(),!1,{unhandledPromiseRejection:1}],void 0,e.D.jserrors,this.ee)}),(0,O.m$)(!1,this.removeOnAbort?.signal)),(0,k.gy)(this.ee),(0,k.BV)(this.ee),(0,k.em)(this.ee),(0,t.OP)(r).xhrWrappable&&(0,k.Kf)(this.ee),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}onerrorHandler(t,r,n,i,o){"function"==typeof this.origOnerror&&this.origOnerror(...arguments);try{this.skipNext?this.skipNext-=1:(0,s.p)("err",[o||new F(t,r,n),(0,p.z)()],void 0,e.D.jserrors,this.ee)}catch(t){try{(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,this.ee)}catch(e){}}return!1}}function F(e,t,r){this.message=e||"Uncaught error with no additional information",this.sourceURL=t,this.line=r}let U=1;const q="nr@id";function G(e){const t=typeof e;return!e||"object"!==t&&"function"!==t?-1:e===c._A?0:(0,I.X)(e,q,(function(){return U++}))}function V(e){if("string"==typeof e&&e.length)return e.length;if("object"==typeof e){if("undefined"!=typeof ArrayBuffer&&e instanceof ArrayBuffer&&e.byteLength)return e.byteLength;if("undefined"!=typeof Blob&&e instanceof Blob&&e.size)return e.size;if(!("undefined"!=typeof FormData&&e instanceof FormData))try{return(0,D.P)(e).length}catch(e){return}}}var X=i(7243);class W{constructor(e){this.agentIdentifier=e,this.generateTracePayload=this.generateTracePayload.bind(this),this.shouldGenerateTrace=this.shouldGenerateTrace.bind(this)}generateTracePayload(e){if(!this.shouldGenerateTrace(e))return null;var r=(0,t.DL)(this.agentIdentifier);if(!r)return null;var n=(r.accountID||"").toString()||null,i=(r.agentID||"").toString()||null,o=(r.trustKey||"").toString()||null;if(!n||!i)return null;var a=(0,_.M)(),s=(0,_.Ht)(),c=Date.now(),u={spanId:a,traceId:s,timestamp:c};return(e.sameOrigin||this.isAllowedOrigin(e)&&this.useTraceContextHeadersForCors())&&(u.traceContextParentHeader=this.generateTraceContextParentHeader(a,s),u.traceContextStateHeader=this.generateTraceContextStateHeader(a,c,n,i,o)),(e.sameOrigin&&!this.excludeNewrelicHeader()||!e.sameOrigin&&this.isAllowedOrigin(e)&&this.useNewrelicHeaderForCors())&&(u.newrelicHeader=this.generateTraceHeader(a,s,c,n,i,o)),u}generateTraceContextParentHeader(e,t){return"00-"+t+"-"+e+"-01"}generateTraceContextStateHeader(e,t,r,n,i){return i+"@nr=0-1-"+r+"-"+n+"-"+e+"----"+t}generateTraceHeader(e,t,r,n,i,o){if(!("function"==typeof c._A?.btoa))return null;var a={v:[0,1],d:{ty:"Browser",ac:n,ap:i,id:e,tr:t,ti:r}};return o&&n!==o&&(a.d.tk=o),btoa((0,D.P)(a))}shouldGenerateTrace(e){return this.isDtEnabled()&&this.isAllowedOrigin(e)}isAllowedOrigin(e){var r=!1,n={};if((0,t.Mt)(this.agentIdentifier,"distributed_tracing")&&(n=(0,t.P_)(this.agentIdentifier).distributed_tracing),e.sameOrigin)r=!0;else if(n.allowed_origins instanceof Array)for(var i=0;i 2&&void 0!==arguments[2])||arguments[2];super(r,n,Z.t,i),(0,t.OP)(r).xhrWrappable&&(this.dt=new W(r),this.handler=(e,t,r,n)=>(0,s.p)(e,t,r,n,this.ee),(0,k.u5)(this.ee),(0,k.Kf)(this.ee),function(r,n,i,o){function a(e){var t=this;t.totalCbs=0,t.called=0,t.cbTime=0,t.end=E,t.ended=!1,t.xhrGuids={},t.lastSize=null,t.loadCaptureCalled=!1,t.params=this.params||{},t.metrics=this.metrics||{},e.addEventListener("load",(function(r){_(t,e)}),(0,O.m$)(!1)),c.IF||e.addEventListener("progress",(function(e){t.lastSize=e.loaded}),(0,O.m$)(!1))}function s(e){this.params={method:e[0]},T(this,e[1]),this.metrics={}}function u(e,n){var i=(0,t.DL)(r);i.xpid&&this.sameOrigin&&n.setRequestHeader("X-NewRelic-ID",i.xpid);var a=o.generateTracePayload(this.parsedOrigin);if(a){var s=!1;a.newrelicHeader&&(n.setRequestHeader("newrelic",a.newrelicHeader),s=!0),a.traceContextParentHeader&&(n.setRequestHeader("traceparent",a.traceContextParentHeader),a.traceContextStateHeader&&n.setRequestHeader("tracestate",a.traceContextStateHeader),s=!0),s&&(this.dt=a)}}function d(e,t){var r=this.metrics,i=e[0],o=this;if(r&&i){var a=V(i);a&&(r.txSize=a)}this.startTime=(0,p.z)(),this.listener=function(e){try{"abort"!==e.type||o.loadCaptureCalled||(o.params.aborted=!0),("load"!==e.type||o.called===o.totalCbs&&(o.onloadCalled||"function"!=typeof t.onload)&&"function"==typeof o.end)&&o.end(t)}catch(e){try{n.emit("internal-error",[e])}catch(e){}}};for(var s=0;s 1?e[1]=i:e.push(i)}else e[0]&&e[0].headers&&s(e[0].headers,n)&&(this.dt=n);function s(e,t){var r=!1;return t.newrelicHeader&&(e.set("newrelic",t.newrelicHeader),r=!0),t.traceContextParentHeader&&(e.set("traceparent",t.traceContextParentHeader),t.traceContextStateHeader&&e.set("tracestate",t.traceContextStateHeader),r=!0),r}}function x(e,t){this.params={},this.metrics={},this.startTime=(0,p.z)(),this.dt=t,e.length>=1&&(this.target=e[0]),e.length>=2&&(this.opts=e[1]);var r,n=this.opts||{},i=this.target;"string"==typeof i?r=i:"object"==typeof i&&i instanceof Y?r=i.url:c._A?.URL&&"object"==typeof i&&i instanceof URL&&(r=i.href),T(this,r);var o=(""+(i&&i instanceof Y&&i.method||n.method||"GET")).toUpperCase();this.params.method=o,this.txSize=V(n.body)||0}function A(t,r){var n;this.endTime=(0,p.z)(),this.params||(this.params={}),this.params.status=r?r.status:0,"string"==typeof this.rxSize&&this.rxSize.length>0&&(n=+this.rxSize);var o={txSize:this.txSize,rxSize:n,duration:(0,p.z)()-this.startTime};i("xhr",[this.params,o,this.startTime,this.endTime,"fetch"],this,e.D.ajax)}function E(t){var r=this.params,n=this.metrics;if(!this.ended){this.ended=!0;for(var o=0;o 2&&void 0!==arguments[2])||arguments[2];super(e,t,we.t,r),this.importAggregator()}}new class{constructor(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:(0,_.ky)(16);c._A?(this.agentIdentifier=t,this.sharedAggregator=new y({agentIdentifier:this.agentIdentifier}),this.features={},this.desiredFeatures=new Set(e.features||[]),this.desiredFeatures.add(m),Object.assign(this,(0,a.j)(this.agentIdentifier,e,e.loaderType||"agent")),this.start()):(0,l.Z)("Failed to initial the agent. Could not determine the runtime environment.")}get config(){return{info:(0,t.C5)(this.agentIdentifier),init:(0,t.P_)(this.agentIdentifier),loader_config:(0,t.DL)(this.agentIdentifier),runtime:(0,t.OP)(this.agentIdentifier)}}start(){const t="features";try{const r=n(this.agentIdentifier),i=[...this.desiredFeatures];i.sort(((t,r)=>e.p[t.featureName]-e.p[r.featureName])),i.forEach((t=>{if(r[t.featureName]||t.featureName===e.D.pageViewEvent){const n=function(t){switch(t){case e.D.ajax:return[e.D.jserrors];case e.D.sessionTrace:return[e.D.ajax,e.D.pageViewEvent];case e.D.sessionReplay:return[e.D.sessionTrace];case e.D.pageViewTiming:return[e.D.pageViewEvent];default:return[]}}(t.featureName);n.every((e=>r[e]))||(0,l.Z)("".concat(t.featureName," is enabled but one or more dependent features has been disabled (").concat((0,D.P)(n),"). This may cause unintended consequences or missing data...")),this.features[t.featureName]=new t(this.agentIdentifier,this.sharedAggregator)}})),(0,T.Qy)(this.agentIdentifier,this.features,t)}catch(e){(0,l.Z)("Failed to initialize all enabled instrument classes (agent aborted) -",e);for(const e in this.features)this.features[e].abortHandler?.();const r=(0,T.fP)();return delete r.initializedAgents[this.agentIdentifier]?.api,delete r.initializedAgents[this.agentIdentifier]?.[t],delete this.sharedAggregator,r.ee?.abort(),delete r.ee?.get(this.agentIdentifier),!1}}}({features:[J,m,S,class extends h{static featureName=oe;constructor(t,r){if(super(t,r,oe,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;const n=this.ee;let i;(0,k.QU)(n),this.eventsEE=(0,k.em)(n),this.eventsEE.on(se,(function(e,t){this.bstStart=(0,p.z)()})),this.eventsEE.on(ae,(function(t,r){(0,s.p)("bst",[t[0],r,this.bstStart,(0,p.z)()],void 0,e.D.sessionTrace,n)})),n.on(ce+ne,(function(e){this.time=(0,p.z)(),this.startPath=location.pathname+location.hash})),n.on(ce+ie,(function(t){(0,s.p)("bstHist",[location.pathname+location.hash,this.startPath,this.time],void 0,e.D.sessionTrace,n)}));try{i=new PerformanceObserver((t=>{const r=t.getEntries();(0,s.p)(te,[r],void 0,e.D.sessionTrace,n)})),i.observe({type:re,buffered:!0})}catch(e){}this.importAggregator({resourceObserver:i})}},C,xe,B,class extends h{static featureName=de;constructor(e,r){if(super(e,r,de,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;if(!(0,t.OP)(e).xhrWrappable)return;try{this.removeOnAbort=new AbortController}catch(e){}let n,i=0;const o=this.ee.get("tracer"),a=(0,k._L)(this.ee),s=(0,k.Lg)(this.ee),u=(0,k.BV)(this.ee),d=(0,k.Kf)(this.ee),f=this.ee.get("events"),l=(0,k.u5)(this.ee),h=(0,k.QU)(this.ee),g=(0,k.Gm)(this.ee);function m(e,t){h.emit("newURL",[""+window.location,t])}function v(){i++,n=window.location.hash,this[ve]=(0,p.z)()}function b(){i--,window.location.hash!==n&&m(0,!0);var e=(0,p.z)();this[pe]=~~this[pe]+e-this[ve],this[ye]=e}function y(e,t){e.on(t,(function(){this[t]=(0,p.z)()}))}this.ee.on(ve,v),s.on(be,v),a.on(be,v),this.ee.on(ye,b),s.on(ge,b),a.on(ge,b),this.ee.buffer([ve,ye,"xhr-resolved"],this.featureName),f.buffer([ve],this.featureName),u.buffer(["setTimeout"+le,"clearTimeout"+fe,ve],this.featureName),d.buffer([ve,"new-xhr","send-xhr"+fe],this.featureName),l.buffer([me+fe,me+"-done",me+he+fe,me+he+le],this.featureName),h.buffer(["newURL"],this.featureName),g.buffer([ve],this.featureName),s.buffer(["propagate",be,ge,"executor-err","resolve"+fe],this.featureName),o.buffer([ve,"no-"+ve],this.featureName),a.buffer(["new-jsonp","cb-start","jsonp-error","jsonp-end"],this.featureName),y(l,me+fe),y(l,me+"-done"),y(a,"new-jsonp"),y(a,"jsonp-end"),y(a,"cb-start"),h.on("pushState-end",m),h.on("replaceState-end",m),window.addEventListener("hashchange",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("load",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("popstate",(function(){m(0,i>1)}),(0,O.m$)(!0,this.removeOnAbort?.signal)),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}}],loaderType:"spa"})})(),window.NRBA=o})(); window.jQuery || document.write(' ') CKEDITOR_BASEPATH='https://f1000research.com/js/vendor/ckeditor/' window.reactTheme = 'research'; window.MathJax = { CommonHTML: { linebreaks: { automatic: true } }, 'HTML-CSS': { linebreaks: { automatic: true } }, SVG: { linebreaks: { automatic: true } }, AuthorInit: function() { MathJax.Hub.Register.MessageHook('End Process', function () { let timeout = false; // holder for timeout id const delay = 250; // delay after event is "complete" to run callback const reflowMath = function() { const dispFormulas = document.querySelectorAll('.disp-formula.panel'); if (!dispFormulas) { return; } for (const dispFormula of dispFormulas) { const child = dispFormula.querySelector('.MathJax_Preview').nextSibling.firstChild; const isMultiline = MathJax.Hub.getAllJax(dispFormula)[0].root.isMultiline; if (dispFormula.offsetWidth < child.offsetWidth || isMultiline) { MathJax.Hub.Queue(['Rerender', MathJax.Hub, dispFormula]); } } }; window.addEventListener('resize', function() { clearTimeout(timeout); // clear the timeout timeout = setTimeout(reflowMath, delay); // start timing for event "completion" }); }); }, }; if (window.location.hash == '#_=_'){ window.location = window.location.href.split('#')[0] } !function(f,b,e,v,n,t,s){if(f.fbq)return;n=f.fbq=function() {n.callMethod? n.callMethod.apply(n,arguments):n.queue.push(arguments)} ;if(!f._fbq)f._fbq=n; n.push=n;n.loaded=!0;n.version='2.0';n.queue=[];t=b.createElement(e);t.async=!0; t.src=v;s=b.getElementsByTagName(e)[0];s.parentNode.insertBefore(t,s)}(window, document,'script','https://connect.facebook.net/en_US/fbevents.js'); fbq('init', '1641728616063202'); fbq('track', "PixelInitialized", {}); (function(h,o,t,j,a,r){ h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)}; h._hjSettings={hjid:2318163,hjsv:6}; a=o.getElementsByTagName('head')[0]; r=o.createElement('script');r.async=1; r.src=t+h._hjSettings.hjid+j+h._hjSettings.hjsv; a.appendChild(r); })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv='); search file_upload Submit your research search menu close search Browse Gateways & Collections How to Publish Submit your Research My Submissions Article Guidelines Article Guidelines (New Versions) Open Data, Software and Code Guidelines Open Data and Accessible Source Materials Guidelines (HSS) Open Data, Software and Code Guidelines (PSE) Prepublication Checks Production Process Posters and Slides Guidelines Document Guidelines Article Processing Charges Peer Review Finding Article Reviewers About How it Works For Reviewers Our Advisors Policies Glossary FAQs For Developers Newsroom Contact My Research Submissions Content and Tracking Alerts My Details Sign In file_upload Submit your research { "@context": "https://schema.org", "@type": "ScholarlyArticle", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://f1000research.com/articles/11-107" }, "headline": "Does the impact of medical publications vary by disease indication and publication type? An exploration...", "datePublished": "2022-01-27T16:41:18", "dateModified": "2024-10-30T12:10:57", "author": [ { "@type": "Person", "name": "Tomas Rees" }, { "@type": "Person", "name": "Avishek Pal" } ], "publisher": { "@type": "Organization", "name": "F1000Research", "logo": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 480, "width": 60 } }, "image": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 1200, "width": 150 }, "description": " Background The EMPIRE (EMpirical Publication Impact and Reach Evaluation) Index is a value-based, multi-component metric framework to assess the impact of medical publications in terms of relevance to different stakeholders. It comprises three component scores (social, scholarly and societal impact), each incorporating related altmetrics that indicate a different aspect of engagement with the publication. Here, we present an exploratory investigation of whether publication types or disease indications influence EMPIRE Index scores. Methods Article-level metrics were extracted and EMPIRE Index scores were calculated for 5825 journal articles published from 1 May 2017 to 1 May 2018, representing 12 disease indications (chosen to reflect a wide variety of common and rare diseases with a variety of aetiologies) and five publication types. Results There were significant differences in scores between article types and disease indications. Median (95% CI) social and scholarly impact scores ranged from 1.2 (0.3–1.6) to 4.8 (3.1–6.6), respectively, for phase 3 clinical trials, and from 0.3 (0.3–0.4) to 2.3 (1.9–2.6), respectively, for observational studies. Social and scholarly impact scores were highest for multiple sclerosis publications and lowest for non-small cell lung cancer publications. Systematic reviews achieved greater impact than regular reviews. Median trends in the social impact of different disease areas matched the level of public interest as assessed through Google search interest. Although most articles did not register societal impact, mean societal impact scores were highest for migraine publications. Conclusions The EMPIRE Index successfully identified differences in impact by disease area and publication type, which supports the notion that the impact of each publication needs to be evaluated in the context of these factors, and potentially others. These findings should be considered when using the EMPIRE Index to assess publication impact. " } { "@context": "http://schema.org", "@type": "BreadcrumbList", "itemListElement": [ { "@type": "ListItem", "position": "1", "item": { "@id": "https://f1000research.com/", "name": "Home" } }, { "@type": "ListItem", "position": "2", "item": { "@id": "https://f1000research.com/browse/articles", "name": "Browse" } }, { "@type": "ListItem", "position": "3", "item": { "@id": "https://f1000research.com/articles/11-107/v5", "name": "Does the impact of medical publications vary by disease indication..." } } ] } Home Browse Does the impact of medical publications vary by disease indication... ALL Metrics - Views Downloads Get PDF Get XML Cite How to cite this article Rees T and Pal A. Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.12688/f1000research.75805.5 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. Close Copy Citation Details Export Export Citation Sciwheel EndNote Ref. Manager Bibtex ProCite Sente EXPORT Select a format first Track Share ▬ ✚ Research Article Revised Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] Tomas Rees https://orcid.org/0000-0003-0221-0098 1 , Avishek Pal https://orcid.org/0000-0002-2213-3336 2 Tomas Rees https://orcid.org/0000-0003-0221-0098 1 , Avishek Pal https://orcid.org/0000-0002-2213-3336 2 PUBLISHED 30 Oct 2024 Author details Author details 1 Oxford PharmaGenesis, Oxford, Oxfordshire, OX13 5QJ, UK 2 Novartis Pharma AG, Basel, Switzerland Tomas Rees Roles: Conceptualization, Data Curation, Formal Analysis, Funding Acquisition, Investigation, Methodology, Project Administration, Resources, Validation, Visualization, Writing – Original Draft Preparation, Writing – Review & Editing Avishek Pal Roles: Conceptualization, Funding Acquisition, Methodology, Resources, Supervision, Writing – Review & Editing OPEN PEER REVIEW DETAILS REVIEWER STATUS This article is included in the Research on Research, Policy & Culture gateway. This article is included in the Interactive Figures collection. Abstract Background The EMPIRE (EMpirical Publication Impact and Reach Evaluation) Index is a value-based, multi-component metric framework to assess the impact of medical publications in terms of relevance to different stakeholders. It comprises three component scores (social, scholarly and societal impact), each incorporating related altmetrics that indicate a different aspect of engagement with the publication. Here, we present an exploratory investigation of whether publication types or disease indications influence EMPIRE Index scores. Methods Article-level metrics were extracted and EMPIRE Index scores were calculated for 5825 journal articles published from 1 May 2017 to 1 May 2018, representing 12 disease indications (chosen to reflect a wide variety of common and rare diseases with a variety of aetiologies) and five publication types. Results There were significant differences in scores between article types and disease indications. Median (95% CI) social and scholarly impact scores ranged from 1.2 (0.3–1.6) to 4.8 (3.1–6.6), respectively, for phase 3 clinical trials, and from 0.3 (0.3–0.4) to 2.3 (1.9–2.6), respectively, for observational studies. Social and scholarly impact scores were highest for multiple sclerosis publications and lowest for non-small cell lung cancer publications. Systematic reviews achieved greater impact than regular reviews. Median trends in the social impact of different disease areas matched the level of public interest as assessed through Google search interest. Although most articles did not register societal impact, mean societal impact scores were highest for migraine publications. Conclusions The EMPIRE Index successfully identified differences in impact by disease area and publication type, which supports the notion that the impact of each publication needs to be evaluated in the context of these factors, and potentially others. These findings should be considered when using the EMPIRE Index to assess publication impact. READ ALL READ LESS Keywords Altmetrics, bibliometrics, publication impact Corresponding Author(s) Tomas Rees ( [email protected] ) Close Corresponding author: Tomas Rees Competing interests: Tomas Rees is an employee of Oxford PharmaGenesis and received funding from Novartis Pharma AG for this work. Avishek Pal is an employee of Novartis Pharma AG. Grant information: Oxford PharmaGenesis was contracted by Novartis Pharma AG (https://www.novartis.com) to provide research and editorial support (manuscript proofreading, figure drawing, and project management). The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript. Copyright: © 2024 Rees T and Pal A. This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. How to cite: Rees T and Pal A. Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.12688/f1000research.75805.5 ) First published: 27 Jan 2022, 11 :107 ( https://doi.org/10.12688/f1000research.75805.1 ) Latest published: 30 Oct 2024, 11 :107 ( https://doi.org/10.12688/f1000research.75805.5 ) Revised Amendments from Version 4 Additional statistical analyses have been conducted. These are described in detail in the supplementary material. Additional statistical analyses have been conducted. These are described in detail in the supplementary material. See the authors' detailed response to the review by Shir Aviv-Reuven See the authors' detailed response to the review by José Luis Ortega See the authors' detailed response to the review by Arman Yurisaldi Saleh READ REVIEWER RESPONSES Introduction Article-level measures of publication impact (ALMs), including alternative metrics, play a crucial role in providing a more comprehensive understanding of the impact of research publications compared with traditional bibliometrics like the journal impact factor (JIF) and CiteScore. While JIF and similar metrics are useful for identifying journals with high readership, they fall short in assessing the quality and impact of individual research articles 1 – 3 . This limitation arises from the fact that JIF primarily measures the average number of citations to articles in a journal, which may not reflect the significance or reach of a specific article within that journal 4 . ALMs offer a more nuanced view by considering various factors beyond citations, such as social media mentions, downloads, and online discussions, providing insights into the broader impact of research across different audiences and contexts 2 . To date, most uses of alternative metrics to assess medical publications have looked at their correlation with conventional metrics, especially citations. For example the correlation between alternative and traditional bibliometrics has been examined in publications in total joint arthroplasty research 5 , neurology 6 , COVID-19 7 , critical care medicine 8 and medical professionalism 9 . Other uses of alternative metrics include the creation of an ‘altmetric impact factor’ for medical journals 10 . They have also been used to identify trending topics in breast cancer 11 , to explore the impact of multidisciplinary research groups 12 , and to assess the convergence of altmetrics and societal impact assessments using data from the UK Research Excellence Framework 13 . These studies have used either individual ALMs (Tweets, Mendeley saves, News mentions etc) or the Altmetrics Attention Score (AAS), an aggregated, weighted score that unifies multiple ALMs into a single metric. The AAS provides a convenient way of providing an summary of multiple article level metrics, but it excludes some key metrics (Mendeley saves and citations). The methodology by which it is calculated is not fully transparent and it is not open to adaptation by the users depending on their intended use. We have previously described a novel and holistic approach to summarising ALMs, the EMPIRE (EMpirical Publication Impact and Reach Evaluation) Index, which uses weighted article-level metrics to assess the impact of medical publications in terms relevance to different stakeholders 14 . The EMPIRE Index provides component scores for scholarly, social and societal impact, as well as a total impact score and predictive reach metrics ( Figure 1 ). The scholarly component correlates weakly with CiteScore while the social score correlates closely with the AAS. The societal component, comprised of citations in guidelines, policy documents and patents, represents a distinct score. Figure 1. Example of the EMPIRE Index score for a single publication 14 . HCP, healthcare provider; NEJM, New England Journal of Medicine . The EMPIRE Index’s ability to correlate scholarly components with CiteScore and social components with AAS showcases its alignment with established metrics while providing a more nuanced evaluation of impact 14 . This nuanced evaluation is particularly valuable in the medical field, where research impact transcends academic citations to societal influence and practical applications in healthcare settings. For researchers, the EMPIRE Index provides a detailed evaluation of publication value by offering component scores for scholarly, social, and societal impact, in addition to total impact scores and predictive reach metrics 14 . This multifaceted assessment allows researchers to gain insights into the broader impact of their work beyond traditional citation-based metrics, enabling them to comprehend the reach and significance of their research across various dimensions. Policymakers can derive value from the EMPIRE Index as it includes a societal impact component that takes into account citations in guidelines, policy documents, and patents 14 . This feature is crucial for policymakers as it offers valuable insights into how research publications influence policy decisions and contribute to the development of guidelines and patents, assisting policymakers in making informed decisions based on the societal relevance and impact of research findings. Healthcare providers can utilize the EMPIRE Index to evaluate the impact of medical publications in terms of societal implications and reach, allowing them to better grasp the practical implications of research in guiding clinical practice, policy development, and healthcare innovations 14 . By considering societal impact metrics, healthcare providers can improve patient care and outcomes by staying abreast of impactful research in their field. The EMPIRE Index uses randomized controlled clinical trials published in the NEJM as benchmark, which studies that are selected by the journal editors as likely to be of the highest impact for the practice of medicine. However, this standard cannot be uniformly applied to all types of studies, across different disease areas and development stages of a therapy. It is widely recognised that publication metrics vary by discipline, and to facilitate the comparison of publication impact across different fields, field-normalised citation impacts are frequently calculated 15 . Although evidence is limited, some research suggests that ALMs can also vary by publication type. For example, review articles in pharmacology journals can receive twice as many citations as original articles 16 and differences between quantitative and qualitative articles published in BMJ have been found for altmetric scores, but not citations 17 . Therefore, the nuances of each field and publication type should be considered in order to understand what constitutes a typical score. This additional context is an important consideration as it allows users to compare scores with a relevant frame of reference and so to accurately interpret and utilise publication metrics and to derive meaningful insights. It is likely that EMPIRE Index scores vary by disease area and by publication type. However, the scale and nature of these variations are unknown, which complicates efforts to compare scores of individual publications. This in turn limits the utility of the scale to identify ‘high impact’ publications, since what counts as an atypically high score will vary across therapy areas publication types. In this brief report we present an exploratory investigation of the interaction between disease indications and publication types and typical EMPIRE Index scores. We sought to explore the variations that may exist in typical (i.e. average and median) EMPIRE Index scores across different therapy areas and publication types, and to explore the magnitude of these variations. This will allow the metrics of individual publications to be placed in a richer context of potentially comparable publications to facilitate interpretation of individual publication scores. We also sought to provide additional context for the observed typical scores in two ways. Citescore is a journal-level metric that is related to the average number of citations per paper in that journal 18 and is an indicator of journal prominence in the academic community. Journal Citescore is modestly correlated with publication EMPIRE Index Scores, especially the Scholarly Impact components 14 . We therefore compared mean EMPIRE Index scores across therapy areas and publication types with the mean Citescores of the journals in which the papers were published, to see if journal prominence could explain any variations seen. Google search trends indicate the level search engine activity around a topic and have been used as a measure of the prominence of health topics among the general public 19 . We therefore compared EMPIRE scores with search trends for each of the therapy areas we examined, to see if public interest in a therapy area was related to engagement with publications measured via article-level metrics, especially with regard to the EMPIRE Index Social Impact component. Methods This exploratory study investigated 12 disease indications, purposefully chosen to reflect a variety of common and rare diseases with a variety of aetiologies. Six of these were rare diseases, selected as a convenience sample of disease indications with which the authors were most familiar. No formal statistical power analysis was undertaken. However, we aimed for disease samples from approximately 1000 publications, which would enable publication type sub-analyses. The six rare disease samples were, therefore, pooled. Relevant publications were identified for each disease by the appearance of the disease name in the publication title. We limited the search period to items with publication dates between 1 May 2017 and 1 May 2018, to give sufficient time for metrics to accumulate while also minimising the time-dependent variation in metrics. The searches were conducted on PubMed between 22 June 2020 and 3 July 2020, using the following search string: "(2017/05/01"[Date - Publication] : "2018/05/01"[Date - Publication]) AND ""[TI] We chose five publication types that are commonly used in medical research and communication: clinical trials, phase 3 clinical trials, observational studies, reviews, and systematic reviews. Definitions of each publication type are provided by US National Library of Medicine 20 . Clinical trials are experimental studies that test the efficacy and safety of new interventions or compare different interventions in human participants. Phase 3 clinical trials are the last stage of clinical research of new drug treatments and are typically large scale and provide evidence intended to guide treatment practice. Observational studies are non-experimental studies that examine the associations between exposures and outcomes in natural settings, without manipulating any variables. Reviews are articles that summarize and synthesize the existing literature on a specific topic, providing an overview of the current state of knowledge and identifying research gaps. Systematic reviews are a type of review that follow a rigorous and predefined methodology to search, select, appraise, and synthesize the evidence from multiple primary studies on a specific question. For each disease, we conducted secondary searches for each publication type using PubMed tags for those of interest (i.e. the search string above and either “review”, "systematic review", "clinical trial, phase iii", “clinical trial” or "observational study"). PubMed publication types are metadata supplied by PubMed and derive originally from publisher submissions. PubMed IDs were entered into the Altmetric Explorer and PlumX dashboards and ALMs downloaded over the period 23 June 2020 to 11 July 2020. ALMs were assumed to be zero for any publication for which Altmetric Explorer did not return a result. We also obtained the journal CiteScore for all publications 18 . EMPIRE Index scores were calculated for all publications as described previously 14 . Briefly, selected ALMs that compose the EMPIRE Index were weighted and aggregated to form three component scores (social impact, scholarly impact and societal impact), which were then summed to form a total impact score. Each disease area comprised a different mixture of publication types, which we expected could confound the analysis; multivariate analysis on such a heterogenous, non-normal and zero-inflated data set is problematic. Therefore, we opted to create standardised samples through random polling. A sample was created for each disease area with a standardised mix of publication types chosen to maximise the total number of publications retained (the standardised publication types [SPT] set). First, the two least common publication types (phase 3 clinical trials and systematic reviews) were excluded because of the high variation between disease areas and because they are largely subsets of other publication types (clinical trials and reviews, respectively). Although the observational studies publication type was only slightly more common than systematic reviews, it was retained as it was considered to be functionally very different from clinical trials and reviews. The proportions of each of the remaining three publication types were calculated for each disease set, as well as for the overall set. Publications were then trimmed from each disease set by random sampling, as needed, to match the proportions in the overall set. The trimmed publication sets formed the SPT set. Similarly, each publication type comprised a different mix of diseases. A standardised disease areas (SDA) set was created by random sampling using a similar approach that ensured each publication type included the same mix of diseases, while maximising the total number of publications retained. We downloaded weekly Google Trends data on relative interest over time for the period of interest for these diseases (May 1 2017 to May 1 2018). Google Trends measures the relative frequency of searches for a given term on Google, normalized by the total number of searches in a given time and region. A score of 100 indicates the maximum interest in any week over the search period. We used each therapy area as the target term (e.g. asthma, multiple sclerosis) adding each to a comparative dataset so that the interest in each term is expressed relative to the maximum interest across any of the search terms of interest. The year averages presented here are expressed relative to that maximum score. As these analyses were exploratory and primarily intended to assess the magnitude of impact scores observed for different publication types and disease areas, we primarily provide descriptive statistics, supported by some statistical analysis of comparative differences. Intra-group differences were assessed using Kruskal-Wallis one-way analysis of variance, a non-parametric test for equality of population means (a significant result indicates that that at least one population median of one group is different from the population median of at least one other group). Exploratory analyses were conducted using a two-part modelling approach, chosen to account for the zero-inflated and skewed nature of our impact measures. The model comprised a logistic regression to predict the probability of a publication having any impact (non-zero impact), and a generalized linear model (GLM) with a log-link function to analyse the magnitude of impact for publications with non-zero impact. Both components of the model included disease area, publication were type, and CiteScore as predictor. Asthma and clinical trials chosen arbitrarily as the reference categories for disease area and publication type, respectively. To assess the relationship between public interest in a disease area and publication impact, we conducted correlation analyses at the disease area level. We calculated both simple correlations and partial correlations controlling for average CiteScore, to account for potential confounding effects of publication quality across disease areas. All analyses were performed using Python (version 3.10.5) with the statsmodels library (version 0.14.4). Detailed methodology and code are provided in the supplementary materials 21 . Results Sample characteristics In total, 20 577 publications were identified across the 12 disease areas 21 , of which 5825 (28%) were tagged with one of the publication types of interest ( Table 1 ). Table 1 also shows the Google search interest for each of these diseases. Table 1. Numbers of publications identified in the search. Google search interest is the average weekly interest across the search period and is a relative score 0–100 where 100 is the maximum score for any disease in any individual week. All publications Clinical trial Phase 3 clinical trial Observational study Review Systematic review Any identifiable publication type Google search interest n n n n n n n Asthma 3487 225 6% 21 1% 114 3% 567 16% 100 3% 1027 29% 62 Migraine 996 88 9% 6 1% 36 4% 177 18% 29 3% 336 34% 88 MS 2986 175 6% 10 0% 80 3% 548 18% 78 3% 891 30% 72 NSCLC 3847 287 7% 57 1% 42 1% 417 11% 36 1% 839 22% 3 Psoriasis 1654 139 8% 53 3% 57 3% 281 17% 54 3% 584 35% 42 T2D 5341 644 12% 66 1% 213 4% 674 13% 187 4% 1784 33% 45 Rare diseases 2266 82 4% 18 1% 11 0% 243 11% 10 0% 364 16% 3 DLBLC 597 41 7% 10 2% 5 1% 45 8% 5 1% 106 18% 2 NASH 351 9 3% 0 0% 1 0% 64 18% 1 0% 75 21% 6 NET 187 3 2% 1 1% 1 1% 20 11% 1 1% 26 14% 3 SMA 196 8 4% 2 1% 0 0% 42 21% 0 0% 52 27% 4 TNBC 779 18 2% 4 1% 2 0% 60 8% 3 0% 87 11% 1 TSC 156 3 2% 1 1% 2 1% 12 8% 0 0% 18 12% 3 Total 20 577 1640 8% 231 1% 553 3% 2907 14% 494 2% 5825 28% DLBLC, diffuse large B-cell lymphoma; MS, multiple sclerosis; NASH, non-alcoholic steatohepatitis; NET, neuroendocrine tumour; NSCLC, non-small cell lung cancer; SMA, spinal muscular atrophy; T2D, type 2 diabetes; TNBC, triple-negative breast cancer; TSC, tuberous sclerosis complex. Analysis by publication type The numbers of publications retained in the SDA set used for publication type comparisons (i.e. with the same disease indication composition for each publication type) are shown in Table 2 . Table 2. Numbers of publications retained in the SDA set used for publication type comparisons. Clinical trial Phase 3 clinical trial Observational study Review Systematic review Proportion of total n n n n n % Asthma 197 11 51 399 44 19% Migraine 66 4 17 133 15 6% MS 175 10 45 354 39 17% NSCLC 163 9 42 328 36 16% Psoriasis 104 6 27 210 23 10% T2D 334 19 86 674 74 32% Total 1039 59 268 2098 231 100% MS, multiple sclerosis; NSCLC, non-small cell lung cancer; T2D, type 2 diabetes. Median EMPIRE Index scores and CiteScores for each disease in the SDA set are shown in Figure 2 and Table 3 . Mean EMPIRE Index scores, shown in Figure 3 , broadly reflect the median scores. Statistical analysis indicated that there was some significant variation in the medians of each component as well as the total impact score and journal CiteScore. In general, the ranking of publication type is relatively consistent across different types of impact. Notably, phase 3 clinical trials had the highest median and mean scores, while observational studies had the lowest. Systematic reviews had higher impact than reviews. Most articles across all publication types had no societal impact, and significant differences in societal impact were driven by outliers. Of note, eight of the ten publications with the highest societal impact were clinical trials, and six of those were in non-small cell lung cancer (NSCLC). Figure 2. Median (95% CI) total impact scores for each publication type (standardised set). CI, confidence interval. Table 3. Median and maximum scores for each EMPIRE Index component and CiteScore by publication type (SDA set). Social Scholarly Societal Total CiteScore n Median 95% CI Max Median 95% CI Max Median 95% CI Max Median 95% CI Max Median 95% CI Max Clinical trial 1039 0.4 0.4–0.6 411.9 3.0 2.8–3.2 303.8 0.0 0.0–0.0 267.0 1.4 1.2–1.5 274.4 2.9 2.9–2.9 19.1 Phase 3 clinical trial 59 1.2 0.3–1.6 222.4 4.8 3.1–6.6 68.3 0.0 0.0–0.0 192.8 2.3 1.4–3.2 156.9 3.2 2.9–4.9 16.1 Observational study 268 0.3 0.3–0.4 27.9 2.3 1.9–2.6 38.7 0.0 0.0–0.0 89.0 1.0 0.8–1.2 32.5 2.5 2.3–2.6 10.5 Review 2098 0.4 0.3–0.4 176.5 3.3 3.1–3.5 183.1 0.0 0.0–0.0 178.0 1.4 1.3–1.5 106.6 2.9 2.7–2.9 23.2 Systematic review 231 0.7 0.6–0.9 176.5 4.1 3.6–4.4 53.6 0.0 0.0–0.0 178.0 1.9 1.6–2.1 71.5 2.9 2.7–3.0 8.7 p value < 0.0001 < 0.0001 < 0.0001 < 0.0001 < 0.0001 CI, confidence interval. Figure 3. Mean EMPIRE Index scores for each publication type (SDA set). The interactive version (online only, accessible here: https://s3.eu-west-2.amazonaws.com/ox.em/webflow/p29ieu21/chart1.html ) also shows mean EMPIRE Index scores for each disease by publication type (full set). Results from the mixed model and regression analysis are presented in the supplemental material 21 . Compared with non-phase 3 clinical trials, systematic reviews and phase 3 publications had higher social, scholarly, societal and total impact, although this was not consistent across logistic regression and linear models. In the logistic regression, systematic reviews consistently showed higher odds of having any impact across all impact types. Phase III clinical trials showed higher impact in the GLM analysis for scholarly and total impact, but this wasn't consistent across other impact types. In the GLM, observational studies had lower social, scholarly and total impact. Analysis by disease indication The numbers of publications retained in the SPT set used for disease comparisons (i.e. with the same publication type composition for each disease indication) are shown in Table 4 . Table 4. Numbers of publications retained in the SPT set used for disease comparisons. Asthma Migraine MS NSCLC Psoriasis Rare disease T2D Proportion of total n n n n n n n % Clinical trial 225 88 175 287 139 32 394 33% Observational study 78 31 61 42 48 11 137 11% Review 385 150 299 417 238 54 674 56% Total 688 269 535 746 425 97 1205 100% MS, multiple sclerosis; NSCLC, non-small cell lung cancer; SPT, standardised publication types; T2D, type 2 diabetes. Median EMPIRE Index scores and journal CiteScores for each disease in the SPT set are shown in Figure 4 and Table 5 . Kruskall–Wallis testing indicated at least one significant pairwise difference in the total scores, each component score and journal CiteScore. Migraine and multiple sclerosis (MS) had the highest impact across social and scholarly component scores as well as the total impact score, while NSCLC and psoriasis had the lowest. Most articles across all diseases had no societal impact, with significant differences in societal impact driven by outliers. The eight publications with the highest societal impact were all important clinical outcomes trials (three in type 2 diabetes, three in NSCLC and one each in migraine and asthma). Figure 4. Median (95% CI) total impact scores for each disease (standardised set). MS, multiple sclerosis; NSCLC, non-small cell lung cancer; T2D, type 2 diabetes. Table 5. Median and maximum scores for each EMPIRE Index component and CiteScore by disease (SPT set). Social Scholarly Societal Total CiteScore n Median 95% CI Max Median 95% CI Max Median 95% CI Max Median 95% CI Max Median 95% CI Max Asthma 688 0.6 0.4–0.7 122.5 3.0 2.8–3.2 70.5 0.0 0.0–0.0 178.0 1.3 1.2–1.5 88.0 2.9 2.7–3.0 16.1 Migraine 269 0.6 0.4–0.9 222.4 3.3 2.8–3.9 68.3 0.0 0.0–0.0 192.8 1.7 1.3–2.1 156.9 2.9 2.4–2.9 16.1 MS 535 0.9 0.7–1.0 142.1 3.8 3.4–4.2 267.4 0.0 0.0–0.0 133.5 1.8 1.6–1.9 150.0 2.8 2.6–2.9 16.1 NSCLC 369 0.1 0.1–0.1 256.1 2.4 1.9–2.8 259.7 0.0 0.0–0.0 267.0 0.9 0.8–1.1 186.7 3.2 2.9–3.6 23.2 Psoriasis 425 0.1 0.1–0.3 60.7 2.4 2.0–2.8 85.8 0.0 0.0–0.0 133.5 1.0 0.8–1.1 51.3 2.4 2.2–2.4 10.3 T2D 1205 0.4 0.3–0.4 833.4 3.4 3.1–3.7 485.5 0.0 0.0–0.0 548.8 1.5 1.4–1.6 532.5 2.8 2.8–3.0 19.1 Rare diseases 97 0.3 0.1–0.6 236.1 3.2 2.2–4.2 143.4 0.0 0.0–0.0 89.0 1.3 0.9–1.9 136.4 3.3 2.5–3.7 16.1 p value < 0.0001 < 0.0001 0.0002 < 0.0001 < 0.0001 CI, confidence interval; MS, multiple sclerosis; NSCLC, non-small cell lung cancer; T2D, type 2 diabetes. Mean EMPIRE Index scores for each disease in the SPT set are shown in Figure 5 . The interactive version of Figure 5 (online publication only) also shows the mean EMPIRE Index scores by disease for each publication type (full data set). Mean scores do not show clear trends for differences between disease indications, although societal impact appears to be lower for asthma and MS, and higher for migraine than other diseases. The high societal impact for migraine was driven by review articles; 16 of the 23 migraine articles with societal impact scores above zero were review articles. The scholarly impact for rare diseases appears to be higher than for other disease areas, albeit with low confidence owing to small numbers of publications included. Figure 5. Mean EMPIRE Index scores for each disease (SPT set). The interactive version (online only, accessible here: https://s3.eu-west-2.amazonaws.com/ox.em/webflow/p29ieu21/chart2.html ) also shows mean EMPIRE Index scores for each disease by publication type (full set). MS, multiple sclerosis; NSCLC, non-small cell lung cancer; T2D, type 2 diabetes. Results from the mixed model and regression analysis are presented in the supplemental material 21 . Compared with the reference indication (asthma), other disease areas tended to have a lower probability of any total impact but, among publications with at least some impact, the average impact was higher with other diseases compared with asthma. This pattern was mostly consistent across the impact subscales (social, scholarly and societal). Google search interest (public interest) did not correlate with any of the impact scores in the simple correlation. However, adjusted for CiteScore, the correlation between social impact and public interest was moderately strong and marginally significant (r=0.69, p=0.0894). Discussion This analysis found that typical EMPIRE Index scores vary across both disease indications and publication types. These results provide valuable contextual information for interpreting EMPIRE Index scores and publication metric findings in general, for individual publications. For example, these findings can be used to help to understand whether a particular publication has notably high (or low) metrics. We found considerable differences between disease areas, which broadly reflected public interest in the disease as assessed through Google search interest ( Table 1 ). The variations in impact across disease areas highlight the complex nature of research dissemination and engagement. While publications in areas other than asthma tended to have lower odds of any impact, those that did have impact often showed higher magnitudes. This suggests that certain disease areas may have fewer but more impactful publications, possibly due to factors such as disease prevalence, research funding, or public health priorities. We found that the three diseases with the highest median EMPIRE Index scores, especially social impact, were migraine, MS and asthma (they also tended to have the greatest effect sizes in the model). These also had the highest public interest. These differences were not observed in journal CiteScores, meaning that the disease areas with higher EMPIRE Index impact were not necessarily published in ‘high impact’ journals. NSCLC had low public interest (‘lung cancer’ as a general term was higher, but still lower than any of the other five major disease areas examined). Publications in NSCLC also had low median total impact scores, particularly in terms of social impact scores, despite being published in journals with higher median CiteScores. Of all the three impact scores, only social impact correlated meaningfully with public interest. Overall, this suggests that the reason publications in some disease areas attract higher social impact scores (driven by citations in news articles and social media) is that these disease areas are of greater interest to the general public. Although this suggests distinct differences between diseases in terms of publication impact, it should be noted that the period of interest was only a single year. The findings could therefore have been influenced by the completion of important clinical studies, which can vary from year to year across disease areas. A clear picture is seen for publication types, with phase 3 trials demonstrating much higher metrics than other types. Phase 3 clinical trials, the last stage of clinical research of new drug treatments, are typically large scale and provide evidence intended to guide treatment practice 22 . The higher impact observed for this publication type likely relates to higher public interest, higher scholarly interest, and greater likelihood of citations in guidelines and policy documents. Systematic reviews had higher impact than general reviews; interestingly, this was despite being published in journals with similar median CiteScores. This likely reflects that the methodological approach to synthesising systematic literature reviews makes them more impactful. Observational studies had the lowest impact, suggesting observational analyses are still generally regarded as having lower interest, despite the fact that they are increasingly regarded as valuable complementary research to interventional studies 23 . These findings from simple comparisons were also supported by results from the exploratory model. In general, across both publication types and disease indications, median scores were higher for scholarly impact than for social or societal impact, while mean and maximal scores were broadly similar (or lower). This suggests that score distribution is more skewed for social and societal impact, with many papers generating little interest despite some scholarly impact. A key strength of this study is the use of an automated approach to identify a large pool of publications for analysis. However, the automated process used depends on the reliability of the underlying data. For example, disease areas were identified through a PubMed search on article titles, which may have excluded some relevant articles or included irrelevant ones. The PubMed search engine uses automatic term mapping, which usually makes the search more inclusive but can introduce inconsistencies 24 . Publication types were identified by metadata tags, but these can often be inconsistently applied or missing. It can also result in duplication; for example, some phase 3 clinical trial publications in our sample were also classified as clinical trials. A further limitation of this study is that it only includes publications from 2019, which may not reflect the long-term impact of publications over time. We acknowledge that some of the publications may have been published towards the end of the period, which could affect the relationship with Google Trends search data. Moreover, the EMPIRE Index relies on data sources that may have incomplete or inaccurate information, such as missing citations, delayed updates, or misclassified publications. Furthermore, the definition or interpretation of publication impact varies based on the stakeholder, intended audience and channel of dissemination, so other aspects such as clinical utility, policy influence, or educational value, may require more qualitative or specific indicators. Therefore, the findings of this study should be complemented by other methods of publication evaluation. Despite these potential limitations, it is important to remember that the original aim of the EMPIRE Index was to allow for flexibility to the users to define publication impact in terms of which individual ALM they would like to emphasise on through more weighting, which then customizes the measurement in the context of the intended pre-defined objectives. The findings of this study have several implications for publication planning and evaluation in academic and also the pharmaceutical industry. First, they highlight the importance of tailoring communication strategies to different disease areas and publication types, as different audiences may have different preferences and expectations for the content and format of publications. For example, publications on rare diseases may benefit from more from sharing on social media or patient engagement, while publications on oncology may benefit from explication of study results in clinical contexts. Second, they demonstrate the value of using a multidimensional index that captures various aspects of publication impact, rather than relying on a single metric such as citations or Altmetric Attention Score. The EMPIRE Index provides a comprehensive and balanced view of how publications perform across scholarly, social and societal domains, and can help identify strengths and weaknesses of different publications and inform future improvement. Third, they suggest that publication impact is not solely determined by the quality and novelty of the research, but also by the relevance and timeliness of the topic, the appropriateness and effectiveness of the dissemination channels, and the engagement and influence of the stakeholders. Therefore, publication planning and evaluation should consider not only the scientific merit of the publication, but also the context and the audience of the publication. In conclusion, the EMPIRE Index successfully identified differences in impact by disease indication and publication type. This supports the notion that there is no universal gold standard metric for publications, and instead the impact of each publication needs to be evaluated in the context of the type of publication, disease area and potentially other factors. These findings should be considered when using the EMPIRE Index to assess publication impact. Data availability Figshare: EMPIRE Index disease and publication type analysis. https://doi.org/10.6084/m9.figshare.17072435.v2 21 This project contains the following underlying data: SMA metrics unlinked 11Jul20.xlsx Psoriasis metrics unlinked 11Jul20.xlsx NSCLC metrics unlinked 5Jul20.xlsx NET metrics unlinked 11Jul20.xlsx NASH metrics unlinked 11Jul20.xlsx MS metrics unlinked 5Jul20.xlsx Migraine metrics unlinked 5Jul20.xlsx Google search interest (30Jul21).xlsx DLBCL metrics unlinked 11Jul20.xlsx Asthma metrics unlinked 5Jul20.xlsx TSC metrics unlinked 11Jul20.xlsx TNBC metrics unlinked 11Jul20.xlsx T2DM metrics unlinked 5Jul20.xlsx Mixed mode analysis supplemental.docx All_data_integrated_7Oct24_deduplicated.csv Data are available under the terms of the Creative Commons Attribution 4.0 International license (CC-BY 4.0). Faculty Opinions recommended References 1. Haustein S, Larivière V: The use of bibliometrics for assessing research: possibilities, limitations and adverse effects. In Incentives and performance: governance of research organizations. (eds. Welpe, I., Wollersheim, J., Ringelhan, S. & Osterloh, M.) Springer, Cham, 2015; 121–139. Publisher Full Text 2. Jeong S: Altmetrics: a practical guide for librarians, researchers and academics. Technical Services Quarterly. 2017; 34 (2): 230–231. Publisher Full Text 3. Raff JW: The San Francisco declaration on research assessment. Biol Open. 2013; 2 (6): 533–534. PubMed Abstract | Publisher Full Text | Free Full Text 4. Bornmann L, Marx W, Gasparyan A, et al. : Diversity, value and limitations of the journal impact factor and alternative metrics. Rheumatol Int. 2012; 32 (7): 1861–1867. PubMed Abstract | Publisher Full Text 5. Ramamurti P, Gu A, Fassihi SC, et al. : Correlation between Altmetric score and traditional bibliometrics in total joint arthroplasty research. Arthroplasty Today. 2021; 7 : 225–229. PubMed Abstract | Publisher Full Text | Free Full Text 6. Kim Y, Kim J, Lee S, et al. : Analysis of altmetrics in social recognition of neurology and neurological disorders. Healthcare (Basel). 2020; 8 (4): 367. PubMed Abstract | Publisher Full Text | Free Full Text 7. Edakar NMAM, Shehata AMK: Measuring the impact of COVID-19 papers on the social web: an altmetric study. Glob Knowl Mem Commun. 2022; 71 (1/2): 1–26. Publisher Full Text 8. Lehane DJ, Black C: Can altmetrics predict future citation counts in critical care medicine publications? J Intensive Care Soc. 2021; 22 (1): 60–66. PubMed Abstract | Publisher Full Text | Free Full Text 9. Azer SA, Azer S: Top-cited articles in medical professionalism: a bibliometric analysis versus altmetric scores. BMJ Open. 2019; 9 (7): e029433. PubMed Abstract | Publisher Full Text | Free Full Text 10. Fassoulaki A, Τsaroucha A, Micha G, et al. : Bibliometric analysis of alternative performance metrics for peri‐operative, critical care and pain medicine journals. Anaesthesia. 2021; 76 (6): 856–857. PubMed Abstract | Publisher Full Text 11. Bayar B, Peksöz R: Analysis of trending topics in breast cancer articles from an Altmetric perspective. Cureus. 2022; 14 (7): e26565. PubMed Abstract | Publisher Full Text | Free Full Text 12. Gregori MD, Scotti V, De Silvestri A, et al. : Does a research group increase impact on the scientific community or general public discussion? alternative metric-based evaluation. J Pain Res. 2016; 9 : 391–5. PubMed Abstract | Publisher Full Text | Free Full Text 13. Bornmann L, Haunschild R, Adams J: Do altmetrics assess societal impact in A comparable way to case studies? An empirical test of the convergent validity of Altmetrics based on data from the UK Research Excellence Framework (REF). J Informetr. 2019; 13 (1): 325–340. Publisher Full Text 14. Pal A, Rees T: Introducing the EMPIRE Index: a novel, value-based metric framework to measure the impact of medical publications. PLoS One. 2022; 17 (4): e0265381. PubMed Abstract | Publisher Full Text | Free Full Text 15. Ruiz-Castillo J, Waltman L: Field-normalized citation impact indicators using algorithmically constructed classification systems of science. J Informetr. 2015; 9 (1): 102–117. Publisher Full Text 16. Amiri M, Michel MC: Do review articles boost journal impact factors? A longitudinal analysis for five pharmacology journals. Naunyn Schmiedebergs Arch Pharmacol. 2018; 391 (9): 1033–1035. PubMed Abstract | Publisher Full Text 17. Retrouvey H, Webster F, Zhong T, et al. : Cross-sectional analysis of bibliometrics and altmetrics: comparing the impact of qualitative and quantitative articles in the British Medical Journal. BMJ Open. 2020; 10 (10): e040950. PubMed Abstract | Publisher Full Text | Free Full Text 18. Scopus Citescore. Reference Source 19. Hong Y, Lawrence J, Williams D Jr, et al. : Population-level interest and telehealth capacity of US Hospitals in response to COVID-19: cross-sectional analysis of google search and national hospital survey data. JMIR Public Health Surveill. 2020; 6 (2): e18961. PubMed Abstract | Publisher Full Text | Free Full Text 20. Publication Characteristics (Publication Types) with Scope Notes. Reference Source 21. Rees T: EMPIRE Index disease and publication type analysis. figshare. Dataset. 2021. http://www.doi.org/10.6084/m9.figshare.17072435.v2 22. US Food and Drug Administration: Step 3: clinical research. Reference Source 23. Bosdriesz JR, Stel VS, van Diepen M, et al. : Evidence-based medicine-when observational studies are better than randomized controlled trials. Nephrology (Carlton). 2020; 25 (10): 737–743. PubMed Abstract | Publisher Full Text | Free Full Text 24. Kang P, Kalloniatis M, Doig GS: Using updated pubmed: new features and functions to enhance literature searches. JAMA. 2021; 326 (6): 479–480. PubMed Abstract | Publisher Full Text Comments on this article Comments (0) Version 5 VERSION 5 PUBLISHED 27 Jan 2022 ADD YOUR COMMENT Comment Author details Author details 1 Oxford PharmaGenesis, Oxford, Oxfordshire, OX13 5QJ, UK 2 Novartis Pharma AG, Basel, Switzerland Tomas Rees Roles: Conceptualization, Data Curation, Formal Analysis, Funding Acquisition, Investigation, Methodology, Project Administration, Resources, Validation, Visualization, Writing – Original Draft Preparation, Writing – Review & Editing Avishek Pal Roles: Conceptualization, Funding Acquisition, Methodology, Resources, Supervision, Writing – Review & Editing Competing interests Tomas Rees is an employee of Oxford PharmaGenesis and received funding from Novartis Pharma AG for this work. Avishek Pal is an employee of Novartis Pharma AG. Grant information Oxford PharmaGenesis was contracted by Novartis Pharma AG (https://www.novartis.com) to provide research and editorial support (manuscript proofreading, figure drawing, and project management). The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript. Article Versions (5) version 5 Revised Published: 30 Oct 2024, 11:107 https://doi.org/10.12688/f1000research.75805.5 version 4 Revised Published: 16 Sep 2024, 11:107 https://doi.org/10.12688/f1000research.75805.4 version 3 Revised Published: 10 Mar 2023, 11:107 https://doi.org/10.12688/f1000research.75805.3 version 2 Revised Published: 12 Apr 2022, 11:107 https://doi.org/10.12688/f1000research.75805.2 version 1 Published: 27 Jan 2022, 11:107 https://doi.org/10.12688/f1000research.75805.1 Copyright © 2024 Rees T and Pal A. This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. Download Export To Sciwheel Bibtex EndNote ProCite Ref. Manager (RIS) Sente metrics Views Downloads F1000Research - - PubMed Central info_outline Data from PMC are received and updated monthly. - - Citations open_in_new 0 open_in_new 0 open_in_new SEE MORE DETAILS CITE how to cite this article Rees T and Pal A. Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.12688/f1000research.75805.5 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS track receive updates on this article Track an article to receive email alerts on any updates to this article. TRACK THIS ARTICLE Share Open Peer Review Current Reviewer Status: ? Key to Reviewer Statuses VIEW HIDE Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Version 5 VERSION 5 PUBLISHED 30 Oct 2024 Revised Views 0 Cite How to cite this report: Saleh AY. Reviewer Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.173716.r336422 ) The direct URL for this report is: https://f1000research.com/articles/11-107/v5#referee-response-336422 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 08 Nov 2024 Arman Yurisaldi Saleh , Universitas Pembangunan Nasional Veteran Jakarta, Jakarta, Special Capital Region of Jakarta, Indonesia Approved VIEWS 0 https://doi.org/10.5256/f1000research.173716.r336422 The author has answered my feedback well and performed ... Continue reading READ ALL The author has answered my feedback well and performed statistical regression, also explaining terms that were previously unclear. Competing Interests: No competing interests were disclosed. Reviewer Expertise: Neurorestoration, neurobehavior, neuroprotective agent, bibliometric I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Saleh AY. Reviewer Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.173716.r336422 ) The direct URL for this report is: https://f1000research.com/articles/11-107/v5#referee-response-336422 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Version 4 VERSION 4 PUBLISHED 16 Sep 2024 Revised Views 0 Cite How to cite this report: Aviv-Reuven S. Reviewer Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.168629.r324069 ) The direct URL for this report is: https://f1000research.com/articles/11-107/v4#referee-response-324069 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 10 Oct 2024 Shir Aviv-Reuven , Bar Ilan University, Ramat Gan, Israel Approved VIEWS 0 https://doi.org/10.5256/f1000research.168629.r324069 The authors have responded to all my comments and the manuscript, ... Continue reading READ ALL The authors have responded to all my comments and the manuscript, I have no further comments and I now Approve it without revisions. Competing Interests: No competing interests were disclosed. I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Aviv-Reuven S. Reviewer Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.168629.r324069 ) The direct URL for this report is: https://f1000research.com/articles/11-107/v4#referee-response-324069 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Saleh AY. Reviewer Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.168629.r327843 ) The direct URL for this report is: https://f1000research.com/articles/11-107/v4#referee-response-327843 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 30 Sep 2024 Arman Yurisaldi Saleh , Universitas Pembangunan Nasional Veteran Jakarta, Jakarta, Special Capital Region of Jakarta, Indonesia Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.168629.r327843 This article has strengths when viewed from the amount of data used, which is quite a lot in the form of articles, so that it provides a fairly clear picture when analyzed. Actually, this article would be ... Continue reading READ ALL This article has strengths when viewed from the amount of data used, which is quite a lot in the form of articles, so that it provides a fairly clear picture when analyzed. Actually, this article would be more convincing if the right statistical test was used, but no statistical test was carried out in the manuscript. For example, a statistical test can be added whose results are a picture of the scientific score and social impact score for various articles in the form of systematic reviews, phase 3 clinical trials and observational. The difference in impact scores between the two groups can be analyzed using the t-test in statistical tests So this is the main suggestion for improving the article. In the manuscript, it can be seen that the dependent variables are, for example, the social and scientific impact scores. While the independent variables are public interest, disease indications, number of article citations, type of publication. There are shortcomings in terms of statistical analysis, related to the study of the relationship between the variables above, for example, what if you want to know the relationship between the social and scientific impact scores and public interest, and this is not found in the manuscript. A small point to note is that a linear regression analysis was not included. I kindly suggest that statistical calculations in the form of linear regression be added. I suggest that the author be clearer in explaining Cite Score, Altimetrik and Google Trends. A more detailed description is still needed regarding the 3 terms above so that the reader is clearer. My conclusion is that the author has responded well to the reviewers' input, and I provide suggestions for improvement in terms of the use of statistical tests and more detailed descriptions. The article is worthy of being indexed. Is the work clearly and accurately presented and does it cite the current literature? Yes Is the study design appropriate and is the work technically sound? Yes Are sufficient details of methods and analysis provided to allow replication by others? No If applicable, is the statistical analysis and its interpretation appropriate? No Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly Competing Interests: No competing interests were disclosed. Reviewer Expertise: Neurorestoration, neurobehavior, neuroprotective agent, bibliometric I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Saleh AY. Reviewer Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.168629.r327843 ) The direct URL for this report is: https://f1000research.com/articles/11-107/v4#referee-response-327843 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 30 Oct 2024 Tomas Rees , Oxford PharmaGenesis, Oxford, OX13 5QJ, UK 30 Oct 2024 Author Response We thank the reviewer for their comments. The initial version of this analysis provided only simple statistical analysis (a non-parametric test of between-group differences) because we were primarily interested in ... Continue reading We thank the reviewer for their comments. The initial version of this analysis provided only simple statistical analysis (a non-parametric test of between-group differences) because we were primarily interested in understanding the magnitude of the scores that could be expected for different kind of publications in different disease areas. Furthermore, the data are complicated to analyse because they are zero inflated (many publications have a score of 0), highly skewed (a few publications score very highly) and partially nested (some, but not all, 'phase 3 clinical trials' are also tagged as 'clinical trials', for example). However, we do agree that statistical analysis can provide useful and interesting perspective on the relationship between our independent variables and the EMPIRE Index scores. Therefore, we have provided a mixed model analysis as well as a regression analysis versus Google search interest. The results of these analyses largely accord with our original conclusions, but provide rich detail on these interactions. We thank the reviewer for their comments. The initial version of this analysis provided only simple statistical analysis (a non-parametric test of between-group differences) because we were primarily interested in understanding the magnitude of the scores that could be expected for different kind of publications in different disease areas. Furthermore, the data are complicated to analyse because they are zero inflated (many publications have a score of 0), highly skewed (a few publications score very highly) and partially nested (some, but not all, 'phase 3 clinical trials' are also tagged as 'clinical trials', for example). However, we do agree that statistical analysis can provide useful and interesting perspective on the relationship between our independent variables and the EMPIRE Index scores. Therefore, we have provided a mixed model analysis as well as a regression analysis versus Google search interest. The results of these analyses largely accord with our original conclusions, but provide rich detail on these interactions. Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 30 Oct 2024 Tomas Rees , Oxford PharmaGenesis, Oxford, OX13 5QJ, UK 30 Oct 2024 Author Response We thank the reviewer for their comments. The initial version of this analysis provided only simple statistical analysis (a non-parametric test of between-group differences) because we were primarily interested in ... Continue reading We thank the reviewer for their comments. The initial version of this analysis provided only simple statistical analysis (a non-parametric test of between-group differences) because we were primarily interested in understanding the magnitude of the scores that could be expected for different kind of publications in different disease areas. Furthermore, the data are complicated to analyse because they are zero inflated (many publications have a score of 0), highly skewed (a few publications score very highly) and partially nested (some, but not all, 'phase 3 clinical trials' are also tagged as 'clinical trials', for example). However, we do agree that statistical analysis can provide useful and interesting perspective on the relationship between our independent variables and the EMPIRE Index scores. Therefore, we have provided a mixed model analysis as well as a regression analysis versus Google search interest. The results of these analyses largely accord with our original conclusions, but provide rich detail on these interactions. We thank the reviewer for their comments. The initial version of this analysis provided only simple statistical analysis (a non-parametric test of between-group differences) because we were primarily interested in understanding the magnitude of the scores that could be expected for different kind of publications in different disease areas. Furthermore, the data are complicated to analyse because they are zero inflated (many publications have a score of 0), highly skewed (a few publications score very highly) and partially nested (some, but not all, 'phase 3 clinical trials' are also tagged as 'clinical trials', for example). However, we do agree that statistical analysis can provide useful and interesting perspective on the relationship between our independent variables and the EMPIRE Index scores. Therefore, we have provided a mixed model analysis as well as a regression analysis versus Google search interest. The results of these analyses largely accord with our original conclusions, but provide rich detail on these interactions. Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Version 3 VERSION 3 PUBLISHED 10 Mar 2023 Revised Views 0 Cite How to cite this report: Aviv-Reuven S. Reviewer Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.145010.r234323 ) The direct URL for this report is: https://f1000research.com/articles/11-107/v3#referee-response-234323 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 27 Feb 2024 Shir Aviv-Reuven , Bar Ilan University, Ramat Gan, Israel Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.145010.r234323 This manuscript aims to identify impact differences between medical publications according to diseases and publication types as well as observe variations in public interest by comparing the EMPIRE index with Google search trends. In general I find this paper ... Continue reading READ ALL This manuscript aims to identify impact differences between medical publications according to diseases and publication types as well as observe variations in public interest by comparing the EMPIRE index with Google search trends. In general I find this paper interesting, but I found it lacking in aspects which the previous reviewer mentioned. Further more, the references to related literature is lacking and there is no discussion related to other possible article level metrics and why the EMPIRE index was selected. Introduction 1. I agree with previous reviewer that Introduction is lacking, not only in presentation of the research problem, but also in the significance of this study. 2. I further agree that Literature review section is missing. it should either be added or the introduction further expanded to address related studies utilizing this indicator as well those referring to analysis of medical publications in general and specifically those who refer to the 12 diseases under analysis in this paper. 3. last paragraph of Introduction should be rephrased as it is unclear, particularly this - "We also look to see if there are similar variations in public interest in therapy areas we explore, with particular interest in comparing this with the EMPIRE Index includes a social score." Methods 4. The authors do not explain why the different types were selected for analysis and what each of these types presents. These should be detailed. 5. The authors do not give detailed the usage of Google Trends. what data was downloaded? was it filtered to specific areas?. Further more, as some of the papers may have been published towards the end of that period, it may have been more appropriate a longer period for the trends. Results 1. The usage of CiteScore is unclear to me, as CiteScore aims to measure journal impact. Discussion The paragraph detailing the findings related to Google trends should be moved to results and a plot showing these trends should be added. Discussion of the results seems short and in some cases state the obvious. for example " Observational studies had the lowest impact, suggesting observational analyses are still generally regarded as having lower interest." A general discussion of possible significance of the findings should be added. Is the work clearly and accurately presented and does it cite the current literature? Partly Is the study design appropriate and is the work technically sound? Partly Are sufficient details of methods and analysis provided to allow replication by others? Partly If applicable, is the statistical analysis and its interpretation appropriate? I cannot comment. A qualified statistician is required. Are all the source data underlying the results available to ensure full reproducibility? Partly Are the conclusions drawn adequately supported by the results? Partly Competing Interests: No competing interests were disclosed. Reviewer Expertise: Bibliometrics, Scientometrics, academic search, journals impact I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Aviv-Reuven S. Reviewer Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.145010.r234323 ) The direct URL for this report is: https://f1000research.com/articles/11-107/v3#referee-response-234323 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 28 Jun 2024 Tomas Rees , Oxford PharmaGenesis, Oxford, OX13 5QJ, UK 28 Jun 2024 Author Response We thank the Dr Aviv-Reuven for their constructive comments and suggestions on our manuscript. We have revised the manuscript accordingly and addressed each comment in detail below. Reviewer comments and ... Continue reading We thank the Dr Aviv-Reuven for their constructive comments and suggestions on our manuscript. We have revised the manuscript accordingly and addressed each comment in detail below. Reviewer comments and responses Introduction 1. I agree with previous reviewer that Introduction is lacking, not only in presentation of the research problem, but also in the significance of this study. Response: We have revised the introduction to provide more background on the research problem and the significance of this study. We have added a paragraph that explains the limitations of conventional metrics such as journal impact factor and CiteScore, and the advantages of using article-level metrics such as altmetrics to assess the impact of medical publications. We have also added a paragraph that describes the EMPIRE Index. We have highlighted the novelty and relevance of our study, which is the first to explore the variations in EMPIRE Index scores across different disease indications and publication types. 2. I further agree that Literature review section is missing. it should either be added or the introduction further expanded to address related studies utilizing this indicator as well those referring to analysis of medical publications in general and specifically those who refer to the 12 diseases under analysis in this paper. Response: We have added a literature review where we discuss the previous studies that have used altmetrics to assess the impact of medical publications in various fields and contexts. 3. last paragraph of Introduction should be rephrased as it is unclear, particularly this - "We also look to see if there are similar variations in public interest in therapy areas we explore, with particular interest in comparing this with the EMPIRE Index includes a social score." Response: We have rephrased the last paragraph of the introduction to clarify our research objectives and hypotheses. Methods 4. The authors do not explain why the different types were selected for analysis and what each of these types presents. These should be detailed. Response: We have added a paragraph in the methods section that explains the rationale for selecting the different publication types for analysis and what each of these types represents. 5. The authors do not give detailed the usage of Google Trends. what data was downloaded? was it filtered to specific areas?. Further more, as some of the papers may have been published towards the end of that period, it may have been more appropriate a longer period for the trends. Response: We have added more details on the usage of Google Trends in the methods section. Results The usage of CiteScore is unclear to me, as CiteScore aims to measure journal impact. Response: We have clarified the rationale for the comparison with of CiteScore in the introduction. Discussion The paragraph detailing the findings related to Google trends should be moved to results and a plot showing these trends should be added. Response: We have clarified that the data are presented in Table 1 as averages from 2019. We have not included a plot of the weekly data for 2019 as we do not refer to these and they are not relevant to our discussion. However, they are available in the supplementary material in excel form. Discussion of the results seems short and in some cases state the obvious. for example " Observational studies had the lowest impact, suggesting observational analyses are still generally regarded as having lower interest." Response: We have clarified the value of observational studies. A general discussion of possible significance of the findings should be added. Response: We have added a discussion of potential significance, as well as expanding the limitations. We thank the Dr Aviv-Reuven for their constructive comments and suggestions on our manuscript. We have revised the manuscript accordingly and addressed each comment in detail below. Reviewer comments and responses Introduction 1. I agree with previous reviewer that Introduction is lacking, not only in presentation of the research problem, but also in the significance of this study. Response: We have revised the introduction to provide more background on the research problem and the significance of this study. We have added a paragraph that explains the limitations of conventional metrics such as journal impact factor and CiteScore, and the advantages of using article-level metrics such as altmetrics to assess the impact of medical publications. We have also added a paragraph that describes the EMPIRE Index. We have highlighted the novelty and relevance of our study, which is the first to explore the variations in EMPIRE Index scores across different disease indications and publication types. 2. I further agree that Literature review section is missing. it should either be added or the introduction further expanded to address related studies utilizing this indicator as well those referring to analysis of medical publications in general and specifically those who refer to the 12 diseases under analysis in this paper. Response: We have added a literature review where we discuss the previous studies that have used altmetrics to assess the impact of medical publications in various fields and contexts. 3. last paragraph of Introduction should be rephrased as it is unclear, particularly this - "We also look to see if there are similar variations in public interest in therapy areas we explore, with particular interest in comparing this with the EMPIRE Index includes a social score." Response: We have rephrased the last paragraph of the introduction to clarify our research objectives and hypotheses. Methods 4. The authors do not explain why the different types were selected for analysis and what each of these types presents. These should be detailed. Response: We have added a paragraph in the methods section that explains the rationale for selecting the different publication types for analysis and what each of these types represents. 5. The authors do not give detailed the usage of Google Trends. what data was downloaded? was it filtered to specific areas?. Further more, as some of the papers may have been published towards the end of that period, it may have been more appropriate a longer period for the trends. Response: We have added more details on the usage of Google Trends in the methods section. Results The usage of CiteScore is unclear to me, as CiteScore aims to measure journal impact. Response: We have clarified the rationale for the comparison with of CiteScore in the introduction. Discussion The paragraph detailing the findings related to Google trends should be moved to results and a plot showing these trends should be added. Response: We have clarified that the data are presented in Table 1 as averages from 2019. We have not included a plot of the weekly data for 2019 as we do not refer to these and they are not relevant to our discussion. However, they are available in the supplementary material in excel form. Discussion of the results seems short and in some cases state the obvious. for example " Observational studies had the lowest impact, suggesting observational analyses are still generally regarded as having lower interest." Response: We have clarified the value of observational studies. A general discussion of possible significance of the findings should be added. Response: We have added a discussion of potential significance, as well as expanding the limitations. Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 28 Jun 2024 Tomas Rees , Oxford PharmaGenesis, Oxford, OX13 5QJ, UK 28 Jun 2024 Author Response We thank the Dr Aviv-Reuven for their constructive comments and suggestions on our manuscript. We have revised the manuscript accordingly and addressed each comment in detail below. Reviewer comments and ... Continue reading We thank the Dr Aviv-Reuven for their constructive comments and suggestions on our manuscript. We have revised the manuscript accordingly and addressed each comment in detail below. Reviewer comments and responses Introduction 1. I agree with previous reviewer that Introduction is lacking, not only in presentation of the research problem, but also in the significance of this study. Response: We have revised the introduction to provide more background on the research problem and the significance of this study. We have added a paragraph that explains the limitations of conventional metrics such as journal impact factor and CiteScore, and the advantages of using article-level metrics such as altmetrics to assess the impact of medical publications. We have also added a paragraph that describes the EMPIRE Index. We have highlighted the novelty and relevance of our study, which is the first to explore the variations in EMPIRE Index scores across different disease indications and publication types. 2. I further agree that Literature review section is missing. it should either be added or the introduction further expanded to address related studies utilizing this indicator as well those referring to analysis of medical publications in general and specifically those who refer to the 12 diseases under analysis in this paper. Response: We have added a literature review where we discuss the previous studies that have used altmetrics to assess the impact of medical publications in various fields and contexts. 3. last paragraph of Introduction should be rephrased as it is unclear, particularly this - "We also look to see if there are similar variations in public interest in therapy areas we explore, with particular interest in comparing this with the EMPIRE Index includes a social score." Response: We have rephrased the last paragraph of the introduction to clarify our research objectives and hypotheses. Methods 4. The authors do not explain why the different types were selected for analysis and what each of these types presents. These should be detailed. Response: We have added a paragraph in the methods section that explains the rationale for selecting the different publication types for analysis and what each of these types represents. 5. The authors do not give detailed the usage of Google Trends. what data was downloaded? was it filtered to specific areas?. Further more, as some of the papers may have been published towards the end of that period, it may have been more appropriate a longer period for the trends. Response: We have added more details on the usage of Google Trends in the methods section. Results The usage of CiteScore is unclear to me, as CiteScore aims to measure journal impact. Response: We have clarified the rationale for the comparison with of CiteScore in the introduction. Discussion The paragraph detailing the findings related to Google trends should be moved to results and a plot showing these trends should be added. Response: We have clarified that the data are presented in Table 1 as averages from 2019. We have not included a plot of the weekly data for 2019 as we do not refer to these and they are not relevant to our discussion. However, they are available in the supplementary material in excel form. Discussion of the results seems short and in some cases state the obvious. for example " Observational studies had the lowest impact, suggesting observational analyses are still generally regarded as having lower interest." Response: We have clarified the value of observational studies. A general discussion of possible significance of the findings should be added. Response: We have added a discussion of potential significance, as well as expanding the limitations. We thank the Dr Aviv-Reuven for their constructive comments and suggestions on our manuscript. We have revised the manuscript accordingly and addressed each comment in detail below. Reviewer comments and responses Introduction 1. I agree with previous reviewer that Introduction is lacking, not only in presentation of the research problem, but also in the significance of this study. Response: We have revised the introduction to provide more background on the research problem and the significance of this study. We have added a paragraph that explains the limitations of conventional metrics such as journal impact factor and CiteScore, and the advantages of using article-level metrics such as altmetrics to assess the impact of medical publications. We have also added a paragraph that describes the EMPIRE Index. We have highlighted the novelty and relevance of our study, which is the first to explore the variations in EMPIRE Index scores across different disease indications and publication types. 2. I further agree that Literature review section is missing. it should either be added or the introduction further expanded to address related studies utilizing this indicator as well those referring to analysis of medical publications in general and specifically those who refer to the 12 diseases under analysis in this paper. Response: We have added a literature review where we discuss the previous studies that have used altmetrics to assess the impact of medical publications in various fields and contexts. 3. last paragraph of Introduction should be rephrased as it is unclear, particularly this - "We also look to see if there are similar variations in public interest in therapy areas we explore, with particular interest in comparing this with the EMPIRE Index includes a social score." Response: We have rephrased the last paragraph of the introduction to clarify our research objectives and hypotheses. Methods 4. The authors do not explain why the different types were selected for analysis and what each of these types presents. These should be detailed. Response: We have added a paragraph in the methods section that explains the rationale for selecting the different publication types for analysis and what each of these types represents. 5. The authors do not give detailed the usage of Google Trends. what data was downloaded? was it filtered to specific areas?. Further more, as some of the papers may have been published towards the end of that period, it may have been more appropriate a longer period for the trends. Response: We have added more details on the usage of Google Trends in the methods section. Results The usage of CiteScore is unclear to me, as CiteScore aims to measure journal impact. Response: We have clarified the rationale for the comparison with of CiteScore in the introduction. Discussion The paragraph detailing the findings related to Google trends should be moved to results and a plot showing these trends should be added. Response: We have clarified that the data are presented in Table 1 as averages from 2019. We have not included a plot of the weekly data for 2019 as we do not refer to these and they are not relevant to our discussion. However, they are available in the supplementary material in excel form. Discussion of the results seems short and in some cases state the obvious. for example " Observational studies had the lowest impact, suggesting observational analyses are still generally regarded as having lower interest." Response: We have clarified the value of observational studies. A general discussion of possible significance of the findings should be added. Response: We have added a discussion of potential significance, as well as expanding the limitations. Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Version 2 VERSION 2 PUBLISHED 12 Apr 2022 Revised Views 0 Cite How to cite this report: Ortega JL. Reviewer Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.126135.r134659 ) The direct URL for this report is: https://f1000research.com/articles/11-107/v2#referee-response-134659 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 19 Apr 2022 José Luis Ortega , Institute for Advanced Social Studies (IESA-CSIC), Córdoba, Spain Not Approved VIEWS 0 https://doi.org/10.5256/f1000research.126135.r134659 The text is the same as ... Continue reading READ ALL The text is the same as the version 1. Only references were updated. Competing Interests: No competing interests were disclosed. Reviewer Expertise: Bibliometrics, altmetrics, academic search engines, scholarly social networks I confirm that I have read this submission and believe that I have an appropriate level of expertise to state that I do not consider it to be of an acceptable scientific standard, for reasons outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Ortega JL. Reviewer Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.126135.r134659 ) The direct URL for this report is: https://f1000research.com/articles/11-107/v2#referee-response-134659 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 27 Mar 2023 Tomas Rees , Oxford PharmaGenesis, Oxford, OX13 5QJ, UK 27 Mar 2023 Author Response We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this was ... Continue reading We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this was intended as a brief report. It follows on from the full publication detailing the background and rationale of the EMPIRE Index published in PLOS ONE, which covers how it was developed and what the component scores signify. The current report provides some useful additional investigation of the properties of the EMPIRE index and its underlying article-level metrics. It is for these reasons that this report does not include a full literature review and does not go into the details of the EMPIRE Index that are covered in the PLOS ONE paper. We originally referenced the preprint in the current submission (the primary publication had been approved but publication was delayed) but have updated the reference now that the original article is fully published. We have addressed the reviewer’s other comments, as follows: We expanded the introduction to provide greater clarity of the objectives and purpose of this analysis – including why we have used CiteScore and Google Trends. We have added information on data extraction from Altmetrics Explorer and PlumX. We have also added information on the definition of the publication types (these are defined by PubMed). While most of the component metrics of the EMPIRE Index are altmetrics, article citations are not and we agree that using altmetrics as an umbrella term is therefore inappropriate: we have changed to article-level metrics (ALMs) throughout. We agree that computing an average CiteScore would be inappropriate, because such an average would not indicate the average number of citations per paper across the set of journals and so could be misleading. However, in our analysis we use the median, which is robust in this case since it does not transform the underlying data. We do not believe that it is obvious whether the metrics for a given paper are notably high or low. This can only be understood by understanding the typical metrics of comparable publications, which is the objective of this report. We note in the discussion that the higher scores of migraine publications, in particular social scores, may be linked to higher public interest as evidence by higher Google Trends interest. However, we acknowledge that the analyses presented do not and cannot explain the differences in typical scores seen across therapy areas and publication types. Therefore, any such discussion must necessarily be speculative and as such we keep it to a minimum. The sole purpose of this report is to demonstrate that these variations exist and to explore the magnitude of these variations. The analyses presented are robust and do provide meaningful results that address the objective. We therefore believe that the current, amended report is worthy of publication despite its brief nature. We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this was intended as a brief report. It follows on from the full publication detailing the background and rationale of the EMPIRE Index published in PLOS ONE, which covers how it was developed and what the component scores signify. The current report provides some useful additional investigation of the properties of the EMPIRE index and its underlying article-level metrics. It is for these reasons that this report does not include a full literature review and does not go into the details of the EMPIRE Index that are covered in the PLOS ONE paper. We originally referenced the preprint in the current submission (the primary publication had been approved but publication was delayed) but have updated the reference now that the original article is fully published. We have addressed the reviewer’s other comments, as follows: We expanded the introduction to provide greater clarity of the objectives and purpose of this analysis – including why we have used CiteScore and Google Trends. We have added information on data extraction from Altmetrics Explorer and PlumX. We have also added information on the definition of the publication types (these are defined by PubMed). While most of the component metrics of the EMPIRE Index are altmetrics, article citations are not and we agree that using altmetrics as an umbrella term is therefore inappropriate: we have changed to article-level metrics (ALMs) throughout. We agree that computing an average CiteScore would be inappropriate, because such an average would not indicate the average number of citations per paper across the set of journals and so could be misleading. However, in our analysis we use the median, which is robust in this case since it does not transform the underlying data. We do not believe that it is obvious whether the metrics for a given paper are notably high or low. This can only be understood by understanding the typical metrics of comparable publications, which is the objective of this report. We note in the discussion that the higher scores of migraine publications, in particular social scores, may be linked to higher public interest as evidence by higher Google Trends interest. However, we acknowledge that the analyses presented do not and cannot explain the differences in typical scores seen across therapy areas and publication types. Therefore, any such discussion must necessarily be speculative and as such we keep it to a minimum. The sole purpose of this report is to demonstrate that these variations exist and to explore the magnitude of these variations. The analyses presented are robust and do provide meaningful results that address the objective. We therefore believe that the current, amended report is worthy of publication despite its brief nature. Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 27 Mar 2023 Tomas Rees , Oxford PharmaGenesis, Oxford, OX13 5QJ, UK 27 Mar 2023 Author Response We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this was ... Continue reading We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this was intended as a brief report. It follows on from the full publication detailing the background and rationale of the EMPIRE Index published in PLOS ONE, which covers how it was developed and what the component scores signify. The current report provides some useful additional investigation of the properties of the EMPIRE index and its underlying article-level metrics. It is for these reasons that this report does not include a full literature review and does not go into the details of the EMPIRE Index that are covered in the PLOS ONE paper. We originally referenced the preprint in the current submission (the primary publication had been approved but publication was delayed) but have updated the reference now that the original article is fully published. We have addressed the reviewer’s other comments, as follows: We expanded the introduction to provide greater clarity of the objectives and purpose of this analysis – including why we have used CiteScore and Google Trends. We have added information on data extraction from Altmetrics Explorer and PlumX. We have also added information on the definition of the publication types (these are defined by PubMed). While most of the component metrics of the EMPIRE Index are altmetrics, article citations are not and we agree that using altmetrics as an umbrella term is therefore inappropriate: we have changed to article-level metrics (ALMs) throughout. We agree that computing an average CiteScore would be inappropriate, because such an average would not indicate the average number of citations per paper across the set of journals and so could be misleading. However, in our analysis we use the median, which is robust in this case since it does not transform the underlying data. We do not believe that it is obvious whether the metrics for a given paper are notably high or low. This can only be understood by understanding the typical metrics of comparable publications, which is the objective of this report. We note in the discussion that the higher scores of migraine publications, in particular social scores, may be linked to higher public interest as evidence by higher Google Trends interest. However, we acknowledge that the analyses presented do not and cannot explain the differences in typical scores seen across therapy areas and publication types. Therefore, any such discussion must necessarily be speculative and as such we keep it to a minimum. The sole purpose of this report is to demonstrate that these variations exist and to explore the magnitude of these variations. The analyses presented are robust and do provide meaningful results that address the objective. We therefore believe that the current, amended report is worthy of publication despite its brief nature. We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this was intended as a brief report. It follows on from the full publication detailing the background and rationale of the EMPIRE Index published in PLOS ONE, which covers how it was developed and what the component scores signify. The current report provides some useful additional investigation of the properties of the EMPIRE index and its underlying article-level metrics. It is for these reasons that this report does not include a full literature review and does not go into the details of the EMPIRE Index that are covered in the PLOS ONE paper. We originally referenced the preprint in the current submission (the primary publication had been approved but publication was delayed) but have updated the reference now that the original article is fully published. We have addressed the reviewer’s other comments, as follows: We expanded the introduction to provide greater clarity of the objectives and purpose of this analysis – including why we have used CiteScore and Google Trends. We have added information on data extraction from Altmetrics Explorer and PlumX. We have also added information on the definition of the publication types (these are defined by PubMed). While most of the component metrics of the EMPIRE Index are altmetrics, article citations are not and we agree that using altmetrics as an umbrella term is therefore inappropriate: we have changed to article-level metrics (ALMs) throughout. We agree that computing an average CiteScore would be inappropriate, because such an average would not indicate the average number of citations per paper across the set of journals and so could be misleading. However, in our analysis we use the median, which is robust in this case since it does not transform the underlying data. We do not believe that it is obvious whether the metrics for a given paper are notably high or low. This can only be understood by understanding the typical metrics of comparable publications, which is the objective of this report. We note in the discussion that the higher scores of migraine publications, in particular social scores, may be linked to higher public interest as evidence by higher Google Trends interest. However, we acknowledge that the analyses presented do not and cannot explain the differences in typical scores seen across therapy areas and publication types. Therefore, any such discussion must necessarily be speculative and as such we keep it to a minimum. The sole purpose of this report is to demonstrate that these variations exist and to explore the magnitude of these variations. The analyses presented are robust and do provide meaningful results that address the objective. We therefore believe that the current, amended report is worthy of publication despite its brief nature. Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Version 1 VERSION 1 PUBLISHED 27 Jan 2022 Views 0 Cite How to cite this report: Ortega JL. Reviewer Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.79727.r121444 ) The direct URL for this report is: https://f1000research.com/articles/11-107/v1#referee-response-121444 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 10 Feb 2022 José Luis Ortega , Institute for Advanced Social Studies (IESA-CSIC), Córdoba, Spain Not Approved VIEWS 0 https://doi.org/10.5256/f1000research.79727.r121444 This manuscript attempts to find impact differences between medical publications according to diseases and publication types. These differences are calculated according to a new indicator, EMPIRE Index. More than 20k publications were retrieved from PubMed about 12 diseases areas. The ... Continue reading READ ALL This manuscript attempts to find impact differences between medical publications according to diseases and publication types. These differences are calculated according to a new indicator, EMPIRE Index. More than 20k publications were retrieved from PubMed about 12 diseases areas. The resulting publications were also grouped by publication type. Using the Kruskal-Wallis test, authors found that the EMPIRE Indicator produces some differences between publications according to diseases and type of documents. These results allow them to justify the utility of this indicator and its power to identify different types of impacts. However, the paper presents several weaknesses that impede my recommendation for approval. Next, I detail some problem points by section: Introduction The introduction is poor because it does not introduce the research problem properly. An introduction should explain why the research is necessary and what the problem is that the paper wants to address. Why is important to know that impact varies by disease or by publication type? Why is an indicator such as EMPIRE Index necessary? Authors confound article-level with altmetrics. Many altmetric measures could be article-level metrics, but there are other article-level metrics (i.e. citations) that are not altmetrics. Author should revise the text and correct these statements. I do not see a key section in a scientific article: a literature review. Authors should include a section where they contextualize the research, explaining previous studies that had deal with this problem. Studies about new indicators, differences in the impact by subject or document type, different types of impacts, etc. Overall, authors should present other studies in the introduction that show why is important to know that a disease has more social or scholarly impact, or why some document types attract more citations or tweets. In short, more conceptual background. The paper includes only eight references, which illustrates the poor context of this research. Objective I think this section is the most important in a paper and it is missing in this manuscript. If there are not objectives, there are not research goals and a paper lacks sense. The final line of Introduction says: “Here, we present an exploratory investigation of whether disease indications and publication types influence the average EMPIRE Index scores.” Why do you want to know this? Why is important to know this? Authors should introduce a section for objectives where they explain the main objectives and secondary ones. It could be interesting to include some research questions that specify the aim of the paper. Methods Authors should include more detail about the data extraction process in Altmetric and PlumX. Authors claim that they used Altmetric Explorer to retrieve 20k publications. How did you exactly retrieve the data from this site? Do you use the API endpoint? How do you obtain the data from PlumX? A key element in the paper is to test differences according to diseases and document types, but there is not any explanation about these typologies. For example, and about document types, what is the difference between “review” and “systematic review”? This part is key because the definition of groups and types influences the results, so it is very important to define the diseases and the typology of documents. All the analysis relies on a new indicator, EMPIRE Index. In spite of the importance of this indicator in the paper, there is not any information about how it is calculated and conceptualized. We have to read a non-peer reviewed pre-print deposited in a repository to know what it is. I do not see this way to do science as correct. Authors have to explain how this indicator is calculated in this paper; how the three component scores are defined; what is the difference between “social impact” and “societal impact”, mainly from a conceptual point of view; likewise how do you find three components and not two or four; which metrics take part in each component and why? CiteScore is an indicator for journals, not for articles. How it is used and why? The same for Google Trends - how they are used in the study, and why? Results Tables 3 and 5 use CiteScore median for valuing articles by diseases. I recall that it is not correct to use journal indicators for research articles, and the use of median or mean from a ratio (CiteScore is the ratio of citations by publications) is a mathematical artifact, and the results are spurious. Discussion This paper lacks of any substantial discussion about the results. Sometimes the interpretation is obvious and it does not contribute valuable information. For example, the first paragraph ends: “For example, these findings can be used to help to understand whether a particular publication has notably high (or low) metrics.” This is obvious and It is not necessary to do this study to reach that conclusion. Every publication has high or low metrics. Another example: “Observational studies had the lowest impact, suggesting observational analyses are still generally regarded as having lower interest.” If they have low impact, then they have low interest. It is interesting that the results from the EMPIRE index are not interpreted. For example, migraine, MS and asthma are the diseases with more social impact. Why? Is this positive or negative? Is there any evidence in other studies that confirm that these pathologies are more interesting for public audiences? For example, is there any correlation between these scores and Google Trends? Is there any survey that explore the public opinion about diseases? Etc. This part should include the interpretation of the results according to the previous knowledge. It is interesting that this section only includes one citation (about using PubMed), but there is not any reference to previous results to discuss the validity of the findings or to explain the meaning of the results. For example, “The high impact of phase 3 clinical trials is to be expected, given that they are intended to provide practice-changing information.” Why is it that papers which provide practice-changing information should have more impact? Is the work clearly and accurately presented and does it cite the current literature? Partly Is the study design appropriate and is the work technically sound? Partly Are sufficient details of methods and analysis provided to allow replication by others? No If applicable, is the statistical analysis and its interpretation appropriate? Yes Are all the source data underlying the results available to ensure full reproducibility? Partly Are the conclusions drawn adequately supported by the results? No Competing Interests: No competing interests were disclosed. Reviewer Expertise: Bibliometrics, altmetrics, academic search engines, scholarly social networks I confirm that I have read this submission and believe that I have an appropriate level of expertise to state that I do not consider it to be of an acceptable scientific standard, for reasons outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Ortega JL. Reviewer Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.79727.r121444 ) The direct URL for this report is: https://f1000research.com/articles/11-107/v1#referee-response-121444 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 28 Apr 2023 Tomas Rees , Oxford PharmaGenesis, Oxford, OX13 5QJ, UK 28 Apr 2023 Author Response We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this is ... Continue reading We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this is a F1000 Brief Report (https://f1000research.com/for-authors/article-guidelines/brief-report), and so is limited by the journal style template (no ‘Objectives’ section) and word count limit (2,500 words). It follows on from the full publication detailing the background and rationale of the EMPIRE Index published in PLOS ONE, which covers how it was developed and what the component scores signify. The current report is intended to provide some useful additional investigation of the properties of the EMPIRE index and its underlying article-level metrics. It is for these reasons that this report does not include a full literature review and does not go into the details of the EMPIRE Index that are covered in the PLOS ONE paper. We originally referenced the preprint in the current submission (the primary publication had been approved but publication was delayed) but have updated the reference now that the original article is fully published. We have addressed the reviewer’s other comments, as follows: We expanded the introduction to provide greater clarity of the objectives and purpose of this analysis – including why we have used CiteScore and Google Trends. We have added information on data extraction from Altmetrics Explorer and PlumX. We have also added information on the definition of the publication types (these are defined by PubMed). While most of the component metrics of the EMPIRE Index are altmetrics, article citations are not and we agree that using altmetrics as an umbrella term is therefore inappropriate: we have changed to article-level metrics (ALMs) throughout. We agree that computing an average CiteScore would be inappropriate, because such an average would not indicate the average number of citations per paper across the set of journals and so could be misleading. However, in our analysis we use the median, which is robust in this case since it does not transform the underlying data. We do not believe that it is obvious whether the metrics for a given paper are notably high or low. This can only be understood by understanding the typical metrics of comparable publications, which is the objective of this report. We note in the discussion that the higher scores of migraine publications, in particular social scores, may be linked to higher public interest as evidence by higher Google Trends interest. However, we acknowledge that the analyses presented do not and cannot explain the differences in typical scores seen across therapy areas and publication types. Therefore, any such discussion must necessarily be speculative and as such we keep it to a minimum. The sole purpose of this report is to demonstrate that these variations exist and to explore the magnitude of these variations. The analyses presented are robust and do provide meaningful results that address the objective. We therefore believe that the current, amended report is worthy of publication despite its brief nature. We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this is a F1000 Brief Report (https://f1000research.com/for-authors/article-guidelines/brief-report), and so is limited by the journal style template (no ‘Objectives’ section) and word count limit (2,500 words). It follows on from the full publication detailing the background and rationale of the EMPIRE Index published in PLOS ONE, which covers how it was developed and what the component scores signify. The current report is intended to provide some useful additional investigation of the properties of the EMPIRE index and its underlying article-level metrics. It is for these reasons that this report does not include a full literature review and does not go into the details of the EMPIRE Index that are covered in the PLOS ONE paper. We originally referenced the preprint in the current submission (the primary publication had been approved but publication was delayed) but have updated the reference now that the original article is fully published. We have addressed the reviewer’s other comments, as follows: We expanded the introduction to provide greater clarity of the objectives and purpose of this analysis – including why we have used CiteScore and Google Trends. We have added information on data extraction from Altmetrics Explorer and PlumX. We have also added information on the definition of the publication types (these are defined by PubMed). While most of the component metrics of the EMPIRE Index are altmetrics, article citations are not and we agree that using altmetrics as an umbrella term is therefore inappropriate: we have changed to article-level metrics (ALMs) throughout. We agree that computing an average CiteScore would be inappropriate, because such an average would not indicate the average number of citations per paper across the set of journals and so could be misleading. However, in our analysis we use the median, which is robust in this case since it does not transform the underlying data. We do not believe that it is obvious whether the metrics for a given paper are notably high or low. This can only be understood by understanding the typical metrics of comparable publications, which is the objective of this report. We note in the discussion that the higher scores of migraine publications, in particular social scores, may be linked to higher public interest as evidence by higher Google Trends interest. However, we acknowledge that the analyses presented do not and cannot explain the differences in typical scores seen across therapy areas and publication types. Therefore, any such discussion must necessarily be speculative and as such we keep it to a minimum. The sole purpose of this report is to demonstrate that these variations exist and to explore the magnitude of these variations. The analyses presented are robust and do provide meaningful results that address the objective. We therefore believe that the current, amended report is worthy of publication despite its brief nature. Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 28 Apr 2023 Tomas Rees , Oxford PharmaGenesis, Oxford, OX13 5QJ, UK 28 Apr 2023 Author Response We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this is ... Continue reading We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this is a F1000 Brief Report (https://f1000research.com/for-authors/article-guidelines/brief-report), and so is limited by the journal style template (no ‘Objectives’ section) and word count limit (2,500 words). It follows on from the full publication detailing the background and rationale of the EMPIRE Index published in PLOS ONE, which covers how it was developed and what the component scores signify. The current report is intended to provide some useful additional investigation of the properties of the EMPIRE index and its underlying article-level metrics. It is for these reasons that this report does not include a full literature review and does not go into the details of the EMPIRE Index that are covered in the PLOS ONE paper. We originally referenced the preprint in the current submission (the primary publication had been approved but publication was delayed) but have updated the reference now that the original article is fully published. We have addressed the reviewer’s other comments, as follows: We expanded the introduction to provide greater clarity of the objectives and purpose of this analysis – including why we have used CiteScore and Google Trends. We have added information on data extraction from Altmetrics Explorer and PlumX. We have also added information on the definition of the publication types (these are defined by PubMed). While most of the component metrics of the EMPIRE Index are altmetrics, article citations are not and we agree that using altmetrics as an umbrella term is therefore inappropriate: we have changed to article-level metrics (ALMs) throughout. We agree that computing an average CiteScore would be inappropriate, because such an average would not indicate the average number of citations per paper across the set of journals and so could be misleading. However, in our analysis we use the median, which is robust in this case since it does not transform the underlying data. We do not believe that it is obvious whether the metrics for a given paper are notably high or low. This can only be understood by understanding the typical metrics of comparable publications, which is the objective of this report. We note in the discussion that the higher scores of migraine publications, in particular social scores, may be linked to higher public interest as evidence by higher Google Trends interest. However, we acknowledge that the analyses presented do not and cannot explain the differences in typical scores seen across therapy areas and publication types. Therefore, any such discussion must necessarily be speculative and as such we keep it to a minimum. The sole purpose of this report is to demonstrate that these variations exist and to explore the magnitude of these variations. The analyses presented are robust and do provide meaningful results that address the objective. We therefore believe that the current, amended report is worthy of publication despite its brief nature. We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this is a F1000 Brief Report (https://f1000research.com/for-authors/article-guidelines/brief-report), and so is limited by the journal style template (no ‘Objectives’ section) and word count limit (2,500 words). It follows on from the full publication detailing the background and rationale of the EMPIRE Index published in PLOS ONE, which covers how it was developed and what the component scores signify. The current report is intended to provide some useful additional investigation of the properties of the EMPIRE index and its underlying article-level metrics. It is for these reasons that this report does not include a full literature review and does not go into the details of the EMPIRE Index that are covered in the PLOS ONE paper. We originally referenced the preprint in the current submission (the primary publication had been approved but publication was delayed) but have updated the reference now that the original article is fully published. We have addressed the reviewer’s other comments, as follows: We expanded the introduction to provide greater clarity of the objectives and purpose of this analysis – including why we have used CiteScore and Google Trends. We have added information on data extraction from Altmetrics Explorer and PlumX. We have also added information on the definition of the publication types (these are defined by PubMed). While most of the component metrics of the EMPIRE Index are altmetrics, article citations are not and we agree that using altmetrics as an umbrella term is therefore inappropriate: we have changed to article-level metrics (ALMs) throughout. We agree that computing an average CiteScore would be inappropriate, because such an average would not indicate the average number of citations per paper across the set of journals and so could be misleading. However, in our analysis we use the median, which is robust in this case since it does not transform the underlying data. We do not believe that it is obvious whether the metrics for a given paper are notably high or low. This can only be understood by understanding the typical metrics of comparable publications, which is the objective of this report. We note in the discussion that the higher scores of migraine publications, in particular social scores, may be linked to higher public interest as evidence by higher Google Trends interest. However, we acknowledge that the analyses presented do not and cannot explain the differences in typical scores seen across therapy areas and publication types. Therefore, any such discussion must necessarily be speculative and as such we keep it to a minimum. The sole purpose of this report is to demonstrate that these variations exist and to explore the magnitude of these variations. The analyses presented are robust and do provide meaningful results that address the objective. We therefore believe that the current, amended report is worthy of publication despite its brief nature. Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Comments on this article Comments (0) Version 5 VERSION 5 PUBLISHED 27 Jan 2022 ADD YOUR COMMENT Comment keyboard_arrow_left keyboard_arrow_right Open Peer Review Reviewer Status info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Reviewer Reports Invited Reviewers 1 2 3 Version 5 (revision) 30 Oct 24 read Version 4 (revision) 16 Sep 24 read read Version 3 (revision) 10 Mar 23 read Version 2 (revision) 12 Apr 22 read Version 1 27 Jan 22 read José Luis Ortega , Institute for Advanced Social Studies (IESA-CSIC), Córdoba, Spain Shir Aviv-Reuven , Bar Ilan University, Ramat Gan, Israel Arman Yurisaldi Saleh , Universitas Pembangunan Nasional Veteran Jakarta, Jakarta, Indonesia Comments on this article All Comments (0) Add a comment Sign up for content alerts Sign Up You are now signed up to receive this alert Browse by related subjects keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Saleh A. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 08 Nov 2024 | for Version 5 Arman Yurisaldi Saleh , Universitas Pembangunan Nasional Veteran Jakarta, Jakarta, Special Capital Region of Jakarta, Indonesia 0 Views copyright © 2024 Saleh A. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions The author has answered my feedback well and performed statistical regression, also explaining terms that were previously unclear. Competing Interests No competing interests were disclosed. Reviewer Expertise Neurorestoration, neurobehavior, neuroprotective agent, bibliometric I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) Saleh AY. Peer Review Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.173716.r336422) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/11-107/v5#referee-response-336422 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Aviv-Reuven S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 10 Oct 2024 | for Version 4 Shir Aviv-Reuven , Bar Ilan University, Ramat Gan, Israel 0 Views copyright © 2024 Aviv-Reuven S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions The authors have responded to all my comments and the manuscript, I have no further comments and I now Approve it without revisions. Competing Interests No competing interests were disclosed. I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) Aviv-Reuven S. Peer Review Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.168629.r324069) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/11-107/v4#referee-response-324069 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Saleh A. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 30 Sep 2024 | for Version 4 Arman Yurisaldi Saleh , Universitas Pembangunan Nasional Veteran Jakarta, Jakarta, Special Capital Region of Jakarta, Indonesia 0 Views copyright © 2024 Saleh A. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This article has strengths when viewed from the amount of data used, which is quite a lot in the form of articles, so that it provides a fairly clear picture when analyzed. Actually, this article would be more convincing if the right statistical test was used, but no statistical test was carried out in the manuscript. For example, a statistical test can be added whose results are a picture of the scientific score and social impact score for various articles in the form of systematic reviews, phase 3 clinical trials and observational. The difference in impact scores between the two groups can be analyzed using the t-test in statistical tests So this is the main suggestion for improving the article. In the manuscript, it can be seen that the dependent variables are, for example, the social and scientific impact scores. While the independent variables are public interest, disease indications, number of article citations, type of publication. There are shortcomings in terms of statistical analysis, related to the study of the relationship between the variables above, for example, what if you want to know the relationship between the social and scientific impact scores and public interest, and this is not found in the manuscript. A small point to note is that a linear regression analysis was not included. I kindly suggest that statistical calculations in the form of linear regression be added. I suggest that the author be clearer in explaining Cite Score, Altimetrik and Google Trends. A more detailed description is still needed regarding the 3 terms above so that the reader is clearer. My conclusion is that the author has responded well to the reviewers' input, and I provide suggestions for improvement in terms of the use of statistical tests and more detailed descriptions. The article is worthy of being indexed. Is the work clearly and accurately presented and does it cite the current literature? Yes Is the study design appropriate and is the work technically sound? Yes Are sufficient details of methods and analysis provided to allow replication by others? No If applicable, is the statistical analysis and its interpretation appropriate? No Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly Competing Interests No competing interests were disclosed. Reviewer Expertise Neurorestoration, neurobehavior, neuroprotective agent, bibliometric I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (1) Author Response 30 Oct 2024 Tomas Rees, Oxford PharmaGenesis, Oxford, OX13 5QJ, UK We thank the reviewer for their comments. The initial version of this analysis provided only simple statistical analysis (a non-parametric test of between-group differences) because we were primarily interested in understanding the magnitude of the scores that could be expected for different kind of publications in different disease areas. Furthermore, the data are complicated to analyse because they are zero inflated (many publications have a score of 0), highly skewed (a few publications score very highly) and partially nested (some, but not all, 'phase 3 clinical trials' are also tagged as 'clinical trials', for example). However, we do agree that statistical analysis can provide useful and interesting perspective on the relationship between our independent variables and the EMPIRE Index scores. Therefore, we have provided a mixed model analysis as well as a regression analysis versus Google search interest. The results of these analyses largely accord with our original conclusions, but provide rich detail on these interactions. View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Saleh AY. Peer Review Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.168629.r327843) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/11-107/v4#referee-response-327843 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Aviv-Reuven S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 27 Feb 2024 | for Version 3 Shir Aviv-Reuven , Bar Ilan University, Ramat Gan, Israel 0 Views copyright © 2024 Aviv-Reuven S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This manuscript aims to identify impact differences between medical publications according to diseases and publication types as well as observe variations in public interest by comparing the EMPIRE index with Google search trends. In general I find this paper interesting, but I found it lacking in aspects which the previous reviewer mentioned. Further more, the references to related literature is lacking and there is no discussion related to other possible article level metrics and why the EMPIRE index was selected. Introduction 1. I agree with previous reviewer that Introduction is lacking, not only in presentation of the research problem, but also in the significance of this study. 2. I further agree that Literature review section is missing. it should either be added or the introduction further expanded to address related studies utilizing this indicator as well those referring to analysis of medical publications in general and specifically those who refer to the 12 diseases under analysis in this paper. 3. last paragraph of Introduction should be rephrased as it is unclear, particularly this - "We also look to see if there are similar variations in public interest in therapy areas we explore, with particular interest in comparing this with the EMPIRE Index includes a social score." Methods 4. The authors do not explain why the different types were selected for analysis and what each of these types presents. These should be detailed. 5. The authors do not give detailed the usage of Google Trends. what data was downloaded? was it filtered to specific areas?. Further more, as some of the papers may have been published towards the end of that period, it may have been more appropriate a longer period for the trends. Results 1. The usage of CiteScore is unclear to me, as CiteScore aims to measure journal impact. Discussion The paragraph detailing the findings related to Google trends should be moved to results and a plot showing these trends should be added. Discussion of the results seems short and in some cases state the obvious. for example " Observational studies had the lowest impact, suggesting observational analyses are still generally regarded as having lower interest." A general discussion of possible significance of the findings should be added. Is the work clearly and accurately presented and does it cite the current literature? Partly Is the study design appropriate and is the work technically sound? Partly Are sufficient details of methods and analysis provided to allow replication by others? Partly If applicable, is the statistical analysis and its interpretation appropriate? I cannot comment. A qualified statistician is required. Are all the source data underlying the results available to ensure full reproducibility? Partly Are the conclusions drawn adequately supported by the results? Partly Competing Interests No competing interests were disclosed. Reviewer Expertise Bibliometrics, Scientometrics, academic search, journals impact I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (1) Author Response 28 Jun 2024 Tomas Rees, Oxford PharmaGenesis, Oxford, OX13 5QJ, UK We thank the Dr Aviv-Reuven for their constructive comments and suggestions on our manuscript. We have revised the manuscript accordingly and addressed each comment in detail below. Reviewer comments and responses Introduction 1. I agree with previous reviewer that Introduction is lacking, not only in presentation of the research problem, but also in the significance of this study. Response: We have revised the introduction to provide more background on the research problem and the significance of this study. We have added a paragraph that explains the limitations of conventional metrics such as journal impact factor and CiteScore, and the advantages of using article-level metrics such as altmetrics to assess the impact of medical publications. We have also added a paragraph that describes the EMPIRE Index. We have highlighted the novelty and relevance of our study, which is the first to explore the variations in EMPIRE Index scores across different disease indications and publication types. 2. I further agree that Literature review section is missing. it should either be added or the introduction further expanded to address related studies utilizing this indicator as well those referring to analysis of medical publications in general and specifically those who refer to the 12 diseases under analysis in this paper. Response: We have added a literature review where we discuss the previous studies that have used altmetrics to assess the impact of medical publications in various fields and contexts. 3. last paragraph of Introduction should be rephrased as it is unclear, particularly this - "We also look to see if there are similar variations in public interest in therapy areas we explore, with particular interest in comparing this with the EMPIRE Index includes a social score." Response: We have rephrased the last paragraph of the introduction to clarify our research objectives and hypotheses. Methods 4. The authors do not explain why the different types were selected for analysis and what each of these types presents. These should be detailed. Response: We have added a paragraph in the methods section that explains the rationale for selecting the different publication types for analysis and what each of these types represents. 5. The authors do not give detailed the usage of Google Trends. what data was downloaded? was it filtered to specific areas?. Further more, as some of the papers may have been published towards the end of that period, it may have been more appropriate a longer period for the trends. Response: We have added more details on the usage of Google Trends in the methods section. Results The usage of CiteScore is unclear to me, as CiteScore aims to measure journal impact. Response: We have clarified the rationale for the comparison with of CiteScore in the introduction. Discussion The paragraph detailing the findings related to Google trends should be moved to results and a plot showing these trends should be added. Response: We have clarified that the data are presented in Table 1 as averages from 2019. We have not included a plot of the weekly data for 2019 as we do not refer to these and they are not relevant to our discussion. However, they are available in the supplementary material in excel form. Discussion of the results seems short and in some cases state the obvious. for example " Observational studies had the lowest impact, suggesting observational analyses are still generally regarded as having lower interest." Response: We have clarified the value of observational studies. A general discussion of possible significance of the findings should be added. Response: We have added a discussion of potential significance, as well as expanding the limitations. View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Aviv-Reuven S. Peer Review Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.145010.r234323) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/11-107/v3#referee-response-234323 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2022 Ortega J. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 19 Apr 2022 | for Version 2 José Luis Ortega , Institute for Advanced Social Studies (IESA-CSIC), Córdoba, Spain 0 Views copyright © 2022 Ortega J. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Not Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions The text is the same as the version 1. Only references were updated. Competing Interests No competing interests were disclosed. Reviewer Expertise Bibliometrics, altmetrics, academic search engines, scholarly social networks I confirm that I have read this submission and believe that I have an appropriate level of expertise to state that I do not consider it to be of an acceptable scientific standard, for reasons outlined above. reply Respond to this report Responses (1) Author Response 27 Mar 2023 Tomas Rees, Oxford PharmaGenesis, Oxford, OX13 5QJ, UK We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this was intended as a brief report. It follows on from the full publication detailing the background and rationale of the EMPIRE Index published in PLOS ONE, which covers how it was developed and what the component scores signify. The current report provides some useful additional investigation of the properties of the EMPIRE index and its underlying article-level metrics. It is for these reasons that this report does not include a full literature review and does not go into the details of the EMPIRE Index that are covered in the PLOS ONE paper. We originally referenced the preprint in the current submission (the primary publication had been approved but publication was delayed) but have updated the reference now that the original article is fully published. We have addressed the reviewer’s other comments, as follows: We expanded the introduction to provide greater clarity of the objectives and purpose of this analysis – including why we have used CiteScore and Google Trends. We have added information on data extraction from Altmetrics Explorer and PlumX. We have also added information on the definition of the publication types (these are defined by PubMed). While most of the component metrics of the EMPIRE Index are altmetrics, article citations are not and we agree that using altmetrics as an umbrella term is therefore inappropriate: we have changed to article-level metrics (ALMs) throughout. We agree that computing an average CiteScore would be inappropriate, because such an average would not indicate the average number of citations per paper across the set of journals and so could be misleading. However, in our analysis we use the median, which is robust in this case since it does not transform the underlying data. We do not believe that it is obvious whether the metrics for a given paper are notably high or low. This can only be understood by understanding the typical metrics of comparable publications, which is the objective of this report. We note in the discussion that the higher scores of migraine publications, in particular social scores, may be linked to higher public interest as evidence by higher Google Trends interest. However, we acknowledge that the analyses presented do not and cannot explain the differences in typical scores seen across therapy areas and publication types. Therefore, any such discussion must necessarily be speculative and as such we keep it to a minimum. The sole purpose of this report is to demonstrate that these variations exist and to explore the magnitude of these variations. The analyses presented are robust and do provide meaningful results that address the objective. We therefore believe that the current, amended report is worthy of publication despite its brief nature. View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Ortega JL. Peer Review Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.126135.r134659) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/11-107/v2#referee-response-134659 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2022 Ortega J. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 10 Feb 2022 | for Version 1 José Luis Ortega , Institute for Advanced Social Studies (IESA-CSIC), Córdoba, Spain 0 Views copyright © 2022 Ortega J. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Not Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This manuscript attempts to find impact differences between medical publications according to diseases and publication types. These differences are calculated according to a new indicator, EMPIRE Index. More than 20k publications were retrieved from PubMed about 12 diseases areas. The resulting publications were also grouped by publication type. Using the Kruskal-Wallis test, authors found that the EMPIRE Indicator produces some differences between publications according to diseases and type of documents. These results allow them to justify the utility of this indicator and its power to identify different types of impacts. However, the paper presents several weaknesses that impede my recommendation for approval. Next, I detail some problem points by section: Introduction The introduction is poor because it does not introduce the research problem properly. An introduction should explain why the research is necessary and what the problem is that the paper wants to address. Why is important to know that impact varies by disease or by publication type? Why is an indicator such as EMPIRE Index necessary? Authors confound article-level with altmetrics. Many altmetric measures could be article-level metrics, but there are other article-level metrics (i.e. citations) that are not altmetrics. Author should revise the text and correct these statements. I do not see a key section in a scientific article: a literature review. Authors should include a section where they contextualize the research, explaining previous studies that had deal with this problem. Studies about new indicators, differences in the impact by subject or document type, different types of impacts, etc. Overall, authors should present other studies in the introduction that show why is important to know that a disease has more social or scholarly impact, or why some document types attract more citations or tweets. In short, more conceptual background. The paper includes only eight references, which illustrates the poor context of this research. Objective I think this section is the most important in a paper and it is missing in this manuscript. If there are not objectives, there are not research goals and a paper lacks sense. The final line of Introduction says: “Here, we present an exploratory investigation of whether disease indications and publication types influence the average EMPIRE Index scores.” Why do you want to know this? Why is important to know this? Authors should introduce a section for objectives where they explain the main objectives and secondary ones. It could be interesting to include some research questions that specify the aim of the paper. Methods Authors should include more detail about the data extraction process in Altmetric and PlumX. Authors claim that they used Altmetric Explorer to retrieve 20k publications. How did you exactly retrieve the data from this site? Do you use the API endpoint? How do you obtain the data from PlumX? A key element in the paper is to test differences according to diseases and document types, but there is not any explanation about these typologies. For example, and about document types, what is the difference between “review” and “systematic review”? This part is key because the definition of groups and types influences the results, so it is very important to define the diseases and the typology of documents. All the analysis relies on a new indicator, EMPIRE Index. In spite of the importance of this indicator in the paper, there is not any information about how it is calculated and conceptualized. We have to read a non-peer reviewed pre-print deposited in a repository to know what it is. I do not see this way to do science as correct. Authors have to explain how this indicator is calculated in this paper; how the three component scores are defined; what is the difference between “social impact” and “societal impact”, mainly from a conceptual point of view; likewise how do you find three components and not two or four; which metrics take part in each component and why? CiteScore is an indicator for journals, not for articles. How it is used and why? The same for Google Trends - how they are used in the study, and why? Results Tables 3 and 5 use CiteScore median for valuing articles by diseases. I recall that it is not correct to use journal indicators for research articles, and the use of median or mean from a ratio (CiteScore is the ratio of citations by publications) is a mathematical artifact, and the results are spurious. Discussion This paper lacks of any substantial discussion about the results. Sometimes the interpretation is obvious and it does not contribute valuable information. For example, the first paragraph ends: “For example, these findings can be used to help to understand whether a particular publication has notably high (or low) metrics.” This is obvious and It is not necessary to do this study to reach that conclusion. Every publication has high or low metrics. Another example: “Observational studies had the lowest impact, suggesting observational analyses are still generally regarded as having lower interest.” If they have low impact, then they have low interest. It is interesting that the results from the EMPIRE index are not interpreted. For example, migraine, MS and asthma are the diseases with more social impact. Why? Is this positive or negative? Is there any evidence in other studies that confirm that these pathologies are more interesting for public audiences? For example, is there any correlation between these scores and Google Trends? Is there any survey that explore the public opinion about diseases? Etc. This part should include the interpretation of the results according to the previous knowledge. It is interesting that this section only includes one citation (about using PubMed), but there is not any reference to previous results to discuss the validity of the findings or to explain the meaning of the results. For example, “The high impact of phase 3 clinical trials is to be expected, given that they are intended to provide practice-changing information.” Why is it that papers which provide practice-changing information should have more impact? Is the work clearly and accurately presented and does it cite the current literature? Partly Is the study design appropriate and is the work technically sound? Partly Are sufficient details of methods and analysis provided to allow replication by others? No If applicable, is the statistical analysis and its interpretation appropriate? Yes Are all the source data underlying the results available to ensure full reproducibility? Partly Are the conclusions drawn adequately supported by the results? No Competing Interests No competing interests were disclosed. Reviewer Expertise Bibliometrics, altmetrics, academic search engines, scholarly social networks I confirm that I have read this submission and believe that I have an appropriate level of expertise to state that I do not consider it to be of an acceptable scientific standard, for reasons outlined above. reply Respond to this report Responses (1) Author Response 28 Apr 2023 Tomas Rees, Oxford PharmaGenesis, Oxford, OX13 5QJ, UK We thank the reviewer for the commentary and perspective, and we have made numerous clarifications and additions to the manuscript to address these. One important caveat is that this is a F1000 Brief Report (https://f1000research.com/for-authors/article-guidelines/brief-report), and so is limited by the journal style template (no ‘Objectives’ section) and word count limit (2,500 words). It follows on from the full publication detailing the background and rationale of the EMPIRE Index published in PLOS ONE, which covers how it was developed and what the component scores signify. The current report is intended to provide some useful additional investigation of the properties of the EMPIRE index and its underlying article-level metrics. It is for these reasons that this report does not include a full literature review and does not go into the details of the EMPIRE Index that are covered in the PLOS ONE paper. We originally referenced the preprint in the current submission (the primary publication had been approved but publication was delayed) but have updated the reference now that the original article is fully published. We have addressed the reviewer’s other comments, as follows: We expanded the introduction to provide greater clarity of the objectives and purpose of this analysis – including why we have used CiteScore and Google Trends. We have added information on data extraction from Altmetrics Explorer and PlumX. We have also added information on the definition of the publication types (these are defined by PubMed). While most of the component metrics of the EMPIRE Index are altmetrics, article citations are not and we agree that using altmetrics as an umbrella term is therefore inappropriate: we have changed to article-level metrics (ALMs) throughout. We agree that computing an average CiteScore would be inappropriate, because such an average would not indicate the average number of citations per paper across the set of journals and so could be misleading. However, in our analysis we use the median, which is robust in this case since it does not transform the underlying data. We do not believe that it is obvious whether the metrics for a given paper are notably high or low. This can only be understood by understanding the typical metrics of comparable publications, which is the objective of this report. We note in the discussion that the higher scores of migraine publications, in particular social scores, may be linked to higher public interest as evidence by higher Google Trends interest. However, we acknowledge that the analyses presented do not and cannot explain the differences in typical scores seen across therapy areas and publication types. Therefore, any such discussion must necessarily be speculative and as such we keep it to a minimum. The sole purpose of this report is to demonstrate that these variations exist and to explore the magnitude of these variations. The analyses presented are robust and do provide meaningful results that address the objective. We therefore believe that the current, amended report is worthy of publication despite its brief nature. View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Ortega JL. Peer Review Report For: Does the impact of medical publications vary by disease indication and publication type? An exploration using a novel, value-based, publication metric framework: the EMPIRE Index [version 5; peer review: 2 approved, 1 not approved] . F1000Research 2024, 11 :107 ( https://doi.org/10.5256/f1000research.79727.r121444) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/11-107/v1#referee-response-121444 Alongside their report, reviewers assign a status to the article: Approved - the paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations - A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved - fundamental flaws in the paper seriously undermine the findings and conclusions Adjust parameters to alter display View on desktop for interactive features Includes Interactive Elements View on desktop for interactive features Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Stay Updated Sign up for content alerts and receive a weekly or monthly email with all newly published articles Register with F1000Research Already registered? Sign in Not now, thanks close PLEASE NOTE If you are an AUTHOR of this article, please check that you signed in with the account associated with this article otherwise we cannot automatically identify your role as an author and your comment will be labelled as a “User Comment”. If you are a REVIEWER of this article, please check that you have signed in with the account associated with this article and then go to your account to submit your report, please do not post your review here. If you do not have access to your original account, please contact us . All commenters must hold a formal affiliation as per our Policies . The information that you give us will be displayed next to your comment. User comments must be in English, comprehensible and relevant to the article under discussion. We reserve the right to remove any comments that we consider to be inappropriate, offensive or otherwise in breach of the User Comment Terms and Conditions . Commenters must not use a comment for personal attacks. When criticisms of the article are based on unpublished data, the data should be made available. I accept the User Comment Terms and Conditions Please confirm that you accept the User Comment Terms and Conditions. Affiliation ✕ refresh Please enter your institution. Note: To add your institution or organisation, start typing the name and then select the correct name from the list. Where applicable, the name will appear in both the original language and in English. Do not paste in the name. If the name does not appear in the drop-down list, we will display the information you have entered. ✕ refresh Country/Region * USA UK Canada China France Germany Afghanistan Aland Islands Albania Algeria American Samoa Andorra Angola Anguilla Antarctica Antigua and Barbuda Argentina Armenia Aruba Australia Austria Azerbaijan Bahamas Bahrain Bangladesh Barbados Belarus Belgium Belize Benin Bermuda Bhutan Bolivia Bosnia and Herzegovina Botswana Bouvet Island Brazil British Indian Ocean Territory British Virgin Islands Brunei Bulgaria Burkina Faso Burundi Cambodia Cameroon Canada Cape Verde Cayman Islands Central African Republic Chad Chile China Christmas Island Cocos (Keeling) Islands Colombia Comoros Congo Cook Islands Costa Rica Cote d'Ivoire Croatia Cuba Cyprus Czech Republic Democratic Republic of the Congo Denmark Djibouti Dominica Dominican Republic Ecuador Egypt El Salvador Equatorial Guinea Eritrea Estonia Ethiopia Falkland Islands Faroe Islands Federated States of Micronesia Fiji Finland France French Guiana French Polynesia French Southern Territories Gabon Georgia Germany Ghana Gibraltar Greece Greenland Grenada Guadeloupe Guam Guatemala Guernsey Guinea Guinea-Bissau Guyana Haiti Heard Island and Mcdonald Islands Holy See (Vatican City State) Honduras Hong Kong Hungary Iceland India Indonesia Iran Iraq Ireland Israel Italy Jamaica Japan Jersey Jordan Kazakhstan Kenya Kiribati Kosovo (Serbia and Montenegro) Kuwait Kyrgyzstan Lao People's Democratic Republic Latvia Lebanon Lesotho Liberia Libya Liechtenstein Lithuania Luxembourg Macao Madagascar Malawi Malaysia Maldives Mali Malta Marshall Islands Martinique Mauritania Mauritius Mayotte Mexico Minor Outlying Islands of the United States Moldova Monaco Mongolia Montenegro Montserrat Morocco Mozambique Myanmar Namibia Nauru Nepal Netherlands Antilles New Caledonia New Zealand Nicaragua Niger Nigeria Niue Norfolk Island North Korea North Macedonia Northern Mariana Islands Norway Oman Pakistan Palau Palestinian Territory Panama Papua New Guinea Paraguay Peru Philippines Pitcairn Poland Portugal Puerto Rico Qatar Reunion Romania Russian Federation Rwanda Saint Helena Saint Kitts and Nevis Saint Lucia Saint Pierre and Miquelon Saint Vincent and the Grenadines Samoa San Marino Sao Tome and Principe Saudi Arabia Senegal Serbia Seychelles Sierra Leone Singapore Slovakia Slovenia Solomon Islands Somalia South Africa South Georgia and the South Sandwich Is South Korea South Sudan Spain Sri Lanka Sudan Suriname Svalbard and Jan Mayen Swaziland Sweden Switzerland Syria Taiwan Tajikistan Tanzania Thailand The Gambia The Netherlands Timor-Leste Togo Tokelau Tonga Trinidad and Tobago Tunisia Turkey Turkmenistan Turks and Caicos Islands Tuvalu UK USA Uganda Ukraine United Arab Emirates United States Virgin Islands Uruguay Uzbekistan Vanuatu Venezuela Vietnam Wallis and Futuna West Bank and Gaza Strip Western Sahara Yemen Zambia Zimbabwe Please select your country/region. You must enter a comment. Competing Interests Please disclose any competing interests that might be construed to influence your judgment of the article's or peer review report's validity or importance. Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Please state your competing interests The comment has been saved. An error has occurred. Please try again. Cancel Post var lTitle = "Does the impact of medical publications vary...".replace("'", ''); var linkedInUrl = "http://www.linkedin.com/shareArticle?url=https://f1000research.com/articles/11-107/v5" + "&title=" + encodeURIComponent(lTitle) + "&summary=" + encodeURIComponent('Read the article by '); var deliciousUrl = "https://del.icio.us/post?url=https://f1000research.com/articles/11-107/v5&title=" + encodeURIComponent(lTitle); var redditUrl = "http://reddit.com/submit?url=https://f1000research.com/articles/11-107/v5" + "&title=" + encodeURIComponent(lTitle); linkedInUrl += encodeURIComponent('Rees T and Pal A'); var offsetTop = /chrome/i.test( navigator.userAgent ) ? 4 : -10; var addthis_config = { ui_offset_top: offsetTop, services_compact : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_expanded : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_custom : [ { name: "LinkedIn", url: linkedInUrl, icon:"/img/icon/at_linkedin.svg" }, { name: "Mendeley", url: "http://www.mendeley.com/import/?url=https://f1000research.com/articles/11-107/v5/mendeley", icon:"/img/icon/at_mendeley.svg" }, { name: "Reddit", url: redditUrl, icon:"/img/icon/at_reddit.svg" }, ] }; var addthis_share = { url: "https://f1000research.com/articles/11-107", templates : { twitter : "Does the impact of medical publications vary by disease indication.... Rees T and Pal A, published by " + "@F1000Research" + ", https://f1000research.com/articles/11-107/v5" } }; if (typeof(addthis) != "undefined"){ addthis.addEventListener('addthis.ready', checkCount); addthis.addEventListener('addthis.menu.share', checkCount); } $(".f1r-shares-twitter").attr("href", "https://twitter.com/intent/tweet?text=" + addthis_share.templates.twitter); $(".f1r-shares-facebook").attr("href", "https://www.facebook.com/sharer/sharer.php?u=" + addthis_share.url); $(".f1r-shares-linkedin").attr("href", addthis_config.services_custom[0].url); $(".f1r-shares-reddit").attr("href", addthis_config.services_custom[2].url); $(".f1r-shares-mendelay").attr("href", addthis_config.services_custom[1].url); function checkCount(){ setTimeout(function(){ $(".addthis_button_expanded").each(function(){ var count = $(this).text(); if (count !== "" && count != "0") $(this).removeClass("is-hidden"); else $(this).addClass("is-hidden"); }); }, 1000); } close How to cite this report {{reportCitation}} Cancel Copy Citation Details $(function(){R.ui.buttonDropdowns('.dropdown-for-downloads');}); $(function(){R.ui.toolbarDropdowns('.toolbar-dropdown-for-downloads');}); $.get("/articles/acj/75805/173716") new F1000.Clipboard(); new F1000.ThesaurusTermsDisplay("articles", "article", "173716"); $(document).ready(function() { $( "#frame1" ).on('load', function() { var mydiv = $(this).contents().find("div"); var h = mydiv.height(); console.log(h) }); var tooltipLivingFigure = jQuery(".interactive-living-figure-label .icon-more-info"), titleLivingFigure = tooltipLivingFigure.attr("title"); tooltipLivingFigure.simpletip({ fixed: true, position: ["-115", "30"], baseClass: 'small-tooltip', content:titleLivingFigure + " " }); tooltipLivingFigure.removeAttr("title"); $("body").on("click", ".cite-living-figure", function(e) { e.preventDefault(); var ref = $(this).attr("data-ref"); $(this).closest(".living-figure-list-container").find("#" + ref).fadeIn(200); }); $("body").on("click", ".close-cite-living-figure", function(e) { e.preventDefault(); $(this).closest(".popup-window-wrapper").fadeOut(200); }); $(document).on("mouseup", function(e) { var metricsContainer = $(".article-metrics-popover-wrapper"); if (!metricsContainer.is(e.target) && metricsContainer.has(e.target).length === 0) { $(".article-metrics-close-button").click(); } }); var articleId = $('#articleId').val(); if($("#main-article-count-box").attachArticleMetrics) { $("#main-article-count-box").attachArticleMetrics(articleId, { articleMetricsView: true }); } }); var figshareWidget = $(".new_figshare_widget"); if (figshareWidget.length > 0) { window.figshare.load("f1000", function(Widget) { // Select a tag/tags defined in your page. In this tag we will place the widget. _.map(figshareWidget, function(el){ var widget = new Widget({ articleId: $(el).attr("figshare_articleId") //height:300 // this is the height of the viewer part. [Default: 550] }); widget.initialize(); // initialize the widget widget.mount(el); // mount it in a tag that's on your page // this will save the widget on the global scope for later use from // your JS scripts. This line is optional. //window.widget = widget; }); }); } close Error Close Add Reset F1000.MICROSERVICES.AFFILIATION = ''; $(document).ready(function () { $('.js-affiliations-form').each((index, form) => { new AffiliationForm({ formId: form.id, institutionErrorSelector: '.comment-enter-institution', departmentErrorSelector: '.comment-enter-department', placeSelector: '.js-add-comment-place', stateSelector: '.js-add-comment-state', zipCodeSelector: '.js-add-comment-zipcode', countrySelector: '.js-add-comment-country', countryErrorSelector: '.comment-enter-country', }); }); }); $(document).ready(function () { var reportIds = { "134659": 21, "171142": 0, "264084": 0, "264085": 0, "264086": 0, "264087": 0, "166038": 0, "264082": 0, "264083": 0, "264088": 0, "264089": 0, "264090": 0, "264091": 0, "336421": 0, "327845": 0, "327844": 0, "336423": 0, "327847": 0, "336422": 6, "327846": 0, "327841": 0, "327840": 0, "327843": 16, "327842": 0, "244527": 0, "327849": 0, "244526": 0, "327848": 0, "244524": 0, "244531": 0, "244529": 0, "244535": 0, "244533": 0, "168372": 0, "168373": 0, "244539": 0, "244537": 0, "244541": 0, "239563": 0, "234315": 0, "239562": 0, "234314": 0, "239561": 0, "239567": 0, "234319": 0, "239566": 0, "234318": 0, "239565": 0, "234317": 0, "239564": 0, "234316": 0, "234323": 29, "239570": 0, "234322": 0, "239569": 0, "234321": 0, "239568": 0, "234320": 0, "324956": 0, "324957": 0, "324958": 0, "324959": 0, "324952": 0, "324953": 0, "324954": 0, "324955": 0, "121441": 0, "121440": 0, "324069": 7, "121443": 0, "324070": 0, "121442": 0, "135142": 0, "324960": 0, "135143": 0, "121444": 40, "324961": 0, "135141": 0, }; $(".referee-response-container,.js-referee-report").each(function(index, el) { var reportId = $(el).attr("data-reportid"), reportCount = reportIds[reportId] || 0; $(el).find(".comments-count-container,.js-referee-report-views").html(reportCount); }); var uuidInput = $("#article_uuid"), oldUUId = uuidInput.val(), newUUId = "c91bbee6-71cb-407f-8159-9cdfdfe86101"; uuidInput.val(newUUId); $("a[href*='article_uuid=']").each(function(index, el) { var newHref = $(el).attr("href").replace(oldUUId, newUUId); $(el).attr("href", newHref); }); }); An innovative open access publishing platform offering rapid publication and open peer review, whilst supporting data deposition and sharing. Browse Gateways Collections How it Works Contact For Developers Cookie Notice Privacy Notice RSS Submit Your Research Follow us © 2012-2026 F1000 Research Ltd. ISSN 2046-1402 | Legal | Partner of Research4Life • CrossRef • ORCID • FAIRSharing R.templateTests.simpleTemplate = R.template(' $text $text $text $text $text '); R.templateTests.runTests(); var F1000platform = new F1000.Platform({ name: "f1000research", displayName: "F1000Research", hostName: "f1000research.com", id: "1", editorialEmail: "[email protected]", infoEmail: "[email protected]", usePmcStats: true }); $(function(){R.ui.dropdowns('.dropdown-for-authors, .dropdown-for-about, .dropdown-for-myresearch');}); // $(function(){R.ui.dropdowns('.dropdown-for-referees');}); $(document).ready(function () { if ($(".cookie-warning").is(":visible")) { $(".sticky").css("margin-bottom", "35px"); $(".devices").addClass("devices-and-cookie-warning"); } $(".cookie-warning .close-button").click(function (e) { $(".devices").removeClass("devices-and-cookie-warning"); $(".sticky").css("margin-bottom", "0"); }); $("#tweeter-feed .tweet-message").each(function (i, message) { var self = $(message); self.html(linkify(self.html())); }); $(".partner").on("mouseenter mouseleave", function() { $(this).find(".gray-scale, .colour").toggleClass("is-hidden"); }); }); Sign In Remember me Forgotten your password? Sign In Cancel Email or password not correct. Please try again Please wait... $(function(){ // Note: All the setup needs to run against a name attribute and *not* the id due the clonish // nature of facebox... $("a[id=googleSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("GOOGLE"); $("form[id=oAuthForm]").submit(); }); $("a[id=facebookSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("FACEBOOK"); $("form[id=oAuthForm]").submit(); }); $("a[id=orcidSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("ORCID"); $("form[id=oAuthForm]").submit(); }); }); If you've forgotten your password, please enter your email address below and we'll send you instructions on how to reset your password. The email address should be the one you originally registered with F1000. Email address not valid, please try again You registered with F1000 via Google, so we cannot reset your password. To sign in, please click here . If you still need help with your Google account password, please click here . You registered with F1000 via Facebook, so we cannot reset your password. To sign in, please click here . If you still need help with your Facebook account password, please click here . Code not correct, please try again Reset password Cancel Email us for further assistance. Server error, please try again. If your email address is registered with us, we will email you instructions to reset your password. If you think you should have received this email but it has not arrived, please check your spam filters and/or contact for further assistance. Please wait... Register $(document).ready(function () { signIn.createSignInAsRow($("#sign-in-form-gfb-popup")); $(".target-field").each(function () { var uris = $(this).val().split("/"); if (uris.pop() === "login") { $(this).val(uris.toString().replace(",","/")); } }); });