A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code

A libraries reproducibility hackathon: connecting... | F1000Research "use strict";function _typeof(t){return(_typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}!function(){var t=function(){var t,e,o=[],n=window,r=n;for(;r;){try{if(r.frames.__tcfapiLocator){t=r;break}}catch(t){}if(r===n.top)break;r=r.parent}t||(!function t(){var e=n.document,o=!!n.frames.__tcfapiLocator;if(!o)if(e.body){var r=e.createElement("iframe");r.style.cssText="display:none",r.name="__tcfapiLocator",e.body.appendChild(r)}else setTimeout(t,5);return!o}(),n.__tcfapi=function(){for(var t=arguments.length,n=new Array(t),r=0;r 3&&2===parseInt(n[1],10)&&"boolean"==typeof n[3]&&(e=n[3],"function"==typeof n[2]&&n[2]("set",!0)):"ping"===n[0]?"function"==typeof n[2]&&n[2]({gdprApplies:e,cmpLoaded:!1,cmpStatus:"stub"}):o.push(n)},n.addEventListener("message",(function(t){var e="string"==typeof t.data,o={};if(e)try{o=JSON.parse(t.data)}catch(t){}else o=t.data;var n="object"===_typeof(o)&&null!==o?o.__tcfapiCall:null;n&&window.__tcfapi(n.command,n.version,(function(o,r){var a={__tcfapiReturn:{returnValue:o,success:r,callId:n.callId}};t&&t.source&&t.source.postMessage&&t.source.postMessage(e?JSON.stringify(a):a,"*")}),n.parameter)}),!1))};"undefined"!=typeof module?module.exports=t:t()}(); dataLayer = dataLayer || []; // Standard GTM initialization - Google Consent Mode handles consent automatically (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl+ '>m_auth=hzk0Vc3qFsQYhCrIoHz68A>m_preview=env-1>m_cookies_win=x';f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-MWFK8L5J'); ;window.NREUM||(NREUM={});NREUM.init={distributed_tracing:{enabled:true},privacy:{cookies_enabled:true},ajax:{deny_list:["bam.nr-data.net"]}}; ;NREUM.loader_config={accountID:"438030",trustKey:"438030",agentID:"772317073",licenseKey:"97f8f67f26",applicationID:"772317073"} ;NREUM.info={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net",licenseKey:"97f8f67f26",applicationID:"772317073",sa:1} ;/*! For license information please see nr-loader-spa-1.236.0.min.js.LICENSE.txt */ (()=>{"use strict";var e,t,r={5763:(e,t,r)=>{r.d(t,{P_:()=>l,Mt:()=>g,C5:()=>s,DL:()=>v,OP:()=>T,lF:()=>D,Yu:()=>y,Dg:()=>h,CX:()=>c,GE:()=>b,sU:()=>_});var n=r(8632),i=r(9567);const o={beacon:n.ce.beacon,errorBeacon:n.ce.errorBeacon,licenseKey:void 0,applicationID:void 0,sa:void 0,queueTime:void 0,applicationTime:void 0,ttGuid:void 0,user:void 0,account:void 0,product:void 0,extra:void 0,jsAttributes:{},userAttributes:void 0,atts:void 0,transactionName:void 0,tNamePlain:void 0},a={};function s(e){if(!e)throw new Error("All info objects require an agent identifier!");if(!a[e])throw new Error("Info for ".concat(e," was never set"));return a[e]}function c(e,t){if(!e)throw new Error("All info objects require an agent identifier!");a[e]=(0,i.D)(t,o),(0,n.Qy)(e,a[e],"info")}var u=r(7056);const d=()=>{const e={blockSelector:"[data-nr-block]",maskInputOptions:{password:!0}};return{allow_bfcache:!0,privacy:{cookies_enabled:!0},ajax:{deny_list:void 0,enabled:!0,harvestTimeSeconds:10},distributed_tracing:{enabled:void 0,exclude_newrelic_header:void 0,cors_use_newrelic_header:void 0,cors_use_tracecontext_headers:void 0,allowed_origins:void 0},session:{domain:void 0,expiresMs:u.oD,inactiveMs:u.Hb},ssl:void 0,obfuscate:void 0,jserrors:{enabled:!0,harvestTimeSeconds:10},metrics:{enabled:!0},page_action:{enabled:!0,harvestTimeSeconds:30},page_view_event:{enabled:!0},page_view_timing:{enabled:!0,harvestTimeSeconds:30,long_task:!1},session_trace:{enabled:!0,harvestTimeSeconds:10},harvest:{tooManyRequestsDelay:60},session_replay:{enabled:!1,harvestTimeSeconds:60,sampleRate:.1,errorSampleRate:.1,maskTextSelector:"*",maskAllInputs:!0,get blockClass(){return"nr-block"},get ignoreClass(){return"nr-ignore"},get maskTextClass(){return"nr-mask"},get blockSelector(){return e.blockSelector},set blockSelector(t){e.blockSelector+=",".concat(t)},get maskInputOptions(){return e.maskInputOptions},set maskInputOptions(t){e.maskInputOptions={...t,password:!0}}},spa:{enabled:!0,harvestTimeSeconds:10}}},f={};function l(e){if(!e)throw new Error("All configuration objects require an agent identifier!");if(!f[e])throw new Error("Configuration for ".concat(e," was never set"));return f[e]}function h(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");f[e]=(0,i.D)(t,d()),(0,n.Qy)(e,f[e],"config")}function g(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");var r=l(e);if(r){for(var n=t.split("."),i=0;i {r.d(t,{D:()=>i});var n=r(50);function i(e,t){try{if(!e||"object"!=typeof e)return(0,n.Z)("Setting a Configurable requires an object as input");if(!t||"object"!=typeof t)return(0,n.Z)("Setting a Configurable requires a model to set its initial properties");const r=Object.create(Object.getPrototypeOf(t),Object.getOwnPropertyDescriptors(t)),o=0===Object.keys(r).length?e:r;for(let a in o)if(void 0!==e[a])try{"object"==typeof e[a]&&"object"==typeof t[a]?r[a]=i(e[a],t[a]):r[a]=e[a]}catch(e){(0,n.Z)("An error occurred while setting a property of a Configurable",e)}return r}catch(e){(0,n.Z)("An error occured while setting a Configurable",e)}}},6818:(e,t,r)=>{r.d(t,{Re:()=>i,gF:()=>o,q4:()=>n});const n="1.236.0",i="PROD",o="CDN"},385:(e,t,r)=>{r.d(t,{FN:()=>a,IF:()=>u,Nk:()=>f,Tt:()=>s,_A:()=>o,il:()=>n,pL:()=>c,v6:()=>i,w1:()=>d});const n="undefined"!=typeof window&&!!window.document,i="undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self.navigator instanceof WorkerNavigator||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis.navigator instanceof WorkerNavigator),o=n?window:"undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis),a=""+o?.location,s=/iPad|iPhone|iPod/.test(navigator.userAgent),c=s&&"undefined"==typeof SharedWorker,u=(()=>{const e=navigator.userAgent.match(/Firefox[/\s](\d+\.\d+)/);return Array.isArray(e)&&e.length>=2?+e[1]:0})(),d=Boolean(n&&window.document.documentMode),f=!!navigator.sendBeacon},1117:(e,t,r)=>{r.d(t,{w:()=>o});var n=r(50);const i={agentIdentifier:"",ee:void 0};class o{constructor(e){try{if("object"!=typeof e)return(0,n.Z)("shared context requires an object as input");this.sharedContext={},Object.assign(this.sharedContext,i),Object.entries(e).forEach((e=>{let[t,r]=e;Object.keys(i).includes(t)&&(this.sharedContext[t]=r)}))}catch(e){(0,n.Z)("An error occured while setting SharedContext",e)}}}},8e3:(e,t,r)=>{r.d(t,{L:()=>d,R:()=>c});var n=r(2177),i=r(1284),o=r(4322),a=r(3325);const s={};function c(e,t){const r={staged:!1,priority:a.p[t]||0};u(e),s[e].get(t)||s[e].set(t,r)}function u(e){e&&(s[e]||(s[e]=new Map))}function d(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"",t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"feature";if(u(e),!e||!s[e].get(t))return a(t);s[e].get(t).staged=!0;const r=[...s[e]];function a(t){const r=e?n.ee.get(e):n.ee,a=o.X.handlers;if(r.backlog&&a){var s=r.backlog[t],c=a[t];if(c){for(var u=0;s&&u {let[t,r]=e;return r.staged}))&&(r.sort(((e,t)=>e[1].priority-t[1].priority)),r.forEach((e=>{let[t]=e;a(t)})))}function f(e,t){var r=e[1];(0,i.D)(t[r],(function(t,r){var n=e[0];if(r[0]===n){var i=r[1],o=e[3],a=e[2];i.apply(o,a)}}))}},2177:(e,t,r)=>{r.d(t,{c:()=>f,ee:()=>u});var n=r(8632),i=r(2210),o=r(1284),a=r(5763),s="nr@context";let c=(0,n.fP)();var u;function d(){}function f(e){return(0,i.X)(e,s,l)}function l(){return new d}function h(){u.aborted=!0,u.backlog={}}c.ee?u=c.ee:(u=function e(t,r){var n={},c={},f={},g=!1;try{g=16===r.length&&(0,a.OP)(r).isolatedBacklog}catch(e){}var p={on:b,addEventListener:b,removeEventListener:y,emit:v,get:x,listeners:w,context:m,buffer:A,abort:h,aborted:!1,isBuffering:E,debugId:r,backlog:g?{}:t&&"object"==typeof t.backlog?t.backlog:{}};return p;function m(e){return e&&e instanceof d?e:e?(0,i.X)(e,s,l):l()}function v(e,r,n,i,o){if(!1!==o&&(o=!0),!u.aborted||i){t&&o&&t.emit(e,r,n);for(var a=m(n),s=w(e),d=s.length,f=0;fn,p:()=>i});var n=r(2177).ee.get("handle");function i(e,t,r,i,o){o?(o.buffer([e],i),o.emit(e,t,r)):(n.buffer([e],i),n.emit(e,t,r))}},4322:(e,t,r)=>{r.d(t,{X:()=>o});var n=r(5546);o.on=a;var i=o.handlers={};function o(e,t,r,o){a(o||n.E,i,e,t,r)}function a(e,t,r,i,o){o||(o="feature"),e||(e=n.E);var a=t[o]=t[o]||{};(a[r]=a[r]||[]).push([e,i])}},3239:(e,t,r)=>{r.d(t,{bP:()=>s,iz:()=>c,m$:()=>a});var n=r(385);let i=!1,o=!1;try{const e={get passive(){return i=!0,!1},get signal(){return o=!0,!1}};n._A.addEventListener("test",null,e),n._A.removeEventListener("test",null,e)}catch(e){}function a(e,t){return i||o?{capture:!!e,passive:i,signal:t}:!!e}function s(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;window.addEventListener(e,t,a(r,n))}function c(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;document.addEventListener(e,t,a(r,n))}},4402:(e,t,r)=>{r.d(t,{Ht:()=>u,M:()=>c,Rl:()=>a,ky:()=>s});var n=r(385);const i="xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx";function o(e,t){return e?15&e[t]:16*Math.random()|0}function a(){const e=n._A?.crypto||n._A?.msCrypto;let t,r=0;return e&&e.getRandomValues&&(t=e.getRandomValues(new Uint8Array(31))),i.split("").map((e=>"x"===e?o(t,++r).toString(16):"y"===e?(3&o()|8).toString(16):e)).join("")}function s(e){const t=n._A?.crypto||n._A?.msCrypto;let r,i=0;t&&t.getRandomValues&&(r=t.getRandomValues(new Uint8Array(31)));const a=[];for(var s=0;s {r.d(t,{Bq:()=>n,Hb:()=>o,oD:()=>i});const n="NRBA",i=144e5,o=18e5},7894:(e,t,r)=>{function n(){return Math.round(performance.now())}r.d(t,{z:()=>n})},7243:(e,t,r)=>{r.d(t,{e:()=>o});var n=r(385),i={};function o(e){if(e in i)return i[e];if(0===(e||"").indexOf("data:"))return{protocol:"data"};let t;var r=n._A?.location,o={};if(n.il)t=document.createElement("a"),t.href=e;else try{t=new URL(e,r.href)}catch(e){return o}o.port=t.port;var a=t.href.split("://");!o.port&&a[1]&&(o.port=a[1].split("/")[0].split("@").pop().split(":")[1]),o.port&&"0"!==o.port||(o.port="https"===a[0]?"443":"80"),o.hostname=t.hostname||r.hostname,o.pathname=t.pathname,o.protocol=a[0],"/"!==o.pathname.charAt(0)&&(o.pathname="/"+o.pathname);var s=!t.protocol||":"===t.protocol||t.protocol===r.protocol,c=t.hostname===r.hostname&&t.port===r.port;return o.sameOrigin=s&&(!t.hostname||c),"/"===o.pathname&&(i[e]=o),o}},50:(e,t,r)=>{function n(e,t){"function"==typeof console.warn&&(console.warn("New Relic: ".concat(e)),t&&console.warn(t))}r.d(t,{Z:()=>n})},2587:(e,t,r)=>{r.d(t,{N:()=>c,T:()=>u});var n=r(2177),i=r(5546),o=r(8e3),a=r(3325);const s={stn:[a.D.sessionTrace],err:[a.D.jserrors,a.D.metrics],ins:[a.D.pageAction],spa:[a.D.spa],sr:[a.D.sessionReplay,a.D.sessionTrace]};function c(e,t){const r=n.ee.get(t);e&&"object"==typeof e&&(Object.entries(e).forEach((e=>{let[t,n]=e;void 0===u[t]&&(s[t]?s[t].forEach((e=>{n?(0,i.p)("feat-"+t,[],void 0,e,r):(0,i.p)("block-"+t,[],void 0,e,r),(0,i.p)("rumresp-"+t,[Boolean(n)],void 0,e,r)})):n&&(0,i.p)("feat-"+t,[],void 0,void 0,r),u[t]=Boolean(n))})),Object.keys(s).forEach((e=>{void 0===u[e]&&(s[e]?.forEach((t=>(0,i.p)("rumresp-"+e,[!1],void 0,t,r))),u[e]=!1)})),(0,o.L)(t,a.D.pageViewEvent))}const u={}},2210:(e,t,r)=>{r.d(t,{X:()=>i});var n=Object.prototype.hasOwnProperty;function i(e,t,r){if(n.call(e,t))return e[t];var i=r();if(Object.defineProperty&&Object.keys)try{return Object.defineProperty(e,t,{value:i,writable:!0,enumerable:!1}),i}catch(e){}return e[t]=i,i}},1284:(e,t,r)=>{r.d(t,{D:()=>n});const n=(e,t)=>Object.entries(e||{}).map((e=>{let[r,n]=e;return t(r,n)}))},4351:(e,t,r)=>{r.d(t,{P:()=>o});var n=r(2177);const i=()=>{const e=new WeakSet;return(t,r)=>{if("object"==typeof r&&null!==r){if(e.has(r))return;e.add(r)}return r}};function o(e){try{return JSON.stringify(e,i())}catch(e){try{n.ee.emit("internal-error",[e])}catch(e){}}}},3960:(e,t,r)=>{r.d(t,{K:()=>a,b:()=>o});var n=r(3239);function i(){return"undefined"==typeof document||"complete"===document.readyState}function o(e,t){if(i())return e();(0,n.bP)("load",e,t)}function a(e){if(i())return e();(0,n.iz)("DOMContentLoaded",e)}},8632:(e,t,r)=>{r.d(t,{EZ:()=>u,Qy:()=>c,ce:()=>o,fP:()=>a,gG:()=>d,mF:()=>s});var n=r(7894),i=r(385);const o={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net"};function a(){return i._A.NREUM||(i._A.NREUM={}),void 0===i._A.newrelic&&(i._A.newrelic=i._A.NREUM),i._A.NREUM}function s(){let e=a();return e.o||(e.o={ST:i._A.setTimeout,SI:i._A.setImmediate,CT:i._A.clearTimeout,XHR:i._A.XMLHttpRequest,REQ:i._A.Request,EV:i._A.Event,PR:i._A.Promise,MO:i._A.MutationObserver,FETCH:i._A.fetch}),e}function c(e,t,r){let i=a();const o=i.initializedAgents||{},s=o[e]||{};return Object.keys(s).length||(s.initializedAt={ms:(0,n.z)(),date:new Date}),i.initializedAgents={...o,[e]:{...s,[r]:t}},i}function u(e,t){a()[e]=t}function d(){return function(){let e=a();const t=e.info||{};e.info={beacon:o.beacon,errorBeacon:o.errorBeacon,...t}}(),function(){let e=a();const t=e.init||{};e.init={...t}}(),s(),function(){let e=a();const t=e.loader_config||{};e.loader_config={...t}}(),a()}},7956:(e,t,r)=>{r.d(t,{N:()=>i});var n=r(3239);function i(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],r=arguments.length>2?arguments[2]:void 0,i=arguments.length>3?arguments[3]:void 0;return void(0,n.iz)("visibilitychange",(function(){if(t)return void("hidden"==document.visibilityState&&e());e(document.visibilityState)}),r,i)}},1214:(e,t,r)=>{r.d(t,{em:()=>v,u5:()=>N,QU:()=>S,_L:()=>I,Gm:()=>L,Lg:()=>M,gy:()=>U,BV:()=>Q,Kf:()=>ee});var n=r(2177);const i="nr@original";var o=Object.prototype.hasOwnProperty,a=!1;function s(e,t){return e||(e=n.ee),r.inPlace=function(e,t,n,i,o){n||(n="");var a,s,c,u="-"===n.charAt(0);for(c=0;c 2?n-2:0),o=2;o {r(A[T],e,w),r(E[T],e,w)})),r(l._A,"fetch",y),t.on(y+"end",(function(e,r){var n=this;if(r){var i=r.headers.get("content-length");null!==i&&(n.rxSize=i),t.emit(y+"done",[null,r],n)}else t.emit(y+"done",[e],n)})),t}const O={},j=["pushState","replaceState"];function S(e){const t=function(e){return(e||n.ee).get("history")}(e);return!l.il||O[t.debugId]++||(O[t.debugId]=1,s(t).inPlace(window.history,j,"-")),t}var P=r(3239);const C={},R=["appendChild","insertBefore","replaceChild"];function I(e){const t=function(e){return(e||n.ee).get("jsonp")}(e);if(!l.il||C[t.debugId])return t;C[t.debugId]=!0;var r=s(t),i=/[?&](?:callback|cb)=([^&#]+)/,o=/(.*)\.([^.]+)/,a=/^(\w+)(\.|$)(.*)$/;function c(e,t){var r=e.match(a),n=r[1],i=r[3];return i?c(i,t[n]):t[n]}return r.inPlace(Node.prototype,R,"dom-"),t.on("dom-start",(function(e){!function(e){if(!e||"string"!=typeof e.nodeName||"script"!==e.nodeName.toLowerCase())return;if("function"!=typeof e.addEventListener)return;var n=(a=e.src,s=a.match(i),s?s[1]:null);var a,s;if(!n)return;var u=function(e){var t=e.match(o);if(t&&t.length>=3)return{key:t[2],parent:c(t[1],window)};return{key:e,parent:window}}(n);if("function"!=typeof u.parent[u.key])return;var d={};function f(){t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}function l(){t.emit("jsonp-error",[],d),t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}r.inPlace(u.parent,[u.key],"cb-",d),e.addEventListener("load",f,(0,P.m$)(!1)),e.addEventListener("error",l,(0,P.m$)(!1)),t.emit("new-jsonp",[e.src],d)}(e[0])})),t}var k=r(5763);const H={};function L(e){const t=function(e){return(e||n.ee).get("mutation")}(e);if(!l.il||H[t.debugId])return t;H[t.debugId]=!0;var r=s(t),i=k.Yu.MO;return i&&(window.MutationObserver=function(e){return this instanceof i?new i(r(e,"fn-")):i.apply(this,arguments)},MutationObserver.prototype=i.prototype),t}const z={};function M(e){const t=function(e){return(e||n.ee).get("promise")}(e);if(z[t.debugId])return t;z[t.debugId]=!0;var r=n.c,o=s(t),a=k.Yu.PR;return a&&function(){function e(r){var n=t.context(),i=o(r,"executor-",n,null,!1);const s=Reflect.construct(a,[i],e);return t.context(s).getCtx=function(){return n},s}l._A.Promise=e,Object.defineProperty(e,"name",{value:"Promise"}),e.toString=function(){return a.toString()},Object.setPrototypeOf(e,a),["all","race"].forEach((function(r){const n=a[r];e[r]=function(e){let i=!1;[...e||[]].forEach((e=>{this.resolve(e).then(a("all"===r),a(!1))}));const o=n.apply(this,arguments);return o;function a(e){return function(){t.emit("propagate",[null,!i],o,!1,!1),i=i||!e}}}})),["resolve","reject"].forEach((function(r){const n=a[r];e[r]=function(e){const r=n.apply(this,arguments);return e!==r&&t.emit("propagate",[e,!0],r,!1,!1),r}})),e.prototype=a.prototype;const n=a.prototype.then;a.prototype.then=function(){var e=this,i=r(e);i.promise=e;for(var a=arguments.length,s=new Array(a),c=0;c e())),t};function m(e,t){i.inPlace(t,["onreadystatechange"],"fn-",E)}function b(){var e=this,t=r.context(e);e.readyState>3&&!t.resolved&&(t.resolved=!0,r.emit("xhr-resolved",[],e)),i.inPlace(e,f,"fn-",E)}if(function(e,t){for(var r in e)t[r]=e[r]}(o,p),p.prototype=o.prototype,i.inPlace(p.prototype,J,"-xhr-",E),r.on("send-xhr-start",(function(e,t){m(e,t),function(e){h.push(e),a&&(y?y.then(A):u?u(A):(w=-w,x.data=w))}(t)})),r.on("open-xhr-start",m),a){var y=c&&c.resolve();if(!u&&!c){var w=1,x=document.createTextNode(w);new a(A).observe(x,{characterData:!0})}}else t.on("fn-end",(function(e){e[0]&&e[0].type===d||A()}));function A(){for(var e=0;e {r.d(t,{t:()=>n});const n=r(3325).D.ajax},6660:(e,t,r)=>{r.d(t,{A:()=>i,t:()=>n});const n=r(3325).D.jserrors,i="nr@seenError"},3081:(e,t,r)=>{r.d(t,{gF:()=>o,mY:()=>i,t9:()=>n,vz:()=>s,xS:()=>a});const n=r(3325).D.metrics,i="sm",o="cm",a="storeSupportabilityMetrics",s="storeEventMetrics"},4649:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageAction},7633:(e,t,r)=>{r.d(t,{Dz:()=>i,OJ:()=>a,qw:()=>o,t9:()=>n});const n=r(3325).D.pageViewEvent,i="firstbyte",o="domcontent",a="windowload"},9251:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageViewTiming},3614:(e,t,r)=>{r.d(t,{BST_RESOURCE:()=>i,END:()=>s,FEATURE_NAME:()=>n,FN_END:()=>u,FN_START:()=>c,PUSH_STATE:()=>d,RESOURCE:()=>o,START:()=>a});const n=r(3325).D.sessionTrace,i="bstResource",o="resource",a="-start",s="-end",c="fn"+a,u="fn"+s,d="pushState"},7836:(e,t,r)=>{r.d(t,{BODY:()=>A,CB_END:()=>E,CB_START:()=>u,END:()=>x,FEATURE_NAME:()=>i,FETCH:()=>_,FETCH_BODY:()=>v,FETCH_DONE:()=>m,FETCH_START:()=>p,FN_END:()=>c,FN_START:()=>s,INTERACTION:()=>l,INTERACTION_API:()=>d,INTERACTION_EVENTS:()=>o,JSONP_END:()=>b,JSONP_NODE:()=>g,JS_TIME:()=>T,MAX_TIMER_BUDGET:()=>a,REMAINING:()=>f,SPA_NODE:()=>h,START:()=>w,originalSetTimeout:()=>y});var n=r(5763);const i=r(3325).D.spa,o=["click","submit","keypress","keydown","keyup","change"],a=999,s="fn-start",c="fn-end",u="cb-start",d="api-ixn-",f="remaining",l="interaction",h="spaNode",g="jsonpNode",p="fetch-start",m="fetch-done",v="fetch-body-",b="jsonp-end",y=n.Yu.ST,w="-start",x="-end",A="-body",E="cb"+x,T="jsTime",_="fetch"},5938:(e,t,r)=>{r.d(t,{W:()=>o});var n=r(5763),i=r(2177);class o{constructor(e,t,r){this.agentIdentifier=e,this.aggregator=t,this.ee=i.ee.get(e,(0,n.OP)(this.agentIdentifier).isolatedBacklog),this.featureName=r,this.blocked=!1}}},9144:(e,t,r)=>{r.d(t,{j:()=>m});var n=r(3325),i=r(5763),o=r(5546),a=r(2177),s=r(7894),c=r(8e3),u=r(3960),d=r(385),f=r(50),l=r(3081),h=r(8632);function g(){const e=(0,h.gG)();["setErrorHandler","finished","addToTrace","inlineHit","addRelease","addPageAction","setCurrentRouteName","setPageViewName","setCustomAttribute","interaction","noticeError","setUserId"].forEach((t=>{e[t]=function(){for(var r=arguments.length,n=new Array(r),i=0;i 1?r-1:0),i=1;i {e.exposed&&e.api[t]&&o.push(e.api[t](...n))})),o.length>1?o:o[0]}(t,...n)}}))}var p=r(2587);function m(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},m=arguments.length>2?arguments[2]:void 0,v=arguments.length>3?arguments[3]:void 0,{init:b,info:y,loader_config:w,runtime:x={loaderType:m},exposed:A=!0}=t;const E=(0,h.gG)();y||(b=E.init,y=E.info,w=E.loader_config),(0,i.Dg)(e,b||{}),(0,i.GE)(e,w||{}),(0,i.sU)(e,x),y.jsAttributes??={},d.v6&&(y.jsAttributes.isWorker=!0),(0,i.CX)(e,y),g();const T=function(e,t){t||(0,c.R)(e,"api");const h={};var g=a.ee.get(e),p=g.get("tracer"),m="api-",v=m+"ixn-";function b(t,r,n,o){const a=(0,i.C5)(e);return null===r?delete a.jsAttributes[t]:(0,i.CX)(e,{...a,jsAttributes:{...a.jsAttributes,[t]:r}}),x(m,n,!0,o||null===r?"session":void 0)(t,r)}function y(){}["setErrorHandler","finished","addToTrace","inlineHit","addRelease"].forEach((e=>h[e]=x(m,e,!0,"api"))),h.addPageAction=x(m,"addPageAction",!0,n.D.pageAction),h.setCurrentRouteName=x(m,"routeName",!0,n.D.spa),h.setPageViewName=function(t,r){if("string"==typeof t)return"/"!==t.charAt(0)&&(t="/"+t),(0,i.OP)(e).customTransaction=(r||"http://custom.transaction")+t,x(m,"setPageViewName",!0)()},h.setCustomAttribute=function(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2];if("string"==typeof e){if(["string","number"].includes(typeof t)||null===t)return b(e,t,"setCustomAttribute",r);(0,f.Z)("Failed to execute setCustomAttribute.\nNon-null value must be a string or number type, but a type of was provided."))}else(0,f.Z)("Failed to execute setCustomAttribute.\nName must be a string type, but a type of was provided."))},h.setUserId=function(e){if("string"==typeof e||null===e)return b("enduser.id",e,"setUserId",!0);(0,f.Z)("Failed to execute setUserId.\nNon-null value must be a string type, but a type of was provided."))},h.interaction=function(){return(new y).get()};var w=y.prototype={createTracer:function(e,t){var r={},i=this,a="function"==typeof t;return(0,o.p)(v+"tracer",[(0,s.z)(),e,r],i,n.D.spa,g),function(){if(p.emit((a?"":"no-")+"fn-start",[(0,s.z)(),i,a],r),a)try{return t.apply(this,arguments)}catch(e){throw p.emit("fn-err",[arguments,this,"string"==typeof e?new Error(e):e],r),e}finally{p.emit("fn-end",[(0,s.z)()],r)}}}};function x(e,t,r,i){return function(){return(0,o.p)(l.xS,["API/"+t+"/called"],void 0,n.D.metrics,g),i&&(0,o.p)(e+t,[(0,s.z)(),...arguments],r?null:this,i,g),r?void 0:this}}function A(){r.e(439).then(r.bind(r,7438)).then((t=>{let{setAPI:r}=t;r(e),(0,c.L)(e,"api")})).catch((()=>(0,f.Z)("Downloading runtime APIs failed...")))}return["actionText","setName","setAttribute","save","ignore","onEnd","getContext","end","get"].forEach((e=>{w[e]=x(v,e,void 0,n.D.spa)})),h.noticeError=function(e,t){"string"==typeof e&&(e=new Error(e)),(0,o.p)(l.xS,["API/noticeError/called"],void 0,n.D.metrics,g),(0,o.p)("err",[e,(0,s.z)(),!1,t],void 0,n.D.jserrors,g)},d.il?(0,u.b)((()=>A()),!0):A(),h}(e,v);return(0,h.Qy)(e,T,"api"),(0,h.Qy)(e,A,"exposed"),(0,h.EZ)("activatedFeatures",p.T),T}},3325:(e,t,r)=>{r.d(t,{D:()=>n,p:()=>i});const n={ajax:"ajax",jserrors:"jserrors",metrics:"metrics",pageAction:"page_action",pageViewEvent:"page_view_event",pageViewTiming:"page_view_timing",sessionReplay:"session_replay",sessionTrace:"session_trace",spa:"spa"},i={[n.pageViewEvent]:1,[n.pageViewTiming]:2,[n.metrics]:3,[n.jserrors]:4,[n.ajax]:5,[n.sessionTrace]:6,[n.pageAction]:7,[n.spa]:8,[n.sessionReplay]:9}}},n={};function i(e){var t=n[e];if(void 0!==t)return t.exports;var o=n[e]={exports:{}};return r[e](o,o.exports,i),o.exports}i.m=r,i.d=(e,t)=>{for(var r in t)i.o(t,r)&&!i.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},i.f={},i.e=e=>Promise.all(Object.keys(i.f).reduce(((t,r)=>(i.f[r](e,t),t)),[])),i.u=e=>(({78:"page_action-aggregate",147:"metrics-aggregate",242:"session-manager",317:"jserrors-aggregate",348:"page_view_timing-aggregate",412:"lazy-feature-loader",439:"async-api",538:"recorder",590:"session_replay-aggregate",675:"compressor",733:"session_trace-aggregate",786:"page_view_event-aggregate",873:"spa-aggregate",898:"ajax-aggregate"}[e]||e)+"."+{78:"ac76d497",147:"3dc53903",148:"1a20d5fe",242:"2a64278a",317:"49e41428",348:"bd6de33a",412:"2f55ce66",439:"30bd804e",538:"1b18459f",590:"cf0efb30",675:"ae9f91a8",733:"83105561",786:"06482edd",860:"03a8b7a5",873:"e6b09d52",898:"998ef92b"}[e]+"-1.236.0.min.js"),i.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),e={},t="NRBA:",i.l=(r,n,o,a)=>{if(e[r])e[r].push(n);else{var s,c;if(void 0!==o)for(var u=document.getElementsByTagName("script"),d=0;d {s.onerror=s.onload=null,clearTimeout(h);var i=e[r];if(delete e[r],s.parentNode&&s.parentNode.removeChild(s),i&&i.forEach((e=>e(n))),t)return t(n)},h=setTimeout(l.bind(null,void 0,{type:"timeout",target:s}),12e4);s.onerror=l.bind(null,s.onerror),s.onload=l.bind(null,s.onload),c&&document.head.appendChild(s)}},i.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.j=364,i.p="https://js-agent.newrelic.com/",(()=>{var e={364:0,953:0};i.f.j=(t,r)=>{var n=i.o(e,t)?e[t]:void 0;if(0!==n)if(n)r.push(n[2]);else{var o=new Promise(((r,i)=>n=e[t]=[r,i]));r.push(n[2]=o);var a=i.p+i.u(t),s=new Error;i.l(a,(r=>{if(i.o(e,t)&&(0!==(n=e[t])&&(e[t]=void 0),n)){var o=r&&("load"===r.type?"missing":r.type),a=r&&r.target&&r.target.src;s.message="Loading chunk "+t+" failed.\n("+o+": "+a+")",s.name="ChunkLoadError",s.type=o,s.request=a,n[1](s)}}),"chunk-"+t,t)}};var t=(t,r)=>{var n,o,[a,s,c]=r,u=0;if(a.some((t=>0!==e[t]))){for(n in s)i.o(s,n)&&(i.m[n]=s[n]);if(c)c(i)}for(t&&t(r);u {i.r(o);var e=i(3325),t=i(5763);const r=Object.values(e.D);function n(e){const n={};return r.forEach((r=>{n[r]=function(e,r){return!1!==(0,t.Mt)(r,"".concat(e,".enabled"))}(r,e)})),n}var a=i(9144);var s=i(5546),c=i(385),u=i(8e3),d=i(5938),f=i(3960),l=i(50);class h extends d.W{constructor(e,t,r){let n=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];super(e,t,r),this.auto=n,this.abortHandler,this.featAggregate,this.onAggregateImported,n&&(0,u.R)(e,r)}importAggregator(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};if(this.featAggregate||!this.auto)return;const r=c.il&&!0===(0,t.Mt)(this.agentIdentifier,"privacy.cookies_enabled");let n;this.onAggregateImported=new Promise((e=>{n=e}));const o=async()=>{let t;try{if(r){const{setupAgentSession:e}=await Promise.all([i.e(860),i.e(242)]).then(i.bind(i,3228));t=e(this.agentIdentifier)}}catch(e){(0,l.Z)("A problem occurred when starting up session manager. This page will not start or extend any session.",e)}try{if(!this.shouldImportAgg(this.featureName,t))return void(0,u.L)(this.agentIdentifier,this.featureName);const{lazyFeatureLoader:r}=await i.e(412).then(i.bind(i,8582)),{Aggregate:o}=await r(this.featureName,"aggregate");this.featAggregate=new o(this.agentIdentifier,this.aggregator,e),n(!0)}catch(e){(0,l.Z)("Downloading and initializing ".concat(this.featureName," failed..."),e),this.abortHandler?.(),n(!1)}};c.il?(0,f.b)((()=>o()),!0):o()}shouldImportAgg(r,n){return r!==e.D.sessionReplay||!1!==(0,t.Mt)(this.agentIdentifier,"session_trace.enabled")&&(!!n?.isNew||!!n?.state.sessionReplay)}}var g=i(7633),p=i(7894);class m extends h{static featureName=g.t9;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];if(super(r,n,g.t9,i),("undefined"==typeof PerformanceNavigationTiming||c.Tt)&&"undefined"!=typeof PerformanceTiming){const n=(0,t.OP)(r);n[g.Dz]=Math.max(Date.now()-n.offset,0),(0,f.K)((()=>n[g.qw]=Math.max((0,p.z)()-n[g.Dz],0))),(0,f.b)((()=>{const t=(0,p.z)();n[g.OJ]=Math.max(t-n[g.Dz],0),(0,s.p)("timing",["load",t],void 0,e.D.pageViewTiming,this.ee)}))}this.importAggregator()}}var v=i(1117),b=i(1284);class y extends v.w{constructor(e){super(e),this.aggregatedData={}}store(e,t,r,n,i){var o=this.getBucket(e,t,r,i);return o.metrics=function(e,t){t||(t={count:0});return t.count+=1,(0,b.D)(e,(function(e,r){t[e]=w(r,t[e])})),t}(n,o.metrics),o}merge(e,t,r,n,i){var o=this.getBucket(e,t,n,i);if(o.metrics){var a=o.metrics;a.count+=r.count,(0,b.D)(r,(function(e,t){if("count"!==e){var n=a[e],i=r[e];i&&!i.c?a[e]=w(i.t,n):a[e]=function(e,t){if(!t)return e;t.c||(t=x(t.t));return t.min=Math.min(e.min,t.min),t.max=Math.max(e.max,t.max),t.t+=e.t,t.sos+=e.sos,t.c+=e.c,t}(i,a[e])}}))}else o.metrics=r}storeMetric(e,t,r,n){var i=this.getBucket(e,t,r);return i.stats=w(n,i.stats),i}getBucket(e,t,r,n){this.aggregatedData[e]||(this.aggregatedData[e]={});var i=this.aggregatedData[e][t];return i||(i=this.aggregatedData[e][t]={params:r||{}},n&&(i.custom=n)),i}get(e,t){return t?this.aggregatedData[e]&&this.aggregatedData[e][t]:this.aggregatedData[e]}take(e){for(var t={},r="",n=!1,i=0;i t.max&&(t.max=e),e 2&&void 0!==arguments[2])||arguments[2];super(e,r,j.t,n),c.il&&((0,t.OP)(e).initHidden=Boolean("hidden"===document.visibilityState),(0,N.N)((()=>(0,s.p)("docHidden",[(0,p.z)()],void 0,j.t,this.ee)),!0),(0,O.bP)("pagehide",(()=>(0,s.p)("winPagehide",[(0,p.z)()],void 0,j.t,this.ee))),this.importAggregator())}}var P=i(3081);class C extends h{static featureName=P.t9;constructor(e,t){let r=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(e,t,P.t9,r),this.importAggregator()}}var R,I=i(2210),k=i(1214),H=i(2177),L={};try{R=localStorage.getItem("__nr_flags").split(","),console&&"function"==typeof console.log&&(L.console=!0,-1!==R.indexOf("dev")&&(L.dev=!0),-1!==R.indexOf("nr_dev")&&(L.nrDev=!0))}catch(e){}function z(e){try{L.console&&z(e)}catch(e){}}L.nrDev&&H.ee.on("internal-error",(function(e){z(e.stack)})),L.dev&&H.ee.on("fn-err",(function(e,t,r){z(r.stack)})),L.dev&&(z("NR AGENT IN DEVELOPMENT MODE"),z("flags: "+(0,b.D)(L,(function(e,t){return e})).join(", ")));var M=i(6660);class B extends h{static featureName=M.t;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(r,n,M.t,i),this.skipNext=0;try{this.removeOnAbort=new AbortController}catch(e){}const o=this;o.ee.on("fn-start",(function(e,t,r){o.abortHandler&&(o.skipNext+=1)})),o.ee.on("fn-err",(function(t,r,n){o.abortHandler&&!n[M.A]&&((0,I.X)(n,M.A,(function(){return!0})),this.thrown=!0,(0,s.p)("err",[n,(0,p.z)()],void 0,e.D.jserrors,o.ee))})),o.ee.on("fn-end",(function(){o.abortHandler&&!this.thrown&&o.skipNext>0&&(o.skipNext-=1)})),o.ee.on("internal-error",(function(t){(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,o.ee)})),this.origOnerror=c._A.onerror,c._A.onerror=this.onerrorHandler.bind(this),c._A.addEventListener("unhandledrejection",(t=>{const r=function(e){let t="Unhandled Promise Rejection: ";if(e instanceof Error)try{return e.message=t+e.message,e}catch(t){return e}if(void 0===e)return new Error(t);try{return new Error(t+(0,D.P)(e))}catch(e){return new Error(t)}}(t.reason);(0,s.p)("err",[r,(0,p.z)(),!1,{unhandledPromiseRejection:1}],void 0,e.D.jserrors,this.ee)}),(0,O.m$)(!1,this.removeOnAbort?.signal)),(0,k.gy)(this.ee),(0,k.BV)(this.ee),(0,k.em)(this.ee),(0,t.OP)(r).xhrWrappable&&(0,k.Kf)(this.ee),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}onerrorHandler(t,r,n,i,o){"function"==typeof this.origOnerror&&this.origOnerror(...arguments);try{this.skipNext?this.skipNext-=1:(0,s.p)("err",[o||new F(t,r,n),(0,p.z)()],void 0,e.D.jserrors,this.ee)}catch(t){try{(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,this.ee)}catch(e){}}return!1}}function F(e,t,r){this.message=e||"Uncaught error with no additional information",this.sourceURL=t,this.line=r}let U=1;const q="nr@id";function G(e){const t=typeof e;return!e||"object"!==t&&"function"!==t?-1:e===c._A?0:(0,I.X)(e,q,(function(){return U++}))}function V(e){if("string"==typeof e&&e.length)return e.length;if("object"==typeof e){if("undefined"!=typeof ArrayBuffer&&e instanceof ArrayBuffer&&e.byteLength)return e.byteLength;if("undefined"!=typeof Blob&&e instanceof Blob&&e.size)return e.size;if(!("undefined"!=typeof FormData&&e instanceof FormData))try{return(0,D.P)(e).length}catch(e){return}}}var X=i(7243);class W{constructor(e){this.agentIdentifier=e,this.generateTracePayload=this.generateTracePayload.bind(this),this.shouldGenerateTrace=this.shouldGenerateTrace.bind(this)}generateTracePayload(e){if(!this.shouldGenerateTrace(e))return null;var r=(0,t.DL)(this.agentIdentifier);if(!r)return null;var n=(r.accountID||"").toString()||null,i=(r.agentID||"").toString()||null,o=(r.trustKey||"").toString()||null;if(!n||!i)return null;var a=(0,_.M)(),s=(0,_.Ht)(),c=Date.now(),u={spanId:a,traceId:s,timestamp:c};return(e.sameOrigin||this.isAllowedOrigin(e)&&this.useTraceContextHeadersForCors())&&(u.traceContextParentHeader=this.generateTraceContextParentHeader(a,s),u.traceContextStateHeader=this.generateTraceContextStateHeader(a,c,n,i,o)),(e.sameOrigin&&!this.excludeNewrelicHeader()||!e.sameOrigin&&this.isAllowedOrigin(e)&&this.useNewrelicHeaderForCors())&&(u.newrelicHeader=this.generateTraceHeader(a,s,c,n,i,o)),u}generateTraceContextParentHeader(e,t){return"00-"+t+"-"+e+"-01"}generateTraceContextStateHeader(e,t,r,n,i){return i+"@nr=0-1-"+r+"-"+n+"-"+e+"----"+t}generateTraceHeader(e,t,r,n,i,o){if(!("function"==typeof c._A?.btoa))return null;var a={v:[0,1],d:{ty:"Browser",ac:n,ap:i,id:e,tr:t,ti:r}};return o&&n!==o&&(a.d.tk=o),btoa((0,D.P)(a))}shouldGenerateTrace(e){return this.isDtEnabled()&&this.isAllowedOrigin(e)}isAllowedOrigin(e){var r=!1,n={};if((0,t.Mt)(this.agentIdentifier,"distributed_tracing")&&(n=(0,t.P_)(this.agentIdentifier).distributed_tracing),e.sameOrigin)r=!0;else if(n.allowed_origins instanceof Array)for(var i=0;i 2&&void 0!==arguments[2])||arguments[2];super(r,n,Z.t,i),(0,t.OP)(r).xhrWrappable&&(this.dt=new W(r),this.handler=(e,t,r,n)=>(0,s.p)(e,t,r,n,this.ee),(0,k.u5)(this.ee),(0,k.Kf)(this.ee),function(r,n,i,o){function a(e){var t=this;t.totalCbs=0,t.called=0,t.cbTime=0,t.end=E,t.ended=!1,t.xhrGuids={},t.lastSize=null,t.loadCaptureCalled=!1,t.params=this.params||{},t.metrics=this.metrics||{},e.addEventListener("load",(function(r){_(t,e)}),(0,O.m$)(!1)),c.IF||e.addEventListener("progress",(function(e){t.lastSize=e.loaded}),(0,O.m$)(!1))}function s(e){this.params={method:e[0]},T(this,e[1]),this.metrics={}}function u(e,n){var i=(0,t.DL)(r);i.xpid&&this.sameOrigin&&n.setRequestHeader("X-NewRelic-ID",i.xpid);var a=o.generateTracePayload(this.parsedOrigin);if(a){var s=!1;a.newrelicHeader&&(n.setRequestHeader("newrelic",a.newrelicHeader),s=!0),a.traceContextParentHeader&&(n.setRequestHeader("traceparent",a.traceContextParentHeader),a.traceContextStateHeader&&n.setRequestHeader("tracestate",a.traceContextStateHeader),s=!0),s&&(this.dt=a)}}function d(e,t){var r=this.metrics,i=e[0],o=this;if(r&&i){var a=V(i);a&&(r.txSize=a)}this.startTime=(0,p.z)(),this.listener=function(e){try{"abort"!==e.type||o.loadCaptureCalled||(o.params.aborted=!0),("load"!==e.type||o.called===o.totalCbs&&(o.onloadCalled||"function"!=typeof t.onload)&&"function"==typeof o.end)&&o.end(t)}catch(e){try{n.emit("internal-error",[e])}catch(e){}}};for(var s=0;s 1?e[1]=i:e.push(i)}else e[0]&&e[0].headers&&s(e[0].headers,n)&&(this.dt=n);function s(e,t){var r=!1;return t.newrelicHeader&&(e.set("newrelic",t.newrelicHeader),r=!0),t.traceContextParentHeader&&(e.set("traceparent",t.traceContextParentHeader),t.traceContextStateHeader&&e.set("tracestate",t.traceContextStateHeader),r=!0),r}}function x(e,t){this.params={},this.metrics={},this.startTime=(0,p.z)(),this.dt=t,e.length>=1&&(this.target=e[0]),e.length>=2&&(this.opts=e[1]);var r,n=this.opts||{},i=this.target;"string"==typeof i?r=i:"object"==typeof i&&i instanceof Y?r=i.url:c._A?.URL&&"object"==typeof i&&i instanceof URL&&(r=i.href),T(this,r);var o=(""+(i&&i instanceof Y&&i.method||n.method||"GET")).toUpperCase();this.params.method=o,this.txSize=V(n.body)||0}function A(t,r){var n;this.endTime=(0,p.z)(),this.params||(this.params={}),this.params.status=r?r.status:0,"string"==typeof this.rxSize&&this.rxSize.length>0&&(n=+this.rxSize);var o={txSize:this.txSize,rxSize:n,duration:(0,p.z)()-this.startTime};i("xhr",[this.params,o,this.startTime,this.endTime,"fetch"],this,e.D.ajax)}function E(t){var r=this.params,n=this.metrics;if(!this.ended){this.ended=!0;for(var o=0;o 2&&void 0!==arguments[2])||arguments[2];super(e,t,we.t,r),this.importAggregator()}}new class{constructor(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:(0,_.ky)(16);c._A?(this.agentIdentifier=t,this.sharedAggregator=new y({agentIdentifier:this.agentIdentifier}),this.features={},this.desiredFeatures=new Set(e.features||[]),this.desiredFeatures.add(m),Object.assign(this,(0,a.j)(this.agentIdentifier,e,e.loaderType||"agent")),this.start()):(0,l.Z)("Failed to initial the agent. Could not determine the runtime environment.")}get config(){return{info:(0,t.C5)(this.agentIdentifier),init:(0,t.P_)(this.agentIdentifier),loader_config:(0,t.DL)(this.agentIdentifier),runtime:(0,t.OP)(this.agentIdentifier)}}start(){const t="features";try{const r=n(this.agentIdentifier),i=[...this.desiredFeatures];i.sort(((t,r)=>e.p[t.featureName]-e.p[r.featureName])),i.forEach((t=>{if(r[t.featureName]||t.featureName===e.D.pageViewEvent){const n=function(t){switch(t){case e.D.ajax:return[e.D.jserrors];case e.D.sessionTrace:return[e.D.ajax,e.D.pageViewEvent];case e.D.sessionReplay:return[e.D.sessionTrace];case e.D.pageViewTiming:return[e.D.pageViewEvent];default:return[]}}(t.featureName);n.every((e=>r[e]))||(0,l.Z)("".concat(t.featureName," is enabled but one or more dependent features has been disabled (").concat((0,D.P)(n),"). This may cause unintended consequences or missing data...")),this.features[t.featureName]=new t(this.agentIdentifier,this.sharedAggregator)}})),(0,T.Qy)(this.agentIdentifier,this.features,t)}catch(e){(0,l.Z)("Failed to initialize all enabled instrument classes (agent aborted) -",e);for(const e in this.features)this.features[e].abortHandler?.();const r=(0,T.fP)();return delete r.initializedAgents[this.agentIdentifier]?.api,delete r.initializedAgents[this.agentIdentifier]?.[t],delete this.sharedAggregator,r.ee?.abort(),delete r.ee?.get(this.agentIdentifier),!1}}}({features:[J,m,S,class extends h{static featureName=oe;constructor(t,r){if(super(t,r,oe,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;const n=this.ee;let i;(0,k.QU)(n),this.eventsEE=(0,k.em)(n),this.eventsEE.on(se,(function(e,t){this.bstStart=(0,p.z)()})),this.eventsEE.on(ae,(function(t,r){(0,s.p)("bst",[t[0],r,this.bstStart,(0,p.z)()],void 0,e.D.sessionTrace,n)})),n.on(ce+ne,(function(e){this.time=(0,p.z)(),this.startPath=location.pathname+location.hash})),n.on(ce+ie,(function(t){(0,s.p)("bstHist",[location.pathname+location.hash,this.startPath,this.time],void 0,e.D.sessionTrace,n)}));try{i=new PerformanceObserver((t=>{const r=t.getEntries();(0,s.p)(te,[r],void 0,e.D.sessionTrace,n)})),i.observe({type:re,buffered:!0})}catch(e){}this.importAggregator({resourceObserver:i})}},C,xe,B,class extends h{static featureName=de;constructor(e,r){if(super(e,r,de,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;if(!(0,t.OP)(e).xhrWrappable)return;try{this.removeOnAbort=new AbortController}catch(e){}let n,i=0;const o=this.ee.get("tracer"),a=(0,k._L)(this.ee),s=(0,k.Lg)(this.ee),u=(0,k.BV)(this.ee),d=(0,k.Kf)(this.ee),f=this.ee.get("events"),l=(0,k.u5)(this.ee),h=(0,k.QU)(this.ee),g=(0,k.Gm)(this.ee);function m(e,t){h.emit("newURL",[""+window.location,t])}function v(){i++,n=window.location.hash,this[ve]=(0,p.z)()}function b(){i--,window.location.hash!==n&&m(0,!0);var e=(0,p.z)();this[pe]=~~this[pe]+e-this[ve],this[ye]=e}function y(e,t){e.on(t,(function(){this[t]=(0,p.z)()}))}this.ee.on(ve,v),s.on(be,v),a.on(be,v),this.ee.on(ye,b),s.on(ge,b),a.on(ge,b),this.ee.buffer([ve,ye,"xhr-resolved"],this.featureName),f.buffer([ve],this.featureName),u.buffer(["setTimeout"+le,"clearTimeout"+fe,ve],this.featureName),d.buffer([ve,"new-xhr","send-xhr"+fe],this.featureName),l.buffer([me+fe,me+"-done",me+he+fe,me+he+le],this.featureName),h.buffer(["newURL"],this.featureName),g.buffer([ve],this.featureName),s.buffer(["propagate",be,ge,"executor-err","resolve"+fe],this.featureName),o.buffer([ve,"no-"+ve],this.featureName),a.buffer(["new-jsonp","cb-start","jsonp-error","jsonp-end"],this.featureName),y(l,me+fe),y(l,me+"-done"),y(a,"new-jsonp"),y(a,"jsonp-end"),y(a,"cb-start"),h.on("pushState-end",m),h.on("replaceState-end",m),window.addEventListener("hashchange",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("load",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("popstate",(function(){m(0,i>1)}),(0,O.m$)(!0,this.removeOnAbort?.signal)),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}}],loaderType:"spa"})})(),window.NRBA=o})(); window.jQuery || document.write(' ') CKEDITOR_BASEPATH='https://f1000research.com/js/vendor/ckeditor/' window.reactTheme = 'research'; window.MathJax = { CommonHTML: { linebreaks: { automatic: true } }, 'HTML-CSS': { linebreaks: { automatic: true } }, SVG: { linebreaks: { automatic: true } }, AuthorInit: function() { MathJax.Hub.Register.MessageHook('End Process', function () { let timeout = false; // holder for timeout id const delay = 250; // delay after event is "complete" to run callback const reflowMath = function() { const dispFormulas = document.querySelectorAll('.disp-formula.panel'); if (!dispFormulas) { return; } for (const dispFormula of dispFormulas) { const child = dispFormula.querySelector('.MathJax_Preview').nextSibling.firstChild; const isMultiline = MathJax.Hub.getAllJax(dispFormula)[0].root.isMultiline; if (dispFormula.offsetWidth < child.offsetWidth || isMultiline) { MathJax.Hub.Queue(['Rerender', MathJax.Hub, dispFormula]); } } }; window.addEventListener('resize', function() { clearTimeout(timeout); // clear the timeout timeout = setTimeout(reflowMath, delay); // start timing for event "completion" }); }); }, }; if (window.location.hash == '#_=_'){ window.location = window.location.href.split('#')[0] } !function(f,b,e,v,n,t,s){if(f.fbq)return;n=f.fbq=function() {n.callMethod? n.callMethod.apply(n,arguments):n.queue.push(arguments)} ;if(!f._fbq)f._fbq=n; n.push=n;n.loaded=!0;n.version='2.0';n.queue=[];t=b.createElement(e);t.async=!0; t.src=v;s=b.getElementsByTagName(e)[0];s.parentNode.insertBefore(t,s)}(window, document,'script','https://connect.facebook.net/en_US/fbevents.js'); fbq('init', '1641728616063202'); fbq('track', "PixelInitialized", {}); (function(h,o,t,j,a,r){ h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)}; h._hjSettings={hjid:2318163,hjsv:6}; a=o.getElementsByTagName('head')[0]; r=o.createElement('script');r.async=1; r.src=t+h._hjSettings.hjid+j+h._hjSettings.hjsv; a.appendChild(r); })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv='); search file_upload Submit your research search menu close search Browse Gateways & Collections How to Publish Submit your Research My Submissions Article Guidelines Article Guidelines (New Versions) Open Data, Software and Code Guidelines Open Data and Accessible Source Materials Guidelines (HSS) Open Data, Software and Code Guidelines (PSE) Prepublication Checks Production Process Posters and Slides Guidelines Document Guidelines Article Processing Charges Peer Review Finding Article Reviewers About How it Works For Reviewers Our Advisors Policies Glossary FAQs For Developers Newsroom Contact My Research Submissions Content and Tracking Alerts My Details Sign In file_upload Submit your research { "@context": "https://schema.org", "@type": "ScholarlyArticle", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://f1000research.com/articles/13-1305" }, "headline": "A libraries reproducibility hackathon: connecting students to University research and testing the longevity of...", "datePublished": "2024-10-31T10:41:48", "dateModified": "2025-09-09T10:30:41", "author": [ { "@type": "Person", "name": "Chasz Griego" }, { "@type": "Person", "name": "Kristen Scotti" }, { "@type": "Person", "name": "Elizabeth Terveen" }, { "@type": "Person", "name": "Joseph Chan" }, { "@type": "Person", "name": "Daisy Sheng" }, { "@type": "Person", "name": "Alfredo González-Espinoza" }, { "@type": "Person", "name": "Christopher Warren" } ], "publisher": { "@type": "Organization", "name": "F1000Research", "logo": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 480, "width": 60 } }, "image": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 1200, "width": 150 }, "description": "Reproducibility is a basis of scientific integrity, yet it remains a significant challenge across disciplines in computational science. This reproducibility crisis is now being met with an Open Science movement, which has risen to prominence within the scientific community and academic libraries especially. To address the need for reproducible computational research and promote Open Science within the community, members of the Open Science and Data Collaborations Program at Carnegie Mellon University Libraries organized a single-day hackathon centered around reproducibility. Partnering with a faculty researcher in English and Digital Humanities, this event allowed several students an opportunity to interact with real research outputs, test the reproducibility of data analyses with code, and offer feedback for improvements. With Python code and data shared by the researcher in an open repository, we revealed that students could successfully reproduce most of the data visualizations, but they required completing some manual setup and modifications to address depreciated libraries to successfully rerun the code. During the event, we also investigated the option of using ChatGPT to debug and troubleshoot rerunning this code. By interacting with a ChatGPT API in the code, we found and addressed the same roadblocks and successfully reproduced the same figures as the participating students. Assessing a second option, we also collaborated with the researcher to publish a compute capsule in Code Ocean. This option presented an alternative to manual setup and modifications, an accessible option for more limited devices like tablets, and a simple solution for outside researchers to modify or build on existing research code." } { "@context": "http://schema.org", "@type": "BreadcrumbList", "itemListElement": [ { "@type": "ListItem", "position": "1", "item": { "@id": "https://f1000research.com/", "name": "Home" } }, { "@type": "ListItem", "position": "2", "item": { "@id": "https://f1000research.com/browse/articles", "name": "Browse" } }, { "@type": "ListItem", "position": "3", "item": { "@id": "https://f1000research.com/articles/13-1305/v1", "name": "A libraries reproducibility hackathon: connecting students to University..." } } ] } Home Browse A libraries reproducibility hackathon: connecting students to University... ALL Metrics - Views Downloads Get PDF Get XML Cite How to cite this article Griego C, Scotti K, Terveen E et al. A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 :1305 ( https://doi.org/10.12688/f1000research.156917.1 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. Close Copy Citation Details Export Export Citation Sciwheel EndNote Ref. Manager Bibtex ProCite Sente EXPORT Select a format first Track Share ▬ ✚ Case Study A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code [version 1; peer review: 1 approved, 1 approved with reservations] Chasz Griego https://orcid.org/0000-0002-2051-7491 1 , Kristen Scotti 1 , Elizabeth Terveen 2 , [...] Joseph Chan 2 , Daisy Sheng 2 , Alfredo González-Espinoza 1 , Christopher Warren https://orcid.org/0000-0002-9881-682X 3 Chasz Griego https://orcid.org/0000-0002-2051-7491 1 , Kristen Scotti 1 , [...] Elizabeth Terveen 2 , Joseph Chan 2 , Daisy Sheng 2 , Alfredo González-Espinoza 1 , Christopher Warren https://orcid.org/0000-0002-9881-682X 3 PUBLISHED 31 Oct 2024 Author details Author details 1 University Libraries, Carnegie Mellon University, Pittsburgh, Pennsylvania, 15213, USA 2 Carnegie Mellon University School of Computer Science, Pittsburgh, Pennsylvania, 15213, USA 3 Carnegie Mellon University Department of English, Pittsburgh, Pennsylvania, 15213, USA Chasz Griego Roles: Conceptualization, Data Curation, Formal Analysis, Methodology, Project Administration, Resources, Software, Supervision, Validation, Writing – Original Draft Preparation Kristen Scotti Roles: Conceptualization, Data Curation, Formal Analysis, Resources, Software, Validation, Visualization, Writing – Original Draft Preparation Elizabeth Terveen Roles: Data Curation, Formal Analysis, Validation, Visualization Joseph Chan Roles: Data Curation, Formal Analysis, Software, Validation Daisy Sheng Roles: Data Curation, Visualization Alfredo González-Espinoza Roles: Data Curation, Formal Analysis, Validation Christopher Warren Roles: Conceptualization, Data Curation, Resources, Software, Visualization, Writing – Review & Editing OPEN PEER REVIEW DETAILS REVIEWER STATUS This article is included in the Hackathons collection. Abstract Reproducibility is a basis of scientific integrity, yet it remains a significant challenge across disciplines in computational science. This reproducibility crisis is now being met with an Open Science movement, which has risen to prominence within the scientific community and academic libraries especially. To address the need for reproducible computational research and promote Open Science within the community, members of the Open Science and Data Collaborations Program at Carnegie Mellon University Libraries organized a single-day hackathon centered around reproducibility. Partnering with a faculty researcher in English and Digital Humanities, this event allowed several students an opportunity to interact with real research outputs, test the reproducibility of data analyses with code, and offer feedback for improvements. With Python code and data shared by the researcher in an open repository, we revealed that students could successfully reproduce most of the data visualizations, but they required completing some manual setup and modifications to address depreciated libraries to successfully rerun the code. During the event, we also investigated the option of using ChatGPT to debug and troubleshoot rerunning this code. By interacting with a ChatGPT API in the code, we found and addressed the same roadblocks and successfully reproduced the same figures as the participating students. Assessing a second option, we also collaborated with the researcher to publish a compute capsule in Code Ocean. This option presented an alternative to manual setup and modifications, an accessible option for more limited devices like tablets, and a simple solution for outside researchers to modify or build on existing research code. READ ALL READ LESS Keywords Reproducibility, Hackathon, Academic Libraries, Open Science, Metascience, Digital Humanities, Computational Research, Software, Community Engagement Corresponding Author(s) Chasz Griego ( [email protected] ) Close Corresponding author: Chasz Griego Competing interests: No competing interests were disclosed. Grant information: The author(s) declared that no grants were involved in supporting this work. Copyright: © 2024 Griego C et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. How to cite: Griego C, Scotti K, Terveen E et al. A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 :1305 ( https://doi.org/10.12688/f1000research.156917.1 ) First published: 31 Oct 2024, 13 :1305 ( https://doi.org/10.12688/f1000research.156917.1 ) Latest published: 09 Sep 2025, 13 :1305 ( https://doi.org/10.12688/f1000research.156917.2 )  There is a newer version of this article available. Suppress this message for one day. Introduction Reproducibility in scientific research is a highly regarded concept that measures both the credibility and soundness of rigorous studies. 1 , 2 In the context of research, the term reproducibility is often loosely defined or used interchangeably with replicability. The National Academy of Sciences convened in 2019 to develop a definition 3 and differentiate between reproducibility, replicability, and generalizability. 2 , 3 Reproducibility, which is generally consistent with computational reproducibility, is defined as consistent computational outcomes by utilizing identical input data, procedures, methods, code, and analytical conditions. Replicability is defined as uniform outcomes across research projects designed to address the same scientific question, where each project collects its own data. And finally, generalizability is defined as how well findings of a study can be applied to different contexts or populations beyond the original. Modern research continues to face multiple challenges surrounding reproducibility. The adoptable skills and practices that promote reproducibility may vary between fields, 4 – 6 awareness or guidance is often lacking, 4 , 6 , 7 and the time and resources devoted to teaching and practicing reproducibility may be deficient. 8 Computation research, in particular, faces barriers to reproducibility with software inconsistencies, changing versions and dependencies, 9 and insufficient documentation of datasets 10 or methods. 11 For example, Liu and Salganik 10 found that the complexity of employed computational methods impacts reproducibility. The more common methods in a given field may ease reproducibility, as they are familiar and well-documented, but more advanced methods can pose greater challenges, requiring specialized knowledge, environments, and resources that are not as readily available or standardized. Investing in tools and practices around computational reproducibility enhances the transparency and reliability of scientific findings by enabling independent verification of results and fostering further research on established findings. Such a shift in computational research practice is important since it increases research credibility and efficiency and avoids wasted efforts in attempting to build on unreliable results. 12 , 13 The scientific community is increasingly advocating for the use of various tools and practices aimed at improving reproducibility of computational research, 11 , 14 including adopting open-source software, 11 conducting version control (e.g., Git and GitHub), 15 encapsulating computing environments (eg. Docker), 11 , 16 , 17 and following community-driven standards and frameworks. In general, scientific communities are upping advocacy through the Open Science movement. Open Science promotes transparency of research process, data, methodologies, and outputs; enhancing accessibility and usability by the broader scientific community. 18 , 19 Reproducibility is fundamental to open science, 18 , 20 by ensuring verifiable scientific claims to the community; made possible via open sharing of datasets, methodology, code, and using open source software. Overall, alignment with reproducible practices moves the scientific community towards a more collaborative environment. 19 At the Carnegie Mellon University Libraries, a dedicated Open Science & Data Collaborations Program (OSDC) was established in 2018, which highlights the Libraries’ resources and tools that foster open, transparent, and reproducible research. 21 In this program, librarians and functional specialists serve as advocates, consultants, and collaborators for researchers accessing the existing Open Science infrastructure in the Libraries, including CMU’s institutional repository, KiltHub, 22 and research data management services. 23 In addition, the Libraries have held multiple iterations of the Open Science Symposium, 24 which gathered global researchers and thought leaders in academia, industry, and publishing to discuss the ways that Open Science has transformed research. The OSDC program also created roles for STEM PhDs to transition into Open Science and librarianship through an Open Science Postdoctoral Associate position, with Chasz Griego and Kristen Scotti being the first and second candidates in this role, respectively. Through this postdoctoral role, Griego considered ways to explore reproducibility integration and evaluation among researchers and students at CMU. 25 Motivated by the challenges surrounding reproducibility and the growing awareness of Open Science, members of the CMU Libraries and OSDC Program organized a single-day hackathon centered around reproducibility. Hackathons, which are time-bound events for groups to collectively solve or explore a technical problem, often find libraries as a suitable host as a neutral and welcoming space on campus that harbors learning, discovery, and collaboration. 26 The CMU Libraries has effectively hosted multiple hackathons, including Biohackathons and a recent AI Literacy Resource Hackathon. 27 – 29 Libraries and hackathons surrounding reproducibility is also not a novel concept. The Center for Digital Scholarship at Leiden University held ReprohackNL in 2019, where participants attempted to reproduce published results using authors’ provided data and code in a single day event. 30 At this hackathon event, the organizers observed how participants faced challenges with insufficient documentation, data behind a paywall, and problems with code and proprietary software. The CMU Libraries Reproducibility Hackathon aimed to offer a similar model, but here, we focused on the reproducibility of a study with shared data and code around a subject that could create interdisciplinary collaboration across students and researchers at CMU. Purpose of the event The CMU Libraries Reproducibility Hackathon was an event that allowed any participant the opportunity to reproduce research results produced by a university professor. The purpose of such an event is to increase awareness of reproducibility as a realistic piece of the research life cycle, and demonstrate the outcomes of research scrutiny, which could theoretically be done by anyone. Most are familiar with the concept of reproducibility at a surface level, but many may not actually have a grasp on what that may look like, how it may happen, or how impactful it could be for any study. In an extremely idealistic world, a person with any background, experience, or research interest would have access to every material behind a particular study. These materials, again in an ideal case, would include all resources and guidance for a person to reproduce each result, assuring the findings first-hand. In reality, most research is very distanced from such an idealistic scenario, however, an event like a reproducibility hackathon can actually attempt to measure this distance for researchers willing to participate. For this hackathon, we offered one academic researcher the opportunity to put their research outputs up to scrutiny. Regardless of the outcome, a volunteer researcher helps set an example. If the results are highly reproducible, this presents a recognizable feat, and if the results prove more difficult to reproduce, this presents an opportunity to reflect on ways to improve. Work that is less reproducible does not have to be met with shame or guilt, but a humbling opportunity to learn and do better. For the CMU Libraries Reproducibility Hackathon, we teamed up with Christopher Warren, a Professor of English and Associate Department Head in the Dietrich College of Humanities and Social Sciences at CMU. As the subject for a reproducibility assessment, Warren offered the content of a published data analysis of the Oxford Dictionary of National Biography (ODNB), 31 , 32 a collection of biographies for over 60,000 influential figures in British history. Warren’s work is credited as an “audit” of the ODNB, 33 revealing the biases and assumptions hidden beneath the vastness of big data infrastructure. This subject was well aligned with the event, because with an analysis such as Warren’s, which scrutinizes the soundness of a massive data collection, the Python code that reveals the findings must also hold up to its own audit, ensuring that changing versions and dependencies do not prevent repeat analyses. With Warren offering his humanities research as an exemplary case, we offer further dialog around inclusivity of all research areas beyond STEM in the Open Science movement. 34 Event execution We openly distributed a call for participants to anyone interested from CMU and other institutions in the greater Pittsburgh area. The call noted that knowledge and awareness of scientific programming with Python and/or R was a desired, but not required, condition for participation. Participants submitted their interest and background through a short form and had the option to attend one of two information sessions a week prior to the event. During the information session, participants received a link to Warren’s manuscript to provide sufficient time to read into the details of the work. They were asked to not search for any items related to the work, avoiding the risk of early access to data and code posted to a repository. The hackathon was scheduled for just one day, with participants and organizers convening in a library space during typical business hours (9am - 5pm). A project page on Open Science Framework (OSF) 35 was created for the event, where participants could access event information and guidance, a copy of Warren’s manuscript, and documents with links to access data and code. In addition to the published manuscript, Warren deposited supplementary research outputs to Harvard Dataverse, which includes a Jupyter Notebook, CSV files, Open Refine transformation JSON files, and others. 32 This collection of outputs served as one option for participants to use to attempt to reproduce the study. This option is arguably open, having research outputs deposited in a repository to ensure secure storage, public access, and findability with a DOI, and this option used non-proprietary file formats such as CSV and JSON, and open-source Python code written in notebooks. Such attributes greatly promote reusability once users download each output and ensure that Python and all necessary libraries are installed. We also provided a second option that aims to greatly facilitate reproducible computational research. The same data and code found in Harvard Dataverse were organized and configured in a Code Ocean capsule. 36 Code Ocean is a platform that allows researchers to archive research data and code along with preserved language and library versions plus dependencies and a metadata record in a self-contained capsule. Capsules are discoverable with a DOI, and outside users can perform reproducible runs that compile and execute code in a virtual environment, producing results that match the original analysis. Observations and reflections In the following, we summarize the experiences of participants in the reproducibility hackathon. Some notes are also listed briefly in Table 1 . One participant, Elizabeth Terveen, chose the option of downloading the materials from Harvard Dataverse and attempting to run the Jupyter Notebook in a new Python environment working with the Anaconda package manager. To run the code, Terveen had to install four Python libraries (Pandas, Nltk, Matplotlib, and Plotly). One of the existing import statements in the notebook, regarding Plotly, returned an error due to a module depreciation. Table 1. Technical issues experienced from participants attempting to reproduce results by running the code corresponding to Warren’s work. 31 Notable issues Several Python libraries required installation Some import statements needed to be updated because of module depreciation New arguments were required to successfully run a function A function keyword argument name was updated File paths in the code were not consistent with file organization in the repository Of the figures produced in the notebook, fourteen figures were successfully reproduced by Terveen, with six of these figures requiring some troubleshooting. This included supplying values for arguments in functions that were not specified in the original code, but a resulting error stated as required. The name of a keyword argument related to a Matplotlib function had also become depreciated, which required an update to the original code. Eight figures were not successfully reproduced because of file paths in the code that were not consistent with directory structure Terveen had after downloading from the repository. A second participant, Joseph Chan, also chose the option to download the materials from Harvard Dataverse. Chan immediately noted that if a “requirements.txt” file was included with the original deposit, any user trying to reproduce the results would be able to download and install the appropriate packages and versions. Based on Chan’s setup, five python libraries and Jupyter needed to be installed, including Pandas, Nltk, Pickle, Numpy, and Matplotlib. Additionally, a plotting library was deprecated, requiring an import statement to be updated. Once again, keyword arguments were depreciated and had to be updated. Two figures did not display upon running and were unable to be solved. Chan also created a modified version of the research outputs as a zip file uploaded to OSF ( https://osf.io/hvbzm ), which contains a requirements.txt file, all necessary data, and all the code from the notebook in a single Python (.py) file. With this, users can simply reproduce the work by executing a shell script from a terminal. Copies of figures that were successfully reproduced were uploaded to OSF ( https://osf.io/gqk8e/ ), with titles altered with either a note or a name to verify that the figures were original copies (see Figure 1 for examples). Figure 1. Reproduced versions of Figures 1, 21, and 22 from Warren’s work. 31 Overall, this process of reproducing results using files downloaded from a repository produced several challenges with requirements to install packages, understand depreciations for some packages, and troubleshooting changes in syntax from evolving versions of software. Though there was trial and error, participants were able to successfully reproduce figures from this study. Additionally, participants provided their own insight to develop an updated collection of research outputs that help others reproduce this work more easily. Additional outcomes and lessons learned Alfredo González-Espinoza, our colleague who participated in the assessment as well, offered an interesting anecdote while pursuing reproducibility via both Harvard Dataverse and Code Ocean ( https://osf.io/brn6d ). In order to run the code downloaded from the repository, he had to install Python, Jupyter, all required libraries, and found the same depreciation errors as other participants; however, he was successful with reproducibility via the Code Ocean capsule. González-Espinoza’s scenario was interesting because he used a gaming laptop to reproduce from the repository, which doesn’t make a difference when the issue is installing software, but for the capsule he accessed the code via an old Chromebook tablet. This individual experience calls to attention the fact that a platform like Code Ocean may promote accessibility for people that may have limited access to computers, and instead only have access to devices that one can’t use to download and install software as easily. The hackathon offered an additional opportunity for exploration. Kristen Scotti investigated ways to troubleshoot reproducing the output of the code using generative AI. Python, like many other programming languages, undergoes frequent updates, deprecating functions or changing syntax overtime, making author-provided code more difficult to work with or potentially incompatible with newer versions. Here, Scotti ran the author-provided code through ChatGPT to request updates ( https://osf.io/r65ev ). ChatGPT’s programming capabilities include the ability to interpret, debug, and suggest updates to code. 37 ChatGPT updated code successfully for all recreated plots except one, which also provided issues for other participants. The likely culprit was formatting issues due to syntax changes and complexity of the provided code block. A more effective approach might be to ask ChatGPT to convert the code in smaller sections, though this often requires someone experienced in the language to fine-tune both the prompts and the outputs. Outside the hackathon setting, we revisited the capsule version of Warren’s research to test how easily his work could be adapted within this platform. For instance, figures 28 and 29 in the manuscript have prompted discussion in other sources. 33 , 38 Figure 29, in particular, displays the historical significance of selected women in the ODNB, and the span of living years of these women from 1830-2000 (listed here in Figure 2b ). However, horizontal bars representing a span of years could instead incidentally suggest the number of women belonging to the corresponding historical significance. We addressed this ambiguity by creating a new version of the capsule on Code Ocean 39 where additional cells in the Jupyter Notebook create new versions of these figures which explicitly state the numbers of years each vertical bar spans ( Figure 2a ). Figure 2. Adapted version (a) of Figure 29 (b) from Warren’s work. 31 Discussion and conclusions We have outlined the execution and outcomes of a Reproducibility Hackathon, hosted by CMU Libraries, that focused on the reusability of data and code shared by a faculty member from the university. This event allowed students to interact with real research outputs to learn potentially new skills and fields of study while simultaneously offering their perspective. In reviewing the reproducibility of research code from the participating faculty member, Christopher Warren, students showed that manual setup was needed to rerun research code, and along with that, depreciations required them to write updates and modifications. One participating student sought a way to encapsulate the code so that everything was packaged and shareable, enabling an outside user to properly reproduce the run by executing a single script. Lastly, we attempted to rerun the code while using a ChatGPT API to debug. This option identified and addressed the same errors that student participants met and successfully reproduced the same figures. Code Ocean presented an alternative to sharing code without manual installations or depreciations. An alternative of Warren’s outputs were assembled and published in a reproducible capsule on Code Ocean. This option presented itself as a more accessible way for a user to interact with computational research, as a participating colleague revealed the ease of interacting with the capsule on a tablet device. Finally, we adapted Warren’s code in Code Ocean to produce an alternative depiction of data visualizations presented in the original work. Doing this offered an example of how a reproducibility-centered platform can promote research as ongoing conversation that openly invites community members to offer constructive insights. A reproducibility hackathon at an academic institution creates an open environment for researchers to put their published work to the test, opening themselves to constructive feedback around reproducibility, and allowing students or other researchers to interact with real research artifacts. We presented our experience here to offer a blueprint for other academic libraries to execute similar events, boost awareness of the threats to preserving research code, and highlight how new platforms eliminate these threats to reproducibility and enhance adaptation along the research lifecycle. Ethics and consent Ethical approval and consent were not required. Author contributions Chasz G: Conceptualization, Data Curation, Formal Analysis, Methodology, Project Administration, Resources, Software, Supervision, Validation, Writing – Original Draft Preparation; Kristen S: Conceptualization, Data Curation, Formal Analysis, Resources, Software, Validation, Visualization, Writing – Original Draft Preparation; Elizabeth T: Data Curation, Formal Analysis, Validation, Visualization; Joseph C: Data Curation, Formal Analysis, Software, Validation; Daisy S: Data Curation, Visualization; Alfredo G-E: Data Curation, Formal Analysis, Validation; Christopher W: Conceptualization, Data Curation, Resources, Software, Visualization, Writing – Review & Editing Data availability Open Science Framework: Reproducibility Hackathon 2024. https://doi.org/10.17605/OSF.IO/AQ69R . 35 The project contains the following underlying data: • All original documents and materials prepared and produced from the event. Data are available under the terms of the Creative Commons Attribution 4.0 International license (CC-BY 4.0). Harvard Dataverse: Replication Data for: Historiography’s Two Voices: Data Infrastructure and History at Scale in the Oxford Dictionary of National Biography (ODNB). https://doi.org/10.7910/DVN/D3KFLP . 32 The project contains the following underlying data: • All data and code supporting the original work that was the focus of the hackathon. Data are available under the terms of the Creative Commons 1.0 Universal Dead (CC0 1.0) Code Ocean: Replication Data for: Historiography’s Two Voices: Data Infrastructure and History at Scale in the Oxford Dictionary of National Biography (ODNB) (Version 1). https://doi.org/10.24433/CO.6313661.v1 . 36 The project contains the following underlying data: • All data, code, and software environment details supporting the original work that was the focus of the hackathon. Data and code are available under the terms of the Creative Commons CC0 licenses with no rights reserved. Code Ocean: Replication Data for: Historiography’s Two Voices: Data Infrastructure and History at Scale in the Oxford Dictionary of National Biography (ODNB) (Version 2). https://doi.org/10.24433/CO.6313661.v2 . 39 The project contains the following underlying data: • All data, code, and software environment details supporting the original work that was the focus of the hackathon as well as appended code that adapts Figures 28 and 29 from the original study. Data and code are available under the terms of the Creative Commons CC0 1.0 licenses with no rights reserved. Acknowledgements We thank Melanie Gainey and Tom Hughes from the CMU Libraries for useful discussions about hackathons and their support during the event. References 1. Buckheit JB, Donoho DL: WaveLab and Reproducible Research.Antoniadis A, Oppenheim G, editors. Wavelets Stat. New York, NY: Springer; 1995; pp. 55–81. Publisher Full Text 2. Barba LA: Terminologies for reproducible research. ArXiv Prepr. ArXiv180203311. 2018. 3. Reproducibility and Replicability in Science. Natl. Acad. Sci. 2019. Reference Source 4. Baker M, Baker M: 1,500 scientists lift the lid on reproducibility. Nature. 2016; 533 : 452–454. Publisher Full Text 5. Mullane K, Williams M: Enhancing reproducibility: Failures from Reproducibility Initiatives underline core challenges. Biochem. Pharmacol. 2017; 138 : 7–18. PubMed Abstract | Publisher Full Text 6. Kohrs FE, Auer S, Bannach-Brown A, et al. : Eleven strategies for making reproducible research and open science training the norm at research institutions. elife. 2023; 12 : e89736. PubMed Abstract | Publisher Full Text | Free Full Text 7. Rich-Edwards JW, Maney DL: Best practices to promote rigor and reproducibility in the era of sex-inclusive research. elife. 2023; 12 : e90623. PubMed Abstract | Publisher Full Text | Free Full Text 8. Six factors affecting reproducibility in life science research and how to handle them.n.d. (accessed May 28, 2024). Reference Source 9. Implementing Reproducible Research. Routledge CRC Press; n.d. (accessed May 28, 2024). Reference Source 10. Liu DM, Salganik MJ: Successes and struggles with computational reproducibility: lessons from the fragile families challenge. Socius. 2019; 5 : 2378023119849803. PubMed Abstract | Publisher Full Text | Free Full Text 11. Piccolo SR, Frampton MB: Tools and techniques for computational reproducibility. Gigascience. 2016; 5 : s13742–s13016. Publisher Full Text 12. Ioannidis JPA: Why Most Published Research Findings Are False. PLoS Med. 2005; 2 : e124. PubMed Abstract | Publisher Full Text | Free Full Text 13. Peng RD: Reproducible Research in Computational Science. Science. 2011; 334 : 1226–1227. Publisher Full Text 14. Brito JJ, Li J, Moore JH, et al. : Recommendations to enhance rigor and reproducibility in biomedical research. GigaScience. 2020; 9 : giaa056. PubMed Abstract | Publisher Full Text | Free Full Text 15. Ram K: Git can facilitate greater reproducibility and increased transparency in science, Source Code. Biol. Med. 2013; 8 : 1–8. Publisher Full Text 16. Boettiger C: An introduction to Docker for reproducible research. ACM SIGOPS Oper. Syst. Rev. 2015; 49 : 71–79. Publisher Full Text 17. Cheifet B: Promoting reproducibility with Code Ocean. Genome Biol. 2021; 22 : 65. PubMed Abstract | Publisher Full Text | Free Full Text 18. Munafò MR, Nosek BA, Bishop DVM, et al. : A manifesto for reproducible science. Nat. Hum. Behav. 2017; 1 : 0021–0021. PubMed Abstract | Publisher Full Text | Free Full Text 19. McKiernan EC, Bourne PE, Brown CT, et al. : How open science helps researchers succeed. elife. 2016; 5 . PubMed Abstract | Publisher Full Text | Free Full Text 20. Munafò M: Open science and research reproducibility. Ecancermedicalscience. 2016; 10 . PubMed Abstract | Publisher Full Text | Free Full Text 21. Wang H, Gainey M, Campbell P, et al. : Implementation and assessment of an end-to-end Open Science & Data Collaborations program. F1000 Res. 2022; 11 : 501. PubMed Abstract | Publisher Full Text | Free Full Text 22. Scherer D, Valen D: Balancing Multiple Roles of Repositories. Developing a Comprehensive Repository at Carnegie Mellon University, Publications; 2019; vol. 7 . ; p. 30. Publisher Full Text 23. Van Tuyl SI: Developing a research data management services infrastructure at. Carnegie Mellon University, Carnegie Mellon University; 2013. 24. Wang H, Gainey M, Van Gulick A: Carnegie Mellon’s first Open Science Symposium - Themes about research data and their reuse.2019. (accessed April 1, 2024). Reference Source 25. Griego C: Integrating and Evaluating Best Practices in Research Reproducibility. Carnegie Mellon University; 2023. (accessed March 9, 2024). Reference Source 26. Longmeier MM: Hackathons and libraries: the evolving landscape 2014-2020.2021. 27. Bongiovanni E, Slayton E, Agate N, et al. : AI Literacy Resource Hackathon. Open Sci. Framew. 2024. Publisher Full Text 28. Al Khleifat A, Smith J, Blobner B, et al. : SnpReportR: A Tool for Clinical Reporting of RNAseq Expression and Variants, BioHackrXiv.2021. 29. Kubica J, Kumar R, Tan G, et al. : The fourth annual Carnegie Mellon Libraries hackathon for biomedical data management, knowledge graphs, and deep learning, BioHackrXiv.2023. 30. Hettne KM, Proppert R, Nab L, et al. : ReprohackNL 2019: how libraries can promote research reproducibility through community engagement. IASSIST Q. 2020; 44 : 1–10. Publisher Full Text 31. Warren CN: Historiography’s Two Voices: Data Infrastructure and History at Scale in the Oxford Dictionary of National Biography (ODNB). J. Cult. Anal. 2018; 3 . Publisher Full Text 32. Warren C: Replication Data for: Historiography’s Two Voices: Data Infrastructure and History at Scale in the Oxford Dictionary of National Biography (ODNB).2018. Publisher Full Text 33. Guldi J: The Dangerous Art of Text Mining: A Methodology for Digital History. Cambridge: Cambridge University Press; 2023. Publisher Full Text 34. Longley Arthur P, Hearn L: Toward open research: A narrative review of the challenges and opportunities for open humanities. J. Commun. 2021; 71 : 827–853. 35. Griego C, Warren CN, Scotti K, et al. : Reproducibility Hackathon.2024; 2024 . (accessed April 23, 2024). Reference Source 36. Warren C: Replication Data for: Historiography’s Two Voices: Data Infrastructure and History at Scale in the Oxford Dictionary of National Biography (ODNB).2024. Publisher Full Text 37. Biswas S: Role of ChatGPT in Computer Programming.: ChatGPT in Computer Programming., Mesopotamian. J. Comput. Sci. 2023; 2023 : 8–16. 38. Guldi J: From Critique to Audit: A Pragmatic Response to the Climate Emergency from the Humanities and Social Sciences, and a Call to Action. KNOW J. Form. Knowl. 2021; 5 : 169–196. Publisher Full Text 39. Warren C, Scotti K: Replication Data for: Historiography’s Two Voices: Data Infrastructure and History at Scale in the Oxford Dictionary of National Biography (ODNB).2024. Publisher Full Text Comments on this article Comments (0) Version 2 VERSION 2 PUBLISHED 31 Oct 2024 ADD YOUR COMMENT Comment Author details Author details 1 University Libraries, Carnegie Mellon University, Pittsburgh, Pennsylvania, 15213, USA 2 Carnegie Mellon University School of Computer Science, Pittsburgh, Pennsylvania, 15213, USA 3 Carnegie Mellon University Department of English, Pittsburgh, Pennsylvania, 15213, USA Chasz Griego Roles: Conceptualization, Data Curation, Formal Analysis, Methodology, Project Administration, Resources, Software, Supervision, Validation, Writing – Original Draft Preparation Kristen Scotti Roles: Conceptualization, Data Curation, Formal Analysis, Resources, Software, Validation, Visualization, Writing – Original Draft Preparation Elizabeth Terveen Roles: Data Curation, Formal Analysis, Validation, Visualization Joseph Chan Roles: Data Curation, Formal Analysis, Software, Validation Daisy Sheng Roles: Data Curation, Visualization Alfredo González-Espinoza Roles: Data Curation, Formal Analysis, Validation Christopher Warren Roles: Conceptualization, Data Curation, Resources, Software, Visualization, Writing – Review & Editing Competing interests No competing interests were disclosed. Grant information The author(s) declared that no grants were involved in supporting this work. Article Versions (2) version 2 Revised Published: 09 Sep 2025, 13:1305 https://doi.org/10.12688/f1000research.156917.2 version 1 Published: 31 Oct 2024, 13:1305 https://doi.org/10.12688/f1000research.156917.1 Copyright © 2024 Griego C et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. Download Export To Sciwheel Bibtex EndNote ProCite Ref. Manager (RIS) Sente metrics Views Downloads F1000Research - - PubMed Central info_outline Data from PMC are received and updated monthly. - - Citations open_in_new 0 open_in_new 0 open_in_new SEE MORE DETAILS CITE how to cite this article Griego C, Scotti K, Terveen E et al. A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 :1305 ( https://doi.org/10.12688/f1000research.156917.1 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS track receive updates on this article Track an article to receive email alerts on any updates to this article. TRACK THIS ARTICLE Share Open Peer Review Current Reviewer Status: ? Key to Reviewer Statuses VIEW HIDE Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Version 1 VERSION 1 PUBLISHED 31 Oct 2024 Views 0 Cite How to cite this report: Gomez-Diaz T and Recio T. Reviewer Report For: A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 :1305 ( https://doi.org/10.5256/f1000research.172297.r355235 ) The direct URL for this report is: https://f1000research.com/articles/13-1305/v1#referee-response-355235 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 07 Jan 2025 Teresa Gomez-Diaz , Universite Gustave Eiffel / Laboratoire d'informatique Gaspard-Monge / CNRS, Marne-la-Vallée, Île-de-France, France Tomas Recio , Universidad Nebrija, Madrid, Spain Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.172297.r355235 CASE STUDY A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code https://f1000research.com/articles/13-1305/v1 This case study refers to a hackathon conducted by the members of the Carnegie Mellon University (CMU) ... Continue reading READ ALL CASE STUDY A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code https://f1000research.com/articles/13-1305/v1 This case study refers to a hackathon conducted by the members of the Carnegie Mellon University (CMU) Libraries in order to reproduce research results produced by Professor Christopher Warren on data analysis of the Oxford Dictionary of National Biography (ODNB). The event was open for participants coming from CMU and other institutions in the greater Pittsburgh area and was conducted by members of the CMU Libraries. This experience gives an interesting setting to explore reproducibility of published research results by a Professor of the University who also provides data and the produced software involved in the research, and shows a good example of a collaboration between Librarians and members (students?) of the Computer Science School. However, there are a few points in this manuscript that need attention. 1. Abstract. As included in the Case Studies guidelines [ https://f1000research.com/for-authors/article-guidelines/case-studies ], abstract should be structured into Background, Methods, Results, and Conclusions . In particular, the conclusions should be better presented in the abstract (and in the manuscript, see later). 2. Hackathon context. As explained in the document (see also [ https://www.library.cmu.edu/abou)t/news/2024-04/reproducibility-hackathon ], the event was open to a large (institutional) public, but the title of the manuscript refers to students. More insight about the participants (how many, how many students, which level, which scientific background…) should be provided in order to better seize the complexity of the event. Please, clarify if the three authors of the manuscript coming from the Computer Science School where also participants of the event and if they are students or Members of the School. Moreover, it was indicated that “participants could have knowledge and awareness of scientific programming with Python and/or R” was desired, but was not required. Was this a difficulty? How many participants had or had not programming knowledge? 3. Problems with data and code It is mentioned that: “At this hackathon event, the organizers observed how participants faced challenges with insufficient documentation, data behind a paywall, and problems with code and proprietary software.” But there is no further information about these observations. 4. Reproducibility success. It is mentioned that: “Of the figures produced in the notebook, fourteen figures were successfully reproduced by Terveen … and eight figures were not successfully reproduced.” Or that “Though there was trial and error, participants were able to successfully reproduce figures from this study”. But which is the percentage of success? How many figures were in the initial work by Prof. Warren and how many were reproduced by all the participants? 5. Insufficient documentation and code longevity. As noticed by participant Chan, a “requirements.txt” file could help to facilitate the understanding of the software context of the code produced by Prof. Warren. Our work (see Gomez-Diaz T, et al., 2022 [Ref 1]) goes further in this kind of recommendations, proposing flexible procedures for research software and research data dissemination. It is extremely idealistic to expect that software released at some time will find easily runable environments, unless there is the available resources (human, funding) to keep the necessary maintenance of the produced software (who can use Windows version 5?). To accompany the research produced software with good documentation and description of the necessary components will help to get to reproducibility environments, that is, to follow appropriate dissemination procedures will be of help for reproducibility issues. Nevertheless, it will be not always possible to ensure the full reproducibility and, in our opinion, a partial solution consists in to release the research software as free/open source software, where the accessibility and the study of the source code is confirmed by the use of free/open source licenses. 6. Conclusions. Conclusions are poorly described, and more information on the benefits, found drawbacks and other consequences of the event, for participants, students, Prof. Warren and event organizers should be provided to complete this case study. 7. References. We would like to suggest the inclusion of these references: for Open Science: UNESCO: UNESCO Recommendation on Open Science. 2021 [Ref 2] for Free/Open Source Software: K. Fogel (2005-2022). Producing Open Source Software. How to Run a Successful Free Software Project, https://producingoss.com for an example of a Journal giving a reproducibility environment for their publications: IPOL Journal · Image Processing On Line [ https://www.ipol.im/ Nicolaï, A., et al., 2022 [Ref 3] 8. Use of privately owned companies. We would like to raise awareness about the use of privately owned companies like Code Ocean [ https://codeocean.com/ ] for the storage and manipulation of data and code produced within the scientific community. Usually the scientific community is not well aware of legal issues related to the terms of use of these platforms and their consequences, like, for example: `` You hereby grant Code Ocean a worldwide, perpetual, irrevocable, royalty-free right and license to use and exploit any ideas, suggestions, comments, recommendations, enhancement requests or other input provided by you about the Platform to Code Ocean. '' (see point 3. Feedback in [ https://codeocean.com/terms-of-use ]). Regarding OSF, the Center for Open Science, Inc. (referred to as "the COS", "Open Science Framework", "OSF") declares to be a nonprofit organization, but also provides with similar terms of use: `` by depositing any Content (including data and links, but not the linked content) in a COS account, you grant the COS a perpetual, irrevocable, worldwide, royalty-free license to store, reproduce, transmit, distribute, publicly perform, and publicly display such Happy… '' (see further information in [ https://github.com/CenterForOpenScience/cos.io/blob/master/TERMS_OF_USE.md ]). Universities and other Research Performing Institutions, and, in particular, University and other institutional Libraries, could study the proposition of University or institutionally owned platforms and/or services to deal with their scientific production. 9. To complete the background context of this case study, some information on previously hosted hackathons by the CMU Libraries, like Biohackathons and an AI Literacy Resource Hackathon, could be included as they can be of interest for the Hackathon community. Is the background of the case’s history and progression described in sufficient detail? Partly Is the work clearly and accurately presented and does it cite the current literature? Partly If applicable, is the statistical analysis and its interpretation appropriate? Partly Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly Is the case presented with sufficient detail to be useful for teaching or other practitioners? Partly References 1. Gomez-Diaz T, Recio T: Research Software vs. Research Data II: Protocols for Research Data dissemination and evaluation in the Open Science context. F1000Res . 2022; 11 : 117 PubMed Abstract | Publisher Full Text 2. UNESCO: UNESCO Recommendation on Open Science. 2021. Publisher Full Text 3. A, Nicolaï Q, Bammey M, Gardella T, et al.: The approach to reproducible research of the Image Processing On Line (IPOL) journal. http://www.scielo.edu.uy/pdf/info/v27n1/2301-1378-info-27-01-76.pdf . 2022. Competing Interests: No competing interests were disclosed. Reviewer Expertise: Research Software, Research Data, Open Science, dissemination, evaluation We confirm that we have read this submission and believe that we have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however we have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Gomez-Diaz T and Recio T. Reviewer Report For: A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 :1305 ( https://doi.org/10.5256/f1000research.172297.r355235 ) The direct URL for this report is: https://f1000research.com/articles/13-1305/v1#referee-response-355235 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 12 Sep 2025 Chasz Griego , University Libraries, Carnegie Mellon University, Pittsburgh, 15213, USA 12 Sep 2025 Author Response Thank you, we really appreciate your review and recommendations. We are grateful that you could offer your experience and perspective to help strengthen our paper, as well as offer questions ... Continue reading Thank you, we really appreciate your review and recommendations. We are grateful that you could offer your experience and perspective to help strengthen our paper, as well as offer questions that helped us add more clarity and critical points in our writing. Please see our responses to your comments, where we also note any changes made to the original manuscript: “abstract should be structured into Background, Methods, Results, and Conclusions . In particular, the conclusions should be better presented in the abstract (and in the manuscript, see later).” Response: Thank you for bringing this to our attention. We have restructured the abstract, resulting in something that we think will be easier for a reader to navigate. We also revised our conclusions, both here and later in the manuscript (also mentioned later in this response). “More insight about the participants (how many, how many students, which level, which scientific background…).... Please, clarify if the three authors of the manuscript coming from the Computer Science School where also participants of the event and if they are students or Members of the School.” Response: Thank you for noting this ambiguity. We've added a statement clarifying that the three authors from the School of Computer Science were undergraduate students participating in the event. This clarification did significantly improve our narrative and summary. “it was indicated that “participants could have knowledge and awareness of scientific programming with Python and/or R” was desired, but was not required. Was this a difficulty? How many participants had or had not programming knowledge?” Response: This is good to clarify. Each student already had an established skillset with Python programming. With the limited pool of participants, a desire for programming knowledge was not a challenge, but this could be something to consider for future events, especially considering the aspect of reproducibility by any person or team. “It is mentioned that: “At this hackathon event, the organizers observed how participants faced challenges with insufficient documentation, data behind a paywall, and problems with code and proprietary software.” But there is no further information about these observations.” Response: We are grateful that you noted this because it actually revealed an error in our editing. This sentence is in reference to the cited hackathon in the previous statement (ReprohackNL). The paragraph break between these statements was an error. We fixed this error to hopefully make better sense. “But which is the percentage of success? How many figures were in the initial work by Prof. Warren and how many were reproduced by all the participants?” Response: Thank you for pointing out these details, which are a bit vague. We've edited the manuscript to mention that the original work had twenty-five figures to reproduce. We also changed the wording in the latter sentence to read with more accuracy. Our paper is not focusing on if the entire work was reproduced during this event, rather, the paper is focusing on the details and nuances of research code that are revealed when a group comes together to try to reproduce the outcomes. “Our work (see Gomez-Diaz T, et al., 2022 [Ref 1]) goes further in this kind of recommendations, proposing flexible procedures for research software and research data dissemination. It is extremely idealistic to expect that software released at some time will find easily runable environments, unless there is the available resources… Nevertheless, it will be not always possible to ensure the full reproducibility and, in our opinion, a partial solution consists in to release the research software as free/open source software, where the accessibility and the study of the source code is confirmed by the use of free/open source licenses.” Response: These were all excellent points, especially those written in your work. We decided to add another paragraph to the conclusion section to address how reproducibility also relies on adequate resources and systems, which their availability may also depreciate with time, and that documentation offers further guidance and transparency to help other users reuse research software and data, even when resources are scarce. We appreciated the opportunity to add this critical aspect to our conclusions. “Conclusions are poorly described, and more information on the benefits, found drawbacks and other consequences of the event, for participants, students, Prof. Warren and event organizers should be provided to complete this case study.” Response: Thank you for this suggestion. Your review greatly helped us better communicate the impact and implications of running this type of hackathon. It also adds further context for readers to learn how to readapt this model. The final paragraphs of this paper now offer further detail about the benefits, drawbacks, and lessons learned. “We would like to suggest the inclusion of these references…” Response: Thank you for these great recommendations. These references were added to the appropriate places in the manuscript. “Use of privately owned companies…” Response: We found your point here one of the most important to call to attention. In our libraries, and even within our Open Science initiatives, we often promote tools and platforms citing the benefits of places that host your research materials and offer increased accessibility and interoperability. However, it is true that while these companies create these products, they also have business models to sustain and insurance around their products. This was important to note in our concluding section to highlight that researchers and librarians may have rightful reservations when using these products. Thank you, we really appreciate your review and recommendations. We are grateful that you could offer your experience and perspective to help strengthen our paper, as well as offer questions that helped us add more clarity and critical points in our writing. Please see our responses to your comments, where we also note any changes made to the original manuscript: “abstract should be structured into Background, Methods, Results, and Conclusions . In particular, the conclusions should be better presented in the abstract (and in the manuscript, see later).” Response: Thank you for bringing this to our attention. We have restructured the abstract, resulting in something that we think will be easier for a reader to navigate. We also revised our conclusions, both here and later in the manuscript (also mentioned later in this response). “More insight about the participants (how many, how many students, which level, which scientific background…).... Please, clarify if the three authors of the manuscript coming from the Computer Science School where also participants of the event and if they are students or Members of the School.” Response: Thank you for noting this ambiguity. We've added a statement clarifying that the three authors from the School of Computer Science were undergraduate students participating in the event. This clarification did significantly improve our narrative and summary. “it was indicated that “participants could have knowledge and awareness of scientific programming with Python and/or R” was desired, but was not required. Was this a difficulty? How many participants had or had not programming knowledge?” Response: This is good to clarify. Each student already had an established skillset with Python programming. With the limited pool of participants, a desire for programming knowledge was not a challenge, but this could be something to consider for future events, especially considering the aspect of reproducibility by any person or team. “It is mentioned that: “At this hackathon event, the organizers observed how participants faced challenges with insufficient documentation, data behind a paywall, and problems with code and proprietary software.” But there is no further information about these observations.” Response: We are grateful that you noted this because it actually revealed an error in our editing. This sentence is in reference to the cited hackathon in the previous statement (ReprohackNL). The paragraph break between these statements was an error. We fixed this error to hopefully make better sense. “But which is the percentage of success? How many figures were in the initial work by Prof. Warren and how many were reproduced by all the participants?” Response: Thank you for pointing out these details, which are a bit vague. We've edited the manuscript to mention that the original work had twenty-five figures to reproduce. We also changed the wording in the latter sentence to read with more accuracy. Our paper is not focusing on if the entire work was reproduced during this event, rather, the paper is focusing on the details and nuances of research code that are revealed when a group comes together to try to reproduce the outcomes. “Our work (see Gomez-Diaz T, et al., 2022 [Ref 1]) goes further in this kind of recommendations, proposing flexible procedures for research software and research data dissemination. It is extremely idealistic to expect that software released at some time will find easily runable environments, unless there is the available resources… Nevertheless, it will be not always possible to ensure the full reproducibility and, in our opinion, a partial solution consists in to release the research software as free/open source software, where the accessibility and the study of the source code is confirmed by the use of free/open source licenses.” Response: These were all excellent points, especially those written in your work. We decided to add another paragraph to the conclusion section to address how reproducibility also relies on adequate resources and systems, which their availability may also depreciate with time, and that documentation offers further guidance and transparency to help other users reuse research software and data, even when resources are scarce. We appreciated the opportunity to add this critical aspect to our conclusions. “Conclusions are poorly described, and more information on the benefits, found drawbacks and other consequences of the event, for participants, students, Prof. Warren and event organizers should be provided to complete this case study.” Response: Thank you for this suggestion. Your review greatly helped us better communicate the impact and implications of running this type of hackathon. It also adds further context for readers to learn how to readapt this model. The final paragraphs of this paper now offer further detail about the benefits, drawbacks, and lessons learned. “We would like to suggest the inclusion of these references…” Response: Thank you for these great recommendations. These references were added to the appropriate places in the manuscript. “Use of privately owned companies…” Response: We found your point here one of the most important to call to attention. In our libraries, and even within our Open Science initiatives, we often promote tools and platforms citing the benefits of places that host your research materials and offer increased accessibility and interoperability. However, it is true that while these companies create these products, they also have business models to sustain and insurance around their products. This was important to note in our concluding section to highlight that researchers and librarians may have rightful reservations when using these products. Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 12 Sep 2025 Chasz Griego , University Libraries, Carnegie Mellon University, Pittsburgh, 15213, USA 12 Sep 2025 Author Response Thank you, we really appreciate your review and recommendations. We are grateful that you could offer your experience and perspective to help strengthen our paper, as well as offer questions ... Continue reading Thank you, we really appreciate your review and recommendations. We are grateful that you could offer your experience and perspective to help strengthen our paper, as well as offer questions that helped us add more clarity and critical points in our writing. Please see our responses to your comments, where we also note any changes made to the original manuscript: “abstract should be structured into Background, Methods, Results, and Conclusions . In particular, the conclusions should be better presented in the abstract (and in the manuscript, see later).” Response: Thank you for bringing this to our attention. We have restructured the abstract, resulting in something that we think will be easier for a reader to navigate. We also revised our conclusions, both here and later in the manuscript (also mentioned later in this response). “More insight about the participants (how many, how many students, which level, which scientific background…).... Please, clarify if the three authors of the manuscript coming from the Computer Science School where also participants of the event and if they are students or Members of the School.” Response: Thank you for noting this ambiguity. We've added a statement clarifying that the three authors from the School of Computer Science were undergraduate students participating in the event. This clarification did significantly improve our narrative and summary. “it was indicated that “participants could have knowledge and awareness of scientific programming with Python and/or R” was desired, but was not required. Was this a difficulty? How many participants had or had not programming knowledge?” Response: This is good to clarify. Each student already had an established skillset with Python programming. With the limited pool of participants, a desire for programming knowledge was not a challenge, but this could be something to consider for future events, especially considering the aspect of reproducibility by any person or team. “It is mentioned that: “At this hackathon event, the organizers observed how participants faced challenges with insufficient documentation, data behind a paywall, and problems with code and proprietary software.” But there is no further information about these observations.” Response: We are grateful that you noted this because it actually revealed an error in our editing. This sentence is in reference to the cited hackathon in the previous statement (ReprohackNL). The paragraph break between these statements was an error. We fixed this error to hopefully make better sense. “But which is the percentage of success? How many figures were in the initial work by Prof. Warren and how many were reproduced by all the participants?” Response: Thank you for pointing out these details, which are a bit vague. We've edited the manuscript to mention that the original work had twenty-five figures to reproduce. We also changed the wording in the latter sentence to read with more accuracy. Our paper is not focusing on if the entire work was reproduced during this event, rather, the paper is focusing on the details and nuances of research code that are revealed when a group comes together to try to reproduce the outcomes. “Our work (see Gomez-Diaz T, et al., 2022 [Ref 1]) goes further in this kind of recommendations, proposing flexible procedures for research software and research data dissemination. It is extremely idealistic to expect that software released at some time will find easily runable environments, unless there is the available resources… Nevertheless, it will be not always possible to ensure the full reproducibility and, in our opinion, a partial solution consists in to release the research software as free/open source software, where the accessibility and the study of the source code is confirmed by the use of free/open source licenses.” Response: These were all excellent points, especially those written in your work. We decided to add another paragraph to the conclusion section to address how reproducibility also relies on adequate resources and systems, which their availability may also depreciate with time, and that documentation offers further guidance and transparency to help other users reuse research software and data, even when resources are scarce. We appreciated the opportunity to add this critical aspect to our conclusions. “Conclusions are poorly described, and more information on the benefits, found drawbacks and other consequences of the event, for participants, students, Prof. Warren and event organizers should be provided to complete this case study.” Response: Thank you for this suggestion. Your review greatly helped us better communicate the impact and implications of running this type of hackathon. It also adds further context for readers to learn how to readapt this model. The final paragraphs of this paper now offer further detail about the benefits, drawbacks, and lessons learned. “We would like to suggest the inclusion of these references…” Response: Thank you for these great recommendations. These references were added to the appropriate places in the manuscript. “Use of privately owned companies…” Response: We found your point here one of the most important to call to attention. In our libraries, and even within our Open Science initiatives, we often promote tools and platforms citing the benefits of places that host your research materials and offer increased accessibility and interoperability. However, it is true that while these companies create these products, they also have business models to sustain and insurance around their products. This was important to note in our concluding section to highlight that researchers and librarians may have rightful reservations when using these products. Thank you, we really appreciate your review and recommendations. We are grateful that you could offer your experience and perspective to help strengthen our paper, as well as offer questions that helped us add more clarity and critical points in our writing. Please see our responses to your comments, where we also note any changes made to the original manuscript: “abstract should be structured into Background, Methods, Results, and Conclusions . In particular, the conclusions should be better presented in the abstract (and in the manuscript, see later).” Response: Thank you for bringing this to our attention. We have restructured the abstract, resulting in something that we think will be easier for a reader to navigate. We also revised our conclusions, both here and later in the manuscript (also mentioned later in this response). “More insight about the participants (how many, how many students, which level, which scientific background…).... Please, clarify if the three authors of the manuscript coming from the Computer Science School where also participants of the event and if they are students or Members of the School.” Response: Thank you for noting this ambiguity. We've added a statement clarifying that the three authors from the School of Computer Science were undergraduate students participating in the event. This clarification did significantly improve our narrative and summary. “it was indicated that “participants could have knowledge and awareness of scientific programming with Python and/or R” was desired, but was not required. Was this a difficulty? How many participants had or had not programming knowledge?” Response: This is good to clarify. Each student already had an established skillset with Python programming. With the limited pool of participants, a desire for programming knowledge was not a challenge, but this could be something to consider for future events, especially considering the aspect of reproducibility by any person or team. “It is mentioned that: “At this hackathon event, the organizers observed how participants faced challenges with insufficient documentation, data behind a paywall, and problems with code and proprietary software.” But there is no further information about these observations.” Response: We are grateful that you noted this because it actually revealed an error in our editing. This sentence is in reference to the cited hackathon in the previous statement (ReprohackNL). The paragraph break between these statements was an error. We fixed this error to hopefully make better sense. “But which is the percentage of success? How many figures were in the initial work by Prof. Warren and how many were reproduced by all the participants?” Response: Thank you for pointing out these details, which are a bit vague. We've edited the manuscript to mention that the original work had twenty-five figures to reproduce. We also changed the wording in the latter sentence to read with more accuracy. Our paper is not focusing on if the entire work was reproduced during this event, rather, the paper is focusing on the details and nuances of research code that are revealed when a group comes together to try to reproduce the outcomes. “Our work (see Gomez-Diaz T, et al., 2022 [Ref 1]) goes further in this kind of recommendations, proposing flexible procedures for research software and research data dissemination. It is extremely idealistic to expect that software released at some time will find easily runable environments, unless there is the available resources… Nevertheless, it will be not always possible to ensure the full reproducibility and, in our opinion, a partial solution consists in to release the research software as free/open source software, where the accessibility and the study of the source code is confirmed by the use of free/open source licenses.” Response: These were all excellent points, especially those written in your work. We decided to add another paragraph to the conclusion section to address how reproducibility also relies on adequate resources and systems, which their availability may also depreciate with time, and that documentation offers further guidance and transparency to help other users reuse research software and data, even when resources are scarce. We appreciated the opportunity to add this critical aspect to our conclusions. “Conclusions are poorly described, and more information on the benefits, found drawbacks and other consequences of the event, for participants, students, Prof. Warren and event organizers should be provided to complete this case study.” Response: Thank you for this suggestion. Your review greatly helped us better communicate the impact and implications of running this type of hackathon. It also adds further context for readers to learn how to readapt this model. The final paragraphs of this paper now offer further detail about the benefits, drawbacks, and lessons learned. “We would like to suggest the inclusion of these references…” Response: Thank you for these great recommendations. These references were added to the appropriate places in the manuscript. “Use of privately owned companies…” Response: We found your point here one of the most important to call to attention. In our libraries, and even within our Open Science initiatives, we often promote tools and platforms citing the benefits of places that host your research materials and offer increased accessibility and interoperability. However, it is true that while these companies create these products, they also have business models to sustain and insurance around their products. This was important to note in our concluding section to highlight that researchers and librarians may have rightful reservations when using these products. Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Antunes B. Reviewer Report For: A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 :1305 ( https://doi.org/10.5256/f1000research.172297.r339316 ) The direct URL for this report is: https://f1000research.com/articles/13-1305/v1#referee-response-339316 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 04 Dec 2024 Benjamin Antunes , Université Clermont Auvergne, Clermont-Ferrand, France Approved VIEWS 0 https://doi.org/10.5256/f1000research.172297.r339316 This paper presents a reproducibility hackathon where students attempted to replicate the work of Professor Christopher Warren. In my opinion, this is an excellent initiative. Education is a critical avenue for improving reproducibility, as it raises awareness among students and ... Continue reading READ ALL This paper presents a reproducibility hackathon where students attempted to replicate the work of Professor Christopher Warren. In my opinion, this is an excellent initiative. Education is a critical avenue for improving reproducibility, as it raises awareness among students and future researchers about its importance. The paper is not highly technical; instead, it provides an overview of the types of challenges encountered when attempting to reproduce results. It highlights the use of literate programming and ChatGPT to facilitate code reproduction and illustrates common difficulties, such as handling software dependencies and library versioning in Python. The hackathon focused on research in the field of Digital Humanities. My Remarks: Introduction – First Paragraph: As noted by Barba (2018), computer science was one of the only field to use swaped definitions between reproducibility and replicability (reproducibility is doing the same experiment to obtain the same results, while replicability is doing another experiment to obtain the same scientific conclusion). ACM recently adopted the NISO-recommended terminology for computer science: " *As a result of discussions with the National Information Standards Organization (NISO), it was recommended that ACM harmonize its terminology and definitions with those used in the broader scientific research community, and ACM agreed with NISO’s recommendation to swap the terms “reproducibility” and “replication” with the existing definitions used by ACM as part of its artifact review and badging initiative. ACM took action to update all prior badging to ensure consistency. " https://www.acm.org/publications/policies/artifact-review-and-badging-current This shift swaped the definition of reproducibility and replicability. However, from an epistemological standpoint, reproducibility remains the broader, general term. The paper omits a crucial concept in computer science: repeatability (bitwise repeatability), which is indispensable for debugging (in the context of computer science) Code execution depends on the entire software environment. Open-source software is essential, starting with the operating system (e.g., Linux). However, Docker might not be the best tool for encapsulating computing environments (in the context of reproducible research), as it was not initially designed for reproducibility. I suggest exploring Guix, which is purpose-built for reproducible research, and Software Heritage for archiving software. You rightly emphasize the importance of sharing code and data, aligning with the open science movement. It would be helpful to mention tools like Software Heritage and Zenodo for sharing resources. I noticed the mention of a locally managed public repository, KiltHub. If this repository is not accessible to researchers outside Carnegie Mellon University, highlighting alternative platforms like Zenodo remains relevant. The inclusion of Figure 1 might not add value to the reader. This paper is a practical feedback, primarily aimed at showcasing the challenges of reproducing work. Instead, figures illustrating reproducibility workflows or tools (e.g., Table 1) might be more informative. While Python is widely used due to its ease of use, it is not ideal for reproducibility because of its instability compared to languages like Bash or C, that are more stable. Nonetheless, I agree that results should be reproducible by all, including non-experts, so Python remains a good practical choice. The paper briefly mentions Docker in the introduction but does not elaborate on its usage. It may be more relevant to discuss tools like Guix and Software Heritage, which were explicitly created to address reproducibility challenges. While the paper focuses on Digital Humanities, tools such as CodeOcean (originating from bioinformatics) and FAIR principles are broadly applicable. The paper focus on Python Jupyter notebooks and literate programming is a solid first step for enabling reproducibility. However, other fields like biology and physics, among others, also face reproducibility crises. Broader discussions could include data storage, workflow tools (for example Snakemake to keep it simple), virtual machines, containers, floating-point arithmetic issues, or parallel computing. But of course, the objective of the paper is to present the hackathon, so we cannot talk about everything. I just write it here if you want to take a look. Computers, often seen as “black boxes,” create reproducibility challenges across disciplines. A short introduction to computer science and reproducible research should be incorporated into all scientific curricula. I can recommend you the MOOC: https://www.fun-mooc.fr/en/courses/reproducible-research-methodological-principles-transparent-scie/ In addition, I have written one survey about reproducible research, even if it was more oriented to High performance computing, you might benefit from its read: Antunes B, et al., 2024 (Ref 1) One detail: In the PDF version of the paper, the abstract text is not justified. Shouldn’t it be? Conclusion This hackathon is an excellent initiative to raise awareness among students and young researchers about the challenges of reproducibility. While the paper provides a high-level overview, focusing on literate programming with Jupyter notebooks and the problem of software dependency hell, it serves as a good introduction to the topic. Is the background of the case’s history and progression described in sufficient detail? Yes Is the work clearly and accurately presented and does it cite the current literature? Yes If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Yes References 1. Antunes B, Hill D: Reproducibility, Replicability and Repeatability: A survey of reproducible research with a focus on high performance computing. Computer Science Review . 2024; 53 . Publisher Full Text Competing Interests: No competing interests were disclosed. Reviewer Expertise: My research area if reproducible research in high performance computing. This paper could benefit from having broader information about reproducibility tools, however, it is a short paper describing a Hackathon. I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Antunes B. Reviewer Report For: A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 :1305 ( https://doi.org/10.5256/f1000research.172297.r339316 ) The direct URL for this report is: https://f1000research.com/articles/13-1305/v1#referee-response-339316 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 12 Sep 2025 Chasz Griego , University Libraries, Carnegie Mellon University, Pittsburgh, 15213, USA 12 Sep 2025 Author Response Thank you for your review. We really appreciate your thoughtful assessment of our work, and we are thrilled to receive your perspective and recognition. Please see our responses to your ... Continue reading Thank you for your review. We really appreciate your thoughtful assessment of our work, and we are thrilled to receive your perspective and recognition. Please see our responses to your comments, where we also note any changes made to the original manuscript: “ACM recently adopted the NISO-recommended terminology for computer science…” Response: We appreciate you noting the consistency of the definitions of reproducibility and replicability, between ACM (and NISO) with what is mentioned in our paper, especially in the context of artifact badging. This was good to note, showing how there exists multiple guidelines that follow this convention. “The paper omits a crucial concept in computer science: repeatability (bitwise repeatability), which is indispensable for debugging (in the context of computer science)” Response: We definitely agree that bitwise repeatability is highly notable for fully reproducible computer science, and we're grateful that you note it here. However, as this paper is addressing a less technical scope of reproducible computational research (ie. basic data science and analysis, in contrast to high performance computing), we are content with the scope of detail on computational reproducible that we chose to mention. “It would be helpful to mention tools like Software Heritage and Zenodo for sharing resources. I noticed the mention of a locally managed public repository, KiltHub. If this repository is not accessible to researchers outside Carnegie Mellon University, highlighting alternative platforms like Zenodo remains relevant.” Response: Thank you for mentioning both of these platforms and suggesting we mention them. We agree and have included them in the third paragraph of the introduction, as they are greatly contributing to sharing and preserving software. We agree that a repository like Zenodo is a great alternative to an institutional repository, like KiltHub. However, we prefer to not highlight Zotero or other platforms in the paragraph with KiltHub, since the paragraph was designed to focus specifically on the background behind CMU's Open Science infrastructure, rather than the array of options available for open and reproducible research. “Instead, figures illustrating reproducibility workflows or tools (e.g., Table 1) might be more informative.” Response: This is a great suggestion. We instead decided to convert this figure into a diagram that illustrates the two options (paths) to reproducibility. This diagram still uses one of the original data visualizations as an example, but in much better context. “The paper briefly mentions Docker in the introduction but does not elaborate on its usage. It may be more relevant to discuss tools like Guix and Software Heritage, which were explicitly created to address reproducibility challenges.” Response: Those are fair points. Again, we only mention tools/platforms and avoid adding excessive detail for the sake of the paper’s scope. However, we have now included mentions to Guix and Software Heritage based on your recommendation. “Broader discussions could include data storage, workflow tools (for example Snakemake to keep it simple), virtual machines, containers, floating-point arithmetic issues, or parallel computing.” Response: We really appreciate these points you mention. Reproducibility, let alone computational reproducibility, is so nuanced in each field. Issues surrounding data storage, high-throughput computing, and other software architectures are important to consider, and we hope that efforts like this hackathon, and conversations around it, will promote further awareness and practice that leads to these factors being further addressed. “I can recommend you the MOOC…. In addition, I have written one survey about reproducible research, even if it was more oriented to High performance computing, you might benefit from its read…” Response: Thank you for sharing the MOOC. The content is very reflective of the practices we aim to instill in our students through the libraries, especially topics around structured Markdown notes, Git version control, and notebooks. Thank you for also sharing your survey. We hope to learn further about reproducibility challenges, especially with high performance computing, and apply this to further efforts. We are especially grateful that we could receive your comments, seeing that you authored such a thorough and extensive survey on the topic. We will also include this reference in the manuscript. “One detail: In the PDF version of the paper, the abstract text is not justified. Shouldn’t it be?” Response: Thanks for asking this. We will redirect this to the editor Thank you for your review. We really appreciate your thoughtful assessment of our work, and we are thrilled to receive your perspective and recognition. Please see our responses to your comments, where we also note any changes made to the original manuscript: “ACM recently adopted the NISO-recommended terminology for computer science…” Response: We appreciate you noting the consistency of the definitions of reproducibility and replicability, between ACM (and NISO) with what is mentioned in our paper, especially in the context of artifact badging. This was good to note, showing how there exists multiple guidelines that follow this convention. “The paper omits a crucial concept in computer science: repeatability (bitwise repeatability), which is indispensable for debugging (in the context of computer science)” Response: We definitely agree that bitwise repeatability is highly notable for fully reproducible computer science, and we're grateful that you note it here. However, as this paper is addressing a less technical scope of reproducible computational research (ie. basic data science and analysis, in contrast to high performance computing), we are content with the scope of detail on computational reproducible that we chose to mention. “It would be helpful to mention tools like Software Heritage and Zenodo for sharing resources. I noticed the mention of a locally managed public repository, KiltHub. If this repository is not accessible to researchers outside Carnegie Mellon University, highlighting alternative platforms like Zenodo remains relevant.” Response: Thank you for mentioning both of these platforms and suggesting we mention them. We agree and have included them in the third paragraph of the introduction, as they are greatly contributing to sharing and preserving software. We agree that a repository like Zenodo is a great alternative to an institutional repository, like KiltHub. However, we prefer to not highlight Zotero or other platforms in the paragraph with KiltHub, since the paragraph was designed to focus specifically on the background behind CMU's Open Science infrastructure, rather than the array of options available for open and reproducible research. “Instead, figures illustrating reproducibility workflows or tools (e.g., Table 1) might be more informative.” Response: This is a great suggestion. We instead decided to convert this figure into a diagram that illustrates the two options (paths) to reproducibility. This diagram still uses one of the original data visualizations as an example, but in much better context. “The paper briefly mentions Docker in the introduction but does not elaborate on its usage. It may be more relevant to discuss tools like Guix and Software Heritage, which were explicitly created to address reproducibility challenges.” Response: Those are fair points. Again, we only mention tools/platforms and avoid adding excessive detail for the sake of the paper’s scope. However, we have now included mentions to Guix and Software Heritage based on your recommendation. “Broader discussions could include data storage, workflow tools (for example Snakemake to keep it simple), virtual machines, containers, floating-point arithmetic issues, or parallel computing.” Response: We really appreciate these points you mention. Reproducibility, let alone computational reproducibility, is so nuanced in each field. Issues surrounding data storage, high-throughput computing, and other software architectures are important to consider, and we hope that efforts like this hackathon, and conversations around it, will promote further awareness and practice that leads to these factors being further addressed. “I can recommend you the MOOC…. In addition, I have written one survey about reproducible research, even if it was more oriented to High performance computing, you might benefit from its read…” Response: Thank you for sharing the MOOC. The content is very reflective of the practices we aim to instill in our students through the libraries, especially topics around structured Markdown notes, Git version control, and notebooks. Thank you for also sharing your survey. We hope to learn further about reproducibility challenges, especially with high performance computing, and apply this to further efforts. We are especially grateful that we could receive your comments, seeing that you authored such a thorough and extensive survey on the topic. We will also include this reference in the manuscript. “One detail: In the PDF version of the paper, the abstract text is not justified. Shouldn’t it be?” Response: Thanks for asking this. We will redirect this to the editor Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 12 Sep 2025 Chasz Griego , University Libraries, Carnegie Mellon University, Pittsburgh, 15213, USA 12 Sep 2025 Author Response Thank you for your review. We really appreciate your thoughtful assessment of our work, and we are thrilled to receive your perspective and recognition. Please see our responses to your ... Continue reading Thank you for your review. We really appreciate your thoughtful assessment of our work, and we are thrilled to receive your perspective and recognition. Please see our responses to your comments, where we also note any changes made to the original manuscript: “ACM recently adopted the NISO-recommended terminology for computer science…” Response: We appreciate you noting the consistency of the definitions of reproducibility and replicability, between ACM (and NISO) with what is mentioned in our paper, especially in the context of artifact badging. This was good to note, showing how there exists multiple guidelines that follow this convention. “The paper omits a crucial concept in computer science: repeatability (bitwise repeatability), which is indispensable for debugging (in the context of computer science)” Response: We definitely agree that bitwise repeatability is highly notable for fully reproducible computer science, and we're grateful that you note it here. However, as this paper is addressing a less technical scope of reproducible computational research (ie. basic data science and analysis, in contrast to high performance computing), we are content with the scope of detail on computational reproducible that we chose to mention. “It would be helpful to mention tools like Software Heritage and Zenodo for sharing resources. I noticed the mention of a locally managed public repository, KiltHub. If this repository is not accessible to researchers outside Carnegie Mellon University, highlighting alternative platforms like Zenodo remains relevant.” Response: Thank you for mentioning both of these platforms and suggesting we mention them. We agree and have included them in the third paragraph of the introduction, as they are greatly contributing to sharing and preserving software. We agree that a repository like Zenodo is a great alternative to an institutional repository, like KiltHub. However, we prefer to not highlight Zotero or other platforms in the paragraph with KiltHub, since the paragraph was designed to focus specifically on the background behind CMU's Open Science infrastructure, rather than the array of options available for open and reproducible research. “Instead, figures illustrating reproducibility workflows or tools (e.g., Table 1) might be more informative.” Response: This is a great suggestion. We instead decided to convert this figure into a diagram that illustrates the two options (paths) to reproducibility. This diagram still uses one of the original data visualizations as an example, but in much better context. “The paper briefly mentions Docker in the introduction but does not elaborate on its usage. It may be more relevant to discuss tools like Guix and Software Heritage, which were explicitly created to address reproducibility challenges.” Response: Those are fair points. Again, we only mention tools/platforms and avoid adding excessive detail for the sake of the paper’s scope. However, we have now included mentions to Guix and Software Heritage based on your recommendation. “Broader discussions could include data storage, workflow tools (for example Snakemake to keep it simple), virtual machines, containers, floating-point arithmetic issues, or parallel computing.” Response: We really appreciate these points you mention. Reproducibility, let alone computational reproducibility, is so nuanced in each field. Issues surrounding data storage, high-throughput computing, and other software architectures are important to consider, and we hope that efforts like this hackathon, and conversations around it, will promote further awareness and practice that leads to these factors being further addressed. “I can recommend you the MOOC…. In addition, I have written one survey about reproducible research, even if it was more oriented to High performance computing, you might benefit from its read…” Response: Thank you for sharing the MOOC. The content is very reflective of the practices we aim to instill in our students through the libraries, especially topics around structured Markdown notes, Git version control, and notebooks. Thank you for also sharing your survey. We hope to learn further about reproducibility challenges, especially with high performance computing, and apply this to further efforts. We are especially grateful that we could receive your comments, seeing that you authored such a thorough and extensive survey on the topic. We will also include this reference in the manuscript. “One detail: In the PDF version of the paper, the abstract text is not justified. Shouldn’t it be?” Response: Thanks for asking this. We will redirect this to the editor Thank you for your review. We really appreciate your thoughtful assessment of our work, and we are thrilled to receive your perspective and recognition. Please see our responses to your comments, where we also note any changes made to the original manuscript: “ACM recently adopted the NISO-recommended terminology for computer science…” Response: We appreciate you noting the consistency of the definitions of reproducibility and replicability, between ACM (and NISO) with what is mentioned in our paper, especially in the context of artifact badging. This was good to note, showing how there exists multiple guidelines that follow this convention. “The paper omits a crucial concept in computer science: repeatability (bitwise repeatability), which is indispensable for debugging (in the context of computer science)” Response: We definitely agree that bitwise repeatability is highly notable for fully reproducible computer science, and we're grateful that you note it here. However, as this paper is addressing a less technical scope of reproducible computational research (ie. basic data science and analysis, in contrast to high performance computing), we are content with the scope of detail on computational reproducible that we chose to mention. “It would be helpful to mention tools like Software Heritage and Zenodo for sharing resources. I noticed the mention of a locally managed public repository, KiltHub. If this repository is not accessible to researchers outside Carnegie Mellon University, highlighting alternative platforms like Zenodo remains relevant.” Response: Thank you for mentioning both of these platforms and suggesting we mention them. We agree and have included them in the third paragraph of the introduction, as they are greatly contributing to sharing and preserving software. We agree that a repository like Zenodo is a great alternative to an institutional repository, like KiltHub. However, we prefer to not highlight Zotero or other platforms in the paragraph with KiltHub, since the paragraph was designed to focus specifically on the background behind CMU's Open Science infrastructure, rather than the array of options available for open and reproducible research. “Instead, figures illustrating reproducibility workflows or tools (e.g., Table 1) might be more informative.” Response: This is a great suggestion. We instead decided to convert this figure into a diagram that illustrates the two options (paths) to reproducibility. This diagram still uses one of the original data visualizations as an example, but in much better context. “The paper briefly mentions Docker in the introduction but does not elaborate on its usage. It may be more relevant to discuss tools like Guix and Software Heritage, which were explicitly created to address reproducibility challenges.” Response: Those are fair points. Again, we only mention tools/platforms and avoid adding excessive detail for the sake of the paper’s scope. However, we have now included mentions to Guix and Software Heritage based on your recommendation. “Broader discussions could include data storage, workflow tools (for example Snakemake to keep it simple), virtual machines, containers, floating-point arithmetic issues, or parallel computing.” Response: We really appreciate these points you mention. Reproducibility, let alone computational reproducibility, is so nuanced in each field. Issues surrounding data storage, high-throughput computing, and other software architectures are important to consider, and we hope that efforts like this hackathon, and conversations around it, will promote further awareness and practice that leads to these factors being further addressed. “I can recommend you the MOOC…. In addition, I have written one survey about reproducible research, even if it was more oriented to High performance computing, you might benefit from its read…” Response: Thank you for sharing the MOOC. The content is very reflective of the practices we aim to instill in our students through the libraries, especially topics around structured Markdown notes, Git version control, and notebooks. Thank you for also sharing your survey. We hope to learn further about reproducibility challenges, especially with high performance computing, and apply this to further efforts. We are especially grateful that we could receive your comments, seeing that you authored such a thorough and extensive survey on the topic. We will also include this reference in the manuscript. “One detail: In the PDF version of the paper, the abstract text is not justified. Shouldn’t it be?” Response: Thanks for asking this. We will redirect this to the editor Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Comments on this article Comments (0) Version 2 VERSION 2 PUBLISHED 31 Oct 2024 ADD YOUR COMMENT Comment keyboard_arrow_left keyboard_arrow_right Open Peer Review Reviewer Status info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Reviewer Reports Invited Reviewers 1 2 3 Version 2 (revision) 09 Sep 25 read read Version 1 31 Oct 24 read read Benjamin Antunes , Université Clermont Auvergne, Clermont-Ferrand, France Teresa Gomez-Diaz , Universite Gustave Eiffel / Laboratoire d'informatique Gaspard-Monge / CNRS, Marne-la-Vallée, France Tomas Recio , Universidad Nebrija, Madrid, Spain Stephen J Eglen , Cambridge University, Cambridge, UK Comments on this article All Comments (0) Add a comment Sign up for content alerts Sign Up You are now signed up to receive this alert Browse by related subjects keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 Eglen S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 15 Oct 2025 | for Version 2 Stephen J Eglen , Cambridge University, Cambridge, UK 0 Views copyright © 2025 Eglen S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions As this paper has already been through review, it looks like most of the key points have already been noted and corrected. I found the study easy to read and am supportive of indexing. I have just a few minor suggestions that might improve the paper. "Eight figures were not successfully reproduced because of file paths in the code": could you fix the code, or did the author get feedback about what went wrong here? "were was not documented" Was the author present when you did the hackathon? (Our experience in codecheck hackathons is that they go much quicker, as expected, when authors are present to address any issues.) Thank you for providing the supplementary information that shows what you did at the event; could you possibly annotate (e.g. a README) for your osf.io collection so that people can easily see what you have shared? It wasn't immediately clear to me what you had provided. As a general comment, if you were to run another hackathon along these lines, is there anything that you would do differently next time? Is the background of the case’s history and progression described in sufficient detail? Yes Is the work clearly and accurately presented and does it cite the current literature? Yes If applicable, is the statistical analysis and its interpretation appropriate? Yes Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Partly Competing Interests I am one of the leads on the project https://codecheck.org.uk Reviewer Expertise Computational Neuroscience; Reproducible Research (codecheck.org.uk) I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) Eglen SJ. Peer Review Report For: A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 :1305 ( https://doi.org/10.5256/f1000research.187759.r413599) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1305/v2#referee-response-413599 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 Gomez-Diaz T et al. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 06 Oct 2025 | for Version 2 Teresa Gomez-Diaz , Universite Gustave Eiffel / Laboratoire d'informatique Gaspard-Monge / CNRS, Marne-la-Vallée, Île-de-France, France Tomas Recio , Universidad Nebrija, Madrid, Spain 0 Views copyright © 2025 Gomez-Diaz T et al. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions The second version is fine. The authors have replied to all comments, and they have fixed the main issues. This work relates a very interesting reproducibility experience that took place at the Carnegie Mellon University Libraries. Competing Interests No competing interests were disclosed. Reviewer Expertise Research Software, Research Data, Open Science, dissemination, evaluation We confirm that we have read this submission and believe that we have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) Gomez-Diaz T and Recio T. Peer Review Report For: A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 :1305 ( https://doi.org/10.5256/f1000research.187759.r412602) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1305/v2#referee-response-412602 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 Gomez-Diaz T et al. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 07 Jan 2025 | for Version 1 Teresa Gomez-Diaz , Universite Gustave Eiffel / Laboratoire d'informatique Gaspard-Monge / CNRS, Marne-la-Vallée, Île-de-France, France Tomas Recio , Universidad Nebrija, Madrid, Spain 0 Views copyright © 2025 Gomez-Diaz T et al. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions CASE STUDY A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code https://f1000research.com/articles/13-1305/v1 This case study refers to a hackathon conducted by the members of the Carnegie Mellon University (CMU) Libraries in order to reproduce research results produced by Professor Christopher Warren on data analysis of the Oxford Dictionary of National Biography (ODNB). The event was open for participants coming from CMU and other institutions in the greater Pittsburgh area and was conducted by members of the CMU Libraries. This experience gives an interesting setting to explore reproducibility of published research results by a Professor of the University who also provides data and the produced software involved in the research, and shows a good example of a collaboration between Librarians and members (students?) of the Computer Science School. However, there are a few points in this manuscript that need attention. 1. Abstract. As included in the Case Studies guidelines [ https://f1000research.com/for-authors/article-guidelines/case-studies ], abstract should be structured into Background, Methods, Results, and Conclusions . In particular, the conclusions should be better presented in the abstract (and in the manuscript, see later). 2. Hackathon context. As explained in the document (see also [ https://www.library.cmu.edu/abou)t/news/2024-04/reproducibility-hackathon ], the event was open to a large (institutional) public, but the title of the manuscript refers to students. More insight about the participants (how many, how many students, which level, which scientific background…) should be provided in order to better seize the complexity of the event. Please, clarify if the three authors of the manuscript coming from the Computer Science School where also participants of the event and if they are students or Members of the School. Moreover, it was indicated that “participants could have knowledge and awareness of scientific programming with Python and/or R” was desired, but was not required. Was this a difficulty? How many participants had or had not programming knowledge? 3. Problems with data and code It is mentioned that: “At this hackathon event, the organizers observed how participants faced challenges with insufficient documentation, data behind a paywall, and problems with code and proprietary software.” But there is no further information about these observations. 4. Reproducibility success. It is mentioned that: “Of the figures produced in the notebook, fourteen figures were successfully reproduced by Terveen … and eight figures were not successfully reproduced.” Or that “Though there was trial and error, participants were able to successfully reproduce figures from this study”. But which is the percentage of success? How many figures were in the initial work by Prof. Warren and how many were reproduced by all the participants? 5. Insufficient documentation and code longevity. As noticed by participant Chan, a “requirements.txt” file could help to facilitate the understanding of the software context of the code produced by Prof. Warren. Our work (see Gomez-Diaz T, et al., 2022 [Ref 1]) goes further in this kind of recommendations, proposing flexible procedures for research software and research data dissemination. It is extremely idealistic to expect that software released at some time will find easily runable environments, unless there is the available resources (human, funding) to keep the necessary maintenance of the produced software (who can use Windows version 5?). To accompany the research produced software with good documentation and description of the necessary components will help to get to reproducibility environments, that is, to follow appropriate dissemination procedures will be of help for reproducibility issues. Nevertheless, it will be not always possible to ensure the full reproducibility and, in our opinion, a partial solution consists in to release the research software as free/open source software, where the accessibility and the study of the source code is confirmed by the use of free/open source licenses. 6. Conclusions. Conclusions are poorly described, and more information on the benefits, found drawbacks and other consequences of the event, for participants, students, Prof. Warren and event organizers should be provided to complete this case study. 7. References. We would like to suggest the inclusion of these references: for Open Science: UNESCO: UNESCO Recommendation on Open Science. 2021 [Ref 2] for Free/Open Source Software: K. Fogel (2005-2022). Producing Open Source Software. How to Run a Successful Free Software Project, https://producingoss.com for an example of a Journal giving a reproducibility environment for their publications: IPOL Journal · Image Processing On Line [ https://www.ipol.im/ Nicolaï, A., et al., 2022 [Ref 3] 8. Use of privately owned companies. We would like to raise awareness about the use of privately owned companies like Code Ocean [ https://codeocean.com/ ] for the storage and manipulation of data and code produced within the scientific community. Usually the scientific community is not well aware of legal issues related to the terms of use of these platforms and their consequences, like, for example: `` You hereby grant Code Ocean a worldwide, perpetual, irrevocable, royalty-free right and license to use and exploit any ideas, suggestions, comments, recommendations, enhancement requests or other input provided by you about the Platform to Code Ocean. '' (see point 3. Feedback in [ https://codeocean.com/terms-of-use ]). Regarding OSF, the Center for Open Science, Inc. (referred to as "the COS", "Open Science Framework", "OSF") declares to be a nonprofit organization, but also provides with similar terms of use: `` by depositing any Content (including data and links, but not the linked content) in a COS account, you grant the COS a perpetual, irrevocable, worldwide, royalty-free license to store, reproduce, transmit, distribute, publicly perform, and publicly display such Happy… '' (see further information in [ https://github.com/CenterForOpenScience/cos.io/blob/master/TERMS_OF_USE.md ]). Universities and other Research Performing Institutions, and, in particular, University and other institutional Libraries, could study the proposition of University or institutionally owned platforms and/or services to deal with their scientific production. 9. To complete the background context of this case study, some information on previously hosted hackathons by the CMU Libraries, like Biohackathons and an AI Literacy Resource Hackathon, could be included as they can be of interest for the Hackathon community. Is the background of the case’s history and progression described in sufficient detail? Partly Is the work clearly and accurately presented and does it cite the current literature? Partly If applicable, is the statistical analysis and its interpretation appropriate? Partly Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly Is the case presented with sufficient detail to be useful for teaching or other practitioners? Partly References 1. Gomez-Diaz T, Recio T: Research Software vs. Research Data II: Protocols for Research Data dissemination and evaluation in the Open Science context. F1000Res . 2022; 11 : 117 PubMed Abstract | Publisher Full Text 2. UNESCO: UNESCO Recommendation on Open Science. 2021. Publisher Full Text 3. A, Nicolaï Q, Bammey M, Gardella T, et al.: The approach to reproducible research of the Image Processing On Line (IPOL) journal. http://www.scielo.edu.uy/pdf/info/v27n1/2301-1378-info-27-01-76.pdf . 2022. Competing Interests No competing interests were disclosed. Reviewer Expertise Research Software, Research Data, Open Science, dissemination, evaluation We confirm that we have read this submission and believe that we have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however we have significant reservations, as outlined above. reply Respond to this report Responses (1) Author Response 12 Sep 2025 Chasz Griego, University Libraries, Carnegie Mellon University, Pittsburgh, 15213, USA Thank you, we really appreciate your review and recommendations. We are grateful that you could offer your experience and perspective to help strengthen our paper, as well as offer questions that helped us add more clarity and critical points in our writing. Please see our responses to your comments, where we also note any changes made to the original manuscript: “abstract should be structured into Background, Methods, Results, and Conclusions . In particular, the conclusions should be better presented in the abstract (and in the manuscript, see later).” Response: Thank you for bringing this to our attention. We have restructured the abstract, resulting in something that we think will be easier for a reader to navigate. We also revised our conclusions, both here and later in the manuscript (also mentioned later in this response). “More insight about the participants (how many, how many students, which level, which scientific background…).... Please, clarify if the three authors of the manuscript coming from the Computer Science School where also participants of the event and if they are students or Members of the School.” Response: Thank you for noting this ambiguity. We've added a statement clarifying that the three authors from the School of Computer Science were undergraduate students participating in the event. This clarification did significantly improve our narrative and summary. “it was indicated that “participants could have knowledge and awareness of scientific programming with Python and/or R” was desired, but was not required. Was this a difficulty? How many participants had or had not programming knowledge?” Response: This is good to clarify. Each student already had an established skillset with Python programming. With the limited pool of participants, a desire for programming knowledge was not a challenge, but this could be something to consider for future events, especially considering the aspect of reproducibility by any person or team. “It is mentioned that: “At this hackathon event, the organizers observed how participants faced challenges with insufficient documentation, data behind a paywall, and problems with code and proprietary software.” But there is no further information about these observations.” Response: We are grateful that you noted this because it actually revealed an error in our editing. This sentence is in reference to the cited hackathon in the previous statement (ReprohackNL). The paragraph break between these statements was an error. We fixed this error to hopefully make better sense. “But which is the percentage of success? How many figures were in the initial work by Prof. Warren and how many were reproduced by all the participants?” Response: Thank you for pointing out these details, which are a bit vague. We've edited the manuscript to mention that the original work had twenty-five figures to reproduce. We also changed the wording in the latter sentence to read with more accuracy. Our paper is not focusing on if the entire work was reproduced during this event, rather, the paper is focusing on the details and nuances of research code that are revealed when a group comes together to try to reproduce the outcomes. “Our work (see Gomez-Diaz T, et al., 2022 [Ref 1]) goes further in this kind of recommendations, proposing flexible procedures for research software and research data dissemination. It is extremely idealistic to expect that software released at some time will find easily runable environments, unless there is the available resources… Nevertheless, it will be not always possible to ensure the full reproducibility and, in our opinion, a partial solution consists in to release the research software as free/open source software, where the accessibility and the study of the source code is confirmed by the use of free/open source licenses.” Response: These were all excellent points, especially those written in your work. We decided to add another paragraph to the conclusion section to address how reproducibility also relies on adequate resources and systems, which their availability may also depreciate with time, and that documentation offers further guidance and transparency to help other users reuse research software and data, even when resources are scarce. We appreciated the opportunity to add this critical aspect to our conclusions. “Conclusions are poorly described, and more information on the benefits, found drawbacks and other consequences of the event, for participants, students, Prof. Warren and event organizers should be provided to complete this case study.” Response: Thank you for this suggestion. Your review greatly helped us better communicate the impact and implications of running this type of hackathon. It also adds further context for readers to learn how to readapt this model. The final paragraphs of this paper now offer further detail about the benefits, drawbacks, and lessons learned. “We would like to suggest the inclusion of these references…” Response: Thank you for these great recommendations. These references were added to the appropriate places in the manuscript. “Use of privately owned companies…” Response: We found your point here one of the most important to call to attention. In our libraries, and even within our Open Science initiatives, we often promote tools and platforms citing the benefits of places that host your research materials and offer increased accessibility and interoperability. However, it is true that while these companies create these products, they also have business models to sustain and insurance around their products. This was important to note in our concluding section to highlight that researchers and librarians may have rightful reservations when using these products. View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Gomez-Diaz T and Recio T. Peer Review Report For: A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 :1305 ( https://doi.org/10.5256/f1000research.172297.r355235) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1305/v1#referee-response-355235 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Antunes B. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 04 Dec 2024 | for Version 1 Benjamin Antunes , Université Clermont Auvergne, Clermont-Ferrand, France 0 Views copyright © 2024 Antunes B. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This paper presents a reproducibility hackathon where students attempted to replicate the work of Professor Christopher Warren. In my opinion, this is an excellent initiative. Education is a critical avenue for improving reproducibility, as it raises awareness among students and future researchers about its importance. The paper is not highly technical; instead, it provides an overview of the types of challenges encountered when attempting to reproduce results. It highlights the use of literate programming and ChatGPT to facilitate code reproduction and illustrates common difficulties, such as handling software dependencies and library versioning in Python. The hackathon focused on research in the field of Digital Humanities. My Remarks: Introduction – First Paragraph: As noted by Barba (2018), computer science was one of the only field to use swaped definitions between reproducibility and replicability (reproducibility is doing the same experiment to obtain the same results, while replicability is doing another experiment to obtain the same scientific conclusion). ACM recently adopted the NISO-recommended terminology for computer science: " *As a result of discussions with the National Information Standards Organization (NISO), it was recommended that ACM harmonize its terminology and definitions with those used in the broader scientific research community, and ACM agreed with NISO’s recommendation to swap the terms “reproducibility” and “replication” with the existing definitions used by ACM as part of its artifact review and badging initiative. ACM took action to update all prior badging to ensure consistency. " https://www.acm.org/publications/policies/artifact-review-and-badging-current This shift swaped the definition of reproducibility and replicability. However, from an epistemological standpoint, reproducibility remains the broader, general term. The paper omits a crucial concept in computer science: repeatability (bitwise repeatability), which is indispensable for debugging (in the context of computer science) Code execution depends on the entire software environment. Open-source software is essential, starting with the operating system (e.g., Linux). However, Docker might not be the best tool for encapsulating computing environments (in the context of reproducible research), as it was not initially designed for reproducibility. I suggest exploring Guix, which is purpose-built for reproducible research, and Software Heritage for archiving software. You rightly emphasize the importance of sharing code and data, aligning with the open science movement. It would be helpful to mention tools like Software Heritage and Zenodo for sharing resources. I noticed the mention of a locally managed public repository, KiltHub. If this repository is not accessible to researchers outside Carnegie Mellon University, highlighting alternative platforms like Zenodo remains relevant. The inclusion of Figure 1 might not add value to the reader. This paper is a practical feedback, primarily aimed at showcasing the challenges of reproducing work. Instead, figures illustrating reproducibility workflows or tools (e.g., Table 1) might be more informative. While Python is widely used due to its ease of use, it is not ideal for reproducibility because of its instability compared to languages like Bash or C, that are more stable. Nonetheless, I agree that results should be reproducible by all, including non-experts, so Python remains a good practical choice. The paper briefly mentions Docker in the introduction but does not elaborate on its usage. It may be more relevant to discuss tools like Guix and Software Heritage, which were explicitly created to address reproducibility challenges. While the paper focuses on Digital Humanities, tools such as CodeOcean (originating from bioinformatics) and FAIR principles are broadly applicable. The paper focus on Python Jupyter notebooks and literate programming is a solid first step for enabling reproducibility. However, other fields like biology and physics, among others, also face reproducibility crises. Broader discussions could include data storage, workflow tools (for example Snakemake to keep it simple), virtual machines, containers, floating-point arithmetic issues, or parallel computing. But of course, the objective of the paper is to present the hackathon, so we cannot talk about everything. I just write it here if you want to take a look. Computers, often seen as “black boxes,” create reproducibility challenges across disciplines. A short introduction to computer science and reproducible research should be incorporated into all scientific curricula. I can recommend you the MOOC: https://www.fun-mooc.fr/en/courses/reproducible-research-methodological-principles-transparent-scie/ In addition, I have written one survey about reproducible research, even if it was more oriented to High performance computing, you might benefit from its read: Antunes B, et al., 2024 (Ref 1) One detail: In the PDF version of the paper, the abstract text is not justified. Shouldn’t it be? Conclusion This hackathon is an excellent initiative to raise awareness among students and young researchers about the challenges of reproducibility. While the paper provides a high-level overview, focusing on literate programming with Jupyter notebooks and the problem of software dependency hell, it serves as a good introduction to the topic. Is the background of the case’s history and progression described in sufficient detail? Yes Is the work clearly and accurately presented and does it cite the current literature? Yes If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Yes References 1. Antunes B, Hill D: Reproducibility, Replicability and Repeatability: A survey of reproducible research with a focus on high performance computing. Computer Science Review . 2024; 53 . Publisher Full Text Competing Interests No competing interests were disclosed. Reviewer Expertise My research area if reproducible research in high performance computing. This paper could benefit from having broader information about reproducibility tools, however, it is a short paper describing a Hackathon. I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (1) Author Response 12 Sep 2025 Chasz Griego, University Libraries, Carnegie Mellon University, Pittsburgh, 15213, USA Thank you for your review. We really appreciate your thoughtful assessment of our work, and we are thrilled to receive your perspective and recognition. Please see our responses to your comments, where we also note any changes made to the original manuscript: “ACM recently adopted the NISO-recommended terminology for computer science…” Response: We appreciate you noting the consistency of the definitions of reproducibility and replicability, between ACM (and NISO) with what is mentioned in our paper, especially in the context of artifact badging. This was good to note, showing how there exists multiple guidelines that follow this convention. “The paper omits a crucial concept in computer science: repeatability (bitwise repeatability), which is indispensable for debugging (in the context of computer science)” Response: We definitely agree that bitwise repeatability is highly notable for fully reproducible computer science, and we're grateful that you note it here. However, as this paper is addressing a less technical scope of reproducible computational research (ie. basic data science and analysis, in contrast to high performance computing), we are content with the scope of detail on computational reproducible that we chose to mention. “It would be helpful to mention tools like Software Heritage and Zenodo for sharing resources. I noticed the mention of a locally managed public repository, KiltHub. If this repository is not accessible to researchers outside Carnegie Mellon University, highlighting alternative platforms like Zenodo remains relevant.” Response: Thank you for mentioning both of these platforms and suggesting we mention them. We agree and have included them in the third paragraph of the introduction, as they are greatly contributing to sharing and preserving software. We agree that a repository like Zenodo is a great alternative to an institutional repository, like KiltHub. However, we prefer to not highlight Zotero or other platforms in the paragraph with KiltHub, since the paragraph was designed to focus specifically on the background behind CMU's Open Science infrastructure, rather than the array of options available for open and reproducible research. “Instead, figures illustrating reproducibility workflows or tools (e.g., Table 1) might be more informative.” Response: This is a great suggestion. We instead decided to convert this figure into a diagram that illustrates the two options (paths) to reproducibility. This diagram still uses one of the original data visualizations as an example, but in much better context. “The paper briefly mentions Docker in the introduction but does not elaborate on its usage. It may be more relevant to discuss tools like Guix and Software Heritage, which were explicitly created to address reproducibility challenges.” Response: Those are fair points. Again, we only mention tools/platforms and avoid adding excessive detail for the sake of the paper’s scope. However, we have now included mentions to Guix and Software Heritage based on your recommendation. “Broader discussions could include data storage, workflow tools (for example Snakemake to keep it simple), virtual machines, containers, floating-point arithmetic issues, or parallel computing.” Response: We really appreciate these points you mention. Reproducibility, let alone computational reproducibility, is so nuanced in each field. Issues surrounding data storage, high-throughput computing, and other software architectures are important to consider, and we hope that efforts like this hackathon, and conversations around it, will promote further awareness and practice that leads to these factors being further addressed. “I can recommend you the MOOC…. In addition, I have written one survey about reproducible research, even if it was more oriented to High performance computing, you might benefit from its read…” Response: Thank you for sharing the MOOC. The content is very reflective of the practices we aim to instill in our students through the libraries, especially topics around structured Markdown notes, Git version control, and notebooks. Thank you for also sharing your survey. We hope to learn further about reproducibility challenges, especially with high performance computing, and apply this to further efforts. We are especially grateful that we could receive your comments, seeing that you authored such a thorough and extensive survey on the topic. We will also include this reference in the manuscript. “One detail: In the PDF version of the paper, the abstract text is not justified. Shouldn’t it be?” Response: Thanks for asking this. We will redirect this to the editor View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Antunes B. Peer Review Report For: A libraries reproducibility hackathon: connecting students to University research and testing the longevity of published code [version 1; peer review: 1 approved, 1 approved with reservations] . F1000Research 2024, 13 :1305 ( https://doi.org/10.5256/f1000research.172297.r339316) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-1305/v1#referee-response-339316 Alongside their report, reviewers assign a status to the article: Approved - the paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations - A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved - fundamental flaws in the paper seriously undermine the findings and conclusions Adjust parameters to alter display View on desktop for interactive features Includes Interactive Elements View on desktop for interactive features Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Stay Updated Sign up for content alerts and receive a weekly or monthly email with all newly published articles Register with F1000Research Already registered? Sign in Not now, thanks close PLEASE NOTE If you are an AUTHOR of this article, please check that you signed in with the account associated with this article otherwise we cannot automatically identify your role as an author and your comment will be labelled as a “User Comment”. If you are a REVIEWER of this article, please check that you have signed in with the account associated with this article and then go to your account to submit your report, please do not post your review here. If you do not have access to your original account, please contact us . All commenters must hold a formal affiliation as per our Policies . The information that you give us will be displayed next to your comment. User comments must be in English, comprehensible and relevant to the article under discussion. We reserve the right to remove any comments that we consider to be inappropriate, offensive or otherwise in breach of the User Comment Terms and Conditions . Commenters must not use a comment for personal attacks. When criticisms of the article are based on unpublished data, the data should be made available. I accept the User Comment Terms and Conditions Please confirm that you accept the User Comment Terms and Conditions. Affiliation ✕ refresh Please enter your institution. Note: To add your institution or organisation, start typing the name and then select the correct name from the list. Where applicable, the name will appear in both the original language and in English. Do not paste in the name. If the name does not appear in the drop-down list, we will display the information you have entered. ✕ refresh Country/Region * USA UK Canada China France Germany Afghanistan Aland Islands Albania Algeria American Samoa Andorra Angola Anguilla Antarctica Antigua and Barbuda Argentina Armenia Aruba Australia Austria Azerbaijan Bahamas Bahrain Bangladesh Barbados Belarus Belgium Belize Benin Bermuda Bhutan Bolivia Bosnia and Herzegovina Botswana Bouvet Island Brazil British Indian Ocean Territory British Virgin Islands Brunei Bulgaria Burkina Faso Burundi Cambodia Cameroon Canada Cape Verde Cayman Islands Central African Republic Chad Chile China Christmas Island Cocos (Keeling) Islands Colombia Comoros Congo Cook Islands Costa Rica Cote d'Ivoire Croatia Cuba Cyprus Czech Republic Democratic Republic of the Congo Denmark Djibouti Dominica Dominican Republic Ecuador Egypt El Salvador Equatorial Guinea Eritrea Estonia Ethiopia Falkland Islands Faroe Islands Federated States of Micronesia Fiji Finland France French Guiana French Polynesia French Southern Territories Gabon Georgia Germany Ghana Gibraltar Greece Greenland Grenada Guadeloupe Guam Guatemala Guernsey Guinea Guinea-Bissau Guyana Haiti Heard Island and Mcdonald Islands Holy See (Vatican City State) Honduras Hong Kong Hungary Iceland India Indonesia Iran Iraq Ireland Israel Italy Jamaica Japan Jersey Jordan Kazakhstan Kenya Kiribati Kosovo (Serbia and Montenegro) Kuwait Kyrgyzstan Lao People's Democratic Republic Latvia Lebanon Lesotho Liberia Libya Liechtenstein Lithuania Luxembourg Macao Madagascar Malawi Malaysia Maldives Mali Malta Marshall Islands Martinique Mauritania Mauritius Mayotte Mexico Minor Outlying Islands of the United States Moldova Monaco Mongolia Montenegro Montserrat Morocco Mozambique Myanmar Namibia Nauru Nepal Netherlands Antilles New Caledonia New Zealand Nicaragua Niger Nigeria Niue Norfolk Island North Korea North Macedonia Northern Mariana Islands Norway Oman Pakistan Palau Palestinian Territory Panama Papua New Guinea Paraguay Peru Philippines Pitcairn Poland Portugal Puerto Rico Qatar Reunion Romania Russian Federation Rwanda Saint Helena Saint Kitts and Nevis Saint Lucia Saint Pierre and Miquelon Saint Vincent and the Grenadines Samoa San Marino Sao Tome and Principe Saudi Arabia Senegal Serbia Seychelles Sierra Leone Singapore Slovakia Slovenia Solomon Islands Somalia South Africa South Georgia and the South Sandwich Is South Korea South Sudan Spain Sri Lanka Sudan Suriname Svalbard and Jan Mayen Swaziland Sweden Switzerland Syria Taiwan Tajikistan Tanzania Thailand The Gambia The Netherlands Timor-Leste Togo Tokelau Tonga Trinidad and Tobago Tunisia Turkey Turkmenistan Turks and Caicos Islands Tuvalu UK USA Uganda Ukraine United Arab Emirates United States Virgin Islands Uruguay Uzbekistan Vanuatu Venezuela Vietnam Wallis and Futuna West Bank and Gaza Strip Western Sahara Yemen Zambia Zimbabwe Please select your country/region. You must enter a comment. Competing Interests Please disclose any competing interests that might be construed to influence your judgment of the article's or peer review report's validity or importance. Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Please state your competing interests The comment has been saved. An error has occurred. Please try again. Cancel Post var lTitle = "A libraries reproducibility hackathon: connecting...".replace("'", ''); var linkedInUrl = "http://www.linkedin.com/shareArticle?url=https://f1000research.com/articles/13-1305/v1" + "&title=" + encodeURIComponent(lTitle) + "&summary=" + encodeURIComponent('Read the article by '); var deliciousUrl = "https://del.icio.us/post?url=https://f1000research.com/articles/13-1305/v1&title=" + encodeURIComponent(lTitle); var redditUrl = "http://reddit.com/submit?url=https://f1000research.com/articles/13-1305/v1" + "&title=" + encodeURIComponent(lTitle); linkedInUrl += encodeURIComponent('Griego C et al.'); var offsetTop = /chrome/i.test( navigator.userAgent ) ? 4 : -10; var addthis_config = { ui_offset_top: offsetTop, services_compact : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_expanded : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_custom : [ { name: "LinkedIn", url: linkedInUrl, icon:"/img/icon/at_linkedin.svg" }, { name: "Mendeley", url: "http://www.mendeley.com/import/?url=https://f1000research.com/articles/13-1305/v1/mendeley", icon:"/img/icon/at_mendeley.svg" }, { name: "Reddit", url: redditUrl, icon:"/img/icon/at_reddit.svg" }, ] }; var addthis_share = { url: "https://f1000research.com/articles/13-1305", templates : { twitter : "A libraries reproducibility hackathon: connecting students to.... Griego C et al., published by " + "@F1000Research" + ", https://f1000research.com/articles/13-1305/v1" } }; if (typeof(addthis) != "undefined"){ addthis.addEventListener('addthis.ready', checkCount); addthis.addEventListener('addthis.menu.share', checkCount); } $(".f1r-shares-twitter").attr("href", "https://twitter.com/intent/tweet?text=" + addthis_share.templates.twitter); $(".f1r-shares-facebook").attr("href", "https://www.facebook.com/sharer/sharer.php?u=" + addthis_share.url); $(".f1r-shares-linkedin").attr("href", addthis_config.services_custom[0].url); $(".f1r-shares-reddit").attr("href", addthis_config.services_custom[2].url); $(".f1r-shares-mendelay").attr("href", addthis_config.services_custom[1].url); function checkCount(){ setTimeout(function(){ $(".addthis_button_expanded").each(function(){ var count = $(this).text(); if (count !== "" && count != "0") $(this).removeClass("is-hidden"); else $(this).addClass("is-hidden"); }); }, 1000); } close How to cite this report {{reportCitation}} Cancel Copy Citation Details $(function(){R.ui.buttonDropdowns('.dropdown-for-downloads');}); $(function(){R.ui.toolbarDropdowns('.toolbar-dropdown-for-downloads');}); $.get("/articles/acj/156917/172297") new F1000.Clipboard(); new F1000.ThesaurusTermsDisplay("articles", "article", "172297"); $(document).ready(function() { $( "#frame1" ).on('load', function() { var mydiv = $(this).contents().find("div"); var h = mydiv.height(); console.log(h) }); var tooltipLivingFigure = jQuery(".interactive-living-figure-label .icon-more-info"), titleLivingFigure = tooltipLivingFigure.attr("title"); tooltipLivingFigure.simpletip({ fixed: true, position: ["-115", "30"], baseClass: 'small-tooltip', content:titleLivingFigure + " " }); tooltipLivingFigure.removeAttr("title"); $("body").on("click", ".cite-living-figure", function(e) { e.preventDefault(); var ref = $(this).attr("data-ref"); $(this).closest(".living-figure-list-container").find("#" + ref).fadeIn(200); }); $("body").on("click", ".close-cite-living-figure", function(e) { e.preventDefault(); $(this).closest(".popup-window-wrapper").fadeOut(200); }); $(document).on("mouseup", function(e) { var metricsContainer = $(".article-metrics-popover-wrapper"); if (!metricsContainer.is(e.target) && metricsContainer.has(e.target).length === 0) { $(".article-metrics-close-button").click(); } }); var articleId = $('#articleId').val(); if($("#main-article-count-box").attachArticleMetrics) { $("#main-article-count-box").attachArticleMetrics(articleId, { articleMetricsView: true }); } }); var figshareWidget = $(".new_figshare_widget"); if (figshareWidget.length > 0) { window.figshare.load("f1000", function(Widget) { // Select a tag/tags defined in your page. In this tag we will place the widget. _.map(figshareWidget, function(el){ var widget = new Widget({ articleId: $(el).attr("figshare_articleId") //height:300 // this is the height of the viewer part. [Default: 550] }); widget.initialize(); // initialize the widget widget.mount(el); // mount it in a tag that's on your page // this will save the widget on the global scope for later use from // your JS scripts. This line is optional. //window.widget = widget; }); }); } close Error Close Add Reset F1000.MICROSERVICES.AFFILIATION = ''; $(document).ready(function () { $('.js-affiliations-form').each((index, form) => { new AffiliationForm({ formId: form.id, institutionErrorSelector: '.comment-enter-institution', departmentErrorSelector: '.comment-enter-department', placeSelector: '.js-add-comment-place', stateSelector: '.js-add-comment-state', zipCodeSelector: '.js-add-comment-zipcode', countrySelector: '.js-add-comment-country', countryErrorSelector: '.comment-enter-country', }); }); }); $(document).ready(function () { var reportIds = { "413598": 0, "351261": 0, "355229": 0, "413599": 6, "351260": 0, "355228": 0, "413596": 0, "351263": 0, "355231": 0, "413597": 0, "351262": 0, "355230": 0, "413594": 0, "413595": 0, "351259": 0, "355227": 0, "351258": 0, "355236": 0, "413602": 0, "351265": 0, "355233": 0, "413603": 0, "351264": 0, "355232": 0, "413600": 0, "357155": 0, "351267": 0, "355235": 44, "413601": 0, "351266": 0, "355234": 0, "339317": 0, "339316": 26, "339319": 0, "339318": 0, "339325": 0, "339324": 0, "412602": 25, "339321": 0, "412603": 0, "339320": 0, "339323": 0, "339322": 0, }; $(".referee-response-container,.js-referee-report").each(function(index, el) { var reportId = $(el).attr("data-reportid"), reportCount = reportIds[reportId] || 0; $(el).find(".comments-count-container,.js-referee-report-views").html(reportCount); }); var uuidInput = $("#article_uuid"), oldUUId = uuidInput.val(), newUUId = "24d4dc6d-0028-4d62-aa70-d7a1969234b1"; uuidInput.val(newUUId); $("a[href*='article_uuid=']").each(function(index, el) { var newHref = $(el).attr("href").replace(oldUUId, newUUId); $(el).attr("href", newHref); }); }); An innovative open access publishing platform offering rapid publication and open peer review, whilst supporting data deposition and sharing. Browse Gateways Collections How it Works Contact For Developers Cookie Notice Privacy Notice RSS Submit Your Research Follow us © 2012-2026 F1000 Research Ltd. ISSN 2046-1402 | Legal | Partner of Research4Life • CrossRef • ORCID • FAIRSharing R.templateTests.simpleTemplate = R.template(' $text $text $text $text $text '); R.templateTests.runTests(); var F1000platform = new F1000.Platform({ name: "f1000research", displayName: "F1000Research", hostName: "f1000research.com", id: "1", editorialEmail: "[email protected]", infoEmail: "[email protected]", usePmcStats: true }); $(function(){R.ui.dropdowns('.dropdown-for-authors, .dropdown-for-about, .dropdown-for-myresearch');}); // $(function(){R.ui.dropdowns('.dropdown-for-referees');}); $(document).ready(function () { if ($(".cookie-warning").is(":visible")) { $(".sticky").css("margin-bottom", "35px"); $(".devices").addClass("devices-and-cookie-warning"); } $(".cookie-warning .close-button").click(function (e) { $(".devices").removeClass("devices-and-cookie-warning"); $(".sticky").css("margin-bottom", "0"); }); $("#tweeter-feed .tweet-message").each(function (i, message) { var self = $(message); self.html(linkify(self.html())); }); $(".partner").on("mouseenter mouseleave", function() { $(this).find(".gray-scale, .colour").toggleClass("is-hidden"); }); }); Sign In Remember me Forgotten your password? Sign In Cancel Email or password not correct. Please try again Please wait... $(function(){ // Note: All the setup needs to run against a name attribute and *not* the id due the clonish // nature of facebox... $("a[id=googleSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("GOOGLE"); $("form[id=oAuthForm]").submit(); }); $("a[id=facebookSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("FACEBOOK"); $("form[id=oAuthForm]").submit(); }); $("a[id=orcidSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("ORCID"); $("form[id=oAuthForm]").submit(); }); }); If you've forgotten your password, please enter your email address below and we'll send you instructions on how to reset your password. The email address should be the one you originally registered with F1000. Email address not valid, please try again You registered with F1000 via Google, so we cannot reset your password. To sign in, please click here . If you still need help with your Google account password, please click here . You registered with F1000 via Facebook, so we cannot reset your password. To sign in, please click here . If you still need help with your Facebook account password, please click here . Code not correct, please try again Reset password Cancel Email us for further assistance. Server error, please try again. If your email address is registered with us, we will email you instructions to reset your password. If you think you should have received this email but it has not arrived, please check your spam filters and/or contact for further assistance. Please wait... Register $(document).ready(function () { signIn.createSignInAsRow($("#sign-in-form-gfb-popup")); $(".target-field").each(function () { var uris = $(this).val().split("/"); if (uris.pop() === "login") { $(this).val(uris.toString().replace(",","/")); } }); });