Expert Consensus Methods In The Humanities: An Exploration of their Potential

Expert Consensus Methods In The Humanities: An... | F1000Research "use strict";function _typeof(t){return(_typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}!function(){var t=function(){var t,e,o=[],n=window,r=n;for(;r;){try{if(r.frames.__tcfapiLocator){t=r;break}}catch(t){}if(r===n.top)break;r=r.parent}t||(!function t(){var e=n.document,o=!!n.frames.__tcfapiLocator;if(!o)if(e.body){var r=e.createElement("iframe");r.style.cssText="display:none",r.name="__tcfapiLocator",e.body.appendChild(r)}else setTimeout(t,5);return!o}(),n.__tcfapi=function(){for(var t=arguments.length,n=new Array(t),r=0;r 3&&2===parseInt(n[1],10)&&"boolean"==typeof n[3]&&(e=n[3],"function"==typeof n[2]&&n[2]("set",!0)):"ping"===n[0]?"function"==typeof n[2]&&n[2]({gdprApplies:e,cmpLoaded:!1,cmpStatus:"stub"}):o.push(n)},n.addEventListener("message",(function(t){var e="string"==typeof t.data,o={};if(e)try{o=JSON.parse(t.data)}catch(t){}else o=t.data;var n="object"===_typeof(o)&&null!==o?o.__tcfapiCall:null;n&&window.__tcfapi(n.command,n.version,(function(o,r){var a={__tcfapiReturn:{returnValue:o,success:r,callId:n.callId}};t&&t.source&&t.source.postMessage&&t.source.postMessage(e?JSON.stringify(a):a,"*")}),n.parameter)}),!1))};"undefined"!=typeof module?module.exports=t:t()}(); dataLayer = dataLayer || []; // Standard GTM initialization - Google Consent Mode handles consent automatically (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl+ '>m_auth=hzk0Vc3qFsQYhCrIoHz68A>m_preview=env-1>m_cookies_win=x';f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-MWFK8L5J'); ;window.NREUM||(NREUM={});NREUM.init={distributed_tracing:{enabled:true},privacy:{cookies_enabled:true},ajax:{deny_list:["bam.nr-data.net"]}}; ;NREUM.loader_config={accountID:"438030",trustKey:"438030",agentID:"772317073",licenseKey:"97f8f67f26",applicationID:"772317073"} ;NREUM.info={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net",licenseKey:"97f8f67f26",applicationID:"772317073",sa:1} ;/*! For license information please see nr-loader-spa-1.236.0.min.js.LICENSE.txt */ (()=>{"use strict";var e,t,r={5763:(e,t,r)=>{r.d(t,{P_:()=>l,Mt:()=>g,C5:()=>s,DL:()=>v,OP:()=>T,lF:()=>D,Yu:()=>y,Dg:()=>h,CX:()=>c,GE:()=>b,sU:()=>_});var n=r(8632),i=r(9567);const o={beacon:n.ce.beacon,errorBeacon:n.ce.errorBeacon,licenseKey:void 0,applicationID:void 0,sa:void 0,queueTime:void 0,applicationTime:void 0,ttGuid:void 0,user:void 0,account:void 0,product:void 0,extra:void 0,jsAttributes:{},userAttributes:void 0,atts:void 0,transactionName:void 0,tNamePlain:void 0},a={};function s(e){if(!e)throw new Error("All info objects require an agent identifier!");if(!a[e])throw new Error("Info for ".concat(e," was never set"));return a[e]}function c(e,t){if(!e)throw new Error("All info objects require an agent identifier!");a[e]=(0,i.D)(t,o),(0,n.Qy)(e,a[e],"info")}var u=r(7056);const d=()=>{const e={blockSelector:"[data-nr-block]",maskInputOptions:{password:!0}};return{allow_bfcache:!0,privacy:{cookies_enabled:!0},ajax:{deny_list:void 0,enabled:!0,harvestTimeSeconds:10},distributed_tracing:{enabled:void 0,exclude_newrelic_header:void 0,cors_use_newrelic_header:void 0,cors_use_tracecontext_headers:void 0,allowed_origins:void 0},session:{domain:void 0,expiresMs:u.oD,inactiveMs:u.Hb},ssl:void 0,obfuscate:void 0,jserrors:{enabled:!0,harvestTimeSeconds:10},metrics:{enabled:!0},page_action:{enabled:!0,harvestTimeSeconds:30},page_view_event:{enabled:!0},page_view_timing:{enabled:!0,harvestTimeSeconds:30,long_task:!1},session_trace:{enabled:!0,harvestTimeSeconds:10},harvest:{tooManyRequestsDelay:60},session_replay:{enabled:!1,harvestTimeSeconds:60,sampleRate:.1,errorSampleRate:.1,maskTextSelector:"*",maskAllInputs:!0,get blockClass(){return"nr-block"},get ignoreClass(){return"nr-ignore"},get maskTextClass(){return"nr-mask"},get blockSelector(){return e.blockSelector},set blockSelector(t){e.blockSelector+=",".concat(t)},get maskInputOptions(){return e.maskInputOptions},set maskInputOptions(t){e.maskInputOptions={...t,password:!0}}},spa:{enabled:!0,harvestTimeSeconds:10}}},f={};function l(e){if(!e)throw new Error("All configuration objects require an agent identifier!");if(!f[e])throw new Error("Configuration for ".concat(e," was never set"));return f[e]}function h(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");f[e]=(0,i.D)(t,d()),(0,n.Qy)(e,f[e],"config")}function g(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");var r=l(e);if(r){for(var n=t.split("."),i=0;i {r.d(t,{D:()=>i});var n=r(50);function i(e,t){try{if(!e||"object"!=typeof e)return(0,n.Z)("Setting a Configurable requires an object as input");if(!t||"object"!=typeof t)return(0,n.Z)("Setting a Configurable requires a model to set its initial properties");const r=Object.create(Object.getPrototypeOf(t),Object.getOwnPropertyDescriptors(t)),o=0===Object.keys(r).length?e:r;for(let a in o)if(void 0!==e[a])try{"object"==typeof e[a]&&"object"==typeof t[a]?r[a]=i(e[a],t[a]):r[a]=e[a]}catch(e){(0,n.Z)("An error occurred while setting a property of a Configurable",e)}return r}catch(e){(0,n.Z)("An error occured while setting a Configurable",e)}}},6818:(e,t,r)=>{r.d(t,{Re:()=>i,gF:()=>o,q4:()=>n});const n="1.236.0",i="PROD",o="CDN"},385:(e,t,r)=>{r.d(t,{FN:()=>a,IF:()=>u,Nk:()=>f,Tt:()=>s,_A:()=>o,il:()=>n,pL:()=>c,v6:()=>i,w1:()=>d});const n="undefined"!=typeof window&&!!window.document,i="undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self.navigator instanceof WorkerNavigator||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis.navigator instanceof WorkerNavigator),o=n?window:"undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis),a=""+o?.location,s=/iPad|iPhone|iPod/.test(navigator.userAgent),c=s&&"undefined"==typeof SharedWorker,u=(()=>{const e=navigator.userAgent.match(/Firefox[/\s](\d+\.\d+)/);return Array.isArray(e)&&e.length>=2?+e[1]:0})(),d=Boolean(n&&window.document.documentMode),f=!!navigator.sendBeacon},1117:(e,t,r)=>{r.d(t,{w:()=>o});var n=r(50);const i={agentIdentifier:"",ee:void 0};class o{constructor(e){try{if("object"!=typeof e)return(0,n.Z)("shared context requires an object as input");this.sharedContext={},Object.assign(this.sharedContext,i),Object.entries(e).forEach((e=>{let[t,r]=e;Object.keys(i).includes(t)&&(this.sharedContext[t]=r)}))}catch(e){(0,n.Z)("An error occured while setting SharedContext",e)}}}},8e3:(e,t,r)=>{r.d(t,{L:()=>d,R:()=>c});var n=r(2177),i=r(1284),o=r(4322),a=r(3325);const s={};function c(e,t){const r={staged:!1,priority:a.p[t]||0};u(e),s[e].get(t)||s[e].set(t,r)}function u(e){e&&(s[e]||(s[e]=new Map))}function d(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"",t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"feature";if(u(e),!e||!s[e].get(t))return a(t);s[e].get(t).staged=!0;const r=[...s[e]];function a(t){const r=e?n.ee.get(e):n.ee,a=o.X.handlers;if(r.backlog&&a){var s=r.backlog[t],c=a[t];if(c){for(var u=0;s&&u {let[t,r]=e;return r.staged}))&&(r.sort(((e,t)=>e[1].priority-t[1].priority)),r.forEach((e=>{let[t]=e;a(t)})))}function f(e,t){var r=e[1];(0,i.D)(t[r],(function(t,r){var n=e[0];if(r[0]===n){var i=r[1],o=e[3],a=e[2];i.apply(o,a)}}))}},2177:(e,t,r)=>{r.d(t,{c:()=>f,ee:()=>u});var n=r(8632),i=r(2210),o=r(1284),a=r(5763),s="nr@context";let c=(0,n.fP)();var u;function d(){}function f(e){return(0,i.X)(e,s,l)}function l(){return new d}function h(){u.aborted=!0,u.backlog={}}c.ee?u=c.ee:(u=function e(t,r){var n={},c={},f={},g=!1;try{g=16===r.length&&(0,a.OP)(r).isolatedBacklog}catch(e){}var p={on:b,addEventListener:b,removeEventListener:y,emit:v,get:x,listeners:w,context:m,buffer:A,abort:h,aborted:!1,isBuffering:E,debugId:r,backlog:g?{}:t&&"object"==typeof t.backlog?t.backlog:{}};return p;function m(e){return e&&e instanceof d?e:e?(0,i.X)(e,s,l):l()}function v(e,r,n,i,o){if(!1!==o&&(o=!0),!u.aborted||i){t&&o&&t.emit(e,r,n);for(var a=m(n),s=w(e),d=s.length,f=0;fn,p:()=>i});var n=r(2177).ee.get("handle");function i(e,t,r,i,o){o?(o.buffer([e],i),o.emit(e,t,r)):(n.buffer([e],i),n.emit(e,t,r))}},4322:(e,t,r)=>{r.d(t,{X:()=>o});var n=r(5546);o.on=a;var i=o.handlers={};function o(e,t,r,o){a(o||n.E,i,e,t,r)}function a(e,t,r,i,o){o||(o="feature"),e||(e=n.E);var a=t[o]=t[o]||{};(a[r]=a[r]||[]).push([e,i])}},3239:(e,t,r)=>{r.d(t,{bP:()=>s,iz:()=>c,m$:()=>a});var n=r(385);let i=!1,o=!1;try{const e={get passive(){return i=!0,!1},get signal(){return o=!0,!1}};n._A.addEventListener("test",null,e),n._A.removeEventListener("test",null,e)}catch(e){}function a(e,t){return i||o?{capture:!!e,passive:i,signal:t}:!!e}function s(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;window.addEventListener(e,t,a(r,n))}function c(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;document.addEventListener(e,t,a(r,n))}},4402:(e,t,r)=>{r.d(t,{Ht:()=>u,M:()=>c,Rl:()=>a,ky:()=>s});var n=r(385);const i="xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx";function o(e,t){return e?15&e[t]:16*Math.random()|0}function a(){const e=n._A?.crypto||n._A?.msCrypto;let t,r=0;return e&&e.getRandomValues&&(t=e.getRandomValues(new Uint8Array(31))),i.split("").map((e=>"x"===e?o(t,++r).toString(16):"y"===e?(3&o()|8).toString(16):e)).join("")}function s(e){const t=n._A?.crypto||n._A?.msCrypto;let r,i=0;t&&t.getRandomValues&&(r=t.getRandomValues(new Uint8Array(31)));const a=[];for(var s=0;s {r.d(t,{Bq:()=>n,Hb:()=>o,oD:()=>i});const n="NRBA",i=144e5,o=18e5},7894:(e,t,r)=>{function n(){return Math.round(performance.now())}r.d(t,{z:()=>n})},7243:(e,t,r)=>{r.d(t,{e:()=>o});var n=r(385),i={};function o(e){if(e in i)return i[e];if(0===(e||"").indexOf("data:"))return{protocol:"data"};let t;var r=n._A?.location,o={};if(n.il)t=document.createElement("a"),t.href=e;else try{t=new URL(e,r.href)}catch(e){return o}o.port=t.port;var a=t.href.split("://");!o.port&&a[1]&&(o.port=a[1].split("/")[0].split("@").pop().split(":")[1]),o.port&&"0"!==o.port||(o.port="https"===a[0]?"443":"80"),o.hostname=t.hostname||r.hostname,o.pathname=t.pathname,o.protocol=a[0],"/"!==o.pathname.charAt(0)&&(o.pathname="/"+o.pathname);var s=!t.protocol||":"===t.protocol||t.protocol===r.protocol,c=t.hostname===r.hostname&&t.port===r.port;return o.sameOrigin=s&&(!t.hostname||c),"/"===o.pathname&&(i[e]=o),o}},50:(e,t,r)=>{function n(e,t){"function"==typeof console.warn&&(console.warn("New Relic: ".concat(e)),t&&console.warn(t))}r.d(t,{Z:()=>n})},2587:(e,t,r)=>{r.d(t,{N:()=>c,T:()=>u});var n=r(2177),i=r(5546),o=r(8e3),a=r(3325);const s={stn:[a.D.sessionTrace],err:[a.D.jserrors,a.D.metrics],ins:[a.D.pageAction],spa:[a.D.spa],sr:[a.D.sessionReplay,a.D.sessionTrace]};function c(e,t){const r=n.ee.get(t);e&&"object"==typeof e&&(Object.entries(e).forEach((e=>{let[t,n]=e;void 0===u[t]&&(s[t]?s[t].forEach((e=>{n?(0,i.p)("feat-"+t,[],void 0,e,r):(0,i.p)("block-"+t,[],void 0,e,r),(0,i.p)("rumresp-"+t,[Boolean(n)],void 0,e,r)})):n&&(0,i.p)("feat-"+t,[],void 0,void 0,r),u[t]=Boolean(n))})),Object.keys(s).forEach((e=>{void 0===u[e]&&(s[e]?.forEach((t=>(0,i.p)("rumresp-"+e,[!1],void 0,t,r))),u[e]=!1)})),(0,o.L)(t,a.D.pageViewEvent))}const u={}},2210:(e,t,r)=>{r.d(t,{X:()=>i});var n=Object.prototype.hasOwnProperty;function i(e,t,r){if(n.call(e,t))return e[t];var i=r();if(Object.defineProperty&&Object.keys)try{return Object.defineProperty(e,t,{value:i,writable:!0,enumerable:!1}),i}catch(e){}return e[t]=i,i}},1284:(e,t,r)=>{r.d(t,{D:()=>n});const n=(e,t)=>Object.entries(e||{}).map((e=>{let[r,n]=e;return t(r,n)}))},4351:(e,t,r)=>{r.d(t,{P:()=>o});var n=r(2177);const i=()=>{const e=new WeakSet;return(t,r)=>{if("object"==typeof r&&null!==r){if(e.has(r))return;e.add(r)}return r}};function o(e){try{return JSON.stringify(e,i())}catch(e){try{n.ee.emit("internal-error",[e])}catch(e){}}}},3960:(e,t,r)=>{r.d(t,{K:()=>a,b:()=>o});var n=r(3239);function i(){return"undefined"==typeof document||"complete"===document.readyState}function o(e,t){if(i())return e();(0,n.bP)("load",e,t)}function a(e){if(i())return e();(0,n.iz)("DOMContentLoaded",e)}},8632:(e,t,r)=>{r.d(t,{EZ:()=>u,Qy:()=>c,ce:()=>o,fP:()=>a,gG:()=>d,mF:()=>s});var n=r(7894),i=r(385);const o={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net"};function a(){return i._A.NREUM||(i._A.NREUM={}),void 0===i._A.newrelic&&(i._A.newrelic=i._A.NREUM),i._A.NREUM}function s(){let e=a();return e.o||(e.o={ST:i._A.setTimeout,SI:i._A.setImmediate,CT:i._A.clearTimeout,XHR:i._A.XMLHttpRequest,REQ:i._A.Request,EV:i._A.Event,PR:i._A.Promise,MO:i._A.MutationObserver,FETCH:i._A.fetch}),e}function c(e,t,r){let i=a();const o=i.initializedAgents||{},s=o[e]||{};return Object.keys(s).length||(s.initializedAt={ms:(0,n.z)(),date:new Date}),i.initializedAgents={...o,[e]:{...s,[r]:t}},i}function u(e,t){a()[e]=t}function d(){return function(){let e=a();const t=e.info||{};e.info={beacon:o.beacon,errorBeacon:o.errorBeacon,...t}}(),function(){let e=a();const t=e.init||{};e.init={...t}}(),s(),function(){let e=a();const t=e.loader_config||{};e.loader_config={...t}}(),a()}},7956:(e,t,r)=>{r.d(t,{N:()=>i});var n=r(3239);function i(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],r=arguments.length>2?arguments[2]:void 0,i=arguments.length>3?arguments[3]:void 0;return void(0,n.iz)("visibilitychange",(function(){if(t)return void("hidden"==document.visibilityState&&e());e(document.visibilityState)}),r,i)}},1214:(e,t,r)=>{r.d(t,{em:()=>v,u5:()=>N,QU:()=>S,_L:()=>I,Gm:()=>L,Lg:()=>M,gy:()=>U,BV:()=>Q,Kf:()=>ee});var n=r(2177);const i="nr@original";var o=Object.prototype.hasOwnProperty,a=!1;function s(e,t){return e||(e=n.ee),r.inPlace=function(e,t,n,i,o){n||(n="");var a,s,c,u="-"===n.charAt(0);for(c=0;c 2?n-2:0),o=2;o {r(A[T],e,w),r(E[T],e,w)})),r(l._A,"fetch",y),t.on(y+"end",(function(e,r){var n=this;if(r){var i=r.headers.get("content-length");null!==i&&(n.rxSize=i),t.emit(y+"done",[null,r],n)}else t.emit(y+"done",[e],n)})),t}const O={},j=["pushState","replaceState"];function S(e){const t=function(e){return(e||n.ee).get("history")}(e);return!l.il||O[t.debugId]++||(O[t.debugId]=1,s(t).inPlace(window.history,j,"-")),t}var P=r(3239);const C={},R=["appendChild","insertBefore","replaceChild"];function I(e){const t=function(e){return(e||n.ee).get("jsonp")}(e);if(!l.il||C[t.debugId])return t;C[t.debugId]=!0;var r=s(t),i=/[?&](?:callback|cb)=([^&#]+)/,o=/(.*)\.([^.]+)/,a=/^(\w+)(\.|$)(.*)$/;function c(e,t){var r=e.match(a),n=r[1],i=r[3];return i?c(i,t[n]):t[n]}return r.inPlace(Node.prototype,R,"dom-"),t.on("dom-start",(function(e){!function(e){if(!e||"string"!=typeof e.nodeName||"script"!==e.nodeName.toLowerCase())return;if("function"!=typeof e.addEventListener)return;var n=(a=e.src,s=a.match(i),s?s[1]:null);var a,s;if(!n)return;var u=function(e){var t=e.match(o);if(t&&t.length>=3)return{key:t[2],parent:c(t[1],window)};return{key:e,parent:window}}(n);if("function"!=typeof u.parent[u.key])return;var d={};function f(){t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}function l(){t.emit("jsonp-error",[],d),t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}r.inPlace(u.parent,[u.key],"cb-",d),e.addEventListener("load",f,(0,P.m$)(!1)),e.addEventListener("error",l,(0,P.m$)(!1)),t.emit("new-jsonp",[e.src],d)}(e[0])})),t}var k=r(5763);const H={};function L(e){const t=function(e){return(e||n.ee).get("mutation")}(e);if(!l.il||H[t.debugId])return t;H[t.debugId]=!0;var r=s(t),i=k.Yu.MO;return i&&(window.MutationObserver=function(e){return this instanceof i?new i(r(e,"fn-")):i.apply(this,arguments)},MutationObserver.prototype=i.prototype),t}const z={};function M(e){const t=function(e){return(e||n.ee).get("promise")}(e);if(z[t.debugId])return t;z[t.debugId]=!0;var r=n.c,o=s(t),a=k.Yu.PR;return a&&function(){function e(r){var n=t.context(),i=o(r,"executor-",n,null,!1);const s=Reflect.construct(a,[i],e);return t.context(s).getCtx=function(){return n},s}l._A.Promise=e,Object.defineProperty(e,"name",{value:"Promise"}),e.toString=function(){return a.toString()},Object.setPrototypeOf(e,a),["all","race"].forEach((function(r){const n=a[r];e[r]=function(e){let i=!1;[...e||[]].forEach((e=>{this.resolve(e).then(a("all"===r),a(!1))}));const o=n.apply(this,arguments);return o;function a(e){return function(){t.emit("propagate",[null,!i],o,!1,!1),i=i||!e}}}})),["resolve","reject"].forEach((function(r){const n=a[r];e[r]=function(e){const r=n.apply(this,arguments);return e!==r&&t.emit("propagate",[e,!0],r,!1,!1),r}})),e.prototype=a.prototype;const n=a.prototype.then;a.prototype.then=function(){var e=this,i=r(e);i.promise=e;for(var a=arguments.length,s=new Array(a),c=0;c e())),t};function m(e,t){i.inPlace(t,["onreadystatechange"],"fn-",E)}function b(){var e=this,t=r.context(e);e.readyState>3&&!t.resolved&&(t.resolved=!0,r.emit("xhr-resolved",[],e)),i.inPlace(e,f,"fn-",E)}if(function(e,t){for(var r in e)t[r]=e[r]}(o,p),p.prototype=o.prototype,i.inPlace(p.prototype,J,"-xhr-",E),r.on("send-xhr-start",(function(e,t){m(e,t),function(e){h.push(e),a&&(y?y.then(A):u?u(A):(w=-w,x.data=w))}(t)})),r.on("open-xhr-start",m),a){var y=c&&c.resolve();if(!u&&!c){var w=1,x=document.createTextNode(w);new a(A).observe(x,{characterData:!0})}}else t.on("fn-end",(function(e){e[0]&&e[0].type===d||A()}));function A(){for(var e=0;e {r.d(t,{t:()=>n});const n=r(3325).D.ajax},6660:(e,t,r)=>{r.d(t,{A:()=>i,t:()=>n});const n=r(3325).D.jserrors,i="nr@seenError"},3081:(e,t,r)=>{r.d(t,{gF:()=>o,mY:()=>i,t9:()=>n,vz:()=>s,xS:()=>a});const n=r(3325).D.metrics,i="sm",o="cm",a="storeSupportabilityMetrics",s="storeEventMetrics"},4649:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageAction},7633:(e,t,r)=>{r.d(t,{Dz:()=>i,OJ:()=>a,qw:()=>o,t9:()=>n});const n=r(3325).D.pageViewEvent,i="firstbyte",o="domcontent",a="windowload"},9251:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageViewTiming},3614:(e,t,r)=>{r.d(t,{BST_RESOURCE:()=>i,END:()=>s,FEATURE_NAME:()=>n,FN_END:()=>u,FN_START:()=>c,PUSH_STATE:()=>d,RESOURCE:()=>o,START:()=>a});const n=r(3325).D.sessionTrace,i="bstResource",o="resource",a="-start",s="-end",c="fn"+a,u="fn"+s,d="pushState"},7836:(e,t,r)=>{r.d(t,{BODY:()=>A,CB_END:()=>E,CB_START:()=>u,END:()=>x,FEATURE_NAME:()=>i,FETCH:()=>_,FETCH_BODY:()=>v,FETCH_DONE:()=>m,FETCH_START:()=>p,FN_END:()=>c,FN_START:()=>s,INTERACTION:()=>l,INTERACTION_API:()=>d,INTERACTION_EVENTS:()=>o,JSONP_END:()=>b,JSONP_NODE:()=>g,JS_TIME:()=>T,MAX_TIMER_BUDGET:()=>a,REMAINING:()=>f,SPA_NODE:()=>h,START:()=>w,originalSetTimeout:()=>y});var n=r(5763);const i=r(3325).D.spa,o=["click","submit","keypress","keydown","keyup","change"],a=999,s="fn-start",c="fn-end",u="cb-start",d="api-ixn-",f="remaining",l="interaction",h="spaNode",g="jsonpNode",p="fetch-start",m="fetch-done",v="fetch-body-",b="jsonp-end",y=n.Yu.ST,w="-start",x="-end",A="-body",E="cb"+x,T="jsTime",_="fetch"},5938:(e,t,r)=>{r.d(t,{W:()=>o});var n=r(5763),i=r(2177);class o{constructor(e,t,r){this.agentIdentifier=e,this.aggregator=t,this.ee=i.ee.get(e,(0,n.OP)(this.agentIdentifier).isolatedBacklog),this.featureName=r,this.blocked=!1}}},9144:(e,t,r)=>{r.d(t,{j:()=>m});var n=r(3325),i=r(5763),o=r(5546),a=r(2177),s=r(7894),c=r(8e3),u=r(3960),d=r(385),f=r(50),l=r(3081),h=r(8632);function g(){const e=(0,h.gG)();["setErrorHandler","finished","addToTrace","inlineHit","addRelease","addPageAction","setCurrentRouteName","setPageViewName","setCustomAttribute","interaction","noticeError","setUserId"].forEach((t=>{e[t]=function(){for(var r=arguments.length,n=new Array(r),i=0;i 1?r-1:0),i=1;i {e.exposed&&e.api[t]&&o.push(e.api[t](...n))})),o.length>1?o:o[0]}(t,...n)}}))}var p=r(2587);function m(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},m=arguments.length>2?arguments[2]:void 0,v=arguments.length>3?arguments[3]:void 0,{init:b,info:y,loader_config:w,runtime:x={loaderType:m},exposed:A=!0}=t;const E=(0,h.gG)();y||(b=E.init,y=E.info,w=E.loader_config),(0,i.Dg)(e,b||{}),(0,i.GE)(e,w||{}),(0,i.sU)(e,x),y.jsAttributes??={},d.v6&&(y.jsAttributes.isWorker=!0),(0,i.CX)(e,y),g();const T=function(e,t){t||(0,c.R)(e,"api");const h={};var g=a.ee.get(e),p=g.get("tracer"),m="api-",v=m+"ixn-";function b(t,r,n,o){const a=(0,i.C5)(e);return null===r?delete a.jsAttributes[t]:(0,i.CX)(e,{...a,jsAttributes:{...a.jsAttributes,[t]:r}}),x(m,n,!0,o||null===r?"session":void 0)(t,r)}function y(){}["setErrorHandler","finished","addToTrace","inlineHit","addRelease"].forEach((e=>h[e]=x(m,e,!0,"api"))),h.addPageAction=x(m,"addPageAction",!0,n.D.pageAction),h.setCurrentRouteName=x(m,"routeName",!0,n.D.spa),h.setPageViewName=function(t,r){if("string"==typeof t)return"/"!==t.charAt(0)&&(t="/"+t),(0,i.OP)(e).customTransaction=(r||"http://custom.transaction")+t,x(m,"setPageViewName",!0)()},h.setCustomAttribute=function(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2];if("string"==typeof e){if(["string","number"].includes(typeof t)||null===t)return b(e,t,"setCustomAttribute",r);(0,f.Z)("Failed to execute setCustomAttribute.\nNon-null value must be a string or number type, but a type of was provided."))}else(0,f.Z)("Failed to execute setCustomAttribute.\nName must be a string type, but a type of was provided."))},h.setUserId=function(e){if("string"==typeof e||null===e)return b("enduser.id",e,"setUserId",!0);(0,f.Z)("Failed to execute setUserId.\nNon-null value must be a string type, but a type of was provided."))},h.interaction=function(){return(new y).get()};var w=y.prototype={createTracer:function(e,t){var r={},i=this,a="function"==typeof t;return(0,o.p)(v+"tracer",[(0,s.z)(),e,r],i,n.D.spa,g),function(){if(p.emit((a?"":"no-")+"fn-start",[(0,s.z)(),i,a],r),a)try{return t.apply(this,arguments)}catch(e){throw p.emit("fn-err",[arguments,this,"string"==typeof e?new Error(e):e],r),e}finally{p.emit("fn-end",[(0,s.z)()],r)}}}};function x(e,t,r,i){return function(){return(0,o.p)(l.xS,["API/"+t+"/called"],void 0,n.D.metrics,g),i&&(0,o.p)(e+t,[(0,s.z)(),...arguments],r?null:this,i,g),r?void 0:this}}function A(){r.e(439).then(r.bind(r,7438)).then((t=>{let{setAPI:r}=t;r(e),(0,c.L)(e,"api")})).catch((()=>(0,f.Z)("Downloading runtime APIs failed...")))}return["actionText","setName","setAttribute","save","ignore","onEnd","getContext","end","get"].forEach((e=>{w[e]=x(v,e,void 0,n.D.spa)})),h.noticeError=function(e,t){"string"==typeof e&&(e=new Error(e)),(0,o.p)(l.xS,["API/noticeError/called"],void 0,n.D.metrics,g),(0,o.p)("err",[e,(0,s.z)(),!1,t],void 0,n.D.jserrors,g)},d.il?(0,u.b)((()=>A()),!0):A(),h}(e,v);return(0,h.Qy)(e,T,"api"),(0,h.Qy)(e,A,"exposed"),(0,h.EZ)("activatedFeatures",p.T),T}},3325:(e,t,r)=>{r.d(t,{D:()=>n,p:()=>i});const n={ajax:"ajax",jserrors:"jserrors",metrics:"metrics",pageAction:"page_action",pageViewEvent:"page_view_event",pageViewTiming:"page_view_timing",sessionReplay:"session_replay",sessionTrace:"session_trace",spa:"spa"},i={[n.pageViewEvent]:1,[n.pageViewTiming]:2,[n.metrics]:3,[n.jserrors]:4,[n.ajax]:5,[n.sessionTrace]:6,[n.pageAction]:7,[n.spa]:8,[n.sessionReplay]:9}}},n={};function i(e){var t=n[e];if(void 0!==t)return t.exports;var o=n[e]={exports:{}};return r[e](o,o.exports,i),o.exports}i.m=r,i.d=(e,t)=>{for(var r in t)i.o(t,r)&&!i.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},i.f={},i.e=e=>Promise.all(Object.keys(i.f).reduce(((t,r)=>(i.f[r](e,t),t)),[])),i.u=e=>(({78:"page_action-aggregate",147:"metrics-aggregate",242:"session-manager",317:"jserrors-aggregate",348:"page_view_timing-aggregate",412:"lazy-feature-loader",439:"async-api",538:"recorder",590:"session_replay-aggregate",675:"compressor",733:"session_trace-aggregate",786:"page_view_event-aggregate",873:"spa-aggregate",898:"ajax-aggregate"}[e]||e)+"."+{78:"ac76d497",147:"3dc53903",148:"1a20d5fe",242:"2a64278a",317:"49e41428",348:"bd6de33a",412:"2f55ce66",439:"30bd804e",538:"1b18459f",590:"cf0efb30",675:"ae9f91a8",733:"83105561",786:"06482edd",860:"03a8b7a5",873:"e6b09d52",898:"998ef92b"}[e]+"-1.236.0.min.js"),i.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),e={},t="NRBA:",i.l=(r,n,o,a)=>{if(e[r])e[r].push(n);else{var s,c;if(void 0!==o)for(var u=document.getElementsByTagName("script"),d=0;d {s.onerror=s.onload=null,clearTimeout(h);var i=e[r];if(delete e[r],s.parentNode&&s.parentNode.removeChild(s),i&&i.forEach((e=>e(n))),t)return t(n)},h=setTimeout(l.bind(null,void 0,{type:"timeout",target:s}),12e4);s.onerror=l.bind(null,s.onerror),s.onload=l.bind(null,s.onload),c&&document.head.appendChild(s)}},i.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.j=364,i.p="https://js-agent.newrelic.com/",(()=>{var e={364:0,953:0};i.f.j=(t,r)=>{var n=i.o(e,t)?e[t]:void 0;if(0!==n)if(n)r.push(n[2]);else{var o=new Promise(((r,i)=>n=e[t]=[r,i]));r.push(n[2]=o);var a=i.p+i.u(t),s=new Error;i.l(a,(r=>{if(i.o(e,t)&&(0!==(n=e[t])&&(e[t]=void 0),n)){var o=r&&("load"===r.type?"missing":r.type),a=r&&r.target&&r.target.src;s.message="Loading chunk "+t+" failed.\n("+o+": "+a+")",s.name="ChunkLoadError",s.type=o,s.request=a,n[1](s)}}),"chunk-"+t,t)}};var t=(t,r)=>{var n,o,[a,s,c]=r,u=0;if(a.some((t=>0!==e[t]))){for(n in s)i.o(s,n)&&(i.m[n]=s[n]);if(c)c(i)}for(t&&t(r);u {i.r(o);var e=i(3325),t=i(5763);const r=Object.values(e.D);function n(e){const n={};return r.forEach((r=>{n[r]=function(e,r){return!1!==(0,t.Mt)(r,"".concat(e,".enabled"))}(r,e)})),n}var a=i(9144);var s=i(5546),c=i(385),u=i(8e3),d=i(5938),f=i(3960),l=i(50);class h extends d.W{constructor(e,t,r){let n=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];super(e,t,r),this.auto=n,this.abortHandler,this.featAggregate,this.onAggregateImported,n&&(0,u.R)(e,r)}importAggregator(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};if(this.featAggregate||!this.auto)return;const r=c.il&&!0===(0,t.Mt)(this.agentIdentifier,"privacy.cookies_enabled");let n;this.onAggregateImported=new Promise((e=>{n=e}));const o=async()=>{let t;try{if(r){const{setupAgentSession:e}=await Promise.all([i.e(860),i.e(242)]).then(i.bind(i,3228));t=e(this.agentIdentifier)}}catch(e){(0,l.Z)("A problem occurred when starting up session manager. This page will not start or extend any session.",e)}try{if(!this.shouldImportAgg(this.featureName,t))return void(0,u.L)(this.agentIdentifier,this.featureName);const{lazyFeatureLoader:r}=await i.e(412).then(i.bind(i,8582)),{Aggregate:o}=await r(this.featureName,"aggregate");this.featAggregate=new o(this.agentIdentifier,this.aggregator,e),n(!0)}catch(e){(0,l.Z)("Downloading and initializing ".concat(this.featureName," failed..."),e),this.abortHandler?.(),n(!1)}};c.il?(0,f.b)((()=>o()),!0):o()}shouldImportAgg(r,n){return r!==e.D.sessionReplay||!1!==(0,t.Mt)(this.agentIdentifier,"session_trace.enabled")&&(!!n?.isNew||!!n?.state.sessionReplay)}}var g=i(7633),p=i(7894);class m extends h{static featureName=g.t9;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];if(super(r,n,g.t9,i),("undefined"==typeof PerformanceNavigationTiming||c.Tt)&&"undefined"!=typeof PerformanceTiming){const n=(0,t.OP)(r);n[g.Dz]=Math.max(Date.now()-n.offset,0),(0,f.K)((()=>n[g.qw]=Math.max((0,p.z)()-n[g.Dz],0))),(0,f.b)((()=>{const t=(0,p.z)();n[g.OJ]=Math.max(t-n[g.Dz],0),(0,s.p)("timing",["load",t],void 0,e.D.pageViewTiming,this.ee)}))}this.importAggregator()}}var v=i(1117),b=i(1284);class y extends v.w{constructor(e){super(e),this.aggregatedData={}}store(e,t,r,n,i){var o=this.getBucket(e,t,r,i);return o.metrics=function(e,t){t||(t={count:0});return t.count+=1,(0,b.D)(e,(function(e,r){t[e]=w(r,t[e])})),t}(n,o.metrics),o}merge(e,t,r,n,i){var o=this.getBucket(e,t,n,i);if(o.metrics){var a=o.metrics;a.count+=r.count,(0,b.D)(r,(function(e,t){if("count"!==e){var n=a[e],i=r[e];i&&!i.c?a[e]=w(i.t,n):a[e]=function(e,t){if(!t)return e;t.c||(t=x(t.t));return t.min=Math.min(e.min,t.min),t.max=Math.max(e.max,t.max),t.t+=e.t,t.sos+=e.sos,t.c+=e.c,t}(i,a[e])}}))}else o.metrics=r}storeMetric(e,t,r,n){var i=this.getBucket(e,t,r);return i.stats=w(n,i.stats),i}getBucket(e,t,r,n){this.aggregatedData[e]||(this.aggregatedData[e]={});var i=this.aggregatedData[e][t];return i||(i=this.aggregatedData[e][t]={params:r||{}},n&&(i.custom=n)),i}get(e,t){return t?this.aggregatedData[e]&&this.aggregatedData[e][t]:this.aggregatedData[e]}take(e){for(var t={},r="",n=!1,i=0;i t.max&&(t.max=e),e 2&&void 0!==arguments[2])||arguments[2];super(e,r,j.t,n),c.il&&((0,t.OP)(e).initHidden=Boolean("hidden"===document.visibilityState),(0,N.N)((()=>(0,s.p)("docHidden",[(0,p.z)()],void 0,j.t,this.ee)),!0),(0,O.bP)("pagehide",(()=>(0,s.p)("winPagehide",[(0,p.z)()],void 0,j.t,this.ee))),this.importAggregator())}}var P=i(3081);class C extends h{static featureName=P.t9;constructor(e,t){let r=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(e,t,P.t9,r),this.importAggregator()}}var R,I=i(2210),k=i(1214),H=i(2177),L={};try{R=localStorage.getItem("__nr_flags").split(","),console&&"function"==typeof console.log&&(L.console=!0,-1!==R.indexOf("dev")&&(L.dev=!0),-1!==R.indexOf("nr_dev")&&(L.nrDev=!0))}catch(e){}function z(e){try{L.console&&z(e)}catch(e){}}L.nrDev&&H.ee.on("internal-error",(function(e){z(e.stack)})),L.dev&&H.ee.on("fn-err",(function(e,t,r){z(r.stack)})),L.dev&&(z("NR AGENT IN DEVELOPMENT MODE"),z("flags: "+(0,b.D)(L,(function(e,t){return e})).join(", ")));var M=i(6660);class B extends h{static featureName=M.t;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(r,n,M.t,i),this.skipNext=0;try{this.removeOnAbort=new AbortController}catch(e){}const o=this;o.ee.on("fn-start",(function(e,t,r){o.abortHandler&&(o.skipNext+=1)})),o.ee.on("fn-err",(function(t,r,n){o.abortHandler&&!n[M.A]&&((0,I.X)(n,M.A,(function(){return!0})),this.thrown=!0,(0,s.p)("err",[n,(0,p.z)()],void 0,e.D.jserrors,o.ee))})),o.ee.on("fn-end",(function(){o.abortHandler&&!this.thrown&&o.skipNext>0&&(o.skipNext-=1)})),o.ee.on("internal-error",(function(t){(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,o.ee)})),this.origOnerror=c._A.onerror,c._A.onerror=this.onerrorHandler.bind(this),c._A.addEventListener("unhandledrejection",(t=>{const r=function(e){let t="Unhandled Promise Rejection: ";if(e instanceof Error)try{return e.message=t+e.message,e}catch(t){return e}if(void 0===e)return new Error(t);try{return new Error(t+(0,D.P)(e))}catch(e){return new Error(t)}}(t.reason);(0,s.p)("err",[r,(0,p.z)(),!1,{unhandledPromiseRejection:1}],void 0,e.D.jserrors,this.ee)}),(0,O.m$)(!1,this.removeOnAbort?.signal)),(0,k.gy)(this.ee),(0,k.BV)(this.ee),(0,k.em)(this.ee),(0,t.OP)(r).xhrWrappable&&(0,k.Kf)(this.ee),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}onerrorHandler(t,r,n,i,o){"function"==typeof this.origOnerror&&this.origOnerror(...arguments);try{this.skipNext?this.skipNext-=1:(0,s.p)("err",[o||new F(t,r,n),(0,p.z)()],void 0,e.D.jserrors,this.ee)}catch(t){try{(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,this.ee)}catch(e){}}return!1}}function F(e,t,r){this.message=e||"Uncaught error with no additional information",this.sourceURL=t,this.line=r}let U=1;const q="nr@id";function G(e){const t=typeof e;return!e||"object"!==t&&"function"!==t?-1:e===c._A?0:(0,I.X)(e,q,(function(){return U++}))}function V(e){if("string"==typeof e&&e.length)return e.length;if("object"==typeof e){if("undefined"!=typeof ArrayBuffer&&e instanceof ArrayBuffer&&e.byteLength)return e.byteLength;if("undefined"!=typeof Blob&&e instanceof Blob&&e.size)return e.size;if(!("undefined"!=typeof FormData&&e instanceof FormData))try{return(0,D.P)(e).length}catch(e){return}}}var X=i(7243);class W{constructor(e){this.agentIdentifier=e,this.generateTracePayload=this.generateTracePayload.bind(this),this.shouldGenerateTrace=this.shouldGenerateTrace.bind(this)}generateTracePayload(e){if(!this.shouldGenerateTrace(e))return null;var r=(0,t.DL)(this.agentIdentifier);if(!r)return null;var n=(r.accountID||"").toString()||null,i=(r.agentID||"").toString()||null,o=(r.trustKey||"").toString()||null;if(!n||!i)return null;var a=(0,_.M)(),s=(0,_.Ht)(),c=Date.now(),u={spanId:a,traceId:s,timestamp:c};return(e.sameOrigin||this.isAllowedOrigin(e)&&this.useTraceContextHeadersForCors())&&(u.traceContextParentHeader=this.generateTraceContextParentHeader(a,s),u.traceContextStateHeader=this.generateTraceContextStateHeader(a,c,n,i,o)),(e.sameOrigin&&!this.excludeNewrelicHeader()||!e.sameOrigin&&this.isAllowedOrigin(e)&&this.useNewrelicHeaderForCors())&&(u.newrelicHeader=this.generateTraceHeader(a,s,c,n,i,o)),u}generateTraceContextParentHeader(e,t){return"00-"+t+"-"+e+"-01"}generateTraceContextStateHeader(e,t,r,n,i){return i+"@nr=0-1-"+r+"-"+n+"-"+e+"----"+t}generateTraceHeader(e,t,r,n,i,o){if(!("function"==typeof c._A?.btoa))return null;var a={v:[0,1],d:{ty:"Browser",ac:n,ap:i,id:e,tr:t,ti:r}};return o&&n!==o&&(a.d.tk=o),btoa((0,D.P)(a))}shouldGenerateTrace(e){return this.isDtEnabled()&&this.isAllowedOrigin(e)}isAllowedOrigin(e){var r=!1,n={};if((0,t.Mt)(this.agentIdentifier,"distributed_tracing")&&(n=(0,t.P_)(this.agentIdentifier).distributed_tracing),e.sameOrigin)r=!0;else if(n.allowed_origins instanceof Array)for(var i=0;i 2&&void 0!==arguments[2])||arguments[2];super(r,n,Z.t,i),(0,t.OP)(r).xhrWrappable&&(this.dt=new W(r),this.handler=(e,t,r,n)=>(0,s.p)(e,t,r,n,this.ee),(0,k.u5)(this.ee),(0,k.Kf)(this.ee),function(r,n,i,o){function a(e){var t=this;t.totalCbs=0,t.called=0,t.cbTime=0,t.end=E,t.ended=!1,t.xhrGuids={},t.lastSize=null,t.loadCaptureCalled=!1,t.params=this.params||{},t.metrics=this.metrics||{},e.addEventListener("load",(function(r){_(t,e)}),(0,O.m$)(!1)),c.IF||e.addEventListener("progress",(function(e){t.lastSize=e.loaded}),(0,O.m$)(!1))}function s(e){this.params={method:e[0]},T(this,e[1]),this.metrics={}}function u(e,n){var i=(0,t.DL)(r);i.xpid&&this.sameOrigin&&n.setRequestHeader("X-NewRelic-ID",i.xpid);var a=o.generateTracePayload(this.parsedOrigin);if(a){var s=!1;a.newrelicHeader&&(n.setRequestHeader("newrelic",a.newrelicHeader),s=!0),a.traceContextParentHeader&&(n.setRequestHeader("traceparent",a.traceContextParentHeader),a.traceContextStateHeader&&n.setRequestHeader("tracestate",a.traceContextStateHeader),s=!0),s&&(this.dt=a)}}function d(e,t){var r=this.metrics,i=e[0],o=this;if(r&&i){var a=V(i);a&&(r.txSize=a)}this.startTime=(0,p.z)(),this.listener=function(e){try{"abort"!==e.type||o.loadCaptureCalled||(o.params.aborted=!0),("load"!==e.type||o.called===o.totalCbs&&(o.onloadCalled||"function"!=typeof t.onload)&&"function"==typeof o.end)&&o.end(t)}catch(e){try{n.emit("internal-error",[e])}catch(e){}}};for(var s=0;s 1?e[1]=i:e.push(i)}else e[0]&&e[0].headers&&s(e[0].headers,n)&&(this.dt=n);function s(e,t){var r=!1;return t.newrelicHeader&&(e.set("newrelic",t.newrelicHeader),r=!0),t.traceContextParentHeader&&(e.set("traceparent",t.traceContextParentHeader),t.traceContextStateHeader&&e.set("tracestate",t.traceContextStateHeader),r=!0),r}}function x(e,t){this.params={},this.metrics={},this.startTime=(0,p.z)(),this.dt=t,e.length>=1&&(this.target=e[0]),e.length>=2&&(this.opts=e[1]);var r,n=this.opts||{},i=this.target;"string"==typeof i?r=i:"object"==typeof i&&i instanceof Y?r=i.url:c._A?.URL&&"object"==typeof i&&i instanceof URL&&(r=i.href),T(this,r);var o=(""+(i&&i instanceof Y&&i.method||n.method||"GET")).toUpperCase();this.params.method=o,this.txSize=V(n.body)||0}function A(t,r){var n;this.endTime=(0,p.z)(),this.params||(this.params={}),this.params.status=r?r.status:0,"string"==typeof this.rxSize&&this.rxSize.length>0&&(n=+this.rxSize);var o={txSize:this.txSize,rxSize:n,duration:(0,p.z)()-this.startTime};i("xhr",[this.params,o,this.startTime,this.endTime,"fetch"],this,e.D.ajax)}function E(t){var r=this.params,n=this.metrics;if(!this.ended){this.ended=!0;for(var o=0;o 2&&void 0!==arguments[2])||arguments[2];super(e,t,we.t,r),this.importAggregator()}}new class{constructor(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:(0,_.ky)(16);c._A?(this.agentIdentifier=t,this.sharedAggregator=new y({agentIdentifier:this.agentIdentifier}),this.features={},this.desiredFeatures=new Set(e.features||[]),this.desiredFeatures.add(m),Object.assign(this,(0,a.j)(this.agentIdentifier,e,e.loaderType||"agent")),this.start()):(0,l.Z)("Failed to initial the agent. Could not determine the runtime environment.")}get config(){return{info:(0,t.C5)(this.agentIdentifier),init:(0,t.P_)(this.agentIdentifier),loader_config:(0,t.DL)(this.agentIdentifier),runtime:(0,t.OP)(this.agentIdentifier)}}start(){const t="features";try{const r=n(this.agentIdentifier),i=[...this.desiredFeatures];i.sort(((t,r)=>e.p[t.featureName]-e.p[r.featureName])),i.forEach((t=>{if(r[t.featureName]||t.featureName===e.D.pageViewEvent){const n=function(t){switch(t){case e.D.ajax:return[e.D.jserrors];case e.D.sessionTrace:return[e.D.ajax,e.D.pageViewEvent];case e.D.sessionReplay:return[e.D.sessionTrace];case e.D.pageViewTiming:return[e.D.pageViewEvent];default:return[]}}(t.featureName);n.every((e=>r[e]))||(0,l.Z)("".concat(t.featureName," is enabled but one or more dependent features has been disabled (").concat((0,D.P)(n),"). This may cause unintended consequences or missing data...")),this.features[t.featureName]=new t(this.agentIdentifier,this.sharedAggregator)}})),(0,T.Qy)(this.agentIdentifier,this.features,t)}catch(e){(0,l.Z)("Failed to initialize all enabled instrument classes (agent aborted) -",e);for(const e in this.features)this.features[e].abortHandler?.();const r=(0,T.fP)();return delete r.initializedAgents[this.agentIdentifier]?.api,delete r.initializedAgents[this.agentIdentifier]?.[t],delete this.sharedAggregator,r.ee?.abort(),delete r.ee?.get(this.agentIdentifier),!1}}}({features:[J,m,S,class extends h{static featureName=oe;constructor(t,r){if(super(t,r,oe,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;const n=this.ee;let i;(0,k.QU)(n),this.eventsEE=(0,k.em)(n),this.eventsEE.on(se,(function(e,t){this.bstStart=(0,p.z)()})),this.eventsEE.on(ae,(function(t,r){(0,s.p)("bst",[t[0],r,this.bstStart,(0,p.z)()],void 0,e.D.sessionTrace,n)})),n.on(ce+ne,(function(e){this.time=(0,p.z)(),this.startPath=location.pathname+location.hash})),n.on(ce+ie,(function(t){(0,s.p)("bstHist",[location.pathname+location.hash,this.startPath,this.time],void 0,e.D.sessionTrace,n)}));try{i=new PerformanceObserver((t=>{const r=t.getEntries();(0,s.p)(te,[r],void 0,e.D.sessionTrace,n)})),i.observe({type:re,buffered:!0})}catch(e){}this.importAggregator({resourceObserver:i})}},C,xe,B,class extends h{static featureName=de;constructor(e,r){if(super(e,r,de,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;if(!(0,t.OP)(e).xhrWrappable)return;try{this.removeOnAbort=new AbortController}catch(e){}let n,i=0;const o=this.ee.get("tracer"),a=(0,k._L)(this.ee),s=(0,k.Lg)(this.ee),u=(0,k.BV)(this.ee),d=(0,k.Kf)(this.ee),f=this.ee.get("events"),l=(0,k.u5)(this.ee),h=(0,k.QU)(this.ee),g=(0,k.Gm)(this.ee);function m(e,t){h.emit("newURL",[""+window.location,t])}function v(){i++,n=window.location.hash,this[ve]=(0,p.z)()}function b(){i--,window.location.hash!==n&&m(0,!0);var e=(0,p.z)();this[pe]=~~this[pe]+e-this[ve],this[ye]=e}function y(e,t){e.on(t,(function(){this[t]=(0,p.z)()}))}this.ee.on(ve,v),s.on(be,v),a.on(be,v),this.ee.on(ye,b),s.on(ge,b),a.on(ge,b),this.ee.buffer([ve,ye,"xhr-resolved"],this.featureName),f.buffer([ve],this.featureName),u.buffer(["setTimeout"+le,"clearTimeout"+fe,ve],this.featureName),d.buffer([ve,"new-xhr","send-xhr"+fe],this.featureName),l.buffer([me+fe,me+"-done",me+he+fe,me+he+le],this.featureName),h.buffer(["newURL"],this.featureName),g.buffer([ve],this.featureName),s.buffer(["propagate",be,ge,"executor-err","resolve"+fe],this.featureName),o.buffer([ve,"no-"+ve],this.featureName),a.buffer(["new-jsonp","cb-start","jsonp-error","jsonp-end"],this.featureName),y(l,me+fe),y(l,me+"-done"),y(a,"new-jsonp"),y(a,"jsonp-end"),y(a,"cb-start"),h.on("pushState-end",m),h.on("replaceState-end",m),window.addEventListener("hashchange",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("load",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("popstate",(function(){m(0,i>1)}),(0,O.m$)(!0,this.removeOnAbort?.signal)),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}}],loaderType:"spa"})})(),window.NRBA=o})(); window.jQuery || document.write(' ') CKEDITOR_BASEPATH='https://f1000research.com/js/vendor/ckeditor/' window.reactTheme = 'research'; window.MathJax = { CommonHTML: { linebreaks: { automatic: true } }, 'HTML-CSS': { linebreaks: { automatic: true } }, SVG: { linebreaks: { automatic: true } }, AuthorInit: function() { MathJax.Hub.Register.MessageHook('End Process', function () { let timeout = false; // holder for timeout id const delay = 250; // delay after event is "complete" to run callback const reflowMath = function() { const dispFormulas = document.querySelectorAll('.disp-formula.panel'); if (!dispFormulas) { return; } for (const dispFormula of dispFormulas) { const child = dispFormula.querySelector('.MathJax_Preview').nextSibling.firstChild; const isMultiline = MathJax.Hub.getAllJax(dispFormula)[0].root.isMultiline; if (dispFormula.offsetWidth < child.offsetWidth || isMultiline) { MathJax.Hub.Queue(['Rerender', MathJax.Hub, dispFormula]); } } }; window.addEventListener('resize', function() { clearTimeout(timeout); // clear the timeout timeout = setTimeout(reflowMath, delay); // start timing for event "completion" }); }); }, }; if (window.location.hash == '#_=_'){ window.location = window.location.href.split('#')[0] } !function(f,b,e,v,n,t,s){if(f.fbq)return;n=f.fbq=function() {n.callMethod? n.callMethod.apply(n,arguments):n.queue.push(arguments)} ;if(!f._fbq)f._fbq=n; n.push=n;n.loaded=!0;n.version='2.0';n.queue=[];t=b.createElement(e);t.async=!0; t.src=v;s=b.getElementsByTagName(e)[0];s.parentNode.insertBefore(t,s)}(window, document,'script','https://connect.facebook.net/en_US/fbevents.js'); fbq('init', '1641728616063202'); fbq('track', "PixelInitialized", {}); (function(h,o,t,j,a,r){ h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)}; h._hjSettings={hjid:2318163,hjsv:6}; a=o.getElementsByTagName('head')[0]; r=o.createElement('script');r.async=1; r.src=t+h._hjSettings.hjid+j+h._hjSettings.hjsv; a.appendChild(r); })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv='); search file_upload Submit your research search menu close search Browse Gateways & Collections How to Publish Submit your Research My Submissions Article Guidelines Article Guidelines (New Versions) Open Data, Software and Code Guidelines Open Data and Accessible Source Materials Guidelines (HSS) Open Data, Software and Code Guidelines (PSE) Prepublication Checks Production Process Posters and Slides Guidelines Document Guidelines Article Processing Charges Peer Review Finding Article Reviewers About How it Works For Reviewers Our Advisors Policies Glossary FAQs For Developers Newsroom Contact My Research Submissions Content and Tracking Alerts My Details Sign In file_upload Submit your research { "@context": "https://schema.org", "@type": "ScholarlyArticle", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://f1000research.com/articles/13-710" }, "headline": "Expert Consensus Methods In The Humanities: An Exploration of their Potential", "datePublished": "2024-06-28T10:55:38", "dateModified": "2024-12-16T14:22:36", "author": [ { "@type": "Person", "name": "Charlotte C.S. Rulkens" }, { "@type": "Person", "name": "Rik Peels" }, { "@type": "Person", "name": "Lidwine B. Mokkink" }, { "@type": "Person", "name": "Tamarinde Haven" }, { "@type": "Person", "name": "Lex Bouter" } ], "publisher": { "@type": "Organization", "name": "F1000Research", "logo": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 480, "width": 60 } }, "image": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 1200, "width": 150 }, "description": " Background Despite the significant role of consensus and dissensus in knowledge production, formal approaches to consensus are notably less common in the humanities compared to their frequent application in natural, social, and life sciences. This article therefore explores the potential of expert consensus methods in humanities-related research. Methods In order to do so, an interdisciplinary team of both sciences researchers experienced in consensus methods and researchers familiar with the domain of the humanities and epistemology, conducted a literary review and exchanged their expertise in multiple brainstorm sessions. Results This resulted in the identification of six key elements of expert consensus methods. It also provided for an overview of different types of expert consensus methods that regularly used in the natural, social, and life sciences: Delphi studies, nominal groups, consensus conferences, and Glaser’s state of the art method and illustrative examples from both sciences and humanities-related studies. An overview of possible purposes for applying these methods is provided to identify the research contexts in which these methods have proven their value, which can be extrapolated to humanities related issues for which these methods seem promising. Conclusions The comparisons and categorisation show that, when focusing on the purposes, there seem to be humanities-related issues that may lend themselves better to structured expert consensus methods than their subject matter and research methods might suggest. When deliberately applied in context chosen by researchers with expertise in a specific humanities domain, expert consensus methods can accelerate epistemic process, enhance transparency, increase replicability, stimulate diversity, and encourage fair processes in humanities research and the application of its findings. " } { "@context": "http://schema.org", "@type": "BreadcrumbList", "itemListElement": [ { "@type": "ListItem", "position": "1", "item": { "@id": "https://f1000research.com/", "name": "Home" } }, { "@type": "ListItem", "position": "2", "item": { "@id": "https://f1000research.com/browse/articles", "name": "Browse" } }, { "@type": "ListItem", "position": "3", "item": { "@id": "https://f1000research.com/articles/13-710", "name": "Expert Consensus Methods In The Humanities: An Exploration of their..." } } ] } Home Browse Expert Consensus Methods In The Humanities: An Exploration of their... ALL Metrics - Views Downloads Get PDF Get XML Cite How to cite this article Rulkens CCS, Peels R, Mokkink LB et al. Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.12688/f1000research.148726.2 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. Close Copy Citation Details Export Export Citation Sciwheel EndNote Ref. Manager Bibtex ProCite Sente EXPORT Select a format first Track Share ▬ ✚ Method Article Revised Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] Charlotte C.S. Rulkens https://orcid.org/0000-0002-4617-9507 1 , Rik Peels 2,3 , Lidwine B. Mokkink 4,5 , Tamarinde Haven 6 , Lex Bouter https://orcid.org/0000-0002-2659-5482 1,5 Charlotte C.S. Rulkens https://orcid.org/0000-0002-4617-9507 1 , Rik Peels 2,3 , [...] Lidwine B. Mokkink 4,5 , Tamarinde Haven 6 , Lex Bouter https://orcid.org/0000-0002-2659-5482 1,5 PUBLISHED 16 Dec 2024 Author details Author details 1 Department of Philosophy, Vrije Universiteit Amsterdam, Amsterdam, North Holland, The Netherlands 2 Faculty of Religion and Theology and Faculty of Humanities, Vrije Universiteit Amsterdam, Amsterdam, North Holland, The Netherlands 3 African Centre for Epistemology and Philosophy of Science, University of Johannesburg, Auckland Park, Gauteng, South Africa 4 Department of Methodology, Amsterdam Public Health research institute, Amsterdam, The Netherlands 5 Department of Epidemiology and Data Science, Amsterdam University Medical Centres, Duivendrecht, North Holland, The Netherlands 6 Department of Methodology and Statistics, Tilburg School of Social and Behavioural Sciences, Tilburg University, Tilburg, North Brabant, The Netherlands Charlotte C.S. Rulkens Roles: Conceptualization, Project Administration, Writing – Original Draft Preparation, Writing – Review & Editing Rik Peels Roles: Conceptualization, Funding Acquisition, Supervision, Writing – Original Draft Preparation, Writing – Review & Editing Lidwine B. Mokkink Roles: Conceptualization, Writing – Review & Editing Tamarinde Haven Roles: Conceptualization, Writing – Review & Editing Lex Bouter Roles: Conceptualization, Funding Acquisition, Supervision, Writing – Review & Editing OPEN PEER REVIEW DETAILS REVIEWER STATUS This article is included in the Research on Research, Policy & Culture gateway. Abstract Background Despite the significant role of consensus and dissensus in knowledge production, formal approaches to consensus are notably less common in the humanities compared to their frequent application in natural, social, and life sciences. This article therefore explores the potential of expert consensus methods in humanities-related research. Methods In order to do so, an interdisciplinary team of both sciences researchers experienced in consensus methods and researchers familiar with the domain of the humanities and epistemology, conducted a literary review and exchanged their expertise in multiple brainstorm sessions. Results This resulted in the identification of six key elements of expert consensus methods. It also provided for an overview of different types of expert consensus methods that regularly used in the natural, social, and life sciences: Delphi studies, nominal groups, consensus conferences, and Glaser’s state of the art method and illustrative examples from both sciences and humanities-related studies. An overview of possible purposes for applying these methods is provided to identify the research contexts in which these methods have proven their value, which can be extrapolated to humanities related issues for which these methods seem promising. Conclusions The comparisons and categorisation show that, when focusing on the purposes, there seem to be humanities-related issues that may lend themselves better to structured expert consensus methods than their subject matter and research methods might suggest. When deliberately applied in context chosen by researchers with expertise in a specific humanities domain, expert consensus methods can accelerate epistemic process, enhance transparency, increase replicability, stimulate diversity, and encourage fair processes in humanities research and the application of its findings. READ ALL READ LESS Keywords Consensus, Consensus methods, Humanities, Methodology, Expertise, Epistemology Corresponding Author(s) Charlotte C.S. Rulkens ( [email protected] ) Close Corresponding author: Charlotte C.S. Rulkens Competing interests: No competing interests were disclosed. Grant information: Work on this article by Charlotte Rulkens, Lex Bouter, and Rik Peels was made possible through the generous support of a grant on Epistemic Progress in the University (grant number 0436) from the Templeton World Charity Foundation The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript. Copyright: © 2024 Rulkens CCS et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. How to cite: Rulkens CCS, Peels R, Mokkink LB et al. Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.12688/f1000research.148726.2 ) First published: 28 Jun 2024, 13 :710 ( https://doi.org/10.12688/f1000research.148726.1 ) Latest published: 16 Dec 2024, 13 :710 ( https://doi.org/10.12688/f1000research.148726.2 ) Revised Amendments from Version 1 In response to reviewer feedback, we have revised the introduction to include a clearer description of our paper’s goal. We also improved the methodology section and added a new table outlining possible objectives for applying expert consensus method. This table can serve as a guide to humanities researchers interested in the contexts where formal assessment of consensus may be beneficial. Additionally, we included remarks on the wide variation in research and methods across both sciences and humanities disciplines, to indicate that although we use the terms, we do not believe in distinct categorisations between and within these domains. Overall, we created a more cohesive flow throughout the paper and have made more explicit what our line of reasoning is. In response to reviewer feedback, we have revised the introduction to include a clearer description of our paper’s goal. We also improved the methodology section and added a new table outlining possible objectives for applying expert consensus method. This table can serve as a guide to humanities researchers interested in the contexts where formal assessment of consensus may be beneficial. Additionally, we included remarks on the wide variation in research and methods across both sciences and humanities disciplines, to indicate that although we use the terms, we do not believe in distinct categorisations between and within these domains. Overall, we created a more cohesive flow throughout the paper and have made more explicit what our line of reasoning is. See the authors' detailed response to the review by Emanuela Reale See the authors' detailed response to the review by Daniel Leising READ REVIEWER RESPONSES 1. Introduction An important way to make progress in knowledge and understanding is to gradually, via the exchange of evidence, expertise, and critical discussion, reach (more) consensus on a topic. However, whether there is consensus on a certain matter at a moment in time can sometimes remain unknown, be unclear, or remain implicit. Expert consensus methods can lift off this inconclusiveness by intentionally and formally pursuing or assessing consensus. These methods are regularly used in natural, social, and life sciences. Despite the significant role of consensus in the development of (academic) knowledge and understanding (e.g. Kuhn 1962 , Habermas 1996 , thus also Peels 2020 , Grimm et al. forthcoming ), formal expert consensus methods have hardly been explored within in the context of the humanities. 1 Other kinds of valuable expert meetings do take place in various disciplines in the humanities. They usually do not seek to reach a consensus, but to explore a topic in-depth by bringing the relevant experts to the table. Moreover, expert consensus methods described in this article differ from the expert meetings that already take place in the humanities by the former’s procedural and formalised character. In this paper, we aim to highlight to scholars, particularly those in the humanities, what is possible in terms of a systematic approach to academic consensus. In doing so, we aim to initiate exploration of and discussion about the potential of applying such an approach. What would be the added value of applying expert consensus meetings to humanities-related issues? To achieve this, we first explicate our method (§2), before we provide a framework for the application of expert consensus methods by introducing six key elements (§3). Subsequently, we present common types of expert consensus methods utilised in the natural, social, and life sciences. We highlight their proven value across various types of research through relevant examples and their respective objectives (§4). We then consider both the potential benefits and possible limitations to applying these methods within humanistic research (§5), and conclude with suggestions for future research (§6). 2. Methods To explore the potential of applying expert consensus methods in the humanities, we composed an interdisciplinary team of both researchers from the sciences using consensus methods in practice (LM: epidemiology; TH: qualitative research methodology; LB: epidemiology and data science, research integrity), and researchers familiar with the domain of the humanities and epistemology (CR: art history, museology; RP: epistemology and philosophy of science, religion and theology). In a first brainstorm session, knowledge on the topic was shared from both perspectives. This resulted in a list of consensus methods and examples used in the sciences, drafted by LM and TH. CR carried out a literature search in order to find examples thereof that were applied in her domain. A first draft was written and discussed in a second group session and culminated in a final version, which was edited and reviewed by all authors. 3. Six Key elements of expert consensus methods Philosophers have reflected on the value and function of consensus as a means to knowledge (e.g. Kuhn 1962 , Habermas 1996 ). Some researchers defend the importance of consensus as an indicator for academic trustworthiness, and the value of consensus studies to assess pressing issues such as climate change ( Oreskes 2021 ). Within the field of personality psychology, others have stressed the importance of fostering and explicating consensus on key preconditions and future research goals to move the entire field forward (Leising et al. 2022a, 2022b). In this article, we take a more practical approach by presenting means to assess or establish consensus. Before we explore the value of expert consensus methods in the humanities, it is helpful to first define what we mean by ‘consensus’. Ordinary language use of ‘consensus’ might suppose that there is consensus only if a group agrees completely (for 100%) on an issue (see e.g., von der Gracht et al. 2012 ). However, also when there is no full consensus and only a large percentage of a group agrees, one can speak of consensus on an issue ( Diamond et al. 2014 ). Clearly, this entails ambiguity, for when is such a group sufficiently large and who are the experts that belong to that group? Moreover, what counts as ‘sufficient’ consensus may differ from discipline to discipline and from issue to issue. Purely deductive disciplines, such as mathematics and logic, may reach full or almost full consensus on some issues and the bar for consensus may, therefore, be high in those fields. Since the questions addressed in the humanities may leave room for multiple viewpoints that are equally warranted by the evidence base, it can be infeasible or not desirable to get a complete or almost complete consensus. The bar for consensus on such issues could be set lower. Consensus can take the shape of agreeing that something is true or accurate, but also that something is false, or inaccurate, or unreliable. There can even be consensus that we do not know something or that the available evidence does not favour any particular hypothesis. A philosophically informed way to put this is that consensus can take the shape of one out of three doxastic (from the Greek doxa , ‘view’) attitudes: joint belief, joint disbelief, and joint suspension of judgment (Peels 2017, chapter 1). Furthermore, consensus can have different objects: experts can agree on the viability of a model, on the predictive power of a theory, on the adequacy of a hypothesis, on what best explains a phenomenon, on what the most suitable method is to study something, on policies for prevention, on definitions, and much more. And once consensus is reached, it is not guaranteed to remain static. It can change over time with emerging new viewpoints or insights deriving from new research. Whether or not there is consensus on an issue at a certain moment in time often remains implicit or inconclusive. This especially holds when there is no full agreement or when it is not clear whether there is ‘sufficient’ agreement. Expert consensus methods aim to make explicit whether or not there is consensus and what the degree of consensus is. This can be useful when the issue at hand is an important one and when evidence is lacking, it is inconsistent, or its interpretation is not obvious. What expert consensus methods do in such cases can be defined as follows: Expert consensus methods are (i) applied by a process leader or steering committee , (ii) to reach consensus or assess the degree of consensus among a group of experts , (iii) about an issue , (iv) based on pre-set rules of engagement , (v) with the aim to deliver useful output , (vi) for future users. They are characterised by constructive and procedural usage of (dis)agreement and the different arguments that are provided therein. Let us briefly clarify the core terms of this definition. i. A process leader or steering committee identifies an issue and initiates and designs the expert consensus method. They consider which groups have to be represented in the method and then select the experts that are to participate as panellists. The process of how experts are selected and by what criteria should be made fully transparent, because expert consensus methods find part of their credibility and validity in how the process leaders or steering committee determines who counts as an expert. In addition to that, the process leader or steering committee prepares the experts for the overall process and steps of the expert consensus method and provides the experts with rules of engagement. These can be adjusted at the instigation of the experts before starting the method. The process leader or steering committee is also responsible for collecting the data and/or information that is needed to assess the issue at hand and ensures every expert is provided with the same set of information and/or evidence before participating in the method. ii. Experts are the panellists in the consensus procedure that deliver input in the process of measuring and/or reaching consensus. The nature of their expertise can differ, depending on the issue and prospected output of the expert consensus method. They can be experts by training and profession, e.g., a physician making a diagnosis or a historian interpreting a source. But they also can be experts on the basis of their experience, e.g. patients’ knowledge about their own illness or students delivering input on their learning experiences at school. iii. Issues are at the centre of the expert consensus method, they are the problem to be solved, the proposals to be considered, or the question to be answered. iv. Rules of engagement are part of the design of the expert consensus method and specify its preconditions. These preconditions can be, but are not limited to, how high the threshold for consensus is (i.e. what ‘sufficient’ agreement is), which areas of disagreement may be retained, how interaction is structured, when to transition into finalising the output, or what to do when consensus is not reached. Experts may suggest changes and/or improvements to the rules of engagement before engaging as panellists. The rules of engagement should be made fully transparent. v. Output can be, but is not limited to, answers to closed questions, guidelines, taxonomies, questionnaires, definitions, quality criteria, and policy advice. vi. Users are those who have an interest in applying the output of the consensus, including but not limited to the experts involved. They can be researchers (starting follow-up projects), the public (making informed decisions), professionals (working according to a guideline), institutions (determining their mission and ambitions and governments (implementing policies). To successfully apply an expert consensus method, careful preparation by the process leader or steering committee and the effective communication between all participating parties about the process is of vital importance. The roles and responsibilities of different actors (the experts versus the process leader or steering committee) need to be laid out and clearly communicated from the start to enable users to verify the quality of the process. The amount of evidence that is available to the experts determines the part of the issue left about which panellists can reach consensus on: i.e. the inclusion of different viewpoints on the issue is one of the most valuable parts of the process, however, the eventual consensus cannot be in contradiction with the available evidence base. This whole process and its outcomes are made transparent and accessible in a publication by the process leader or steering committee. Figure 1 presents the interrelation of the six elements of expert consensus methods (blue boxes) and the fourteen main steps undertaken in the process (arrows). The boxes and steps that are in the orange field are addressed in the publication and its appendices. Figure 1. The six elements of expert consensus methods (blue boxes) and the fourteen main steps of the process (arrows). All steps in the orange area are described in the final publication. Naturally, the six elements can be specified by the steering committee. If they deem it valuable to include an independent moderator or to incorporate a member with a critical or divergent perspective—especially if the topic is highly controversial—this remains within their discretion. Additionally, it is up to them to decide who will be listed as co-authors, whether that includes only the steering committee, all participants, or a broader group representation such as the panellists as a whole, for instance. 4. Types, examples and purposes After establishing this foundational understanding of consensus and the dynamic of expert consensus methods, Table 1 presents and compares the four expert consensus methods that are used frequently in the natural, social and life sciences: Delphi studies, nominal groups, consensus conferences, and Glaser’s state of the art method. As expert consensus methods can be tailor-made to the issue at hand, many studies combine elements of these methods. Tables 2 and 3 each display four examples of studies employing one or more of these expert consensus methods, both in the natural, social and life sciences and in humanities-related fields. All examples are described by the six key elements as defined by us. The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and subdisciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalizability to other fields of research and were included in Tables 2 and 3 . Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove to be effective. Table 1. Frequently used expert consensus methods in the natural, social, and life sciences and their characteristics. Synthesises of criteria as previously discussed in Jones and Hunter (1995) , Black et al. (1999) and Blazey et al. (2022) . Name Delphi study Nominal Group Technique Consensus conference Glaser’s state of the art method General description of the process A series of (online) surveys where panellists are asked to vote and comment on different topics, interspersed by detailed feedback of the survey results which are shared with panellists Introduction of the topic; silent generation of ideas (panellists write down their own ideas); the facilitator goes round in circles asking one panellist at the time to share their idea, (every idea is recorded) until no further ideas emerge; clarification phase where ideas can be clarified and grouped (seeking explanation); voting and ranking (prioritising the ideas) Formulation of a list of questions that determine the scope and direction of the conference (questions are widely disseminated); organisation of a conference where all relevant data and views are discussed; presenting draft consensus statement by independent panel; amendments are made immediately Levelled approach to obtaining consensus. A facilitator gets selected, this person invites a core group, the core group drafts a position paper (multiple iterations); this core group then invited a bigger group of experts to criticise the draft (multiple iterations); the draft is sent out to independent reviewers via a journal leading to revisions of the paper in waves (a certain percentage of invited reviewers submits their reviews, which are then incorporated, to be sent out to new percentage of reviewers) until point of diminishing returns when the paper gets published by the journal Key references to origins of the method ( Powell 2003 ) ( Goodman 1987 ) ( Linstone and Turoff 1975 ) ( Dalkey and Helmer 1963 ) ( Delbecq et al. 1975 ) ( Stocking et al. 1991 ) ( Glaser 1980 ) Use of questionnaires (taken from: Black et al. 1999 ) Yes No No No Anonymity (taken from: Jones et al. 1995 ) Yes Partly (the group is in-person, the voting is anonymous) No No Structured voting Yes Yes Yes (degree of formality depends on sample size) No Structured interaction (taken from: McMillan et al . 2016 ) High High Intermediate Intermediate Immediate results available (taken from: Potter et al . 2004 ) Partly Yes Partly No Co-ownership (experts generally being co-authors of the publication or not) No No Yes Yes Sample size (number of experts taking part in the process) 15-100+ 2-14 30-3000+ 30-50+ Table 2. Four examples of expert consensus methods in the natural, social, and life sciences described by type and according to the six elements. Method Delphi study Nominal Group Technique Consensus conference Glaser’s state of the art method Publication ( Brinkman et al. 2018 ) ( Paraskevas 2013 ) ( Curigliano et al. 2017 ) ( Hodgkin et al. 1975 ) Field Pharmacology and higher education Safety research (Anti-terrorism and hospitality management) Medicine (Oncology, breast cancer) Medicine (Pulmonary medicine, obstructive airway diseases) (i) Process leader or steering committee The steering committee consisted of a clinical pharmacologist, a junior doctor, an internist-infectious disease specialist, and a senior lecturer in prescribing The process leader was a senior lecturer in Strategic Risk Management The consensus writing committee consisted of 2 chairpersons and 9 researchers and clinicians The process leader was a behavioural psychologist, the steering committee consisted of 11 physician researcher-practitioners (ii) Experts The panel consisted of 129 experts from 27 European countries The panel consisted of 19 hotel security experts as well as members of an international working group on terrorism from 6 European countries The panel consisted of 42 medical specialists from Europe, North America, Asia and North Africa The (unstructured) panel consisted of 120 persons who were identified based on their attendance of previous chronic obstructive airway diseases (COPD) key conferences as well as their publications on the issue, or nomination from the steering committee. (iii) Issue How to modernise and harmonise clinical pharmacology and therapeutics (CPT) education at a European level? How to best prevent terrorist abusing your hotel, and how to minimise damage? What is the best way to treat breast cancer in the face of controversies where data from clinical trials is lacking? What is the status quo of COPD? (iv) Rules of engagemen t Panellists were asked to rate each outcome (1=very unimportant, 2=unimportant, 3=neutral, 4=important, 5=very important), indicating their agreement that the outcome should be included in the undergraduate CPT curriculum and should be expected of European graduates in order that they can prescribe safely and effectively. If panellists awarded an outcome a score of 4 or 5, they were asked to indicate whether that outcome should be acquired during the preclinical (i.e., bachelor’s degree) or clinical (i.e., master’s degree, clerkships) years of the curriculum, or both Panellists had to list various security measures for relevance, and were then asked to share their ideas during the workshops, after which they were asked to vote on each proposed measure Panellists were asked to vote (yes/no/abstain) on a total of 201 different questions related to escalating and de-escalating treatment across subtypes of early-stage breast cancer and treatment types. For some questions, panellists were asked to vote on treatment-specific matters, such as minimal acceptable margins in breast surgery or when the best time is to take a specific biopsy. In cases of a clear difference (88.2% voting “Yes”, 5,9% “No”, and 5,9% “Abstain), the question was transformed into a recommended strategy. For votes close to 50/50, it was reported that the panel was split on the subject and no further recommendations were made Panellists were asked to read and comment on the current draft and send back their suggested changes (v) Output Key learning outcomes for undergraduate CPT education in Europe A six-step baseline anti-terrorism strategy and a series of measures and actions to address the threat of terrorist attacks in hotels Articulation of strategies for early-stage breast cancer treatment interventions, including guidance on which patients should receive adjuvant chemotherapy, considering also less costly alternatives for countries with limited access to therapeutic and diagnostic resources Systematised treatment guidelines that can be used to differentiate between different chronic obstructive airway diseases, as well as how the therapeutic program should look, plus the therapeutic modalities, tools, and where application of rehabilitation medicine is useful (vi) Users Educators, students, health care settings, pharmacologists, patients Hotel management, anti-terrorism strategists, hotel employees Clinicians, patients, students, researchers, insurance representatives Clinicians, patients, students, researchers, insurance representatives Type of purpose(s) (not mutually exclusive) Policy development Guidelines Policy development Guidelines Classifications Policy development Guidelines Classifications Policy development Current state of knowledge Table 3. Four examples of expert consensus methods in humanities related research described by type. Expert consensus method Delphi study Nominal group Mixed and tailor-made method that somewhat resembles a consensus conference Mixed and tailor-made method that somewhat resembles a Glaser’s state of the art method Publication ( Lechner et al. 2023 ) ( Dragouni and Lekakis 2023 ) ( Rulkens et al. 2022-2023 ) ( Standing Committee for the Museum Definition – ICOM Define Final Report (2020-2022) 2022 ) and ( ICOM Webinar Defining the Museum in Times of Change: A Way Forward 2020 ) Field Philosophy (Epistemology) Conservation (Heritage management) Visual Arts (Art History) Museology (i) Process leaders or steering committee Authors of the paper Authors of the paper Multidisciplinary group of researchers International Council of Museums (ICOM) (ii) Experts 46 Researchers on epistemic responsibilities and/or university assessment instruments and/or (former) administrators with practical knowledge and hands-on experience in leading a university 32 Representatives of local government; academic researchers and members of cultural associations, artists, and residents 4 Rembrandt scholars with various kinds of expertise in art history and technical research into paintings 126 National Committees of museum professionals that are members of ICOM, (precise number of experts unknown) (iii) Issue What are the core epistemic responsibilities of universities? What are solutions/actions and/or tactics to protect the rural monuments of the island of Naxos? Are two paintings painted by Rembrandt, partially painted by Rembrandt or not painted by Rembrandt? What should count as an international definition of the museum? (iv) Rules of engagemen t (consensus threshold) Three-round online survey, alternating between closed questions to gain consensus, and open questions to let experts motivate their answers. Panellists rated their agreement to consensus questions on a 5-point Likert scale (strongly agree – somewhat agree – neither agree nor disagree – somewhat disagree – strongly disagree). Consensus defined as 67% (i.e., two-thirds) of panellists (strongly) agreeing to a question. If no consensus was reached, the steering committee made a final decision Panellists were divided in 5 groups which each met for 2 hours. Five main stages were completed: (i) presentation of problem-issue (ii) ideas sharing and generation and recording, (iii) discussion and listing, (iv) voting and (v) vote counting and conclusions. Consensus was establishes through majority and the qualitative generated data was thematically analysed Panellists individually assessed the consensus questions and filled in a form in which they could address in percentages the extend in which they were sure about their answers. Subsequently, they were each interviewed to ask for motivations. This was followed by a focus group discussion, led by a chair. The experts then filled in a second form to examine if answers changed due to the group discussion. This was followed by an interview to ask about their motivations. The method ended with a joint debriefing. Qualitative data was thematically analysed. Consensus was defined as 75%, i.e. when three out of four experts (three-quarters) agreed to a question Committees each shared max 100 keywords for the definition, after discussion each committee submitted max 20 keywords for the definition. Then the keywords were analysed externally and the results were published. Committees reviewed the published results and submitted comments. They could add up to 3 new keywords and comment on keywords they could not accept. A final list of keywords was compiled. Based on this, a small group of special define members submitted 14 proposals for the definition. 5 Were selected by them for publication. Committees identified their preferred proposal. This final proposal was published and put to vote during an international extraordinary general assembly of ICOM members (v) Output 6 core epistemic responsibilities of universities 22 ideas reflecting current needs and proposing responding actions to safeguarding the future of the rural heritage of the island of Naxos Full consensus amongst panellists about the attribution of two 17 th century paintings An internationally supported definition of the museum (vi) Users Higher education policymakers and university leadership Heritage experts, managers of rural resources, local communities Specialists of 17 th century Dutch paintings, the museums that own these paintings, museum visitors Museums, museum professionals, policymakers and governments world-wide Type of purpose(s) (not mutually exclusive) Policy development Policy development Guidelines Current state of knowledge Taxonomies/definitions Table 4. Examples of purposes for which expert consensus methods can be employed, partially based on Gattrell et al. (2024, Table 2). Purpose: How consensus helps, incorporating multiple perspectives and/or expertises: Guidelines Translating evidence into recommendations Classifications Definition of markers, signs or thresholds Research priorities Defining and ranking priorities in the context of limited resources Policy development Analysing and interpreting evidence to inform policies Taxonomies/definitions Defining standard taxonomies and definitions Current state of knowledge Defining the status quo of consensus or dissensus on a research topic 5. Expert consensus methods in the humanities: limitations and potential In the preceding sections, we examined the elements and dynamics of expert consensus methods, along with examples and possibilities for intended objectives. In this section, we delve into the question why formal approaches to consensus in the humanities remain scarce, despite the significant factor of consensus and dissensus in knowledge production across research cultures. Aside from limitations, we also address what the potential added value of applying consensus methods to humanities-related issues could be. First of all, practical limitations likely play a role. Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared ( Leising et al. 2024 ). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasizing consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer 2022; Asendorpf and Gebauer 2022 ; Beck et al. 2022 ; Denissen and Sijtsma 2022 ; Hilbig et al. 2022 ). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Scholars may also disagree due to the role of research paradigms or schools within the humanities, as researchers’ positionality can shape their interpretations. In addition to that, humanities research can be concerned with hermeneutics, i.e., issues that have to do with understanding the values and meanings of texts and other objects. And many humanities fields such as ethics, epistemology, and metaphysics, rely on a-priori methods like thought experiments, reasoning on the basis of intuitions and principles, and formal logic rather than empirical data. This allows for multiple interpretations that can all to some extent be warranted by the evidence base, and may not readily align into a singular consensus. These factors suggest that the complexity and interpretative nature of humanities research might render expert consensus methods to some extent less relevant. Nevertheless, we believe that there are compelling reasons for considering expert consensus methods in humanities research. First of all, opposing the above concerns, one of the important purposes of expert consensus methods is to bring various viewpoints, nuances, and interpretations to the surface, so that they can be discussed. Depending on the design and purpose of the consensus method, it could reveal levels of consensus and/or dissensus on an issue, within or across schools of thought or paradigms. They might even have the potential to articulate both divergences and common grounds regarding an issue, and by extension increase mutual understanding. The application of such a method may therefore stimulate epistemic progress in itself. Moreover, a carefully designed protocol and curated group of experts can foster a fair practice and ensure the consultation of diverse viewpoints. Expert consensus methods provide opportunities to mitigate biases and their ramifications by this cautious selection of experts, but also by blinding certain parts of the process in relation to anonymity of participating experts, for example. Secondly, expert consensus methods can be helpful in getting a firmer grip on the status quo or state of the art on a particular issue in a field (e.g. Hodgkin et al. (1975) mentioned in Table 2 ; Rulkens et al. 2022-2023 , mentioned in Table 3 ). Of course, scholars often already have an idea about that, and such an informed opinion is valuable in itself. However, it can be complemented by a more structured approach, particularly when the number of scholars involved is large. This approach can additionally provide guidance for future research by pointing out the gaps in the current evidence base. And when applied on a recurrent basis, it might even have the potential to gain insight in how views evolve over time. It is important to note that consensus is not a final endpoint; rather, it can be continually re-evaluated in light of new insights and reasonable doubts regarding its validity. Third, an advantage of expert consensus methods is that they enhance the transparency and replicability of research because of their well-documented and systemised approach (for replication in the humanities, see Peels and Bouter (2018) ). It can therefore be instrumental to epistemic progress beyond the consensus reached at a moment in time. Fourth, application or implementation of its outcomes is often easier after the process of a consensus method is completed (e.g. Standing Committee for the Museum Definition (2020-2022) (2022) , mentioned in Table 3 ). And finally, expert consensus methods can aid public understanding by clarifying experts perspectives on specific issues ( Stekelenburg et al. (2022) thereby enabling the public to make informed decisions ( Ordway 2021 ). 6. Conclusion In this article, we have presented a framework comprising six key elements for the application of expert consensus methods, alongside examples that illustrate different types of these methods and their purposes in various research contexts. This serves as a preliminary exploration of the potential value these methods may offer for the humanities. Despite inevitable practical limitations, expert consensus methods can accelerate epistemic process, enhance transparency, increase replicability, stimulate diversity, and encourage fair processes in humanities research and the application of its findings. There seem to be humanities-related issues that may lend themselves better to structured expert consensus methods than their subject matter and research methods might suggest. However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritization of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Author contributions Charlotte C.S. Rulkens and Rik Peels delivered the overall concept and text for this article, which Lidwine B. Mokkink, Tamarinde Haven and Lex Bouter critically read and revised. In addition to that, Tamarinde Haven and Lidwine B. Mokkink were responsible for Tables 1 and 2 . Data availability Data sharing is not applicable to this article as no datasets were generated or analysed during the current study. Acknowledgements The authors are grateful to Hans Ket and Michèle Meijer, information specialists at the library of the Vrije Universiteit Amsterdam, for sharing their advice and knowledge in searching for examples of expert consensus methods in both the natural, social, and life sciences and the humanities. References Asendorpf JB, Gebauer JE: “Good Intentions – Unfortunate Side Effects” In: Open peer commentaries to Leising et al. , Ten steps toward a better personality science: How quality may be rewarded more in research evaluation. Personality Science. 2022; 3 . Publisher Full Text Beck ED, Workman CI, Christensen AP: “CRediT Where Credit Is Due: A Comment on Leising et al.” In: Open peer commentaries to Leising et al. , Ten steps toward a better personality science: How quality may be rewarded more in research evaluation. Personality Science. 2022; 3 . Publisher Full Text Black N, Murphy M, Lamping D, et al. : Consensus Development Methods: A Review of Best Practice in Creating Clinical Guidelines. J. Health Serv. Res. Policy. 1999; 4 (4): 236–248. Publisher Full Text Blazey P, Crossley KM, Ardern CL, et al. : It Is Time for Consensus on ‘Consensus Statements’. Br. J. Sports Med. 2022; 56 (6): 306–307. Publisher Full Text Brinkman DJ, Tichelaar J, Mokkink LB, et al. : Key Learning Outcomes for Clinical Pharmacology and Therapeutics Education in Europe: A Modified Delphi Study. Clin. Pharmacol. Ther. 2018; 104 (2): 317–325. PubMed Abstract | Publisher Full Text | Free Full Text Curigliano G, Burstein HJ, Winer EP, et al. : De-Escalating and Escalating Treatments for Early-Stage Breast Cancer: The St. Gallen International Expert Consensus Conference on the Primary Therapy of Early Breast Cancer 2017. Ann. Oncol. 2017; 28 (8): 1700–1712. PubMed Abstract | Publisher Full Text | Free Full Text Dalkey N, Helmer O: An experimental application of the Delphi method to the use of experts. Manag. Sci. 1963; 9 (3):458–467. Publisher Full Text Reference Source Daly C: Introduction to Philosophical Methods. Peterborough: Broadview; 2010. Delbecq AL, Van de Ven AH, Gustafson DH: Group Techniques for Program Planning: A Guide to Nominal Group and Delphi Processes. Glenview, Illinois: Scott, Foresman and Company; 1975. Denissen JJA, Sijtsma K: A New Academic Incentive Structure: Does It Fit the Psychology of Human Motives? 2022. Diamond IR, Grant RC, Feldman BM, et al. : Defining Consensus: A Systematic Review Recommends Methodologic Criteria for Reporting of Delphi Studies. J. Clin. Epidemiol. 2014; 67 (4): 401–409. PubMed Abstract | Publisher Full Text Dragouni M, Lekakis S: Co-Creating the Future of Heritage in-the-Making: Empirical Evidence from Community Deliberation at Naxos Island, Greece. Int. J. Herit. Stud. 2023; 29 (4): 294–313. Publisher Full Text Glaser EM: Using Behavioral Science Strategies for Defining the State-of-the-Art. J. Appl. Behav. Sci. 1980; 16 (1): 79–92. Publisher Full Text Goodman CM: The Delphi Technique: A Critique. J. Adv. Nurs. 1987; 12 (6): 729–734. PubMed Abstract | Publisher Full Text von der Gracht HA : Consensus Measurement in Delphi Studies: Review and Implications for Future Quality Assurance. Technol. Forecast. Soc. Chang. 2012; 79 (8): 1525–1536. Publisher Full Text Grimm S, Peels R, van Woudenberg R : A Philosophy of the Humanities. New York: Oxford University Press; forthcoming. Habermas J: Between Facts and Norms: Contributions to a Discourse Theory of Law and Democracy. Rehg W, translator. Cambridge, Massachusetts: MIT Press; 1996. Hilbig BE, Moshagen M, Zettler I: “Consensus Is not the Cure; It's Part of the Disease” In: Open peer commentaries to Leising et al. , Ten steps toward a better personality science: How quality may be rewarded more in research evaluation. Personality Science. 2022; 3 Publisher Full Text Hodgkin JE, Balchum OJ, Kass I, et al. : Chronic Obstructive Airway Diseases. Current Concepts in Diagnosis and Comprehensive Care. J. Am. Med. Assoc. 1975; 232 (12): 1243–1260. Publisher Full Text ICOM Webinar Defining the Museum in Times of Change: A Way Forward: 2020. Accessed 11 October 2023. Reference Source Jones J, Hunter D: Consensus Methods for Medical and Health Services Research. BMJ. 1995; 311 (7001): 376–380. PubMed Abstract | Publisher Full Text | Free Full Text Kuhn T: The Structure of Scientific Revolutions. Chicago: University of Chicago Press; 1962. Lechner IM, Mokkink LB, de Ridder GJ , et al. : The Core Epistemic Responsibilities of Universities: Results from a Delphi Study. Account. Res. 2023; 1–21. PubMed Abstract | Publisher Full Text Leising D, Liesefeld HR, Buecker S, et al. : A Tentative Roadmap for Consensus-Building Processes. 2024, May 13. Publisher Full Text Linstone H, Turoff M: The Delphi Method: Techniques and Applications. Reading: Addison-Wesley Publishing Company; 1975. Longino H: The Social Dimensions of Scientific Knowledge. The Stanford Encyclopedia of Philosophy. 2019. (Summer 2019 Edition), Edward N. Zalta (ed.). Reference Source McMillan SS, King M, Tully MP: How to Use the Nominal Group and Delphi Techniques. Int. J. Clin. Pharm. 2016; 38 (3): 655–662. PubMed Abstract | Publisher Full Text | Free Full Text O’Connor C, Goldberg S, Goldman A: Social Epistemology. The Stanford Encyclopedia of Philosophy. 2024. (Forthcoming, Summer 2024 Edition), Edward N. Zalta and Uri Nodelman (eds.). Ordway D-M: Covering Scientific Consensus: What to Avoid and How to Get It Right. The Journalist’s Resource. 2021. Online. (2021). Reference Source Oreskes N: Why Trust Science? Princeton, New Jersey: Princeton University Press; 2021. Paraskevas A: Aligning Strategy to Threat: A Baseline Anti-Terrorism Strategy for Hotels. Int. J. Contemp. Hosp. Manag. 2013; 25 (1): 140–162. Publisher Full Text Peels R: Replicability and Replication in the Humanities. Res. Integr. Peer Rev. 2019; 4 (1): 2. Publisher Full Text Peels R: The Many Roads to Progress in Philosophy and Theology. Philosophy in the Age of Science? Inquiries into Philosophical Progress, Method, and Societal Relevance. Hermann J, Hopster J, Kalf W, et al. , editors. London: Rowman and Littlefield; 2020; 15–34. Peels R, Bouter L: The Possibility and Desirability of Replication in the Humanities. Palgrave Commun. 2018; 4 (1): 95. Publisher Full Text Potter M, Gordon S, Hamer P: The Nominal Group Technique: A Useful Consensus Methodology in Physiotherapy Research. N. Z. J. Physiother. 2004; 32 (3): 126. Powell C: The Delphi Technique: Myths and Realities. J. Adv. Nurs. 2003; 41 (4): 376–382. PubMed Abstract | Publisher Full Text Rulkens CCS, Peels R, Bouter L, et al. : Replicating a Rembrandt Study. Preregistration; 2022-2023. Reference Source Standing Committee for the Museum Definition – ICOM Define Final Report (2020-2022): 2022. Accessed 21 June 2023. Reference Source Stocking B, Jennett B, Spiby J: Criteria for Change: The History and Impact of Consensus Development Conferences in the UK. London: King’s Fund Centre; 1991. van Stekelenburg A , Schaap G, Veling H, et al. : Scientific-Consensus Communication About Contested Science: A Preregistered Meta-Analysis. Psychol. Sci. 2022; 33 (12): 1989–2008. PubMed Abstract | Publisher Full Text Footnotes 1 We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and subfields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorizations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al. ( forthcoming ) . Comments on this article Comments (0) Version 2 VERSION 2 PUBLISHED 28 Jun 2024 ADD YOUR COMMENT Comment Author details Author details 1 Department of Philosophy, Vrije Universiteit Amsterdam, Amsterdam, North Holland, The Netherlands 2 Faculty of Religion and Theology and Faculty of Humanities, Vrije Universiteit Amsterdam, Amsterdam, North Holland, The Netherlands 3 African Centre for Epistemology and Philosophy of Science, University of Johannesburg, Auckland Park, Gauteng, South Africa 4 Department of Methodology, Amsterdam Public Health research institute, Amsterdam, The Netherlands 5 Department of Epidemiology and Data Science, Amsterdam University Medical Centres, Duivendrecht, North Holland, The Netherlands 6 Department of Methodology and Statistics, Tilburg School of Social and Behavioural Sciences, Tilburg University, Tilburg, North Brabant, The Netherlands Charlotte C.S. Rulkens Roles: Conceptualization, Project Administration, Writing – Original Draft Preparation, Writing – Review & Editing Rik Peels Roles: Conceptualization, Funding Acquisition, Supervision, Writing – Original Draft Preparation, Writing – Review & Editing Lidwine B. Mokkink Roles: Conceptualization, Writing – Review & Editing Tamarinde Haven Roles: Conceptualization, Writing – Review & Editing Lex Bouter Roles: Conceptualization, Funding Acquisition, Supervision, Writing – Review & Editing Competing interests No competing interests were disclosed. Grant information Work on this article by Charlotte Rulkens, Lex Bouter, and Rik Peels was made possible through the generous support of a grant on Epistemic Progress in the University (grant number 0436) from the Templeton World Charity Foundation The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript. Article Versions (2) version 2 Revised Published: 16 Dec 2024, 13:710 https://doi.org/10.12688/f1000research.148726.2 version 1 Published: 28 Jun 2024, 13:710 https://doi.org/10.12688/f1000research.148726.1 Copyright © 2024 Rulkens CCS et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. Download Export To Sciwheel Bibtex EndNote ProCite Ref. Manager (RIS) Sente metrics Views Downloads F1000Research - - PubMed Central info_outline Data from PMC are received and updated monthly. - - Citations open_in_new 0 open_in_new 0 open_in_new SEE MORE DETAILS CITE how to cite this article Rulkens CCS, Peels R, Mokkink LB et al. Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.12688/f1000research.148726.2 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS track receive updates on this article Track an article to receive email alerts on any updates to this article. TRACK THIS ARTICLE Share Open Peer Review Current Reviewer Status: ? Key to Reviewer Statuses VIEW HIDE Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Version 2 VERSION 2 PUBLISHED 16 Dec 2024 Revised Views 0 Cite How to cite this report: Grimm SR. Reviewer Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.174565.r355056 ) The direct URL for this report is: https://f1000research.com/articles/13-710/v2#referee-response-355056 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 22 Jan 2025 Stephen R Grimm , Fordham University, Bronx, USA Approved VIEWS 0 https://doi.org/10.5256/f1000research.174565.r355056 This is a very nice, clear overview of the nature and potential importance of consensus methods in the humanities. I especially appreciate the cautions at the end, namely that "applying a consensus method is a costly endeavour in terms of ... Continue reading READ ALL This is a very nice, clear overview of the nature and potential importance of consensus methods in the humanities. I especially appreciate the cautions at the end, namely that "applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for." I too wondered this throughout reading the paper. But, that said, I can also see the benefits of pursuing these methods in the humanities. Is the rationale for developing the new method (or application) clearly explained? Yes Is the description of the method technically sound? Yes Are sufficient details provided to allow replication of the method development and its use by others? Yes If any results are presented, are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions about the method and its performance adequately supported by the findings presented in the article? Yes Competing Interests: No competing interests were disclosed. Reviewer Expertise: Philosophy, epistemology I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Grimm SR. Reviewer Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.174565.r355056 ) The direct URL for this report is: https://f1000research.com/articles/13-710/v2#referee-response-355056 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Reale E. Reviewer Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.174565.r349191 ) The direct URL for this report is: https://f1000research.com/articles/13-710/v2#referee-response-349191 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 09 Jan 2025 Emanuela Reale , CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.174565.r349191 I have read the revised version of the paper and the changes that have been applied. Some of my suggestions have been addressed and the manuscript in the current form is clearer and more consistent than before. However, the manuscript ... Continue reading READ ALL I have read the revised version of the paper and the changes that have been applied. Some of my suggestions have been addressed and the manuscript in the current form is clearer and more consistent than before. However, the manuscript still does not have a clear research question and therefore it remains largely descriptive and the scientific contribution it provides does not emerge. Problems and limitations with the applications of the consensus methods in the Humanities are not discussed with reference to the specific cases that have been presented in tables 3-4 and with empirical evidence on the cases themselves. Moreover, some limitations are not exclusive of Humanities (e.g. time and resources; role of research paradigms bringing to divergent views) and the specific features of Humanities would need a deeper analysis. Competing Interests: No competing interests were disclosed. Reviewer Expertise: Higher education and research policy I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Reale E. Reviewer Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.174565.r349191 ) The direct URL for this report is: https://f1000research.com/articles/13-710/v2#referee-response-349191 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Leising D. Reviewer Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.174565.r349190 ) The direct URL for this report is: https://f1000research.com/articles/13-710/v2#referee-response-349190 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 23 Dec 2024 Daniel Leising , Technische Universitat Dresden, Dresden, Saxony, Germany Approved VIEWS 0 https://doi.org/10.5256/f1000research.174565.r349190 I found that the authors responded to my comments in a careful and throughtful manner, ... Continue reading READ ALL I found that the authors responded to my comments in a careful and throughtful manner, and that the manuscript has improved. I have no further comments or suggestions for changes. Competing Interests: No competing interests were disclosed. Reviewer Expertise: Personality / Social Psychology, Meta-Science I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Leising D. Reviewer Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.174565.r349190 ) The direct URL for this report is: https://f1000research.com/articles/13-710/v2#referee-response-349190 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Version 1 VERSION 1 PUBLISHED 28 Jun 2024 Views 0 Cite How to cite this report: Leising D. Reviewer Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.163073.r309049 ) The direct URL for this report is: https://f1000research.com/articles/13-710/v1#referee-response-309049 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 19 Sep 2024 Daniel Leising , Technische Universitat Dresden, Dresden, Saxony, Germany Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.163073.r309049 I strongly agree that more systematic consensus-building is needed in many fields of science, including the Humanities and Psychology (my own field) and I find little that I would disagree with in the manuscript as is. However, ... Continue reading READ ALL I strongly agree that more systematic consensus-building is needed in many fields of science, including the Humanities and Psychology (my own field) and I find little that I would disagree with in the manuscript as is. However, I also think that the authors could do more than just presenting an overview of established consensus building methods and suggesting that researchers from the Humanities should give these methods a try. The issue is more complex than that. In order to improve on the depth, utility and potential impact of this paper, I would thus like to encourage the authors to think about and – if possible – share their views on a number of additional, intriguing, and sometimes thorny issues regarding consensus building. First, I would argue that the Humanities are actually not that “unique” in terms of their nature and approaches (page 11). To enable a proper discussion of this issue, the authors could explain their own view of what is so specific about the Humanities in a bit more detail. Here is my view: Interviews and text analyses are used in many different branches of science. Exploratory research is also conducted in many fields. The ways in which people give meaning to their experiences are also of interest across various branches of science. And even in many of the “harder” sciences, research is often “coloured by the backgrounds and positionality of the researchers” (page 8). So, much of what is common in research in the Humanities is actually shared with other fields. In my view, two features may actually distinguish “the Humanities” from other fields - but even these distinctions are only gradual in nature. One feature is the common practice of using everyday language as the primary scientific language. The other feature is the relative fuzziness of many concepts - which I would note is not necessarily a weakness but may often be unavoidable given the complexity of the phenomena of interest (and also given the first feature just mentioned). Do the authors agree? Second, I would like to encourage the authors to say a bit more about possible reasons for why so little consensus building is taking place in many fields of science, despite it being widely viewed as desirable. In my view, there are a couple of reasons, most of which are rather extraneous to an idealized image of science (as a collaborative and cumulative effort at building a reliable knowledge base for the good of mankind). For example, consensus building is likely to reduce the publication output of a field, which would be at odds with the interests of commercial publishers. Also, individual researchers have an interest in “making a name for themselves" based on their specific expertise or views regarding certain topics. Consensus building makes it more difficult to maintain a highly distinct profile and thus reduce visibility. This may be risky in a science system that is as competitive as the one we choose to entertain. And then there may be a need to “save face”: The more of your career you have dedicated to certain ideas (e.g., your own theory of some phenomenon), the harder it may be to let those ideas go - even if the evidence against them is overwhelming. There is a reason why Max Planck once stated that scientific progress happens “one funeral at a time”. At least a brief nod to the relevance of such more political factors would be nice - if the authors share my conviction that they are relevant. Who has an actual interest in achieving consensus? Who has an interest in avoiding the emergence of consensus? And what are the external incentives for and agaisnt this type of work? Third, I would also appreciate if the authors could talk some more about “bad consensus” – views that are held by many, despite their being wrong. With a particular eye to the Humanities – what criteria are there, apart from consensus, that may help us determine whether a view is “correct”? Or may truth in the Humanities *only* be defined in terms of relative consensus among researchers? In the harder sciences, the ability to *predict* the occurrence of certain events from a theory (and some data input) plays a crucial role as evidence of the theory's correctness. Are the Humanities able to produce testable predictions, too? Can there be consensus projects in the Humanities that actually test one theory against another? Fourth, on page 3 the authors state that “ Since the questions addressed in the humanities may leave room for multiple viewpoints that are equally warranted by the evidence base, it can be infeasible or not desirable to get a complete or almost complete consensus.” To this I would respond: what are the conditions under which working toward consensus would not be feasible, or desirable? I agree that this may be the case – but when is it the case, and when is it not the case? I personally hold the somewhat skeptical view that many researchers will shy away from working toward consensus even if doing so *would* be both feasible and desirable (see my previous point). Such a “live and let live” attitude may often be relatively comfortable to have (e.g., to avoid conflict). It may, however, be regarded as unethical, because it will foster the preservation of an unnecessary level of ambiguity. Minor points Fifth, the authors state that a steering committee would typically be comprised of the people who initiated the process and who will ultimately author the consensus document. Could there not be cases in which the process should better be overseen by an independent third party (e.g., a moderator)? Especially when issues are highly contentious, for example? And should not all the experts who worked on the consensus be authors on the consensus document? Sixth, something is not quite right on page 4, text lines 16-19. Seventh, I would say that evidence will never “speak for itself” (page 4) - otherwise papers in certain scientific fields would not need any Discussion sections. Together with a variety of colleagues, I have written about several of the above-mentioned issues. If the authors are interested in this, here are a few key references: I referenced some of our own publications that I consider to be related in terms of content, to let the authors know about their existence. The authors are of course entirely free to decide whether to look into them, or not. Leising, D.et.al; 2023 (Ref 1) Leising, D.et.al; 2024 (Ref 2) Leising, D.et.al; 2022 (Ref 3) Leising, D.et.al; 2022 (Ref 4) Is the rationale for developing the new method (or application) clearly explained? Yes Is the description of the method technically sound? Yes Are sufficient details provided to allow replication of the method development and its use by others? Yes If any results are presented, are all the source data underlying the results available to ensure full reproducibility? No source data required Are the conclusions about the method and its performance adequately supported by the findings presented in the article? Yes References 1. Leising D, Grenke O, Cramer M: Visual Argument Structure Tool (VAST) Version 1.0. Meta-Psychology . 2023; 7 . Publisher Full Text 2. Leising D, Liesefeld H, Buecker S, Glöckner A, et al.: A Tentative Roadmap for Consensus-Building Processes. 2024. Publisher Full Text 3. Leising D, Thielmann I, Glöckner A, Gärtner A, et al.: Ten steps toward a better personality science – how quality may be rewarded more in research evaluation. Personality Science . 2022; 3 . Publisher Full Text 4. Leising D, Thielmann I, Glöckner A, Gärtner A, et al.: Ten steps toward a better personality science – a rejoinder to the comments. Personality Science . 2022; 3 . Publisher Full Text Competing Interests: No competing interests were disclosed. Reviewer Expertise: Personality / Social Psychology, Meta-Science I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Leising D. Reviewer Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.163073.r309049 ) The direct URL for this report is: https://f1000research.com/articles/13-710/v1#referee-response-309049 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 16 Dec 2024 Charlotte Rulkens , Department of Philosophy, Vrije Universiteit Amsterdam, Amsterdam, The Netherlands 16 Dec 2024 Author Response Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the ... Continue reading Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the time and effort you have put into evaluating our work and for providing insightful comments and valuable suggestions. We have carefully considered each of them and have revised the manuscript accordingly. Below, we provide detailed responses to each of the comments. We believe that the revisions have improved the manuscript, and we hope that our responses meet any remaining concerns. Response to reviewer 1: Emanuela Reale, CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy Reviewer 1, Comment 1: First, the aim of the paper is clear, but the research question(s) related to that aim are missing. This is an important limitation, since the manuscript reports an exploratory effort and therefore a delimitation of the research problem that you want to investigate is very useful to delineate the perimeter of the work. Response: We acknowledge the concern raised and have addressed this by rewriting the introduction section and by incorporating a clearer description of our study goal. We now make it more clear that our goal was primarily didactic, in the sense that we sought to explore the usefulness of consensus methods utilised in the natural, social, and life sciences for the humanities (see section 1, lines 59-83): An important way to make progress in knowledge and understanding is to gradually, via the exchange of evidence, expertise, and critical discussion, reach (more) consensus on a topic. However, whether there is consensus on a certain matter at a moment in time can sometimes remain unknown, be unclear, or remain implicit. Expert consensus methods can lift off this inconclusiveness by intentionally and formally pursuing or assessing consensus. These methods are regularly used in natural, social, and life sciences. Despite the significant role of consensus in the development of (academic) knowledge and understanding (e.g. Kuhn, 1962; Habermas, 1996; thus also Peels, 2020; Grimm et al., forthcoming ), formal expert consensus methods have hardly been explored within in the context of the humanities. [1] Other kinds of valuable expert meetings do take place in various disciplines in the humanities. They usually do not seek to reach a consensus, but to explore a topic in-depth by bringing the relevant experts to the table. Moreover, expert consensus methods described in this article differ from the expert meetings that already take place in the humanities by the former’s procedural and formalised character. In this paper, we aim to highlight to scholars, particularly those in the humanities, what is possible in terms of a systematic approach to academic consensus. In doing so, we aim to initiate exploration of and discussion about the potential of applying such an approach. What would be the added value of applying expert consensus meetings to humanities-related issues? To achieve this, we first explicate our method (§2), before we provide a framework for the application of expert consensus methods by introducing six key elements (§3). Subsequently, we present common types of expert consensus methods utilized in the natural, social, and life sciences. We highlight their and proven value across various types of research through relevant examples and their respective objectives (§4). We then consider both the potential benefits and possible limitations to applying these methods within humanistic research (§5), and conclude with suggestions for future research (§6). Reviewer 1, Comment 2: Second, the section on Method is poor. The composition of the interdisciplinary team (and the reasons for using this team) is not clearly described, the brainstorm session is not properly defined and illustrated, the ‘informal’ literature review is something new for me (what ‘informal’ means?) and it is not acceptable for a research paper to quote two methods used (‘philosophical methods of conceptual analysis’ and ‘conceptual engineering’) without explaining how they have been used for the purposes of the paper but just putting two references. Also, the process of writing and reviewing the manuscript should be better described and justified. Response: Our article, as we now emphasise more, is meant as an exploratory comment and a didactic contribution to inspire scholars in the humanities to see whether consensus methods may be valuable within their own discipline or sub discipline. It is not an instance of systematic empirical research, which would have required a much more detailed methodological section. But we certainly we agree that the method section can be improved in various ways. Therefore, we extended the section by including a specification of the backgrounds of the involved authors and specified the process of our collaboration as follows (see section 2, lines 85-94): To explore the potential of applying expert consensus methods in the humanities, we composed an interdisciplinary team of both researchers from the sciences using consensus methods in practice (LM: epidemiology; TH: qualitative research methodology; LB: epidemiology and data science, research integrity), and researchers familiar with the domain of the humanities and epistemology (CR: art history, museology; RP: epistemology and philosophy of science, religion and theology). In a first brainstorm session, knowledge on the topic was shared from both perspectives. This resulted in a list of consensus methods and examples used in the sciences, drafted by LM and TH. CR carried out a literature search in order to find examples thereof that were applied in her domain. A first draft was written and discussed in a second group session and culminated in a final version, which was edited and reviewed by all authors. Reviewer 1, Comment 3: Third, and more important, the paper refers always to ‘humanities.’ Humanities is a highly fragmented and differentiated field, where many and different epistemic communities work, interact, shape their own idea and uses of consensus method (if any) and where different instances of relationships with other researchers within the field and with neighboring fields are at place. The paper does not acknowledge this important characteristic of humanities (see footnote 1 where there is a list of disciplines included in the humanities that are considered in the paper). The same holds true for STEMs, which are considered as they are all adopting regularly expert consensus methods and there is no explanation or evidence about it. In sum, the paper ‘explores’ the expert consensus method without considering the set of ideas, values, theories, concepts and practices that pertain to the disciplines analyzed and their scientific work. Response: We acknowledge the concern raised and have taken this into account by adjusting a footnote 1 in section 1 (see above) and by revising section 4. Upon reflection we furthermore adjusted the ways to delineate the contexts in which expert consensus methods may be valuable, by focusing on one’s purpose in applying such a method (see section 4, lines 213-235): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 1, Comment 4: Fourth, sections 3 and 4 provide a clear summary of what expert consensus methods are and a deepen analysis of four types. Then the reader expects Section 5 discusses the results with respect to the exploration done. On the contrary, Section 5 lists of pros and cons related to the expert consensus method, and some examples of applications; the contents are completely unrelated with what the sections 3 and 4 have presented and the motivations for considering the expert consensus method an interesting development for humanities are unclear. Here the suggestion is to revise the sections trying to answer the research questions (when formulated) and highlight what we have learnt from the exploratory work, what is the contribution of the brainstorming to the exploration, what new avenues of research can be envisaged based on the results obtained, and what limitations/obstacles can the expert consensus model have in the humanities with respect to the other fields. Response: Thank you for pointing this out to us. Based on this valuable feedback, we have worked to better align section 5 with the preceding sections to create a more cohesive flow throughout the paper and have made more explicit what our line of reasoning is. However, because the paper is intended as preliminary exploration, part of our manuscript aims to identify the potential value of these methods for the humanities. Section 5 is dedicated to that purpose and we made that more explicit in the revision (see section 5, lines 253-315): In the preceding sections, we examined the elements and dynamics of expert consensus methods, along with examples and possibilities for intended objectives. In this section, we delve into the question why formal approaches to consensus in the humanities remain scarce, despite the significant factor of consensus and dissensus in knowledge production across research cultures. Aside from limitations, we also address what the potential added value of applying consensus methods to humanities-related issues could be. First of all, practical limitations likely play a role. Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Scholars may also disagree due to the role of research paradigms or schools within the humanities, as researchers’ positionality can shape their interpretations. In addition to that, humanities research can be concerned with hermeneutics, i.e., issues that have to do with understanding the values and meanings of texts and other objects. And many humanities fields such as ethics, epistemology, and metaphysics, rely on apriori methods like thought experiments, reasoning on the basis of intuitions and principles, and formal logic rather than empirical data. This allows for multiple interpretations that can all to some extent be warranted by the evidence base, and may not readily align into a singular consensus. These factors suggest that the complexity and interpretative nature of humanities research might render expert consensus methods to some extent less relevant. Nevertheless, we believe that there are compelling reasons for considering expert consensus methods in humanities research. First of all, opposing the above concerns, one of the important purposes of expert consensus methods is to bring various viewpoints, nuances, and interpretations to the surface, so that they can be discussed. Depending on the design and purpose of the consensus method, it could reveal levels of consensus and/or dissensus on an issue, within or across schools of thought or paradigms. They might even have the potential to articulate both divergences and common grounds regarding an issue, and by extension increase mutual understanding. The application of such a method may therefore stimulate epistemic progress in itself. Moreover, a carefully designed protocol and curated group of experts can foster a fair practice and ensure the consultation of diverse viewpoints. Expert consensus methods provide opportunities to mitigate biases and their ramifications by this cautious selection of experts, but also by blinding certain parts of the process in relation to anonymity of participating experts, for example. Secondly, expert consensus methods can be helpful in getting a firmer grip on the status quo or state of the art on a particular issue in a field (e.g. Hodgkin et al. (1975) mentioned in Table 2; Rulkens et al. 2022-2023, mentioned in Table 3). Of course, scholars often already have an idea about that, and such an informed opinion is valuable in itself. However, it can be complemented by a more structured approach, particularly when the number of scholars involved is large. This approach can additionally provide guidance for future research by pointing out the gaps in the current evidence base. And when applied on a recurrent basis, it might even have the potential to gain insight in how views evolve over time. It is important to note that consensus is not a final endpoint; rather, it can be continually re-evaluated in light of new insights and reasonable doubts regarding its validity. Third, an advantage of expert consensus methods is that they enhance the transparency and replicability of research because of their well-documented and systematised approach (for replication in the humanities, see Peels and Bouter (2018)). It can therefore be instrumental to epistemic progress beyond the consensus reached at a moment in time. Fourth, application or implementation of its outcomes is often easier after the process of a consensus method is completed (e.g. Standing Committee for the Museum Definition (2020-2022) (2022), mentioned in Table 3). And finally, expert consensus methods can aid public understanding by clarifying experts perspectives on specific issues (Stekelenburg et al., 2022) thereby enabling the public to make informed decisions (Ordway, 2021). In response to this comments, we furthermore extended suggestions for future research in our conclusion (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Response to reviewer 2: Daniel Leising, Technische Universitat Dresden, Dresden, Saxony, Germany Reviewer 2, Comment 1 : First, I would argue that the Humanities are actually not that “unique” in terms of their nature and approaches (page 11). To enable a proper discussion of this issue, the authors could explain their own view of what is so specific about the Humanities in a bit more detail. Here is my view: Interviews and text analyses are used in many different branches of science. Exploratory research is also conducted in many fields. The ways in which people give meaning to their experiences are also of interest across various branches of science. And even in many of the “harder” sciences, research is often “coloured by the backgrounds and positionality of the researchers” (page 8). So, much of what is common in research in the Humanities is actually shared with other fields. In my view, two features may actually distinguish “the Humanities” from other fields - but even these distinctions are only gradual in nature. One feature is the common practice of using everyday language as the primary scientific language. The other feature is the relative fuzziness of many concepts - which I would note is not necessarily a weakness but may often be unavoidable given the complexity of the phenomena of interest (and also given the first feature just mentioned). Do the authors agree? Response: Thank you for challenging us on this point. We agree that the borders between the general fields of the natural, social, and life sciences, and the humanities are fuzzy, when it comes to their objects, purposes, and methods. Whether there is something distinctive about the humanities – we believe that is compatible with fuzziness at the boarders – is a matter that we cannot possibly explore in this article (one of us does so in a book entitled A Philosophy of the Humanities that is forthcoming in 2025 with Oxford University Press, a volume co-authored by Stephen Grimm, Rik Peels, and René van Woudenberg). We do believe though that ordinary language use is not generally characteristic of the humanities (think of the highly technical language of metaphysics and epistemology), nor do we believe that insufficient clarity about core concepts is characteristic of the humanities. We agree though with the idea, also drawn to attention to by reviewer 1 in comment 3, that the humanities are a highly diversified field. We included that by adjusting footnote 1 in section 1: We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Additionally, to be able to demarcate between contexts in which the formal approach might be beneficial, we decided to put an emphasis on the purposes for which these methods are employed in other domains. This would be a way in which researchers with expertise in their fields could identify opportunities for a formal approach to consensus (see section 4, lines 213-235, also quoted above in response to comment 3 of reviewer 1): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 2, Comment 2: Second, I would like to encourage the authors to say a bit more about possible reasons for why so little consensus building is taking place in many fields of science, despite it being widely viewed as desirable. In my view, there are a couple of reasons, most of which are rather extraneous to an idealised image of science (as a collaborative and cumulative effort at building a reliable knowledge base for the good of mankind). For example, consensus building is likely to reduce the publication output of a field, which would be at odds with the interests of commercial publishers. Also, individual researchers have an interest in “making a name for themselves” based on their specific expertise or views regarding certain topics. Consensus building makes it more difficult to maintain a highly distinct profile and thus reduce visibility. This may be risky in a science system that is as competitive as the one we choose to entertain. And then there may be a need to “save face”: The more of your career you have dedicated to certain ideas (e.g., your own theory of some phenomenon), the harder it may be to let those ideas go - even if the evidence against them is overwhelming. There is a reason why Max Planck once stated that scientific progress happens “one funeral at a time”. At least a brief nod to the relevance of such more political factors would be nice - if the authors share my conviction that they are relevant. Who has an actual interest in achieving consensus? Who has an interest in avoiding the emergence of consensus? And what are the external incentives for and against this type of work ? Response: We think that these are valid points and the reviewers’ publications on the topic shed important light on this. We agree that external incentives may work against a formal approach to consensus as follows and have therefore slightly revised the manuscript on this point. We remain rather brief, though, as the exact explanation of why consensus methods are still relatively rarely used even in the sciences is an issue that is beyond the scope of our paper (see section 5, lines 259-264, also quoted in response to comment 4 of reviewer 1): Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Reviewer 2, Comment 3: Third, I would also appreciate if the authors could talk some more about “bad consensus” – views that are held by many, despite their being wrong. With a particular eye to the Humanities – what criteria are there, apart from consensus, that may help us determine whether a view is “correct”? Or may truth in the Humanities *only* be defined in terms of relative consensus among researchers? In the harder sciences, the ability to *predict* the occurrence of certain events from a theory (and some data input) plays a crucial role as evidence of the theory's correctness. Are the Humanities able to produce testable predictions, too? Can there be consensus projects in the Humanities that actually test one theory against another? Response: This is an interesting point, that would need a elaborative reflection in order to do justice to the complexity of this issue. It is important to note, that we do believe that consensus and truth are two different things, and we made this more clear in our revised paper. Based these suggestions we also incorporated that consensus can be reached based on the wrong reasons and (see section 5, lines 265-271, also quoted in response to comment 4 of reviewer 1): Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Reviewer 2, Comment 4: Fourth, on page 3 the authors state that “Since the questions addressed in the humanities may leave room for multiple viewpoints that are equally warranted by the evidence base, it can be infeasible or not desirable to get a complete or almost complete consensus.” To this I would respond: what are the conditions under which working toward consensus would not be feasible, or desirable? I agree that this may be the case – but when is it the case, and when is it not the case? I personally hold the somewhat sceptical view that many researchers will shy away from working toward consensus even if doing so *would* be both feasible and desirable (see my previous point). Such a “live and let live” attitude may often be relatively comfortable to have (e.g., to avoid conflict). It may, however, be regarded as unethical, because it will foster the preservation of an unnecessary level of ambiguity. Response: We appreciate this insightful suggestion. Our approach to expert consensus methods does not necessarily imply working towards consensus in all instances, but is also related to the assessment of the (lack of) consensus. We view measuring consensus as a means, not an objective in itself. Furthermore, as put forward in paragraph 3, the bar for ‘sufficient’ consensus is something consensus researchers can determine their selves. Applying a consensus methods is a costly endeavour in terms of time and means. There might be conditions under which the effort of applying such a method does not live up to the epistemic benefits it provides for. We furthermore believe that the feasibility and desirability of assessing consensus is an issue that merits further exploration and included this as follows (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Minor comments: Reviewer 2, Comment 5: Fifth, the authors state that a steering committee would typically be comprised of the people who initiated the process and who will ultimately author the consensus document. Could there not be cases in which the process should better be overseen by an independent third party (e.g., a moderator)? Especially when issues are highly contentious, for example? And should not all the experts who worked on the consensus be authors on the consensus document? Response: Thank you for this relevant point, this is certainly the case and we made this more clear in the revised manuscript by the following addition (see section 3, lines 193-198): Naturally, the six elements can be specified at by the steering committee. If they deem it valuable to include an independent moderator or to incorporate a member with a critical or divergent perspective—especially if the topic is highly controversial—this remains within their discretion. Additionally, it is up to them to decide who will be listed as co-authors, whether that includes only the steering committee, all participants, or a broader group representation such as the panellists as a whole, for instance. Reviewer 2, Comment 6: Sixth, something is not quite right on page 4, text lines 16-19. Response: Thank you for noticing, we have corrected this sentence as follows (see section 3, lines 136-140): Expert consensus methods are (i) applied by a process leader or steering committee , (ii) to reach consensus or assess the degree of consensus among a group of experts , (iii) about an issue , (iv) based on preset rules of engagement , (v) with the aim to deliver useful output , (vi) for future users . They are characterised by constructive and procedural usage of (dis)agreement and the different arguments that are provided therein. Reviewer 2, Comment 7: Seventh, I would say that evidence will never “speak for itself” (page 4) - otherwise papers in certain scientific fields would not need any Discussion sections. Response: Thank you for rightly pointing this out to us. We adjusted this as follows (see section 3, lines 132-134): This can be useful when the issue at hand is an important one and when evidence is lacking, it is inconsistent, or its interpretation is not obvious. As a result of our reactions to the comments, we also made adjustments to our abstract and conclusion, which are not all quoted in the above letter. We hope these revisions and clarifications meet the reviewers’ expectations. Please let us know if any further revisions are needed. Sincerely, Charlotte Rulkens MA Prof. dr. Rik Peels Dr. Lidwine Mokkink Dr. Tamarinde Haven Prof. em. dr. Lex Bouter [1] We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the time and effort you have put into evaluating our work and for providing insightful comments and valuable suggestions. We have carefully considered each of them and have revised the manuscript accordingly. Below, we provide detailed responses to each of the comments. We believe that the revisions have improved the manuscript, and we hope that our responses meet any remaining concerns. Response to reviewer 1: Emanuela Reale, CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy Reviewer 1, Comment 1: First, the aim of the paper is clear, but the research question(s) related to that aim are missing. This is an important limitation, since the manuscript reports an exploratory effort and therefore a delimitation of the research problem that you want to investigate is very useful to delineate the perimeter of the work. Response: We acknowledge the concern raised and have addressed this by rewriting the introduction section and by incorporating a clearer description of our study goal. We now make it more clear that our goal was primarily didactic, in the sense that we sought to explore the usefulness of consensus methods utilised in the natural, social, and life sciences for the humanities (see section 1, lines 59-83): An important way to make progress in knowledge and understanding is to gradually, via the exchange of evidence, expertise, and critical discussion, reach (more) consensus on a topic. However, whether there is consensus on a certain matter at a moment in time can sometimes remain unknown, be unclear, or remain implicit. Expert consensus methods can lift off this inconclusiveness by intentionally and formally pursuing or assessing consensus. These methods are regularly used in natural, social, and life sciences. Despite the significant role of consensus in the development of (academic) knowledge and understanding (e.g. Kuhn, 1962; Habermas, 1996; thus also Peels, 2020; Grimm et al., forthcoming ), formal expert consensus methods have hardly been explored within in the context of the humanities. [1] Other kinds of valuable expert meetings do take place in various disciplines in the humanities. They usually do not seek to reach a consensus, but to explore a topic in-depth by bringing the relevant experts to the table. Moreover, expert consensus methods described in this article differ from the expert meetings that already take place in the humanities by the former’s procedural and formalised character. In this paper, we aim to highlight to scholars, particularly those in the humanities, what is possible in terms of a systematic approach to academic consensus. In doing so, we aim to initiate exploration of and discussion about the potential of applying such an approach. What would be the added value of applying expert consensus meetings to humanities-related issues? To achieve this, we first explicate our method (§2), before we provide a framework for the application of expert consensus methods by introducing six key elements (§3). Subsequently, we present common types of expert consensus methods utilized in the natural, social, and life sciences. We highlight their and proven value across various types of research through relevant examples and their respective objectives (§4). We then consider both the potential benefits and possible limitations to applying these methods within humanistic research (§5), and conclude with suggestions for future research (§6). Reviewer 1, Comment 2: Second, the section on Method is poor. The composition of the interdisciplinary team (and the reasons for using this team) is not clearly described, the brainstorm session is not properly defined and illustrated, the ‘informal’ literature review is something new for me (what ‘informal’ means?) and it is not acceptable for a research paper to quote two methods used (‘philosophical methods of conceptual analysis’ and ‘conceptual engineering’) without explaining how they have been used for the purposes of the paper but just putting two references. Also, the process of writing and reviewing the manuscript should be better described and justified. Response: Our article, as we now emphasise more, is meant as an exploratory comment and a didactic contribution to inspire scholars in the humanities to see whether consensus methods may be valuable within their own discipline or sub discipline. It is not an instance of systematic empirical research, which would have required a much more detailed methodological section. But we certainly we agree that the method section can be improved in various ways. Therefore, we extended the section by including a specification of the backgrounds of the involved authors and specified the process of our collaboration as follows (see section 2, lines 85-94): To explore the potential of applying expert consensus methods in the humanities, we composed an interdisciplinary team of both researchers from the sciences using consensus methods in practice (LM: epidemiology; TH: qualitative research methodology; LB: epidemiology and data science, research integrity), and researchers familiar with the domain of the humanities and epistemology (CR: art history, museology; RP: epistemology and philosophy of science, religion and theology). In a first brainstorm session, knowledge on the topic was shared from both perspectives. This resulted in a list of consensus methods and examples used in the sciences, drafted by LM and TH. CR carried out a literature search in order to find examples thereof that were applied in her domain. A first draft was written and discussed in a second group session and culminated in a final version, which was edited and reviewed by all authors. Reviewer 1, Comment 3: Third, and more important, the paper refers always to ‘humanities.’ Humanities is a highly fragmented and differentiated field, where many and different epistemic communities work, interact, shape their own idea and uses of consensus method (if any) and where different instances of relationships with other researchers within the field and with neighboring fields are at place. The paper does not acknowledge this important characteristic of humanities (see footnote 1 where there is a list of disciplines included in the humanities that are considered in the paper). The same holds true for STEMs, which are considered as they are all adopting regularly expert consensus methods and there is no explanation or evidence about it. In sum, the paper ‘explores’ the expert consensus method without considering the set of ideas, values, theories, concepts and practices that pertain to the disciplines analyzed and their scientific work. Response: We acknowledge the concern raised and have taken this into account by adjusting a footnote 1 in section 1 (see above) and by revising section 4. Upon reflection we furthermore adjusted the ways to delineate the contexts in which expert consensus methods may be valuable, by focusing on one’s purpose in applying such a method (see section 4, lines 213-235): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 1, Comment 4: Fourth, sections 3 and 4 provide a clear summary of what expert consensus methods are and a deepen analysis of four types. Then the reader expects Section 5 discusses the results with respect to the exploration done. On the contrary, Section 5 lists of pros and cons related to the expert consensus method, and some examples of applications; the contents are completely unrelated with what the sections 3 and 4 have presented and the motivations for considering the expert consensus method an interesting development for humanities are unclear. Here the suggestion is to revise the sections trying to answer the research questions (when formulated) and highlight what we have learnt from the exploratory work, what is the contribution of the brainstorming to the exploration, what new avenues of research can be envisaged based on the results obtained, and what limitations/obstacles can the expert consensus model have in the humanities with respect to the other fields. Response: Thank you for pointing this out to us. Based on this valuable feedback, we have worked to better align section 5 with the preceding sections to create a more cohesive flow throughout the paper and have made more explicit what our line of reasoning is. However, because the paper is intended as preliminary exploration, part of our manuscript aims to identify the potential value of these methods for the humanities. Section 5 is dedicated to that purpose and we made that more explicit in the revision (see section 5, lines 253-315): In the preceding sections, we examined the elements and dynamics of expert consensus methods, along with examples and possibilities for intended objectives. In this section, we delve into the question why formal approaches to consensus in the humanities remain scarce, despite the significant factor of consensus and dissensus in knowledge production across research cultures. Aside from limitations, we also address what the potential added value of applying consensus methods to humanities-related issues could be. First of all, practical limitations likely play a role. Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Scholars may also disagree due to the role of research paradigms or schools within the humanities, as researchers’ positionality can shape their interpretations. In addition to that, humanities research can be concerned with hermeneutics, i.e., issues that have to do with understanding the values and meanings of texts and other objects. And many humanities fields such as ethics, epistemology, and metaphysics, rely on apriori methods like thought experiments, reasoning on the basis of intuitions and principles, and formal logic rather than empirical data. This allows for multiple interpretations that can all to some extent be warranted by the evidence base, and may not readily align into a singular consensus. These factors suggest that the complexity and interpretative nature of humanities research might render expert consensus methods to some extent less relevant. Nevertheless, we believe that there are compelling reasons for considering expert consensus methods in humanities research. First of all, opposing the above concerns, one of the important purposes of expert consensus methods is to bring various viewpoints, nuances, and interpretations to the surface, so that they can be discussed. Depending on the design and purpose of the consensus method, it could reveal levels of consensus and/or dissensus on an issue, within or across schools of thought or paradigms. They might even have the potential to articulate both divergences and common grounds regarding an issue, and by extension increase mutual understanding. The application of such a method may therefore stimulate epistemic progress in itself. Moreover, a carefully designed protocol and curated group of experts can foster a fair practice and ensure the consultation of diverse viewpoints. Expert consensus methods provide opportunities to mitigate biases and their ramifications by this cautious selection of experts, but also by blinding certain parts of the process in relation to anonymity of participating experts, for example. Secondly, expert consensus methods can be helpful in getting a firmer grip on the status quo or state of the art on a particular issue in a field (e.g. Hodgkin et al. (1975) mentioned in Table 2; Rulkens et al. 2022-2023, mentioned in Table 3). Of course, scholars often already have an idea about that, and such an informed opinion is valuable in itself. However, it can be complemented by a more structured approach, particularly when the number of scholars involved is large. This approach can additionally provide guidance for future research by pointing out the gaps in the current evidence base. And when applied on a recurrent basis, it might even have the potential to gain insight in how views evolve over time. It is important to note that consensus is not a final endpoint; rather, it can be continually re-evaluated in light of new insights and reasonable doubts regarding its validity. Third, an advantage of expert consensus methods is that they enhance the transparency and replicability of research because of their well-documented and systematised approach (for replication in the humanities, see Peels and Bouter (2018)). It can therefore be instrumental to epistemic progress beyond the consensus reached at a moment in time. Fourth, application or implementation of its outcomes is often easier after the process of a consensus method is completed (e.g. Standing Committee for the Museum Definition (2020-2022) (2022), mentioned in Table 3). And finally, expert consensus methods can aid public understanding by clarifying experts perspectives on specific issues (Stekelenburg et al., 2022) thereby enabling the public to make informed decisions (Ordway, 2021). In response to this comments, we furthermore extended suggestions for future research in our conclusion (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Response to reviewer 2: Daniel Leising, Technische Universitat Dresden, Dresden, Saxony, Germany Reviewer 2, Comment 1 : First, I would argue that the Humanities are actually not that “unique” in terms of their nature and approaches (page 11). To enable a proper discussion of this issue, the authors could explain their own view of what is so specific about the Humanities in a bit more detail. Here is my view: Interviews and text analyses are used in many different branches of science. Exploratory research is also conducted in many fields. The ways in which people give meaning to their experiences are also of interest across various branches of science. And even in many of the “harder” sciences, research is often “coloured by the backgrounds and positionality of the researchers” (page 8). So, much of what is common in research in the Humanities is actually shared with other fields. In my view, two features may actually distinguish “the Humanities” from other fields - but even these distinctions are only gradual in nature. One feature is the common practice of using everyday language as the primary scientific language. The other feature is the relative fuzziness of many concepts - which I would note is not necessarily a weakness but may often be unavoidable given the complexity of the phenomena of interest (and also given the first feature just mentioned). Do the authors agree? Response: Thank you for challenging us on this point. We agree that the borders between the general fields of the natural, social, and life sciences, and the humanities are fuzzy, when it comes to their objects, purposes, and methods. Whether there is something distinctive about the humanities – we believe that is compatible with fuzziness at the boarders – is a matter that we cannot possibly explore in this article (one of us does so in a book entitled A Philosophy of the Humanities that is forthcoming in 2025 with Oxford University Press, a volume co-authored by Stephen Grimm, Rik Peels, and René van Woudenberg). We do believe though that ordinary language use is not generally characteristic of the humanities (think of the highly technical language of metaphysics and epistemology), nor do we believe that insufficient clarity about core concepts is characteristic of the humanities. We agree though with the idea, also drawn to attention to by reviewer 1 in comment 3, that the humanities are a highly diversified field. We included that by adjusting footnote 1 in section 1: We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Additionally, to be able to demarcate between contexts in which the formal approach might be beneficial, we decided to put an emphasis on the purposes for which these methods are employed in other domains. This would be a way in which researchers with expertise in their fields could identify opportunities for a formal approach to consensus (see section 4, lines 213-235, also quoted above in response to comment 3 of reviewer 1): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 2, Comment 2: Second, I would like to encourage the authors to say a bit more about possible reasons for why so little consensus building is taking place in many fields of science, despite it being widely viewed as desirable. In my view, there are a couple of reasons, most of which are rather extraneous to an idealised image of science (as a collaborative and cumulative effort at building a reliable knowledge base for the good of mankind). For example, consensus building is likely to reduce the publication output of a field, which would be at odds with the interests of commercial publishers. Also, individual researchers have an interest in “making a name for themselves” based on their specific expertise or views regarding certain topics. Consensus building makes it more difficult to maintain a highly distinct profile and thus reduce visibility. This may be risky in a science system that is as competitive as the one we choose to entertain. And then there may be a need to “save face”: The more of your career you have dedicated to certain ideas (e.g., your own theory of some phenomenon), the harder it may be to let those ideas go - even if the evidence against them is overwhelming. There is a reason why Max Planck once stated that scientific progress happens “one funeral at a time”. At least a brief nod to the relevance of such more political factors would be nice - if the authors share my conviction that they are relevant. Who has an actual interest in achieving consensus? Who has an interest in avoiding the emergence of consensus? And what are the external incentives for and against this type of work ? Response: We think that these are valid points and the reviewers’ publications on the topic shed important light on this. We agree that external incentives may work against a formal approach to consensus as follows and have therefore slightly revised the manuscript on this point. We remain rather brief, though, as the exact explanation of why consensus methods are still relatively rarely used even in the sciences is an issue that is beyond the scope of our paper (see section 5, lines 259-264, also quoted in response to comment 4 of reviewer 1): Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Reviewer 2, Comment 3: Third, I would also appreciate if the authors could talk some more about “bad consensus” – views that are held by many, despite their being wrong. With a particular eye to the Humanities – what criteria are there, apart from consensus, that may help us determine whether a view is “correct”? Or may truth in the Humanities *only* be defined in terms of relative consensus among researchers? In the harder sciences, the ability to *predict* the occurrence of certain events from a theory (and some data input) plays a crucial role as evidence of the theory's correctness. Are the Humanities able to produce testable predictions, too? Can there be consensus projects in the Humanities that actually test one theory against another? Response: This is an interesting point, that would need a elaborative reflection in order to do justice to the complexity of this issue. It is important to note, that we do believe that consensus and truth are two different things, and we made this more clear in our revised paper. Based these suggestions we also incorporated that consensus can be reached based on the wrong reasons and (see section 5, lines 265-271, also quoted in response to comment 4 of reviewer 1): Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Reviewer 2, Comment 4: Fourth, on page 3 the authors state that “Since the questions addressed in the humanities may leave room for multiple viewpoints that are equally warranted by the evidence base, it can be infeasible or not desirable to get a complete or almost complete consensus.” To this I would respond: what are the conditions under which working toward consensus would not be feasible, or desirable? I agree that this may be the case – but when is it the case, and when is it not the case? I personally hold the somewhat sceptical view that many researchers will shy away from working toward consensus even if doing so *would* be both feasible and desirable (see my previous point). Such a “live and let live” attitude may often be relatively comfortable to have (e.g., to avoid conflict). It may, however, be regarded as unethical, because it will foster the preservation of an unnecessary level of ambiguity. Response: We appreciate this insightful suggestion. Our approach to expert consensus methods does not necessarily imply working towards consensus in all instances, but is also related to the assessment of the (lack of) consensus. We view measuring consensus as a means, not an objective in itself. Furthermore, as put forward in paragraph 3, the bar for ‘sufficient’ consensus is something consensus researchers can determine their selves. Applying a consensus methods is a costly endeavour in terms of time and means. There might be conditions under which the effort of applying such a method does not live up to the epistemic benefits it provides for. We furthermore believe that the feasibility and desirability of assessing consensus is an issue that merits further exploration and included this as follows (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Minor comments: Reviewer 2, Comment 5: Fifth, the authors state that a steering committee would typically be comprised of the people who initiated the process and who will ultimately author the consensus document. Could there not be cases in which the process should better be overseen by an independent third party (e.g., a moderator)? Especially when issues are highly contentious, for example? And should not all the experts who worked on the consensus be authors on the consensus document? Response: Thank you for this relevant point, this is certainly the case and we made this more clear in the revised manuscript by the following addition (see section 3, lines 193-198): Naturally, the six elements can be specified at by the steering committee. If they deem it valuable to include an independent moderator or to incorporate a member with a critical or divergent perspective—especially if the topic is highly controversial—this remains within their discretion. Additionally, it is up to them to decide who will be listed as co-authors, whether that includes only the steering committee, all participants, or a broader group representation such as the panellists as a whole, for instance. Reviewer 2, Comment 6: Sixth, something is not quite right on page 4, text lines 16-19. Response: Thank you for noticing, we have corrected this sentence as follows (see section 3, lines 136-140): Expert consensus methods are (i) applied by a process leader or steering committee , (ii) to reach consensus or assess the degree of consensus among a group of experts , (iii) about an issue , (iv) based on preset rules of engagement , (v) with the aim to deliver useful output , (vi) for future users . They are characterised by constructive and procedural usage of (dis)agreement and the different arguments that are provided therein. Reviewer 2, Comment 7: Seventh, I would say that evidence will never “speak for itself” (page 4) - otherwise papers in certain scientific fields would not need any Discussion sections. Response: Thank you for rightly pointing this out to us. We adjusted this as follows (see section 3, lines 132-134): This can be useful when the issue at hand is an important one and when evidence is lacking, it is inconsistent, or its interpretation is not obvious. As a result of our reactions to the comments, we also made adjustments to our abstract and conclusion, which are not all quoted in the above letter. We hope these revisions and clarifications meet the reviewers’ expectations. Please let us know if any further revisions are needed. Sincerely, Charlotte Rulkens MA Prof. dr. Rik Peels Dr. Lidwine Mokkink Dr. Tamarinde Haven Prof. em. dr. Lex Bouter [1] We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 16 Dec 2024 Charlotte Rulkens , Department of Philosophy, Vrije Universiteit Amsterdam, Amsterdam, The Netherlands 16 Dec 2024 Author Response Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the ... Continue reading Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the time and effort you have put into evaluating our work and for providing insightful comments and valuable suggestions. We have carefully considered each of them and have revised the manuscript accordingly. Below, we provide detailed responses to each of the comments. We believe that the revisions have improved the manuscript, and we hope that our responses meet any remaining concerns. Response to reviewer 1: Emanuela Reale, CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy Reviewer 1, Comment 1: First, the aim of the paper is clear, but the research question(s) related to that aim are missing. This is an important limitation, since the manuscript reports an exploratory effort and therefore a delimitation of the research problem that you want to investigate is very useful to delineate the perimeter of the work. Response: We acknowledge the concern raised and have addressed this by rewriting the introduction section and by incorporating a clearer description of our study goal. We now make it more clear that our goal was primarily didactic, in the sense that we sought to explore the usefulness of consensus methods utilised in the natural, social, and life sciences for the humanities (see section 1, lines 59-83): An important way to make progress in knowledge and understanding is to gradually, via the exchange of evidence, expertise, and critical discussion, reach (more) consensus on a topic. However, whether there is consensus on a certain matter at a moment in time can sometimes remain unknown, be unclear, or remain implicit. Expert consensus methods can lift off this inconclusiveness by intentionally and formally pursuing or assessing consensus. These methods are regularly used in natural, social, and life sciences. Despite the significant role of consensus in the development of (academic) knowledge and understanding (e.g. Kuhn, 1962; Habermas, 1996; thus also Peels, 2020; Grimm et al., forthcoming ), formal expert consensus methods have hardly been explored within in the context of the humanities. [1] Other kinds of valuable expert meetings do take place in various disciplines in the humanities. They usually do not seek to reach a consensus, but to explore a topic in-depth by bringing the relevant experts to the table. Moreover, expert consensus methods described in this article differ from the expert meetings that already take place in the humanities by the former’s procedural and formalised character. In this paper, we aim to highlight to scholars, particularly those in the humanities, what is possible in terms of a systematic approach to academic consensus. In doing so, we aim to initiate exploration of and discussion about the potential of applying such an approach. What would be the added value of applying expert consensus meetings to humanities-related issues? To achieve this, we first explicate our method (§2), before we provide a framework for the application of expert consensus methods by introducing six key elements (§3). Subsequently, we present common types of expert consensus methods utilized in the natural, social, and life sciences. We highlight their and proven value across various types of research through relevant examples and their respective objectives (§4). We then consider both the potential benefits and possible limitations to applying these methods within humanistic research (§5), and conclude with suggestions for future research (§6). Reviewer 1, Comment 2: Second, the section on Method is poor. The composition of the interdisciplinary team (and the reasons for using this team) is not clearly described, the brainstorm session is not properly defined and illustrated, the ‘informal’ literature review is something new for me (what ‘informal’ means?) and it is not acceptable for a research paper to quote two methods used (‘philosophical methods of conceptual analysis’ and ‘conceptual engineering’) without explaining how they have been used for the purposes of the paper but just putting two references. Also, the process of writing and reviewing the manuscript should be better described and justified. Response: Our article, as we now emphasise more, is meant as an exploratory comment and a didactic contribution to inspire scholars in the humanities to see whether consensus methods may be valuable within their own discipline or sub discipline. It is not an instance of systematic empirical research, which would have required a much more detailed methodological section. But we certainly we agree that the method section can be improved in various ways. Therefore, we extended the section by including a specification of the backgrounds of the involved authors and specified the process of our collaboration as follows (see section 2, lines 85-94): To explore the potential of applying expert consensus methods in the humanities, we composed an interdisciplinary team of both researchers from the sciences using consensus methods in practice (LM: epidemiology; TH: qualitative research methodology; LB: epidemiology and data science, research integrity), and researchers familiar with the domain of the humanities and epistemology (CR: art history, museology; RP: epistemology and philosophy of science, religion and theology). In a first brainstorm session, knowledge on the topic was shared from both perspectives. This resulted in a list of consensus methods and examples used in the sciences, drafted by LM and TH. CR carried out a literature search in order to find examples thereof that were applied in her domain. A first draft was written and discussed in a second group session and culminated in a final version, which was edited and reviewed by all authors. Reviewer 1, Comment 3: Third, and more important, the paper refers always to ‘humanities.’ Humanities is a highly fragmented and differentiated field, where many and different epistemic communities work, interact, shape their own idea and uses of consensus method (if any) and where different instances of relationships with other researchers within the field and with neighboring fields are at place. The paper does not acknowledge this important characteristic of humanities (see footnote 1 where there is a list of disciplines included in the humanities that are considered in the paper). The same holds true for STEMs, which are considered as they are all adopting regularly expert consensus methods and there is no explanation or evidence about it. In sum, the paper ‘explores’ the expert consensus method without considering the set of ideas, values, theories, concepts and practices that pertain to the disciplines analyzed and their scientific work. Response: We acknowledge the concern raised and have taken this into account by adjusting a footnote 1 in section 1 (see above) and by revising section 4. Upon reflection we furthermore adjusted the ways to delineate the contexts in which expert consensus methods may be valuable, by focusing on one’s purpose in applying such a method (see section 4, lines 213-235): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 1, Comment 4: Fourth, sections 3 and 4 provide a clear summary of what expert consensus methods are and a deepen analysis of four types. Then the reader expects Section 5 discusses the results with respect to the exploration done. On the contrary, Section 5 lists of pros and cons related to the expert consensus method, and some examples of applications; the contents are completely unrelated with what the sections 3 and 4 have presented and the motivations for considering the expert consensus method an interesting development for humanities are unclear. Here the suggestion is to revise the sections trying to answer the research questions (when formulated) and highlight what we have learnt from the exploratory work, what is the contribution of the brainstorming to the exploration, what new avenues of research can be envisaged based on the results obtained, and what limitations/obstacles can the expert consensus model have in the humanities with respect to the other fields. Response: Thank you for pointing this out to us. Based on this valuable feedback, we have worked to better align section 5 with the preceding sections to create a more cohesive flow throughout the paper and have made more explicit what our line of reasoning is. However, because the paper is intended as preliminary exploration, part of our manuscript aims to identify the potential value of these methods for the humanities. Section 5 is dedicated to that purpose and we made that more explicit in the revision (see section 5, lines 253-315): In the preceding sections, we examined the elements and dynamics of expert consensus methods, along with examples and possibilities for intended objectives. In this section, we delve into the question why formal approaches to consensus in the humanities remain scarce, despite the significant factor of consensus and dissensus in knowledge production across research cultures. Aside from limitations, we also address what the potential added value of applying consensus methods to humanities-related issues could be. First of all, practical limitations likely play a role. Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Scholars may also disagree due to the role of research paradigms or schools within the humanities, as researchers’ positionality can shape their interpretations. In addition to that, humanities research can be concerned with hermeneutics, i.e., issues that have to do with understanding the values and meanings of texts and other objects. And many humanities fields such as ethics, epistemology, and metaphysics, rely on apriori methods like thought experiments, reasoning on the basis of intuitions and principles, and formal logic rather than empirical data. This allows for multiple interpretations that can all to some extent be warranted by the evidence base, and may not readily align into a singular consensus. These factors suggest that the complexity and interpretative nature of humanities research might render expert consensus methods to some extent less relevant. Nevertheless, we believe that there are compelling reasons for considering expert consensus methods in humanities research. First of all, opposing the above concerns, one of the important purposes of expert consensus methods is to bring various viewpoints, nuances, and interpretations to the surface, so that they can be discussed. Depending on the design and purpose of the consensus method, it could reveal levels of consensus and/or dissensus on an issue, within or across schools of thought or paradigms. They might even have the potential to articulate both divergences and common grounds regarding an issue, and by extension increase mutual understanding. The application of such a method may therefore stimulate epistemic progress in itself. Moreover, a carefully designed protocol and curated group of experts can foster a fair practice and ensure the consultation of diverse viewpoints. Expert consensus methods provide opportunities to mitigate biases and their ramifications by this cautious selection of experts, but also by blinding certain parts of the process in relation to anonymity of participating experts, for example. Secondly, expert consensus methods can be helpful in getting a firmer grip on the status quo or state of the art on a particular issue in a field (e.g. Hodgkin et al. (1975) mentioned in Table 2; Rulkens et al. 2022-2023, mentioned in Table 3). Of course, scholars often already have an idea about that, and such an informed opinion is valuable in itself. However, it can be complemented by a more structured approach, particularly when the number of scholars involved is large. This approach can additionally provide guidance for future research by pointing out the gaps in the current evidence base. And when applied on a recurrent basis, it might even have the potential to gain insight in how views evolve over time. It is important to note that consensus is not a final endpoint; rather, it can be continually re-evaluated in light of new insights and reasonable doubts regarding its validity. Third, an advantage of expert consensus methods is that they enhance the transparency and replicability of research because of their well-documented and systematised approach (for replication in the humanities, see Peels and Bouter (2018)). It can therefore be instrumental to epistemic progress beyond the consensus reached at a moment in time. Fourth, application or implementation of its outcomes is often easier after the process of a consensus method is completed (e.g. Standing Committee for the Museum Definition (2020-2022) (2022), mentioned in Table 3). And finally, expert consensus methods can aid public understanding by clarifying experts perspectives on specific issues (Stekelenburg et al., 2022) thereby enabling the public to make informed decisions (Ordway, 2021). In response to this comments, we furthermore extended suggestions for future research in our conclusion (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Response to reviewer 2: Daniel Leising, Technische Universitat Dresden, Dresden, Saxony, Germany Reviewer 2, Comment 1 : First, I would argue that the Humanities are actually not that “unique” in terms of their nature and approaches (page 11). To enable a proper discussion of this issue, the authors could explain their own view of what is so specific about the Humanities in a bit more detail. Here is my view: Interviews and text analyses are used in many different branches of science. Exploratory research is also conducted in many fields. The ways in which people give meaning to their experiences are also of interest across various branches of science. And even in many of the “harder” sciences, research is often “coloured by the backgrounds and positionality of the researchers” (page 8). So, much of what is common in research in the Humanities is actually shared with other fields. In my view, two features may actually distinguish “the Humanities” from other fields - but even these distinctions are only gradual in nature. One feature is the common practice of using everyday language as the primary scientific language. The other feature is the relative fuzziness of many concepts - which I would note is not necessarily a weakness but may often be unavoidable given the complexity of the phenomena of interest (and also given the first feature just mentioned). Do the authors agree? Response: Thank you for challenging us on this point. We agree that the borders between the general fields of the natural, social, and life sciences, and the humanities are fuzzy, when it comes to their objects, purposes, and methods. Whether there is something distinctive about the humanities – we believe that is compatible with fuzziness at the boarders – is a matter that we cannot possibly explore in this article (one of us does so in a book entitled A Philosophy of the Humanities that is forthcoming in 2025 with Oxford University Press, a volume co-authored by Stephen Grimm, Rik Peels, and René van Woudenberg). We do believe though that ordinary language use is not generally characteristic of the humanities (think of the highly technical language of metaphysics and epistemology), nor do we believe that insufficient clarity about core concepts is characteristic of the humanities. We agree though with the idea, also drawn to attention to by reviewer 1 in comment 3, that the humanities are a highly diversified field. We included that by adjusting footnote 1 in section 1: We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Additionally, to be able to demarcate between contexts in which the formal approach might be beneficial, we decided to put an emphasis on the purposes for which these methods are employed in other domains. This would be a way in which researchers with expertise in their fields could identify opportunities for a formal approach to consensus (see section 4, lines 213-235, also quoted above in response to comment 3 of reviewer 1): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 2, Comment 2: Second, I would like to encourage the authors to say a bit more about possible reasons for why so little consensus building is taking place in many fields of science, despite it being widely viewed as desirable. In my view, there are a couple of reasons, most of which are rather extraneous to an idealised image of science (as a collaborative and cumulative effort at building a reliable knowledge base for the good of mankind). For example, consensus building is likely to reduce the publication output of a field, which would be at odds with the interests of commercial publishers. Also, individual researchers have an interest in “making a name for themselves” based on their specific expertise or views regarding certain topics. Consensus building makes it more difficult to maintain a highly distinct profile and thus reduce visibility. This may be risky in a science system that is as competitive as the one we choose to entertain. And then there may be a need to “save face”: The more of your career you have dedicated to certain ideas (e.g., your own theory of some phenomenon), the harder it may be to let those ideas go - even if the evidence against them is overwhelming. There is a reason why Max Planck once stated that scientific progress happens “one funeral at a time”. At least a brief nod to the relevance of such more political factors would be nice - if the authors share my conviction that they are relevant. Who has an actual interest in achieving consensus? Who has an interest in avoiding the emergence of consensus? And what are the external incentives for and against this type of work ? Response: We think that these are valid points and the reviewers’ publications on the topic shed important light on this. We agree that external incentives may work against a formal approach to consensus as follows and have therefore slightly revised the manuscript on this point. We remain rather brief, though, as the exact explanation of why consensus methods are still relatively rarely used even in the sciences is an issue that is beyond the scope of our paper (see section 5, lines 259-264, also quoted in response to comment 4 of reviewer 1): Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Reviewer 2, Comment 3: Third, I would also appreciate if the authors could talk some more about “bad consensus” – views that are held by many, despite their being wrong. With a particular eye to the Humanities – what criteria are there, apart from consensus, that may help us determine whether a view is “correct”? Or may truth in the Humanities *only* be defined in terms of relative consensus among researchers? In the harder sciences, the ability to *predict* the occurrence of certain events from a theory (and some data input) plays a crucial role as evidence of the theory's correctness. Are the Humanities able to produce testable predictions, too? Can there be consensus projects in the Humanities that actually test one theory against another? Response: This is an interesting point, that would need a elaborative reflection in order to do justice to the complexity of this issue. It is important to note, that we do believe that consensus and truth are two different things, and we made this more clear in our revised paper. Based these suggestions we also incorporated that consensus can be reached based on the wrong reasons and (see section 5, lines 265-271, also quoted in response to comment 4 of reviewer 1): Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Reviewer 2, Comment 4: Fourth, on page 3 the authors state that “Since the questions addressed in the humanities may leave room for multiple viewpoints that are equally warranted by the evidence base, it can be infeasible or not desirable to get a complete or almost complete consensus.” To this I would respond: what are the conditions under which working toward consensus would not be feasible, or desirable? I agree that this may be the case – but when is it the case, and when is it not the case? I personally hold the somewhat sceptical view that many researchers will shy away from working toward consensus even if doing so *would* be both feasible and desirable (see my previous point). Such a “live and let live” attitude may often be relatively comfortable to have (e.g., to avoid conflict). It may, however, be regarded as unethical, because it will foster the preservation of an unnecessary level of ambiguity. Response: We appreciate this insightful suggestion. Our approach to expert consensus methods does not necessarily imply working towards consensus in all instances, but is also related to the assessment of the (lack of) consensus. We view measuring consensus as a means, not an objective in itself. Furthermore, as put forward in paragraph 3, the bar for ‘sufficient’ consensus is something consensus researchers can determine their selves. Applying a consensus methods is a costly endeavour in terms of time and means. There might be conditions under which the effort of applying such a method does not live up to the epistemic benefits it provides for. We furthermore believe that the feasibility and desirability of assessing consensus is an issue that merits further exploration and included this as follows (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Minor comments: Reviewer 2, Comment 5: Fifth, the authors state that a steering committee would typically be comprised of the people who initiated the process and who will ultimately author the consensus document. Could there not be cases in which the process should better be overseen by an independent third party (e.g., a moderator)? Especially when issues are highly contentious, for example? And should not all the experts who worked on the consensus be authors on the consensus document? Response: Thank you for this relevant point, this is certainly the case and we made this more clear in the revised manuscript by the following addition (see section 3, lines 193-198): Naturally, the six elements can be specified at by the steering committee. If they deem it valuable to include an independent moderator or to incorporate a member with a critical or divergent perspective—especially if the topic is highly controversial—this remains within their discretion. Additionally, it is up to them to decide who will be listed as co-authors, whether that includes only the steering committee, all participants, or a broader group representation such as the panellists as a whole, for instance. Reviewer 2, Comment 6: Sixth, something is not quite right on page 4, text lines 16-19. Response: Thank you for noticing, we have corrected this sentence as follows (see section 3, lines 136-140): Expert consensus methods are (i) applied by a process leader or steering committee , (ii) to reach consensus or assess the degree of consensus among a group of experts , (iii) about an issue , (iv) based on preset rules of engagement , (v) with the aim to deliver useful output , (vi) for future users . They are characterised by constructive and procedural usage of (dis)agreement and the different arguments that are provided therein. Reviewer 2, Comment 7: Seventh, I would say that evidence will never “speak for itself” (page 4) - otherwise papers in certain scientific fields would not need any Discussion sections. Response: Thank you for rightly pointing this out to us. We adjusted this as follows (see section 3, lines 132-134): This can be useful when the issue at hand is an important one and when evidence is lacking, it is inconsistent, or its interpretation is not obvious. As a result of our reactions to the comments, we also made adjustments to our abstract and conclusion, which are not all quoted in the above letter. We hope these revisions and clarifications meet the reviewers’ expectations. Please let us know if any further revisions are needed. Sincerely, Charlotte Rulkens MA Prof. dr. Rik Peels Dr. Lidwine Mokkink Dr. Tamarinde Haven Prof. em. dr. Lex Bouter [1] We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the time and effort you have put into evaluating our work and for providing insightful comments and valuable suggestions. We have carefully considered each of them and have revised the manuscript accordingly. Below, we provide detailed responses to each of the comments. We believe that the revisions have improved the manuscript, and we hope that our responses meet any remaining concerns. Response to reviewer 1: Emanuela Reale, CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy Reviewer 1, Comment 1: First, the aim of the paper is clear, but the research question(s) related to that aim are missing. This is an important limitation, since the manuscript reports an exploratory effort and therefore a delimitation of the research problem that you want to investigate is very useful to delineate the perimeter of the work. Response: We acknowledge the concern raised and have addressed this by rewriting the introduction section and by incorporating a clearer description of our study goal. We now make it more clear that our goal was primarily didactic, in the sense that we sought to explore the usefulness of consensus methods utilised in the natural, social, and life sciences for the humanities (see section 1, lines 59-83): An important way to make progress in knowledge and understanding is to gradually, via the exchange of evidence, expertise, and critical discussion, reach (more) consensus on a topic. However, whether there is consensus on a certain matter at a moment in time can sometimes remain unknown, be unclear, or remain implicit. Expert consensus methods can lift off this inconclusiveness by intentionally and formally pursuing or assessing consensus. These methods are regularly used in natural, social, and life sciences. Despite the significant role of consensus in the development of (academic) knowledge and understanding (e.g. Kuhn, 1962; Habermas, 1996; thus also Peels, 2020; Grimm et al., forthcoming ), formal expert consensus methods have hardly been explored within in the context of the humanities. [1] Other kinds of valuable expert meetings do take place in various disciplines in the humanities. They usually do not seek to reach a consensus, but to explore a topic in-depth by bringing the relevant experts to the table. Moreover, expert consensus methods described in this article differ from the expert meetings that already take place in the humanities by the former’s procedural and formalised character. In this paper, we aim to highlight to scholars, particularly those in the humanities, what is possible in terms of a systematic approach to academic consensus. In doing so, we aim to initiate exploration of and discussion about the potential of applying such an approach. What would be the added value of applying expert consensus meetings to humanities-related issues? To achieve this, we first explicate our method (§2), before we provide a framework for the application of expert consensus methods by introducing six key elements (§3). Subsequently, we present common types of expert consensus methods utilized in the natural, social, and life sciences. We highlight their and proven value across various types of research through relevant examples and their respective objectives (§4). We then consider both the potential benefits and possible limitations to applying these methods within humanistic research (§5), and conclude with suggestions for future research (§6). Reviewer 1, Comment 2: Second, the section on Method is poor. The composition of the interdisciplinary team (and the reasons for using this team) is not clearly described, the brainstorm session is not properly defined and illustrated, the ‘informal’ literature review is something new for me (what ‘informal’ means?) and it is not acceptable for a research paper to quote two methods used (‘philosophical methods of conceptual analysis’ and ‘conceptual engineering’) without explaining how they have been used for the purposes of the paper but just putting two references. Also, the process of writing and reviewing the manuscript should be better described and justified. Response: Our article, as we now emphasise more, is meant as an exploratory comment and a didactic contribution to inspire scholars in the humanities to see whether consensus methods may be valuable within their own discipline or sub discipline. It is not an instance of systematic empirical research, which would have required a much more detailed methodological section. But we certainly we agree that the method section can be improved in various ways. Therefore, we extended the section by including a specification of the backgrounds of the involved authors and specified the process of our collaboration as follows (see section 2, lines 85-94): To explore the potential of applying expert consensus methods in the humanities, we composed an interdisciplinary team of both researchers from the sciences using consensus methods in practice (LM: epidemiology; TH: qualitative research methodology; LB: epidemiology and data science, research integrity), and researchers familiar with the domain of the humanities and epistemology (CR: art history, museology; RP: epistemology and philosophy of science, religion and theology). In a first brainstorm session, knowledge on the topic was shared from both perspectives. This resulted in a list of consensus methods and examples used in the sciences, drafted by LM and TH. CR carried out a literature search in order to find examples thereof that were applied in her domain. A first draft was written and discussed in a second group session and culminated in a final version, which was edited and reviewed by all authors. Reviewer 1, Comment 3: Third, and more important, the paper refers always to ‘humanities.’ Humanities is a highly fragmented and differentiated field, where many and different epistemic communities work, interact, shape their own idea and uses of consensus method (if any) and where different instances of relationships with other researchers within the field and with neighboring fields are at place. The paper does not acknowledge this important characteristic of humanities (see footnote 1 where there is a list of disciplines included in the humanities that are considered in the paper). The same holds true for STEMs, which are considered as they are all adopting regularly expert consensus methods and there is no explanation or evidence about it. In sum, the paper ‘explores’ the expert consensus method without considering the set of ideas, values, theories, concepts and practices that pertain to the disciplines analyzed and their scientific work. Response: We acknowledge the concern raised and have taken this into account by adjusting a footnote 1 in section 1 (see above) and by revising section 4. Upon reflection we furthermore adjusted the ways to delineate the contexts in which expert consensus methods may be valuable, by focusing on one’s purpose in applying such a method (see section 4, lines 213-235): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 1, Comment 4: Fourth, sections 3 and 4 provide a clear summary of what expert consensus methods are and a deepen analysis of four types. Then the reader expects Section 5 discusses the results with respect to the exploration done. On the contrary, Section 5 lists of pros and cons related to the expert consensus method, and some examples of applications; the contents are completely unrelated with what the sections 3 and 4 have presented and the motivations for considering the expert consensus method an interesting development for humanities are unclear. Here the suggestion is to revise the sections trying to answer the research questions (when formulated) and highlight what we have learnt from the exploratory work, what is the contribution of the brainstorming to the exploration, what new avenues of research can be envisaged based on the results obtained, and what limitations/obstacles can the expert consensus model have in the humanities with respect to the other fields. Response: Thank you for pointing this out to us. Based on this valuable feedback, we have worked to better align section 5 with the preceding sections to create a more cohesive flow throughout the paper and have made more explicit what our line of reasoning is. However, because the paper is intended as preliminary exploration, part of our manuscript aims to identify the potential value of these methods for the humanities. Section 5 is dedicated to that purpose and we made that more explicit in the revision (see section 5, lines 253-315): In the preceding sections, we examined the elements and dynamics of expert consensus methods, along with examples and possibilities for intended objectives. In this section, we delve into the question why formal approaches to consensus in the humanities remain scarce, despite the significant factor of consensus and dissensus in knowledge production across research cultures. Aside from limitations, we also address what the potential added value of applying consensus methods to humanities-related issues could be. First of all, practical limitations likely play a role. Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Scholars may also disagree due to the role of research paradigms or schools within the humanities, as researchers’ positionality can shape their interpretations. In addition to that, humanities research can be concerned with hermeneutics, i.e., issues that have to do with understanding the values and meanings of texts and other objects. And many humanities fields such as ethics, epistemology, and metaphysics, rely on apriori methods like thought experiments, reasoning on the basis of intuitions and principles, and formal logic rather than empirical data. This allows for multiple interpretations that can all to some extent be warranted by the evidence base, and may not readily align into a singular consensus. These factors suggest that the complexity and interpretative nature of humanities research might render expert consensus methods to some extent less relevant. Nevertheless, we believe that there are compelling reasons for considering expert consensus methods in humanities research. First of all, opposing the above concerns, one of the important purposes of expert consensus methods is to bring various viewpoints, nuances, and interpretations to the surface, so that they can be discussed. Depending on the design and purpose of the consensus method, it could reveal levels of consensus and/or dissensus on an issue, within or across schools of thought or paradigms. They might even have the potential to articulate both divergences and common grounds regarding an issue, and by extension increase mutual understanding. The application of such a method may therefore stimulate epistemic progress in itself. Moreover, a carefully designed protocol and curated group of experts can foster a fair practice and ensure the consultation of diverse viewpoints. Expert consensus methods provide opportunities to mitigate biases and their ramifications by this cautious selection of experts, but also by blinding certain parts of the process in relation to anonymity of participating experts, for example. Secondly, expert consensus methods can be helpful in getting a firmer grip on the status quo or state of the art on a particular issue in a field (e.g. Hodgkin et al. (1975) mentioned in Table 2; Rulkens et al. 2022-2023, mentioned in Table 3). Of course, scholars often already have an idea about that, and such an informed opinion is valuable in itself. However, it can be complemented by a more structured approach, particularly when the number of scholars involved is large. This approach can additionally provide guidance for future research by pointing out the gaps in the current evidence base. And when applied on a recurrent basis, it might even have the potential to gain insight in how views evolve over time. It is important to note that consensus is not a final endpoint; rather, it can be continually re-evaluated in light of new insights and reasonable doubts regarding its validity. Third, an advantage of expert consensus methods is that they enhance the transparency and replicability of research because of their well-documented and systematised approach (for replication in the humanities, see Peels and Bouter (2018)). It can therefore be instrumental to epistemic progress beyond the consensus reached at a moment in time. Fourth, application or implementation of its outcomes is often easier after the process of a consensus method is completed (e.g. Standing Committee for the Museum Definition (2020-2022) (2022), mentioned in Table 3). And finally, expert consensus methods can aid public understanding by clarifying experts perspectives on specific issues (Stekelenburg et al., 2022) thereby enabling the public to make informed decisions (Ordway, 2021). In response to this comments, we furthermore extended suggestions for future research in our conclusion (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Response to reviewer 2: Daniel Leising, Technische Universitat Dresden, Dresden, Saxony, Germany Reviewer 2, Comment 1 : First, I would argue that the Humanities are actually not that “unique” in terms of their nature and approaches (page 11). To enable a proper discussion of this issue, the authors could explain their own view of what is so specific about the Humanities in a bit more detail. Here is my view: Interviews and text analyses are used in many different branches of science. Exploratory research is also conducted in many fields. The ways in which people give meaning to their experiences are also of interest across various branches of science. And even in many of the “harder” sciences, research is often “coloured by the backgrounds and positionality of the researchers” (page 8). So, much of what is common in research in the Humanities is actually shared with other fields. In my view, two features may actually distinguish “the Humanities” from other fields - but even these distinctions are only gradual in nature. One feature is the common practice of using everyday language as the primary scientific language. The other feature is the relative fuzziness of many concepts - which I would note is not necessarily a weakness but may often be unavoidable given the complexity of the phenomena of interest (and also given the first feature just mentioned). Do the authors agree? Response: Thank you for challenging us on this point. We agree that the borders between the general fields of the natural, social, and life sciences, and the humanities are fuzzy, when it comes to their objects, purposes, and methods. Whether there is something distinctive about the humanities – we believe that is compatible with fuzziness at the boarders – is a matter that we cannot possibly explore in this article (one of us does so in a book entitled A Philosophy of the Humanities that is forthcoming in 2025 with Oxford University Press, a volume co-authored by Stephen Grimm, Rik Peels, and René van Woudenberg). We do believe though that ordinary language use is not generally characteristic of the humanities (think of the highly technical language of metaphysics and epistemology), nor do we believe that insufficient clarity about core concepts is characteristic of the humanities. We agree though with the idea, also drawn to attention to by reviewer 1 in comment 3, that the humanities are a highly diversified field. We included that by adjusting footnote 1 in section 1: We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Additionally, to be able to demarcate between contexts in which the formal approach might be beneficial, we decided to put an emphasis on the purposes for which these methods are employed in other domains. This would be a way in which researchers with expertise in their fields could identify opportunities for a formal approach to consensus (see section 4, lines 213-235, also quoted above in response to comment 3 of reviewer 1): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 2, Comment 2: Second, I would like to encourage the authors to say a bit more about possible reasons for why so little consensus building is taking place in many fields of science, despite it being widely viewed as desirable. In my view, there are a couple of reasons, most of which are rather extraneous to an idealised image of science (as a collaborative and cumulative effort at building a reliable knowledge base for the good of mankind). For example, consensus building is likely to reduce the publication output of a field, which would be at odds with the interests of commercial publishers. Also, individual researchers have an interest in “making a name for themselves” based on their specific expertise or views regarding certain topics. Consensus building makes it more difficult to maintain a highly distinct profile and thus reduce visibility. This may be risky in a science system that is as competitive as the one we choose to entertain. And then there may be a need to “save face”: The more of your career you have dedicated to certain ideas (e.g., your own theory of some phenomenon), the harder it may be to let those ideas go - even if the evidence against them is overwhelming. There is a reason why Max Planck once stated that scientific progress happens “one funeral at a time”. At least a brief nod to the relevance of such more political factors would be nice - if the authors share my conviction that they are relevant. Who has an actual interest in achieving consensus? Who has an interest in avoiding the emergence of consensus? And what are the external incentives for and against this type of work ? Response: We think that these are valid points and the reviewers’ publications on the topic shed important light on this. We agree that external incentives may work against a formal approach to consensus as follows and have therefore slightly revised the manuscript on this point. We remain rather brief, though, as the exact explanation of why consensus methods are still relatively rarely used even in the sciences is an issue that is beyond the scope of our paper (see section 5, lines 259-264, also quoted in response to comment 4 of reviewer 1): Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Reviewer 2, Comment 3: Third, I would also appreciate if the authors could talk some more about “bad consensus” – views that are held by many, despite their being wrong. With a particular eye to the Humanities – what criteria are there, apart from consensus, that may help us determine whether a view is “correct”? Or may truth in the Humanities *only* be defined in terms of relative consensus among researchers? In the harder sciences, the ability to *predict* the occurrence of certain events from a theory (and some data input) plays a crucial role as evidence of the theory's correctness. Are the Humanities able to produce testable predictions, too? Can there be consensus projects in the Humanities that actually test one theory against another? Response: This is an interesting point, that would need a elaborative reflection in order to do justice to the complexity of this issue. It is important to note, that we do believe that consensus and truth are two different things, and we made this more clear in our revised paper. Based these suggestions we also incorporated that consensus can be reached based on the wrong reasons and (see section 5, lines 265-271, also quoted in response to comment 4 of reviewer 1): Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Reviewer 2, Comment 4: Fourth, on page 3 the authors state that “Since the questions addressed in the humanities may leave room for multiple viewpoints that are equally warranted by the evidence base, it can be infeasible or not desirable to get a complete or almost complete consensus.” To this I would respond: what are the conditions under which working toward consensus would not be feasible, or desirable? I agree that this may be the case – but when is it the case, and when is it not the case? I personally hold the somewhat sceptical view that many researchers will shy away from working toward consensus even if doing so *would* be both feasible and desirable (see my previous point). Such a “live and let live” attitude may often be relatively comfortable to have (e.g., to avoid conflict). It may, however, be regarded as unethical, because it will foster the preservation of an unnecessary level of ambiguity. Response: We appreciate this insightful suggestion. Our approach to expert consensus methods does not necessarily imply working towards consensus in all instances, but is also related to the assessment of the (lack of) consensus. We view measuring consensus as a means, not an objective in itself. Furthermore, as put forward in paragraph 3, the bar for ‘sufficient’ consensus is something consensus researchers can determine their selves. Applying a consensus methods is a costly endeavour in terms of time and means. There might be conditions under which the effort of applying such a method does not live up to the epistemic benefits it provides for. We furthermore believe that the feasibility and desirability of assessing consensus is an issue that merits further exploration and included this as follows (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Minor comments: Reviewer 2, Comment 5: Fifth, the authors state that a steering committee would typically be comprised of the people who initiated the process and who will ultimately author the consensus document. Could there not be cases in which the process should better be overseen by an independent third party (e.g., a moderator)? Especially when issues are highly contentious, for example? And should not all the experts who worked on the consensus be authors on the consensus document? Response: Thank you for this relevant point, this is certainly the case and we made this more clear in the revised manuscript by the following addition (see section 3, lines 193-198): Naturally, the six elements can be specified at by the steering committee. If they deem it valuable to include an independent moderator or to incorporate a member with a critical or divergent perspective—especially if the topic is highly controversial—this remains within their discretion. Additionally, it is up to them to decide who will be listed as co-authors, whether that includes only the steering committee, all participants, or a broader group representation such as the panellists as a whole, for instance. Reviewer 2, Comment 6: Sixth, something is not quite right on page 4, text lines 16-19. Response: Thank you for noticing, we have corrected this sentence as follows (see section 3, lines 136-140): Expert consensus methods are (i) applied by a process leader or steering committee , (ii) to reach consensus or assess the degree of consensus among a group of experts , (iii) about an issue , (iv) based on preset rules of engagement , (v) with the aim to deliver useful output , (vi) for future users . They are characterised by constructive and procedural usage of (dis)agreement and the different arguments that are provided therein. Reviewer 2, Comment 7: Seventh, I would say that evidence will never “speak for itself” (page 4) - otherwise papers in certain scientific fields would not need any Discussion sections. Response: Thank you for rightly pointing this out to us. We adjusted this as follows (see section 3, lines 132-134): This can be useful when the issue at hand is an important one and when evidence is lacking, it is inconsistent, or its interpretation is not obvious. As a result of our reactions to the comments, we also made adjustments to our abstract and conclusion, which are not all quoted in the above letter. We hope these revisions and clarifications meet the reviewers’ expectations. Please let us know if any further revisions are needed. Sincerely, Charlotte Rulkens MA Prof. dr. Rik Peels Dr. Lidwine Mokkink Dr. Tamarinde Haven Prof. em. dr. Lex Bouter [1] We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Reale E. Reviewer Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.163073.r296789 ) The direct URL for this report is: https://f1000research.com/articles/13-710/v1#referee-response-296789 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 23 Aug 2024 Emanuela Reale , CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy Not Approved VIEWS 0 https://doi.org/10.5256/f1000research.163073.r296789 Expert Consensus Methods In The Humanities – An Exploration of their Potential The paper wants to explore the value of applying expert consensus methods in the humanities. The topic is very interesting and deserve attention. The manuscript ... Continue reading READ ALL Expert Consensus Methods In The Humanities – An Exploration of their Potential The paper wants to explore the value of applying expert consensus methods in the humanities. The topic is very interesting and deserve attention. The manuscript is clearly written, and the literature mobilized is appropriate and relevant for the aim of the investigation proposed. The effort of synthetize concepts and findings using tables is valuable for the clarity of the argument. However, the manuscript in its current form has several weaknesses and, in my opinion, needs some substantial re-work. Hereby I try to summarize the most important critical issues that must be solved. First, the aim of the paper is clear, but the research question(s) related to that aim are missing. This is an important limitation, since the manuscript reports an exploratory effort and therefore a delimitation of the research problem that you want to investigate is very useful to delineate the perimeter of the work. Second, the section on Method is poor. The composition of the interdisciplinary team (and the reasons for using this team) is not clearly described, the brainstorm session is not properly defined and illustrated, the ‘informal’ literature review is something new for me (what ‘informal’ means?) and it is not acceptable for a research paper to quote two methods used (‘philosophical methods of conceptual analysis’ and ‘conceptual engineering’) without explaining how they have been used for the purposes of the paper but just putting two references. Also, the process of writing and reviewing the manuscript should be better described and justified. Third, and more important, the paper refers always to ‘humanities.’ Humanities is a highly fragmented and differentiated field, where many and different epistemic communities work, interact, shape their own idea and uses of consensus method (if any) and where different instances of relationships with other researchers within the field and with neighboring fields are at place. The paper does not acknowledge this important characteristic of humanities (see footnote 1 where there is a list of disciplines included in the humanities that are considered in the paper). The same holds true for STEMs, which are considered as they are all adopting regularly expert consensus methods and there is no explanation or evidence about it. In sum, the paper ‘explores’ the expert consensus method without considering the set of ideas, values, theories, concepts and practices that pertain to the disciplines analyzed and their scientific work. Fourth, sections 3 and 4 provide a clear summary of what expert consensus methods are and a deepen analysis of four types. Then the reader expects Section 5 discusses the results with respect to the exploration done. On the contrary, Section 5 lists of pros and cons related to the expert consensus method, and some examples of applications; the contents are completely unrelated with what the sections 3 and 4 have presented and the motivations for considering the expert consensus method an interesting development for humanities are unclear. Here the suggestion is to revise the sections trying to answer the research questions (when formulated) and highlight what we have learnt from the exploratory work, what is the contribution of the brainstorming to the exploration, what new avenues of research can be envisaged based on the results obtained, and what limitations/obstacles can the expert consensus model have in the humanities with respect to the other fields. Is the rationale for developing the new method (or application) clearly explained? No Is the description of the method technically sound? No Are sufficient details provided to allow replication of the method development and its use by others? No If any results are presented, are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions about the method and its performance adequately supported by the findings presented in the article? No Competing Interests: No competing interests were disclosed. Reviewer Expertise: Higher education and research policy I confirm that I have read this submission and believe that I have an appropriate level of expertise to state that I do not consider it to be of an acceptable scientific standard, for reasons outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Reale E. Reviewer Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.163073.r296789 ) The direct URL for this report is: https://f1000research.com/articles/13-710/v1#referee-response-296789 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 16 Dec 2024 Charlotte Rulkens , Department of Philosophy, Vrije Universiteit Amsterdam, Amsterdam, The Netherlands 16 Dec 2024 Author Response We sincerely thank the reviewer for her valuable comments, all of which we will address in the revision of our paper. We plan to begin this revision as soon as ... Continue reading We sincerely thank the reviewer for her valuable comments, all of which we will address in the revision of our paper. We plan to begin this revision as soon as we receive the second review. We agree with the first and second suggestions to clarify our research questions, as well as the opportunities mentioned for improving the method section. While our method section may differ from those typically found in research papers, it is important to note that this paper is intended as a perspective piece with a didactic focus. It is aimed at highlighting to humanities scholars what is possible in terms of a systematic approach to academic ​​consensus on a topic and to initiate exploration of and discussion about the possible value of applying such an approach. Regarding the third comment, we will ensure that we address the issue of differentiation within research fields and discuss its implications for the application of consensus methods. Concerning the fourth point, we will work to better integrate section 5 with the preceding sections to create a more cohesive flow throughout the paper. We sincerely thank the reviewer for her valuable comments, all of which we will address in the revision of our paper. We plan to begin this revision as soon as we receive the second review. We agree with the first and second suggestions to clarify our research questions, as well as the opportunities mentioned for improving the method section. While our method section may differ from those typically found in research papers, it is important to note that this paper is intended as a perspective piece with a didactic focus. It is aimed at highlighting to humanities scholars what is possible in terms of a systematic approach to academic ​​consensus on a topic and to initiate exploration of and discussion about the possible value of applying such an approach. Regarding the third comment, we will ensure that we address the issue of differentiation within research fields and discuss its implications for the application of consensus methods. Concerning the fourth point, we will work to better integrate section 5 with the preceding sections to create a more cohesive flow throughout the paper. Competing Interests: No competing interests were disclosed. Close Report a concern Author Response 16 Dec 2024 Charlotte Rulkens , Department of Philosophy, Vrije Universiteit Amsterdam, Amsterdam, The Netherlands 16 Dec 2024 Author Response Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the ... Continue reading Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the time and effort you have put into evaluating our work and for providing insightful comments and valuable suggestions. We have carefully considered each of them and have revised the manuscript accordingly. Below, we provide detailed responses to each of the comments. We believe that the revisions have improved the manuscript, and we hope that our responses meet any remaining concerns. Response to reviewer 1: Emanuela Reale, CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy Reviewer 1, Comment 1: First, the aim of the paper is clear, but the research question(s) related to that aim are missing. This is an important limitation, since the manuscript reports an exploratory effort and therefore a delimitation of the research problem that you want to investigate is very useful to delineate the perimeter of the work. Response: We acknowledge the concern raised and have addressed this by rewriting the introduction section and by incorporating a clearer description of our study goal. We now make it more clear that our goal was primarily didactic, in the sense that we sought to explore the usefulness of consensus methods utilised in the natural, social, and life sciences for the humanities (see section 1, lines 59-83): An important way to make progress in knowledge and understanding is to gradually, via the exchange of evidence, expertise, and critical discussion, reach (more) consensus on a topic. However, whether there is consensus on a certain matter at a moment in time can sometimes remain unknown, be unclear, or remain implicit. Expert consensus methods can lift off this inconclusiveness by intentionally and formally pursuing or assessing consensus. These methods are regularly used in natural, social, and life sciences. Despite the significant role of consensus in the development of (academic) knowledge and understanding (e.g. Kuhn, 1962; Habermas, 1996; thus also Peels, 2020; Grimm et al., forthcoming ), formal expert consensus methods have hardly been explored within in the context of the humanities. [1] Other kinds of valuable expert meetings do take place in various disciplines in the humanities. They usually do not seek to reach a consensus, but to explore a topic in-depth by bringing the relevant experts to the table. Moreover, expert consensus methods described in this article differ from the expert meetings that already take place in the humanities by the former’s procedural and formalised character. In this paper, we aim to highlight to scholars, particularly those in the humanities, what is possible in terms of a systematic approach to academic consensus. In doing so, we aim to initiate exploration of and discussion about the potential of applying such an approach. What would be the added value of applying expert consensus meetings to humanities-related issues? To achieve this, we first explicate our method (§2), before we provide a framework for the application of expert consensus methods by introducing six key elements (§3). Subsequently, we present common types of expert consensus methods utilized in the natural, social, and life sciences. We highlight their and proven value across various types of research through relevant examples and their respective objectives (§4). We then consider both the potential benefits and possible limitations to applying these methods within humanistic research (§5), and conclude with suggestions for future research (§6). Reviewer 1, Comment 2: Second, the section on Method is poor. The composition of the interdisciplinary team (and the reasons for using this team) is not clearly described, the brainstorm session is not properly defined and illustrated, the ‘informal’ literature review is something new for me (what ‘informal’ means?) and it is not acceptable for a research paper to quote two methods used (‘philosophical methods of conceptual analysis’ and ‘conceptual engineering’) without explaining how they have been used for the purposes of the paper but just putting two references. Also, the process of writing and reviewing the manuscript should be better described and justified. Response: Our article, as we now emphasise more, is meant as an exploratory comment and a didactic contribution to inspire scholars in the humanities to see whether consensus methods may be valuable within their own discipline or sub discipline. It is not an instance of systematic empirical research, which would have required a much more detailed methodological section. But we certainly we agree that the method section can be improved in various ways. Therefore, we extended the section by including a specification of the backgrounds of the involved authors and specified the process of our collaboration as follows (see section 2, lines 85-94): To explore the potential of applying expert consensus methods in the humanities, we composed an interdisciplinary team of both researchers from the sciences using consensus methods in practice (LM: epidemiology; TH: qualitative research methodology; LB: epidemiology and data science, research integrity), and researchers familiar with the domain of the humanities and epistemology (CR: art history, museology; RP: epistemology and philosophy of science, religion and theology). In a first brainstorm session, knowledge on the topic was shared from both perspectives. This resulted in a list of consensus methods and examples used in the sciences, drafted by LM and TH. CR carried out a literature search in order to find examples thereof that were applied in her domain. A first draft was written and discussed in a second group session and culminated in a final version, which was edited and reviewed by all authors. Reviewer 1, Comment 3: Third, and more important, the paper refers always to ‘humanities.’ Humanities is a highly fragmented and differentiated field, where many and different epistemic communities work, interact, shape their own idea and uses of consensus method (if any) and where different instances of relationships with other researchers within the field and with neighboring fields are at place. The paper does not acknowledge this important characteristic of humanities (see footnote 1 where there is a list of disciplines included in the humanities that are considered in the paper). The same holds true for STEMs, which are considered as they are all adopting regularly expert consensus methods and there is no explanation or evidence about it. In sum, the paper ‘explores’ the expert consensus method without considering the set of ideas, values, theories, concepts and practices that pertain to the disciplines analyzed and their scientific work. Response: We acknowledge the concern raised and have taken this into account by adjusting a footnote 1 in section 1 (see above) and by revising section 4. Upon reflection we furthermore adjusted the ways to delineate the contexts in which expert consensus methods may be valuable, by focusing on one’s purpose in applying such a method (see section 4, lines 213-235): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 1, Comment 4: Fourth, sections 3 and 4 provide a clear summary of what expert consensus methods are and a deepen analysis of four types. Then the reader expects Section 5 discusses the results with respect to the exploration done. On the contrary, Section 5 lists of pros and cons related to the expert consensus method, and some examples of applications; the contents are completely unrelated with what the sections 3 and 4 have presented and the motivations for considering the expert consensus method an interesting development for humanities are unclear. Here the suggestion is to revise the sections trying to answer the research questions (when formulated) and highlight what we have learnt from the exploratory work, what is the contribution of the brainstorming to the exploration, what new avenues of research can be envisaged based on the results obtained, and what limitations/obstacles can the expert consensus model have in the humanities with respect to the other fields. Response: Thank you for pointing this out to us. Based on this valuable feedback, we have worked to better align section 5 with the preceding sections to create a more cohesive flow throughout the paper and have made more explicit what our line of reasoning is. However, because the paper is intended as preliminary exploration, part of our manuscript aims to identify the potential value of these methods for the humanities. Section 5 is dedicated to that purpose and we made that more explicit in the revision (see section 5, lines 253-315): In the preceding sections, we examined the elements and dynamics of expert consensus methods, along with examples and possibilities for intended objectives. In this section, we delve into the question why formal approaches to consensus in the humanities remain scarce, despite the significant factor of consensus and dissensus in knowledge production across research cultures. Aside from limitations, we also address what the potential added value of applying consensus methods to humanities-related issues could be. First of all, practical limitations likely play a role. Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Scholars may also disagree due to the role of research paradigms or schools within the humanities, as researchers’ positionality can shape their interpretations. In addition to that, humanities research can be concerned with hermeneutics, i.e., issues that have to do with understanding the values and meanings of texts and other objects. And many humanities fields such as ethics, epistemology, and metaphysics, rely on apriori methods like thought experiments, reasoning on the basis of intuitions and principles, and formal logic rather than empirical data. This allows for multiple interpretations that can all to some extent be warranted by the evidence base, and may not readily align into a singular consensus. These factors suggest that the complexity and interpretative nature of humanities research might render expert consensus methods to some extent less relevant. Nevertheless, we believe that there are compelling reasons for considering expert consensus methods in humanities research. First of all, opposing the above concerns, one of the important purposes of expert consensus methods is to bring various viewpoints, nuances, and interpretations to the surface, so that they can be discussed. Depending on the design and purpose of the consensus method, it could reveal levels of consensus and/or dissensus on an issue, within or across schools of thought or paradigms. They might even have the potential to articulate both divergences and common grounds regarding an issue, and by extension increase mutual understanding. The application of such a method may therefore stimulate epistemic progress in itself. Moreover, a carefully designed protocol and curated group of experts can foster a fair practice and ensure the consultation of diverse viewpoints. Expert consensus methods provide opportunities to mitigate biases and their ramifications by this cautious selection of experts, but also by blinding certain parts of the process in relation to anonymity of participating experts, for example. Secondly, expert consensus methods can be helpful in getting a firmer grip on the status quo or state of the art on a particular issue in a field (e.g. Hodgkin et al. (1975) mentioned in Table 2; Rulkens et al. 2022-2023, mentioned in Table 3). Of course, scholars often already have an idea about that, and such an informed opinion is valuable in itself. However, it can be complemented by a more structured approach, particularly when the number of scholars involved is large. This approach can additionally provide guidance for future research by pointing out the gaps in the current evidence base. And when applied on a recurrent basis, it might even have the potential to gain insight in how views evolve over time. It is important to note that consensus is not a final endpoint; rather, it can be continually re-evaluated in light of new insights and reasonable doubts regarding its validity. Third, an advantage of expert consensus methods is that they enhance the transparency and replicability of research because of their well-documented and systematised approach (for replication in the humanities, see Peels and Bouter (2018)). It can therefore be instrumental to epistemic progress beyond the consensus reached at a moment in time. Fourth, application or implementation of its outcomes is often easier after the process of a consensus method is completed (e.g. Standing Committee for the Museum Definition (2020-2022) (2022), mentioned in Table 3). And finally, expert consensus methods can aid public understanding by clarifying experts perspectives on specific issues (Stekelenburg et al., 2022) thereby enabling the public to make informed decisions (Ordway, 2021). In response to this comments, we furthermore extended suggestions for future research in our conclusion (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Response to reviewer 2: Daniel Leising, Technische Universitat Dresden, Dresden, Saxony, Germany Reviewer 2, Comment 1 : First, I would argue that the Humanities are actually not that “unique” in terms of their nature and approaches (page 11). To enable a proper discussion of this issue, the authors could explain their own view of what is so specific about the Humanities in a bit more detail. Here is my view: Interviews and text analyses are used in many different branches of science. Exploratory research is also conducted in many fields. The ways in which people give meaning to their experiences are also of interest across various branches of science. And even in many of the “harder” sciences, research is often “coloured by the backgrounds and positionality of the researchers” (page 8). So, much of what is common in research in the Humanities is actually shared with other fields. In my view, two features may actually distinguish “the Humanities” from other fields - but even these distinctions are only gradual in nature. One feature is the common practice of using everyday language as the primary scientific language. The other feature is the relative fuzziness of many concepts - which I would note is not necessarily a weakness but may often be unavoidable given the complexity of the phenomena of interest (and also given the first feature just mentioned). Do the authors agree? Response: Thank you for challenging us on this point. We agree that the borders between the general fields of the natural, social, and life sciences, and the humanities are fuzzy, when it comes to their objects, purposes, and methods. Whether there is something distinctive about the humanities – we believe that is compatible with fuzziness at the boarders – is a matter that we cannot possibly explore in this article (one of us does so in a book entitled A Philosophy of the Humanities that is forthcoming in 2025 with Oxford University Press, a volume co-authored by Stephen Grimm, Rik Peels, and René van Woudenberg). We do believe though that ordinary language use is not generally characteristic of the humanities (think of the highly technical language of metaphysics and epistemology), nor do we believe that insufficient clarity about core concepts is characteristic of the humanities. We agree though with the idea, also drawn to attention to by reviewer 1 in comment 3, that the humanities are a highly diversified field. We included that by adjusting footnote 1 in section 1: We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Additionally, to be able to demarcate between contexts in which the formal approach might be beneficial, we decided to put an emphasis on the purposes for which these methods are employed in other domains. This would be a way in which researchers with expertise in their fields could identify opportunities for a formal approach to consensus (see section 4, lines 213-235, also quoted above in response to comment 3 of reviewer 1): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 2, Comment 2: Second, I would like to encourage the authors to say a bit more about possible reasons for why so little consensus building is taking place in many fields of science, despite it being widely viewed as desirable. In my view, there are a couple of reasons, most of which are rather extraneous to an idealised image of science (as a collaborative and cumulative effort at building a reliable knowledge base for the good of mankind). For example, consensus building is likely to reduce the publication output of a field, which would be at odds with the interests of commercial publishers. Also, individual researchers have an interest in “making a name for themselves” based on their specific expertise or views regarding certain topics. Consensus building makes it more difficult to maintain a highly distinct profile and thus reduce visibility. This may be risky in a science system that is as competitive as the one we choose to entertain. And then there may be a need to “save face”: The more of your career you have dedicated to certain ideas (e.g., your own theory of some phenomenon), the harder it may be to let those ideas go - even if the evidence against them is overwhelming. There is a reason why Max Planck once stated that scientific progress happens “one funeral at a time”. At least a brief nod to the relevance of such more political factors would be nice - if the authors share my conviction that they are relevant. Who has an actual interest in achieving consensus? Who has an interest in avoiding the emergence of consensus? And what are the external incentives for and against this type of work ? Response: We think that these are valid points and the reviewers’ publications on the topic shed important light on this. We agree that external incentives may work against a formal approach to consensus as follows and have therefore slightly revised the manuscript on this point. We remain rather brief, though, as the exact explanation of why consensus methods are still relatively rarely used even in the sciences is an issue that is beyond the scope of our paper (see section 5, lines 259-264, also quoted in response to comment 4 of reviewer 1): Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Reviewer 2, Comment 3: Third, I would also appreciate if the authors could talk some more about “bad consensus” – views that are held by many, despite their being wrong. With a particular eye to the Humanities – what criteria are there, apart from consensus, that may help us determine whether a view is “correct”? Or may truth in the Humanities *only* be defined in terms of relative consensus among researchers? In the harder sciences, the ability to *predict* the occurrence of certain events from a theory (and some data input) plays a crucial role as evidence of the theory's correctness. Are the Humanities able to produce testable predictions, too? Can there be consensus projects in the Humanities that actually test one theory against another? Response: This is an interesting point, that would need a elaborative reflection in order to do justice to the complexity of this issue. It is important to note, that we do believe that consensus and truth are two different things, and we made this more clear in our revised paper. Based these suggestions we also incorporated that consensus can be reached based on the wrong reasons and (see section 5, lines 265-271, also quoted in response to comment 4 of reviewer 1): Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Reviewer 2, Comment 4: Fourth, on page 3 the authors state that “Since the questions addressed in the humanities may leave room for multiple viewpoints that are equally warranted by the evidence base, it can be infeasible or not desirable to get a complete or almost complete consensus.” To this I would respond: what are the conditions under which working toward consensus would not be feasible, or desirable? I agree that this may be the case – but when is it the case, and when is it not the case? I personally hold the somewhat sceptical view that many researchers will shy away from working toward consensus even if doing so *would* be both feasible and desirable (see my previous point). Such a “live and let live” attitude may often be relatively comfortable to have (e.g., to avoid conflict). It may, however, be regarded as unethical, because it will foster the preservation of an unnecessary level of ambiguity. Response: We appreciate this insightful suggestion. Our approach to expert consensus methods does not necessarily imply working towards consensus in all instances, but is also related to the assessment of the (lack of) consensus. We view measuring consensus as a means, not an objective in itself. Furthermore, as put forward in paragraph 3, the bar for ‘sufficient’ consensus is something consensus researchers can determine their selves. Applying a consensus methods is a costly endeavour in terms of time and means. There might be conditions under which the effort of applying such a method does not live up to the epistemic benefits it provides for. We furthermore believe that the feasibility and desirability of assessing consensus is an issue that merits further exploration and included this as follows (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Minor comments: Reviewer 2, Comment 5: Fifth, the authors state that a steering committee would typically be comprised of the people who initiated the process and who will ultimately author the consensus document. Could there not be cases in which the process should better be overseen by an independent third party (e.g., a moderator)? Especially when issues are highly contentious, for example? And should not all the experts who worked on the consensus be authors on the consensus document? Response: Thank you for this relevant point, this is certainly the case and we made this more clear in the revised manuscript by the following addition (see section 3, lines 193-198): Naturally, the six elements can be specified at by the steering committee. If they deem it valuable to include an independent moderator or to incorporate a member with a critical or divergent perspective—especially if the topic is highly controversial—this remains within their discretion. Additionally, it is up to them to decide who will be listed as co-authors, whether that includes only the steering committee, all participants, or a broader group representation such as the panellists as a whole, for instance. Reviewer 2, Comment 6: Sixth, something is not quite right on page 4, text lines 16-19. Response: Thank you for noticing, we have corrected this sentence as follows (see section 3, lines 136-140): Expert consensus methods are (i) applied by a process leader or steering committee , (ii) to reach consensus or assess the degree of consensus among a group of experts , (iii) about an issue , (iv) based on preset rules of engagement , (v) with the aim to deliver useful output , (vi) for future users . They are characterised by constructive and procedural usage of (dis)agreement and the different arguments that are provided therein. Reviewer 2, Comment 7: Seventh, I would say that evidence will never “speak for itself” (page 4) - otherwise papers in certain scientific fields would not need any Discussion sections. Response: Thank you for rightly pointing this out to us. We adjusted this as follows (see section 3, lines 132-134): This can be useful when the issue at hand is an important one and when evidence is lacking, it is inconsistent, or its interpretation is not obvious. As a result of our reactions to the comments, we also made adjustments to our abstract and conclusion, which are not all quoted in the above letter. We hope these revisions and clarifications meet the reviewers’ expectations. Please let us know if any further revisions are needed. Sincerely, Charlotte Rulkens MA Prof. dr. Rik Peels Dr. Lidwine Mokkink Dr. Tamarinde Haven Prof. em. dr. Lex Bouter [1] We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the time and effort you have put into evaluating our work and for providing insightful comments and valuable suggestions. We have carefully considered each of them and have revised the manuscript accordingly. Below, we provide detailed responses to each of the comments. We believe that the revisions have improved the manuscript, and we hope that our responses meet any remaining concerns. Response to reviewer 1: Emanuela Reale, CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy Reviewer 1, Comment 1: First, the aim of the paper is clear, but the research question(s) related to that aim are missing. This is an important limitation, since the manuscript reports an exploratory effort and therefore a delimitation of the research problem that you want to investigate is very useful to delineate the perimeter of the work. Response: We acknowledge the concern raised and have addressed this by rewriting the introduction section and by incorporating a clearer description of our study goal. We now make it more clear that our goal was primarily didactic, in the sense that we sought to explore the usefulness of consensus methods utilised in the natural, social, and life sciences for the humanities (see section 1, lines 59-83): An important way to make progress in knowledge and understanding is to gradually, via the exchange of evidence, expertise, and critical discussion, reach (more) consensus on a topic. However, whether there is consensus on a certain matter at a moment in time can sometimes remain unknown, be unclear, or remain implicit. Expert consensus methods can lift off this inconclusiveness by intentionally and formally pursuing or assessing consensus. These methods are regularly used in natural, social, and life sciences. Despite the significant role of consensus in the development of (academic) knowledge and understanding (e.g. Kuhn, 1962; Habermas, 1996; thus also Peels, 2020; Grimm et al., forthcoming ), formal expert consensus methods have hardly been explored within in the context of the humanities. [1] Other kinds of valuable expert meetings do take place in various disciplines in the humanities. They usually do not seek to reach a consensus, but to explore a topic in-depth by bringing the relevant experts to the table. Moreover, expert consensus methods described in this article differ from the expert meetings that already take place in the humanities by the former’s procedural and formalised character. In this paper, we aim to highlight to scholars, particularly those in the humanities, what is possible in terms of a systematic approach to academic consensus. In doing so, we aim to initiate exploration of and discussion about the potential of applying such an approach. What would be the added value of applying expert consensus meetings to humanities-related issues? To achieve this, we first explicate our method (§2), before we provide a framework for the application of expert consensus methods by introducing six key elements (§3). Subsequently, we present common types of expert consensus methods utilized in the natural, social, and life sciences. We highlight their and proven value across various types of research through relevant examples and their respective objectives (§4). We then consider both the potential benefits and possible limitations to applying these methods within humanistic research (§5), and conclude with suggestions for future research (§6). Reviewer 1, Comment 2: Second, the section on Method is poor. The composition of the interdisciplinary team (and the reasons for using this team) is not clearly described, the brainstorm session is not properly defined and illustrated, the ‘informal’ literature review is something new for me (what ‘informal’ means?) and it is not acceptable for a research paper to quote two methods used (‘philosophical methods of conceptual analysis’ and ‘conceptual engineering’) without explaining how they have been used for the purposes of the paper but just putting two references. Also, the process of writing and reviewing the manuscript should be better described and justified. Response: Our article, as we now emphasise more, is meant as an exploratory comment and a didactic contribution to inspire scholars in the humanities to see whether consensus methods may be valuable within their own discipline or sub discipline. It is not an instance of systematic empirical research, which would have required a much more detailed methodological section. But we certainly we agree that the method section can be improved in various ways. Therefore, we extended the section by including a specification of the backgrounds of the involved authors and specified the process of our collaboration as follows (see section 2, lines 85-94): To explore the potential of applying expert consensus methods in the humanities, we composed an interdisciplinary team of both researchers from the sciences using consensus methods in practice (LM: epidemiology; TH: qualitative research methodology; LB: epidemiology and data science, research integrity), and researchers familiar with the domain of the humanities and epistemology (CR: art history, museology; RP: epistemology and philosophy of science, religion and theology). In a first brainstorm session, knowledge on the topic was shared from both perspectives. This resulted in a list of consensus methods and examples used in the sciences, drafted by LM and TH. CR carried out a literature search in order to find examples thereof that were applied in her domain. A first draft was written and discussed in a second group session and culminated in a final version, which was edited and reviewed by all authors. Reviewer 1, Comment 3: Third, and more important, the paper refers always to ‘humanities.’ Humanities is a highly fragmented and differentiated field, where many and different epistemic communities work, interact, shape their own idea and uses of consensus method (if any) and where different instances of relationships with other researchers within the field and with neighboring fields are at place. The paper does not acknowledge this important characteristic of humanities (see footnote 1 where there is a list of disciplines included in the humanities that are considered in the paper). The same holds true for STEMs, which are considered as they are all adopting regularly expert consensus methods and there is no explanation or evidence about it. In sum, the paper ‘explores’ the expert consensus method without considering the set of ideas, values, theories, concepts and practices that pertain to the disciplines analyzed and their scientific work. Response: We acknowledge the concern raised and have taken this into account by adjusting a footnote 1 in section 1 (see above) and by revising section 4. Upon reflection we furthermore adjusted the ways to delineate the contexts in which expert consensus methods may be valuable, by focusing on one’s purpose in applying such a method (see section 4, lines 213-235): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 1, Comment 4: Fourth, sections 3 and 4 provide a clear summary of what expert consensus methods are and a deepen analysis of four types. Then the reader expects Section 5 discusses the results with respect to the exploration done. On the contrary, Section 5 lists of pros and cons related to the expert consensus method, and some examples of applications; the contents are completely unrelated with what the sections 3 and 4 have presented and the motivations for considering the expert consensus method an interesting development for humanities are unclear. Here the suggestion is to revise the sections trying to answer the research questions (when formulated) and highlight what we have learnt from the exploratory work, what is the contribution of the brainstorming to the exploration, what new avenues of research can be envisaged based on the results obtained, and what limitations/obstacles can the expert consensus model have in the humanities with respect to the other fields. Response: Thank you for pointing this out to us. Based on this valuable feedback, we have worked to better align section 5 with the preceding sections to create a more cohesive flow throughout the paper and have made more explicit what our line of reasoning is. However, because the paper is intended as preliminary exploration, part of our manuscript aims to identify the potential value of these methods for the humanities. Section 5 is dedicated to that purpose and we made that more explicit in the revision (see section 5, lines 253-315): In the preceding sections, we examined the elements and dynamics of expert consensus methods, along with examples and possibilities for intended objectives. In this section, we delve into the question why formal approaches to consensus in the humanities remain scarce, despite the significant factor of consensus and dissensus in knowledge production across research cultures. Aside from limitations, we also address what the potential added value of applying consensus methods to humanities-related issues could be. First of all, practical limitations likely play a role. Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Scholars may also disagree due to the role of research paradigms or schools within the humanities, as researchers’ positionality can shape their interpretations. In addition to that, humanities research can be concerned with hermeneutics, i.e., issues that have to do with understanding the values and meanings of texts and other objects. And many humanities fields such as ethics, epistemology, and metaphysics, rely on apriori methods like thought experiments, reasoning on the basis of intuitions and principles, and formal logic rather than empirical data. This allows for multiple interpretations that can all to some extent be warranted by the evidence base, and may not readily align into a singular consensus. These factors suggest that the complexity and interpretative nature of humanities research might render expert consensus methods to some extent less relevant. Nevertheless, we believe that there are compelling reasons for considering expert consensus methods in humanities research. First of all, opposing the above concerns, one of the important purposes of expert consensus methods is to bring various viewpoints, nuances, and interpretations to the surface, so that they can be discussed. Depending on the design and purpose of the consensus method, it could reveal levels of consensus and/or dissensus on an issue, within or across schools of thought or paradigms. They might even have the potential to articulate both divergences and common grounds regarding an issue, and by extension increase mutual understanding. The application of such a method may therefore stimulate epistemic progress in itself. Moreover, a carefully designed protocol and curated group of experts can foster a fair practice and ensure the consultation of diverse viewpoints. Expert consensus methods provide opportunities to mitigate biases and their ramifications by this cautious selection of experts, but also by blinding certain parts of the process in relation to anonymity of participating experts, for example. Secondly, expert consensus methods can be helpful in getting a firmer grip on the status quo or state of the art on a particular issue in a field (e.g. Hodgkin et al. (1975) mentioned in Table 2; Rulkens et al. 2022-2023, mentioned in Table 3). Of course, scholars often already have an idea about that, and such an informed opinion is valuable in itself. However, it can be complemented by a more structured approach, particularly when the number of scholars involved is large. This approach can additionally provide guidance for future research by pointing out the gaps in the current evidence base. And when applied on a recurrent basis, it might even have the potential to gain insight in how views evolve over time. It is important to note that consensus is not a final endpoint; rather, it can be continually re-evaluated in light of new insights and reasonable doubts regarding its validity. Third, an advantage of expert consensus methods is that they enhance the transparency and replicability of research because of their well-documented and systematised approach (for replication in the humanities, see Peels and Bouter (2018)). It can therefore be instrumental to epistemic progress beyond the consensus reached at a moment in time. Fourth, application or implementation of its outcomes is often easier after the process of a consensus method is completed (e.g. Standing Committee for the Museum Definition (2020-2022) (2022), mentioned in Table 3). And finally, expert consensus methods can aid public understanding by clarifying experts perspectives on specific issues (Stekelenburg et al., 2022) thereby enabling the public to make informed decisions (Ordway, 2021). In response to this comments, we furthermore extended suggestions for future research in our conclusion (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Response to reviewer 2: Daniel Leising, Technische Universitat Dresden, Dresden, Saxony, Germany Reviewer 2, Comment 1 : First, I would argue that the Humanities are actually not that “unique” in terms of their nature and approaches (page 11). To enable a proper discussion of this issue, the authors could explain their own view of what is so specific about the Humanities in a bit more detail. Here is my view: Interviews and text analyses are used in many different branches of science. Exploratory research is also conducted in many fields. The ways in which people give meaning to their experiences are also of interest across various branches of science. And even in many of the “harder” sciences, research is often “coloured by the backgrounds and positionality of the researchers” (page 8). So, much of what is common in research in the Humanities is actually shared with other fields. In my view, two features may actually distinguish “the Humanities” from other fields - but even these distinctions are only gradual in nature. One feature is the common practice of using everyday language as the primary scientific language. The other feature is the relative fuzziness of many concepts - which I would note is not necessarily a weakness but may often be unavoidable given the complexity of the phenomena of interest (and also given the first feature just mentioned). Do the authors agree? Response: Thank you for challenging us on this point. We agree that the borders between the general fields of the natural, social, and life sciences, and the humanities are fuzzy, when it comes to their objects, purposes, and methods. Whether there is something distinctive about the humanities – we believe that is compatible with fuzziness at the boarders – is a matter that we cannot possibly explore in this article (one of us does so in a book entitled A Philosophy of the Humanities that is forthcoming in 2025 with Oxford University Press, a volume co-authored by Stephen Grimm, Rik Peels, and René van Woudenberg). We do believe though that ordinary language use is not generally characteristic of the humanities (think of the highly technical language of metaphysics and epistemology), nor do we believe that insufficient clarity about core concepts is characteristic of the humanities. We agree though with the idea, also drawn to attention to by reviewer 1 in comment 3, that the humanities are a highly diversified field. We included that by adjusting footnote 1 in section 1: We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Additionally, to be able to demarcate between contexts in which the formal approach might be beneficial, we decided to put an emphasis on the purposes for which these methods are employed in other domains. This would be a way in which researchers with expertise in their fields could identify opportunities for a formal approach to consensus (see section 4, lines 213-235, also quoted above in response to comment 3 of reviewer 1): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 2, Comment 2: Second, I would like to encourage the authors to say a bit more about possible reasons for why so little consensus building is taking place in many fields of science, despite it being widely viewed as desirable. In my view, there are a couple of reasons, most of which are rather extraneous to an idealised image of science (as a collaborative and cumulative effort at building a reliable knowledge base for the good of mankind). For example, consensus building is likely to reduce the publication output of a field, which would be at odds with the interests of commercial publishers. Also, individual researchers have an interest in “making a name for themselves” based on their specific expertise or views regarding certain topics. Consensus building makes it more difficult to maintain a highly distinct profile and thus reduce visibility. This may be risky in a science system that is as competitive as the one we choose to entertain. And then there may be a need to “save face”: The more of your career you have dedicated to certain ideas (e.g., your own theory of some phenomenon), the harder it may be to let those ideas go - even if the evidence against them is overwhelming. There is a reason why Max Planck once stated that scientific progress happens “one funeral at a time”. At least a brief nod to the relevance of such more political factors would be nice - if the authors share my conviction that they are relevant. Who has an actual interest in achieving consensus? Who has an interest in avoiding the emergence of consensus? And what are the external incentives for and against this type of work ? Response: We think that these are valid points and the reviewers’ publications on the topic shed important light on this. We agree that external incentives may work against a formal approach to consensus as follows and have therefore slightly revised the manuscript on this point. We remain rather brief, though, as the exact explanation of why consensus methods are still relatively rarely used even in the sciences is an issue that is beyond the scope of our paper (see section 5, lines 259-264, also quoted in response to comment 4 of reviewer 1): Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Reviewer 2, Comment 3: Third, I would also appreciate if the authors could talk some more about “bad consensus” – views that are held by many, despite their being wrong. With a particular eye to the Humanities – what criteria are there, apart from consensus, that may help us determine whether a view is “correct”? Or may truth in the Humanities *only* be defined in terms of relative consensus among researchers? In the harder sciences, the ability to *predict* the occurrence of certain events from a theory (and some data input) plays a crucial role as evidence of the theory's correctness. Are the Humanities able to produce testable predictions, too? Can there be consensus projects in the Humanities that actually test one theory against another? Response: This is an interesting point, that would need a elaborative reflection in order to do justice to the complexity of this issue. It is important to note, that we do believe that consensus and truth are two different things, and we made this more clear in our revised paper. Based these suggestions we also incorporated that consensus can be reached based on the wrong reasons and (see section 5, lines 265-271, also quoted in response to comment 4 of reviewer 1): Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Reviewer 2, Comment 4: Fourth, on page 3 the authors state that “Since the questions addressed in the humanities may leave room for multiple viewpoints that are equally warranted by the evidence base, it can be infeasible or not desirable to get a complete or almost complete consensus.” To this I would respond: what are the conditions under which working toward consensus would not be feasible, or desirable? I agree that this may be the case – but when is it the case, and when is it not the case? I personally hold the somewhat sceptical view that many researchers will shy away from working toward consensus even if doing so *would* be both feasible and desirable (see my previous point). Such a “live and let live” attitude may often be relatively comfortable to have (e.g., to avoid conflict). It may, however, be regarded as unethical, because it will foster the preservation of an unnecessary level of ambiguity. Response: We appreciate this insightful suggestion. Our approach to expert consensus methods does not necessarily imply working towards consensus in all instances, but is also related to the assessment of the (lack of) consensus. We view measuring consensus as a means, not an objective in itself. Furthermore, as put forward in paragraph 3, the bar for ‘sufficient’ consensus is something consensus researchers can determine their selves. Applying a consensus methods is a costly endeavour in terms of time and means. There might be conditions under which the effort of applying such a method does not live up to the epistemic benefits it provides for. We furthermore believe that the feasibility and desirability of assessing consensus is an issue that merits further exploration and included this as follows (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Minor comments: Reviewer 2, Comment 5: Fifth, the authors state that a steering committee would typically be comprised of the people who initiated the process and who will ultimately author the consensus document. Could there not be cases in which the process should better be overseen by an independent third party (e.g., a moderator)? Especially when issues are highly contentious, for example? And should not all the experts who worked on the consensus be authors on the consensus document? Response: Thank you for this relevant point, this is certainly the case and we made this more clear in the revised manuscript by the following addition (see section 3, lines 193-198): Naturally, the six elements can be specified at by the steering committee. If they deem it valuable to include an independent moderator or to incorporate a member with a critical or divergent perspective—especially if the topic is highly controversial—this remains within their discretion. Additionally, it is up to them to decide who will be listed as co-authors, whether that includes only the steering committee, all participants, or a broader group representation such as the panellists as a whole, for instance. Reviewer 2, Comment 6: Sixth, something is not quite right on page 4, text lines 16-19. Response: Thank you for noticing, we have corrected this sentence as follows (see section 3, lines 136-140): Expert consensus methods are (i) applied by a process leader or steering committee , (ii) to reach consensus or assess the degree of consensus among a group of experts , (iii) about an issue , (iv) based on preset rules of engagement , (v) with the aim to deliver useful output , (vi) for future users . They are characterised by constructive and procedural usage of (dis)agreement and the different arguments that are provided therein. Reviewer 2, Comment 7: Seventh, I would say that evidence will never “speak for itself” (page 4) - otherwise papers in certain scientific fields would not need any Discussion sections. Response: Thank you for rightly pointing this out to us. We adjusted this as follows (see section 3, lines 132-134): This can be useful when the issue at hand is an important one and when evidence is lacking, it is inconsistent, or its interpretation is not obvious. As a result of our reactions to the comments, we also made adjustments to our abstract and conclusion, which are not all quoted in the above letter. We hope these revisions and clarifications meet the reviewers’ expectations. Please let us know if any further revisions are needed. Sincerely, Charlotte Rulkens MA Prof. dr. Rik Peels Dr. Lidwine Mokkink Dr. Tamarinde Haven Prof. em. dr. Lex Bouter [1] We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 16 Dec 2024 Charlotte Rulkens , Department of Philosophy, Vrije Universiteit Amsterdam, Amsterdam, The Netherlands 16 Dec 2024 Author Response We sincerely thank the reviewer for her valuable comments, all of which we will address in the revision of our paper. We plan to begin this revision as soon as ... Continue reading We sincerely thank the reviewer for her valuable comments, all of which we will address in the revision of our paper. We plan to begin this revision as soon as we receive the second review. We agree with the first and second suggestions to clarify our research questions, as well as the opportunities mentioned for improving the method section. While our method section may differ from those typically found in research papers, it is important to note that this paper is intended as a perspective piece with a didactic focus. It is aimed at highlighting to humanities scholars what is possible in terms of a systematic approach to academic ​​consensus on a topic and to initiate exploration of and discussion about the possible value of applying such an approach. Regarding the third comment, we will ensure that we address the issue of differentiation within research fields and discuss its implications for the application of consensus methods. Concerning the fourth point, we will work to better integrate section 5 with the preceding sections to create a more cohesive flow throughout the paper. We sincerely thank the reviewer for her valuable comments, all of which we will address in the revision of our paper. We plan to begin this revision as soon as we receive the second review. We agree with the first and second suggestions to clarify our research questions, as well as the opportunities mentioned for improving the method section. While our method section may differ from those typically found in research papers, it is important to note that this paper is intended as a perspective piece with a didactic focus. It is aimed at highlighting to humanities scholars what is possible in terms of a systematic approach to academic ​​consensus on a topic and to initiate exploration of and discussion about the possible value of applying such an approach. Regarding the third comment, we will ensure that we address the issue of differentiation within research fields and discuss its implications for the application of consensus methods. Concerning the fourth point, we will work to better integrate section 5 with the preceding sections to create a more cohesive flow throughout the paper. Competing Interests: No competing interests were disclosed. Close Report a concern Author Response 16 Dec 2024 Charlotte Rulkens , Department of Philosophy, Vrije Universiteit Amsterdam, Amsterdam, The Netherlands 16 Dec 2024 Author Response Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the ... Continue reading Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the time and effort you have put into evaluating our work and for providing insightful comments and valuable suggestions. We have carefully considered each of them and have revised the manuscript accordingly. Below, we provide detailed responses to each of the comments. We believe that the revisions have improved the manuscript, and we hope that our responses meet any remaining concerns. Response to reviewer 1: Emanuela Reale, CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy Reviewer 1, Comment 1: First, the aim of the paper is clear, but the research question(s) related to that aim are missing. This is an important limitation, since the manuscript reports an exploratory effort and therefore a delimitation of the research problem that you want to investigate is very useful to delineate the perimeter of the work. Response: We acknowledge the concern raised and have addressed this by rewriting the introduction section and by incorporating a clearer description of our study goal. We now make it more clear that our goal was primarily didactic, in the sense that we sought to explore the usefulness of consensus methods utilised in the natural, social, and life sciences for the humanities (see section 1, lines 59-83): An important way to make progress in knowledge and understanding is to gradually, via the exchange of evidence, expertise, and critical discussion, reach (more) consensus on a topic. However, whether there is consensus on a certain matter at a moment in time can sometimes remain unknown, be unclear, or remain implicit. Expert consensus methods can lift off this inconclusiveness by intentionally and formally pursuing or assessing consensus. These methods are regularly used in natural, social, and life sciences. Despite the significant role of consensus in the development of (academic) knowledge and understanding (e.g. Kuhn, 1962; Habermas, 1996; thus also Peels, 2020; Grimm et al., forthcoming ), formal expert consensus methods have hardly been explored within in the context of the humanities. [1] Other kinds of valuable expert meetings do take place in various disciplines in the humanities. They usually do not seek to reach a consensus, but to explore a topic in-depth by bringing the relevant experts to the table. Moreover, expert consensus methods described in this article differ from the expert meetings that already take place in the humanities by the former’s procedural and formalised character. In this paper, we aim to highlight to scholars, particularly those in the humanities, what is possible in terms of a systematic approach to academic consensus. In doing so, we aim to initiate exploration of and discussion about the potential of applying such an approach. What would be the added value of applying expert consensus meetings to humanities-related issues? To achieve this, we first explicate our method (§2), before we provide a framework for the application of expert consensus methods by introducing six key elements (§3). Subsequently, we present common types of expert consensus methods utilized in the natural, social, and life sciences. We highlight their and proven value across various types of research through relevant examples and their respective objectives (§4). We then consider both the potential benefits and possible limitations to applying these methods within humanistic research (§5), and conclude with suggestions for future research (§6). Reviewer 1, Comment 2: Second, the section on Method is poor. The composition of the interdisciplinary team (and the reasons for using this team) is not clearly described, the brainstorm session is not properly defined and illustrated, the ‘informal’ literature review is something new for me (what ‘informal’ means?) and it is not acceptable for a research paper to quote two methods used (‘philosophical methods of conceptual analysis’ and ‘conceptual engineering’) without explaining how they have been used for the purposes of the paper but just putting two references. Also, the process of writing and reviewing the manuscript should be better described and justified. Response: Our article, as we now emphasise more, is meant as an exploratory comment and a didactic contribution to inspire scholars in the humanities to see whether consensus methods may be valuable within their own discipline or sub discipline. It is not an instance of systematic empirical research, which would have required a much more detailed methodological section. But we certainly we agree that the method section can be improved in various ways. Therefore, we extended the section by including a specification of the backgrounds of the involved authors and specified the process of our collaboration as follows (see section 2, lines 85-94): To explore the potential of applying expert consensus methods in the humanities, we composed an interdisciplinary team of both researchers from the sciences using consensus methods in practice (LM: epidemiology; TH: qualitative research methodology; LB: epidemiology and data science, research integrity), and researchers familiar with the domain of the humanities and epistemology (CR: art history, museology; RP: epistemology and philosophy of science, religion and theology). In a first brainstorm session, knowledge on the topic was shared from both perspectives. This resulted in a list of consensus methods and examples used in the sciences, drafted by LM and TH. CR carried out a literature search in order to find examples thereof that were applied in her domain. A first draft was written and discussed in a second group session and culminated in a final version, which was edited and reviewed by all authors. Reviewer 1, Comment 3: Third, and more important, the paper refers always to ‘humanities.’ Humanities is a highly fragmented and differentiated field, where many and different epistemic communities work, interact, shape their own idea and uses of consensus method (if any) and where different instances of relationships with other researchers within the field and with neighboring fields are at place. The paper does not acknowledge this important characteristic of humanities (see footnote 1 where there is a list of disciplines included in the humanities that are considered in the paper). The same holds true for STEMs, which are considered as they are all adopting regularly expert consensus methods and there is no explanation or evidence about it. In sum, the paper ‘explores’ the expert consensus method without considering the set of ideas, values, theories, concepts and practices that pertain to the disciplines analyzed and their scientific work. Response: We acknowledge the concern raised and have taken this into account by adjusting a footnote 1 in section 1 (see above) and by revising section 4. Upon reflection we furthermore adjusted the ways to delineate the contexts in which expert consensus methods may be valuable, by focusing on one’s purpose in applying such a method (see section 4, lines 213-235): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 1, Comment 4: Fourth, sections 3 and 4 provide a clear summary of what expert consensus methods are and a deepen analysis of four types. Then the reader expects Section 5 discusses the results with respect to the exploration done. On the contrary, Section 5 lists of pros and cons related to the expert consensus method, and some examples of applications; the contents are completely unrelated with what the sections 3 and 4 have presented and the motivations for considering the expert consensus method an interesting development for humanities are unclear. Here the suggestion is to revise the sections trying to answer the research questions (when formulated) and highlight what we have learnt from the exploratory work, what is the contribution of the brainstorming to the exploration, what new avenues of research can be envisaged based on the results obtained, and what limitations/obstacles can the expert consensus model have in the humanities with respect to the other fields. Response: Thank you for pointing this out to us. Based on this valuable feedback, we have worked to better align section 5 with the preceding sections to create a more cohesive flow throughout the paper and have made more explicit what our line of reasoning is. However, because the paper is intended as preliminary exploration, part of our manuscript aims to identify the potential value of these methods for the humanities. Section 5 is dedicated to that purpose and we made that more explicit in the revision (see section 5, lines 253-315): In the preceding sections, we examined the elements and dynamics of expert consensus methods, along with examples and possibilities for intended objectives. In this section, we delve into the question why formal approaches to consensus in the humanities remain scarce, despite the significant factor of consensus and dissensus in knowledge production across research cultures. Aside from limitations, we also address what the potential added value of applying consensus methods to humanities-related issues could be. First of all, practical limitations likely play a role. Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Scholars may also disagree due to the role of research paradigms or schools within the humanities, as researchers’ positionality can shape their interpretations. In addition to that, humanities research can be concerned with hermeneutics, i.e., issues that have to do with understanding the values and meanings of texts and other objects. And many humanities fields such as ethics, epistemology, and metaphysics, rely on apriori methods like thought experiments, reasoning on the basis of intuitions and principles, and formal logic rather than empirical data. This allows for multiple interpretations that can all to some extent be warranted by the evidence base, and may not readily align into a singular consensus. These factors suggest that the complexity and interpretative nature of humanities research might render expert consensus methods to some extent less relevant. Nevertheless, we believe that there are compelling reasons for considering expert consensus methods in humanities research. First of all, opposing the above concerns, one of the important purposes of expert consensus methods is to bring various viewpoints, nuances, and interpretations to the surface, so that they can be discussed. Depending on the design and purpose of the consensus method, it could reveal levels of consensus and/or dissensus on an issue, within or across schools of thought or paradigms. They might even have the potential to articulate both divergences and common grounds regarding an issue, and by extension increase mutual understanding. The application of such a method may therefore stimulate epistemic progress in itself. Moreover, a carefully designed protocol and curated group of experts can foster a fair practice and ensure the consultation of diverse viewpoints. Expert consensus methods provide opportunities to mitigate biases and their ramifications by this cautious selection of experts, but also by blinding certain parts of the process in relation to anonymity of participating experts, for example. Secondly, expert consensus methods can be helpful in getting a firmer grip on the status quo or state of the art on a particular issue in a field (e.g. Hodgkin et al. (1975) mentioned in Table 2; Rulkens et al. 2022-2023, mentioned in Table 3). Of course, scholars often already have an idea about that, and such an informed opinion is valuable in itself. However, it can be complemented by a more structured approach, particularly when the number of scholars involved is large. This approach can additionally provide guidance for future research by pointing out the gaps in the current evidence base. And when applied on a recurrent basis, it might even have the potential to gain insight in how views evolve over time. It is important to note that consensus is not a final endpoint; rather, it can be continually re-evaluated in light of new insights and reasonable doubts regarding its validity. Third, an advantage of expert consensus methods is that they enhance the transparency and replicability of research because of their well-documented and systematised approach (for replication in the humanities, see Peels and Bouter (2018)). It can therefore be instrumental to epistemic progress beyond the consensus reached at a moment in time. Fourth, application or implementation of its outcomes is often easier after the process of a consensus method is completed (e.g. Standing Committee for the Museum Definition (2020-2022) (2022), mentioned in Table 3). And finally, expert consensus methods can aid public understanding by clarifying experts perspectives on specific issues (Stekelenburg et al., 2022) thereby enabling the public to make informed decisions (Ordway, 2021). In response to this comments, we furthermore extended suggestions for future research in our conclusion (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Response to reviewer 2: Daniel Leising, Technische Universitat Dresden, Dresden, Saxony, Germany Reviewer 2, Comment 1 : First, I would argue that the Humanities are actually not that “unique” in terms of their nature and approaches (page 11). To enable a proper discussion of this issue, the authors could explain their own view of what is so specific about the Humanities in a bit more detail. Here is my view: Interviews and text analyses are used in many different branches of science. Exploratory research is also conducted in many fields. The ways in which people give meaning to their experiences are also of interest across various branches of science. And even in many of the “harder” sciences, research is often “coloured by the backgrounds and positionality of the researchers” (page 8). So, much of what is common in research in the Humanities is actually shared with other fields. In my view, two features may actually distinguish “the Humanities” from other fields - but even these distinctions are only gradual in nature. One feature is the common practice of using everyday language as the primary scientific language. The other feature is the relative fuzziness of many concepts - which I would note is not necessarily a weakness but may often be unavoidable given the complexity of the phenomena of interest (and also given the first feature just mentioned). Do the authors agree? Response: Thank you for challenging us on this point. We agree that the borders between the general fields of the natural, social, and life sciences, and the humanities are fuzzy, when it comes to their objects, purposes, and methods. Whether there is something distinctive about the humanities – we believe that is compatible with fuzziness at the boarders – is a matter that we cannot possibly explore in this article (one of us does so in a book entitled A Philosophy of the Humanities that is forthcoming in 2025 with Oxford University Press, a volume co-authored by Stephen Grimm, Rik Peels, and René van Woudenberg). We do believe though that ordinary language use is not generally characteristic of the humanities (think of the highly technical language of metaphysics and epistemology), nor do we believe that insufficient clarity about core concepts is characteristic of the humanities. We agree though with the idea, also drawn to attention to by reviewer 1 in comment 3, that the humanities are a highly diversified field. We included that by adjusting footnote 1 in section 1: We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Additionally, to be able to demarcate between contexts in which the formal approach might be beneficial, we decided to put an emphasis on the purposes for which these methods are employed in other domains. This would be a way in which researchers with expertise in their fields could identify opportunities for a formal approach to consensus (see section 4, lines 213-235, also quoted above in response to comment 3 of reviewer 1): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 2, Comment 2: Second, I would like to encourage the authors to say a bit more about possible reasons for why so little consensus building is taking place in many fields of science, despite it being widely viewed as desirable. In my view, there are a couple of reasons, most of which are rather extraneous to an idealised image of science (as a collaborative and cumulative effort at building a reliable knowledge base for the good of mankind). For example, consensus building is likely to reduce the publication output of a field, which would be at odds with the interests of commercial publishers. Also, individual researchers have an interest in “making a name for themselves” based on their specific expertise or views regarding certain topics. Consensus building makes it more difficult to maintain a highly distinct profile and thus reduce visibility. This may be risky in a science system that is as competitive as the one we choose to entertain. And then there may be a need to “save face”: The more of your career you have dedicated to certain ideas (e.g., your own theory of some phenomenon), the harder it may be to let those ideas go - even if the evidence against them is overwhelming. There is a reason why Max Planck once stated that scientific progress happens “one funeral at a time”. At least a brief nod to the relevance of such more political factors would be nice - if the authors share my conviction that they are relevant. Who has an actual interest in achieving consensus? Who has an interest in avoiding the emergence of consensus? And what are the external incentives for and against this type of work ? Response: We think that these are valid points and the reviewers’ publications on the topic shed important light on this. We agree that external incentives may work against a formal approach to consensus as follows and have therefore slightly revised the manuscript on this point. We remain rather brief, though, as the exact explanation of why consensus methods are still relatively rarely used even in the sciences is an issue that is beyond the scope of our paper (see section 5, lines 259-264, also quoted in response to comment 4 of reviewer 1): Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Reviewer 2, Comment 3: Third, I would also appreciate if the authors could talk some more about “bad consensus” – views that are held by many, despite their being wrong. With a particular eye to the Humanities – what criteria are there, apart from consensus, that may help us determine whether a view is “correct”? Or may truth in the Humanities *only* be defined in terms of relative consensus among researchers? In the harder sciences, the ability to *predict* the occurrence of certain events from a theory (and some data input) plays a crucial role as evidence of the theory's correctness. Are the Humanities able to produce testable predictions, too? Can there be consensus projects in the Humanities that actually test one theory against another? Response: This is an interesting point, that would need a elaborative reflection in order to do justice to the complexity of this issue. It is important to note, that we do believe that consensus and truth are two different things, and we made this more clear in our revised paper. Based these suggestions we also incorporated that consensus can be reached based on the wrong reasons and (see section 5, lines 265-271, also quoted in response to comment 4 of reviewer 1): Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Reviewer 2, Comment 4: Fourth, on page 3 the authors state that “Since the questions addressed in the humanities may leave room for multiple viewpoints that are equally warranted by the evidence base, it can be infeasible or not desirable to get a complete or almost complete consensus.” To this I would respond: what are the conditions under which working toward consensus would not be feasible, or desirable? I agree that this may be the case – but when is it the case, and when is it not the case? I personally hold the somewhat sceptical view that many researchers will shy away from working toward consensus even if doing so *would* be both feasible and desirable (see my previous point). Such a “live and let live” attitude may often be relatively comfortable to have (e.g., to avoid conflict). It may, however, be regarded as unethical, because it will foster the preservation of an unnecessary level of ambiguity. Response: We appreciate this insightful suggestion. Our approach to expert consensus methods does not necessarily imply working towards consensus in all instances, but is also related to the assessment of the (lack of) consensus. We view measuring consensus as a means, not an objective in itself. Furthermore, as put forward in paragraph 3, the bar for ‘sufficient’ consensus is something consensus researchers can determine their selves. Applying a consensus methods is a costly endeavour in terms of time and means. There might be conditions under which the effort of applying such a method does not live up to the epistemic benefits it provides for. We furthermore believe that the feasibility and desirability of assessing consensus is an issue that merits further exploration and included this as follows (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Minor comments: Reviewer 2, Comment 5: Fifth, the authors state that a steering committee would typically be comprised of the people who initiated the process and who will ultimately author the consensus document. Could there not be cases in which the process should better be overseen by an independent third party (e.g., a moderator)? Especially when issues are highly contentious, for example? And should not all the experts who worked on the consensus be authors on the consensus document? Response: Thank you for this relevant point, this is certainly the case and we made this more clear in the revised manuscript by the following addition (see section 3, lines 193-198): Naturally, the six elements can be specified at by the steering committee. If they deem it valuable to include an independent moderator or to incorporate a member with a critical or divergent perspective—especially if the topic is highly controversial—this remains within their discretion. Additionally, it is up to them to decide who will be listed as co-authors, whether that includes only the steering committee, all participants, or a broader group representation such as the panellists as a whole, for instance. Reviewer 2, Comment 6: Sixth, something is not quite right on page 4, text lines 16-19. Response: Thank you for noticing, we have corrected this sentence as follows (see section 3, lines 136-140): Expert consensus methods are (i) applied by a process leader or steering committee , (ii) to reach consensus or assess the degree of consensus among a group of experts , (iii) about an issue , (iv) based on preset rules of engagement , (v) with the aim to deliver useful output , (vi) for future users . They are characterised by constructive and procedural usage of (dis)agreement and the different arguments that are provided therein. Reviewer 2, Comment 7: Seventh, I would say that evidence will never “speak for itself” (page 4) - otherwise papers in certain scientific fields would not need any Discussion sections. Response: Thank you for rightly pointing this out to us. We adjusted this as follows (see section 3, lines 132-134): This can be useful when the issue at hand is an important one and when evidence is lacking, it is inconsistent, or its interpretation is not obvious. As a result of our reactions to the comments, we also made adjustments to our abstract and conclusion, which are not all quoted in the above letter. We hope these revisions and clarifications meet the reviewers’ expectations. Please let us know if any further revisions are needed. Sincerely, Charlotte Rulkens MA Prof. dr. Rik Peels Dr. Lidwine Mokkink Dr. Tamarinde Haven Prof. em. dr. Lex Bouter [1] We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the time and effort you have put into evaluating our work and for providing insightful comments and valuable suggestions. We have carefully considered each of them and have revised the manuscript accordingly. Below, we provide detailed responses to each of the comments. We believe that the revisions have improved the manuscript, and we hope that our responses meet any remaining concerns. Response to reviewer 1: Emanuela Reale, CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy Reviewer 1, Comment 1: First, the aim of the paper is clear, but the research question(s) related to that aim are missing. This is an important limitation, since the manuscript reports an exploratory effort and therefore a delimitation of the research problem that you want to investigate is very useful to delineate the perimeter of the work. Response: We acknowledge the concern raised and have addressed this by rewriting the introduction section and by incorporating a clearer description of our study goal. We now make it more clear that our goal was primarily didactic, in the sense that we sought to explore the usefulness of consensus methods utilised in the natural, social, and life sciences for the humanities (see section 1, lines 59-83): An important way to make progress in knowledge and understanding is to gradually, via the exchange of evidence, expertise, and critical discussion, reach (more) consensus on a topic. However, whether there is consensus on a certain matter at a moment in time can sometimes remain unknown, be unclear, or remain implicit. Expert consensus methods can lift off this inconclusiveness by intentionally and formally pursuing or assessing consensus. These methods are regularly used in natural, social, and life sciences. Despite the significant role of consensus in the development of (academic) knowledge and understanding (e.g. Kuhn, 1962; Habermas, 1996; thus also Peels, 2020; Grimm et al., forthcoming ), formal expert consensus methods have hardly been explored within in the context of the humanities. [1] Other kinds of valuable expert meetings do take place in various disciplines in the humanities. They usually do not seek to reach a consensus, but to explore a topic in-depth by bringing the relevant experts to the table. Moreover, expert consensus methods described in this article differ from the expert meetings that already take place in the humanities by the former’s procedural and formalised character. In this paper, we aim to highlight to scholars, particularly those in the humanities, what is possible in terms of a systematic approach to academic consensus. In doing so, we aim to initiate exploration of and discussion about the potential of applying such an approach. What would be the added value of applying expert consensus meetings to humanities-related issues? To achieve this, we first explicate our method (§2), before we provide a framework for the application of expert consensus methods by introducing six key elements (§3). Subsequently, we present common types of expert consensus methods utilized in the natural, social, and life sciences. We highlight their and proven value across various types of research through relevant examples and their respective objectives (§4). We then consider both the potential benefits and possible limitations to applying these methods within humanistic research (§5), and conclude with suggestions for future research (§6). Reviewer 1, Comment 2: Second, the section on Method is poor. The composition of the interdisciplinary team (and the reasons for using this team) is not clearly described, the brainstorm session is not properly defined and illustrated, the ‘informal’ literature review is something new for me (what ‘informal’ means?) and it is not acceptable for a research paper to quote two methods used (‘philosophical methods of conceptual analysis’ and ‘conceptual engineering’) without explaining how they have been used for the purposes of the paper but just putting two references. Also, the process of writing and reviewing the manuscript should be better described and justified. Response: Our article, as we now emphasise more, is meant as an exploratory comment and a didactic contribution to inspire scholars in the humanities to see whether consensus methods may be valuable within their own discipline or sub discipline. It is not an instance of systematic empirical research, which would have required a much more detailed methodological section. But we certainly we agree that the method section can be improved in various ways. Therefore, we extended the section by including a specification of the backgrounds of the involved authors and specified the process of our collaboration as follows (see section 2, lines 85-94): To explore the potential of applying expert consensus methods in the humanities, we composed an interdisciplinary team of both researchers from the sciences using consensus methods in practice (LM: epidemiology; TH: qualitative research methodology; LB: epidemiology and data science, research integrity), and researchers familiar with the domain of the humanities and epistemology (CR: art history, museology; RP: epistemology and philosophy of science, religion and theology). In a first brainstorm session, knowledge on the topic was shared from both perspectives. This resulted in a list of consensus methods and examples used in the sciences, drafted by LM and TH. CR carried out a literature search in order to find examples thereof that were applied in her domain. A first draft was written and discussed in a second group session and culminated in a final version, which was edited and reviewed by all authors. Reviewer 1, Comment 3: Third, and more important, the paper refers always to ‘humanities.’ Humanities is a highly fragmented and differentiated field, where many and different epistemic communities work, interact, shape their own idea and uses of consensus method (if any) and where different instances of relationships with other researchers within the field and with neighboring fields are at place. The paper does not acknowledge this important characteristic of humanities (see footnote 1 where there is a list of disciplines included in the humanities that are considered in the paper). The same holds true for STEMs, which are considered as they are all adopting regularly expert consensus methods and there is no explanation or evidence about it. In sum, the paper ‘explores’ the expert consensus method without considering the set of ideas, values, theories, concepts and practices that pertain to the disciplines analyzed and their scientific work. Response: We acknowledge the concern raised and have taken this into account by adjusting a footnote 1 in section 1 (see above) and by revising section 4. Upon reflection we furthermore adjusted the ways to delineate the contexts in which expert consensus methods may be valuable, by focusing on one’s purpose in applying such a method (see section 4, lines 213-235): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 1, Comment 4: Fourth, sections 3 and 4 provide a clear summary of what expert consensus methods are and a deepen analysis of four types. Then the reader expects Section 5 discusses the results with respect to the exploration done. On the contrary, Section 5 lists of pros and cons related to the expert consensus method, and some examples of applications; the contents are completely unrelated with what the sections 3 and 4 have presented and the motivations for considering the expert consensus method an interesting development for humanities are unclear. Here the suggestion is to revise the sections trying to answer the research questions (when formulated) and highlight what we have learnt from the exploratory work, what is the contribution of the brainstorming to the exploration, what new avenues of research can be envisaged based on the results obtained, and what limitations/obstacles can the expert consensus model have in the humanities with respect to the other fields. Response: Thank you for pointing this out to us. Based on this valuable feedback, we have worked to better align section 5 with the preceding sections to create a more cohesive flow throughout the paper and have made more explicit what our line of reasoning is. However, because the paper is intended as preliminary exploration, part of our manuscript aims to identify the potential value of these methods for the humanities. Section 5 is dedicated to that purpose and we made that more explicit in the revision (see section 5, lines 253-315): In the preceding sections, we examined the elements and dynamics of expert consensus methods, along with examples and possibilities for intended objectives. In this section, we delve into the question why formal approaches to consensus in the humanities remain scarce, despite the significant factor of consensus and dissensus in knowledge production across research cultures. Aside from limitations, we also address what the potential added value of applying consensus methods to humanities-related issues could be. First of all, practical limitations likely play a role. Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Scholars may also disagree due to the role of research paradigms or schools within the humanities, as researchers’ positionality can shape their interpretations. In addition to that, humanities research can be concerned with hermeneutics, i.e., issues that have to do with understanding the values and meanings of texts and other objects. And many humanities fields such as ethics, epistemology, and metaphysics, rely on apriori methods like thought experiments, reasoning on the basis of intuitions and principles, and formal logic rather than empirical data. This allows for multiple interpretations that can all to some extent be warranted by the evidence base, and may not readily align into a singular consensus. These factors suggest that the complexity and interpretative nature of humanities research might render expert consensus methods to some extent less relevant. Nevertheless, we believe that there are compelling reasons for considering expert consensus methods in humanities research. First of all, opposing the above concerns, one of the important purposes of expert consensus methods is to bring various viewpoints, nuances, and interpretations to the surface, so that they can be discussed. Depending on the design and purpose of the consensus method, it could reveal levels of consensus and/or dissensus on an issue, within or across schools of thought or paradigms. They might even have the potential to articulate both divergences and common grounds regarding an issue, and by extension increase mutual understanding. The application of such a method may therefore stimulate epistemic progress in itself. Moreover, a carefully designed protocol and curated group of experts can foster a fair practice and ensure the consultation of diverse viewpoints. Expert consensus methods provide opportunities to mitigate biases and their ramifications by this cautious selection of experts, but also by blinding certain parts of the process in relation to anonymity of participating experts, for example. Secondly, expert consensus methods can be helpful in getting a firmer grip on the status quo or state of the art on a particular issue in a field (e.g. Hodgkin et al. (1975) mentioned in Table 2; Rulkens et al. 2022-2023, mentioned in Table 3). Of course, scholars often already have an idea about that, and such an informed opinion is valuable in itself. However, it can be complemented by a more structured approach, particularly when the number of scholars involved is large. This approach can additionally provide guidance for future research by pointing out the gaps in the current evidence base. And when applied on a recurrent basis, it might even have the potential to gain insight in how views evolve over time. It is important to note that consensus is not a final endpoint; rather, it can be continually re-evaluated in light of new insights and reasonable doubts regarding its validity. Third, an advantage of expert consensus methods is that they enhance the transparency and replicability of research because of their well-documented and systematised approach (for replication in the humanities, see Peels and Bouter (2018)). It can therefore be instrumental to epistemic progress beyond the consensus reached at a moment in time. Fourth, application or implementation of its outcomes is often easier after the process of a consensus method is completed (e.g. Standing Committee for the Museum Definition (2020-2022) (2022), mentioned in Table 3). And finally, expert consensus methods can aid public understanding by clarifying experts perspectives on specific issues (Stekelenburg et al., 2022) thereby enabling the public to make informed decisions (Ordway, 2021). In response to this comments, we furthermore extended suggestions for future research in our conclusion (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Response to reviewer 2: Daniel Leising, Technische Universitat Dresden, Dresden, Saxony, Germany Reviewer 2, Comment 1 : First, I would argue that the Humanities are actually not that “unique” in terms of their nature and approaches (page 11). To enable a proper discussion of this issue, the authors could explain their own view of what is so specific about the Humanities in a bit more detail. Here is my view: Interviews and text analyses are used in many different branches of science. Exploratory research is also conducted in many fields. The ways in which people give meaning to their experiences are also of interest across various branches of science. And even in many of the “harder” sciences, research is often “coloured by the backgrounds and positionality of the researchers” (page 8). So, much of what is common in research in the Humanities is actually shared with other fields. In my view, two features may actually distinguish “the Humanities” from other fields - but even these distinctions are only gradual in nature. One feature is the common practice of using everyday language as the primary scientific language. The other feature is the relative fuzziness of many concepts - which I would note is not necessarily a weakness but may often be unavoidable given the complexity of the phenomena of interest (and also given the first feature just mentioned). Do the authors agree? Response: Thank you for challenging us on this point. We agree that the borders between the general fields of the natural, social, and life sciences, and the humanities are fuzzy, when it comes to their objects, purposes, and methods. Whether there is something distinctive about the humanities – we believe that is compatible with fuzziness at the boarders – is a matter that we cannot possibly explore in this article (one of us does so in a book entitled A Philosophy of the Humanities that is forthcoming in 2025 with Oxford University Press, a volume co-authored by Stephen Grimm, Rik Peels, and René van Woudenberg). We do believe though that ordinary language use is not generally characteristic of the humanities (think of the highly technical language of metaphysics and epistemology), nor do we believe that insufficient clarity about core concepts is characteristic of the humanities. We agree though with the idea, also drawn to attention to by reviewer 1 in comment 3, that the humanities are a highly diversified field. We included that by adjusting footnote 1 in section 1: We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Additionally, to be able to demarcate between contexts in which the formal approach might be beneficial, we decided to put an emphasis on the purposes for which these methods are employed in other domains. This would be a way in which researchers with expertise in their fields could identify opportunities for a formal approach to consensus (see section 4, lines 213-235, also quoted above in response to comment 3 of reviewer 1): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 2, Comment 2: Second, I would like to encourage the authors to say a bit more about possible reasons for why so little consensus building is taking place in many fields of science, despite it being widely viewed as desirable. In my view, there are a couple of reasons, most of which are rather extraneous to an idealised image of science (as a collaborative and cumulative effort at building a reliable knowledge base for the good of mankind). For example, consensus building is likely to reduce the publication output of a field, which would be at odds with the interests of commercial publishers. Also, individual researchers have an interest in “making a name for themselves” based on their specific expertise or views regarding certain topics. Consensus building makes it more difficult to maintain a highly distinct profile and thus reduce visibility. This may be risky in a science system that is as competitive as the one we choose to entertain. And then there may be a need to “save face”: The more of your career you have dedicated to certain ideas (e.g., your own theory of some phenomenon), the harder it may be to let those ideas go - even if the evidence against them is overwhelming. There is a reason why Max Planck once stated that scientific progress happens “one funeral at a time”. At least a brief nod to the relevance of such more political factors would be nice - if the authors share my conviction that they are relevant. Who has an actual interest in achieving consensus? Who has an interest in avoiding the emergence of consensus? And what are the external incentives for and against this type of work ? Response: We think that these are valid points and the reviewers’ publications on the topic shed important light on this. We agree that external incentives may work against a formal approach to consensus as follows and have therefore slightly revised the manuscript on this point. We remain rather brief, though, as the exact explanation of why consensus methods are still relatively rarely used even in the sciences is an issue that is beyond the scope of our paper (see section 5, lines 259-264, also quoted in response to comment 4 of reviewer 1): Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Reviewer 2, Comment 3: Third, I would also appreciate if the authors could talk some more about “bad consensus” – views that are held by many, despite their being wrong. With a particular eye to the Humanities – what criteria are there, apart from consensus, that may help us determine whether a view is “correct”? Or may truth in the Humanities *only* be defined in terms of relative consensus among researchers? In the harder sciences, the ability to *predict* the occurrence of certain events from a theory (and some data input) plays a crucial role as evidence of the theory's correctness. Are the Humanities able to produce testable predictions, too? Can there be consensus projects in the Humanities that actually test one theory against another? Response: This is an interesting point, that would need a elaborative reflection in order to do justice to the complexity of this issue. It is important to note, that we do believe that consensus and truth are two different things, and we made this more clear in our revised paper. Based these suggestions we also incorporated that consensus can be reached based on the wrong reasons and (see section 5, lines 265-271, also quoted in response to comment 4 of reviewer 1): Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Reviewer 2, Comment 4: Fourth, on page 3 the authors state that “Since the questions addressed in the humanities may leave room for multiple viewpoints that are equally warranted by the evidence base, it can be infeasible or not desirable to get a complete or almost complete consensus.” To this I would respond: what are the conditions under which working toward consensus would not be feasible, or desirable? I agree that this may be the case – but when is it the case, and when is it not the case? I personally hold the somewhat sceptical view that many researchers will shy away from working toward consensus even if doing so *would* be both feasible and desirable (see my previous point). Such a “live and let live” attitude may often be relatively comfortable to have (e.g., to avoid conflict). It may, however, be regarded as unethical, because it will foster the preservation of an unnecessary level of ambiguity. Response: We appreciate this insightful suggestion. Our approach to expert consensus methods does not necessarily imply working towards consensus in all instances, but is also related to the assessment of the (lack of) consensus. We view measuring consensus as a means, not an objective in itself. Furthermore, as put forward in paragraph 3, the bar for ‘sufficient’ consensus is something consensus researchers can determine their selves. Applying a consensus methods is a costly endeavour in terms of time and means. There might be conditions under which the effort of applying such a method does not live up to the epistemic benefits it provides for. We furthermore believe that the feasibility and desirability of assessing consensus is an issue that merits further exploration and included this as follows (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Minor comments: Reviewer 2, Comment 5: Fifth, the authors state that a steering committee would typically be comprised of the people who initiated the process and who will ultimately author the consensus document. Could there not be cases in which the process should better be overseen by an independent third party (e.g., a moderator)? Especially when issues are highly contentious, for example? And should not all the experts who worked on the consensus be authors on the consensus document? Response: Thank you for this relevant point, this is certainly the case and we made this more clear in the revised manuscript by the following addition (see section 3, lines 193-198): Naturally, the six elements can be specified at by the steering committee. If they deem it valuable to include an independent moderator or to incorporate a member with a critical or divergent perspective—especially if the topic is highly controversial—this remains within their discretion. Additionally, it is up to them to decide who will be listed as co-authors, whether that includes only the steering committee, all participants, or a broader group representation such as the panellists as a whole, for instance. Reviewer 2, Comment 6: Sixth, something is not quite right on page 4, text lines 16-19. Response: Thank you for noticing, we have corrected this sentence as follows (see section 3, lines 136-140): Expert consensus methods are (i) applied by a process leader or steering committee , (ii) to reach consensus or assess the degree of consensus among a group of experts , (iii) about an issue , (iv) based on preset rules of engagement , (v) with the aim to deliver useful output , (vi) for future users . They are characterised by constructive and procedural usage of (dis)agreement and the different arguments that are provided therein. Reviewer 2, Comment 7: Seventh, I would say that evidence will never “speak for itself” (page 4) - otherwise papers in certain scientific fields would not need any Discussion sections. Response: Thank you for rightly pointing this out to us. We adjusted this as follows (see section 3, lines 132-134): This can be useful when the issue at hand is an important one and when evidence is lacking, it is inconsistent, or its interpretation is not obvious. As a result of our reactions to the comments, we also made adjustments to our abstract and conclusion, which are not all quoted in the above letter. We hope these revisions and clarifications meet the reviewers’ expectations. Please let us know if any further revisions are needed. Sincerely, Charlotte Rulkens MA Prof. dr. Rik Peels Dr. Lidwine Mokkink Dr. Tamarinde Haven Prof. em. dr. Lex Bouter [1] We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Comments on this article Comments (0) Version 2 VERSION 2 PUBLISHED 28 Jun 2024 ADD YOUR COMMENT Comment keyboard_arrow_left keyboard_arrow_right Open Peer Review Reviewer Status info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Reviewer Reports Invited Reviewers 1 2 3 Version 2 (revision) 16 Dec 24 read read read Version 1 28 Jun 24 read read Emanuela Reale , CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy Daniel Leising , Technische Universitat Dresden, Dresden, Germany Stephen R Grimm , Fordham University, Bronx, USA Comments on this article All Comments (0) Add a comment Sign up for content alerts Sign Up You are now signed up to receive this alert Browse by related subjects keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 Grimm S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 22 Jan 2025 | for Version 2 Stephen R Grimm , Fordham University, Bronx, USA 0 Views copyright © 2025 Grimm S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This is a very nice, clear overview of the nature and potential importance of consensus methods in the humanities. I especially appreciate the cautions at the end, namely that "applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for." I too wondered this throughout reading the paper. But, that said, I can also see the benefits of pursuing these methods in the humanities. Is the rationale for developing the new method (or application) clearly explained? Yes Is the description of the method technically sound? Yes Are sufficient details provided to allow replication of the method development and its use by others? Yes If any results are presented, are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions about the method and its performance adequately supported by the findings presented in the article? Yes Competing Interests No competing interests were disclosed. Reviewer Expertise Philosophy, epistemology I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) Grimm SR. Peer Review Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.174565.r355056) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-710/v2#referee-response-355056 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 Reale E. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 09 Jan 2025 | for Version 2 Emanuela Reale , CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy 0 Views copyright © 2025 Reale E. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions I have read the revised version of the paper and the changes that have been applied. Some of my suggestions have been addressed and the manuscript in the current form is clearer and more consistent than before. However, the manuscript still does not have a clear research question and therefore it remains largely descriptive and the scientific contribution it provides does not emerge. Problems and limitations with the applications of the consensus methods in the Humanities are not discussed with reference to the specific cases that have been presented in tables 3-4 and with empirical evidence on the cases themselves. Moreover, some limitations are not exclusive of Humanities (e.g. time and resources; role of research paradigms bringing to divergent views) and the specific features of Humanities would need a deeper analysis. Competing Interests No competing interests were disclosed. Reviewer Expertise Higher education and research policy I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (0) Reale E. Peer Review Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.174565.r349191) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-710/v2#referee-response-349191 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Leising D. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 23 Dec 2024 | for Version 2 Daniel Leising , Technische Universitat Dresden, Dresden, Saxony, Germany 0 Views copyright © 2024 Leising D. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions I found that the authors responded to my comments in a careful and throughtful manner, and that the manuscript has improved. I have no further comments or suggestions for changes. Competing Interests No competing interests were disclosed. Reviewer Expertise Personality / Social Psychology, Meta-Science I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) Leising D. Peer Review Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.174565.r349190) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-710/v2#referee-response-349190 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Leising D. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 19 Sep 2024 | for Version 1 Daniel Leising , Technische Universitat Dresden, Dresden, Saxony, Germany 0 Views copyright © 2024 Leising D. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions I strongly agree that more systematic consensus-building is needed in many fields of science, including the Humanities and Psychology (my own field) and I find little that I would disagree with in the manuscript as is. However, I also think that the authors could do more than just presenting an overview of established consensus building methods and suggesting that researchers from the Humanities should give these methods a try. The issue is more complex than that. In order to improve on the depth, utility and potential impact of this paper, I would thus like to encourage the authors to think about and – if possible – share their views on a number of additional, intriguing, and sometimes thorny issues regarding consensus building. First, I would argue that the Humanities are actually not that “unique” in terms of their nature and approaches (page 11). To enable a proper discussion of this issue, the authors could explain their own view of what is so specific about the Humanities in a bit more detail. Here is my view: Interviews and text analyses are used in many different branches of science. Exploratory research is also conducted in many fields. The ways in which people give meaning to their experiences are also of interest across various branches of science. And even in many of the “harder” sciences, research is often “coloured by the backgrounds and positionality of the researchers” (page 8). So, much of what is common in research in the Humanities is actually shared with other fields. In my view, two features may actually distinguish “the Humanities” from other fields - but even these distinctions are only gradual in nature. One feature is the common practice of using everyday language as the primary scientific language. The other feature is the relative fuzziness of many concepts - which I would note is not necessarily a weakness but may often be unavoidable given the complexity of the phenomena of interest (and also given the first feature just mentioned). Do the authors agree? Second, I would like to encourage the authors to say a bit more about possible reasons for why so little consensus building is taking place in many fields of science, despite it being widely viewed as desirable. In my view, there are a couple of reasons, most of which are rather extraneous to an idealized image of science (as a collaborative and cumulative effort at building a reliable knowledge base for the good of mankind). For example, consensus building is likely to reduce the publication output of a field, which would be at odds with the interests of commercial publishers. Also, individual researchers have an interest in “making a name for themselves" based on their specific expertise or views regarding certain topics. Consensus building makes it more difficult to maintain a highly distinct profile and thus reduce visibility. This may be risky in a science system that is as competitive as the one we choose to entertain. And then there may be a need to “save face”: The more of your career you have dedicated to certain ideas (e.g., your own theory of some phenomenon), the harder it may be to let those ideas go - even if the evidence against them is overwhelming. There is a reason why Max Planck once stated that scientific progress happens “one funeral at a time”. At least a brief nod to the relevance of such more political factors would be nice - if the authors share my conviction that they are relevant. Who has an actual interest in achieving consensus? Who has an interest in avoiding the emergence of consensus? And what are the external incentives for and agaisnt this type of work? Third, I would also appreciate if the authors could talk some more about “bad consensus” – views that are held by many, despite their being wrong. With a particular eye to the Humanities – what criteria are there, apart from consensus, that may help us determine whether a view is “correct”? Or may truth in the Humanities *only* be defined in terms of relative consensus among researchers? In the harder sciences, the ability to *predict* the occurrence of certain events from a theory (and some data input) plays a crucial role as evidence of the theory's correctness. Are the Humanities able to produce testable predictions, too? Can there be consensus projects in the Humanities that actually test one theory against another? Fourth, on page 3 the authors state that “ Since the questions addressed in the humanities may leave room for multiple viewpoints that are equally warranted by the evidence base, it can be infeasible or not desirable to get a complete or almost complete consensus.” To this I would respond: what are the conditions under which working toward consensus would not be feasible, or desirable? I agree that this may be the case – but when is it the case, and when is it not the case? I personally hold the somewhat skeptical view that many researchers will shy away from working toward consensus even if doing so *would* be both feasible and desirable (see my previous point). Such a “live and let live” attitude may often be relatively comfortable to have (e.g., to avoid conflict). It may, however, be regarded as unethical, because it will foster the preservation of an unnecessary level of ambiguity. Minor points Fifth, the authors state that a steering committee would typically be comprised of the people who initiated the process and who will ultimately author the consensus document. Could there not be cases in which the process should better be overseen by an independent third party (e.g., a moderator)? Especially when issues are highly contentious, for example? And should not all the experts who worked on the consensus be authors on the consensus document? Sixth, something is not quite right on page 4, text lines 16-19. Seventh, I would say that evidence will never “speak for itself” (page 4) - otherwise papers in certain scientific fields would not need any Discussion sections. Together with a variety of colleagues, I have written about several of the above-mentioned issues. If the authors are interested in this, here are a few key references: I referenced some of our own publications that I consider to be related in terms of content, to let the authors know about their existence. The authors are of course entirely free to decide whether to look into them, or not. Leising, D.et.al; 2023 (Ref 1) Leising, D.et.al; 2024 (Ref 2) Leising, D.et.al; 2022 (Ref 3) Leising, D.et.al; 2022 (Ref 4) Is the rationale for developing the new method (or application) clearly explained? Yes Is the description of the method technically sound? Yes Are sufficient details provided to allow replication of the method development and its use by others? Yes If any results are presented, are all the source data underlying the results available to ensure full reproducibility? No source data required Are the conclusions about the method and its performance adequately supported by the findings presented in the article? Yes References 1. Leising D, Grenke O, Cramer M: Visual Argument Structure Tool (VAST) Version 1.0. Meta-Psychology . 2023; 7 . Publisher Full Text 2. Leising D, Liesefeld H, Buecker S, Glöckner A, et al.: A Tentative Roadmap for Consensus-Building Processes. 2024. Publisher Full Text 3. Leising D, Thielmann I, Glöckner A, Gärtner A, et al.: Ten steps toward a better personality science – how quality may be rewarded more in research evaluation. Personality Science . 2022; 3 . Publisher Full Text 4. Leising D, Thielmann I, Glöckner A, Gärtner A, et al.: Ten steps toward a better personality science – a rejoinder to the comments. Personality Science . 2022; 3 . Publisher Full Text Competing Interests No competing interests were disclosed. Reviewer Expertise Personality / Social Psychology, Meta-Science I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (1) Author Response 16 Dec 2024 Charlotte Rulkens, Department of Philosophy, Vrije Universiteit Amsterdam, Amsterdam, The Netherlands Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the time and effort you have put into evaluating our work and for providing insightful comments and valuable suggestions. We have carefully considered each of them and have revised the manuscript accordingly. Below, we provide detailed responses to each of the comments. We believe that the revisions have improved the manuscript, and we hope that our responses meet any remaining concerns. Response to reviewer 1: Emanuela Reale, CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy Reviewer 1, Comment 1: First, the aim of the paper is clear, but the research question(s) related to that aim are missing. This is an important limitation, since the manuscript reports an exploratory effort and therefore a delimitation of the research problem that you want to investigate is very useful to delineate the perimeter of the work. Response: We acknowledge the concern raised and have addressed this by rewriting the introduction section and by incorporating a clearer description of our study goal. We now make it more clear that our goal was primarily didactic, in the sense that we sought to explore the usefulness of consensus methods utilised in the natural, social, and life sciences for the humanities (see section 1, lines 59-83): An important way to make progress in knowledge and understanding is to gradually, via the exchange of evidence, expertise, and critical discussion, reach (more) consensus on a topic. However, whether there is consensus on a certain matter at a moment in time can sometimes remain unknown, be unclear, or remain implicit. Expert consensus methods can lift off this inconclusiveness by intentionally and formally pursuing or assessing consensus. These methods are regularly used in natural, social, and life sciences. Despite the significant role of consensus in the development of (academic) knowledge and understanding (e.g. Kuhn, 1962; Habermas, 1996; thus also Peels, 2020; Grimm et al., forthcoming ), formal expert consensus methods have hardly been explored within in the context of the humanities. [1] Other kinds of valuable expert meetings do take place in various disciplines in the humanities. They usually do not seek to reach a consensus, but to explore a topic in-depth by bringing the relevant experts to the table. Moreover, expert consensus methods described in this article differ from the expert meetings that already take place in the humanities by the former’s procedural and formalised character. In this paper, we aim to highlight to scholars, particularly those in the humanities, what is possible in terms of a systematic approach to academic consensus. In doing so, we aim to initiate exploration of and discussion about the potential of applying such an approach. What would be the added value of applying expert consensus meetings to humanities-related issues? To achieve this, we first explicate our method (§2), before we provide a framework for the application of expert consensus methods by introducing six key elements (§3). Subsequently, we present common types of expert consensus methods utilized in the natural, social, and life sciences. We highlight their and proven value across various types of research through relevant examples and their respective objectives (§4). We then consider both the potential benefits and possible limitations to applying these methods within humanistic research (§5), and conclude with suggestions for future research (§6). Reviewer 1, Comment 2: Second, the section on Method is poor. The composition of the interdisciplinary team (and the reasons for using this team) is not clearly described, the brainstorm session is not properly defined and illustrated, the ‘informal’ literature review is something new for me (what ‘informal’ means?) and it is not acceptable for a research paper to quote two methods used (‘philosophical methods of conceptual analysis’ and ‘conceptual engineering’) without explaining how they have been used for the purposes of the paper but just putting two references. Also, the process of writing and reviewing the manuscript should be better described and justified. Response: Our article, as we now emphasise more, is meant as an exploratory comment and a didactic contribution to inspire scholars in the humanities to see whether consensus methods may be valuable within their own discipline or sub discipline. It is not an instance of systematic empirical research, which would have required a much more detailed methodological section. But we certainly we agree that the method section can be improved in various ways. Therefore, we extended the section by including a specification of the backgrounds of the involved authors and specified the process of our collaboration as follows (see section 2, lines 85-94): To explore the potential of applying expert consensus methods in the humanities, we composed an interdisciplinary team of both researchers from the sciences using consensus methods in practice (LM: epidemiology; TH: qualitative research methodology; LB: epidemiology and data science, research integrity), and researchers familiar with the domain of the humanities and epistemology (CR: art history, museology; RP: epistemology and philosophy of science, religion and theology). In a first brainstorm session, knowledge on the topic was shared from both perspectives. This resulted in a list of consensus methods and examples used in the sciences, drafted by LM and TH. CR carried out a literature search in order to find examples thereof that were applied in her domain. A first draft was written and discussed in a second group session and culminated in a final version, which was edited and reviewed by all authors. Reviewer 1, Comment 3: Third, and more important, the paper refers always to ‘humanities.’ Humanities is a highly fragmented and differentiated field, where many and different epistemic communities work, interact, shape their own idea and uses of consensus method (if any) and where different instances of relationships with other researchers within the field and with neighboring fields are at place. The paper does not acknowledge this important characteristic of humanities (see footnote 1 where there is a list of disciplines included in the humanities that are considered in the paper). The same holds true for STEMs, which are considered as they are all adopting regularly expert consensus methods and there is no explanation or evidence about it. In sum, the paper ‘explores’ the expert consensus method without considering the set of ideas, values, theories, concepts and practices that pertain to the disciplines analyzed and their scientific work. Response: We acknowledge the concern raised and have taken this into account by adjusting a footnote 1 in section 1 (see above) and by revising section 4. Upon reflection we furthermore adjusted the ways to delineate the contexts in which expert consensus methods may be valuable, by focusing on one’s purpose in applying such a method (see section 4, lines 213-235): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 1, Comment 4: Fourth, sections 3 and 4 provide a clear summary of what expert consensus methods are and a deepen analysis of four types. Then the reader expects Section 5 discusses the results with respect to the exploration done. On the contrary, Section 5 lists of pros and cons related to the expert consensus method, and some examples of applications; the contents are completely unrelated with what the sections 3 and 4 have presented and the motivations for considering the expert consensus method an interesting development for humanities are unclear. Here the suggestion is to revise the sections trying to answer the research questions (when formulated) and highlight what we have learnt from the exploratory work, what is the contribution of the brainstorming to the exploration, what new avenues of research can be envisaged based on the results obtained, and what limitations/obstacles can the expert consensus model have in the humanities with respect to the other fields. Response: Thank you for pointing this out to us. Based on this valuable feedback, we have worked to better align section 5 with the preceding sections to create a more cohesive flow throughout the paper and have made more explicit what our line of reasoning is. However, because the paper is intended as preliminary exploration, part of our manuscript aims to identify the potential value of these methods for the humanities. Section 5 is dedicated to that purpose and we made that more explicit in the revision (see section 5, lines 253-315): In the preceding sections, we examined the elements and dynamics of expert consensus methods, along with examples and possibilities for intended objectives. In this section, we delve into the question why formal approaches to consensus in the humanities remain scarce, despite the significant factor of consensus and dissensus in knowledge production across research cultures. Aside from limitations, we also address what the potential added value of applying consensus methods to humanities-related issues could be. First of all, practical limitations likely play a role. Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Scholars may also disagree due to the role of research paradigms or schools within the humanities, as researchers’ positionality can shape their interpretations. In addition to that, humanities research can be concerned with hermeneutics, i.e., issues that have to do with understanding the values and meanings of texts and other objects. And many humanities fields such as ethics, epistemology, and metaphysics, rely on apriori methods like thought experiments, reasoning on the basis of intuitions and principles, and formal logic rather than empirical data. This allows for multiple interpretations that can all to some extent be warranted by the evidence base, and may not readily align into a singular consensus. These factors suggest that the complexity and interpretative nature of humanities research might render expert consensus methods to some extent less relevant. Nevertheless, we believe that there are compelling reasons for considering expert consensus methods in humanities research. First of all, opposing the above concerns, one of the important purposes of expert consensus methods is to bring various viewpoints, nuances, and interpretations to the surface, so that they can be discussed. Depending on the design and purpose of the consensus method, it could reveal levels of consensus and/or dissensus on an issue, within or across schools of thought or paradigms. They might even have the potential to articulate both divergences and common grounds regarding an issue, and by extension increase mutual understanding. The application of such a method may therefore stimulate epistemic progress in itself. Moreover, a carefully designed protocol and curated group of experts can foster a fair practice and ensure the consultation of diverse viewpoints. Expert consensus methods provide opportunities to mitigate biases and their ramifications by this cautious selection of experts, but also by blinding certain parts of the process in relation to anonymity of participating experts, for example. Secondly, expert consensus methods can be helpful in getting a firmer grip on the status quo or state of the art on a particular issue in a field (e.g. Hodgkin et al. (1975) mentioned in Table 2; Rulkens et al. 2022-2023, mentioned in Table 3). Of course, scholars often already have an idea about that, and such an informed opinion is valuable in itself. However, it can be complemented by a more structured approach, particularly when the number of scholars involved is large. This approach can additionally provide guidance for future research by pointing out the gaps in the current evidence base. And when applied on a recurrent basis, it might even have the potential to gain insight in how views evolve over time. It is important to note that consensus is not a final endpoint; rather, it can be continually re-evaluated in light of new insights and reasonable doubts regarding its validity. Third, an advantage of expert consensus methods is that they enhance the transparency and replicability of research because of their well-documented and systematised approach (for replication in the humanities, see Peels and Bouter (2018)). It can therefore be instrumental to epistemic progress beyond the consensus reached at a moment in time. Fourth, application or implementation of its outcomes is often easier after the process of a consensus method is completed (e.g. Standing Committee for the Museum Definition (2020-2022) (2022), mentioned in Table 3). And finally, expert consensus methods can aid public understanding by clarifying experts perspectives on specific issues (Stekelenburg et al., 2022) thereby enabling the public to make informed decisions (Ordway, 2021). In response to this comments, we furthermore extended suggestions for future research in our conclusion (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Response to reviewer 2: Daniel Leising, Technische Universitat Dresden, Dresden, Saxony, Germany Reviewer 2, Comment 1 : First, I would argue that the Humanities are actually not that “unique” in terms of their nature and approaches (page 11). To enable a proper discussion of this issue, the authors could explain their own view of what is so specific about the Humanities in a bit more detail. Here is my view: Interviews and text analyses are used in many different branches of science. Exploratory research is also conducted in many fields. The ways in which people give meaning to their experiences are also of interest across various branches of science. And even in many of the “harder” sciences, research is often “coloured by the backgrounds and positionality of the researchers” (page 8). So, much of what is common in research in the Humanities is actually shared with other fields. In my view, two features may actually distinguish “the Humanities” from other fields - but even these distinctions are only gradual in nature. One feature is the common practice of using everyday language as the primary scientific language. The other feature is the relative fuzziness of many concepts - which I would note is not necessarily a weakness but may often be unavoidable given the complexity of the phenomena of interest (and also given the first feature just mentioned). Do the authors agree? Response: Thank you for challenging us on this point. We agree that the borders between the general fields of the natural, social, and life sciences, and the humanities are fuzzy, when it comes to their objects, purposes, and methods. Whether there is something distinctive about the humanities – we believe that is compatible with fuzziness at the boarders – is a matter that we cannot possibly explore in this article (one of us does so in a book entitled A Philosophy of the Humanities that is forthcoming in 2025 with Oxford University Press, a volume co-authored by Stephen Grimm, Rik Peels, and René van Woudenberg). We do believe though that ordinary language use is not generally characteristic of the humanities (think of the highly technical language of metaphysics and epistemology), nor do we believe that insufficient clarity about core concepts is characteristic of the humanities. We agree though with the idea, also drawn to attention to by reviewer 1 in comment 3, that the humanities are a highly diversified field. We included that by adjusting footnote 1 in section 1: We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Additionally, to be able to demarcate between contexts in which the formal approach might be beneficial, we decided to put an emphasis on the purposes for which these methods are employed in other domains. This would be a way in which researchers with expertise in their fields could identify opportunities for a formal approach to consensus (see section 4, lines 213-235, also quoted above in response to comment 3 of reviewer 1): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 2, Comment 2: Second, I would like to encourage the authors to say a bit more about possible reasons for why so little consensus building is taking place in many fields of science, despite it being widely viewed as desirable. In my view, there are a couple of reasons, most of which are rather extraneous to an idealised image of science (as a collaborative and cumulative effort at building a reliable knowledge base for the good of mankind). For example, consensus building is likely to reduce the publication output of a field, which would be at odds with the interests of commercial publishers. Also, individual researchers have an interest in “making a name for themselves” based on their specific expertise or views regarding certain topics. Consensus building makes it more difficult to maintain a highly distinct profile and thus reduce visibility. This may be risky in a science system that is as competitive as the one we choose to entertain. And then there may be a need to “save face”: The more of your career you have dedicated to certain ideas (e.g., your own theory of some phenomenon), the harder it may be to let those ideas go - even if the evidence against them is overwhelming. There is a reason why Max Planck once stated that scientific progress happens “one funeral at a time”. At least a brief nod to the relevance of such more political factors would be nice - if the authors share my conviction that they are relevant. Who has an actual interest in achieving consensus? Who has an interest in avoiding the emergence of consensus? And what are the external incentives for and against this type of work ? Response: We think that these are valid points and the reviewers’ publications on the topic shed important light on this. We agree that external incentives may work against a formal approach to consensus as follows and have therefore slightly revised the manuscript on this point. We remain rather brief, though, as the exact explanation of why consensus methods are still relatively rarely used even in the sciences is an issue that is beyond the scope of our paper (see section 5, lines 259-264, also quoted in response to comment 4 of reviewer 1): Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Reviewer 2, Comment 3: Third, I would also appreciate if the authors could talk some more about “bad consensus” – views that are held by many, despite their being wrong. With a particular eye to the Humanities – what criteria are there, apart from consensus, that may help us determine whether a view is “correct”? Or may truth in the Humanities *only* be defined in terms of relative consensus among researchers? In the harder sciences, the ability to *predict* the occurrence of certain events from a theory (and some data input) plays a crucial role as evidence of the theory's correctness. Are the Humanities able to produce testable predictions, too? Can there be consensus projects in the Humanities that actually test one theory against another? Response: This is an interesting point, that would need a elaborative reflection in order to do justice to the complexity of this issue. It is important to note, that we do believe that consensus and truth are two different things, and we made this more clear in our revised paper. Based these suggestions we also incorporated that consensus can be reached based on the wrong reasons and (see section 5, lines 265-271, also quoted in response to comment 4 of reviewer 1): Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Reviewer 2, Comment 4: Fourth, on page 3 the authors state that “Since the questions addressed in the humanities may leave room for multiple viewpoints that are equally warranted by the evidence base, it can be infeasible or not desirable to get a complete or almost complete consensus.” To this I would respond: what are the conditions under which working toward consensus would not be feasible, or desirable? I agree that this may be the case – but when is it the case, and when is it not the case? I personally hold the somewhat sceptical view that many researchers will shy away from working toward consensus even if doing so *would* be both feasible and desirable (see my previous point). Such a “live and let live” attitude may often be relatively comfortable to have (e.g., to avoid conflict). It may, however, be regarded as unethical, because it will foster the preservation of an unnecessary level of ambiguity. Response: We appreciate this insightful suggestion. Our approach to expert consensus methods does not necessarily imply working towards consensus in all instances, but is also related to the assessment of the (lack of) consensus. We view measuring consensus as a means, not an objective in itself. Furthermore, as put forward in paragraph 3, the bar for ‘sufficient’ consensus is something consensus researchers can determine their selves. Applying a consensus methods is a costly endeavour in terms of time and means. There might be conditions under which the effort of applying such a method does not live up to the epistemic benefits it provides for. We furthermore believe that the feasibility and desirability of assessing consensus is an issue that merits further exploration and included this as follows (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Minor comments: Reviewer 2, Comment 5: Fifth, the authors state that a steering committee would typically be comprised of the people who initiated the process and who will ultimately author the consensus document. Could there not be cases in which the process should better be overseen by an independent third party (e.g., a moderator)? Especially when issues are highly contentious, for example? And should not all the experts who worked on the consensus be authors on the consensus document? Response: Thank you for this relevant point, this is certainly the case and we made this more clear in the revised manuscript by the following addition (see section 3, lines 193-198): Naturally, the six elements can be specified at by the steering committee. If they deem it valuable to include an independent moderator or to incorporate a member with a critical or divergent perspective—especially if the topic is highly controversial—this remains within their discretion. Additionally, it is up to them to decide who will be listed as co-authors, whether that includes only the steering committee, all participants, or a broader group representation such as the panellists as a whole, for instance. Reviewer 2, Comment 6: Sixth, something is not quite right on page 4, text lines 16-19. Response: Thank you for noticing, we have corrected this sentence as follows (see section 3, lines 136-140): Expert consensus methods are (i) applied by a process leader or steering committee , (ii) to reach consensus or assess the degree of consensus among a group of experts , (iii) about an issue , (iv) based on preset rules of engagement , (v) with the aim to deliver useful output , (vi) for future users . They are characterised by constructive and procedural usage of (dis)agreement and the different arguments that are provided therein. Reviewer 2, Comment 7: Seventh, I would say that evidence will never “speak for itself” (page 4) - otherwise papers in certain scientific fields would not need any Discussion sections. Response: Thank you for rightly pointing this out to us. We adjusted this as follows (see section 3, lines 132-134): This can be useful when the issue at hand is an important one and when evidence is lacking, it is inconsistent, or its interpretation is not obvious. As a result of our reactions to the comments, we also made adjustments to our abstract and conclusion, which are not all quoted in the above letter. We hope these revisions and clarifications meet the reviewers’ expectations. Please let us know if any further revisions are needed. Sincerely, Charlotte Rulkens MA Prof. dr. Rik Peels Dr. Lidwine Mokkink Dr. Tamarinde Haven Prof. em. dr. Lex Bouter [1] We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Leising D. Peer Review Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.163073.r309049) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-710/v1#referee-response-309049 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Reale E. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 23 Aug 2024 | for Version 1 Emanuela Reale , CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy 0 Views copyright © 2024 Reale E. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (2) Not Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Expert Consensus Methods In The Humanities – An Exploration of their Potential The paper wants to explore the value of applying expert consensus methods in the humanities. The topic is very interesting and deserve attention. The manuscript is clearly written, and the literature mobilized is appropriate and relevant for the aim of the investigation proposed. The effort of synthetize concepts and findings using tables is valuable for the clarity of the argument. However, the manuscript in its current form has several weaknesses and, in my opinion, needs some substantial re-work. Hereby I try to summarize the most important critical issues that must be solved. First, the aim of the paper is clear, but the research question(s) related to that aim are missing. This is an important limitation, since the manuscript reports an exploratory effort and therefore a delimitation of the research problem that you want to investigate is very useful to delineate the perimeter of the work. Second, the section on Method is poor. The composition of the interdisciplinary team (and the reasons for using this team) is not clearly described, the brainstorm session is not properly defined and illustrated, the ‘informal’ literature review is something new for me (what ‘informal’ means?) and it is not acceptable for a research paper to quote two methods used (‘philosophical methods of conceptual analysis’ and ‘conceptual engineering’) without explaining how they have been used for the purposes of the paper but just putting two references. Also, the process of writing and reviewing the manuscript should be better described and justified. Third, and more important, the paper refers always to ‘humanities.’ Humanities is a highly fragmented and differentiated field, where many and different epistemic communities work, interact, shape their own idea and uses of consensus method (if any) and where different instances of relationships with other researchers within the field and with neighboring fields are at place. The paper does not acknowledge this important characteristic of humanities (see footnote 1 where there is a list of disciplines included in the humanities that are considered in the paper). The same holds true for STEMs, which are considered as they are all adopting regularly expert consensus methods and there is no explanation or evidence about it. In sum, the paper ‘explores’ the expert consensus method without considering the set of ideas, values, theories, concepts and practices that pertain to the disciplines analyzed and their scientific work. Fourth, sections 3 and 4 provide a clear summary of what expert consensus methods are and a deepen analysis of four types. Then the reader expects Section 5 discusses the results with respect to the exploration done. On the contrary, Section 5 lists of pros and cons related to the expert consensus method, and some examples of applications; the contents are completely unrelated with what the sections 3 and 4 have presented and the motivations for considering the expert consensus method an interesting development for humanities are unclear. Here the suggestion is to revise the sections trying to answer the research questions (when formulated) and highlight what we have learnt from the exploratory work, what is the contribution of the brainstorming to the exploration, what new avenues of research can be envisaged based on the results obtained, and what limitations/obstacles can the expert consensus model have in the humanities with respect to the other fields. Is the rationale for developing the new method (or application) clearly explained? No Is the description of the method technically sound? No Are sufficient details provided to allow replication of the method development and its use by others? No If any results are presented, are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions about the method and its performance adequately supported by the findings presented in the article? No Competing Interests No competing interests were disclosed. Reviewer Expertise Higher education and research policy I confirm that I have read this submission and believe that I have an appropriate level of expertise to state that I do not consider it to be of an acceptable scientific standard, for reasons outlined above. reply Respond to this report Responses (2) Author Response 16 Dec 2024 Charlotte Rulkens, Department of Philosophy, Vrije Universiteit Amsterdam, Amsterdam, The Netherlands We sincerely thank the reviewer for her valuable comments, all of which we will address in the revision of our paper. We plan to begin this revision as soon as we receive the second review. We agree with the first and second suggestions to clarify our research questions, as well as the opportunities mentioned for improving the method section. While our method section may differ from those typically found in research papers, it is important to note that this paper is intended as a perspective piece with a didactic focus. It is aimed at highlighting to humanities scholars what is possible in terms of a systematic approach to academic ​​consensus on a topic and to initiate exploration of and discussion about the possible value of applying such an approach. Regarding the third comment, we will ensure that we address the issue of differentiation within research fields and discuss its implications for the application of consensus methods. Concerning the fourth point, we will work to better integrate section 5 with the preceding sections to create a more cohesive flow throughout the paper. View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Author Response 16 Dec 2024 Charlotte Rulkens, Department of Philosophy, Vrije Universiteit Amsterdam, Amsterdam, The Netherlands Dear reviewers, We sincerely thank you for the thorough reviews of our manuscript titled Expert Consensus Methods In The Humanities - An Exploration of their Potential. We appreciate the time and effort you have put into evaluating our work and for providing insightful comments and valuable suggestions. We have carefully considered each of them and have revised the manuscript accordingly. Below, we provide detailed responses to each of the comments. We believe that the revisions have improved the manuscript, and we hope that our responses meet any remaining concerns. Response to reviewer 1: Emanuela Reale, CNR IRCRES—Research Institute for Sustainable Economic Growth, Rome, Italy Reviewer 1, Comment 1: First, the aim of the paper is clear, but the research question(s) related to that aim are missing. This is an important limitation, since the manuscript reports an exploratory effort and therefore a delimitation of the research problem that you want to investigate is very useful to delineate the perimeter of the work. Response: We acknowledge the concern raised and have addressed this by rewriting the introduction section and by incorporating a clearer description of our study goal. We now make it more clear that our goal was primarily didactic, in the sense that we sought to explore the usefulness of consensus methods utilised in the natural, social, and life sciences for the humanities (see section 1, lines 59-83): An important way to make progress in knowledge and understanding is to gradually, via the exchange of evidence, expertise, and critical discussion, reach (more) consensus on a topic. However, whether there is consensus on a certain matter at a moment in time can sometimes remain unknown, be unclear, or remain implicit. Expert consensus methods can lift off this inconclusiveness by intentionally and formally pursuing or assessing consensus. These methods are regularly used in natural, social, and life sciences. Despite the significant role of consensus in the development of (academic) knowledge and understanding (e.g. Kuhn, 1962; Habermas, 1996; thus also Peels, 2020; Grimm et al., forthcoming ), formal expert consensus methods have hardly been explored within in the context of the humanities. [1] Other kinds of valuable expert meetings do take place in various disciplines in the humanities. They usually do not seek to reach a consensus, but to explore a topic in-depth by bringing the relevant experts to the table. Moreover, expert consensus methods described in this article differ from the expert meetings that already take place in the humanities by the former’s procedural and formalised character. In this paper, we aim to highlight to scholars, particularly those in the humanities, what is possible in terms of a systematic approach to academic consensus. In doing so, we aim to initiate exploration of and discussion about the potential of applying such an approach. What would be the added value of applying expert consensus meetings to humanities-related issues? To achieve this, we first explicate our method (§2), before we provide a framework for the application of expert consensus methods by introducing six key elements (§3). Subsequently, we present common types of expert consensus methods utilized in the natural, social, and life sciences. We highlight their and proven value across various types of research through relevant examples and their respective objectives (§4). We then consider both the potential benefits and possible limitations to applying these methods within humanistic research (§5), and conclude with suggestions for future research (§6). Reviewer 1, Comment 2: Second, the section on Method is poor. The composition of the interdisciplinary team (and the reasons for using this team) is not clearly described, the brainstorm session is not properly defined and illustrated, the ‘informal’ literature review is something new for me (what ‘informal’ means?) and it is not acceptable for a research paper to quote two methods used (‘philosophical methods of conceptual analysis’ and ‘conceptual engineering’) without explaining how they have been used for the purposes of the paper but just putting two references. Also, the process of writing and reviewing the manuscript should be better described and justified. Response: Our article, as we now emphasise more, is meant as an exploratory comment and a didactic contribution to inspire scholars in the humanities to see whether consensus methods may be valuable within their own discipline or sub discipline. It is not an instance of systematic empirical research, which would have required a much more detailed methodological section. But we certainly we agree that the method section can be improved in various ways. Therefore, we extended the section by including a specification of the backgrounds of the involved authors and specified the process of our collaboration as follows (see section 2, lines 85-94): To explore the potential of applying expert consensus methods in the humanities, we composed an interdisciplinary team of both researchers from the sciences using consensus methods in practice (LM: epidemiology; TH: qualitative research methodology; LB: epidemiology and data science, research integrity), and researchers familiar with the domain of the humanities and epistemology (CR: art history, museology; RP: epistemology and philosophy of science, religion and theology). In a first brainstorm session, knowledge on the topic was shared from both perspectives. This resulted in a list of consensus methods and examples used in the sciences, drafted by LM and TH. CR carried out a literature search in order to find examples thereof that were applied in her domain. A first draft was written and discussed in a second group session and culminated in a final version, which was edited and reviewed by all authors. Reviewer 1, Comment 3: Third, and more important, the paper refers always to ‘humanities.’ Humanities is a highly fragmented and differentiated field, where many and different epistemic communities work, interact, shape their own idea and uses of consensus method (if any) and where different instances of relationships with other researchers within the field and with neighboring fields are at place. The paper does not acknowledge this important characteristic of humanities (see footnote 1 where there is a list of disciplines included in the humanities that are considered in the paper). The same holds true for STEMs, which are considered as they are all adopting regularly expert consensus methods and there is no explanation or evidence about it. In sum, the paper ‘explores’ the expert consensus method without considering the set of ideas, values, theories, concepts and practices that pertain to the disciplines analyzed and their scientific work. Response: We acknowledge the concern raised and have taken this into account by adjusting a footnote 1 in section 1 (see above) and by revising section 4. Upon reflection we furthermore adjusted the ways to delineate the contexts in which expert consensus methods may be valuable, by focusing on one’s purpose in applying such a method (see section 4, lines 213-235): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 1, Comment 4: Fourth, sections 3 and 4 provide a clear summary of what expert consensus methods are and a deepen analysis of four types. Then the reader expects Section 5 discusses the results with respect to the exploration done. On the contrary, Section 5 lists of pros and cons related to the expert consensus method, and some examples of applications; the contents are completely unrelated with what the sections 3 and 4 have presented and the motivations for considering the expert consensus method an interesting development for humanities are unclear. Here the suggestion is to revise the sections trying to answer the research questions (when formulated) and highlight what we have learnt from the exploratory work, what is the contribution of the brainstorming to the exploration, what new avenues of research can be envisaged based on the results obtained, and what limitations/obstacles can the expert consensus model have in the humanities with respect to the other fields. Response: Thank you for pointing this out to us. Based on this valuable feedback, we have worked to better align section 5 with the preceding sections to create a more cohesive flow throughout the paper and have made more explicit what our line of reasoning is. However, because the paper is intended as preliminary exploration, part of our manuscript aims to identify the potential value of these methods for the humanities. Section 5 is dedicated to that purpose and we made that more explicit in the revision (see section 5, lines 253-315): In the preceding sections, we examined the elements and dynamics of expert consensus methods, along with examples and possibilities for intended objectives. In this section, we delve into the question why formal approaches to consensus in the humanities remain scarce, despite the significant factor of consensus and dissensus in knowledge production across research cultures. Aside from limitations, we also address what the potential added value of applying consensus methods to humanities-related issues could be. First of all, practical limitations likely play a role. Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Scholars may also disagree due to the role of research paradigms or schools within the humanities, as researchers’ positionality can shape their interpretations. In addition to that, humanities research can be concerned with hermeneutics, i.e., issues that have to do with understanding the values and meanings of texts and other objects. And many humanities fields such as ethics, epistemology, and metaphysics, rely on apriori methods like thought experiments, reasoning on the basis of intuitions and principles, and formal logic rather than empirical data. This allows for multiple interpretations that can all to some extent be warranted by the evidence base, and may not readily align into a singular consensus. These factors suggest that the complexity and interpretative nature of humanities research might render expert consensus methods to some extent less relevant. Nevertheless, we believe that there are compelling reasons for considering expert consensus methods in humanities research. First of all, opposing the above concerns, one of the important purposes of expert consensus methods is to bring various viewpoints, nuances, and interpretations to the surface, so that they can be discussed. Depending on the design and purpose of the consensus method, it could reveal levels of consensus and/or dissensus on an issue, within or across schools of thought or paradigms. They might even have the potential to articulate both divergences and common grounds regarding an issue, and by extension increase mutual understanding. The application of such a method may therefore stimulate epistemic progress in itself. Moreover, a carefully designed protocol and curated group of experts can foster a fair practice and ensure the consultation of diverse viewpoints. Expert consensus methods provide opportunities to mitigate biases and their ramifications by this cautious selection of experts, but also by blinding certain parts of the process in relation to anonymity of participating experts, for example. Secondly, expert consensus methods can be helpful in getting a firmer grip on the status quo or state of the art on a particular issue in a field (e.g. Hodgkin et al. (1975) mentioned in Table 2; Rulkens et al. 2022-2023, mentioned in Table 3). Of course, scholars often already have an idea about that, and such an informed opinion is valuable in itself. However, it can be complemented by a more structured approach, particularly when the number of scholars involved is large. This approach can additionally provide guidance for future research by pointing out the gaps in the current evidence base. And when applied on a recurrent basis, it might even have the potential to gain insight in how views evolve over time. It is important to note that consensus is not a final endpoint; rather, it can be continually re-evaluated in light of new insights and reasonable doubts regarding its validity. Third, an advantage of expert consensus methods is that they enhance the transparency and replicability of research because of their well-documented and systematised approach (for replication in the humanities, see Peels and Bouter (2018)). It can therefore be instrumental to epistemic progress beyond the consensus reached at a moment in time. Fourth, application or implementation of its outcomes is often easier after the process of a consensus method is completed (e.g. Standing Committee for the Museum Definition (2020-2022) (2022), mentioned in Table 3). And finally, expert consensus methods can aid public understanding by clarifying experts perspectives on specific issues (Stekelenburg et al., 2022) thereby enabling the public to make informed decisions (Ordway, 2021). In response to this comments, we furthermore extended suggestions for future research in our conclusion (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Response to reviewer 2: Daniel Leising, Technische Universitat Dresden, Dresden, Saxony, Germany Reviewer 2, Comment 1 : First, I would argue that the Humanities are actually not that “unique” in terms of their nature and approaches (page 11). To enable a proper discussion of this issue, the authors could explain their own view of what is so specific about the Humanities in a bit more detail. Here is my view: Interviews and text analyses are used in many different branches of science. Exploratory research is also conducted in many fields. The ways in which people give meaning to their experiences are also of interest across various branches of science. And even in many of the “harder” sciences, research is often “coloured by the backgrounds and positionality of the researchers” (page 8). So, much of what is common in research in the Humanities is actually shared with other fields. In my view, two features may actually distinguish “the Humanities” from other fields - but even these distinctions are only gradual in nature. One feature is the common practice of using everyday language as the primary scientific language. The other feature is the relative fuzziness of many concepts - which I would note is not necessarily a weakness but may often be unavoidable given the complexity of the phenomena of interest (and also given the first feature just mentioned). Do the authors agree? Response: Thank you for challenging us on this point. We agree that the borders between the general fields of the natural, social, and life sciences, and the humanities are fuzzy, when it comes to their objects, purposes, and methods. Whether there is something distinctive about the humanities – we believe that is compatible with fuzziness at the boarders – is a matter that we cannot possibly explore in this article (one of us does so in a book entitled A Philosophy of the Humanities that is forthcoming in 2025 with Oxford University Press, a volume co-authored by Stephen Grimm, Rik Peels, and René van Woudenberg). We do believe though that ordinary language use is not generally characteristic of the humanities (think of the highly technical language of metaphysics and epistemology), nor do we believe that insufficient clarity about core concepts is characteristic of the humanities. We agree though with the idea, also drawn to attention to by reviewer 1 in comment 3, that the humanities are a highly diversified field. We included that by adjusting footnote 1 in section 1: We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). Additionally, to be able to demarcate between contexts in which the formal approach might be beneficial, we decided to put an emphasis on the purposes for which these methods are employed in other domains. This would be a way in which researchers with expertise in their fields could identify opportunities for a formal approach to consensus (see section 4, lines 213-235, also quoted above in response to comment 3 of reviewer 1): The subsequent question would be to determine the contexts in which this approach to consensus would prove most valuable. Research in both the humanities and the sciences spans a wide array of disciplines and sub disciplines, with a vast range of variation in types of research and methods, even within the most specialised areas. Classification of research into one of these domains is often gradual in nature, especially since interdisciplinary research increasingly combines approaches traditionally linked to the ‘humanities’ or ‘sciences’. Making general statements on the types of research in which expert consensus methods might be valuable related to the ‘humanities’ versus the ‘sciences’ is therefore difficult. We decided to examine objectives for employing consensus methods in various contexts in which they have demonstrated their value. This could contribute to a directory of instances in which these methods may prove useful in humanities-related fields. Gattrell et al. (2024, Table 2) present examples of these objectives in context of the health sciences, such as the establishment of clinical practice guidelines, diagnostic guidelines or reporting guidelines but also the classification of diseases, the setting research priorities, or the formulation of policy. These aims are summarised, adjusted and extended in Table 4 in a manner that enhances their generalisability to other fields of research and were included in Table 2 and 3. Although these examples highlight specific applications, we think that there is a broader potential for its use in the humanities. For instance, in establishing consensus and guidelines on the best way to preserve certain historical objects (guidelines/ policy development), involving eyewitnesses in the reconstruction of an historical event (current state of knowledge), or consulting various experts in establishing how to interpret an ancient source (current state of knowledge). Future research and case studies exploring the potential applications outlined in Table 4 could provide insights into specific instances where the pursuit of formal dis- or consensus may prove effective. Reviewer 2, Comment 2: Second, I would like to encourage the authors to say a bit more about possible reasons for why so little consensus building is taking place in many fields of science, despite it being widely viewed as desirable. In my view, there are a couple of reasons, most of which are rather extraneous to an idealised image of science (as a collaborative and cumulative effort at building a reliable knowledge base for the good of mankind). For example, consensus building is likely to reduce the publication output of a field, which would be at odds with the interests of commercial publishers. Also, individual researchers have an interest in “making a name for themselves” based on their specific expertise or views regarding certain topics. Consensus building makes it more difficult to maintain a highly distinct profile and thus reduce visibility. This may be risky in a science system that is as competitive as the one we choose to entertain. And then there may be a need to “save face”: The more of your career you have dedicated to certain ideas (e.g., your own theory of some phenomenon), the harder it may be to let those ideas go - even if the evidence against them is overwhelming. There is a reason why Max Planck once stated that scientific progress happens “one funeral at a time”. At least a brief nod to the relevance of such more political factors would be nice - if the authors share my conviction that they are relevant. Who has an actual interest in achieving consensus? Who has an interest in avoiding the emergence of consensus? And what are the external incentives for and against this type of work ? Response: We think that these are valid points and the reviewers’ publications on the topic shed important light on this. We agree that external incentives may work against a formal approach to consensus as follows and have therefore slightly revised the manuscript on this point. We remain rather brief, though, as the exact explanation of why consensus methods are still relatively rarely used even in the sciences is an issue that is beyond the scope of our paper (see section 5, lines 259-264, also quoted in response to comment 4 of reviewer 1): Expert consensus methods and their subsequent publication demand significant time and resources, which are rarely in abundance in the humanities. Additionally, a factor that may affect academia broadly is the emphasis on publication quantity and individual contributions, which can discourage scholars from lengthy endeavours for which recognition is shared (Leising et al., 2024). Moreover, expert consensus methods are not interwoven in humanities research traditions. Researchers may not be familiar with consensus methods, and the advantages they may have for their area of research. Reviewer 2, Comment 3: Third, I would also appreciate if the authors could talk some more about “bad consensus” – views that are held by many, despite their being wrong. With a particular eye to the Humanities – what criteria are there, apart from consensus, that may help us determine whether a view is “correct”? Or may truth in the Humanities *only* be defined in terms of relative consensus among researchers? In the harder sciences, the ability to *predict* the occurrence of certain events from a theory (and some data input) plays a crucial role as evidence of the theory's correctness. Are the Humanities able to produce testable predictions, too? Can there be consensus projects in the Humanities that actually test one theory against another? Response: This is an interesting point, that would need a elaborative reflection in order to do justice to the complexity of this issue. It is important to note, that we do believe that consensus and truth are two different things, and we made this more clear in our revised paper. Based these suggestions we also incorporated that consensus can be reached based on the wrong reasons and (see section 5, lines 265-271, also quoted in response to comment 4 of reviewer 1): Secondly, concerns raised in the field of psychology that may apply in the humanities as well, highlight risks of consensus-building. Emphasising consensus may lead to agreement for the wrong reasons, such as conformity, which may be harmful to innovation, creativity and minority views. It can sideline dissent, whilst that is a vital element in the evolution of science (Gollwitzer, 2022; Asendorpf & Gebauer, 2022; Beck et al., 2022; Denissen & Sijtsma, 2022; Hilbig et al., 2022). Additionally, experts agreeing on something does not necessarily mean they are correct. They may be in full agreement on a point that is incorrect or untrue. Reviewer 2, Comment 4: Fourth, on page 3 the authors state that “Since the questions addressed in the humanities may leave room for multiple viewpoints that are equally warranted by the evidence base, it can be infeasible or not desirable to get a complete or almost complete consensus.” To this I would respond: what are the conditions under which working toward consensus would not be feasible, or desirable? I agree that this may be the case – but when is it the case, and when is it not the case? I personally hold the somewhat sceptical view that many researchers will shy away from working toward consensus even if doing so *would* be both feasible and desirable (see my previous point). Such a “live and let live” attitude may often be relatively comfortable to have (e.g., to avoid conflict). It may, however, be regarded as unethical, because it will foster the preservation of an unnecessary level of ambiguity. Response: We appreciate this insightful suggestion. Our approach to expert consensus methods does not necessarily imply working towards consensus in all instances, but is also related to the assessment of the (lack of) consensus. We view measuring consensus as a means, not an objective in itself. Furthermore, as put forward in paragraph 3, the bar for ‘sufficient’ consensus is something consensus researchers can determine their selves. Applying a consensus methods is a costly endeavour in terms of time and means. There might be conditions under which the effort of applying such a method does not live up to the epistemic benefits it provides for. We furthermore believe that the feasibility and desirability of assessing consensus is an issue that merits further exploration and included this as follows (see section 5, lines 325-335): However, applying a consensus method is a costly endeavour in terms of time and means. There might be conditions under which these efforts do not live up to the epistemic benefits they provide for. The identification of the particular areas and issues where these methods hold most promise can be done by assessing the purposes these methods have served, and translating those to a specific area of research. The assessment and prioritisation of issues, especially when resources are limited, should be carried out locally by researchers who are familiar with the epistemologies and issues relevant to their field. Consequently, it is essential to carry out future research involving humanities case studies that employ expert consensus methods, to evaluate their feasibility and effectiveness. Collaboration between humanities researchers with scholars from other domains experienced in designing consensus methods could facilitate mutual learning and foster innovative approaches. Minor comments: Reviewer 2, Comment 5: Fifth, the authors state that a steering committee would typically be comprised of the people who initiated the process and who will ultimately author the consensus document. Could there not be cases in which the process should better be overseen by an independent third party (e.g., a moderator)? Especially when issues are highly contentious, for example? And should not all the experts who worked on the consensus be authors on the consensus document? Response: Thank you for this relevant point, this is certainly the case and we made this more clear in the revised manuscript by the following addition (see section 3, lines 193-198): Naturally, the six elements can be specified at by the steering committee. If they deem it valuable to include an independent moderator or to incorporate a member with a critical or divergent perspective—especially if the topic is highly controversial—this remains within their discretion. Additionally, it is up to them to decide who will be listed as co-authors, whether that includes only the steering committee, all participants, or a broader group representation such as the panellists as a whole, for instance. Reviewer 2, Comment 6: Sixth, something is not quite right on page 4, text lines 16-19. Response: Thank you for noticing, we have corrected this sentence as follows (see section 3, lines 136-140): Expert consensus methods are (i) applied by a process leader or steering committee , (ii) to reach consensus or assess the degree of consensus among a group of experts , (iii) about an issue , (iv) based on preset rules of engagement , (v) with the aim to deliver useful output , (vi) for future users . They are characterised by constructive and procedural usage of (dis)agreement and the different arguments that are provided therein. Reviewer 2, Comment 7: Seventh, I would say that evidence will never “speak for itself” (page 4) - otherwise papers in certain scientific fields would not need any Discussion sections. Response: Thank you for rightly pointing this out to us. We adjusted this as follows (see section 3, lines 132-134): This can be useful when the issue at hand is an important one and when evidence is lacking, it is inconsistent, or its interpretation is not obvious. As a result of our reactions to the comments, we also made adjustments to our abstract and conclusion, which are not all quoted in the above letter. We hope these revisions and clarifications meet the reviewers’ expectations. Please let us know if any further revisions are needed. Sincerely, Charlotte Rulkens MA Prof. dr. Rik Peels Dr. Lidwine Mokkink Dr. Tamarinde Haven Prof. em. dr. Lex Bouter [1] We acknowledge that as is the case in the sciences, the humanities encompass a large amount of disciplines and sub fields, exhibiting significant diversity in research types and methodologies. However, in this paper we refer to ‘the humanities’ to clarify difference for the reader, rather than out of belief in rigid categorisations. For readers interested in this topic, see e.g. Woudenberg (2018) and Grimm et al., ( forthcoming ). View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Reale E. Peer Review Report For: Expert Consensus Methods In The Humanities: An Exploration of their Potential [version 2; peer review: 2 approved, 1 approved with reservations] . F1000Research 2024, 13 :710 ( https://doi.org/10.5256/f1000research.163073.r296789) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-710/v1#referee-response-296789 Alongside their report, reviewers assign a status to the article: Approved - the paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations - A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved - fundamental flaws in the paper seriously undermine the findings and conclusions Adjust parameters to alter display View on desktop for interactive features Includes Interactive Elements View on desktop for interactive features Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Stay Updated Sign up for content alerts and receive a weekly or monthly email with all newly published articles Register with F1000Research Already registered? Sign in Not now, thanks close PLEASE NOTE If you are an AUTHOR of this article, please check that you signed in with the account associated with this article otherwise we cannot automatically identify your role as an author and your comment will be labelled as a “User Comment”. If you are a REVIEWER of this article, please check that you have signed in with the account associated with this article and then go to your account to submit your report, please do not post your review here. If you do not have access to your original account, please contact us . All commenters must hold a formal affiliation as per our Policies . The information that you give us will be displayed next to your comment. User comments must be in English, comprehensible and relevant to the article under discussion. We reserve the right to remove any comments that we consider to be inappropriate, offensive or otherwise in breach of the User Comment Terms and Conditions . Commenters must not use a comment for personal attacks. When criticisms of the article are based on unpublished data, the data should be made available. I accept the User Comment Terms and Conditions Please confirm that you accept the User Comment Terms and Conditions. Affiliation ✕ refresh Please enter your institution. Note: To add your institution or organisation, start typing the name and then select the correct name from the list. Where applicable, the name will appear in both the original language and in English. Do not paste in the name. If the name does not appear in the drop-down list, we will display the information you have entered. ✕ refresh Country/Region * USA UK Canada China France Germany Afghanistan Aland Islands Albania Algeria American Samoa Andorra Angola Anguilla Antarctica Antigua and Barbuda Argentina Armenia Aruba Australia Austria Azerbaijan Bahamas Bahrain Bangladesh Barbados Belarus Belgium Belize Benin Bermuda Bhutan Bolivia Bosnia and Herzegovina Botswana Bouvet Island Brazil British Indian Ocean Territory British Virgin Islands Brunei Bulgaria Burkina Faso Burundi Cambodia Cameroon Canada Cape Verde Cayman Islands Central African Republic Chad Chile China Christmas Island Cocos (Keeling) Islands Colombia Comoros Congo Cook Islands Costa Rica Cote d'Ivoire Croatia Cuba Cyprus Czech Republic Democratic Republic of the Congo Denmark Djibouti Dominica Dominican Republic Ecuador Egypt El Salvador Equatorial Guinea Eritrea Estonia Ethiopia Falkland Islands Faroe Islands Federated States of Micronesia Fiji Finland France French Guiana French Polynesia French Southern Territories Gabon Georgia Germany Ghana Gibraltar Greece Greenland Grenada Guadeloupe Guam Guatemala Guernsey Guinea Guinea-Bissau Guyana Haiti Heard Island and Mcdonald Islands Holy See (Vatican City State) Honduras Hong Kong Hungary Iceland India Indonesia Iran Iraq Ireland Israel Italy Jamaica Japan Jersey Jordan Kazakhstan Kenya Kiribati Kosovo (Serbia and Montenegro) Kuwait Kyrgyzstan Lao People's Democratic Republic Latvia Lebanon Lesotho Liberia Libya Liechtenstein Lithuania Luxembourg Macao Madagascar Malawi Malaysia Maldives Mali Malta Marshall Islands Martinique Mauritania Mauritius Mayotte Mexico Minor Outlying Islands of the United States Moldova Monaco Mongolia Montenegro Montserrat Morocco Mozambique Myanmar Namibia Nauru Nepal Netherlands Antilles New Caledonia New Zealand Nicaragua Niger Nigeria Niue Norfolk Island North Korea North Macedonia Northern Mariana Islands Norway Oman Pakistan Palau Palestinian Territory Panama Papua New Guinea Paraguay Peru Philippines Pitcairn Poland Portugal Puerto Rico Qatar Reunion Romania Russian Federation Rwanda Saint Helena Saint Kitts and Nevis Saint Lucia Saint Pierre and Miquelon Saint Vincent and the Grenadines Samoa San Marino Sao Tome and Principe Saudi Arabia Senegal Serbia Seychelles Sierra Leone Singapore Slovakia Slovenia Solomon Islands Somalia South Africa South Georgia and the South Sandwich Is South Korea South Sudan Spain Sri Lanka Sudan Suriname Svalbard and Jan Mayen Swaziland Sweden Switzerland Syria Taiwan Tajikistan Tanzania Thailand The Gambia The Netherlands Timor-Leste Togo Tokelau Tonga Trinidad and Tobago Tunisia Turkey Turkmenistan Turks and Caicos Islands Tuvalu UK USA Uganda Ukraine United Arab Emirates United States Virgin Islands Uruguay Uzbekistan Vanuatu Venezuela Vietnam Wallis and Futuna West Bank and Gaza Strip Western Sahara Yemen Zambia Zimbabwe Please select your country/region. You must enter a comment. Competing Interests Please disclose any competing interests that might be construed to influence your judgment of the article's or peer review report's validity or importance. Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Please state your competing interests The comment has been saved. An error has occurred. Please try again. Cancel Post var lTitle = "Expert Consensus Methods In The Humanities:...".replace("'", ''); var linkedInUrl = "http://www.linkedin.com/shareArticle?url=https://f1000research.com/articles/13-710/v2" + "&title=" + encodeURIComponent(lTitle) + "&summary=" + encodeURIComponent('Read the article by '); var deliciousUrl = "https://del.icio.us/post?url=https://f1000research.com/articles/13-710/v2&title=" + encodeURIComponent(lTitle); var redditUrl = "http://reddit.com/submit?url=https://f1000research.com/articles/13-710/v2" + "&title=" + encodeURIComponent(lTitle); linkedInUrl += encodeURIComponent('Rulkens CCS et al.'); var offsetTop = /chrome/i.test( navigator.userAgent ) ? 4 : -10; var addthis_config = { ui_offset_top: offsetTop, services_compact : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_expanded : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_custom : [ { name: "LinkedIn", url: linkedInUrl, icon:"/img/icon/at_linkedin.svg" }, { name: "Mendeley", url: "http://www.mendeley.com/import/?url=https://f1000research.com/articles/13-710/v2/mendeley", icon:"/img/icon/at_mendeley.svg" }, { name: "Reddit", url: redditUrl, icon:"/img/icon/at_reddit.svg" }, ] }; var addthis_share = { url: "https://f1000research.com/articles/13-710", templates : { twitter : "Expert Consensus Methods In The Humanities: An Exploration of.... Rulkens CCS et al., published by " + "@F1000Research" + ", https://f1000research.com/articles/13-710/v2" } }; if (typeof(addthis) != "undefined"){ addthis.addEventListener('addthis.ready', checkCount); addthis.addEventListener('addthis.menu.share', checkCount); } $(".f1r-shares-twitter").attr("href", "https://twitter.com/intent/tweet?text=" + addthis_share.templates.twitter); $(".f1r-shares-facebook").attr("href", "https://www.facebook.com/sharer/sharer.php?u=" + addthis_share.url); $(".f1r-shares-linkedin").attr("href", addthis_config.services_custom[0].url); $(".f1r-shares-reddit").attr("href", addthis_config.services_custom[2].url); $(".f1r-shares-mendelay").attr("href", addthis_config.services_custom[1].url); function checkCount(){ setTimeout(function(){ $(".addthis_button_expanded").each(function(){ var count = $(this).text(); if (count !== "" && count != "0") $(this).removeClass("is-hidden"); else $(this).addClass("is-hidden"); }); }, 1000); } close How to cite this report {{reportCitation}} Cancel Copy Citation Details $(function(){R.ui.buttonDropdowns('.dropdown-for-downloads');}); $(function(){R.ui.toolbarDropdowns('.toolbar-dropdown-for-downloads');}); $.get("/articles/acj/148726/174565") new F1000.Clipboard(); new F1000.ThesaurusTermsDisplay("articles", "article", "174565"); $(document).ready(function() { $( "#frame1" ).on('load', function() { var mydiv = $(this).contents().find("div"); var h = mydiv.height(); console.log(h) }); var tooltipLivingFigure = jQuery(".interactive-living-figure-label .icon-more-info"), titleLivingFigure = tooltipLivingFigure.attr("title"); tooltipLivingFigure.simpletip({ fixed: true, position: ["-115", "30"], baseClass: 'small-tooltip', content:titleLivingFigure + " " }); tooltipLivingFigure.removeAttr("title"); $("body").on("click", ".cite-living-figure", function(e) { e.preventDefault(); var ref = $(this).attr("data-ref"); $(this).closest(".living-figure-list-container").find("#" + ref).fadeIn(200); }); $("body").on("click", ".close-cite-living-figure", function(e) { e.preventDefault(); $(this).closest(".popup-window-wrapper").fadeOut(200); }); $(document).on("mouseup", function(e) { var metricsContainer = $(".article-metrics-popover-wrapper"); if (!metricsContainer.is(e.target) && metricsContainer.has(e.target).length === 0) { $(".article-metrics-close-button").click(); } }); var articleId = $('#articleId').val(); if($("#main-article-count-box").attachArticleMetrics) { $("#main-article-count-box").attachArticleMetrics(articleId, { articleMetricsView: true }); } }); var figshareWidget = $(".new_figshare_widget"); if (figshareWidget.length > 0) { window.figshare.load("f1000", function(Widget) { // Select a tag/tags defined in your page. In this tag we will place the widget. _.map(figshareWidget, function(el){ var widget = new Widget({ articleId: $(el).attr("figshare_articleId") //height:300 // this is the height of the viewer part. [Default: 550] }); widget.initialize(); // initialize the widget widget.mount(el); // mount it in a tag that's on your page // this will save the widget on the global scope for later use from // your JS scripts. This line is optional. //window.widget = widget; }); }); } close Error Close Add Reset F1000.MICROSERVICES.AFFILIATION = ''; $(document).ready(function () { $('.js-affiliations-form').each((index, form) => { new AffiliationForm({ formId: form.id, institutionErrorSelector: '.comment-enter-institution', departmentErrorSelector: '.comment-enter-department', placeSelector: '.js-add-comment-place', stateSelector: '.js-add-comment-state', zipCodeSelector: '.js-add-comment-zipcode', countrySelector: '.js-add-comment-country', countryErrorSelector: '.comment-enter-country', }); }); }); $(document).ready(function () { var reportIds = { "349191": 17, "349190": 10, "352671": 0, "352670": 0, "352677": 0, "352676": 0, "352678": 0, "352673": 0, "352672": 0, "352675": 0, "352674": 0, "309044": 0, "309045": 0, "309046": 0, "309047": 0, "309052": 0, "309053": 0, "309048": 0, "309049": 44, "309050": 0, "309051": 0, "301125": 0, "301126": 0, "301127": 0, "301132": 0, "301133": 0, "301134": 0, "301128": 0, "301129": 0, "301130": 0, "301131": 0, "350421": 0, "350420": 0, "296789": 40, "350423": 0, "296790": 0, "350422": 0, "296791": 0, "350417": 0, "350416": 0, "350419": 0, "350418": 0, "296796": 0, "296797": 0, "296798": 0, "350425": 0, "296792": 0, "350424": 0, "296793": 0, "296794": 0, "296795": 0, "355053": 0, "355052": 0, "355055": 0, "355054": 0, "355051": 0, "355050": 0, "355057": 0, "355056": 8, "355059": 0, "355058": 0, }; $(".referee-response-container,.js-referee-report").each(function(index, el) { var reportId = $(el).attr("data-reportid"), reportCount = reportIds[reportId] || 0; $(el).find(".comments-count-container,.js-referee-report-views").html(reportCount); }); var uuidInput = $("#article_uuid"), oldUUId = uuidInput.val(), newUUId = "ade0c1be-ca74-41af-9c61-fe175faee274"; uuidInput.val(newUUId); $("a[href*='article_uuid=']").each(function(index, el) { var newHref = $(el).attr("href").replace(oldUUId, newUUId); $(el).attr("href", newHref); }); }); An innovative open access publishing platform offering rapid publication and open peer review, whilst supporting data deposition and sharing. Browse Gateways Collections How it Works Contact For Developers Cookie Notice Privacy Notice RSS Submit Your Research Follow us © 2012-2026 F1000 Research Ltd. ISSN 2046-1402 | Legal | Partner of Research4Life • CrossRef • ORCID • FAIRSharing R.templateTests.simpleTemplate = R.template(' $text $text $text $text $text '); R.templateTests.runTests(); var F1000platform = new F1000.Platform({ name: "f1000research", displayName: "F1000Research", hostName: "f1000research.com", id: "1", editorialEmail: "[email protected]", infoEmail: "[email protected]", usePmcStats: true }); $(function(){R.ui.dropdowns('.dropdown-for-authors, .dropdown-for-about, .dropdown-for-myresearch');}); // $(function(){R.ui.dropdowns('.dropdown-for-referees');}); $(document).ready(function () { if ($(".cookie-warning").is(":visible")) { $(".sticky").css("margin-bottom", "35px"); $(".devices").addClass("devices-and-cookie-warning"); } $(".cookie-warning .close-button").click(function (e) { $(".devices").removeClass("devices-and-cookie-warning"); $(".sticky").css("margin-bottom", "0"); }); $("#tweeter-feed .tweet-message").each(function (i, message) { var self = $(message); self.html(linkify(self.html())); }); $(".partner").on("mouseenter mouseleave", function() { $(this).find(".gray-scale, .colour").toggleClass("is-hidden"); }); }); Sign In Remember me Forgotten your password? Sign In Cancel Email or password not correct. Please try again Please wait... $(function(){ // Note: All the setup needs to run against a name attribute and *not* the id due the clonish // nature of facebox... $("a[id=googleSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("GOOGLE"); $("form[id=oAuthForm]").submit(); }); $("a[id=facebookSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("FACEBOOK"); $("form[id=oAuthForm]").submit(); }); $("a[id=orcidSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("ORCID"); $("form[id=oAuthForm]").submit(); }); }); If you've forgotten your password, please enter your email address below and we'll send you instructions on how to reset your password. The email address should be the one you originally registered with F1000. Email address not valid, please try again You registered with F1000 via Google, so we cannot reset your password. To sign in, please click here . If you still need help with your Google account password, please click here . You registered with F1000 via Facebook, so we cannot reset your password. To sign in, please click here . If you still need help with your Facebook account password, please click here . Code not correct, please try again Reset password Cancel Email us for further assistance. Server error, please try again. If your email address is registered with us, we will email you instructions to reset your password. If you think you should have received this email but it has not arrived, please check your spam filters and/or contact for further assistance. Please wait... Register $(document).ready(function () { signIn.createSignInAsRow($("#sign-in-form-gfb-popup")); $(".target-field").each(function () { var uris = $(this).val().split("/"); if (uris.pop() === "login") { $(this).val(uris.toString().replace(",","/")); } }); });