Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal

Towards an Integrated Performance Management and... | F1000Research "use strict";function _typeof(t){return(_typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}!function(){var t=function(){var t,e,o=[],n=window,r=n;for(;r;){try{if(r.frames.__tcfapiLocator){t=r;break}}catch(t){}if(r===n.top)break;r=r.parent}t||(!function t(){var e=n.document,o=!!n.frames.__tcfapiLocator;if(!o)if(e.body){var r=e.createElement("iframe");r.style.cssText="display:none",r.name="__tcfapiLocator",e.body.appendChild(r)}else setTimeout(t,5);return!o}(),n.__tcfapi=function(){for(var t=arguments.length,n=new Array(t),r=0;r 3&&2===parseInt(n[1],10)&&"boolean"==typeof n[3]&&(e=n[3],"function"==typeof n[2]&&n[2]("set",!0)):"ping"===n[0]?"function"==typeof n[2]&&n[2]({gdprApplies:e,cmpLoaded:!1,cmpStatus:"stub"}):o.push(n)},n.addEventListener("message",(function(t){var e="string"==typeof t.data,o={};if(e)try{o=JSON.parse(t.data)}catch(t){}else o=t.data;var n="object"===_typeof(o)&&null!==o?o.__tcfapiCall:null;n&&window.__tcfapi(n.command,n.version,(function(o,r){var a={__tcfapiReturn:{returnValue:o,success:r,callId:n.callId}};t&&t.source&&t.source.postMessage&&t.source.postMessage(e?JSON.stringify(a):a,"*")}),n.parameter)}),!1))};"undefined"!=typeof module?module.exports=t:t()}(); dataLayer = dataLayer || []; // Standard GTM initialization - Google Consent Mode handles consent automatically (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl+ '>m_auth=hzk0Vc3qFsQYhCrIoHz68A>m_preview=env-1>m_cookies_win=x';f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-MWFK8L5J'); ;window.NREUM||(NREUM={});NREUM.init={distributed_tracing:{enabled:true},privacy:{cookies_enabled:true},ajax:{deny_list:["bam.nr-data.net"]}}; ;NREUM.loader_config={accountID:"438030",trustKey:"438030",agentID:"772317073",licenseKey:"97f8f67f26",applicationID:"772317073"} ;NREUM.info={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net",licenseKey:"97f8f67f26",applicationID:"772317073",sa:1} ;/*! For license information please see nr-loader-spa-1.236.0.min.js.LICENSE.txt */ (()=>{"use strict";var e,t,r={5763:(e,t,r)=>{r.d(t,{P_:()=>l,Mt:()=>g,C5:()=>s,DL:()=>v,OP:()=>T,lF:()=>D,Yu:()=>y,Dg:()=>h,CX:()=>c,GE:()=>b,sU:()=>_});var n=r(8632),i=r(9567);const o={beacon:n.ce.beacon,errorBeacon:n.ce.errorBeacon,licenseKey:void 0,applicationID:void 0,sa:void 0,queueTime:void 0,applicationTime:void 0,ttGuid:void 0,user:void 0,account:void 0,product:void 0,extra:void 0,jsAttributes:{},userAttributes:void 0,atts:void 0,transactionName:void 0,tNamePlain:void 0},a={};function s(e){if(!e)throw new Error("All info objects require an agent identifier!");if(!a[e])throw new Error("Info for ".concat(e," was never set"));return a[e]}function c(e,t){if(!e)throw new Error("All info objects require an agent identifier!");a[e]=(0,i.D)(t,o),(0,n.Qy)(e,a[e],"info")}var u=r(7056);const d=()=>{const e={blockSelector:"[data-nr-block]",maskInputOptions:{password:!0}};return{allow_bfcache:!0,privacy:{cookies_enabled:!0},ajax:{deny_list:void 0,enabled:!0,harvestTimeSeconds:10},distributed_tracing:{enabled:void 0,exclude_newrelic_header:void 0,cors_use_newrelic_header:void 0,cors_use_tracecontext_headers:void 0,allowed_origins:void 0},session:{domain:void 0,expiresMs:u.oD,inactiveMs:u.Hb},ssl:void 0,obfuscate:void 0,jserrors:{enabled:!0,harvestTimeSeconds:10},metrics:{enabled:!0},page_action:{enabled:!0,harvestTimeSeconds:30},page_view_event:{enabled:!0},page_view_timing:{enabled:!0,harvestTimeSeconds:30,long_task:!1},session_trace:{enabled:!0,harvestTimeSeconds:10},harvest:{tooManyRequestsDelay:60},session_replay:{enabled:!1,harvestTimeSeconds:60,sampleRate:.1,errorSampleRate:.1,maskTextSelector:"*",maskAllInputs:!0,get blockClass(){return"nr-block"},get ignoreClass(){return"nr-ignore"},get maskTextClass(){return"nr-mask"},get blockSelector(){return e.blockSelector},set blockSelector(t){e.blockSelector+=",".concat(t)},get maskInputOptions(){return e.maskInputOptions},set maskInputOptions(t){e.maskInputOptions={...t,password:!0}}},spa:{enabled:!0,harvestTimeSeconds:10}}},f={};function l(e){if(!e)throw new Error("All configuration objects require an agent identifier!");if(!f[e])throw new Error("Configuration for ".concat(e," was never set"));return f[e]}function h(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");f[e]=(0,i.D)(t,d()),(0,n.Qy)(e,f[e],"config")}function g(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");var r=l(e);if(r){for(var n=t.split("."),i=0;i {r.d(t,{D:()=>i});var n=r(50);function i(e,t){try{if(!e||"object"!=typeof e)return(0,n.Z)("Setting a Configurable requires an object as input");if(!t||"object"!=typeof t)return(0,n.Z)("Setting a Configurable requires a model to set its initial properties");const r=Object.create(Object.getPrototypeOf(t),Object.getOwnPropertyDescriptors(t)),o=0===Object.keys(r).length?e:r;for(let a in o)if(void 0!==e[a])try{"object"==typeof e[a]&&"object"==typeof t[a]?r[a]=i(e[a],t[a]):r[a]=e[a]}catch(e){(0,n.Z)("An error occurred while setting a property of a Configurable",e)}return r}catch(e){(0,n.Z)("An error occured while setting a Configurable",e)}}},6818:(e,t,r)=>{r.d(t,{Re:()=>i,gF:()=>o,q4:()=>n});const n="1.236.0",i="PROD",o="CDN"},385:(e,t,r)=>{r.d(t,{FN:()=>a,IF:()=>u,Nk:()=>f,Tt:()=>s,_A:()=>o,il:()=>n,pL:()=>c,v6:()=>i,w1:()=>d});const n="undefined"!=typeof window&&!!window.document,i="undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self.navigator instanceof WorkerNavigator||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis.navigator instanceof WorkerNavigator),o=n?window:"undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis),a=""+o?.location,s=/iPad|iPhone|iPod/.test(navigator.userAgent),c=s&&"undefined"==typeof SharedWorker,u=(()=>{const e=navigator.userAgent.match(/Firefox[/\s](\d+\.\d+)/);return Array.isArray(e)&&e.length>=2?+e[1]:0})(),d=Boolean(n&&window.document.documentMode),f=!!navigator.sendBeacon},1117:(e,t,r)=>{r.d(t,{w:()=>o});var n=r(50);const i={agentIdentifier:"",ee:void 0};class o{constructor(e){try{if("object"!=typeof e)return(0,n.Z)("shared context requires an object as input");this.sharedContext={},Object.assign(this.sharedContext,i),Object.entries(e).forEach((e=>{let[t,r]=e;Object.keys(i).includes(t)&&(this.sharedContext[t]=r)}))}catch(e){(0,n.Z)("An error occured while setting SharedContext",e)}}}},8e3:(e,t,r)=>{r.d(t,{L:()=>d,R:()=>c});var n=r(2177),i=r(1284),o=r(4322),a=r(3325);const s={};function c(e,t){const r={staged:!1,priority:a.p[t]||0};u(e),s[e].get(t)||s[e].set(t,r)}function u(e){e&&(s[e]||(s[e]=new Map))}function d(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"",t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"feature";if(u(e),!e||!s[e].get(t))return a(t);s[e].get(t).staged=!0;const r=[...s[e]];function a(t){const r=e?n.ee.get(e):n.ee,a=o.X.handlers;if(r.backlog&&a){var s=r.backlog[t],c=a[t];if(c){for(var u=0;s&&u {let[t,r]=e;return r.staged}))&&(r.sort(((e,t)=>e[1].priority-t[1].priority)),r.forEach((e=>{let[t]=e;a(t)})))}function f(e,t){var r=e[1];(0,i.D)(t[r],(function(t,r){var n=e[0];if(r[0]===n){var i=r[1],o=e[3],a=e[2];i.apply(o,a)}}))}},2177:(e,t,r)=>{r.d(t,{c:()=>f,ee:()=>u});var n=r(8632),i=r(2210),o=r(1284),a=r(5763),s="nr@context";let c=(0,n.fP)();var u;function d(){}function f(e){return(0,i.X)(e,s,l)}function l(){return new d}function h(){u.aborted=!0,u.backlog={}}c.ee?u=c.ee:(u=function e(t,r){var n={},c={},f={},g=!1;try{g=16===r.length&&(0,a.OP)(r).isolatedBacklog}catch(e){}var p={on:b,addEventListener:b,removeEventListener:y,emit:v,get:x,listeners:w,context:m,buffer:A,abort:h,aborted:!1,isBuffering:E,debugId:r,backlog:g?{}:t&&"object"==typeof t.backlog?t.backlog:{}};return p;function m(e){return e&&e instanceof d?e:e?(0,i.X)(e,s,l):l()}function v(e,r,n,i,o){if(!1!==o&&(o=!0),!u.aborted||i){t&&o&&t.emit(e,r,n);for(var a=m(n),s=w(e),d=s.length,f=0;fn,p:()=>i});var n=r(2177).ee.get("handle");function i(e,t,r,i,o){o?(o.buffer([e],i),o.emit(e,t,r)):(n.buffer([e],i),n.emit(e,t,r))}},4322:(e,t,r)=>{r.d(t,{X:()=>o});var n=r(5546);o.on=a;var i=o.handlers={};function o(e,t,r,o){a(o||n.E,i,e,t,r)}function a(e,t,r,i,o){o||(o="feature"),e||(e=n.E);var a=t[o]=t[o]||{};(a[r]=a[r]||[]).push([e,i])}},3239:(e,t,r)=>{r.d(t,{bP:()=>s,iz:()=>c,m$:()=>a});var n=r(385);let i=!1,o=!1;try{const e={get passive(){return i=!0,!1},get signal(){return o=!0,!1}};n._A.addEventListener("test",null,e),n._A.removeEventListener("test",null,e)}catch(e){}function a(e,t){return i||o?{capture:!!e,passive:i,signal:t}:!!e}function s(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;window.addEventListener(e,t,a(r,n))}function c(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;document.addEventListener(e,t,a(r,n))}},4402:(e,t,r)=>{r.d(t,{Ht:()=>u,M:()=>c,Rl:()=>a,ky:()=>s});var n=r(385);const i="xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx";function o(e,t){return e?15&e[t]:16*Math.random()|0}function a(){const e=n._A?.crypto||n._A?.msCrypto;let t,r=0;return e&&e.getRandomValues&&(t=e.getRandomValues(new Uint8Array(31))),i.split("").map((e=>"x"===e?o(t,++r).toString(16):"y"===e?(3&o()|8).toString(16):e)).join("")}function s(e){const t=n._A?.crypto||n._A?.msCrypto;let r,i=0;t&&t.getRandomValues&&(r=t.getRandomValues(new Uint8Array(31)));const a=[];for(var s=0;s {r.d(t,{Bq:()=>n,Hb:()=>o,oD:()=>i});const n="NRBA",i=144e5,o=18e5},7894:(e,t,r)=>{function n(){return Math.round(performance.now())}r.d(t,{z:()=>n})},7243:(e,t,r)=>{r.d(t,{e:()=>o});var n=r(385),i={};function o(e){if(e in i)return i[e];if(0===(e||"").indexOf("data:"))return{protocol:"data"};let t;var r=n._A?.location,o={};if(n.il)t=document.createElement("a"),t.href=e;else try{t=new URL(e,r.href)}catch(e){return o}o.port=t.port;var a=t.href.split("://");!o.port&&a[1]&&(o.port=a[1].split("/")[0].split("@").pop().split(":")[1]),o.port&&"0"!==o.port||(o.port="https"===a[0]?"443":"80"),o.hostname=t.hostname||r.hostname,o.pathname=t.pathname,o.protocol=a[0],"/"!==o.pathname.charAt(0)&&(o.pathname="/"+o.pathname);var s=!t.protocol||":"===t.protocol||t.protocol===r.protocol,c=t.hostname===r.hostname&&t.port===r.port;return o.sameOrigin=s&&(!t.hostname||c),"/"===o.pathname&&(i[e]=o),o}},50:(e,t,r)=>{function n(e,t){"function"==typeof console.warn&&(console.warn("New Relic: ".concat(e)),t&&console.warn(t))}r.d(t,{Z:()=>n})},2587:(e,t,r)=>{r.d(t,{N:()=>c,T:()=>u});var n=r(2177),i=r(5546),o=r(8e3),a=r(3325);const s={stn:[a.D.sessionTrace],err:[a.D.jserrors,a.D.metrics],ins:[a.D.pageAction],spa:[a.D.spa],sr:[a.D.sessionReplay,a.D.sessionTrace]};function c(e,t){const r=n.ee.get(t);e&&"object"==typeof e&&(Object.entries(e).forEach((e=>{let[t,n]=e;void 0===u[t]&&(s[t]?s[t].forEach((e=>{n?(0,i.p)("feat-"+t,[],void 0,e,r):(0,i.p)("block-"+t,[],void 0,e,r),(0,i.p)("rumresp-"+t,[Boolean(n)],void 0,e,r)})):n&&(0,i.p)("feat-"+t,[],void 0,void 0,r),u[t]=Boolean(n))})),Object.keys(s).forEach((e=>{void 0===u[e]&&(s[e]?.forEach((t=>(0,i.p)("rumresp-"+e,[!1],void 0,t,r))),u[e]=!1)})),(0,o.L)(t,a.D.pageViewEvent))}const u={}},2210:(e,t,r)=>{r.d(t,{X:()=>i});var n=Object.prototype.hasOwnProperty;function i(e,t,r){if(n.call(e,t))return e[t];var i=r();if(Object.defineProperty&&Object.keys)try{return Object.defineProperty(e,t,{value:i,writable:!0,enumerable:!1}),i}catch(e){}return e[t]=i,i}},1284:(e,t,r)=>{r.d(t,{D:()=>n});const n=(e,t)=>Object.entries(e||{}).map((e=>{let[r,n]=e;return t(r,n)}))},4351:(e,t,r)=>{r.d(t,{P:()=>o});var n=r(2177);const i=()=>{const e=new WeakSet;return(t,r)=>{if("object"==typeof r&&null!==r){if(e.has(r))return;e.add(r)}return r}};function o(e){try{return JSON.stringify(e,i())}catch(e){try{n.ee.emit("internal-error",[e])}catch(e){}}}},3960:(e,t,r)=>{r.d(t,{K:()=>a,b:()=>o});var n=r(3239);function i(){return"undefined"==typeof document||"complete"===document.readyState}function o(e,t){if(i())return e();(0,n.bP)("load",e,t)}function a(e){if(i())return e();(0,n.iz)("DOMContentLoaded",e)}},8632:(e,t,r)=>{r.d(t,{EZ:()=>u,Qy:()=>c,ce:()=>o,fP:()=>a,gG:()=>d,mF:()=>s});var n=r(7894),i=r(385);const o={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net"};function a(){return i._A.NREUM||(i._A.NREUM={}),void 0===i._A.newrelic&&(i._A.newrelic=i._A.NREUM),i._A.NREUM}function s(){let e=a();return e.o||(e.o={ST:i._A.setTimeout,SI:i._A.setImmediate,CT:i._A.clearTimeout,XHR:i._A.XMLHttpRequest,REQ:i._A.Request,EV:i._A.Event,PR:i._A.Promise,MO:i._A.MutationObserver,FETCH:i._A.fetch}),e}function c(e,t,r){let i=a();const o=i.initializedAgents||{},s=o[e]||{};return Object.keys(s).length||(s.initializedAt={ms:(0,n.z)(),date:new Date}),i.initializedAgents={...o,[e]:{...s,[r]:t}},i}function u(e,t){a()[e]=t}function d(){return function(){let e=a();const t=e.info||{};e.info={beacon:o.beacon,errorBeacon:o.errorBeacon,...t}}(),function(){let e=a();const t=e.init||{};e.init={...t}}(),s(),function(){let e=a();const t=e.loader_config||{};e.loader_config={...t}}(),a()}},7956:(e,t,r)=>{r.d(t,{N:()=>i});var n=r(3239);function i(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],r=arguments.length>2?arguments[2]:void 0,i=arguments.length>3?arguments[3]:void 0;return void(0,n.iz)("visibilitychange",(function(){if(t)return void("hidden"==document.visibilityState&&e());e(document.visibilityState)}),r,i)}},1214:(e,t,r)=>{r.d(t,{em:()=>v,u5:()=>N,QU:()=>S,_L:()=>I,Gm:()=>L,Lg:()=>M,gy:()=>U,BV:()=>Q,Kf:()=>ee});var n=r(2177);const i="nr@original";var o=Object.prototype.hasOwnProperty,a=!1;function s(e,t){return e||(e=n.ee),r.inPlace=function(e,t,n,i,o){n||(n="");var a,s,c,u="-"===n.charAt(0);for(c=0;c 2?n-2:0),o=2;o {r(A[T],e,w),r(E[T],e,w)})),r(l._A,"fetch",y),t.on(y+"end",(function(e,r){var n=this;if(r){var i=r.headers.get("content-length");null!==i&&(n.rxSize=i),t.emit(y+"done",[null,r],n)}else t.emit(y+"done",[e],n)})),t}const O={},j=["pushState","replaceState"];function S(e){const t=function(e){return(e||n.ee).get("history")}(e);return!l.il||O[t.debugId]++||(O[t.debugId]=1,s(t).inPlace(window.history,j,"-")),t}var P=r(3239);const C={},R=["appendChild","insertBefore","replaceChild"];function I(e){const t=function(e){return(e||n.ee).get("jsonp")}(e);if(!l.il||C[t.debugId])return t;C[t.debugId]=!0;var r=s(t),i=/[?&](?:callback|cb)=([^&#]+)/,o=/(.*)\.([^.]+)/,a=/^(\w+)(\.|$)(.*)$/;function c(e,t){var r=e.match(a),n=r[1],i=r[3];return i?c(i,t[n]):t[n]}return r.inPlace(Node.prototype,R,"dom-"),t.on("dom-start",(function(e){!function(e){if(!e||"string"!=typeof e.nodeName||"script"!==e.nodeName.toLowerCase())return;if("function"!=typeof e.addEventListener)return;var n=(a=e.src,s=a.match(i),s?s[1]:null);var a,s;if(!n)return;var u=function(e){var t=e.match(o);if(t&&t.length>=3)return{key:t[2],parent:c(t[1],window)};return{key:e,parent:window}}(n);if("function"!=typeof u.parent[u.key])return;var d={};function f(){t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}function l(){t.emit("jsonp-error",[],d),t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}r.inPlace(u.parent,[u.key],"cb-",d),e.addEventListener("load",f,(0,P.m$)(!1)),e.addEventListener("error",l,(0,P.m$)(!1)),t.emit("new-jsonp",[e.src],d)}(e[0])})),t}var k=r(5763);const H={};function L(e){const t=function(e){return(e||n.ee).get("mutation")}(e);if(!l.il||H[t.debugId])return t;H[t.debugId]=!0;var r=s(t),i=k.Yu.MO;return i&&(window.MutationObserver=function(e){return this instanceof i?new i(r(e,"fn-")):i.apply(this,arguments)},MutationObserver.prototype=i.prototype),t}const z={};function M(e){const t=function(e){return(e||n.ee).get("promise")}(e);if(z[t.debugId])return t;z[t.debugId]=!0;var r=n.c,o=s(t),a=k.Yu.PR;return a&&function(){function e(r){var n=t.context(),i=o(r,"executor-",n,null,!1);const s=Reflect.construct(a,[i],e);return t.context(s).getCtx=function(){return n},s}l._A.Promise=e,Object.defineProperty(e,"name",{value:"Promise"}),e.toString=function(){return a.toString()},Object.setPrototypeOf(e,a),["all","race"].forEach((function(r){const n=a[r];e[r]=function(e){let i=!1;[...e||[]].forEach((e=>{this.resolve(e).then(a("all"===r),a(!1))}));const o=n.apply(this,arguments);return o;function a(e){return function(){t.emit("propagate",[null,!i],o,!1,!1),i=i||!e}}}})),["resolve","reject"].forEach((function(r){const n=a[r];e[r]=function(e){const r=n.apply(this,arguments);return e!==r&&t.emit("propagate",[e,!0],r,!1,!1),r}})),e.prototype=a.prototype;const n=a.prototype.then;a.prototype.then=function(){var e=this,i=r(e);i.promise=e;for(var a=arguments.length,s=new Array(a),c=0;c e())),t};function m(e,t){i.inPlace(t,["onreadystatechange"],"fn-",E)}function b(){var e=this,t=r.context(e);e.readyState>3&&!t.resolved&&(t.resolved=!0,r.emit("xhr-resolved",[],e)),i.inPlace(e,f,"fn-",E)}if(function(e,t){for(var r in e)t[r]=e[r]}(o,p),p.prototype=o.prototype,i.inPlace(p.prototype,J,"-xhr-",E),r.on("send-xhr-start",(function(e,t){m(e,t),function(e){h.push(e),a&&(y?y.then(A):u?u(A):(w=-w,x.data=w))}(t)})),r.on("open-xhr-start",m),a){var y=c&&c.resolve();if(!u&&!c){var w=1,x=document.createTextNode(w);new a(A).observe(x,{characterData:!0})}}else t.on("fn-end",(function(e){e[0]&&e[0].type===d||A()}));function A(){for(var e=0;e {r.d(t,{t:()=>n});const n=r(3325).D.ajax},6660:(e,t,r)=>{r.d(t,{A:()=>i,t:()=>n});const n=r(3325).D.jserrors,i="nr@seenError"},3081:(e,t,r)=>{r.d(t,{gF:()=>o,mY:()=>i,t9:()=>n,vz:()=>s,xS:()=>a});const n=r(3325).D.metrics,i="sm",o="cm",a="storeSupportabilityMetrics",s="storeEventMetrics"},4649:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageAction},7633:(e,t,r)=>{r.d(t,{Dz:()=>i,OJ:()=>a,qw:()=>o,t9:()=>n});const n=r(3325).D.pageViewEvent,i="firstbyte",o="domcontent",a="windowload"},9251:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageViewTiming},3614:(e,t,r)=>{r.d(t,{BST_RESOURCE:()=>i,END:()=>s,FEATURE_NAME:()=>n,FN_END:()=>u,FN_START:()=>c,PUSH_STATE:()=>d,RESOURCE:()=>o,START:()=>a});const n=r(3325).D.sessionTrace,i="bstResource",o="resource",a="-start",s="-end",c="fn"+a,u="fn"+s,d="pushState"},7836:(e,t,r)=>{r.d(t,{BODY:()=>A,CB_END:()=>E,CB_START:()=>u,END:()=>x,FEATURE_NAME:()=>i,FETCH:()=>_,FETCH_BODY:()=>v,FETCH_DONE:()=>m,FETCH_START:()=>p,FN_END:()=>c,FN_START:()=>s,INTERACTION:()=>l,INTERACTION_API:()=>d,INTERACTION_EVENTS:()=>o,JSONP_END:()=>b,JSONP_NODE:()=>g,JS_TIME:()=>T,MAX_TIMER_BUDGET:()=>a,REMAINING:()=>f,SPA_NODE:()=>h,START:()=>w,originalSetTimeout:()=>y});var n=r(5763);const i=r(3325).D.spa,o=["click","submit","keypress","keydown","keyup","change"],a=999,s="fn-start",c="fn-end",u="cb-start",d="api-ixn-",f="remaining",l="interaction",h="spaNode",g="jsonpNode",p="fetch-start",m="fetch-done",v="fetch-body-",b="jsonp-end",y=n.Yu.ST,w="-start",x="-end",A="-body",E="cb"+x,T="jsTime",_="fetch"},5938:(e,t,r)=>{r.d(t,{W:()=>o});var n=r(5763),i=r(2177);class o{constructor(e,t,r){this.agentIdentifier=e,this.aggregator=t,this.ee=i.ee.get(e,(0,n.OP)(this.agentIdentifier).isolatedBacklog),this.featureName=r,this.blocked=!1}}},9144:(e,t,r)=>{r.d(t,{j:()=>m});var n=r(3325),i=r(5763),o=r(5546),a=r(2177),s=r(7894),c=r(8e3),u=r(3960),d=r(385),f=r(50),l=r(3081),h=r(8632);function g(){const e=(0,h.gG)();["setErrorHandler","finished","addToTrace","inlineHit","addRelease","addPageAction","setCurrentRouteName","setPageViewName","setCustomAttribute","interaction","noticeError","setUserId"].forEach((t=>{e[t]=function(){for(var r=arguments.length,n=new Array(r),i=0;i 1?r-1:0),i=1;i {e.exposed&&e.api[t]&&o.push(e.api[t](...n))})),o.length>1?o:o[0]}(t,...n)}}))}var p=r(2587);function m(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},m=arguments.length>2?arguments[2]:void 0,v=arguments.length>3?arguments[3]:void 0,{init:b,info:y,loader_config:w,runtime:x={loaderType:m},exposed:A=!0}=t;const E=(0,h.gG)();y||(b=E.init,y=E.info,w=E.loader_config),(0,i.Dg)(e,b||{}),(0,i.GE)(e,w||{}),(0,i.sU)(e,x),y.jsAttributes??={},d.v6&&(y.jsAttributes.isWorker=!0),(0,i.CX)(e,y),g();const T=function(e,t){t||(0,c.R)(e,"api");const h={};var g=a.ee.get(e),p=g.get("tracer"),m="api-",v=m+"ixn-";function b(t,r,n,o){const a=(0,i.C5)(e);return null===r?delete a.jsAttributes[t]:(0,i.CX)(e,{...a,jsAttributes:{...a.jsAttributes,[t]:r}}),x(m,n,!0,o||null===r?"session":void 0)(t,r)}function y(){}["setErrorHandler","finished","addToTrace","inlineHit","addRelease"].forEach((e=>h[e]=x(m,e,!0,"api"))),h.addPageAction=x(m,"addPageAction",!0,n.D.pageAction),h.setCurrentRouteName=x(m,"routeName",!0,n.D.spa),h.setPageViewName=function(t,r){if("string"==typeof t)return"/"!==t.charAt(0)&&(t="/"+t),(0,i.OP)(e).customTransaction=(r||"http://custom.transaction")+t,x(m,"setPageViewName",!0)()},h.setCustomAttribute=function(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2];if("string"==typeof e){if(["string","number"].includes(typeof t)||null===t)return b(e,t,"setCustomAttribute",r);(0,f.Z)("Failed to execute setCustomAttribute.\nNon-null value must be a string or number type, but a type of was provided."))}else(0,f.Z)("Failed to execute setCustomAttribute.\nName must be a string type, but a type of was provided."))},h.setUserId=function(e){if("string"==typeof e||null===e)return b("enduser.id",e,"setUserId",!0);(0,f.Z)("Failed to execute setUserId.\nNon-null value must be a string type, but a type of was provided."))},h.interaction=function(){return(new y).get()};var w=y.prototype={createTracer:function(e,t){var r={},i=this,a="function"==typeof t;return(0,o.p)(v+"tracer",[(0,s.z)(),e,r],i,n.D.spa,g),function(){if(p.emit((a?"":"no-")+"fn-start",[(0,s.z)(),i,a],r),a)try{return t.apply(this,arguments)}catch(e){throw p.emit("fn-err",[arguments,this,"string"==typeof e?new Error(e):e],r),e}finally{p.emit("fn-end",[(0,s.z)()],r)}}}};function x(e,t,r,i){return function(){return(0,o.p)(l.xS,["API/"+t+"/called"],void 0,n.D.metrics,g),i&&(0,o.p)(e+t,[(0,s.z)(),...arguments],r?null:this,i,g),r?void 0:this}}function A(){r.e(439).then(r.bind(r,7438)).then((t=>{let{setAPI:r}=t;r(e),(0,c.L)(e,"api")})).catch((()=>(0,f.Z)("Downloading runtime APIs failed...")))}return["actionText","setName","setAttribute","save","ignore","onEnd","getContext","end","get"].forEach((e=>{w[e]=x(v,e,void 0,n.D.spa)})),h.noticeError=function(e,t){"string"==typeof e&&(e=new Error(e)),(0,o.p)(l.xS,["API/noticeError/called"],void 0,n.D.metrics,g),(0,o.p)("err",[e,(0,s.z)(),!1,t],void 0,n.D.jserrors,g)},d.il?(0,u.b)((()=>A()),!0):A(),h}(e,v);return(0,h.Qy)(e,T,"api"),(0,h.Qy)(e,A,"exposed"),(0,h.EZ)("activatedFeatures",p.T),T}},3325:(e,t,r)=>{r.d(t,{D:()=>n,p:()=>i});const n={ajax:"ajax",jserrors:"jserrors",metrics:"metrics",pageAction:"page_action",pageViewEvent:"page_view_event",pageViewTiming:"page_view_timing",sessionReplay:"session_replay",sessionTrace:"session_trace",spa:"spa"},i={[n.pageViewEvent]:1,[n.pageViewTiming]:2,[n.metrics]:3,[n.jserrors]:4,[n.ajax]:5,[n.sessionTrace]:6,[n.pageAction]:7,[n.spa]:8,[n.sessionReplay]:9}}},n={};function i(e){var t=n[e];if(void 0!==t)return t.exports;var o=n[e]={exports:{}};return r[e](o,o.exports,i),o.exports}i.m=r,i.d=(e,t)=>{for(var r in t)i.o(t,r)&&!i.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},i.f={},i.e=e=>Promise.all(Object.keys(i.f).reduce(((t,r)=>(i.f[r](e,t),t)),[])),i.u=e=>(({78:"page_action-aggregate",147:"metrics-aggregate",242:"session-manager",317:"jserrors-aggregate",348:"page_view_timing-aggregate",412:"lazy-feature-loader",439:"async-api",538:"recorder",590:"session_replay-aggregate",675:"compressor",733:"session_trace-aggregate",786:"page_view_event-aggregate",873:"spa-aggregate",898:"ajax-aggregate"}[e]||e)+"."+{78:"ac76d497",147:"3dc53903",148:"1a20d5fe",242:"2a64278a",317:"49e41428",348:"bd6de33a",412:"2f55ce66",439:"30bd804e",538:"1b18459f",590:"cf0efb30",675:"ae9f91a8",733:"83105561",786:"06482edd",860:"03a8b7a5",873:"e6b09d52",898:"998ef92b"}[e]+"-1.236.0.min.js"),i.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),e={},t="NRBA:",i.l=(r,n,o,a)=>{if(e[r])e[r].push(n);else{var s,c;if(void 0!==o)for(var u=document.getElementsByTagName("script"),d=0;d {s.onerror=s.onload=null,clearTimeout(h);var i=e[r];if(delete e[r],s.parentNode&&s.parentNode.removeChild(s),i&&i.forEach((e=>e(n))),t)return t(n)},h=setTimeout(l.bind(null,void 0,{type:"timeout",target:s}),12e4);s.onerror=l.bind(null,s.onerror),s.onload=l.bind(null,s.onload),c&&document.head.appendChild(s)}},i.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.j=364,i.p="https://js-agent.newrelic.com/",(()=>{var e={364:0,953:0};i.f.j=(t,r)=>{var n=i.o(e,t)?e[t]:void 0;if(0!==n)if(n)r.push(n[2]);else{var o=new Promise(((r,i)=>n=e[t]=[r,i]));r.push(n[2]=o);var a=i.p+i.u(t),s=new Error;i.l(a,(r=>{if(i.o(e,t)&&(0!==(n=e[t])&&(e[t]=void 0),n)){var o=r&&("load"===r.type?"missing":r.type),a=r&&r.target&&r.target.src;s.message="Loading chunk "+t+" failed.\n("+o+": "+a+")",s.name="ChunkLoadError",s.type=o,s.request=a,n[1](s)}}),"chunk-"+t,t)}};var t=(t,r)=>{var n,o,[a,s,c]=r,u=0;if(a.some((t=>0!==e[t]))){for(n in s)i.o(s,n)&&(i.m[n]=s[n]);if(c)c(i)}for(t&&t(r);u {i.r(o);var e=i(3325),t=i(5763);const r=Object.values(e.D);function n(e){const n={};return r.forEach((r=>{n[r]=function(e,r){return!1!==(0,t.Mt)(r,"".concat(e,".enabled"))}(r,e)})),n}var a=i(9144);var s=i(5546),c=i(385),u=i(8e3),d=i(5938),f=i(3960),l=i(50);class h extends d.W{constructor(e,t,r){let n=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];super(e,t,r),this.auto=n,this.abortHandler,this.featAggregate,this.onAggregateImported,n&&(0,u.R)(e,r)}importAggregator(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};if(this.featAggregate||!this.auto)return;const r=c.il&&!0===(0,t.Mt)(this.agentIdentifier,"privacy.cookies_enabled");let n;this.onAggregateImported=new Promise((e=>{n=e}));const o=async()=>{let t;try{if(r){const{setupAgentSession:e}=await Promise.all([i.e(860),i.e(242)]).then(i.bind(i,3228));t=e(this.agentIdentifier)}}catch(e){(0,l.Z)("A problem occurred when starting up session manager. This page will not start or extend any session.",e)}try{if(!this.shouldImportAgg(this.featureName,t))return void(0,u.L)(this.agentIdentifier,this.featureName);const{lazyFeatureLoader:r}=await i.e(412).then(i.bind(i,8582)),{Aggregate:o}=await r(this.featureName,"aggregate");this.featAggregate=new o(this.agentIdentifier,this.aggregator,e),n(!0)}catch(e){(0,l.Z)("Downloading and initializing ".concat(this.featureName," failed..."),e),this.abortHandler?.(),n(!1)}};c.il?(0,f.b)((()=>o()),!0):o()}shouldImportAgg(r,n){return r!==e.D.sessionReplay||!1!==(0,t.Mt)(this.agentIdentifier,"session_trace.enabled")&&(!!n?.isNew||!!n?.state.sessionReplay)}}var g=i(7633),p=i(7894);class m extends h{static featureName=g.t9;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];if(super(r,n,g.t9,i),("undefined"==typeof PerformanceNavigationTiming||c.Tt)&&"undefined"!=typeof PerformanceTiming){const n=(0,t.OP)(r);n[g.Dz]=Math.max(Date.now()-n.offset,0),(0,f.K)((()=>n[g.qw]=Math.max((0,p.z)()-n[g.Dz],0))),(0,f.b)((()=>{const t=(0,p.z)();n[g.OJ]=Math.max(t-n[g.Dz],0),(0,s.p)("timing",["load",t],void 0,e.D.pageViewTiming,this.ee)}))}this.importAggregator()}}var v=i(1117),b=i(1284);class y extends v.w{constructor(e){super(e),this.aggregatedData={}}store(e,t,r,n,i){var o=this.getBucket(e,t,r,i);return o.metrics=function(e,t){t||(t={count:0});return t.count+=1,(0,b.D)(e,(function(e,r){t[e]=w(r,t[e])})),t}(n,o.metrics),o}merge(e,t,r,n,i){var o=this.getBucket(e,t,n,i);if(o.metrics){var a=o.metrics;a.count+=r.count,(0,b.D)(r,(function(e,t){if("count"!==e){var n=a[e],i=r[e];i&&!i.c?a[e]=w(i.t,n):a[e]=function(e,t){if(!t)return e;t.c||(t=x(t.t));return t.min=Math.min(e.min,t.min),t.max=Math.max(e.max,t.max),t.t+=e.t,t.sos+=e.sos,t.c+=e.c,t}(i,a[e])}}))}else o.metrics=r}storeMetric(e,t,r,n){var i=this.getBucket(e,t,r);return i.stats=w(n,i.stats),i}getBucket(e,t,r,n){this.aggregatedData[e]||(this.aggregatedData[e]={});var i=this.aggregatedData[e][t];return i||(i=this.aggregatedData[e][t]={params:r||{}},n&&(i.custom=n)),i}get(e,t){return t?this.aggregatedData[e]&&this.aggregatedData[e][t]:this.aggregatedData[e]}take(e){for(var t={},r="",n=!1,i=0;i t.max&&(t.max=e),e 2&&void 0!==arguments[2])||arguments[2];super(e,r,j.t,n),c.il&&((0,t.OP)(e).initHidden=Boolean("hidden"===document.visibilityState),(0,N.N)((()=>(0,s.p)("docHidden",[(0,p.z)()],void 0,j.t,this.ee)),!0),(0,O.bP)("pagehide",(()=>(0,s.p)("winPagehide",[(0,p.z)()],void 0,j.t,this.ee))),this.importAggregator())}}var P=i(3081);class C extends h{static featureName=P.t9;constructor(e,t){let r=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(e,t,P.t9,r),this.importAggregator()}}var R,I=i(2210),k=i(1214),H=i(2177),L={};try{R=localStorage.getItem("__nr_flags").split(","),console&&"function"==typeof console.log&&(L.console=!0,-1!==R.indexOf("dev")&&(L.dev=!0),-1!==R.indexOf("nr_dev")&&(L.nrDev=!0))}catch(e){}function z(e){try{L.console&&z(e)}catch(e){}}L.nrDev&&H.ee.on("internal-error",(function(e){z(e.stack)})),L.dev&&H.ee.on("fn-err",(function(e,t,r){z(r.stack)})),L.dev&&(z("NR AGENT IN DEVELOPMENT MODE"),z("flags: "+(0,b.D)(L,(function(e,t){return e})).join(", ")));var M=i(6660);class B extends h{static featureName=M.t;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(r,n,M.t,i),this.skipNext=0;try{this.removeOnAbort=new AbortController}catch(e){}const o=this;o.ee.on("fn-start",(function(e,t,r){o.abortHandler&&(o.skipNext+=1)})),o.ee.on("fn-err",(function(t,r,n){o.abortHandler&&!n[M.A]&&((0,I.X)(n,M.A,(function(){return!0})),this.thrown=!0,(0,s.p)("err",[n,(0,p.z)()],void 0,e.D.jserrors,o.ee))})),o.ee.on("fn-end",(function(){o.abortHandler&&!this.thrown&&o.skipNext>0&&(o.skipNext-=1)})),o.ee.on("internal-error",(function(t){(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,o.ee)})),this.origOnerror=c._A.onerror,c._A.onerror=this.onerrorHandler.bind(this),c._A.addEventListener("unhandledrejection",(t=>{const r=function(e){let t="Unhandled Promise Rejection: ";if(e instanceof Error)try{return e.message=t+e.message,e}catch(t){return e}if(void 0===e)return new Error(t);try{return new Error(t+(0,D.P)(e))}catch(e){return new Error(t)}}(t.reason);(0,s.p)("err",[r,(0,p.z)(),!1,{unhandledPromiseRejection:1}],void 0,e.D.jserrors,this.ee)}),(0,O.m$)(!1,this.removeOnAbort?.signal)),(0,k.gy)(this.ee),(0,k.BV)(this.ee),(0,k.em)(this.ee),(0,t.OP)(r).xhrWrappable&&(0,k.Kf)(this.ee),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}onerrorHandler(t,r,n,i,o){"function"==typeof this.origOnerror&&this.origOnerror(...arguments);try{this.skipNext?this.skipNext-=1:(0,s.p)("err",[o||new F(t,r,n),(0,p.z)()],void 0,e.D.jserrors,this.ee)}catch(t){try{(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,this.ee)}catch(e){}}return!1}}function F(e,t,r){this.message=e||"Uncaught error with no additional information",this.sourceURL=t,this.line=r}let U=1;const q="nr@id";function G(e){const t=typeof e;return!e||"object"!==t&&"function"!==t?-1:e===c._A?0:(0,I.X)(e,q,(function(){return U++}))}function V(e){if("string"==typeof e&&e.length)return e.length;if("object"==typeof e){if("undefined"!=typeof ArrayBuffer&&e instanceof ArrayBuffer&&e.byteLength)return e.byteLength;if("undefined"!=typeof Blob&&e instanceof Blob&&e.size)return e.size;if(!("undefined"!=typeof FormData&&e instanceof FormData))try{return(0,D.P)(e).length}catch(e){return}}}var X=i(7243);class W{constructor(e){this.agentIdentifier=e,this.generateTracePayload=this.generateTracePayload.bind(this),this.shouldGenerateTrace=this.shouldGenerateTrace.bind(this)}generateTracePayload(e){if(!this.shouldGenerateTrace(e))return null;var r=(0,t.DL)(this.agentIdentifier);if(!r)return null;var n=(r.accountID||"").toString()||null,i=(r.agentID||"").toString()||null,o=(r.trustKey||"").toString()||null;if(!n||!i)return null;var a=(0,_.M)(),s=(0,_.Ht)(),c=Date.now(),u={spanId:a,traceId:s,timestamp:c};return(e.sameOrigin||this.isAllowedOrigin(e)&&this.useTraceContextHeadersForCors())&&(u.traceContextParentHeader=this.generateTraceContextParentHeader(a,s),u.traceContextStateHeader=this.generateTraceContextStateHeader(a,c,n,i,o)),(e.sameOrigin&&!this.excludeNewrelicHeader()||!e.sameOrigin&&this.isAllowedOrigin(e)&&this.useNewrelicHeaderForCors())&&(u.newrelicHeader=this.generateTraceHeader(a,s,c,n,i,o)),u}generateTraceContextParentHeader(e,t){return"00-"+t+"-"+e+"-01"}generateTraceContextStateHeader(e,t,r,n,i){return i+"@nr=0-1-"+r+"-"+n+"-"+e+"----"+t}generateTraceHeader(e,t,r,n,i,o){if(!("function"==typeof c._A?.btoa))return null;var a={v:[0,1],d:{ty:"Browser",ac:n,ap:i,id:e,tr:t,ti:r}};return o&&n!==o&&(a.d.tk=o),btoa((0,D.P)(a))}shouldGenerateTrace(e){return this.isDtEnabled()&&this.isAllowedOrigin(e)}isAllowedOrigin(e){var r=!1,n={};if((0,t.Mt)(this.agentIdentifier,"distributed_tracing")&&(n=(0,t.P_)(this.agentIdentifier).distributed_tracing),e.sameOrigin)r=!0;else if(n.allowed_origins instanceof Array)for(var i=0;i 2&&void 0!==arguments[2])||arguments[2];super(r,n,Z.t,i),(0,t.OP)(r).xhrWrappable&&(this.dt=new W(r),this.handler=(e,t,r,n)=>(0,s.p)(e,t,r,n,this.ee),(0,k.u5)(this.ee),(0,k.Kf)(this.ee),function(r,n,i,o){function a(e){var t=this;t.totalCbs=0,t.called=0,t.cbTime=0,t.end=E,t.ended=!1,t.xhrGuids={},t.lastSize=null,t.loadCaptureCalled=!1,t.params=this.params||{},t.metrics=this.metrics||{},e.addEventListener("load",(function(r){_(t,e)}),(0,O.m$)(!1)),c.IF||e.addEventListener("progress",(function(e){t.lastSize=e.loaded}),(0,O.m$)(!1))}function s(e){this.params={method:e[0]},T(this,e[1]),this.metrics={}}function u(e,n){var i=(0,t.DL)(r);i.xpid&&this.sameOrigin&&n.setRequestHeader("X-NewRelic-ID",i.xpid);var a=o.generateTracePayload(this.parsedOrigin);if(a){var s=!1;a.newrelicHeader&&(n.setRequestHeader("newrelic",a.newrelicHeader),s=!0),a.traceContextParentHeader&&(n.setRequestHeader("traceparent",a.traceContextParentHeader),a.traceContextStateHeader&&n.setRequestHeader("tracestate",a.traceContextStateHeader),s=!0),s&&(this.dt=a)}}function d(e,t){var r=this.metrics,i=e[0],o=this;if(r&&i){var a=V(i);a&&(r.txSize=a)}this.startTime=(0,p.z)(),this.listener=function(e){try{"abort"!==e.type||o.loadCaptureCalled||(o.params.aborted=!0),("load"!==e.type||o.called===o.totalCbs&&(o.onloadCalled||"function"!=typeof t.onload)&&"function"==typeof o.end)&&o.end(t)}catch(e){try{n.emit("internal-error",[e])}catch(e){}}};for(var s=0;s 1?e[1]=i:e.push(i)}else e[0]&&e[0].headers&&s(e[0].headers,n)&&(this.dt=n);function s(e,t){var r=!1;return t.newrelicHeader&&(e.set("newrelic",t.newrelicHeader),r=!0),t.traceContextParentHeader&&(e.set("traceparent",t.traceContextParentHeader),t.traceContextStateHeader&&e.set("tracestate",t.traceContextStateHeader),r=!0),r}}function x(e,t){this.params={},this.metrics={},this.startTime=(0,p.z)(),this.dt=t,e.length>=1&&(this.target=e[0]),e.length>=2&&(this.opts=e[1]);var r,n=this.opts||{},i=this.target;"string"==typeof i?r=i:"object"==typeof i&&i instanceof Y?r=i.url:c._A?.URL&&"object"==typeof i&&i instanceof URL&&(r=i.href),T(this,r);var o=(""+(i&&i instanceof Y&&i.method||n.method||"GET")).toUpperCase();this.params.method=o,this.txSize=V(n.body)||0}function A(t,r){var n;this.endTime=(0,p.z)(),this.params||(this.params={}),this.params.status=r?r.status:0,"string"==typeof this.rxSize&&this.rxSize.length>0&&(n=+this.rxSize);var o={txSize:this.txSize,rxSize:n,duration:(0,p.z)()-this.startTime};i("xhr",[this.params,o,this.startTime,this.endTime,"fetch"],this,e.D.ajax)}function E(t){var r=this.params,n=this.metrics;if(!this.ended){this.ended=!0;for(var o=0;o 2&&void 0!==arguments[2])||arguments[2];super(e,t,we.t,r),this.importAggregator()}}new class{constructor(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:(0,_.ky)(16);c._A?(this.agentIdentifier=t,this.sharedAggregator=new y({agentIdentifier:this.agentIdentifier}),this.features={},this.desiredFeatures=new Set(e.features||[]),this.desiredFeatures.add(m),Object.assign(this,(0,a.j)(this.agentIdentifier,e,e.loaderType||"agent")),this.start()):(0,l.Z)("Failed to initial the agent. Could not determine the runtime environment.")}get config(){return{info:(0,t.C5)(this.agentIdentifier),init:(0,t.P_)(this.agentIdentifier),loader_config:(0,t.DL)(this.agentIdentifier),runtime:(0,t.OP)(this.agentIdentifier)}}start(){const t="features";try{const r=n(this.agentIdentifier),i=[...this.desiredFeatures];i.sort(((t,r)=>e.p[t.featureName]-e.p[r.featureName])),i.forEach((t=>{if(r[t.featureName]||t.featureName===e.D.pageViewEvent){const n=function(t){switch(t){case e.D.ajax:return[e.D.jserrors];case e.D.sessionTrace:return[e.D.ajax,e.D.pageViewEvent];case e.D.sessionReplay:return[e.D.sessionTrace];case e.D.pageViewTiming:return[e.D.pageViewEvent];default:return[]}}(t.featureName);n.every((e=>r[e]))||(0,l.Z)("".concat(t.featureName," is enabled but one or more dependent features has been disabled (").concat((0,D.P)(n),"). This may cause unintended consequences or missing data...")),this.features[t.featureName]=new t(this.agentIdentifier,this.sharedAggregator)}})),(0,T.Qy)(this.agentIdentifier,this.features,t)}catch(e){(0,l.Z)("Failed to initialize all enabled instrument classes (agent aborted) -",e);for(const e in this.features)this.features[e].abortHandler?.();const r=(0,T.fP)();return delete r.initializedAgents[this.agentIdentifier]?.api,delete r.initializedAgents[this.agentIdentifier]?.[t],delete this.sharedAggregator,r.ee?.abort(),delete r.ee?.get(this.agentIdentifier),!1}}}({features:[J,m,S,class extends h{static featureName=oe;constructor(t,r){if(super(t,r,oe,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;const n=this.ee;let i;(0,k.QU)(n),this.eventsEE=(0,k.em)(n),this.eventsEE.on(se,(function(e,t){this.bstStart=(0,p.z)()})),this.eventsEE.on(ae,(function(t,r){(0,s.p)("bst",[t[0],r,this.bstStart,(0,p.z)()],void 0,e.D.sessionTrace,n)})),n.on(ce+ne,(function(e){this.time=(0,p.z)(),this.startPath=location.pathname+location.hash})),n.on(ce+ie,(function(t){(0,s.p)("bstHist",[location.pathname+location.hash,this.startPath,this.time],void 0,e.D.sessionTrace,n)}));try{i=new PerformanceObserver((t=>{const r=t.getEntries();(0,s.p)(te,[r],void 0,e.D.sessionTrace,n)})),i.observe({type:re,buffered:!0})}catch(e){}this.importAggregator({resourceObserver:i})}},C,xe,B,class extends h{static featureName=de;constructor(e,r){if(super(e,r,de,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;if(!(0,t.OP)(e).xhrWrappable)return;try{this.removeOnAbort=new AbortController}catch(e){}let n,i=0;const o=this.ee.get("tracer"),a=(0,k._L)(this.ee),s=(0,k.Lg)(this.ee),u=(0,k.BV)(this.ee),d=(0,k.Kf)(this.ee),f=this.ee.get("events"),l=(0,k.u5)(this.ee),h=(0,k.QU)(this.ee),g=(0,k.Gm)(this.ee);function m(e,t){h.emit("newURL",[""+window.location,t])}function v(){i++,n=window.location.hash,this[ve]=(0,p.z)()}function b(){i--,window.location.hash!==n&&m(0,!0);var e=(0,p.z)();this[pe]=~~this[pe]+e-this[ve],this[ye]=e}function y(e,t){e.on(t,(function(){this[t]=(0,p.z)()}))}this.ee.on(ve,v),s.on(be,v),a.on(be,v),this.ee.on(ye,b),s.on(ge,b),a.on(ge,b),this.ee.buffer([ve,ye,"xhr-resolved"],this.featureName),f.buffer([ve],this.featureName),u.buffer(["setTimeout"+le,"clearTimeout"+fe,ve],this.featureName),d.buffer([ve,"new-xhr","send-xhr"+fe],this.featureName),l.buffer([me+fe,me+"-done",me+he+fe,me+he+le],this.featureName),h.buffer(["newURL"],this.featureName),g.buffer([ve],this.featureName),s.buffer(["propagate",be,ge,"executor-err","resolve"+fe],this.featureName),o.buffer([ve,"no-"+ve],this.featureName),a.buffer(["new-jsonp","cb-start","jsonp-error","jsonp-end"],this.featureName),y(l,me+fe),y(l,me+"-done"),y(a,"new-jsonp"),y(a,"jsonp-end"),y(a,"cb-start"),h.on("pushState-end",m),h.on("replaceState-end",m),window.addEventListener("hashchange",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("load",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("popstate",(function(){m(0,i>1)}),(0,O.m$)(!0,this.removeOnAbort?.signal)),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}}],loaderType:"spa"})})(),window.NRBA=o})(); window.jQuery || document.write(' ') CKEDITOR_BASEPATH='https://f1000research.com/js/vendor/ckeditor/' window.reactTheme = 'research'; window.MathJax = { CommonHTML: { linebreaks: { automatic: true } }, 'HTML-CSS': { linebreaks: { automatic: true } }, SVG: { linebreaks: { automatic: true } }, AuthorInit: function() { MathJax.Hub.Register.MessageHook('End Process', function () { let timeout = false; // holder for timeout id const delay = 250; // delay after event is "complete" to run callback const reflowMath = function() { const dispFormulas = document.querySelectorAll('.disp-formula.panel'); if (!dispFormulas) { return; } for (const dispFormula of dispFormulas) { const child = dispFormula.querySelector('.MathJax_Preview').nextSibling.firstChild; const isMultiline = MathJax.Hub.getAllJax(dispFormula)[0].root.isMultiline; if (dispFormula.offsetWidth < child.offsetWidth || isMultiline) { MathJax.Hub.Queue(['Rerender', MathJax.Hub, dispFormula]); } } }; window.addEventListener('resize', function() { clearTimeout(timeout); // clear the timeout timeout = setTimeout(reflowMath, delay); // start timing for event "completion" }); }); }, }; if (window.location.hash == '#_=_'){ window.location = window.location.href.split('#')[0] } !function(f,b,e,v,n,t,s){if(f.fbq)return;n=f.fbq=function() {n.callMethod? n.callMethod.apply(n,arguments):n.queue.push(arguments)} ;if(!f._fbq)f._fbq=n; n.push=n;n.loaded=!0;n.version='2.0';n.queue=[];t=b.createElement(e);t.async=!0; t.src=v;s=b.getElementsByTagName(e)[0];s.parentNode.insertBefore(t,s)}(window, document,'script','https://connect.facebook.net/en_US/fbevents.js'); fbq('init', '1641728616063202'); fbq('track', "PixelInitialized", {}); (function(h,o,t,j,a,r){ h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)}; h._hjSettings={hjid:2318163,hjsv:6}; a=o.getElementsByTagName('head')[0]; r=o.createElement('script');r.async=1; r.src=t+h._hjSettings.hjid+j+h._hjSettings.hjsv; a.appendChild(r); })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv='); search file_upload Submit your research search menu close search Browse Gateways & Collections How to Publish Submit your Research My Submissions Article Guidelines Article Guidelines (New Versions) Open Data, Software and Code Guidelines Open Data and Accessible Source Materials Guidelines (HSS) Open Data, Software and Code Guidelines (PSE) Prepublication Checks Production Process Posters and Slides Guidelines Document Guidelines Article Processing Charges Peer Review Finding Article Reviewers About How it Works For Reviewers Our Advisors Policies Glossary FAQs For Developers Newsroom Contact My Research Submissions Content and Tracking Alerts My Details Sign In file_upload Submit your research { "@context": "https://schema.org", "@type": "ScholarlyArticle", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://f1000research.com/articles/12-1420" }, "headline": "Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study...", "datePublished": "2023-10-30T16:55:00", "dateModified": "2024-12-09T16:57:44", "author": [ { "@type": "Person", "name": "Anes Ben Fradj" }, { "@type": "Person", "name": "Neila El Asli" }, { "@type": "Person", "name": "Tasseda Boukherroub" }, { "@type": "Person", "name": "Claude Olivier" } ], "publisher": { "@type": "Organization", "name": "F1000Research", "logo": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 480, "width": 60 } }, "image": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 1200, "width": 150 }, "description": "This study proposes an approach for developing or improving performance management and measurement systems (PMMSs) for healthcare organisations. First, data is collected to analyse and understand the current organisation’s performance management system. Second, the SWOT (Strengths, Weaknesses, Opportunities, Threats) method is used to identify the main aspects of the performance management system to be improved. Third, based on the scientific literature and SWOT analysis, BSC principles are integrated to this performance management system to better align the organisation’s performance objectives and indicators with its strategy. Finally, we develop a performance indicator structure and select indicators to be used as well as how these indicators could be integrated and shared with higher hierarchical levels in the organisation by using AHP (Analytic Hierarchy Process). Our approach is applied to the CIUSSS du Centre-Sud-de-l’île-de-Montreal (CCSMTL), a large healthcare network, in the province of Québec, Canada." } { "@context": "http://schema.org", "@type": "BreadcrumbList", "itemListElement": [ { "@type": "ListItem", "position": "1", "item": { "@id": "https://f1000research.com/", "name": "Home" } }, { "@type": "ListItem", "position": "2", "item": { "@id": "https://f1000research.com/browse/articles", "name": "Browse" } }, { "@type": "ListItem", "position": "3", "item": { "@id": "https://f1000research.com/articles/12-1420", "name": "Towards an Integrated Performance Management and Measurement System..." } } ] } Home Browse Towards an Integrated Performance Management and Measurement System... ALL Metrics - Views Downloads Get PDF Get XML Cite How to cite this article Ben Fradj A, El Asli N, Boukherroub T and Olivier C. Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.12688/f1000research.138430.2 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. Close Copy Citation Details Export Export Citation Sciwheel EndNote Ref. Manager Bibtex ProCite Sente EXPORT Select a format first Track Share ▬ ✚ Case Study Revised Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] Previously titled: Developing an integrated performance management and measurement system in healthcare organisations: a Canadian case study Anes Ben Fradj 1,2 , Neila El Asli https://orcid.org/0000-0002-3477-5416 1-4 , Tasseda Boukherroub 1-3 , Claude Olivier 1,2 Anes Ben Fradj 1,2 , Neila El Asli https://orcid.org/0000-0002-3477-5416 1-4 , Tasseda Boukherroub 1-3 , Claude Olivier 1,2 PUBLISHED 09 Dec 2024 Author details Author details 1 Numérix Laboratory, École de technologie superieure, Montreal, Québec, Canada 2 Systems Engineering Department, École de technologie superieure, Montreal, Québec, Canada 3 Interuniversity Research Centre on Enterprise Networks, Logistics and Transportation, Montreal, Québec, Canada 4 Industrial Engineering Department, Université de Quebec à Trois Rivieres, Quebec, Canada Anes Ben Fradj Roles: Conceptualization, Formal Analysis, Writing – Original Draft Preparation Neila El Asli Roles: Formal Analysis, Validation, Writing – Original Draft Preparation, Writing – Review & Editing Tasseda Boukherroub Roles: Supervision, Writing – Review & Editing Claude Olivier Roles: Supervision OPEN PEER REVIEW DETAILS REVIEWER STATUS Abstract This study proposes an approach for developing or improving performance management and measurement systems (PMMSs) for healthcare organisations. First, data is collected to analyse and understand the current organisation’s performance management system. Second, the SWOT (Strengths, Weaknesses, Opportunities, Threats) method is used to identify the main aspects of the performance management system to be improved. Third, based on the scientific literature and SWOT analysis, BSC principles are integrated to this performance management system to better align the organisation’s performance objectives and indicators with its strategy. Finally, we develop a performance indicator structure and select indicators to be used as well as how these indicators could be integrated and shared with higher hierarchical levels in the organisation by using AHP (Analytic Hierarchy Process). Our approach is applied to the CIUSSS du Centre-Sud-de-l’île-de-Montreal (CCSMTL), a large healthcare network, in the province of Québec, Canada. READ ALL READ LESS Keywords AHP, BSC, performance management, healthcare Corresponding Author(s) Neila El Asli ( [email protected] ) Close Corresponding author: Neila El Asli Competing interests: No competing interests were disclosed. Grant information: This work is supported by the MITACS Globalink program, École de technologie supérieure (ÉTS), CIUSSS du-Centre-Sud-de-l'île-de-Montréal, and Natural Sciences and Engineering Research Council of Canada (NSERC) grant 2018-04555. The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript. Copyright: © 2024 Ben Fradj A et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. How to cite: Ben Fradj A, El Asli N, Boukherroub T and Olivier C. Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.12688/f1000research.138430.2 ) First published: 30 Oct 2023, 12 :1420 ( https://doi.org/10.12688/f1000research.138430.1 ) Latest published: 09 Dec 2024, 12 :1420 ( https://doi.org/10.12688/f1000research.138430.2 ) Revised Amendments from Version 1 The changes made to the previous version are the result of the reviewers' recommendations. The most notable change is that the structure of the paper has been changed to highlight the results of the study. The title has also been modified and figure 10 has been added. The changes made to the previous version are the result of the reviewers' recommendations. The most notable change is that the structure of the paper has been changed to highlight the results of the study. The title has also been modified and figure 10 has been added. See the authors' detailed response to the review by Jean-Baptiste Gartner See the authors' detailed response to the review by Wolfgang Munar See the authors' detailed response to the review by Paulino Silva READ REVIEWER RESPONSES Introduction In the province of Québec, Canada, public health and social services are provided by networks of institutions called CI (U)SSSs (integrated (University) centres of healthcare and social services), which fall under the responsibility of the Ministry of Healthcare and Social Services (Mimistère de la Santé et des Services Sociaux referred to as MSSS). Their main mission is to provide healthcare and social services to the population while ensuring accessibility, quality, safety, and efficiency. This study focusses on the CIUSSS serving the South-Centre of Montreal region (CIUSSS du Centre-Sud-de-l’île-de-Montréal referred to as CCSMTL). It provides 28 different healthcare services to a population of 311,000 people, it owns 150 facilities and employs more than 21,300 people, including 56 senior managers and nearly 800 physicians. It manages a budget of more than $1.7 billion Canadian dollars ( Annual Report 2019-2020, CIUSSS ). When the CCSMTL was created in 2015, it had to develop a reference framework for its performance management and to define a vision regarding its organisational performance. A Management and Accountability Agreement , involving the MSSS and the CCSMTL management, was introduced by the MSSS, with the obligation to implement a visual management system as a means for performance management and decision-making. Therefore, the CCSMTL has built its Quality-Performance Model (referred to as QPM), which encompasses four dimensions of performance; customer, accessibility/quality, mobilisation, and optimisation. It has also implemented its visual management system. However, QPM deployment is very challenging: 1) It must be implemented at different decision-making levels, organisational structures (programs, sub-programs, services, teams, etc.), and territories. For instance, performance indicators may differ greatly from one service to another or from one territory to another. 2) Some performance aspects such as customer and mobilisation dimensions are not sufficiently or well measured, which leads to an unbalanced performance management system. 3) CCSMTL managers need to measure the overall performance of a given service or program, through integrated information. Currently, such information is not available nor easy to obtain. This study contributes to addressing the aforementioned issues. More precisely, it proposes a four-phases based approach to support CCSMTL managers to improve and transform the QPM into a more balanced and effective performance management and measurement system (PMMS). Even though recent studies in the literature discuss performance management systems in healthcare organisations and recognise both the importance and the complexity of measuring performance, most of them are descriptive and do not propose PMMSs capable of effectively structuring and measuring the performance of healthcare organisations. Our work contributes to the theory and practice by proposing a comprehensive and quantitative performance management and measurement approach adapted to large healthcare organisations such as the CCSMTL. It can be used by theoreticians and practitioners as roadmap to develop or improve healthcare organisations’ PMMSs. The remainder of this paper is organised as follows: the next section presents the QPM in more detail. After that, we present our literature review and then describe our approach. Next, we present the results of applying our approach to the CCSMTL case study. We also discuss the preliminary results. Finally, we conclude and present some research perspectives. The quality-performance model The QPM (see Figure 1 ) is based on the MSSS strategic plan (2019-2023 period) and framework for assessing the performance of the public healthcare and social services system ( MSSS, 2012 ), Accreditation Canada’s quality management framework ( Agrément-Canada, 2014 ), and PLANTREE’s person-centred approach ( Cosgrove, 1994 ). The QPM encompasses four dimensions of performance: customer, accessibility/quality, mobilisation, and optimisation. Each dimension presents an aspect of the CCSMTL performance and encompasses a number of sub-dimensions enriched by the perspectives of the customers and the community (see Table 1 ). Figure 1. Quality-Performance Model (QPM) ( Couillard, 2018 ). Table 1. QPM dimensions and sub-dimensions enriched by the customer and community perspective ( Couillard, 2018 ). Dimension Sub-Dimension Definition Customer Partnership Involve myself and my relatives as partners in the care and services you provide Experience Make sure you have a human relationship with me so that I have a positive experience Communication Listen to me, inform me of my rights and give me complete information in accessible language Accessibility / Quality Accessibility Welcome me with an open mind and provide me in a timely fashion with appropriate care and services Security (safety) Ensure my safety Effectiveness Do what it takes to provide me with the best care and services Continuity Coordinate my care and services throughout my trajectory Optimisation Efficiency Avoid waste when using your resources Viability Consider all perspectives to meet my present and future needs Agility Be proactive and responsive to my expectations, values, and rights Innovation Dare to introduce new ways to offer care and services Learning Learn from experience how to do better Mobilisation Healthy work environment Take care of those who care for me and humanise the physical environment Focus on people Work with community partners who can help us Knowledge development Support those who care for me to maintain and develop their knowledge Collaboration Talk to each other and work together Accessibility/quality and customer are the two main dimensions of the QPM. They are presented in the form of a compass needle that points in the right direction to achieve better performance. Positioned at the centre of the model, customer dimension focusses on customer satisfaction regarding services and care provided. It presents three sub-dimensions: partnership, experience, and communication. Customers refer to people who receive a service or care and their relatives, as well as the community and the population in general. Accessibility/quality is the organisation’s ability to meet safely the needs and expectations of customers and ensure accessible and continuous services. It presents three sub-dimensions; accessibility, safety, and relevance/effectiveness of services. Mobilisation is the use of skills and talents of all people working in the organisation as well as fostering personal achievement and commitment to fulfil collectively the organisation’s mission. It encompasses four sub-dimensions, efficiency, agility, innovation, learning, and sustainability. Finally, optimisation is the search for continuous improvement in order to use efficiently available resources and provide services that are adapted to customer needs. Its sub-dimensions include healthy work environment, knowledge development, collaboration, and focus on people. The CCSMTL also implemented its visual management system formed by a network of control rooms at different decision levels of the organisation. Put simply, a control room is a dedicated space where managers and employees meet regularly to acquire information about current situation and initiate discussions to improve future performance ( Lagacé et Landry, 2016 ). At the highest organisational level, we find the strategic room, which is used by the CEO and senior managers. Data is used to evaluate the organisational performance, monitor current strategy, and support strategic decision-making (e.g. prioritising major projects). Tactical rooms are deployed in the different programs and sub-programs. Managers discuss their departments’ performance and decisions to be made at their level. Some of these decisions flow-up to the strategic room for validation while others flow-down to ‘intermediate’ tactical rooms or operational control rooms to be executed, in a cascade-escalation mechanism ( MSSS, 2015 ). Some tactical rooms referred to as “intermediate tactical rooms” might play an intermediate role between tactical and operational levels. Operational control rooms (or visual stations) ensure day-to-day operations management ( MSSS, 2015 ). The QPM is aimed at being implemented in all CCSMTL control room networks to support performance management. As mentioned earlier, QPM deployment is very challenging for the CCSMTL. First, it must be implemented at different decision-making levels, organisational structures, and territories. However, performance indicators may differ greatly from one service to another or from one territory to another. Second, some performance aspects such as customer and mobilisation dimensions are not sufficiently or well measured, which leads to an unbalanced performance management system. Finally, CCSMTL managers need to measure the overall performance of a given service or program, through integrated information. Currently, such information is not available nor easy to obtain. Literature review Measuring the performance of healthcare organisations might be very challenging ( Ferreira et al., 2009 ; Fache, 2016 ; Solomon, 2013 ) notably in complex organisations. Among recent studies that reported on performance management in healthcare organisations, we can cite Moisan (2019) who focused on the CISSS of Gaspésie region (province of Québec). This study highlighted the experience of this institution in deploying its performance management system and described how it contributed to improve the client trajectory performance in the context of social services dedicated to youth. Fache et al. (2016) investigated the role of the Department of Health in Quebec which faces the situation of "indicator chaos". To the authors, this chaos is at the quantitative level (too many indicators) and at the qualitative level (different measures for the same indicator), and consequently, this situation complicates effective organisational management and evaluation. Ferreira and Otely (2009) introduced the performance management system framework as a research tool aimed at providing a more comprehensive description of the structure and functionality of performance management systems. Their framework was developed based on a synthesis of relevant literature and insights gathered from their observations and experience. Patient-reported outcome and experience measures (PROMs and PREMs) for system performance measurement is considered as a way to enhance and promote the shift of healthcare systems towards a more coordinated, integrated and people-centred care. This approach has been the focus of recent studies such as Bull and Callander (2022) and Nuti et al . (2017) who discussed the importance of using patient-reported measures as well as indicators based on administrative data to evaluate cross-sectional healthcare services in a multidimensional healthcare performance system. Bull and Callander (2022) discussed the different performance measurement programs employed across England, the US and Australia, and suggested key recommendations for advancing PROM and PREM programs internationally. A performance measurement system could be seen as the process of assessing the organisation’s progress in achieving its goals ( Kairu et al ., 2013 ). It contains financial and non-financial measures ( Okwo & Marire, 2012 ). It should enable the correct deployment of the strategic and tactical objectives and provide a structured framework enabling the relevant feedback of information to the appropriate levels to facilitate decision-making and control ( Bititci et al. , 1997 ). Botton et al. (2012) classified performance measurement systems according to three categories of approaches: strategy-oriented (BSC ( Kaplan & Norton, 1996 ), performance prism ( Neely et al ., 2002 )), hierarchical (performance pyramid), and process-based (macro-process model ( Brown, 2008 )). The QPM can be considered as strategy-oriented. Due to its multi-dimensional and strategy-focussed aspects, it is consistent with the BSC model. In their recent systematic review, Amer et al . (2021) provided managers and policymakers with evidence to support using BSC in the healthcare sector. They conclude that BSC implementation demonstrated positive outcomes for patient satisfaction and financial performance. In their systematic literature review, Betto et al . (2022) attempted to understand the evolution of BSC in healthcare and their results revealed that the majority of studies focussed mainly on the BSC design process, rather than BSC implementation, use, or review. To Bohm et al . (2021) , the composition of BSC development teams in healthcare should consider including patients as well as their families where appropriate and select highly achievable and valid performance measures aligned with the organisational strategy. Developed in 1992 by Kaplan and Norton, the BSC is the result of a re-examination of performance measurement systems that usually give too much importance to the financial aspect. It is designed to measure four perspectives: finance (measures financial performance and the efficient use of financial resources), customer (concerns the organisation’s impact on the market in terms of customer satisfaction), internal processes (focusses on the efficiency and quality of internal processes) and learning & growth (concerns the culture and perspectives of the organisation, human resources, and information system). The term “balanced” underlines the concept of balance between all four axes of the model ( Kaplan & Norton, 1992 ). In BSC model, we distinguish leading indicators, which measure processes, and lagging indicators that measure outcomes. Causality is one of the most important aspects in the implementation of BSC: there is an implicit causal relationship between BSC perspectives and lag-lead indicators ( Porporato, Tsasis, et Vinuesa, 2017 ). To this end, a visual map (or strategy map) is built to show the objectives and causal links needed to execute a strategy. Scholey (2005) provides a well-structured six-step based process for creating a strategy map. Kaplan and Norton stressed the need to measure causal relationships, by using a complementary analytical approach ( Barnabè et al ., 2012 ) even though, it is not easy to analytically validate all causal relationships ( Lorino, 2001 ; Schrage & Kiron, 2018 ). This is also the case for healthcare ( Porporato et al ., 2017 ). Canada leads in hospital performance measurement and historically, it is among the first countries to develop scorecards in healthcare ( Porporato et al ., 2017 ). In Toronto, Ontario, Women’s College Hospital started using a scorecard based on key objectives in the 1990’s ( Porporato et al ., 2017 ). Since then, Ontario hospitals have gained great experience in designing and using scorecards. Chan (2009) presented the evolution of healthcare systems in implementing BSC and strategy maps in Ontario. From 1992 to date, BSC has evolved from a strategic alignment tool (normative form) to a strategy construction tool. The normative version proposes a “top-down” deployment approach to operationalise the organisational strategy. Studies have shown that BSC can contribute to the emergence of a new strategy, and it would be better to opt for a “bottom-up” approach, where strategic objectives would be the result of collaborative management ( Choffel & Meyssonnier, 2005 ; Ittner & Larcker, 2003 ; Norreklit, 2000 ; Simons, 1994 ). In addition, the learning & growth axis should aim to shape the organisation’s culture. Kaplan (2020) presented the case of merging St. Mary’s Hospital and Duluth Clinic (Minnesota, US) who co-created a new strategy, a strategy map and a common BSC. Montalan and Vincent (2011) proposed an approach for the co-construction of the strategy of a French hospital. They developed and validated a strategy map by a multidisciplinary teamwork following a bottom-up approach. Figure 2 shows the evolution of BSC over time and application in healthcare. Figure 2. BSC evolution over time and application in healthcare. In order to measure the performance while considering the different dimensions of the BSC (healthcare and other sectors), some studies in the literature combined the BSC with multi-criteria decision-making techniques (MCDM). Most of these studies used the well-known Analytic Hierarchy (or Network) Process (AHP/ANP) ( Anjomshoae et al ., 2019 ; Chan, 2006 ; Modak et al. , 2019 ; Regragui et al. , 2018 ; Yaghoobi & Haddadi, 2016 ). Chan (2006) combined AHP and BSC to identify global metrics and compare performance across Canadian healthcare organisations. Regragui et al. (2018) also combined BSC and AHP to develop a framework for assessing performance in hospitals. By using AHP, the authors weighted different performance perspectives and indicators, leading to a ranking of these indicators. In other studies, AHP has been successfully used to consolidate, prioritise, and weight performance indicators ( Bentes et al ., 2012 ; Hossain et al. , 2019 ; Yaghoobi & Haddadi, 2016 ). For more on AHP applications in healthcare, see Hummel and IJzerman (2011) . Recently, Daneshmand et al . (2022) developed a BSC-AHP based model for assessing the health system of the Social Security Organisation in Iran. They conclude that, the most important performance dimensions are internal processes and social obligations. Table 2 presents most recent BSC applications in healthcare including studies that combined BSC and AHP. Our proposed approach is inspired by the work of Regragui et al. (2018) . Table 2. Examples of most recent BSC applications in healthcare. Country Perspective Reference Australia Validate a non-profit version of the BSC ( Soysa et al. , 2019 ) Integrating an environmental dimension with BSC ( Khalid et al. , 2019 ) Brazil Performance Management - survey on performance management practices in Brazil ( Portulhak et al. , 2017 ) Canada Performance measurement - implementation of a bottom-up approach ( Backman et al. , 2016 ) Integrating AHP to hospital scorecards in performance assessment ( Chan, 2006 ) Information management - key indicators - strategic alignment ( Nippak et al. , 2016 ) Causal Relationships ( Porporato et al. , 2017 ) Queue performance evaluation ( Breton et al. , 2017 ) China Establishing an Indicator System using the BSC ( Gao et al. , 2018 ) Ethiopia Performance measurement - development of a scoring methodology ( Teklehaimanot et al. , 2016 ) Germany Prioritise strategies ( Dekrita et al. , 2019 ) India Dynamic Balanced scorecard and its connectivity with Corporate Social Responsibility ( Ghosh & Singh, 2021 ) Iran Performance measurement system for the Social Security Organisation's health system. ( Daneshmand et al. , 2022 ) Morocco Conceptual framework for evaluating performance ( Regragui et al. , 2018 ) Saudi Arabia Cloud computing - Strategy map ( Alharbi et al. , 2016 ) UK Success factors for performance management ( Moullin, 2017 ) USA Accreditation in hospitals (ISO9001) ( Ritchie et al. , 2019 ) Merging health care organisations ( Kaplan, 2020 ) Vietnam Performance evaluation ( Pham et al. , 2020 ) Methods Our approach encompasses two parts as illustrated in Figure 3 . The first part (Phases I and II) aims at describing and analysing the current situation while the second part aims at developing or improving a PMMS (Phases II and IV). Figure 3. Proposed approach. Current situation analysis (Phases I and II) The data collection (phase I) began in May 2019 (the study started in January 2019). Collected data was used to gain a more in-depth understanding of CCSMTL internal processes and current practices and progress regarding the implementation and use of its visual management system and QPM (deployment level, indicators, implementing obstacles and issues, etc.). To this end, three methods were used: review of documents, field observations, and structured interviews (see Table 3 ). Table 3. Description of the used methods for phase I. Data collection methods Description Document review MSSS’s Law on healthcare and social services, Health Standards Organization (HSO) standards ( HSO, 2021 ), Accreditation Canada ( Agrément-Canada, 2014 ), Ministerial reference framework for evaluating performance in public healthcare and social services systems ( MSSS, 2012 ), MSSS strategic plan 2019-2023, CCSMTL annual reports ( CCSMTL, 2016 ), and PLANETREE documents (e.g. Cosgrove, 1994 ). Observations 1-Participation in the QPM Coordinating Committee and the QPM Advisory Committee meetings. 2- Participation in a performance improvement workshop. 3-Visiting several control rooms of the CCSMTL at different levels of the organisation: the strategic room, five tactical and intermediate rooms, and 10 operational rooms. 4-Visiting two hospitals and two rehabilitation centres. Structured interview 12 structured interviews involving 13 managers (Directors, assistant managers, executive advisors, department heads) and four Programs and departments (i.e. different decision-making levels and services) between May 2019 and September 2021. The following are examples of questions asked during the interviews: - Are the strategic objectives targeted by the implementation of the QPM? - What are the practices put in place for implementing the QPM? - How does the CCSMTL measure performance and quality in the organisation? - What are the difficulties met during your performance measurement process? The Coordinating Committee of the CCSMTL is composed of a limited number of managers from the Quality, Evaluation, Performance and Ethics department. It aims at implementing the QPM within the CCSMTL, in accordance with the MSSS strategic plan. Periodically, the Coordinating Committee invites the heads of all CCSMTL departments to discuss decisions to be made related to the QPM and collect their feedback. This enlarged group forms the QPM Advisory Committee. The objective of the interviews (with a duration of 1 to 2 hours) was to validate and consolidate our understanding of the performance management and measurement methods and tools being used and the progress in implementing the QPM and visual management system. The structured interview was used ( DiCicco‐Bloom and Crabtree, 2006 ) the interviewer asks several structured questions, and the interviewee responds freely to the questions. The participants were selected in collaboration with the deputy director of organisational performance at the CCSMTL, who is responsible for QPM implementation. When required, a second meeting was scheduled with the same participant(s). For more details, the reader may refer to Ben Fradj et al. (2020) . Next (Phase II), the SWOT (Strengths, Weaknesses, Opportunities, Threats) method was used to structure and analyse the collected data. SWOT analysis is a popular tool from which strategies can be developed and which improves organisational performance ( Pickton and Wright, 1998 ). It helps to clarify the strengths to be maintained, the weaknesses to be addressed, the opportunities to benefit from, and the constraints to be handled. In our case, we used the SWOT to identify the strengths and weaknesses related to the current implementation of the QPM model. Once the data was collected, the research team brainstormed to cocreate the preliminary version of the SWOT matrix. Next, four discussion meetings have been organised with three different managers from the CCSMTL to validate and complement the final version of the SWOT (see Table 4 in the Results Section). PMMS Development (Phases III and IV) Balanced-QPM and strategy map development (Phase III) Based on the results of our SWOT analysis and inspired from the literature, we use BSC principles to create a more balanced QPM. In other words, we use the BSC in order to address the weaknesses of the QPM model identified in our SWOT analysis. Indeed, by comparing BSC and QPM models, we identified similarities and complementary aspects. First, there is a strong similarity between the dimensions of performance of the QPM and the axes of the BSC, in the way that they express equivalent pillars of performance (customer, internal processes, resource management, etc.). Second, both models enable defining performance indicators and creating a measurement system. Third, the QPM and the BSC provide a basis for stakeholders to share and discuss strategic objectives. Finally, the BSC is often recommended in the healthcare sector as mentioned in the literature review section. First, the axes of the BSC, i.e., the version adapted for non-profit organisations by Kaplan, and the dimensions of the QPM are compared to each other. For example, BSC’s customer perspective corresponds to the customer dimension of the QPM. In the CCSMTL context, achieving the goals of this dimension reflects the organisation’s main mission and therefore would take the first position in the Balanced-QPM. Similarly, BSC’s internal process perspective is equivalent to the accessibility/quality dimension in the QPM. We recall that accessibility/quality refers to the ability to safely meet customer needs and expectations by providing accessible and continuous services. This dimension would take the second position in the Balanced-QPM. The positioning of customer and accessibility/quality dimensions is consistent with the CCSMTL vision which considers the two dimensions as the “True North” of the organisation. Second, based on the causal relationship between the BSC axes, we identify the causes and effect links specific to the dimensions of the QPM model. This aspect was absent in the intitial QPM. For instance, the optimisation dimension has direct impact on the accessibility/quality dimension, which in turn has a direct effect on the customer dimension. The result of these two processes (positioning the QPM dimensions and identifying causal relationships between them) leads to what call the Balanced-QMP (see Figure 5 in the Results Section). To create the CCSMTL’s strategy map, the causal links between the Balanced-QPM dimensions are further analysed. We were inspired by the six-step guidelines suggested by Scholey (2005) . To the author, this process leads to the creation of a comprehensive strategy map, ensuring a clear and communicable strategy throughout the organization. It consists in 1) identifying the primary objective, 2) selecting the value proposition, 3) determining general financial strategies to follow, 4) establishing customer-focused strategies, 5) aligning internal processes to support strategy execution, and finally 6) implementing necessary skills, capabilities, and employee programs to achieve the strategy. For the CCSMTL, the primary objective is to satisfy its customers by providing general and specialised healthcare and social services while insuring accessibility, security, efficiency and quality. An example of a target to achieve for this primary objective would be to increase the access delay compliance rate to 95% within one year. For the value proposition selection, we relied on the study by Treacy and Wiersema (2007) , who identified three value propositions; operational excellence, product excellence and customer relationship. Since the CCSMTL follows a customer-oriented approach, its value proposal focusses on customer relationship. The remaining step are based on the QPM sub-dimension definitions enriched by the customer perspective (see Table 1 ) that could be further expanded by the objectives set by the department managers, in relation to their specific mission. In addition, rigorous criteria for selecting relevant objectives such as specificity, validity, sustainability, reliability, and utility could be used to support this process. The resulting strategy map is shown in Figure 6 in the Results Section. Indicator structure design and performance assessment (Phase IV) This phase aims at providing relevant indicators measuring performance following the objectives specified in the strategy map in each performance dimension. Step 1: Indicator system structure We propose to create a structured system that provides indicators as well as performance indexes for each performance dimension at each hierarchical level of the CCSMTL. This general structure is based on the CCSMTL organisational chart and its visual management system. To provide and monitor the performance indicators, scorecards presenting the four dimensions (and their indicators) can be used at each department and hierarchical level, forming a network of scorecards (information is presented and discussed in the control rooms during regular meeting. This proposal is inspired from the study of Voyer ( Voyer & Voyer, 1999 ) who developed a network of interrelated dashboards used at different levels within an organisation. Béland and Abran (2004) also used this concept (see Figure 7 in the Results Section). Step 2: Indicator identification and selection From the established strategy map, performance indicators are selected for each dimension following strategic objectives that are relevant to the department/service. Existing indicators should be reviewed first. As suggested by Bull and Callander (2022) and Nuti et al. (2017) patient-reported measures and indicators based on administrative data can be used. Existing indicators and new ones (selected based on the literature or on recognized standards) should be elected according to criteria ensuring specificity, validity, sustainability, reliability, utility, etc. Such criteria are well documented in the literature for selecting relevant performance objectives and indicators. At the end of this step, each department will have a set of indicators for each dimension of the Balanced-QPM called “before normalisation indicators”, noted as Xbn in the following paragraphs. Step 3: Indicator normalisation The “before normalisation indicators” Xbn may have different units of measurement and different magnitudes in healthcare contexts. In some cases, the measurement unit is expressed in monetary units (e.g. expenses), and in others, in time units (e.g. hours) such as waiting time. In other cases, there is no measurement unit (e.g. number of services provided). In addition, the same indicator may have distinct targets in different departments. For example, waiting time target in an emergency department is significantly less compared to psychosocial services. Therefore, it might be difficult to compare them to each other based on their absolute values. Due to this, normalising the indicators is very helpful. Normalisation techniques for processing data have been used by many researchers in different fields. We refer the reader to Singh and Singh (2020) for more on normalisation. We selected the Min-Max normalisation technique ( Singh & Singh, 2020 ), mainly because minimum and maximum limits can be easily distinguished in available data provided by the CCSMTL. Following this technique, data is usually rescaled within the range 0 to 1 or (-1) to 1. The general equation is given as follows: (1) X norm = ( Nmin − Nmax ) X bn − min max − min + Nmin With: X norm : Normalised value of the indicator X bn : Indicator value before normalisation min : Minimum value in the data max : Maximum value Nmin ; Nmax : Normalised Min and Max For the CCSMTL context X norm is expressed as a percentage ranging from 0% to 100%, therefore Nmin et Nmax values are 0% and 100%, respectively. Given that in some cases the maximum value of an indicator is the undesirable value while the minimum value is the desired value (e.g. the maximum value for waiting time is non-desired while the minimum value is desired), we set the best value in the data set as max parameter (which is not necessarily the highest value) and the worst value in the data as min (which is not necessarily the lowest value) before normalisation. Outliers must be eliminated before choosing the best and worst values. To link a given indicator to a desired objective and to ensure its continuous improvement, max value corresponds to the indicator target. For each performance dimension, there is a set of associated indicators. Each indicator takes a numerical index i in the set and each indicator has an index dim referring to the first letter of the dimension to which it belongs (e.g. C for Customer). Indicators before normalisation take the index bn , referring to “before normalisation”. Normalised indicators in the original formula ( Equation (1) ) X norm become X i dim in the new formula adapted to the CCSMTL ( Equation (2) ). We remove the index norm for the sake of simplicity. (2) X i dim = ( 100 ) × Xbn i dim − min i target i − min i Each department/service at different hierarchical levels calculates its normalised indicators ( X i C , X i A , X i M , X i O ). These are the inputs of the fourth step. Step 4: Indicator weighting Step 4 aims at weighting the normalised indicators of the same dimension at the same hierarchical level. AHP technique is used. AHP is usually used to evaluate and rank a set of evaluated based on several criteria. In this study, there is no need to rank or choose an option, but rather weighting the indicators within each performance dimension and aggregating them to obtain performance indexes for each dimension for a given department (see Figure 4 ). We selected AHP due its simplicity ( Kumar et al ., 2017 ) since it is very convenient for decision-makers who are not familiar with analytical tools. Moreover, it relies on pairwise comparisons, and according to Ishizaka and Labib ( Ishizaka & Labib, 2011 ), psychologists argue that it is easier to express an opinion based on two elements rather than on all elements simultaneously. Figure 4. AHP adaptation for performance measurement for the CCSMTL case. Criteria and indicators that are usually used in AHP, correspond to the four dimensions of the Balanced-QPM and to the normalised indicators X i dim , respectively. The normalised indicators are compared to each other by using pair-wise comparison matrices (one matrix for each dimension) according to their relative importance regarding the dimension to which they belong. These preferences are expressed by managers. A given element a ij of a matrix A (see Equation (4) below) corresponds to the relative importance of indicator i (in the row) compared to indicator j (in the column). The preference values range from one to five, which is adapted from the preference scale proposed by Saaty (2005) . For example, a ij = 1 means that the indicators are equally important and a ij = 5 that indicator i is extremely important compared to indicator j . The weights of the performance indicators are calculated by using a specific algorithm described in detail by Brunelli (2014) . All AHP calculation steps are presented in the next paragraphs. Comparisons make sense only if the pair-wise comparison matrices are coherent or quasi-coherent. A consistency ratio (CR) is used to check this aspect. If CR is less than 10%, the matrix is consistent, otherwise, the pair-wise comparisons are repeated (see Equations (5) and ( 6 ) below). Once consistency is validated, the weights of the indicators, noted as W i dim ( W i C , W i A , W i M , W i O ) are obtained. Step 5: Indicator aggregation Indicators within each dimension are aggregated (see Figure 8 ), and four performance indexes ( P C , P A , P M , P O ) are calculated (one index for each dimension) following Equation (3) : (3) P dim = ∑ i n W i dim × X i dim P dim : Performance index of dimension dim W i dim : the weight of indicator i within the performance dimension dim X i dim : i th normalised indicator within performance dimension dim Equations (4) to (6) provide more details about the AHP calculation steps. (4) A = ( a 11 ⋯ a 1 j ⋯ a 1 n ⋮ ⋱ ⋮ ⋮ a i 1 a ij a in ⋮ ⋮ ⋱ ⋮ a n 1 ⋯ a nj ⋯ a nn ) With: A: pair-wise comparison matrix n: number of indicators to be compared, which corresponds to the matrix size A = a ij , a ij > 0 ; i , j ∈ ⟦ 1 , n ⟧ 1 a ij = a ji ∀ i , j ∈ ⟦ 1 , n ⟧ a ij =1 ∀ i = j 2. Calculation of consistency ratio (CR) (5) CR = CI RI (6) CI = λ max − n n − 1 With: CR: Consistency ratio CI: Consistency index RI: Random index (depends on n ) λ max : Maximum eigenvalue of the matrix n: Matrix size Results In this section, the results of phases II (SWOT analysis), III (Balanced-QPM and strategy map) and VI (indicator structure design and performance assessment) are presented. First, the results of the SWOT analysis are presented ( Table 4 ). After that, the Balanced-QPM ( Figure 5 ), the CCSMTL’s strategy map ( Figure 6 ), and the indicator structure system proposed for the CCSMTL are introduced. Finally, to illustrate how a strategy map could be built for a specific department, and how the performance indicator structure could be applied and the performance measured, the physical disability, intellectual disability, and autism spectrum disorder case, referred to as DI-TSA-DP, is used. Results of the SWOT analysis From Table 4 , we can see that one of the strengths identified is the well implementation of the QPM at the strategic level. There is strong interest by senior managers in aligning the organisation’s practices with QPM objectives. In addition, there are several practices put in place to ensure service quality and customer satisfaction and there is a real willingness to embed a customer-focused culture through the implementation of purposeful projects. The deployment and use (even partially at the moment of conducting this study) of the visual management system at different decision-making levels demonstrates a real desire to build an effective management and measurement performance system. We also identified that there are enough indicators in the accessibility/quality dimension. Another important aspect is that managers at different hierarchical levels have the possibility to propose performance indicators in compliance with general ministerial objectives (even though some indicators are imposed by the MSSS). Moreover, each department holds regular meetings, which helps to maintain communication within the departments and support the information cascade-escalation process. While the QPM is well implemented at the strategic level, it is not sufficiently operationalised at the tactical and operational levels. We also identified that the QPM’s dimensions are unbalanced in terms of number of performance indicators used within each dimension. We find many indicators measuring the quality/accessibility dimension, but there is a lack of indicators in optimisation, mobilisation, and customer dimensions. There is also a lack of specific targets for those indicators. With the current performance management system, it is not possible for senior managers to track performance indicators at tactical/operational levels or to verify the overall performance without consulting a large amount of information. In addition, it is not possible to examine easily the performance of a specific service or department. There is a discontinuity in the assessment of performance between hierarchical levels. That is, each department/service measures its performance independently of the higher hierarchical level and other departments/services. We identified that dependencies and conflicts among some indicators exist, and this is challenging for managers to deal with. For example, there is a need to simultaneously reduce waiting time and improve direct service time (time with customers). An improvement in one indicator can decrease the other and vice versa. Regarding opportunities, the CCSMTL must continuously challenge itself to maintain/obtain accreditations and certifications such as PLANTREE, Accreditation Canada and to also comply with the MSSS’s strategic plans. Finally, like most organisations in a continuously changing socio-economic context, the CCSMTL might be threatened by pandemics such as COVID-19 or workforce shortage. These aspects can notably lead to disrupt service accessibility. Table 4. Results of the SWOT analysis (Phase II). Strengths Weaknesses Internal aspects - The QPM is well implemented at the strategic level. - Several practices ensuring high-quality services are implemented. - There is a willingness to embed a customer-focussed culture through the implementation of purposeful projects. - The visual management system is implemented at different decision-making levels. - There is a sufficient number of indicators within accessibility/quality dimension. - CCSMTL managers have a certain flexibility in proposing performance indicators for their departments. - There is good communication within the organisation. - The QPM is known at the tactical and operational levels, but not used for performance management and measurement purposes. - The QPM dimensions are unbalanced in terms of number of indicators within each dimension. - There is a lack of global and aggregated indicators. - It is not possible for senior managers to easily examine the performance of a specific service or department. - There is a discontinuity in the assessment of performance between the hierarchical levels. - The information presented in dashboards is not well discussed. - Some indicators are interdependent, and other ones are conflictual. External aspects Opportunities Threats - Several opportunities are provided by institutions such as PLANTREE, Agrément Canada, MSSS, etc. to implement best performance management practices. - Socio-economic context (pandemics such as COVID-19 and population aging). - Workforce shortage issue. Balanced-QPM, strategy map and indicator structure system (CCSMTL level) Figure 5 shows the dimensions of the proposed Balanced-QPM. BSC’s customer perspective corresponds to the customer dimension of the QPM. BSC’s internal process perspective is equivalent to the accessibility/quality dimension in the QPM. Learning & growth axis is found in both mobilisation and optimisation dimensions of the QPM. Learning & growth presents three main components: human capital, information capital, and organisational capital. Human capital refers to the management of skills and behaviours of employees. This aspect is part of the mobilisation dimension of the QPM (use of skills and talents of everyone in the organisation, partners, customers, and their families, encouraging personal development and commitment to accomplish the mission of the CCSMTL). The mobilisation dimension takes the third position in the Balanced-QPM. Organisational capital is defined as the organisation’s ability to align employee objectives with the strategy while information capital is the way organisations use their information systems, networks, and databases to support accomplishing their operations. Both aspects are part of the optimisation dimension of the QPM. We recall that optimisation dimension aims at continuous improvement and innovation to ensure, over time, that the services offered are adapted to the needs of customers. BSC’s financial perspective is also part of optimisation dimension. Kaplan & Norton (1999) stated that the financial perspective serves as a constraint, not an objective, for non-profit organisations such as healthcare systems. Therefore, optimisation dimension, takes the last position in the Balanced-QPM. Figure 5. Proposed Balanced-QPM. Additionally, Figure 5 shows global causal links between the four dimensions. The optimisation dimension has direct impact on the accessibility/quality dimension. There is also a mutual relationship between optimisation and mobilisation. Mobilisation has an impact on the accessibility/quality of services provided by the CCSMTL, which in turn has direct impact on customer satisfaction. Finally, customers put pressure on the MSSS to provide the necessary funding supporting the CCSMTL capacity in terms of optimisation, mobilisation, accessibility/quality aspects. Figure 6 shows the strategy map of the CCSMTL. The causal links between the four dimensions shown in the figure correspond to those identified in Figure 5 . For a customised strategy maps, e.g. DI-TSA-DP program, these causal links are defined more precisely (see Figure 8 for an example). The CCSMTL’s strategy map is the basis for building “customised” strategy maps for the different departments of the organisation. Figure 6. Generic strategy map proposed for the CCSMTL. Finally, Figure 7 shows the indicator structure system of the CCSMTL. We recall that this structure is based on the CCSMTL organisational chart and visual management system (CCSMTL’s network of control rooms). We propose to use a network of scorecards presenting the performance indexes and indicators of each dimension of the Balanced-QPM of the different departments of the CCSMTL to evaluate and discuss the performance at different hierarchical levels of the organisation. In other words, information on performance would be presented in scorecards (For an example, see Figure 10 ) and discussed in CCSMTL’s control rooms, and then would flow-up from operational to tactical and strategic levels (through tactical and strategic control rooms) or flow-down from strategic to tactical and operational levels, following the cascade-escalation mechanism used in the current visual management system of the CCSMTL. Figure 7. Proposed hierarchy and network-based scorecard structure. Balanced-QPM scoredcard at the visual station level. DI-TSA-DP case DI-TSA-DP program offers services to 2,200 customers and receives approximately 10,000 service requests per year, mainly in the South-Centre of Montreal region and throughout the province of Quebec. 2,150 employees and 58 physicians work in this department. It includes three main sub-programs; DI-TSA, which provides services to customers with an intellectual disability or autism spectrum disorder, DP, which provides services to customers with a physical disability, and RMVS, which provides rehabilitation services in substitute living environments such as elderly homes. DI-TSA-DP mission is to provide specific, specialised and super-specialised habilitation and rehabilitation services to its customers to promote their integration and social participation. At the moment of conducting this study, DI-TSA-DP has put in place one tactical room (top management level of the program), and three intermediate control rooms at the sub-program level (DI-TSA, DP, and RMVS). Operational rooms are established at the team unit level within the three sub-programs. Control room deployment at the team unit level was still in progress. Therefore, each sub-program had only reached a certain deployment rate, which is seen as a measure of performance-culture maturity level in the CCSMTL. Following the recommendation of our collaborators, we focussed on DP sub-program, since it had the highest deployment rate. The approach can be applied to the two other sub-programs as well. DP includes three services: AT (Technical Aids), which provides special assistance devices (prosthesis, wheelchair, etc.), LN (Locomotor-Neurology), which provides services to customers with physical disabilities due to neurological or locomotor disorders, and SL (Sensory-Language), which provides services to customers with linguistic or sensory disabilities. We focus on LN service, again based on its operational room deployment rate (100%). There are nine care team units within LN service, and each unit has its own operational room. We worked in close collaboration with two teams, Locomotor team (A-BOG), which provides services for customers with an amputation or serious orthopaedic injuries, and Neurological team (AVC), which provides services for customers who have had a stroke. DI-TSA-DP strategy map The proposed strategy map for DI-TSA-DP ( Figure 8 ) is consistent with the generic strategy map ( Figure 6 ). It was co-created with the DI-TSA-DP program executive manager. Figure 8. Proposed strategy map for DI-TSA-DP program. It illustrates objectives determined following the QPM sub-dimensions (including the primary objective) and causal links established between these objectives and/or sub-dimensions. The objectives selected are based on the internal DI-TSA-DP strategic vision 2018-2021 provided by DI-TSA-DP executive manager. Each objective was checked against the sub-dimensions of the QPM to identify to which one it is related. In total 16 objectives were classified within the sub-dimensions and dimensions of the QPM. The DI-TSA-SP mission was chosen as the primary objective and added in the customer dimension. Since no objective exists for customer experience, a new objective was proposed based on customers’ definition of this sub-dimension ( Table 1 ). Additional objectives should be set for this sub-dimension and other ones that currently do not present any specific objectives (i.e., communication, learning and focus on people). Examples of causal links in the strategy map, are the relationship between objectives within customer sub-dimensions (‘partnership’ and experience’) and the primary objective (DI-TSA-DP mission). It is clear that positive customer experience, customer involvement as a partner in the system, and providing a climate of trust for the customers contribute to achieving DI-TSA-DP mission. It is interesting to notice that causal links also exist between sub-dimensions/objectives within the same dimension. As an example, partnership objectives contribute to achieving customer experience objectives. That is, when customers are involved as partners in the system, or when a climate of trust and collaborations is established with them, they are likely to have a positive experience. Some objectives/sub-dimensions mutually impact each-other, e.g. efficiency (optimisation dimension) and healthy work environment (mobilisation dimension). Reducing indirect labour and extra working hours (efficiency sub-dimension) leads to a healthy work environment, which in turn has a positive impact on efficiency. The strategy map was key for aligning DI-TSA-DP vision and strategy with the four dimensions/sub-dimensions of the QPM. This step helped to bring clarity and logic to the strategic objective determination process in relation to the QPM dimensions/sub-dimensions and performance measurement. Based on the strategy map, it was also possible to align LN service performance indicators with the four dimensions of the QPM through the strategic objectives and sub-dimensions selected as shown in the next paragraphs. DI-TSA-DP performance measurement (phase IV) Indicator structure design and performance measurement We conducted five working meetings of two hours each with different DI-TSA-DP managers. We started with the operational level (A-BOG and AVC) and moved to the tactical level (LN and DP service and sub-program, respectively), and finally, to the highest level (DI-TSA-DP program). The following paragraphs present the process of applying the developed PMMS to DI-TSA-DP case. Step 1: Indicator system structure Figure 9 shows the indicator system structure proposed for DI-TSA-DP. It is based on DI-TSA-DP organisational chart (and visual management system) and is part of the CCSMTL indicator system structure ( Figure 7 ). Figure 9 shows only the departments needed to illustrate our approach. We distinguish four levels: 1) CCSMTL strategic level, 2) tactical level, which includes DI-TSA-DP program and DI-TSA, DP, and RMVS sub-programs, and 3) operational level (within DP sub-program) that includes SL, LN, and AT teams. Figure 9 also shows the network of control rooms deployed in these different levels. The performance indicator structure follows the organisational chart and control room network structure. They are presented in the form of scorecards following the four Balanced-QPM dimensions (indicators, values and targets, performance indexes). For an example of a scorecard, see Figure 10 . Figure 9. Simplified indicator system structure proposed for DI-TSA-DP. Step 2: Indicator identification and selection To apply the rest of Phase IV steps, LN service was considered, and the process was performed in collaboration with LN coordinator. First, indicators being used by LN service were classified according to the four dimensions of the QPM and the strategy map’s sub-dimensions/objectives. Next, we checked the relevance of each performance indicator (against the criteria specificity, validity, simplicity, relevance, sustainability, reliability, and utility). When an indicator does not meet all criteria, it is either changed for another indicator that satisfies the criteria, or more information is gathered to ensure that the indicator meets all criteria. As an example, indicator ‘Customer satisfaction rate’ ( Xan 1 C ) ( Table 5 ), which was proposed to measure customer experience sub-dimension objective, did not meet simplicity criterion. LN managers did not have information on customers’ satisfaction and did not put in place specific methods to collect this information. To address this issue, we relied on the results of a survey used to collect the feedback of customers regarding their satisfaction level regarding experience and service quality. Table 5 shows the 10 final indicators selected for LN service. Table 5. Indicators selected and their values after normalisation (LN service). Dimensions Measured sub-dimensions / Objectives Indicator description Indicator notation ( Xbn i dim ) Xbn i dim value Targets (Targeti) Worst value (min i ) Normalized indicator ( X i dim ) value Customer Experience Customer satisfaction rate Xbn 1 C 75% 90% 75% Experience Complaint rate Xbn 2 C 60% 40% 100% 66.7% Accessibility/Quality Accessibility Bed occupancy rate Xbn 1 A 85% 95% 85% Accessibility Continuity Effectiveness Respect of access deadlines Xbn 2 A 80% 90% 80% Continuity Average length of stay (DMS) Xbn 3 A 17 days 10 days 50 days 82.5% Mobilisation Healthy work environment Collaboration Staffing level Xbn 1 M 90% 98% 92% Healthy work environment Ratio of salary insurance hours Xbn 2 M 7% 5,9% 13% 84.5% Healthy work environment Collaboration Employee satisfaction rate Xbn 3 M 70% 90% 70% Optimisation Innovation Reference number change rate Xbn 1 O 80% 90% 80% Innovation Number of programs that have implemented the ‘compassion’ approach Xbn 2 O 7 programs 9 programs 0 programs 77.8% Step 3: Indicator normalisation The columns of Table 5 show the indicators measuring the objectives (and therefore sub-dimensions) selected in previous step, their notations, their values before normalisation, their targets, their worst values, and their values after normalisation (obtained by using Equation (2) ). Due to confidentiality reasons, all real values of the indicators have been modified. Note that indicators that are already expressed as percentages before normalisation and which are aimed to be maximised to reach the desired target have not been normalised. The worst value is set to zero for those indicators (e.g. Customer satisfaction rate, Bed occupancy rate, and Respect of access deadlines). Moreover, indicators before normalisation meant to be minimised to reach their target have been converted into indicators to be maximisation after normalisation (e.g. Complaint rate and Ratio of salary insurance hours). Steps 4 and 5: Indicator weighting and aggregation Pairwise comparisons of normalised indicators within each QPM dimension were performed with the LN coordinator. The values obtained were entered in a VBA program that checks the consistency of the matrices and generates the weights following AHP algorithm (see Equations (4) to (6) ). The final results that represent the weights of the indicators are presented in Table 6 . Table 6. Calculating performance indexes for LN sub-service. Dimension Customer Accessibility/Quality Mobilisation Optimisation X i dim X 1 C X 2 C X 1 A X 2 A X 3 A X 1 M X 2 M X 3 M X 1 O X 2 O X i dim (%) 75% 66.7% 85% 80% 82.5% 92% 84.5% 70% 80% 77.8% W i dim 067 0.33 0.44 0.44 0.11 0.25 0.50 0.25 0.33 0.67 Performance index ( PI LN dim ) 72.2% 82.5% 82.3% 78.5% Step 5 enables us to calculate aggregated performance indexes for each dimension that can be used by the service managers or by higher hierarchy level departments (DP sub-program and DI-TSA-DP programs in our case) to evaluate how the service performs globally in each dimension of the QPM. For instance, if the global performance of a given dimension is not satisfying, the performance indicators within that dimension should be analysed in more detail to identify the problem and take actions to improve the situation. The results of calculating the four performance indexes ( PI LN C , PI LN A , PI LN M , PI LN O ) of LN are given in Table 6 . Each performance index corresponds to one dimension of the Balanced-QPM. Again, the VBA program is used to automate these calculations following AHP algorithm. These aggregated performance indexes are finally visualised on scorecard so that managers can examine each performance dimension and put actions to improve the critical ones. Figure 10 illustrates the LN scorecard with the performance indexes selected for LN for each QPM dimension and performance indicators for each dimension. Figure 10. LN scorecard. It is worth noting that, when using AHP, we identified a compensation effect between indicators that results from aggregating two (or more) indicators having opposite values, which is not desired. For instance, within Mobilisation dimension, the normalised indicator ‘Staffing level’ ( X 1 M ) has a value of 92% while the normalised indicator ‘Employee satisfaction rates ( X 13 M ) has a value of 70%. Their aggregation together with the normalised indicator ‘Ratio of salary insurance hours’ ( X 2 M ) (value of 84.5%), leads to a performance index with a value of 82,3% ( PI LN M ) ( Table 6 ). It is easy to see that the high value of ‘Staffing level indicator’ compensates the lower value of ‘Employee satisfaction rate’ indicator. This result also depends on the weights assigned to the three indicators (0.25, 0.25, and 0.50, respectively in our case). Discussion In this study, we have proposed a comprehensive and quantitative approach to develop or improve a PMMS. We have illustrated our approach by using CCSMTL’s current performance management model, the QPM. Recent studies in the literature recognise the importance and at the same time the complexity of measuring performance in healthcare organisations. However, most of these studies are descriptive and do not propose PMMSs capable of effectively structuring and measuring performance. The development of a strategy map enables the operationalisation of an organisation’s staregy and identifying relevant performance indicators aligned with the strategy. In other words, the strategy map plays a pivotal role in translating the organisation’s vision and strategy into performance objectives and measurable indicators. Moreover, in the literature, the BSC and AHP method (and its variants, e.g. ANP) have been used to identify global indicators, ranking and prioritizing those indicators, or compare performance across different healthcare organisations. Our approach allows managers to measure performance at different hierarchical levels of an organisation and aggregate the information to facilitate global performance assessment and support decision-making at the strategic level. This is an important problem faced by many healthcare organisations, notably by large institutions, yet it not well addressed in the literature. In the CCSMTL context, aggregating the performance indicators and calculating one performance index for each dimension by using AHP was well appreciated by the managers. In addition, our approach contributed to balancing the four dimensions of the CCSMTL’s QPM by ensuring that a sufficient number of performance indicators measure each dimension. Finally, performance indicators selection relied on a transparent and rigorous process that involved the participation of managers. This contributes to foster positive changes in the organisation regarding perfrormance measurement and performance culture in general. CCSMTL case study helped us to better understand the challenges and difficulties managers may face during strategy map construction and PMMS development/implementation processes. Regarding the strategy map, iterative meetings with managers were necessary to build the version presented in this work (DI-TSA-DP program). More collaboration is required to improve this preliminary version (i.e. determining more precisely the strategic objectives and causal links between them). We have also observed that choosing targets for the indicators was not an easy task for managers. In fact, most existing indicators are associated with organisational targets that are set by strategic management. Due to this, it is challenging for managers to select reasonable targets for new indicators. In addition, deep discussions took place between managers during the processes of comparing indicators to each other to agree on common preferences. This shows the difficulty of making common pair-wise comparisons and deriving the right weights, and therefore the right performance index values. This aspect was also observed in Boukherroub et al. (2017) . It deserves to be addressed in future work, for instance, by considering group-decision-making techniques in the weighting process or by discussing this issue at the strategic management to set clear guidelines to support managers at different hierarchical levels in defining the weights that best reflect the vision of the organisation. Dependencies among aggregated indicators might also occur. As an example, in psychosocial services, waiting time and customer service time evolve in opposite directions as an improvement in one indicator decreases the other and vice versa (i.e. when a social worker spends more time with his/her current customers, new customers assigned to him/her will wait longer before they can be met) ( Boukherroub et al. , 2022 ). Therefore, methods that address this issue are also required. Typically, the ANP technique, which is a generalisation of AHP, is appropriate for this situation. BWM (Best Worst Method) ( Singh & Singh, 2020 ) and MACBETH (Measuring Attractiveness by a Categorical Based Evaluation Technique) ( Bana e Costa & Vansnick, 1994 ) are other possibilities. This study has some limitations. Regarding data collection, more managers and employees coming from different services and hierarchical levels should have been included in the interview process. Due to the COVID-19 pandemic, it was not possible to include more people. Strategy maps should be built for all programs of the CCSMTL prior to indicator selection, validation, information aggregation, etc. to align the measured performance with strategic objectives. Brainstorming workshops should be carried out with managers and coordinators to establish more precisely their objectives and causal links between these objectives. More work is also required to design additional indicators. In particular, means of measuring and monitoring indicators over time need to be developed. Since AHP does not take into consideration potential interdependences between indicators, ANP, BWM, and MACBETH techniques could be explored. Undesired compensation effects might also occur in the aggregation process. Finally, it is not recommended to limit the use of performance indexes alone for monitoring performance. Operational indicators related for instance to emergency services are very important to monitor at the highest management level. In sum, more research is needed to refine our proposed approach. Conclusions and research perspectives The CCSMTL has developed a multi-dimensional performance model (Quality-Performance Model - QPM), however, its implementation across the organisation is very challenging. This study proposes an approach that contributes to improving this model and its effective deployment. First, our proposed strategy map supports managers in identifying indicators that are aligned with their strategic objectives and vision. Second, the proposed PMMS helps them focus on all four dimensions of the QPM by explicitly linking performance indicators to each dimension. This, notably, supports proposing performance indicators within the customer dimension, which contributes to promoting customer-focussed culture. This would also allow departments to concretely measure their performance and compare themselves in a way that fosters competitiveness and anchors performance-oriented practices in the organisation. Third, the aggregated performance measures (performance indexes) that we propose to calculate based on AHP method will help managers at higher hierarchy levels to have a better visibility and understanding of the performance of operational teams, and make better decisions to support their improvement. Finally, our approach would support standardising performance management practices within the organisation. Our work contributes to the theory and practice by proposing a performance management and measurement approach adapted to large healthcare organisations, which is has been reported in practice and in the literature as a very challenging problem ( Moisan, 2019 ). The following is a testimonial from the deputy director of organisational performance at the CCSMTL: “As part of this project, several working sessions were held with stakeholders [CCSMTL managers]. These structured interviews enabled the research team to well capture the QPM and then develop their approach accordingly. The data [performance indicators] organisation structure is very relevant in the sense that it addresses both the dimensions and the sub-dimensions of the QPM, and it is declined according to the hierarchical administrative structure of the CCSMTL, thus matching with our approach of continuous improvement, which aims to dynamise the information cascade-escalation process through all layers of the organisation. Finally, the rigorous process of validating and weighting the indicators was carried out with the stakeholders, thus ensuring better reliability of the indicators developed, which will optimise their use to eventually support decision-making.” Data availability No data are associated with this article. Acknowledgments The authors acknowledge the financial support of MITACS Globalink program, ÉTS, CCSMTL, and Natural Sciences and Engineering Research Council of Canada (NSERC grant 2018-04555). The authors are grateful to Mr. Jacques Couillard, Mr. Jacques Drôlet, Mrs. Nancy Brunet, and all CCSMTL managers involved in this project. References Agrément-Canada: Guide sur l’évaluation de l’amélioration de la qualités les indicateurs de qualité.2014. Alharbi F, Atkins A, Stanier C, et al. : Strategic value of cloud computing in healthcare organisations using the Balanced Scorecard approach: a case study from a Saudi hospital. Procedia Computer Science. 2016; 98 : 332–339. Publisher Full Text Amer F, Hammoud S, Khatatbeh H, et al. : A Systematic Review: The Dimensions Utilized in the Performance Evaluation of Healthcare- An Implication during the COVID-19 Pandemic.2021. 2021.06.26.21259568. Anjomshoae A, Hassan A, Wong KY: An integrated AHP-based scheme for performance measurement in humanitarian supply chains. Int. J. Product. Perform. Manag. 2019; 68 : 938–957. Publisher Full Text Backman C, Vanderloo S, Forster J, et al. : Measuring and improving quality in university hospitals in Canada: The Collaborative for Excellence in Healthcare Quality. Health Policy. 2016; 120 (9): 982‑986. PubMed Abstract | Publisher Full Text Bana e Costa CA, Vansnick J-C: MACBETH — An Interactive Path towards the Construction of Cardinal Value Functions. Int. Trans. Oper. Res. 1994; 1 (4): 489–500. Publisher Full Text Barnabè F, Hoque Z, Busco C: The causal relationships between performance drivers and outcomes. J. Account. Organ. Chang. 2012; 8 (4): 528–538. Publisher Full Text Béland S, Abran A: Utilisation d’un « balanced scoreboard » pour l’intégration des mesures associées à un programme d’assurance qualité.2004; 14 . Ben Fradj A, Boukherroub T, Olivier C: A framework for developing a performance measurement system in healthcare: a case study in montreal présentée à the 13th International Conference on Modeling, Optimisation and Simulation - MOSIM’20. Agadir – Morocco. 2020. Bentes AV, Carneiro J, da Silva JF , et al. : Multidimensional Assessment of Organizational Performance: Integrating BSC and AHP. J. Bus. Res. 2012; 65 (12): 1790–1799. Publisher Full Text Betto F, Sardi A, Garengo P, et al. : The Evolution of Balanced Scorecard in Healthcare: A Systematic Review of Its Design, Implementation, Use, and Review. Int. J. Environ. Res. Public Health. 2022; 19 (16): 10291. PubMed Abstract | Publisher Full Text | Free Full Text Bititci U, Carrie A, McDevitt L: Integrated performance measurement systems: A development guide. Int. J. Oper. Prod. Manage. 1997; 17 : 522–534. Publisher Full Text Bohm V, Lacaille D, Spencer N, et al. : Scoping Review of Balanced Scorecards for Use in Healthcare Settings: Development and Implementation. BMJ Open Quality. 2021; 10 (3): e001293. PubMed Abstract | Publisher Full Text | Free Full Text Botton C, Jobin M-H, Haithem N: Performance management : the conditions for success. Gestion. 2000. 2012; 29 (2): 37–52. Publisher Full Text Boukherroub T, LeBel L, Ruiz A: A Framework for Sustainable Forest Resource Allocation: A Canadian Case Study. Omega. 2017; 66 (PB): 224–235. Publisher Full Text Boukherroub T, Ouellet L, Lemay G, et al. : Improving Access to Frontline Psychosocial Services for Youths in Difficulty by Using Lean Approach: an action research case study. Int. J. Lean Six Sigma. 2022; 13 (4): 937–958. Breton M, Smithman MA, Brousselle A, et al. : Assessing the performance of centralized waiting lists for patients without a regular family physician using clinical-administrative data. BMC Fam. Pract. 2017; 18 (1): 1–13. PubMed Abstract | Publisher Full Text | Free Full Text Brown MG: Keeping Score: Using the Right Metrics to Drive World-Class Performance. Nachdr. New York: Productivity Press; 2008. Brunelli M: Introduction to the analytic hierarchy process. Springer; 2014. Bull C, Callander E: Current PROM and PREM use in health system performance measurement: Still a way to go. Patient Exp. J. 2022; 9 (1): 12–18. Publisher Full Text Chan Y‐CL: An analytic hierarchy framework for evaluating balanced scorecards of healthcare organizations. Canadian Journal of Administrative Sciences/Revue Canadienne des Sciences de l’Administration. 2006; 23 (2): 85–104. Publisher Full Text Choffel D, Meyssonnier F: Dix ans de débats autour du Balanced Scorecard. Comptabilité-Contrôle-Audit. 2005; Tome 11 (2): 61–81. Publisher Full Text CIUSSS: Annual Report. « CIUSSS du Centre-Sud-de-l’Île-de-Montréal ». Consulté 11 novembre 2021.2020. Reference Source Cosgrove TL: Planetree health information services: public access to the health information people want. Bull. Med. Libr. Assoc. 1994; 82 (1): 57–63. PubMed Abstract Couillard J: Vers une culture qualité performance centrée sur le client. colloque CvPAC. 2018; 39 . CCSMTL: 2016. Reference Source Dekrita YA, Yunus R, Citta AB, et al. : Integration of Balanced Scorecard and Analytical Hierarchy Process as a Tool for Determining the Priority of the Program Strategy: Case Study in Dr. Tc. Hillers Maumere Hospital. 3rd International Conference on Accounting, Management and Economics 2018 (ICAME 2018). Atlantis Press; 2019. DiCicco‐Bloom B, Crabtree BF: The qualitative research interview. Med. Educ. 2006; 40 (4): 314–321. Publisher Full Text Daneshmand E, Sedigheh TM, Mehdizadeh A, et al. : Developing A Model for Measuring the Performance of the Health System Based on the Balanced Scorecard BSC Method in Social Security Organization. IAU Int. J. Soc. Sci. 2022; 12 (2). Publisher Full Text Fache P, Sicotte C, Minvielle É:The Search is on for Coherent Performance Measurement in Healthcare Organizations. Has Quebec Reached a Crossroads? Healthc. Policy. 2016; 11 (4):60–69. Ferreira A, Otley D:The design and use of performance management systems: An extended framework for analysis. Manag. Account. Res. 2009; 20 (4):263–282. Publisher Full Text Gao H, Chen H, Feng J, et al. : Balanced scorecard-based performance evaluation of Chinese county hospitals in underdeveloped areas. J. Int. Med. Res. 2018; 46 (5): 1947–1962. PubMed Abstract | Publisher Full Text | Free Full Text Ghosh R, Singh S: A System Dynamics Approach to Strategic Planning and Performance in Healthcare: Dynamic Balanced Scorecard Leveraged with Corporate Social Responsibility. The Management Accountant Journal. 2021; 56 (2): 32–36. Publisher Full Text Hossain M, Guest R, Smith C: Performance Indicators of Public Private Partnership in Bangladesh: An Implication for Developing Countries. Int. J. Product. Perform. Manag. 2019; 68 (1): 46–68. Publisher Full Text Hummel M, IJzerman M: The past and future of the AHP in health care decision making.2011. HSO: L’Organisation de normes en santé (HSO) - Normes et évaluations en matière de soins de santé. L’Organisation de normes en santé (HSO). Consulté 11 novembre 2021. 2021. Reference Source Ishizaka A, Labib A: Review of the Main Developments in the Analytic Hierarchy Process. Expert Syst. Appl. 2011; 38 (11): 14336–14345. Publisher Full Text Ittner CD, Larcker DF: Coming up short on nonfinancial performance measurement. Harv. Bus. Rev. 2003; 81 (11): 88–95, 139. PubMed Abstract Kairu EW, Wafula MO, Okaka O, et al. : Effects of Balanced Scorecard on Performance of Firms in the Service Sector. European Journal of Business and Management. 2013; 9 . Kaplan RS: Using the Balanced Scorecard for Successful Health Care M&A Integration.2020. Kaplan RS, Norton DP: The Balanced Scorecard œ Measures That Drive Performance—, in: Harvard Business Review, January-February 1992. Putting the balanced scorecard to work. 1992. Kaplan RS, Norton DP: Using the balanced scorecard as a strategic management system.1996. Kaplan RS, Norton DP: The balanced scorecard for public-sector organizations. Balanced Scorecard Report. 1999; 15 (11): 1999. Khalid S, Beattie C, Sands J, et al. : Incorporating the environmental dimension into the balanced scorecard. Meditari Accountancy Research. 2019. Kumar A, Sah B, Singh AR, et al. : A review of multi criteria decision making (MCDM) towards sustainable renewable energy development. Renew. Sust. Energ. Rev. 2017; 69 : 596–609. Publisher Full Text Lagacé D, Landry S: Salles de pilotage: un nouveau mode de gestion de la performance. Gestion. 2016; 41 (3): 90–93. Publisher Full Text Lilian Chan Y‐C: How strategy map works for Ontario’s health system. Int. J. Public Sect. Manag. 2009; 22 (4): 349–363. Publisher Full Text Lorino P: Le balanced Scorecard Revisite: dynamique stratégique et pilotage de performance exemple d’une entreprise énergétique.2001. Modak M, Ghosh KK, Pathak K: A BSC-ANP approach to organizational outsourcing decision support-A case study. J. Bus. Res. 2019; 103 : 432–447. Publisher Full Text Moisan L: Système intégré de gestion de la performance : catalyseur d’intégration horizontale et de gestion du changement au CISSS de la Gaspésie. phd, Université du Québec à Trois-Rivières; Université de Sherbrooke, Trois-Rivières; Sherbrooke. 2019. Montalan M-A, Vincent B: Élaboration d’un balanced scorecard en milieu hospitalier. Rev. Fr. Gest. 2011; 37 : 93–102. Publisher Full Text Moullin M: Improving and evaluating performance with the Public Sector Scorecard. Int. J. Product. Perform. Manag. 2017; 66 : 442–458. Publisher Full Text MSSS: Cadre de référence ministériel d’évaluation de la performance du système public de santé et de services sociaux à des fins de gestion.2012. MSSS: Cadre de référence - salle de pilotage stratégique.2015. MSSS: Plan stratégique 2019-2023 du ministère de la Santé et des Services sociaux.2023. Neely AD, Adams C, Kennerley M: The performance prism: The scorecard for measuring and managing business success. Prentice Hall Financial Times London; 2002. Nippak PMD, Veracion JI, Muia M, et al. : Designing and evaluating a balanced scorecard for a health information management department in a Canadian urban non-teaching hospital. Health Informatics J. 2016; 22 (2): 120–139. Publisher Full Text Norreklit H: The balance on the balanced scorecard a critical analysis of some of its assumptions. Manag. Account. Res. 2000; 11 (1): 65–88. Publisher Full Text Nuti S, De Rosis S, Bonciani M, et al. : Rethinking Healthcare Performance Evaluation Systems towards the People-Centredness Approach: Their Pathways, their Experience, their Evaluation. HealthcarePapers. 2017; 17 (2): 56–64. PubMed Abstract | Publisher Full Text Okwo IM, Mary MI: Performance Measurement in Business Organizations: An Empirical Analysis of the Financial Performance of Some Breweries in Nigeria. Res. J. Financ. Account. 2012. PHAM CD, Sen Thi VU, PHAM YTK, et al. : Evaluating Performance of Vietnamese Public Hospitals Based on Balanced Scorecard. The Journal of Asian Finance, Economics, and Business. 2020; 7 (6): 339–349. Publisher Full Text Pickton DW, Wright S: What's swot in strategic analysis? Strateg. Chang. 1998; 7 (2): 101–109. Publisher Full Text Porporato M, Tsasis P, Vinuesa LMM: Do hospital balanced scorecard measures reflect cause-effect relationships? Int. J. Product. Perform. Manag. 2017; 66 : 338–361. Publisher Full Text Portulhak H, Martins DB, Maria M, et al. : Business performance management in university hospitals: a diagnosis in Brazilian institutions. Revista de Salud Pública. 2017; 19 : 697–703. PubMed Abstract | Publisher Full Text Regragui H, Sefiani N, Azzouzi H: Improving performance through measurement: the application of BSC and AHP in healthcare organization. 2018 International Colloquium on Logistics and Supply Chain Management (LOGISTIQUA). IEEE; 2018; pp. 51‑56. Ritchie WJ, Ni J, Stark EM, et al. : The Effectiveness of ISO 9001-Based Healthcare Accreditation Surveyors and Standards on Hospital Performance Outcomes: A Balanced Scorecard Perspective. Qual. Manag. J. 2019; 26 (4): 162–173. Publisher Full Text Saaty TL: Theory and Applications of the Analytic Network Process: Decision Making With Benefits, Opportunities, Costs, and Risks. 3e édition.Pittsburgh: Rws Pubns; 2005. Scholey C: Strategy maps: a step‐by‐step guide to measuring, managing and communicating the plan. J. Bus. Strateg. 2005; 26 : 12–19. Publisher Full Text Schrage M, Kiron D: Leading with next-generation key performance indicators. MIT Sloan Manag. Rev. 2018; 16 . Simons R: How managers use innovative control systems to drive strategic renewal. Strateg. Manag. J. 1994; 15 (3): 169–189. Publisher Full Text Singh D, Singh B: Investigating the impact of data normalization on classification performance. Appl. Soft Comput. 2020; 97 : 105524. Publisher Full Text Solomon R, Damba C, Bryant S:Measuring quality at a system level: an impossible task? The Toronto Central LHIN Experience. Healthc. Q. 2013; 16 (4):36–42. PubMed Abstract | Publisher Full Text Soysa IB, Jayamaha NP, Grigg NP: Validating the balanced scorecard framework for nonprofit organisations: an empirical study involving Australasian healthcare. Total Qual. Manag. Bus. Excell. 2019; 30 (9‑10): 1005–1025. Publisher Full Text Teklehaimanot HD, Teklehaimanot A, Tedella AA, et al. : Use of balanced scorecard methodology for performance measurement of the Health Extension Program in Ethiopia. Am. J. Trop. Med. Hyg. 2016; 94 (5): 1157–1169. PubMed Abstract | Publisher Full Text | Free Full Text Treacy M, Wiersema F: The discipline of market leaders: Choose your customers, narrow your focus, dominate your market. Basic Books; 2007. Voyer P, Voyer P: Tableaux de bord de gestion et indicateurs de performance. 2e éd.Sainte-Foy, Québec: Presses de l’Université du Québec; 1999. Yaghoobi T, Haddadi F: Organizational performance measurement by a framework integrating BSC and AHP. Int. J. Product. Perform. Manag. 2016; 65 : 959–976. Publisher Full Text Comments on this article Comments (0) Version 2 VERSION 2 PUBLISHED 30 Oct 2023 ADD YOUR COMMENT Comment Author details Author details 1 Numérix Laboratory, École de technologie superieure, Montreal, Québec, Canada 2 Systems Engineering Department, École de technologie superieure, Montreal, Québec, Canada 3 Interuniversity Research Centre on Enterprise Networks, Logistics and Transportation, Montreal, Québec, Canada 4 Industrial Engineering Department, Université de Quebec à Trois Rivieres, Quebec, Canada Anes Ben Fradj Roles: Conceptualization, Formal Analysis, Writing – Original Draft Preparation Neila El Asli Roles: Formal Analysis, Validation, Writing – Original Draft Preparation, Writing – Review & Editing Tasseda Boukherroub Roles: Supervision, Writing – Review & Editing Claude Olivier Roles: Supervision Competing interests No competing interests were disclosed. Grant information This work is supported by the MITACS Globalink program, École de technologie supérieure (ÉTS), CIUSSS du-Centre-Sud-de-l'île-de-Montréal, and Natural Sciences and Engineering Research Council of Canada (NSERC) grant 2018-04555. The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript. Article Versions (2) version 2 Revised Published: 09 Dec 2024, 12:1420 https://doi.org/10.12688/f1000research.138430.2 version 1 Published: 30 Oct 2023, 12:1420 https://doi.org/10.12688/f1000research.138430.1 Copyright © 2024 Ben Fradj A et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. Download Export To Sciwheel Bibtex EndNote ProCite Ref. Manager (RIS) Sente metrics Views Downloads F1000Research - - PubMed Central info_outline Data from PMC are received and updated monthly. - - Citations open_in_new 0 open_in_new 0 open_in_new SEE MORE DETAILS CITE how to cite this article Ben Fradj A, El Asli N, Boukherroub T and Olivier C. Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.12688/f1000research.138430.2 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS track receive updates on this article Track an article to receive email alerts on any updates to this article. TRACK THIS ARTICLE Share Open Peer Review Current Reviewer Status: ? Key to Reviewer Statuses VIEW HIDE Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Version 2 VERSION 2 PUBLISHED 09 Dec 2024 Revised Views 0 Cite How to cite this report: Gartner JB. Reviewer Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.169268.r347038 ) The direct URL for this report is: https://f1000research.com/articles/12-1420/v2#referee-response-347038 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 23 Dec 2024 Jean-Baptiste Gartner , Universite Laval, Québec City, Québec, Canada Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.169268.r347038 This study aims to develop a performance management and measurement system for a large healthcare organisation (integrated (University) centres of healthcare and social services) in Montreal, Quebec, Canada. Overall, the revisions have greatly improved the document. However, ... Continue reading READ ALL This study aims to develop a performance management and measurement system for a large healthcare organisation (integrated (University) centres of healthcare and social services) in Montreal, Quebec, Canada. Overall, the revisions have greatly improved the document. However, there are still some points that need to be improved before it can be accepted. INTRODUCTI ON/LITERATURE REVIEW - The background and the context are clearly stated. - "Even though recent studies in the literature discuss performance management systems in healthcare organisations and recognise both the importance and the complexity of measuring performance, most of them are descriptive and do not propose PMMSs capable of effectively structuring and measuring the performance of healthcare organisations. " This statement must be supported by at least one reference. Otherwise, it appears to be a personal opinion. - "The QPM (see Figure 1 ) is based on the MSSS strategic plan (2019-2023 period)" This statement does not work (the legal deposit of the document of the MSSS dates from after the article was published), remove or explain. LITTERATURE REVIEW This part has been suitably enhanced. METHODS The addition of the table is appropriate. The addition of details of the method is welcome. RESULTS Given the number of data sources (interview, observation, focus group), it is surprising to have been left with a single case study description. With this in mind, it is unfortunate that assertions are not supported by your empirical material, for example : - "From Table 4 , we can see that one of the strengths identified is the well implementation of the QPM at the strategic level". (This is an assertion, but the reader has no data to corroborate the assertion (interview excerpts, focus groups, changes in a quantitative/qualitative indicator, etc.). If the intention is to be descriptive only, this can remain unchanged. DISCUSSION The discussion still does not contextualise the results with existing knowledge (present in your Literature review paragraph). Contribution of your article ? - Future work: Ok, your opening is relevant. Some typos and errors : Introduction: - " In the province of Qu é bec " (Quebec) - " which fall under the responsibility of the Ministry of Healthcare and Social Services (Mi m istère de la Santé et des Services Sociaux". Competing Interests: No competing interests were disclosed. Reviewer Expertise: Healthcare management research I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Gartner JB. Reviewer Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.169268.r347038 ) The direct URL for this report is: https://f1000research.com/articles/12-1420/v2#referee-response-347038 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Version 1 VERSION 1 PUBLISHED 30 Oct 2023 Views 0 Cite How to cite this report: Forget P. Reviewer Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.151623.r219887 ) The direct URL for this report is: https://f1000research.com/articles/12-1420/v1#referee-response-219887 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 16 Sep 2024 Pascal Forget , Université du Québec à Trois-Rivières, Trois-Rivières, Quebec, Canada Approved VIEWS 0 https://doi.org/10.5256/f1000research.151623.r219887 This article proposes an approach to facilitate the implementation of a performance management model in a government healthcare system. This article is really interesting and offer a pertinent contribution. It is clear and well written. The introduction, the literature, ... Continue reading READ ALL This article proposes an approach to facilitate the implementation of a performance management model in a government healthcare system. This article is really interesting and offer a pertinent contribution. It is clear and well written. The introduction, the literature, the method and discussion are clear and well presented. There is no specific point to address and the paper could be index as it is. Is the background of the case’s history and progression described in sufficient detail? Yes Is the work clearly and accurately presented and does it cite the current literature? Yes If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Yes Competing Interests: No competing interests were disclosed. Reviewer Expertise: Industrial engineering, operation research, simulation I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Forget P. Reviewer Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.151623.r219887 ) The direct URL for this report is: https://f1000research.com/articles/12-1420/v1#referee-response-219887 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Silva P. Reviewer Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.151623.r238355 ) The direct URL for this report is: https://f1000research.com/articles/12-1420/v1#referee-response-238355 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 04 Apr 2024 Paulino Silva , ISCAP Polytechnic of Porto, Porto, Porto District, Portugal Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.151623.r238355 The article presents a very relevant topic with great added value, both for the organizational performance management literature and for management practitioners. The contribution of the article can be relevant for the body of knowledge. However, there are some methodological ... Continue reading READ ALL The article presents a very relevant topic with great added value, both for the organizational performance management literature and for management practitioners. The contribution of the article can be relevant for the body of knowledge. However, there are some methodological weaknesses that should be considered. The article claims to develop an integrated performance management system for healthcare organizations, but perhaps it would be more appropriate to define it as an improvement to the existing system. This is because it makes a diagnosis of the existing system and, through SWOT analysis, presents the main strengths, weaknesses, opportunities, and threats. The used case study seems too specific (Physical disability, intellectual disability, and autism spectrum disorder" of CIUSSS du Centre-Sud-de-l'île-de-Montreal, in the province of Québec, Canada) to be considered a reference of a performance management system in healthcare organizations. As the study began in 2019, and the interviews between May 2019 and September 2021, could the results have been too affected by the Covid-19 pandemic? The number of interviews seems to be very low (only 12) and the variety of jobs very limited (only managers). Consider including additional literature specifically related to performance management in healthcare organizations, such as : [1] Some “typos” in the article: The first paragraph of the methods section is incorrect, it should end with phases III and IV. The abbreviation for Ministry of Healthcare and Social Services presented in the article is MSSS and not MHSS, why? Customer-focused instead of customer-focussed in line 6 of the section Conclusions and research perspectives. Is the background of the case’s history and progression described in sufficient detail? Yes Is the work clearly and accurately presented and does it cite the current literature? Partly If applicable, is the statistical analysis and its interpretation appropriate? I cannot comment. A qualified statistician is required. Are all the source data underlying the results available to ensure full reproducibility? Partly Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Yes References 1. Silva P, Ferreira A: Performance management in primary healthcare services: evidence from a field study. Qualitative Research in Accounting & Management . 2010; 7 (4): 424-449 Publisher Full Text Competing Interests: No competing interests were disclosed. Reviewer Expertise: Management Accounting I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Silva P. Reviewer Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.151623.r238355 ) The direct URL for this report is: https://f1000research.com/articles/12-1420/v1#referee-response-238355 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 09 Dec 2024 NEILA EL ASLI , Systems Engineering Department, École de technologie superieure, Montreal, Canada 09 Dec 2024 Author Response Reviewer #3: APPROVED WITH RESERVATIONS The article presents a very relevant topic with great added value, both for the organizational performance management literature and for management practitioners. The contribution of ... Continue reading Reviewer #3: APPROVED WITH RESERVATIONS The article presents a very relevant topic with great added value, both for the organizational performance management literature and for management practitioners. The contribution of the article can be relevant for the body of knowledge. However, there are some methodological weaknesses that should be considered. The article claims to develop an integrated performance management system for healthcare organizations, but perhaps it would be more appropriate to define it as an improvement to the existing system. This is because it makes a diagnosis of the existing system and, through SWOT analysis, presents the main strengths, weaknesses, opportunities, and threats. You are right. The PMMS (Performance Management and Measurement System) we proposed is an improved Quality-Performance Model (QPM) in the context of the CCSMTL. However, in the absence of a performance model, our approach (or at least part of it if we remove the SWOT analysis), could be used to develop a PMMS. We clarified this in the text in the new version of the manuscript at different levels: We changed the title from “Developing an integrated ….” to “ Towrads an integarted …”. In the abstract, the first sentence is now: “This study proposes an approach for developing or improving performance management and measurement systems (PMMSs) for healthcare organisations.”. In the Introduction (last paragraph): “…it proposes a four-phases based approach to support CCSMTL managers to improve and transform the QPM into a more balanced and effective performance management and measurement system (PMMS) . … It can be used by theoreticians and practitioners as roadmap to develop or improve healthcare organisations’ PMMSs. In the Discussion section: ““In this study, we have proposed a comprehensive and quantitative approach to develop or improve a PMMS.” In the Conclusion section: “The CCSMTL has developed a multi-dimensional performance model (Quality-Performance Model - QPM), however, its implementation across the organisation is very challenging. This study proposes an approach that contributes to improving this model and its effective deployment .” The used case study seems too specific (Physical disability, intellectual disability, and autism spectrum disorder" of CIUSSS du Centre-Sud-de-l'île-de-Montreal, in the province of Québec, Canada) to be considered a reference of a performance management system in healthcare organizations. There are two levels of application of our PMMS. We applied Phases I, II, III, and partially IV to the entire CIUSSS, and then Phases III and IV to the program Physical disability, intellectual disability, and autism spectrum disorder . We have made changes in the new version of the manuscript to clarify this (please see the Results section). We agree that more work is needed before we can consider this system as a reference in healthcare organizations. We have acknowledged this limitation in the discussion Section, and as mentioned before, we also have changed the title. As the study began in 2019, and the interviews between May 2019 and September 2021, could the results have been too affected by the Covid-19 pandemic? This is right, part of the data collection took place during the COVID-19 pandemic (2020). During this period, the process slowed down; it was difficult to reach people. This resulted in a lesser number of interviews than what we targeted initially. However, it is difficult to estimate the impact on the results though. The number of interviews seems to be very low (only 12) and the variety of jobs very limited (only managers). As mentioned above, initially we targeted more interviews. Even though all the participants involved are managers, there is a certain variety considering that they don’t have the same function, hierarchy level in the organization, and they don’t come from the same programs/services (please see Table 3 in the new version of the manuscript). Employees were also met during the data collection phase. However, their knowledge level regarding performance measurement, notably in relation to the QPM, was too low or inexistant (in some services, there are no performance indicators put in place at all). This lack of knowledge at the operational level has been mentioned as a weakness in the SWOT analysis “The QPM is known at the tactical and operational levels, but not used for performance management and measurement purposes”. We appreciate your comment. We have discussed this as a limitation in the new version of the manuscript. Consider including additional literature specifically related to performance management in healthcare organizations, such as : [1] 1. Silva P, Ferreira A: Performance management in primary healthcare services: evidence from a field study. Qualitative Research in Accounting & Management. 2010; 7 (4): 424-449 Publisher Full Text Thanks for this suggestion. We have included it in our literature review. We have also added three other references suggested by Reviewer #1. Some “typos” in the article: The first paragraph of the methods section is incorrect, it should end with phases III and IV. Y ou are right. It has been corrected. The abbreviation for Ministry of Healthcare and Social Services presented in the article is MSSS and not MHSS, why? MSSS stands for Ministère de la Santé et des Services Sociaux in French (official title of the Ministry). We will have added this information in the new version of the manuscript to avoid confusion. Customer-focused instead of customer-focussed in line 6 of the section Conclusions and research perspectives. Thanks, we have corrected it. Reviewer #3: APPROVED WITH RESERVATIONS The article presents a very relevant topic with great added value, both for the organizational performance management literature and for management practitioners. The contribution of the article can be relevant for the body of knowledge. However, there are some methodological weaknesses that should be considered. The article claims to develop an integrated performance management system for healthcare organizations, but perhaps it would be more appropriate to define it as an improvement to the existing system. This is because it makes a diagnosis of the existing system and, through SWOT analysis, presents the main strengths, weaknesses, opportunities, and threats. You are right. The PMMS (Performance Management and Measurement System) we proposed is an improved Quality-Performance Model (QPM) in the context of the CCSMTL. However, in the absence of a performance model, our approach (or at least part of it if we remove the SWOT analysis), could be used to develop a PMMS. We clarified this in the text in the new version of the manuscript at different levels: We changed the title from “Developing an integrated ….” to “ Towrads an integarted …”. In the abstract, the first sentence is now: “This study proposes an approach for developing or improving performance management and measurement systems (PMMSs) for healthcare organisations.”. In the Introduction (last paragraph): “…it proposes a four-phases based approach to support CCSMTL managers to improve and transform the QPM into a more balanced and effective performance management and measurement system (PMMS) . … It can be used by theoreticians and practitioners as roadmap to develop or improve healthcare organisations’ PMMSs. In the Discussion section: ““In this study, we have proposed a comprehensive and quantitative approach to develop or improve a PMMS.” In the Conclusion section: “The CCSMTL has developed a multi-dimensional performance model (Quality-Performance Model - QPM), however, its implementation across the organisation is very challenging. This study proposes an approach that contributes to improving this model and its effective deployment .” The used case study seems too specific (Physical disability, intellectual disability, and autism spectrum disorder" of CIUSSS du Centre-Sud-de-l'île-de-Montreal, in the province of Québec, Canada) to be considered a reference of a performance management system in healthcare organizations. There are two levels of application of our PMMS. We applied Phases I, II, III, and partially IV to the entire CIUSSS, and then Phases III and IV to the program Physical disability, intellectual disability, and autism spectrum disorder . We have made changes in the new version of the manuscript to clarify this (please see the Results section). We agree that more work is needed before we can consider this system as a reference in healthcare organizations. We have acknowledged this limitation in the discussion Section, and as mentioned before, we also have changed the title. As the study began in 2019, and the interviews between May 2019 and September 2021, could the results have been too affected by the Covid-19 pandemic? This is right, part of the data collection took place during the COVID-19 pandemic (2020). During this period, the process slowed down; it was difficult to reach people. This resulted in a lesser number of interviews than what we targeted initially. However, it is difficult to estimate the impact on the results though. The number of interviews seems to be very low (only 12) and the variety of jobs very limited (only managers). As mentioned above, initially we targeted more interviews. Even though all the participants involved are managers, there is a certain variety considering that they don’t have the same function, hierarchy level in the organization, and they don’t come from the same programs/services (please see Table 3 in the new version of the manuscript). Employees were also met during the data collection phase. However, their knowledge level regarding performance measurement, notably in relation to the QPM, was too low or inexistant (in some services, there are no performance indicators put in place at all). This lack of knowledge at the operational level has been mentioned as a weakness in the SWOT analysis “The QPM is known at the tactical and operational levels, but not used for performance management and measurement purposes”. We appreciate your comment. We have discussed this as a limitation in the new version of the manuscript. Consider including additional literature specifically related to performance management in healthcare organizations, such as : [1] 1. Silva P, Ferreira A: Performance management in primary healthcare services: evidence from a field study. Qualitative Research in Accounting & Management. 2010; 7 (4): 424-449 Publisher Full Text Thanks for this suggestion. We have included it in our literature review. We have also added three other references suggested by Reviewer #1. Some “typos” in the article: The first paragraph of the methods section is incorrect, it should end with phases III and IV. Y ou are right. It has been corrected. The abbreviation for Ministry of Healthcare and Social Services presented in the article is MSSS and not MHSS, why? MSSS stands for Ministère de la Santé et des Services Sociaux in French (official title of the Ministry). We will have added this information in the new version of the manuscript to avoid confusion. Customer-focused instead of customer-focussed in line 6 of the section Conclusions and research perspectives. Thanks, we have corrected it. Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 09 Dec 2024 NEILA EL ASLI , Systems Engineering Department, École de technologie superieure, Montreal, Canada 09 Dec 2024 Author Response Reviewer #3: APPROVED WITH RESERVATIONS The article presents a very relevant topic with great added value, both for the organizational performance management literature and for management practitioners. The contribution of ... Continue reading Reviewer #3: APPROVED WITH RESERVATIONS The article presents a very relevant topic with great added value, both for the organizational performance management literature and for management practitioners. The contribution of the article can be relevant for the body of knowledge. However, there are some methodological weaknesses that should be considered. The article claims to develop an integrated performance management system for healthcare organizations, but perhaps it would be more appropriate to define it as an improvement to the existing system. This is because it makes a diagnosis of the existing system and, through SWOT analysis, presents the main strengths, weaknesses, opportunities, and threats. You are right. The PMMS (Performance Management and Measurement System) we proposed is an improved Quality-Performance Model (QPM) in the context of the CCSMTL. However, in the absence of a performance model, our approach (or at least part of it if we remove the SWOT analysis), could be used to develop a PMMS. We clarified this in the text in the new version of the manuscript at different levels: We changed the title from “Developing an integrated ….” to “ Towrads an integarted …”. In the abstract, the first sentence is now: “This study proposes an approach for developing or improving performance management and measurement systems (PMMSs) for healthcare organisations.”. In the Introduction (last paragraph): “…it proposes a four-phases based approach to support CCSMTL managers to improve and transform the QPM into a more balanced and effective performance management and measurement system (PMMS) . … It can be used by theoreticians and practitioners as roadmap to develop or improve healthcare organisations’ PMMSs. In the Discussion section: ““In this study, we have proposed a comprehensive and quantitative approach to develop or improve a PMMS.” In the Conclusion section: “The CCSMTL has developed a multi-dimensional performance model (Quality-Performance Model - QPM), however, its implementation across the organisation is very challenging. This study proposes an approach that contributes to improving this model and its effective deployment .” The used case study seems too specific (Physical disability, intellectual disability, and autism spectrum disorder" of CIUSSS du Centre-Sud-de-l'île-de-Montreal, in the province of Québec, Canada) to be considered a reference of a performance management system in healthcare organizations. There are two levels of application of our PMMS. We applied Phases I, II, III, and partially IV to the entire CIUSSS, and then Phases III and IV to the program Physical disability, intellectual disability, and autism spectrum disorder . We have made changes in the new version of the manuscript to clarify this (please see the Results section). We agree that more work is needed before we can consider this system as a reference in healthcare organizations. We have acknowledged this limitation in the discussion Section, and as mentioned before, we also have changed the title. As the study began in 2019, and the interviews between May 2019 and September 2021, could the results have been too affected by the Covid-19 pandemic? This is right, part of the data collection took place during the COVID-19 pandemic (2020). During this period, the process slowed down; it was difficult to reach people. This resulted in a lesser number of interviews than what we targeted initially. However, it is difficult to estimate the impact on the results though. The number of interviews seems to be very low (only 12) and the variety of jobs very limited (only managers). As mentioned above, initially we targeted more interviews. Even though all the participants involved are managers, there is a certain variety considering that they don’t have the same function, hierarchy level in the organization, and they don’t come from the same programs/services (please see Table 3 in the new version of the manuscript). Employees were also met during the data collection phase. However, their knowledge level regarding performance measurement, notably in relation to the QPM, was too low or inexistant (in some services, there are no performance indicators put in place at all). This lack of knowledge at the operational level has been mentioned as a weakness in the SWOT analysis “The QPM is known at the tactical and operational levels, but not used for performance management and measurement purposes”. We appreciate your comment. We have discussed this as a limitation in the new version of the manuscript. Consider including additional literature specifically related to performance management in healthcare organizations, such as : [1] 1. Silva P, Ferreira A: Performance management in primary healthcare services: evidence from a field study. Qualitative Research in Accounting & Management. 2010; 7 (4): 424-449 Publisher Full Text Thanks for this suggestion. We have included it in our literature review. We have also added three other references suggested by Reviewer #1. Some “typos” in the article: The first paragraph of the methods section is incorrect, it should end with phases III and IV. Y ou are right. It has been corrected. The abbreviation for Ministry of Healthcare and Social Services presented in the article is MSSS and not MHSS, why? MSSS stands for Ministère de la Santé et des Services Sociaux in French (official title of the Ministry). We will have added this information in the new version of the manuscript to avoid confusion. Customer-focused instead of customer-focussed in line 6 of the section Conclusions and research perspectives. Thanks, we have corrected it. Reviewer #3: APPROVED WITH RESERVATIONS The article presents a very relevant topic with great added value, both for the organizational performance management literature and for management practitioners. The contribution of the article can be relevant for the body of knowledge. However, there are some methodological weaknesses that should be considered. The article claims to develop an integrated performance management system for healthcare organizations, but perhaps it would be more appropriate to define it as an improvement to the existing system. This is because it makes a diagnosis of the existing system and, through SWOT analysis, presents the main strengths, weaknesses, opportunities, and threats. You are right. The PMMS (Performance Management and Measurement System) we proposed is an improved Quality-Performance Model (QPM) in the context of the CCSMTL. However, in the absence of a performance model, our approach (or at least part of it if we remove the SWOT analysis), could be used to develop a PMMS. We clarified this in the text in the new version of the manuscript at different levels: We changed the title from “Developing an integrated ….” to “ Towrads an integarted …”. In the abstract, the first sentence is now: “This study proposes an approach for developing or improving performance management and measurement systems (PMMSs) for healthcare organisations.”. In the Introduction (last paragraph): “…it proposes a four-phases based approach to support CCSMTL managers to improve and transform the QPM into a more balanced and effective performance management and measurement system (PMMS) . … It can be used by theoreticians and practitioners as roadmap to develop or improve healthcare organisations’ PMMSs. In the Discussion section: ““In this study, we have proposed a comprehensive and quantitative approach to develop or improve a PMMS.” In the Conclusion section: “The CCSMTL has developed a multi-dimensional performance model (Quality-Performance Model - QPM), however, its implementation across the organisation is very challenging. This study proposes an approach that contributes to improving this model and its effective deployment .” The used case study seems too specific (Physical disability, intellectual disability, and autism spectrum disorder" of CIUSSS du Centre-Sud-de-l'île-de-Montreal, in the province of Québec, Canada) to be considered a reference of a performance management system in healthcare organizations. There are two levels of application of our PMMS. We applied Phases I, II, III, and partially IV to the entire CIUSSS, and then Phases III and IV to the program Physical disability, intellectual disability, and autism spectrum disorder . We have made changes in the new version of the manuscript to clarify this (please see the Results section). We agree that more work is needed before we can consider this system as a reference in healthcare organizations. We have acknowledged this limitation in the discussion Section, and as mentioned before, we also have changed the title. As the study began in 2019, and the interviews between May 2019 and September 2021, could the results have been too affected by the Covid-19 pandemic? This is right, part of the data collection took place during the COVID-19 pandemic (2020). During this period, the process slowed down; it was difficult to reach people. This resulted in a lesser number of interviews than what we targeted initially. However, it is difficult to estimate the impact on the results though. The number of interviews seems to be very low (only 12) and the variety of jobs very limited (only managers). As mentioned above, initially we targeted more interviews. Even though all the participants involved are managers, there is a certain variety considering that they don’t have the same function, hierarchy level in the organization, and they don’t come from the same programs/services (please see Table 3 in the new version of the manuscript). Employees were also met during the data collection phase. However, their knowledge level regarding performance measurement, notably in relation to the QPM, was too low or inexistant (in some services, there are no performance indicators put in place at all). This lack of knowledge at the operational level has been mentioned as a weakness in the SWOT analysis “The QPM is known at the tactical and operational levels, but not used for performance management and measurement purposes”. We appreciate your comment. We have discussed this as a limitation in the new version of the manuscript. Consider including additional literature specifically related to performance management in healthcare organizations, such as : [1] 1. Silva P, Ferreira A: Performance management in primary healthcare services: evidence from a field study. Qualitative Research in Accounting & Management. 2010; 7 (4): 424-449 Publisher Full Text Thanks for this suggestion. We have included it in our literature review. We have also added three other references suggested by Reviewer #1. Some “typos” in the article: The first paragraph of the methods section is incorrect, it should end with phases III and IV. Y ou are right. It has been corrected. The abbreviation for Ministry of Healthcare and Social Services presented in the article is MSSS and not MHSS, why? MSSS stands for Ministère de la Santé et des Services Sociaux in French (official title of the Ministry). We will have added this information in the new version of the manuscript to avoid confusion. Customer-focused instead of customer-focussed in line 6 of the section Conclusions and research perspectives. Thanks, we have corrected it. Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Munar W. Reviewer Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.151623.r253136 ) The direct URL for this report is: https://f1000research.com/articles/12-1420/v1#referee-response-253136 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 28 Mar 2024 Wolfgang Munar , The George Washington University, Washington, District of Columbia, USA Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.151623.r253136 This solid paper contributes to ongoing debates on designing performance measurement and management systems in healthcare organizations. The authors describe the process followed to complete the system's design (i.e., situational analysis, SWOT analysis, and literature review), leading to the development ... Continue reading READ ALL This solid paper contributes to ongoing debates on designing performance measurement and management systems in healthcare organizations. The authors describe the process followed to complete the system's design (i.e., situational analysis, SWOT analysis, and literature review), leading to the development of a performance indicator framework. An indicative application of the resulting design was then applied to the CIUSSS program. The main two areas to consider for improving the paper are described below. 1. While the authors recognize the complexity inherent to PMM system design (and implementation), there is insufficient analysis/discussion of the proposed design's strengths and limitations to address such complexity (e.g., interdependence and recurrence between indicators, gaming, users' cognitive biases, etc.). 2. The authors' use of the concept of "implementation" requires clarification. While the paper clearly presents a prototype for a PMM system, it cannot be said that it describes its implementation. Is the background of the case’s history and progression described in sufficient detail? Yes Is the work clearly and accurately presented and does it cite the current literature? Yes If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly Is the case presented with sufficient detail to be useful for teaching or other practitioners? Yes Competing Interests: No competing interests were disclosed. Reviewer Expertise: Health policy and systems research, and theory-driven evaluation. I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Munar W. Reviewer Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.151623.r253136 ) The direct URL for this report is: https://f1000research.com/articles/12-1420/v1#referee-response-253136 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 09 Dec 2024 NEILA EL ASLI , Systems Engineering Department, École de technologie superieure, Montreal, Canada 09 Dec 2024 Author Response This solid paper contributes to ongoing debates on designing performance measurement and management systems in healthcare organizations. The authors describe the process followed to complete the system's design (i.e., situational ... Continue reading This solid paper contributes to ongoing debates on designing performance measurement and management systems in healthcare organizations. The authors describe the process followed to complete the system's design (i.e., situational analysis, SWOT analysis, and literature review), leading to the development of a performance indicator framework. An indicative application of the resulting design was then applied to the CIUSSS program. The main two areas to consider for improving the paper are described below. 1. While the authors recognize the complexity inherent to PMM system design (and implementation), there is insufficient analysis/discussion of the proposed design's strengths and limitations to address such complexity (e.g., interdependence and recurrence between indicators, gaming, users' cognitive biases, etc.). Author Response: Thanks for this comment. We have improved the discussion section in the new version of the manuscript. 2. The authors' use of the concept of "implementation" requires clarification. While the paper clearly presents a prototype for a PMM system, it cannot be said that it describes its implementation. Author Response: We agree that the us of the term “implementation” in the text can be confusing. We clarified this in the new version of the manuscript at different levels: We changed the title from “Developing an integrated ….” to “ Towrads an integarted …”. In the abstract, the first sentence is now: “This study proposes an approach for developing or improving performance management and measurement systems (PMMSs) for healthcare organisations.”. In the Introduction (last paragraph): “…it proposes a four-phases based approach to support CCSMTL managers to improve and transform the QPM into a more balanced and effective performance management and measurement system (PMMS) . … It can be used by theoreticians and practitioners as roadmap to develop or improve healthcare organisations’ PMMSs. In the Discussion section: ““In this study, we have proposed a comprehensive and quantitative approach to develop or improve a PMMS.” In the Conclusion section: “The CCSMTL has developed a multi-dimensional performance model (Quality-Performance Model - QPM), however, its implementation across the organisation is very challenging. This study proposes an approach that contributes to improving this model and its effective deployment .” This solid paper contributes to ongoing debates on designing performance measurement and management systems in healthcare organizations. The authors describe the process followed to complete the system's design (i.e., situational analysis, SWOT analysis, and literature review), leading to the development of a performance indicator framework. An indicative application of the resulting design was then applied to the CIUSSS program. The main two areas to consider for improving the paper are described below. 1. While the authors recognize the complexity inherent to PMM system design (and implementation), there is insufficient analysis/discussion of the proposed design's strengths and limitations to address such complexity (e.g., interdependence and recurrence between indicators, gaming, users' cognitive biases, etc.). Author Response: Thanks for this comment. We have improved the discussion section in the new version of the manuscript. 2. The authors' use of the concept of "implementation" requires clarification. While the paper clearly presents a prototype for a PMM system, it cannot be said that it describes its implementation. Author Response: We agree that the us of the term “implementation” in the text can be confusing. We clarified this in the new version of the manuscript at different levels: We changed the title from “Developing an integrated ….” to “ Towrads an integarted …”. In the abstract, the first sentence is now: “This study proposes an approach for developing or improving performance management and measurement systems (PMMSs) for healthcare organisations.”. In the Introduction (last paragraph): “…it proposes a four-phases based approach to support CCSMTL managers to improve and transform the QPM into a more balanced and effective performance management and measurement system (PMMS) . … It can be used by theoreticians and practitioners as roadmap to develop or improve healthcare organisations’ PMMSs. In the Discussion section: ““In this study, we have proposed a comprehensive and quantitative approach to develop or improve a PMMS.” In the Conclusion section: “The CCSMTL has developed a multi-dimensional performance model (Quality-Performance Model - QPM), however, its implementation across the organisation is very challenging. This study proposes an approach that contributes to improving this model and its effective deployment .” Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 09 Dec 2024 NEILA EL ASLI , Systems Engineering Department, École de technologie superieure, Montreal, Canada 09 Dec 2024 Author Response This solid paper contributes to ongoing debates on designing performance measurement and management systems in healthcare organizations. The authors describe the process followed to complete the system's design (i.e., situational ... Continue reading This solid paper contributes to ongoing debates on designing performance measurement and management systems in healthcare organizations. The authors describe the process followed to complete the system's design (i.e., situational analysis, SWOT analysis, and literature review), leading to the development of a performance indicator framework. An indicative application of the resulting design was then applied to the CIUSSS program. The main two areas to consider for improving the paper are described below. 1. While the authors recognize the complexity inherent to PMM system design (and implementation), there is insufficient analysis/discussion of the proposed design's strengths and limitations to address such complexity (e.g., interdependence and recurrence between indicators, gaming, users' cognitive biases, etc.). Author Response: Thanks for this comment. We have improved the discussion section in the new version of the manuscript. 2. The authors' use of the concept of "implementation" requires clarification. While the paper clearly presents a prototype for a PMM system, it cannot be said that it describes its implementation. Author Response: We agree that the us of the term “implementation” in the text can be confusing. We clarified this in the new version of the manuscript at different levels: We changed the title from “Developing an integrated ….” to “ Towrads an integarted …”. In the abstract, the first sentence is now: “This study proposes an approach for developing or improving performance management and measurement systems (PMMSs) for healthcare organisations.”. In the Introduction (last paragraph): “…it proposes a four-phases based approach to support CCSMTL managers to improve and transform the QPM into a more balanced and effective performance management and measurement system (PMMS) . … It can be used by theoreticians and practitioners as roadmap to develop or improve healthcare organisations’ PMMSs. In the Discussion section: ““In this study, we have proposed a comprehensive and quantitative approach to develop or improve a PMMS.” In the Conclusion section: “The CCSMTL has developed a multi-dimensional performance model (Quality-Performance Model - QPM), however, its implementation across the organisation is very challenging. This study proposes an approach that contributes to improving this model and its effective deployment .” This solid paper contributes to ongoing debates on designing performance measurement and management systems in healthcare organizations. The authors describe the process followed to complete the system's design (i.e., situational analysis, SWOT analysis, and literature review), leading to the development of a performance indicator framework. An indicative application of the resulting design was then applied to the CIUSSS program. The main two areas to consider for improving the paper are described below. 1. While the authors recognize the complexity inherent to PMM system design (and implementation), there is insufficient analysis/discussion of the proposed design's strengths and limitations to address such complexity (e.g., interdependence and recurrence between indicators, gaming, users' cognitive biases, etc.). Author Response: Thanks for this comment. We have improved the discussion section in the new version of the manuscript. 2. The authors' use of the concept of "implementation" requires clarification. While the paper clearly presents a prototype for a PMM system, it cannot be said that it describes its implementation. Author Response: We agree that the us of the term “implementation” in the text can be confusing. We clarified this in the new version of the manuscript at different levels: We changed the title from “Developing an integrated ….” to “ Towrads an integarted …”. In the abstract, the first sentence is now: “This study proposes an approach for developing or improving performance management and measurement systems (PMMSs) for healthcare organisations.”. In the Introduction (last paragraph): “…it proposes a four-phases based approach to support CCSMTL managers to improve and transform the QPM into a more balanced and effective performance management and measurement system (PMMS) . … It can be used by theoreticians and practitioners as roadmap to develop or improve healthcare organisations’ PMMSs. In the Discussion section: ““In this study, we have proposed a comprehensive and quantitative approach to develop or improve a PMMS.” In the Conclusion section: “The CCSMTL has developed a multi-dimensional performance model (Quality-Performance Model - QPM), however, its implementation across the organisation is very challenging. This study proposes an approach that contributes to improving this model and its effective deployment .” Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Gartner JB. Reviewer Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.151623.r253135 ) The direct URL for this report is: https://f1000research.com/articles/12-1420/v1#referee-response-253135 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 09 Mar 2024 Jean-Baptiste Gartner , Universite Laval, Québec City, Québec, Canada Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.151623.r253135 Comments to the author(s) This study aims to develop a performance management and measurement system for a large healthcare organisation (integrated (University) centres of healthcare and social services) in Quebec, Canada. INTRODUCTION/LITERATURE REVIEW - The ... Continue reading READ ALL Comments to the author(s) This study aims to develop a performance management and measurement system for a large healthcare organisation (integrated (University) centres of healthcare and social services) in Quebec, Canada. INTRODUCTION/LITERATURE REVIEW - The background and the context are clearly stated. - However, the construction of your introduction between PMMS literature and context needs to be better realized to better accompany the reader. Indeed, the construction of your introduction goes from: 1. Contextualizing your case 2. Recent cases in literature 3. Model of QPM and its deployment 4. Literature review on PMS. I invite you to review the structure of the introduction. It might be relevant to merge the literature review section with the cases described in the literature (2 and 4), to conclude your introduction with the QPM model and its implementation issues, justifying your study. And then, conclude with the aim of the study. Apart from the structure, certain points could be enhanced: - Avoid talking about the methods and results of the study in your introduction (paragraph “This study contributes to addressing”). - To further strengthen your literature: PMMS: Ferreira A, et. al. 2009 (Ref 1) Solomon R, et.al. 2013 (Ref 2) Québec context: Fache P, et. al. 2016 (Ref 3) METHODS The methodology is interesting and diverse. Some points could be improved or better justified: - Data collection (Phase I): The material should be better described and quantified (approximate total duration of observations, socio-demographic characteristics of interviewees). A table summarizing the material collected would be a great help. In addition, you do not specify the methodology for collecting information (notebook or reflective notebook for the observations, Semi-structured interviews (recorded or not)). Finally, the interview guide should be included as an appendix. - Data analysis (Phase II): Using SWOT to structure your results is very interesting. But you say nothing about the method of analysis and synthesis (thematic coding by two independent authors and comparison of results? or a coder and then pooling). Here you need to focus on the method description. Have you used methods to ensure the robustness of the analysis (internal and external validity, data triangulation, consistency)? If so, please specify. Table 3 and the following paragraph should be put in the results section, as it is results. - Balanced-QPM development (Phase III) and Indicator structure design and performance measurement (Phase IV): Stay with the process (the method) and keep the results for the results section. RESULTS Your results are interesting and relevant. Create a results section separate from the methods section. DISCUSSION The discussion needs to be further enriched and your results better compared with current knowledge (recent cases in literature and literature review). - From my point of view, the first two paragraphs of the discussion are still part of the results (implementation results). - Future work: Ok, your opening is relevant. Some typos and errors : Introduction: “The Coordinating Committee, which is composed of a limited number of managers, aims at insuring ” (ensuring?) Methods: “Due of this, it is challenging” (to) Conclusion: “which is has been reported in practice and in the literature as a very challenging problem” In summary: - The topic is interesting, and the study mobilizes a wealth of material to propose results relevant to the field. - There are several study structuring issues that need to be addressed (structure of introduction, absence of results title, structure of discussion. A number of tips are given for this purpose. - A number of methodological points need to be reinforced in order to fully convince the reader. - The literature could be strengthened to develop the discussion on the contribution of this study. Proposals to this effect have also been made. Good luck for the next steps. Is the background of the case’s history and progression described in sufficient detail? Yes Is the work clearly and accurately presented and does it cite the current literature? Partly If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? No Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Partly References 1. Ferreira A, Otley D: The design and use of performance management systems: An extended framework for analysis. Management Accounting Research . 2009; 20 (4): 263-282 Publisher Full Text 2. Solomon R, Damba C, Bryant S: Measuring quality at a system level: an impossible task? The Toronto Central LHIN Experience. Healthc Q . 2013; 16 (4): 36-42 PubMed Abstract | Publisher Full Text 3. Fache P, Sicotte C, Minvielle É: The Search is on for Coherent Performance Measurement in Healthcare Organizations. Has Quebec Reached a Crossroads?. Healthc Policy . 2016; 11 (4): 60-9 PubMed Abstract Competing Interests: No competing interests were disclosed. Reviewer Expertise: Healthcare management sciences, Performance management systems, Care pathways optimization, Learning Health Systems I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Gartner JB. Reviewer Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.151623.r253135 ) The direct URL for this report is: https://f1000research.com/articles/12-1420/v1#referee-response-253135 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 09 Dec 2024 NEILA EL ASLI , Systems Engineering Department, École de technologie superieure, Montreal, Canada 09 Dec 2024 Author Response In the following, you will find our responses in bold style. This study aims to develop a performance management and measurement system for a large healthcare organisation (integrated (University) ... Continue reading In the following, you will find our responses in bold style. This study aims to develop a performance management and measurement system for a large healthcare organisation (integrated (University) centres of healthcare and social services) in Quebec, Canada. INTRODUCTION/LITERATURE REVIEW - The background and the context are clearly stated. - However, the construction of your introduction between PMMS literature and context needs to be better realized to better accompany the reader. Indeed, the construction of your introduction goes from: 1. Contextualizing your case 2. Recent cases in literature 3. Model of QPM and its deployment 4. Literature review on PMS. I invite you to review the structure of the introduction. It might be relevant to merge the literature review section with the cases described in the literature (2 and 4), to conclude your introduction with the QPM model and its implementation issues, justifying your study. And then, conclude with the aim of the study. Thanks for this suggestion. Parts 3 and 4 are normally two distinct sections separated from the Introduction. The formatting of the journal merged them with the Introduction. We have corrected this. We agree that it is better to merge parts 2 and 4 and to conclude the introduction with the QPM model and the aim of the study. Please see the changes we have made in the new version of the manuscript. Apart from the structure, certain points could be enhanced: - Avoid talking about the methods and results of the study in your introduction (paragraph “This study contributes to addressing”). Thanks for this suggestion. We have removed the details concerning the approach proposed as well as the description of the results in the new version of the manuscript. - To further strengthen your literature: PMMS: Ferreira A, et. al. 2009 (Ref 1) Solomon R, et.al. 2013 (Ref 2) Québec context: Fache P, et. al. 2016 (Ref 3) 1. Ferreira A, Otley D: The design and use of performance management systems: An extended framework for analysis. Management Accounting Research. 2009; 20 (4): 263-282 Publisher Full Text 2. Solomon R, Damba C, Bryant S: Measuring quality at a system level: an impossible task? The Toronto Central LHIN Experience.Healthc Q. 2013; 16 (4): 36-42 PubMed Abstract | Publisher Full Text 3. Fache P, Sicotte C, Minvielle É: The Search is on for Coherent Performance Measurement in Healthcare Organizations. Has Quebec Reached a Crossroads?. Healthc Policy. 2016; 11 (4): 60-9 PubMed Abstract Thanks for these references. We have added them in the literature review section of the new version of the Manuscript. METHODS The methodology is interesting and diverse. Some points could be improved or better justified: - Data collection (Phase I): The material should be better described and quantified (approximate total duration of observations , socio-demographic characteristics of interviewees). A table summarizing the material collected would be a great help. In addition, you do not specify the methodology for collecting information (notebook or reflective notebook for the observations, Semi-structured interviews (recorded or not)). Finally, the interview guide should be included as an appendix. In the new version of the manuscript, we have added a table summarizing the data collection methods used for conducting phase I and short descriptions (Table 3). We also referred the reader to a work presenting more in detail on the collection data methods in the text (Ben Fradj, 2020). Unfortunately, the journal does not allow including any appendix. - Data analysis (Phase II): Using SWOT to structure your results is very interesting. But you say nothing about the method of analysis and synthesis (thematic coding by two independent authors and comparison of results? or a coder and then pooling). Here you need to focus on the method description. Have you used methods to ensure the robustness of the analysis (internal and external validity, data triangulation, consistency)? If so, please specify. Table 3 and the following paragraph should be put in the results section, as it is results. We have added more information in the text to describe how we conducted the analysis with and moved Table 3 and the paragraphs commenting it to the Results section (Table 4 in the new version of the manuscript). - Balanced-QPM development (Phase III) and Indicator structure design and performance measurement (Phase IV): Stay with the process (the method) and keep the results for the results section. Thanks for this comment. In the new version of the manuscript, we have kept only the description of the methods used and moved all the results to the Results section. RESULTS Your results are interesting and relevant. Create a results section separate from the methods section. Thanks for this suggestion. We have now a Results section. DISCUSSION The discussion needs to be further enriched and your results better compared with current knowledge (recent cases in literature and literature review). From my point of view, the first two paragraphs of the discussion are still part of the results (implementation results). You are right, we improved the discussion section following your recommendation. - Future work: Ok, your opening is relevant. Thanks. Some typos and errors : Introduction: “The Coordinating Committee, which is composed of a limited number of managers, aims at insuring ” (ensuring?) Right “ensuring”. Methods: “Due of this, it is challenging” (to) Indeed, “to” is the correct term to use. Conclusion: “which is has been reported in practice and in the literature as a very challenging problem” Corrected, “which has been”. In summary: - The topic is interesting, and the study mobilizes a wealth of material to propose results relevant to the field. - There are several study structuring issues that need to be addressed (structure of introduction, absence of results title, structure of discussion. A number of tips are given for this purpose. - A number of methodological points need to be reinforced in order to fully convince the reader. - The literature could be strengthened to develop the discussion on the contribution of this study. Proposals to this effect have also been made. Good luck for the next steps. Thanks for the relevant comments and suggestions that we believe significantly improved our manuscript. In the following, you will find our responses in bold style. This study aims to develop a performance management and measurement system for a large healthcare organisation (integrated (University) centres of healthcare and social services) in Quebec, Canada. INTRODUCTION/LITERATURE REVIEW - The background and the context are clearly stated. - However, the construction of your introduction between PMMS literature and context needs to be better realized to better accompany the reader. Indeed, the construction of your introduction goes from: 1. Contextualizing your case 2. Recent cases in literature 3. Model of QPM and its deployment 4. Literature review on PMS. I invite you to review the structure of the introduction. It might be relevant to merge the literature review section with the cases described in the literature (2 and 4), to conclude your introduction with the QPM model and its implementation issues, justifying your study. And then, conclude with the aim of the study. Thanks for this suggestion. Parts 3 and 4 are normally two distinct sections separated from the Introduction. The formatting of the journal merged them with the Introduction. We have corrected this. We agree that it is better to merge parts 2 and 4 and to conclude the introduction with the QPM model and the aim of the study. Please see the changes we have made in the new version of the manuscript. Apart from the structure, certain points could be enhanced: - Avoid talking about the methods and results of the study in your introduction (paragraph “This study contributes to addressing”). Thanks for this suggestion. We have removed the details concerning the approach proposed as well as the description of the results in the new version of the manuscript. - To further strengthen your literature: PMMS: Ferreira A, et. al. 2009 (Ref 1) Solomon R, et.al. 2013 (Ref 2) Québec context: Fache P, et. al. 2016 (Ref 3) 1. Ferreira A, Otley D: The design and use of performance management systems: An extended framework for analysis. Management Accounting Research. 2009; 20 (4): 263-282 Publisher Full Text 2. Solomon R, Damba C, Bryant S: Measuring quality at a system level: an impossible task? The Toronto Central LHIN Experience.Healthc Q. 2013; 16 (4): 36-42 PubMed Abstract | Publisher Full Text 3. Fache P, Sicotte C, Minvielle É: The Search is on for Coherent Performance Measurement in Healthcare Organizations. Has Quebec Reached a Crossroads?. Healthc Policy. 2016; 11 (4): 60-9 PubMed Abstract Thanks for these references. We have added them in the literature review section of the new version of the Manuscript. METHODS The methodology is interesting and diverse. Some points could be improved or better justified: - Data collection (Phase I): The material should be better described and quantified (approximate total duration of observations , socio-demographic characteristics of interviewees). A table summarizing the material collected would be a great help. In addition, you do not specify the methodology for collecting information (notebook or reflective notebook for the observations, Semi-structured interviews (recorded or not)). Finally, the interview guide should be included as an appendix. In the new version of the manuscript, we have added a table summarizing the data collection methods used for conducting phase I and short descriptions (Table 3). We also referred the reader to a work presenting more in detail on the collection data methods in the text (Ben Fradj, 2020). Unfortunately, the journal does not allow including any appendix. - Data analysis (Phase II): Using SWOT to structure your results is very interesting. But you say nothing about the method of analysis and synthesis (thematic coding by two independent authors and comparison of results? or a coder and then pooling). Here you need to focus on the method description. Have you used methods to ensure the robustness of the analysis (internal and external validity, data triangulation, consistency)? If so, please specify. Table 3 and the following paragraph should be put in the results section, as it is results. We have added more information in the text to describe how we conducted the analysis with and moved Table 3 and the paragraphs commenting it to the Results section (Table 4 in the new version of the manuscript). - Balanced-QPM development (Phase III) and Indicator structure design and performance measurement (Phase IV): Stay with the process (the method) and keep the results for the results section. Thanks for this comment. In the new version of the manuscript, we have kept only the description of the methods used and moved all the results to the Results section. RESULTS Your results are interesting and relevant. Create a results section separate from the methods section. Thanks for this suggestion. We have now a Results section. DISCUSSION The discussion needs to be further enriched and your results better compared with current knowledge (recent cases in literature and literature review). From my point of view, the first two paragraphs of the discussion are still part of the results (implementation results). You are right, we improved the discussion section following your recommendation. - Future work: Ok, your opening is relevant. Thanks. Some typos and errors : Introduction: “The Coordinating Committee, which is composed of a limited number of managers, aims at insuring ” (ensuring?) Right “ensuring”. Methods: “Due of this, it is challenging” (to) Indeed, “to” is the correct term to use. Conclusion: “which is has been reported in practice and in the literature as a very challenging problem” Corrected, “which has been”. In summary: - The topic is interesting, and the study mobilizes a wealth of material to propose results relevant to the field. - There are several study structuring issues that need to be addressed (structure of introduction, absence of results title, structure of discussion. A number of tips are given for this purpose. - A number of methodological points need to be reinforced in order to fully convince the reader. - The literature could be strengthened to develop the discussion on the contribution of this study. Proposals to this effect have also been made. Good luck for the next steps. Thanks for the relevant comments and suggestions that we believe significantly improved our manuscript. Competing Interests: No competing interests were disclosed. Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 09 Dec 2024 NEILA EL ASLI , Systems Engineering Department, École de technologie superieure, Montreal, Canada 09 Dec 2024 Author Response In the following, you will find our responses in bold style. This study aims to develop a performance management and measurement system for a large healthcare organisation (integrated (University) ... Continue reading In the following, you will find our responses in bold style. This study aims to develop a performance management and measurement system for a large healthcare organisation (integrated (University) centres of healthcare and social services) in Quebec, Canada. INTRODUCTION/LITERATURE REVIEW - The background and the context are clearly stated. - However, the construction of your introduction between PMMS literature and context needs to be better realized to better accompany the reader. Indeed, the construction of your introduction goes from: 1. Contextualizing your case 2. Recent cases in literature 3. Model of QPM and its deployment 4. Literature review on PMS. I invite you to review the structure of the introduction. It might be relevant to merge the literature review section with the cases described in the literature (2 and 4), to conclude your introduction with the QPM model and its implementation issues, justifying your study. And then, conclude with the aim of the study. Thanks for this suggestion. Parts 3 and 4 are normally two distinct sections separated from the Introduction. The formatting of the journal merged them with the Introduction. We have corrected this. We agree that it is better to merge parts 2 and 4 and to conclude the introduction with the QPM model and the aim of the study. Please see the changes we have made in the new version of the manuscript. Apart from the structure, certain points could be enhanced: - Avoid talking about the methods and results of the study in your introduction (paragraph “This study contributes to addressing”). Thanks for this suggestion. We have removed the details concerning the approach proposed as well as the description of the results in the new version of the manuscript. - To further strengthen your literature: PMMS: Ferreira A, et. al. 2009 (Ref 1) Solomon R, et.al. 2013 (Ref 2) Québec context: Fache P, et. al. 2016 (Ref 3) 1. Ferreira A, Otley D: The design and use of performance management systems: An extended framework for analysis. Management Accounting Research. 2009; 20 (4): 263-282 Publisher Full Text 2. Solomon R, Damba C, Bryant S: Measuring quality at a system level: an impossible task? The Toronto Central LHIN Experience.Healthc Q. 2013; 16 (4): 36-42 PubMed Abstract | Publisher Full Text 3. Fache P, Sicotte C, Minvielle É: The Search is on for Coherent Performance Measurement in Healthcare Organizations. Has Quebec Reached a Crossroads?. Healthc Policy. 2016; 11 (4): 60-9 PubMed Abstract Thanks for these references. We have added them in the literature review section of the new version of the Manuscript. METHODS The methodology is interesting and diverse. Some points could be improved or better justified: - Data collection (Phase I): The material should be better described and quantified (approximate total duration of observations , socio-demographic characteristics of interviewees). A table summarizing the material collected would be a great help. In addition, you do not specify the methodology for collecting information (notebook or reflective notebook for the observations, Semi-structured interviews (recorded or not)). Finally, the interview guide should be included as an appendix. In the new version of the manuscript, we have added a table summarizing the data collection methods used for conducting phase I and short descriptions (Table 3). We also referred the reader to a work presenting more in detail on the collection data methods in the text (Ben Fradj, 2020). Unfortunately, the journal does not allow including any appendix. - Data analysis (Phase II): Using SWOT to structure your results is very interesting. But you say nothing about the method of analysis and synthesis (thematic coding by two independent authors and comparison of results? or a coder and then pooling). Here you need to focus on the method description. Have you used methods to ensure the robustness of the analysis (internal and external validity, data triangulation, consistency)? If so, please specify. Table 3 and the following paragraph should be put in the results section, as it is results. We have added more information in the text to describe how we conducted the analysis with and moved Table 3 and the paragraphs commenting it to the Results section (Table 4 in the new version of the manuscript). - Balanced-QPM development (Phase III) and Indicator structure design and performance measurement (Phase IV): Stay with the process (the method) and keep the results for the results section. Thanks for this comment. In the new version of the manuscript, we have kept only the description of the methods used and moved all the results to the Results section. RESULTS Your results are interesting and relevant. Create a results section separate from the methods section. Thanks for this suggestion. We have now a Results section. DISCUSSION The discussion needs to be further enriched and your results better compared with current knowledge (recent cases in literature and literature review). From my point of view, the first two paragraphs of the discussion are still part of the results (implementation results). You are right, we improved the discussion section following your recommendation. - Future work: Ok, your opening is relevant. Thanks. Some typos and errors : Introduction: “The Coordinating Committee, which is composed of a limited number of managers, aims at insuring ” (ensuring?) Right “ensuring”. Methods: “Due of this, it is challenging” (to) Indeed, “to” is the correct term to use. Conclusion: “which is has been reported in practice and in the literature as a very challenging problem” Corrected, “which has been”. In summary: - The topic is interesting, and the study mobilizes a wealth of material to propose results relevant to the field. - There are several study structuring issues that need to be addressed (structure of introduction, absence of results title, structure of discussion. A number of tips are given for this purpose. - A number of methodological points need to be reinforced in order to fully convince the reader. - The literature could be strengthened to develop the discussion on the contribution of this study. Proposals to this effect have also been made. Good luck for the next steps. Thanks for the relevant comments and suggestions that we believe significantly improved our manuscript. In the following, you will find our responses in bold style. This study aims to develop a performance management and measurement system for a large healthcare organisation (integrated (University) centres of healthcare and social services) in Quebec, Canada. INTRODUCTION/LITERATURE REVIEW - The background and the context are clearly stated. - However, the construction of your introduction between PMMS literature and context needs to be better realized to better accompany the reader. Indeed, the construction of your introduction goes from: 1. Contextualizing your case 2. Recent cases in literature 3. Model of QPM and its deployment 4. Literature review on PMS. I invite you to review the structure of the introduction. It might be relevant to merge the literature review section with the cases described in the literature (2 and 4), to conclude your introduction with the QPM model and its implementation issues, justifying your study. And then, conclude with the aim of the study. Thanks for this suggestion. Parts 3 and 4 are normally two distinct sections separated from the Introduction. The formatting of the journal merged them with the Introduction. We have corrected this. We agree that it is better to merge parts 2 and 4 and to conclude the introduction with the QPM model and the aim of the study. Please see the changes we have made in the new version of the manuscript. Apart from the structure, certain points could be enhanced: - Avoid talking about the methods and results of the study in your introduction (paragraph “This study contributes to addressing”). Thanks for this suggestion. We have removed the details concerning the approach proposed as well as the description of the results in the new version of the manuscript. - To further strengthen your literature: PMMS: Ferreira A, et. al. 2009 (Ref 1) Solomon R, et.al. 2013 (Ref 2) Québec context: Fache P, et. al. 2016 (Ref 3) 1. Ferreira A, Otley D: The design and use of performance management systems: An extended framework for analysis. Management Accounting Research. 2009; 20 (4): 263-282 Publisher Full Text 2. Solomon R, Damba C, Bryant S: Measuring quality at a system level: an impossible task? The Toronto Central LHIN Experience.Healthc Q. 2013; 16 (4): 36-42 PubMed Abstract | Publisher Full Text 3. Fache P, Sicotte C, Minvielle É: The Search is on for Coherent Performance Measurement in Healthcare Organizations. Has Quebec Reached a Crossroads?. Healthc Policy. 2016; 11 (4): 60-9 PubMed Abstract Thanks for these references. We have added them in the literature review section of the new version of the Manuscript. METHODS The methodology is interesting and diverse. Some points could be improved or better justified: - Data collection (Phase I): The material should be better described and quantified (approximate total duration of observations , socio-demographic characteristics of interviewees). A table summarizing the material collected would be a great help. In addition, you do not specify the methodology for collecting information (notebook or reflective notebook for the observations, Semi-structured interviews (recorded or not)). Finally, the interview guide should be included as an appendix. In the new version of the manuscript, we have added a table summarizing the data collection methods used for conducting phase I and short descriptions (Table 3). We also referred the reader to a work presenting more in detail on the collection data methods in the text (Ben Fradj, 2020). Unfortunately, the journal does not allow including any appendix. - Data analysis (Phase II): Using SWOT to structure your results is very interesting. But you say nothing about the method of analysis and synthesis (thematic coding by two independent authors and comparison of results? or a coder and then pooling). Here you need to focus on the method description. Have you used methods to ensure the robustness of the analysis (internal and external validity, data triangulation, consistency)? If so, please specify. Table 3 and the following paragraph should be put in the results section, as it is results. We have added more information in the text to describe how we conducted the analysis with and moved Table 3 and the paragraphs commenting it to the Results section (Table 4 in the new version of the manuscript). - Balanced-QPM development (Phase III) and Indicator structure design and performance measurement (Phase IV): Stay with the process (the method) and keep the results for the results section. Thanks for this comment. In the new version of the manuscript, we have kept only the description of the methods used and moved all the results to the Results section. RESULTS Your results are interesting and relevant. Create a results section separate from the methods section. Thanks for this suggestion. We have now a Results section. DISCUSSION The discussion needs to be further enriched and your results better compared with current knowledge (recent cases in literature and literature review). From my point of view, the first two paragraphs of the discussion are still part of the results (implementation results). You are right, we improved the discussion section following your recommendation. - Future work: Ok, your opening is relevant. Thanks. Some typos and errors : Introduction: “The Coordinating Committee, which is composed of a limited number of managers, aims at insuring ” (ensuring?) Right “ensuring”. Methods: “Due of this, it is challenging” (to) Indeed, “to” is the correct term to use. Conclusion: “which is has been reported in practice and in the literature as a very challenging problem” Corrected, “which has been”. In summary: - The topic is interesting, and the study mobilizes a wealth of material to propose results relevant to the field. - There are several study structuring issues that need to be addressed (structure of introduction, absence of results title, structure of discussion. A number of tips are given for this purpose. - A number of methodological points need to be reinforced in order to fully convince the reader. - The literature could be strengthened to develop the discussion on the contribution of this study. Proposals to this effect have also been made. Good luck for the next steps. Thanks for the relevant comments and suggestions that we believe significantly improved our manuscript. Competing Interests: No competing interests were disclosed. Close Report a concern COMMENT ON THIS REPORT Comments on this article Comments (0) Version 2 VERSION 2 PUBLISHED 30 Oct 2023 ADD YOUR COMMENT Comment keyboard_arrow_left keyboard_arrow_right Open Peer Review Reviewer Status info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Reviewer Reports Invited Reviewers 1 2 3 4 Version 2 (revision) 09 Dec 24 read Version 1 30 Oct 23 read read read read Jean-Baptiste Gartner , Universite Laval, Québec City, Canada Wolfgang Munar , The George Washington University, Washington, USA Paulino Silva , ISCAP Polytechnic of Porto, Porto, Portugal Pascal Forget , Université du Québec à Trois-Rivières, Trois-Rivières, Canada Comments on this article All Comments (0) Add a comment Sign up for content alerts Sign Up You are now signed up to receive this alert Browse by related subjects keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Gartner J. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 23 Dec 2024 | for Version 2 Jean-Baptiste Gartner , Universite Laval, Québec City, Québec, Canada 0 Views copyright © 2024 Gartner J. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This study aims to develop a performance management and measurement system for a large healthcare organisation (integrated (University) centres of healthcare and social services) in Montreal, Quebec, Canada. Overall, the revisions have greatly improved the document. However, there are still some points that need to be improved before it can be accepted. INTRODUCTI ON/LITERATURE REVIEW - The background and the context are clearly stated. - "Even though recent studies in the literature discuss performance management systems in healthcare organisations and recognise both the importance and the complexity of measuring performance, most of them are descriptive and do not propose PMMSs capable of effectively structuring and measuring the performance of healthcare organisations. " This statement must be supported by at least one reference. Otherwise, it appears to be a personal opinion. - "The QPM (see Figure 1 ) is based on the MSSS strategic plan (2019-2023 period)" This statement does not work (the legal deposit of the document of the MSSS dates from after the article was published), remove or explain. LITTERATURE REVIEW This part has been suitably enhanced. METHODS The addition of the table is appropriate. The addition of details of the method is welcome. RESULTS Given the number of data sources (interview, observation, focus group), it is surprising to have been left with a single case study description. With this in mind, it is unfortunate that assertions are not supported by your empirical material, for example : - "From Table 4 , we can see that one of the strengths identified is the well implementation of the QPM at the strategic level". (This is an assertion, but the reader has no data to corroborate the assertion (interview excerpts, focus groups, changes in a quantitative/qualitative indicator, etc.). If the intention is to be descriptive only, this can remain unchanged. DISCUSSION The discussion still does not contextualise the results with existing knowledge (present in your Literature review paragraph). Contribution of your article ? - Future work: Ok, your opening is relevant. Some typos and errors : Introduction: - " In the province of Qu é bec " (Quebec) - " which fall under the responsibility of the Ministry of Healthcare and Social Services (Mi m istère de la Santé et des Services Sociaux". Competing Interests No competing interests were disclosed. Reviewer Expertise Healthcare management research I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (0) Gartner JB. Peer Review Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.169268.r347038) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/12-1420/v2#referee-response-347038 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Forget P. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 16 Sep 2024 | for Version 1 Pascal Forget , Université du Québec à Trois-Rivières, Trois-Rivières, Quebec, Canada 0 Views copyright © 2024 Forget P. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This article proposes an approach to facilitate the implementation of a performance management model in a government healthcare system. This article is really interesting and offer a pertinent contribution. It is clear and well written. The introduction, the literature, the method and discussion are clear and well presented. There is no specific point to address and the paper could be index as it is. Is the background of the case’s history and progression described in sufficient detail? Yes Is the work clearly and accurately presented and does it cite the current literature? Yes If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Yes Competing Interests No competing interests were disclosed. Reviewer Expertise Industrial engineering, operation research, simulation I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) Forget P. Peer Review Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.151623.r219887) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/12-1420/v1#referee-response-219887 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Silva P. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 04 Apr 2024 | for Version 1 Paulino Silva , ISCAP Polytechnic of Porto, Porto, Porto District, Portugal 0 Views copyright © 2024 Silva P. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions The article presents a very relevant topic with great added value, both for the organizational performance management literature and for management practitioners. The contribution of the article can be relevant for the body of knowledge. However, there are some methodological weaknesses that should be considered. The article claims to develop an integrated performance management system for healthcare organizations, but perhaps it would be more appropriate to define it as an improvement to the existing system. This is because it makes a diagnosis of the existing system and, through SWOT analysis, presents the main strengths, weaknesses, opportunities, and threats. The used case study seems too specific (Physical disability, intellectual disability, and autism spectrum disorder" of CIUSSS du Centre-Sud-de-l'île-de-Montreal, in the province of Québec, Canada) to be considered a reference of a performance management system in healthcare organizations. As the study began in 2019, and the interviews between May 2019 and September 2021, could the results have been too affected by the Covid-19 pandemic? The number of interviews seems to be very low (only 12) and the variety of jobs very limited (only managers). Consider including additional literature specifically related to performance management in healthcare organizations, such as : [1] Some “typos” in the article: The first paragraph of the methods section is incorrect, it should end with phases III and IV. The abbreviation for Ministry of Healthcare and Social Services presented in the article is MSSS and not MHSS, why? Customer-focused instead of customer-focussed in line 6 of the section Conclusions and research perspectives. Is the background of the case’s history and progression described in sufficient detail? Yes Is the work clearly and accurately presented and does it cite the current literature? Partly If applicable, is the statistical analysis and its interpretation appropriate? I cannot comment. A qualified statistician is required. Are all the source data underlying the results available to ensure full reproducibility? Partly Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Yes References 1. Silva P, Ferreira A: Performance management in primary healthcare services: evidence from a field study. Qualitative Research in Accounting & Management . 2010; 7 (4): 424-449 Publisher Full Text Competing Interests No competing interests were disclosed. Reviewer Expertise Management Accounting I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (1) Author Response 09 Dec 2024 NEILA EL ASLI, Systems Engineering Department, École de technologie superieure, Montreal, Canada Reviewer #3: APPROVED WITH RESERVATIONS The article presents a very relevant topic with great added value, both for the organizational performance management literature and for management practitioners. The contribution of the article can be relevant for the body of knowledge. However, there are some methodological weaknesses that should be considered. The article claims to develop an integrated performance management system for healthcare organizations, but perhaps it would be more appropriate to define it as an improvement to the existing system. This is because it makes a diagnosis of the existing system and, through SWOT analysis, presents the main strengths, weaknesses, opportunities, and threats. You are right. The PMMS (Performance Management and Measurement System) we proposed is an improved Quality-Performance Model (QPM) in the context of the CCSMTL. However, in the absence of a performance model, our approach (or at least part of it if we remove the SWOT analysis), could be used to develop a PMMS. We clarified this in the text in the new version of the manuscript at different levels: We changed the title from “Developing an integrated ….” to “ Towrads an integarted …”. In the abstract, the first sentence is now: “This study proposes an approach for developing or improving performance management and measurement systems (PMMSs) for healthcare organisations.”. In the Introduction (last paragraph): “…it proposes a four-phases based approach to support CCSMTL managers to improve and transform the QPM into a more balanced and effective performance management and measurement system (PMMS) . … It can be used by theoreticians and practitioners as roadmap to develop or improve healthcare organisations’ PMMSs. In the Discussion section: ““In this study, we have proposed a comprehensive and quantitative approach to develop or improve a PMMS.” In the Conclusion section: “The CCSMTL has developed a multi-dimensional performance model (Quality-Performance Model - QPM), however, its implementation across the organisation is very challenging. This study proposes an approach that contributes to improving this model and its effective deployment .” The used case study seems too specific (Physical disability, intellectual disability, and autism spectrum disorder" of CIUSSS du Centre-Sud-de-l'île-de-Montreal, in the province of Québec, Canada) to be considered a reference of a performance management system in healthcare organizations. There are two levels of application of our PMMS. We applied Phases I, II, III, and partially IV to the entire CIUSSS, and then Phases III and IV to the program Physical disability, intellectual disability, and autism spectrum disorder . We have made changes in the new version of the manuscript to clarify this (please see the Results section). We agree that more work is needed before we can consider this system as a reference in healthcare organizations. We have acknowledged this limitation in the discussion Section, and as mentioned before, we also have changed the title. As the study began in 2019, and the interviews between May 2019 and September 2021, could the results have been too affected by the Covid-19 pandemic? This is right, part of the data collection took place during the COVID-19 pandemic (2020). During this period, the process slowed down; it was difficult to reach people. This resulted in a lesser number of interviews than what we targeted initially. However, it is difficult to estimate the impact on the results though. The number of interviews seems to be very low (only 12) and the variety of jobs very limited (only managers). As mentioned above, initially we targeted more interviews. Even though all the participants involved are managers, there is a certain variety considering that they don’t have the same function, hierarchy level in the organization, and they don’t come from the same programs/services (please see Table 3 in the new version of the manuscript). Employees were also met during the data collection phase. However, their knowledge level regarding performance measurement, notably in relation to the QPM, was too low or inexistant (in some services, there are no performance indicators put in place at all). This lack of knowledge at the operational level has been mentioned as a weakness in the SWOT analysis “The QPM is known at the tactical and operational levels, but not used for performance management and measurement purposes”. We appreciate your comment. We have discussed this as a limitation in the new version of the manuscript. Consider including additional literature specifically related to performance management in healthcare organizations, such as : [1] 1. Silva P, Ferreira A: Performance management in primary healthcare services: evidence from a field study. Qualitative Research in Accounting & Management. 2010; 7 (4): 424-449 Publisher Full Text Thanks for this suggestion. We have included it in our literature review. We have also added three other references suggested by Reviewer #1. Some “typos” in the article: The first paragraph of the methods section is incorrect, it should end with phases III and IV. Y ou are right. It has been corrected. The abbreviation for Ministry of Healthcare and Social Services presented in the article is MSSS and not MHSS, why? MSSS stands for Ministère de la Santé et des Services Sociaux in French (official title of the Ministry). We will have added this information in the new version of the manuscript to avoid confusion. Customer-focused instead of customer-focussed in line 6 of the section Conclusions and research perspectives. Thanks, we have corrected it. View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Silva P. Peer Review Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.151623.r238355) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/12-1420/v1#referee-response-238355 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Munar W. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 28 Mar 2024 | for Version 1 Wolfgang Munar , The George Washington University, Washington, District of Columbia, USA 0 Views copyright © 2024 Munar W. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This solid paper contributes to ongoing debates on designing performance measurement and management systems in healthcare organizations. The authors describe the process followed to complete the system's design (i.e., situational analysis, SWOT analysis, and literature review), leading to the development of a performance indicator framework. An indicative application of the resulting design was then applied to the CIUSSS program. The main two areas to consider for improving the paper are described below. 1. While the authors recognize the complexity inherent to PMM system design (and implementation), there is insufficient analysis/discussion of the proposed design's strengths and limitations to address such complexity (e.g., interdependence and recurrence between indicators, gaming, users' cognitive biases, etc.). 2. The authors' use of the concept of "implementation" requires clarification. While the paper clearly presents a prototype for a PMM system, it cannot be said that it describes its implementation. Is the background of the case’s history and progression described in sufficient detail? Yes Is the work clearly and accurately presented and does it cite the current literature? Yes If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly Is the case presented with sufficient detail to be useful for teaching or other practitioners? Yes Competing Interests No competing interests were disclosed. Reviewer Expertise Health policy and systems research, and theory-driven evaluation. I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (1) Author Response 09 Dec 2024 NEILA EL ASLI, Systems Engineering Department, École de technologie superieure, Montreal, Canada This solid paper contributes to ongoing debates on designing performance measurement and management systems in healthcare organizations. The authors describe the process followed to complete the system's design (i.e., situational analysis, SWOT analysis, and literature review), leading to the development of a performance indicator framework. An indicative application of the resulting design was then applied to the CIUSSS program. The main two areas to consider for improving the paper are described below. 1. While the authors recognize the complexity inherent to PMM system design (and implementation), there is insufficient analysis/discussion of the proposed design's strengths and limitations to address such complexity (e.g., interdependence and recurrence between indicators, gaming, users' cognitive biases, etc.). Author Response: Thanks for this comment. We have improved the discussion section in the new version of the manuscript. 2. The authors' use of the concept of "implementation" requires clarification. While the paper clearly presents a prototype for a PMM system, it cannot be said that it describes its implementation. Author Response: We agree that the us of the term “implementation” in the text can be confusing. We clarified this in the new version of the manuscript at different levels: We changed the title from “Developing an integrated ….” to “ Towrads an integarted …”. In the abstract, the first sentence is now: “This study proposes an approach for developing or improving performance management and measurement systems (PMMSs) for healthcare organisations.”. In the Introduction (last paragraph): “…it proposes a four-phases based approach to support CCSMTL managers to improve and transform the QPM into a more balanced and effective performance management and measurement system (PMMS) . … It can be used by theoreticians and practitioners as roadmap to develop or improve healthcare organisations’ PMMSs. In the Discussion section: ““In this study, we have proposed a comprehensive and quantitative approach to develop or improve a PMMS.” In the Conclusion section: “The CCSMTL has developed a multi-dimensional performance model (Quality-Performance Model - QPM), however, its implementation across the organisation is very challenging. This study proposes an approach that contributes to improving this model and its effective deployment .” View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Munar W. Peer Review Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.151623.r253136) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/12-1420/v1#referee-response-253136 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Gartner J. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 09 Mar 2024 | for Version 1 Jean-Baptiste Gartner , Universite Laval, Québec City, Québec, Canada 0 Views copyright © 2024 Gartner J. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Comments to the author(s) This study aims to develop a performance management and measurement system for a large healthcare organisation (integrated (University) centres of healthcare and social services) in Quebec, Canada. INTRODUCTION/LITERATURE REVIEW - The background and the context are clearly stated. - However, the construction of your introduction between PMMS literature and context needs to be better realized to better accompany the reader. Indeed, the construction of your introduction goes from: 1. Contextualizing your case 2. Recent cases in literature 3. Model of QPM and its deployment 4. Literature review on PMS. I invite you to review the structure of the introduction. It might be relevant to merge the literature review section with the cases described in the literature (2 and 4), to conclude your introduction with the QPM model and its implementation issues, justifying your study. And then, conclude with the aim of the study. Apart from the structure, certain points could be enhanced: - Avoid talking about the methods and results of the study in your introduction (paragraph “This study contributes to addressing”). - To further strengthen your literature: PMMS: Ferreira A, et. al. 2009 (Ref 1) Solomon R, et.al. 2013 (Ref 2) Québec context: Fache P, et. al. 2016 (Ref 3) METHODS The methodology is interesting and diverse. Some points could be improved or better justified: - Data collection (Phase I): The material should be better described and quantified (approximate total duration of observations, socio-demographic characteristics of interviewees). A table summarizing the material collected would be a great help. In addition, you do not specify the methodology for collecting information (notebook or reflective notebook for the observations, Semi-structured interviews (recorded or not)). Finally, the interview guide should be included as an appendix. - Data analysis (Phase II): Using SWOT to structure your results is very interesting. But you say nothing about the method of analysis and synthesis (thematic coding by two independent authors and comparison of results? or a coder and then pooling). Here you need to focus on the method description. Have you used methods to ensure the robustness of the analysis (internal and external validity, data triangulation, consistency)? If so, please specify. Table 3 and the following paragraph should be put in the results section, as it is results. - Balanced-QPM development (Phase III) and Indicator structure design and performance measurement (Phase IV): Stay with the process (the method) and keep the results for the results section. RESULTS Your results are interesting and relevant. Create a results section separate from the methods section. DISCUSSION The discussion needs to be further enriched and your results better compared with current knowledge (recent cases in literature and literature review). - From my point of view, the first two paragraphs of the discussion are still part of the results (implementation results). - Future work: Ok, your opening is relevant. Some typos and errors : Introduction: “The Coordinating Committee, which is composed of a limited number of managers, aims at insuring ” (ensuring?) Methods: “Due of this, it is challenging” (to) Conclusion: “which is has been reported in practice and in the literature as a very challenging problem” In summary: - The topic is interesting, and the study mobilizes a wealth of material to propose results relevant to the field. - There are several study structuring issues that need to be addressed (structure of introduction, absence of results title, structure of discussion. A number of tips are given for this purpose. - A number of methodological points need to be reinforced in order to fully convince the reader. - The literature could be strengthened to develop the discussion on the contribution of this study. Proposals to this effect have also been made. Good luck for the next steps. Is the background of the case’s history and progression described in sufficient detail? Yes Is the work clearly and accurately presented and does it cite the current literature? Partly If applicable, is the statistical analysis and its interpretation appropriate? Not applicable Are all the source data underlying the results available to ensure full reproducibility? No Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Partly References 1. Ferreira A, Otley D: The design and use of performance management systems: An extended framework for analysis. Management Accounting Research . 2009; 20 (4): 263-282 Publisher Full Text 2. Solomon R, Damba C, Bryant S: Measuring quality at a system level: an impossible task? The Toronto Central LHIN Experience. Healthc Q . 2013; 16 (4): 36-42 PubMed Abstract | Publisher Full Text 3. Fache P, Sicotte C, Minvielle É: The Search is on for Coherent Performance Measurement in Healthcare Organizations. Has Quebec Reached a Crossroads?. Healthc Policy . 2016; 11 (4): 60-9 PubMed Abstract Competing Interests No competing interests were disclosed. Reviewer Expertise Healthcare management sciences, Performance management systems, Care pathways optimization, Learning Health Systems I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (1) Author Response 09 Dec 2024 NEILA EL ASLI, Systems Engineering Department, École de technologie superieure, Montreal, Canada In the following, you will find our responses in bold style. This study aims to develop a performance management and measurement system for a large healthcare organisation (integrated (University) centres of healthcare and social services) in Quebec, Canada. INTRODUCTION/LITERATURE REVIEW - The background and the context are clearly stated. - However, the construction of your introduction between PMMS literature and context needs to be better realized to better accompany the reader. Indeed, the construction of your introduction goes from: 1. Contextualizing your case 2. Recent cases in literature 3. Model of QPM and its deployment 4. Literature review on PMS. I invite you to review the structure of the introduction. It might be relevant to merge the literature review section with the cases described in the literature (2 and 4), to conclude your introduction with the QPM model and its implementation issues, justifying your study. And then, conclude with the aim of the study. Thanks for this suggestion. Parts 3 and 4 are normally two distinct sections separated from the Introduction. The formatting of the journal merged them with the Introduction. We have corrected this. We agree that it is better to merge parts 2 and 4 and to conclude the introduction with the QPM model and the aim of the study. Please see the changes we have made in the new version of the manuscript. Apart from the structure, certain points could be enhanced: - Avoid talking about the methods and results of the study in your introduction (paragraph “This study contributes to addressing”). Thanks for this suggestion. We have removed the details concerning the approach proposed as well as the description of the results in the new version of the manuscript. - To further strengthen your literature: PMMS: Ferreira A, et. al. 2009 (Ref 1) Solomon R, et.al. 2013 (Ref 2) Québec context: Fache P, et. al. 2016 (Ref 3) 1. Ferreira A, Otley D: The design and use of performance management systems: An extended framework for analysis. Management Accounting Research. 2009; 20 (4): 263-282 Publisher Full Text 2. Solomon R, Damba C, Bryant S: Measuring quality at a system level: an impossible task? The Toronto Central LHIN Experience.Healthc Q. 2013; 16 (4): 36-42 PubMed Abstract | Publisher Full Text 3. Fache P, Sicotte C, Minvielle É: The Search is on for Coherent Performance Measurement in Healthcare Organizations. Has Quebec Reached a Crossroads?. Healthc Policy. 2016; 11 (4): 60-9 PubMed Abstract Thanks for these references. We have added them in the literature review section of the new version of the Manuscript. METHODS The methodology is interesting and diverse. Some points could be improved or better justified: - Data collection (Phase I): The material should be better described and quantified (approximate total duration of observations , socio-demographic characteristics of interviewees). A table summarizing the material collected would be a great help. In addition, you do not specify the methodology for collecting information (notebook or reflective notebook for the observations, Semi-structured interviews (recorded or not)). Finally, the interview guide should be included as an appendix. In the new version of the manuscript, we have added a table summarizing the data collection methods used for conducting phase I and short descriptions (Table 3). We also referred the reader to a work presenting more in detail on the collection data methods in the text (Ben Fradj, 2020). Unfortunately, the journal does not allow including any appendix. - Data analysis (Phase II): Using SWOT to structure your results is very interesting. But you say nothing about the method of analysis and synthesis (thematic coding by two independent authors and comparison of results? or a coder and then pooling). Here you need to focus on the method description. Have you used methods to ensure the robustness of the analysis (internal and external validity, data triangulation, consistency)? If so, please specify. Table 3 and the following paragraph should be put in the results section, as it is results. We have added more information in the text to describe how we conducted the analysis with and moved Table 3 and the paragraphs commenting it to the Results section (Table 4 in the new version of the manuscript). - Balanced-QPM development (Phase III) and Indicator structure design and performance measurement (Phase IV): Stay with the process (the method) and keep the results for the results section. Thanks for this comment. In the new version of the manuscript, we have kept only the description of the methods used and moved all the results to the Results section. RESULTS Your results are interesting and relevant. Create a results section separate from the methods section. Thanks for this suggestion. We have now a Results section. DISCUSSION The discussion needs to be further enriched and your results better compared with current knowledge (recent cases in literature and literature review). From my point of view, the first two paragraphs of the discussion are still part of the results (implementation results). You are right, we improved the discussion section following your recommendation. - Future work: Ok, your opening is relevant. Thanks. Some typos and errors : Introduction: “The Coordinating Committee, which is composed of a limited number of managers, aims at insuring ” (ensuring?) Right “ensuring”. Methods: “Due of this, it is challenging” (to) Indeed, “to” is the correct term to use. Conclusion: “which is has been reported in practice and in the literature as a very challenging problem” Corrected, “which has been”. In summary: - The topic is interesting, and the study mobilizes a wealth of material to propose results relevant to the field. - There are several study structuring issues that need to be addressed (structure of introduction, absence of results title, structure of discussion. A number of tips are given for this purpose. - A number of methodological points need to be reinforced in order to fully convince the reader. - The literature could be strengthened to develop the discussion on the contribution of this study. Proposals to this effect have also been made. Good luck for the next steps. Thanks for the relevant comments and suggestions that we believe significantly improved our manuscript. View more View less Competing Interests No competing interests were disclosed. reply Respond Report a concern Gartner JB. Peer Review Report For: Towards an Integrated Performance Management and Measurement System for healthcare organisations: a case study in Montreal [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 12 :1420 ( https://doi.org/10.5256/f1000research.151623.r253135) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/12-1420/v1#referee-response-253135 Alongside their report, reviewers assign a status to the article: Approved - the paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations - A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved - fundamental flaws in the paper seriously undermine the findings and conclusions Adjust parameters to alter display View on desktop for interactive features Includes Interactive Elements View on desktop for interactive features Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Stay Updated Sign up for content alerts and receive a weekly or monthly email with all newly published articles Register with F1000Research Already registered? Sign in Not now, thanks close PLEASE NOTE If you are an AUTHOR of this article, please check that you signed in with the account associated with this article otherwise we cannot automatically identify your role as an author and your comment will be labelled as a “User Comment”. If you are a REVIEWER of this article, please check that you have signed in with the account associated with this article and then go to your account to submit your report, please do not post your review here. If you do not have access to your original account, please contact us . All commenters must hold a formal affiliation as per our Policies . The information that you give us will be displayed next to your comment. User comments must be in English, comprehensible and relevant to the article under discussion. We reserve the right to remove any comments that we consider to be inappropriate, offensive or otherwise in breach of the User Comment Terms and Conditions . Commenters must not use a comment for personal attacks. When criticisms of the article are based on unpublished data, the data should be made available. I accept the User Comment Terms and Conditions Please confirm that you accept the User Comment Terms and Conditions. Affiliation ✕ refresh Please enter your institution. Note: To add your institution or organisation, start typing the name and then select the correct name from the list. Where applicable, the name will appear in both the original language and in English. Do not paste in the name. If the name does not appear in the drop-down list, we will display the information you have entered. ✕ refresh Country/Region * USA UK Canada China France Germany Afghanistan Aland Islands Albania Algeria American Samoa Andorra Angola Anguilla Antarctica Antigua and Barbuda Argentina Armenia Aruba Australia Austria Azerbaijan Bahamas Bahrain Bangladesh Barbados Belarus Belgium Belize Benin Bermuda Bhutan Bolivia Bosnia and Herzegovina Botswana Bouvet Island Brazil British Indian Ocean Territory British Virgin Islands Brunei Bulgaria Burkina Faso Burundi Cambodia Cameroon Canada Cape Verde Cayman Islands Central African Republic Chad Chile China Christmas Island Cocos (Keeling) Islands Colombia Comoros Congo Cook Islands Costa Rica Cote d'Ivoire Croatia Cuba Cyprus Czech Republic Democratic Republic of the Congo Denmark Djibouti Dominica Dominican Republic Ecuador Egypt El Salvador Equatorial Guinea Eritrea Estonia Ethiopia Falkland Islands Faroe Islands Federated States of Micronesia Fiji Finland France French Guiana French Polynesia French Southern Territories Gabon Georgia Germany Ghana Gibraltar Greece Greenland Grenada Guadeloupe Guam Guatemala Guernsey Guinea Guinea-Bissau Guyana Haiti Heard Island and Mcdonald Islands Holy See (Vatican City State) Honduras Hong Kong Hungary Iceland India Indonesia Iran Iraq Ireland Israel Italy Jamaica Japan Jersey Jordan Kazakhstan Kenya Kiribati Kosovo (Serbia and Montenegro) Kuwait Kyrgyzstan Lao People's Democratic Republic Latvia Lebanon Lesotho Liberia Libya Liechtenstein Lithuania Luxembourg Macao Madagascar Malawi Malaysia Maldives Mali Malta Marshall Islands Martinique Mauritania Mauritius Mayotte Mexico Minor Outlying Islands of the United States Moldova Monaco Mongolia Montenegro Montserrat Morocco Mozambique Myanmar Namibia Nauru Nepal Netherlands Antilles New Caledonia New Zealand Nicaragua Niger Nigeria Niue Norfolk Island North Korea North Macedonia Northern Mariana Islands Norway Oman Pakistan Palau Palestinian Territory Panama Papua New Guinea Paraguay Peru Philippines Pitcairn Poland Portugal Puerto Rico Qatar Reunion Romania Russian Federation Rwanda Saint Helena Saint Kitts and Nevis Saint Lucia Saint Pierre and Miquelon Saint Vincent and the Grenadines Samoa San Marino Sao Tome and Principe Saudi Arabia Senegal Serbia Seychelles Sierra Leone Singapore Slovakia Slovenia Solomon Islands Somalia South Africa South Georgia and the South Sandwich Is South Korea South Sudan Spain Sri Lanka Sudan Suriname Svalbard and Jan Mayen Swaziland Sweden Switzerland Syria Taiwan Tajikistan Tanzania Thailand The Gambia The Netherlands Timor-Leste Togo Tokelau Tonga Trinidad and Tobago Tunisia Turkey Turkmenistan Turks and Caicos Islands Tuvalu UK USA Uganda Ukraine United Arab Emirates United States Virgin Islands Uruguay Uzbekistan Vanuatu Venezuela Vietnam Wallis and Futuna West Bank and Gaza Strip Western Sahara Yemen Zambia Zimbabwe Please select your country/region. You must enter a comment. Competing Interests Please disclose any competing interests that might be construed to influence your judgment of the article's or peer review report's validity or importance. Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Please state your competing interests The comment has been saved. An error has occurred. Please try again. Cancel Post var lTitle = "Towards an Integrated Performance Management...".replace("'", ''); var linkedInUrl = "http://www.linkedin.com/shareArticle?url=https://f1000research.com/articles/12-1420/v2" + "&title=" + encodeURIComponent(lTitle) + "&summary=" + encodeURIComponent('Read the article by '); var deliciousUrl = "https://del.icio.us/post?url=https://f1000research.com/articles/12-1420/v2&title=" + encodeURIComponent(lTitle); var redditUrl = "http://reddit.com/submit?url=https://f1000research.com/articles/12-1420/v2" + "&title=" + encodeURIComponent(lTitle); linkedInUrl += encodeURIComponent('Ben Fradj A et al.'); var offsetTop = /chrome/i.test( navigator.userAgent ) ? 4 : -10; var addthis_config = { ui_offset_top: offsetTop, services_compact : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_expanded : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_custom : [ { name: "LinkedIn", url: linkedInUrl, icon:"/img/icon/at_linkedin.svg" }, { name: "Mendeley", url: "http://www.mendeley.com/import/?url=https://f1000research.com/articles/12-1420/v2/mendeley", icon:"/img/icon/at_mendeley.svg" }, { name: "Reddit", url: redditUrl, icon:"/img/icon/at_reddit.svg" }, ] }; var addthis_share = { url: "https://f1000research.com/articles/12-1420", templates : { twitter : "Towards an Integrated Performance Management and Measurement.... Ben Fradj A et al., published by " + "@F1000Research" + ", https://f1000research.com/articles/12-1420/v2" } }; if (typeof(addthis) != "undefined"){ addthis.addEventListener('addthis.ready', checkCount); addthis.addEventListener('addthis.menu.share', checkCount); } $(".f1r-shares-twitter").attr("href", "https://twitter.com/intent/tweet?text=" + addthis_share.templates.twitter); $(".f1r-shares-facebook").attr("href", "https://www.facebook.com/sharer/sharer.php?u=" + addthis_share.url); $(".f1r-shares-linkedin").attr("href", addthis_config.services_custom[0].url); $(".f1r-shares-reddit").attr("href", addthis_config.services_custom[2].url); $(".f1r-shares-mendelay").attr("href", addthis_config.services_custom[1].url); function checkCount(){ setTimeout(function(){ $(".addthis_button_expanded").each(function(){ var count = $(this).text(); if (count !== "" && count != "0") $(this).removeClass("is-hidden"); else $(this).addClass("is-hidden"); }); }, 1000); } close How to cite this report {{reportCitation}} Cancel Copy Citation Details $(function(){R.ui.buttonDropdowns('.dropdown-for-downloads');}); $(function(){R.ui.toolbarDropdowns('.toolbar-dropdown-for-downloads');}); $.get("/articles/acj/138430/169268") new F1000.Clipboard(); new F1000.ThesaurusTermsDisplay("articles", "article", "169268"); $(document).ready(function() { $( "#frame1" ).on('load', function() { var mydiv = $(this).contents().find("div"); var h = mydiv.height(); console.log(h) }); var tooltipLivingFigure = jQuery(".interactive-living-figure-label .icon-more-info"), titleLivingFigure = tooltipLivingFigure.attr("title"); tooltipLivingFigure.simpletip({ fixed: true, position: ["-115", "30"], baseClass: 'small-tooltip', content:titleLivingFigure + " " }); tooltipLivingFigure.removeAttr("title"); $("body").on("click", ".cite-living-figure", function(e) { e.preventDefault(); var ref = $(this).attr("data-ref"); $(this).closest(".living-figure-list-container").find("#" + ref).fadeIn(200); }); $("body").on("click", ".close-cite-living-figure", function(e) { e.preventDefault(); $(this).closest(".popup-window-wrapper").fadeOut(200); }); $(document).on("mouseup", function(e) { var metricsContainer = $(".article-metrics-popover-wrapper"); if (!metricsContainer.is(e.target) && metricsContainer.has(e.target).length === 0) { $(".article-metrics-close-button").click(); } }); var articleId = $('#articleId').val(); if($("#main-article-count-box").attachArticleMetrics) { $("#main-article-count-box").attachArticleMetrics(articleId, { articleMetricsView: true }); } }); var figshareWidget = $(".new_figshare_widget"); if (figshareWidget.length > 0) { window.figshare.load("f1000", function(Widget) { // Select a tag/tags defined in your page. In this tag we will place the widget. _.map(figshareWidget, function(el){ var widget = new Widget({ articleId: $(el).attr("figshare_articleId") //height:300 // this is the height of the viewer part. [Default: 550] }); widget.initialize(); // initialize the widget widget.mount(el); // mount it in a tag that's on your page // this will save the widget on the global scope for later use from // your JS scripts. This line is optional. //window.widget = widget; }); }); } close Error Close Add Reset F1000.MICROSERVICES.AFFILIATION = ''; $(document).ready(function () { $('.js-affiliations-form').each((index, form) => { new AffiliationForm({ formId: form.id, institutionErrorSelector: '.comment-enter-institution', departmentErrorSelector: '.comment-enter-department', placeSelector: '.js-add-comment-place', stateSelector: '.js-add-comment-state', zipCodeSelector: '.js-add-comment-zipcode', countrySelector: '.js-add-comment-country', countryErrorSelector: '.comment-enter-country', }); }); }); $(document).ready(function () { var reportIds = { "238347": 0, "238346": 0, "238351": 0, "238350": 0, "238349": 0, "238348": 0, "238355": 20, "238354": 0, "238353": 0, "238352": 0, "229399": 0, "229398": 0, "229397": 0, "229396": 0, "229403": 0, "347037": 0, "229402": 0, "347036": 0, "229401": 0, "229400": 0, "347038": 11, "229405": 0, "347035": 0, "229404": 0, "225575": 0, "225574": 0, "225573": 0, "225579": 0, "225578": 0, "225577": 0, "225576": 0, "245039": 0, "245038": 0, "225582": 0, "225581": 0, "225580": 0, "245043": 0, "245042": 0, "245041": 0, "245040": 0, "245047": 0, "245046": 0, "245045": 0, "245044": 0, "221243": 0, "221242": 0, "221241": 0, "221247": 0, "221246": 0, "221244": 0, "221251": 0, "221250": 0, "221249": 0, "221248": 0, "253131": 0, "347853": 0, "347852": 0, "347855": 0, "347854": 0, "253135": 29, "253133": 0, "347851": 0, "347850": 0, "253139": 0, "347861": 0, "253138": 0, "347860": 0, "253137": 0, "253136": 18, "253141": 0, "347859": 0, "253140": 0, "347858": 0, "350957": 0, "350956": 0, "350959": 0, "350958": 0, "219887": 8, "219886": 0, "350955": 0, "219889": 0, "219888": 0, "350961": 0, "350960": 0, "350963": 0, "350962": 0, "350969": 0, }; $(".referee-response-container,.js-referee-report").each(function(index, el) { var reportId = $(el).attr("data-reportid"), reportCount = reportIds[reportId] || 0; $(el).find(".comments-count-container,.js-referee-report-views").html(reportCount); }); var uuidInput = $("#article_uuid"), oldUUId = uuidInput.val(), newUUId = "c17408fa-c478-47b0-a923-8a7947d76e7d"; uuidInput.val(newUUId); $("a[href*='article_uuid=']").each(function(index, el) { var newHref = $(el).attr("href").replace(oldUUId, newUUId); $(el).attr("href", newHref); }); }); An innovative open access publishing platform offering rapid publication and open peer review, whilst supporting data deposition and sharing. Browse Gateways Collections How it Works Contact For Developers Cookie Notice Privacy Notice RSS Submit Your Research Follow us © 2012-2026 F1000 Research Ltd. ISSN 2046-1402 | Legal | Partner of Research4Life • CrossRef • ORCID • FAIRSharing R.templateTests.simpleTemplate = R.template(' $text $text $text $text $text '); R.templateTests.runTests(); var F1000platform = new F1000.Platform({ name: "f1000research", displayName: "F1000Research", hostName: "f1000research.com", id: "1", editorialEmail: "[email protected]", infoEmail: "[email protected]", usePmcStats: true }); $(function(){R.ui.dropdowns('.dropdown-for-authors, .dropdown-for-about, .dropdown-for-myresearch');}); // $(function(){R.ui.dropdowns('.dropdown-for-referees');}); $(document).ready(function () { if ($(".cookie-warning").is(":visible")) { $(".sticky").css("margin-bottom", "35px"); $(".devices").addClass("devices-and-cookie-warning"); } $(".cookie-warning .close-button").click(function (e) { $(".devices").removeClass("devices-and-cookie-warning"); $(".sticky").css("margin-bottom", "0"); }); $("#tweeter-feed .tweet-message").each(function (i, message) { var self = $(message); self.html(linkify(self.html())); }); $(".partner").on("mouseenter mouseleave", function() { $(this).find(".gray-scale, .colour").toggleClass("is-hidden"); }); }); Sign In Remember me Forgotten your password? Sign In Cancel Email or password not correct. Please try again Please wait... $(function(){ // Note: All the setup needs to run against a name attribute and *not* the id due the clonish // nature of facebox... $("a[id=googleSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("GOOGLE"); $("form[id=oAuthForm]").submit(); }); $("a[id=facebookSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("FACEBOOK"); $("form[id=oAuthForm]").submit(); }); $("a[id=orcidSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("ORCID"); $("form[id=oAuthForm]").submit(); }); }); If you've forgotten your password, please enter your email address below and we'll send you instructions on how to reset your password. The email address should be the one you originally registered with F1000. Email address not valid, please try again You registered with F1000 via Google, so we cannot reset your password. To sign in, please click here . If you still need help with your Google account password, please click here . You registered with F1000 via Facebook, so we cannot reset your password. To sign in, please click here . If you still need help with your Facebook account password, please click here . Code not correct, please try again Reset password Cancel Email us for further assistance. Server error, please try again. If your email address is registered with us, we will email you instructions to reset your password. If you think you should have received this email but it has not arrived, please check your spam filters and/or contact for further assistance. Please wait... Register $(document).ready(function () { signIn.createSignInAsRow($("#sign-in-form-gfb-popup")); $(".target-field").each(function () { var uris = $(this).val().split("/"); if (uris.pop() === "login") { $(this).val(uris.toString().replace(",","/")); } }); });