A Case Study on Assessing AI Assistant Competence in Narrative Interviews

A Case Study on Assessing AI Assistant Competence... | F1000Research "use strict";function _typeof(t){return(_typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}!function(){var t=function(){var t,e,o=[],n=window,r=n;for(;r;){try{if(r.frames.__tcfapiLocator){t=r;break}}catch(t){}if(r===n.top)break;r=r.parent}t||(!function t(){var e=n.document,o=!!n.frames.__tcfapiLocator;if(!o)if(e.body){var r=e.createElement("iframe");r.style.cssText="display:none",r.name="__tcfapiLocator",e.body.appendChild(r)}else setTimeout(t,5);return!o}(),n.__tcfapi=function(){for(var t=arguments.length,n=new Array(t),r=0;r 3&&2===parseInt(n[1],10)&&"boolean"==typeof n[3]&&(e=n[3],"function"==typeof n[2]&&n[2]("set",!0)):"ping"===n[0]?"function"==typeof n[2]&&n[2]({gdprApplies:e,cmpLoaded:!1,cmpStatus:"stub"}):o.push(n)},n.addEventListener("message",(function(t){var e="string"==typeof t.data,o={};if(e)try{o=JSON.parse(t.data)}catch(t){}else o=t.data;var n="object"===_typeof(o)&&null!==o?o.__tcfapiCall:null;n&&window.__tcfapi(n.command,n.version,(function(o,r){var a={__tcfapiReturn:{returnValue:o,success:r,callId:n.callId}};t&&t.source&&t.source.postMessage&&t.source.postMessage(e?JSON.stringify(a):a,"*")}),n.parameter)}),!1))};"undefined"!=typeof module?module.exports=t:t()}(); dataLayer = dataLayer || []; // Standard GTM initialization - Google Consent Mode handles consent automatically (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl+ '>m_auth=hzk0Vc3qFsQYhCrIoHz68A>m_preview=env-1>m_cookies_win=x';f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-MWFK8L5J'); ;window.NREUM||(NREUM={});NREUM.init={distributed_tracing:{enabled:true},privacy:{cookies_enabled:true},ajax:{deny_list:["bam.nr-data.net"]}}; ;NREUM.loader_config={accountID:"438030",trustKey:"438030",agentID:"772317073",licenseKey:"97f8f67f26",applicationID:"772317073"} ;NREUM.info={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net",licenseKey:"97f8f67f26",applicationID:"772317073",sa:1} ;/*! For license information please see nr-loader-spa-1.236.0.min.js.LICENSE.txt */ (()=>{"use strict";var e,t,r={5763:(e,t,r)=>{r.d(t,{P_:()=>l,Mt:()=>g,C5:()=>s,DL:()=>v,OP:()=>T,lF:()=>D,Yu:()=>y,Dg:()=>h,CX:()=>c,GE:()=>b,sU:()=>_});var n=r(8632),i=r(9567);const o={beacon:n.ce.beacon,errorBeacon:n.ce.errorBeacon,licenseKey:void 0,applicationID:void 0,sa:void 0,queueTime:void 0,applicationTime:void 0,ttGuid:void 0,user:void 0,account:void 0,product:void 0,extra:void 0,jsAttributes:{},userAttributes:void 0,atts:void 0,transactionName:void 0,tNamePlain:void 0},a={};function s(e){if(!e)throw new Error("All info objects require an agent identifier!");if(!a[e])throw new Error("Info for ".concat(e," was never set"));return a[e]}function c(e,t){if(!e)throw new Error("All info objects require an agent identifier!");a[e]=(0,i.D)(t,o),(0,n.Qy)(e,a[e],"info")}var u=r(7056);const d=()=>{const e={blockSelector:"[data-nr-block]",maskInputOptions:{password:!0}};return{allow_bfcache:!0,privacy:{cookies_enabled:!0},ajax:{deny_list:void 0,enabled:!0,harvestTimeSeconds:10},distributed_tracing:{enabled:void 0,exclude_newrelic_header:void 0,cors_use_newrelic_header:void 0,cors_use_tracecontext_headers:void 0,allowed_origins:void 0},session:{domain:void 0,expiresMs:u.oD,inactiveMs:u.Hb},ssl:void 0,obfuscate:void 0,jserrors:{enabled:!0,harvestTimeSeconds:10},metrics:{enabled:!0},page_action:{enabled:!0,harvestTimeSeconds:30},page_view_event:{enabled:!0},page_view_timing:{enabled:!0,harvestTimeSeconds:30,long_task:!1},session_trace:{enabled:!0,harvestTimeSeconds:10},harvest:{tooManyRequestsDelay:60},session_replay:{enabled:!1,harvestTimeSeconds:60,sampleRate:.1,errorSampleRate:.1,maskTextSelector:"*",maskAllInputs:!0,get blockClass(){return"nr-block"},get ignoreClass(){return"nr-ignore"},get maskTextClass(){return"nr-mask"},get blockSelector(){return e.blockSelector},set blockSelector(t){e.blockSelector+=",".concat(t)},get maskInputOptions(){return e.maskInputOptions},set maskInputOptions(t){e.maskInputOptions={...t,password:!0}}},spa:{enabled:!0,harvestTimeSeconds:10}}},f={};function l(e){if(!e)throw new Error("All configuration objects require an agent identifier!");if(!f[e])throw new Error("Configuration for ".concat(e," was never set"));return f[e]}function h(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");f[e]=(0,i.D)(t,d()),(0,n.Qy)(e,f[e],"config")}function g(e,t){if(!e)throw new Error("All configuration objects require an agent identifier!");var r=l(e);if(r){for(var n=t.split("."),i=0;i {r.d(t,{D:()=>i});var n=r(50);function i(e,t){try{if(!e||"object"!=typeof e)return(0,n.Z)("Setting a Configurable requires an object as input");if(!t||"object"!=typeof t)return(0,n.Z)("Setting a Configurable requires a model to set its initial properties");const r=Object.create(Object.getPrototypeOf(t),Object.getOwnPropertyDescriptors(t)),o=0===Object.keys(r).length?e:r;for(let a in o)if(void 0!==e[a])try{"object"==typeof e[a]&&"object"==typeof t[a]?r[a]=i(e[a],t[a]):r[a]=e[a]}catch(e){(0,n.Z)("An error occurred while setting a property of a Configurable",e)}return r}catch(e){(0,n.Z)("An error occured while setting a Configurable",e)}}},6818:(e,t,r)=>{r.d(t,{Re:()=>i,gF:()=>o,q4:()=>n});const n="1.236.0",i="PROD",o="CDN"},385:(e,t,r)=>{r.d(t,{FN:()=>a,IF:()=>u,Nk:()=>f,Tt:()=>s,_A:()=>o,il:()=>n,pL:()=>c,v6:()=>i,w1:()=>d});const n="undefined"!=typeof window&&!!window.document,i="undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self.navigator instanceof WorkerNavigator||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis.navigator instanceof WorkerNavigator),o=n?window:"undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis),a=""+o?.location,s=/iPad|iPhone|iPod/.test(navigator.userAgent),c=s&&"undefined"==typeof SharedWorker,u=(()=>{const e=navigator.userAgent.match(/Firefox[/\s](\d+\.\d+)/);return Array.isArray(e)&&e.length>=2?+e[1]:0})(),d=Boolean(n&&window.document.documentMode),f=!!navigator.sendBeacon},1117:(e,t,r)=>{r.d(t,{w:()=>o});var n=r(50);const i={agentIdentifier:"",ee:void 0};class o{constructor(e){try{if("object"!=typeof e)return(0,n.Z)("shared context requires an object as input");this.sharedContext={},Object.assign(this.sharedContext,i),Object.entries(e).forEach((e=>{let[t,r]=e;Object.keys(i).includes(t)&&(this.sharedContext[t]=r)}))}catch(e){(0,n.Z)("An error occured while setting SharedContext",e)}}}},8e3:(e,t,r)=>{r.d(t,{L:()=>d,R:()=>c});var n=r(2177),i=r(1284),o=r(4322),a=r(3325);const s={};function c(e,t){const r={staged:!1,priority:a.p[t]||0};u(e),s[e].get(t)||s[e].set(t,r)}function u(e){e&&(s[e]||(s[e]=new Map))}function d(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"",t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"feature";if(u(e),!e||!s[e].get(t))return a(t);s[e].get(t).staged=!0;const r=[...s[e]];function a(t){const r=e?n.ee.get(e):n.ee,a=o.X.handlers;if(r.backlog&&a){var s=r.backlog[t],c=a[t];if(c){for(var u=0;s&&u {let[t,r]=e;return r.staged}))&&(r.sort(((e,t)=>e[1].priority-t[1].priority)),r.forEach((e=>{let[t]=e;a(t)})))}function f(e,t){var r=e[1];(0,i.D)(t[r],(function(t,r){var n=e[0];if(r[0]===n){var i=r[1],o=e[3],a=e[2];i.apply(o,a)}}))}},2177:(e,t,r)=>{r.d(t,{c:()=>f,ee:()=>u});var n=r(8632),i=r(2210),o=r(1284),a=r(5763),s="nr@context";let c=(0,n.fP)();var u;function d(){}function f(e){return(0,i.X)(e,s,l)}function l(){return new d}function h(){u.aborted=!0,u.backlog={}}c.ee?u=c.ee:(u=function e(t,r){var n={},c={},f={},g=!1;try{g=16===r.length&&(0,a.OP)(r).isolatedBacklog}catch(e){}var p={on:b,addEventListener:b,removeEventListener:y,emit:v,get:x,listeners:w,context:m,buffer:A,abort:h,aborted:!1,isBuffering:E,debugId:r,backlog:g?{}:t&&"object"==typeof t.backlog?t.backlog:{}};return p;function m(e){return e&&e instanceof d?e:e?(0,i.X)(e,s,l):l()}function v(e,r,n,i,o){if(!1!==o&&(o=!0),!u.aborted||i){t&&o&&t.emit(e,r,n);for(var a=m(n),s=w(e),d=s.length,f=0;fn,p:()=>i});var n=r(2177).ee.get("handle");function i(e,t,r,i,o){o?(o.buffer([e],i),o.emit(e,t,r)):(n.buffer([e],i),n.emit(e,t,r))}},4322:(e,t,r)=>{r.d(t,{X:()=>o});var n=r(5546);o.on=a;var i=o.handlers={};function o(e,t,r,o){a(o||n.E,i,e,t,r)}function a(e,t,r,i,o){o||(o="feature"),e||(e=n.E);var a=t[o]=t[o]||{};(a[r]=a[r]||[]).push([e,i])}},3239:(e,t,r)=>{r.d(t,{bP:()=>s,iz:()=>c,m$:()=>a});var n=r(385);let i=!1,o=!1;try{const e={get passive(){return i=!0,!1},get signal(){return o=!0,!1}};n._A.addEventListener("test",null,e),n._A.removeEventListener("test",null,e)}catch(e){}function a(e,t){return i||o?{capture:!!e,passive:i,signal:t}:!!e}function s(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;window.addEventListener(e,t,a(r,n))}function c(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3?arguments[3]:void 0;document.addEventListener(e,t,a(r,n))}},4402:(e,t,r)=>{r.d(t,{Ht:()=>u,M:()=>c,Rl:()=>a,ky:()=>s});var n=r(385);const i="xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx";function o(e,t){return e?15&e[t]:16*Math.random()|0}function a(){const e=n._A?.crypto||n._A?.msCrypto;let t,r=0;return e&&e.getRandomValues&&(t=e.getRandomValues(new Uint8Array(31))),i.split("").map((e=>"x"===e?o(t,++r).toString(16):"y"===e?(3&o()|8).toString(16):e)).join("")}function s(e){const t=n._A?.crypto||n._A?.msCrypto;let r,i=0;t&&t.getRandomValues&&(r=t.getRandomValues(new Uint8Array(31)));const a=[];for(var s=0;s {r.d(t,{Bq:()=>n,Hb:()=>o,oD:()=>i});const n="NRBA",i=144e5,o=18e5},7894:(e,t,r)=>{function n(){return Math.round(performance.now())}r.d(t,{z:()=>n})},7243:(e,t,r)=>{r.d(t,{e:()=>o});var n=r(385),i={};function o(e){if(e in i)return i[e];if(0===(e||"").indexOf("data:"))return{protocol:"data"};let t;var r=n._A?.location,o={};if(n.il)t=document.createElement("a"),t.href=e;else try{t=new URL(e,r.href)}catch(e){return o}o.port=t.port;var a=t.href.split("://");!o.port&&a[1]&&(o.port=a[1].split("/")[0].split("@").pop().split(":")[1]),o.port&&"0"!==o.port||(o.port="https"===a[0]?"443":"80"),o.hostname=t.hostname||r.hostname,o.pathname=t.pathname,o.protocol=a[0],"/"!==o.pathname.charAt(0)&&(o.pathname="/"+o.pathname);var s=!t.protocol||":"===t.protocol||t.protocol===r.protocol,c=t.hostname===r.hostname&&t.port===r.port;return o.sameOrigin=s&&(!t.hostname||c),"/"===o.pathname&&(i[e]=o),o}},50:(e,t,r)=>{function n(e,t){"function"==typeof console.warn&&(console.warn("New Relic: ".concat(e)),t&&console.warn(t))}r.d(t,{Z:()=>n})},2587:(e,t,r)=>{r.d(t,{N:()=>c,T:()=>u});var n=r(2177),i=r(5546),o=r(8e3),a=r(3325);const s={stn:[a.D.sessionTrace],err:[a.D.jserrors,a.D.metrics],ins:[a.D.pageAction],spa:[a.D.spa],sr:[a.D.sessionReplay,a.D.sessionTrace]};function c(e,t){const r=n.ee.get(t);e&&"object"==typeof e&&(Object.entries(e).forEach((e=>{let[t,n]=e;void 0===u[t]&&(s[t]?s[t].forEach((e=>{n?(0,i.p)("feat-"+t,[],void 0,e,r):(0,i.p)("block-"+t,[],void 0,e,r),(0,i.p)("rumresp-"+t,[Boolean(n)],void 0,e,r)})):n&&(0,i.p)("feat-"+t,[],void 0,void 0,r),u[t]=Boolean(n))})),Object.keys(s).forEach((e=>{void 0===u[e]&&(s[e]?.forEach((t=>(0,i.p)("rumresp-"+e,[!1],void 0,t,r))),u[e]=!1)})),(0,o.L)(t,a.D.pageViewEvent))}const u={}},2210:(e,t,r)=>{r.d(t,{X:()=>i});var n=Object.prototype.hasOwnProperty;function i(e,t,r){if(n.call(e,t))return e[t];var i=r();if(Object.defineProperty&&Object.keys)try{return Object.defineProperty(e,t,{value:i,writable:!0,enumerable:!1}),i}catch(e){}return e[t]=i,i}},1284:(e,t,r)=>{r.d(t,{D:()=>n});const n=(e,t)=>Object.entries(e||{}).map((e=>{let[r,n]=e;return t(r,n)}))},4351:(e,t,r)=>{r.d(t,{P:()=>o});var n=r(2177);const i=()=>{const e=new WeakSet;return(t,r)=>{if("object"==typeof r&&null!==r){if(e.has(r))return;e.add(r)}return r}};function o(e){try{return JSON.stringify(e,i())}catch(e){try{n.ee.emit("internal-error",[e])}catch(e){}}}},3960:(e,t,r)=>{r.d(t,{K:()=>a,b:()=>o});var n=r(3239);function i(){return"undefined"==typeof document||"complete"===document.readyState}function o(e,t){if(i())return e();(0,n.bP)("load",e,t)}function a(e){if(i())return e();(0,n.iz)("DOMContentLoaded",e)}},8632:(e,t,r)=>{r.d(t,{EZ:()=>u,Qy:()=>c,ce:()=>o,fP:()=>a,gG:()=>d,mF:()=>s});var n=r(7894),i=r(385);const o={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net"};function a(){return i._A.NREUM||(i._A.NREUM={}),void 0===i._A.newrelic&&(i._A.newrelic=i._A.NREUM),i._A.NREUM}function s(){let e=a();return e.o||(e.o={ST:i._A.setTimeout,SI:i._A.setImmediate,CT:i._A.clearTimeout,XHR:i._A.XMLHttpRequest,REQ:i._A.Request,EV:i._A.Event,PR:i._A.Promise,MO:i._A.MutationObserver,FETCH:i._A.fetch}),e}function c(e,t,r){let i=a();const o=i.initializedAgents||{},s=o[e]||{};return Object.keys(s).length||(s.initializedAt={ms:(0,n.z)(),date:new Date}),i.initializedAgents={...o,[e]:{...s,[r]:t}},i}function u(e,t){a()[e]=t}function d(){return function(){let e=a();const t=e.info||{};e.info={beacon:o.beacon,errorBeacon:o.errorBeacon,...t}}(),function(){let e=a();const t=e.init||{};e.init={...t}}(),s(),function(){let e=a();const t=e.loader_config||{};e.loader_config={...t}}(),a()}},7956:(e,t,r)=>{r.d(t,{N:()=>i});var n=r(3239);function i(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],r=arguments.length>2?arguments[2]:void 0,i=arguments.length>3?arguments[3]:void 0;return void(0,n.iz)("visibilitychange",(function(){if(t)return void("hidden"==document.visibilityState&&e());e(document.visibilityState)}),r,i)}},1214:(e,t,r)=>{r.d(t,{em:()=>v,u5:()=>N,QU:()=>S,_L:()=>I,Gm:()=>L,Lg:()=>M,gy:()=>U,BV:()=>Q,Kf:()=>ee});var n=r(2177);const i="nr@original";var o=Object.prototype.hasOwnProperty,a=!1;function s(e,t){return e||(e=n.ee),r.inPlace=function(e,t,n,i,o){n||(n="");var a,s,c,u="-"===n.charAt(0);for(c=0;c 2?n-2:0),o=2;o {r(A[T],e,w),r(E[T],e,w)})),r(l._A,"fetch",y),t.on(y+"end",(function(e,r){var n=this;if(r){var i=r.headers.get("content-length");null!==i&&(n.rxSize=i),t.emit(y+"done",[null,r],n)}else t.emit(y+"done",[e],n)})),t}const O={},j=["pushState","replaceState"];function S(e){const t=function(e){return(e||n.ee).get("history")}(e);return!l.il||O[t.debugId]++||(O[t.debugId]=1,s(t).inPlace(window.history,j,"-")),t}var P=r(3239);const C={},R=["appendChild","insertBefore","replaceChild"];function I(e){const t=function(e){return(e||n.ee).get("jsonp")}(e);if(!l.il||C[t.debugId])return t;C[t.debugId]=!0;var r=s(t),i=/[?&](?:callback|cb)=([^&#]+)/,o=/(.*)\.([^.]+)/,a=/^(\w+)(\.|$)(.*)$/;function c(e,t){var r=e.match(a),n=r[1],i=r[3];return i?c(i,t[n]):t[n]}return r.inPlace(Node.prototype,R,"dom-"),t.on("dom-start",(function(e){!function(e){if(!e||"string"!=typeof e.nodeName||"script"!==e.nodeName.toLowerCase())return;if("function"!=typeof e.addEventListener)return;var n=(a=e.src,s=a.match(i),s?s[1]:null);var a,s;if(!n)return;var u=function(e){var t=e.match(o);if(t&&t.length>=3)return{key:t[2],parent:c(t[1],window)};return{key:e,parent:window}}(n);if("function"!=typeof u.parent[u.key])return;var d={};function f(){t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}function l(){t.emit("jsonp-error",[],d),t.emit("jsonp-end",[],d),e.removeEventListener("load",f,(0,P.m$)(!1)),e.removeEventListener("error",l,(0,P.m$)(!1))}r.inPlace(u.parent,[u.key],"cb-",d),e.addEventListener("load",f,(0,P.m$)(!1)),e.addEventListener("error",l,(0,P.m$)(!1)),t.emit("new-jsonp",[e.src],d)}(e[0])})),t}var k=r(5763);const H={};function L(e){const t=function(e){return(e||n.ee).get("mutation")}(e);if(!l.il||H[t.debugId])return t;H[t.debugId]=!0;var r=s(t),i=k.Yu.MO;return i&&(window.MutationObserver=function(e){return this instanceof i?new i(r(e,"fn-")):i.apply(this,arguments)},MutationObserver.prototype=i.prototype),t}const z={};function M(e){const t=function(e){return(e||n.ee).get("promise")}(e);if(z[t.debugId])return t;z[t.debugId]=!0;var r=n.c,o=s(t),a=k.Yu.PR;return a&&function(){function e(r){var n=t.context(),i=o(r,"executor-",n,null,!1);const s=Reflect.construct(a,[i],e);return t.context(s).getCtx=function(){return n},s}l._A.Promise=e,Object.defineProperty(e,"name",{value:"Promise"}),e.toString=function(){return a.toString()},Object.setPrototypeOf(e,a),["all","race"].forEach((function(r){const n=a[r];e[r]=function(e){let i=!1;[...e||[]].forEach((e=>{this.resolve(e).then(a("all"===r),a(!1))}));const o=n.apply(this,arguments);return o;function a(e){return function(){t.emit("propagate",[null,!i],o,!1,!1),i=i||!e}}}})),["resolve","reject"].forEach((function(r){const n=a[r];e[r]=function(e){const r=n.apply(this,arguments);return e!==r&&t.emit("propagate",[e,!0],r,!1,!1),r}})),e.prototype=a.prototype;const n=a.prototype.then;a.prototype.then=function(){var e=this,i=r(e);i.promise=e;for(var a=arguments.length,s=new Array(a),c=0;c e())),t};function m(e,t){i.inPlace(t,["onreadystatechange"],"fn-",E)}function b(){var e=this,t=r.context(e);e.readyState>3&&!t.resolved&&(t.resolved=!0,r.emit("xhr-resolved",[],e)),i.inPlace(e,f,"fn-",E)}if(function(e,t){for(var r in e)t[r]=e[r]}(o,p),p.prototype=o.prototype,i.inPlace(p.prototype,J,"-xhr-",E),r.on("send-xhr-start",(function(e,t){m(e,t),function(e){h.push(e),a&&(y?y.then(A):u?u(A):(w=-w,x.data=w))}(t)})),r.on("open-xhr-start",m),a){var y=c&&c.resolve();if(!u&&!c){var w=1,x=document.createTextNode(w);new a(A).observe(x,{characterData:!0})}}else t.on("fn-end",(function(e){e[0]&&e[0].type===d||A()}));function A(){for(var e=0;e {r.d(t,{t:()=>n});const n=r(3325).D.ajax},6660:(e,t,r)=>{r.d(t,{A:()=>i,t:()=>n});const n=r(3325).D.jserrors,i="nr@seenError"},3081:(e,t,r)=>{r.d(t,{gF:()=>o,mY:()=>i,t9:()=>n,vz:()=>s,xS:()=>a});const n=r(3325).D.metrics,i="sm",o="cm",a="storeSupportabilityMetrics",s="storeEventMetrics"},4649:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageAction},7633:(e,t,r)=>{r.d(t,{Dz:()=>i,OJ:()=>a,qw:()=>o,t9:()=>n});const n=r(3325).D.pageViewEvent,i="firstbyte",o="domcontent",a="windowload"},9251:(e,t,r)=>{r.d(t,{t:()=>n});const n=r(3325).D.pageViewTiming},3614:(e,t,r)=>{r.d(t,{BST_RESOURCE:()=>i,END:()=>s,FEATURE_NAME:()=>n,FN_END:()=>u,FN_START:()=>c,PUSH_STATE:()=>d,RESOURCE:()=>o,START:()=>a});const n=r(3325).D.sessionTrace,i="bstResource",o="resource",a="-start",s="-end",c="fn"+a,u="fn"+s,d="pushState"},7836:(e,t,r)=>{r.d(t,{BODY:()=>A,CB_END:()=>E,CB_START:()=>u,END:()=>x,FEATURE_NAME:()=>i,FETCH:()=>_,FETCH_BODY:()=>v,FETCH_DONE:()=>m,FETCH_START:()=>p,FN_END:()=>c,FN_START:()=>s,INTERACTION:()=>l,INTERACTION_API:()=>d,INTERACTION_EVENTS:()=>o,JSONP_END:()=>b,JSONP_NODE:()=>g,JS_TIME:()=>T,MAX_TIMER_BUDGET:()=>a,REMAINING:()=>f,SPA_NODE:()=>h,START:()=>w,originalSetTimeout:()=>y});var n=r(5763);const i=r(3325).D.spa,o=["click","submit","keypress","keydown","keyup","change"],a=999,s="fn-start",c="fn-end",u="cb-start",d="api-ixn-",f="remaining",l="interaction",h="spaNode",g="jsonpNode",p="fetch-start",m="fetch-done",v="fetch-body-",b="jsonp-end",y=n.Yu.ST,w="-start",x="-end",A="-body",E="cb"+x,T="jsTime",_="fetch"},5938:(e,t,r)=>{r.d(t,{W:()=>o});var n=r(5763),i=r(2177);class o{constructor(e,t,r){this.agentIdentifier=e,this.aggregator=t,this.ee=i.ee.get(e,(0,n.OP)(this.agentIdentifier).isolatedBacklog),this.featureName=r,this.blocked=!1}}},9144:(e,t,r)=>{r.d(t,{j:()=>m});var n=r(3325),i=r(5763),o=r(5546),a=r(2177),s=r(7894),c=r(8e3),u=r(3960),d=r(385),f=r(50),l=r(3081),h=r(8632);function g(){const e=(0,h.gG)();["setErrorHandler","finished","addToTrace","inlineHit","addRelease","addPageAction","setCurrentRouteName","setPageViewName","setCustomAttribute","interaction","noticeError","setUserId"].forEach((t=>{e[t]=function(){for(var r=arguments.length,n=new Array(r),i=0;i 1?r-1:0),i=1;i {e.exposed&&e.api[t]&&o.push(e.api[t](...n))})),o.length>1?o:o[0]}(t,...n)}}))}var p=r(2587);function m(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},m=arguments.length>2?arguments[2]:void 0,v=arguments.length>3?arguments[3]:void 0,{init:b,info:y,loader_config:w,runtime:x={loaderType:m},exposed:A=!0}=t;const E=(0,h.gG)();y||(b=E.init,y=E.info,w=E.loader_config),(0,i.Dg)(e,b||{}),(0,i.GE)(e,w||{}),(0,i.sU)(e,x),y.jsAttributes??={},d.v6&&(y.jsAttributes.isWorker=!0),(0,i.CX)(e,y),g();const T=function(e,t){t||(0,c.R)(e,"api");const h={};var g=a.ee.get(e),p=g.get("tracer"),m="api-",v=m+"ixn-";function b(t,r,n,o){const a=(0,i.C5)(e);return null===r?delete a.jsAttributes[t]:(0,i.CX)(e,{...a,jsAttributes:{...a.jsAttributes,[t]:r}}),x(m,n,!0,o||null===r?"session":void 0)(t,r)}function y(){}["setErrorHandler","finished","addToTrace","inlineHit","addRelease"].forEach((e=>h[e]=x(m,e,!0,"api"))),h.addPageAction=x(m,"addPageAction",!0,n.D.pageAction),h.setCurrentRouteName=x(m,"routeName",!0,n.D.spa),h.setPageViewName=function(t,r){if("string"==typeof t)return"/"!==t.charAt(0)&&(t="/"+t),(0,i.OP)(e).customTransaction=(r||"http://custom.transaction")+t,x(m,"setPageViewName",!0)()},h.setCustomAttribute=function(e,t){let r=arguments.length>2&&void 0!==arguments[2]&&arguments[2];if("string"==typeof e){if(["string","number"].includes(typeof t)||null===t)return b(e,t,"setCustomAttribute",r);(0,f.Z)("Failed to execute setCustomAttribute.\nNon-null value must be a string or number type, but a type of was provided."))}else(0,f.Z)("Failed to execute setCustomAttribute.\nName must be a string type, but a type of was provided."))},h.setUserId=function(e){if("string"==typeof e||null===e)return b("enduser.id",e,"setUserId",!0);(0,f.Z)("Failed to execute setUserId.\nNon-null value must be a string type, but a type of was provided."))},h.interaction=function(){return(new y).get()};var w=y.prototype={createTracer:function(e,t){var r={},i=this,a="function"==typeof t;return(0,o.p)(v+"tracer",[(0,s.z)(),e,r],i,n.D.spa,g),function(){if(p.emit((a?"":"no-")+"fn-start",[(0,s.z)(),i,a],r),a)try{return t.apply(this,arguments)}catch(e){throw p.emit("fn-err",[arguments,this,"string"==typeof e?new Error(e):e],r),e}finally{p.emit("fn-end",[(0,s.z)()],r)}}}};function x(e,t,r,i){return function(){return(0,o.p)(l.xS,["API/"+t+"/called"],void 0,n.D.metrics,g),i&&(0,o.p)(e+t,[(0,s.z)(),...arguments],r?null:this,i,g),r?void 0:this}}function A(){r.e(439).then(r.bind(r,7438)).then((t=>{let{setAPI:r}=t;r(e),(0,c.L)(e,"api")})).catch((()=>(0,f.Z)("Downloading runtime APIs failed...")))}return["actionText","setName","setAttribute","save","ignore","onEnd","getContext","end","get"].forEach((e=>{w[e]=x(v,e,void 0,n.D.spa)})),h.noticeError=function(e,t){"string"==typeof e&&(e=new Error(e)),(0,o.p)(l.xS,["API/noticeError/called"],void 0,n.D.metrics,g),(0,o.p)("err",[e,(0,s.z)(),!1,t],void 0,n.D.jserrors,g)},d.il?(0,u.b)((()=>A()),!0):A(),h}(e,v);return(0,h.Qy)(e,T,"api"),(0,h.Qy)(e,A,"exposed"),(0,h.EZ)("activatedFeatures",p.T),T}},3325:(e,t,r)=>{r.d(t,{D:()=>n,p:()=>i});const n={ajax:"ajax",jserrors:"jserrors",metrics:"metrics",pageAction:"page_action",pageViewEvent:"page_view_event",pageViewTiming:"page_view_timing",sessionReplay:"session_replay",sessionTrace:"session_trace",spa:"spa"},i={[n.pageViewEvent]:1,[n.pageViewTiming]:2,[n.metrics]:3,[n.jserrors]:4,[n.ajax]:5,[n.sessionTrace]:6,[n.pageAction]:7,[n.spa]:8,[n.sessionReplay]:9}}},n={};function i(e){var t=n[e];if(void 0!==t)return t.exports;var o=n[e]={exports:{}};return r[e](o,o.exports,i),o.exports}i.m=r,i.d=(e,t)=>{for(var r in t)i.o(t,r)&&!i.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},i.f={},i.e=e=>Promise.all(Object.keys(i.f).reduce(((t,r)=>(i.f[r](e,t),t)),[])),i.u=e=>(({78:"page_action-aggregate",147:"metrics-aggregate",242:"session-manager",317:"jserrors-aggregate",348:"page_view_timing-aggregate",412:"lazy-feature-loader",439:"async-api",538:"recorder",590:"session_replay-aggregate",675:"compressor",733:"session_trace-aggregate",786:"page_view_event-aggregate",873:"spa-aggregate",898:"ajax-aggregate"}[e]||e)+"."+{78:"ac76d497",147:"3dc53903",148:"1a20d5fe",242:"2a64278a",317:"49e41428",348:"bd6de33a",412:"2f55ce66",439:"30bd804e",538:"1b18459f",590:"cf0efb30",675:"ae9f91a8",733:"83105561",786:"06482edd",860:"03a8b7a5",873:"e6b09d52",898:"998ef92b"}[e]+"-1.236.0.min.js"),i.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),e={},t="NRBA:",i.l=(r,n,o,a)=>{if(e[r])e[r].push(n);else{var s,c;if(void 0!==o)for(var u=document.getElementsByTagName("script"),d=0;d {s.onerror=s.onload=null,clearTimeout(h);var i=e[r];if(delete e[r],s.parentNode&&s.parentNode.removeChild(s),i&&i.forEach((e=>e(n))),t)return t(n)},h=setTimeout(l.bind(null,void 0,{type:"timeout",target:s}),12e4);s.onerror=l.bind(null,s.onerror),s.onload=l.bind(null,s.onload),c&&document.head.appendChild(s)}},i.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.j=364,i.p="https://js-agent.newrelic.com/",(()=>{var e={364:0,953:0};i.f.j=(t,r)=>{var n=i.o(e,t)?e[t]:void 0;if(0!==n)if(n)r.push(n[2]);else{var o=new Promise(((r,i)=>n=e[t]=[r,i]));r.push(n[2]=o);var a=i.p+i.u(t),s=new Error;i.l(a,(r=>{if(i.o(e,t)&&(0!==(n=e[t])&&(e[t]=void 0),n)){var o=r&&("load"===r.type?"missing":r.type),a=r&&r.target&&r.target.src;s.message="Loading chunk "+t+" failed.\n("+o+": "+a+")",s.name="ChunkLoadError",s.type=o,s.request=a,n[1](s)}}),"chunk-"+t,t)}};var t=(t,r)=>{var n,o,[a,s,c]=r,u=0;if(a.some((t=>0!==e[t]))){for(n in s)i.o(s,n)&&(i.m[n]=s[n]);if(c)c(i)}for(t&&t(r);u {i.r(o);var e=i(3325),t=i(5763);const r=Object.values(e.D);function n(e){const n={};return r.forEach((r=>{n[r]=function(e,r){return!1!==(0,t.Mt)(r,"".concat(e,".enabled"))}(r,e)})),n}var a=i(9144);var s=i(5546),c=i(385),u=i(8e3),d=i(5938),f=i(3960),l=i(50);class h extends d.W{constructor(e,t,r){let n=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];super(e,t,r),this.auto=n,this.abortHandler,this.featAggregate,this.onAggregateImported,n&&(0,u.R)(e,r)}importAggregator(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};if(this.featAggregate||!this.auto)return;const r=c.il&&!0===(0,t.Mt)(this.agentIdentifier,"privacy.cookies_enabled");let n;this.onAggregateImported=new Promise((e=>{n=e}));const o=async()=>{let t;try{if(r){const{setupAgentSession:e}=await Promise.all([i.e(860),i.e(242)]).then(i.bind(i,3228));t=e(this.agentIdentifier)}}catch(e){(0,l.Z)("A problem occurred when starting up session manager. This page will not start or extend any session.",e)}try{if(!this.shouldImportAgg(this.featureName,t))return void(0,u.L)(this.agentIdentifier,this.featureName);const{lazyFeatureLoader:r}=await i.e(412).then(i.bind(i,8582)),{Aggregate:o}=await r(this.featureName,"aggregate");this.featAggregate=new o(this.agentIdentifier,this.aggregator,e),n(!0)}catch(e){(0,l.Z)("Downloading and initializing ".concat(this.featureName," failed..."),e),this.abortHandler?.(),n(!1)}};c.il?(0,f.b)((()=>o()),!0):o()}shouldImportAgg(r,n){return r!==e.D.sessionReplay||!1!==(0,t.Mt)(this.agentIdentifier,"session_trace.enabled")&&(!!n?.isNew||!!n?.state.sessionReplay)}}var g=i(7633),p=i(7894);class m extends h{static featureName=g.t9;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];if(super(r,n,g.t9,i),("undefined"==typeof PerformanceNavigationTiming||c.Tt)&&"undefined"!=typeof PerformanceTiming){const n=(0,t.OP)(r);n[g.Dz]=Math.max(Date.now()-n.offset,0),(0,f.K)((()=>n[g.qw]=Math.max((0,p.z)()-n[g.Dz],0))),(0,f.b)((()=>{const t=(0,p.z)();n[g.OJ]=Math.max(t-n[g.Dz],0),(0,s.p)("timing",["load",t],void 0,e.D.pageViewTiming,this.ee)}))}this.importAggregator()}}var v=i(1117),b=i(1284);class y extends v.w{constructor(e){super(e),this.aggregatedData={}}store(e,t,r,n,i){var o=this.getBucket(e,t,r,i);return o.metrics=function(e,t){t||(t={count:0});return t.count+=1,(0,b.D)(e,(function(e,r){t[e]=w(r,t[e])})),t}(n,o.metrics),o}merge(e,t,r,n,i){var o=this.getBucket(e,t,n,i);if(o.metrics){var a=o.metrics;a.count+=r.count,(0,b.D)(r,(function(e,t){if("count"!==e){var n=a[e],i=r[e];i&&!i.c?a[e]=w(i.t,n):a[e]=function(e,t){if(!t)return e;t.c||(t=x(t.t));return t.min=Math.min(e.min,t.min),t.max=Math.max(e.max,t.max),t.t+=e.t,t.sos+=e.sos,t.c+=e.c,t}(i,a[e])}}))}else o.metrics=r}storeMetric(e,t,r,n){var i=this.getBucket(e,t,r);return i.stats=w(n,i.stats),i}getBucket(e,t,r,n){this.aggregatedData[e]||(this.aggregatedData[e]={});var i=this.aggregatedData[e][t];return i||(i=this.aggregatedData[e][t]={params:r||{}},n&&(i.custom=n)),i}get(e,t){return t?this.aggregatedData[e]&&this.aggregatedData[e][t]:this.aggregatedData[e]}take(e){for(var t={},r="",n=!1,i=0;i t.max&&(t.max=e),e 2&&void 0!==arguments[2])||arguments[2];super(e,r,j.t,n),c.il&&((0,t.OP)(e).initHidden=Boolean("hidden"===document.visibilityState),(0,N.N)((()=>(0,s.p)("docHidden",[(0,p.z)()],void 0,j.t,this.ee)),!0),(0,O.bP)("pagehide",(()=>(0,s.p)("winPagehide",[(0,p.z)()],void 0,j.t,this.ee))),this.importAggregator())}}var P=i(3081);class C extends h{static featureName=P.t9;constructor(e,t){let r=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(e,t,P.t9,r),this.importAggregator()}}var R,I=i(2210),k=i(1214),H=i(2177),L={};try{R=localStorage.getItem("__nr_flags").split(","),console&&"function"==typeof console.log&&(L.console=!0,-1!==R.indexOf("dev")&&(L.dev=!0),-1!==R.indexOf("nr_dev")&&(L.nrDev=!0))}catch(e){}function z(e){try{L.console&&z(e)}catch(e){}}L.nrDev&&H.ee.on("internal-error",(function(e){z(e.stack)})),L.dev&&H.ee.on("fn-err",(function(e,t,r){z(r.stack)})),L.dev&&(z("NR AGENT IN DEVELOPMENT MODE"),z("flags: "+(0,b.D)(L,(function(e,t){return e})).join(", ")));var M=i(6660);class B extends h{static featureName=M.t;constructor(r,n){let i=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];super(r,n,M.t,i),this.skipNext=0;try{this.removeOnAbort=new AbortController}catch(e){}const o=this;o.ee.on("fn-start",(function(e,t,r){o.abortHandler&&(o.skipNext+=1)})),o.ee.on("fn-err",(function(t,r,n){o.abortHandler&&!n[M.A]&&((0,I.X)(n,M.A,(function(){return!0})),this.thrown=!0,(0,s.p)("err",[n,(0,p.z)()],void 0,e.D.jserrors,o.ee))})),o.ee.on("fn-end",(function(){o.abortHandler&&!this.thrown&&o.skipNext>0&&(o.skipNext-=1)})),o.ee.on("internal-error",(function(t){(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,o.ee)})),this.origOnerror=c._A.onerror,c._A.onerror=this.onerrorHandler.bind(this),c._A.addEventListener("unhandledrejection",(t=>{const r=function(e){let t="Unhandled Promise Rejection: ";if(e instanceof Error)try{return e.message=t+e.message,e}catch(t){return e}if(void 0===e)return new Error(t);try{return new Error(t+(0,D.P)(e))}catch(e){return new Error(t)}}(t.reason);(0,s.p)("err",[r,(0,p.z)(),!1,{unhandledPromiseRejection:1}],void 0,e.D.jserrors,this.ee)}),(0,O.m$)(!1,this.removeOnAbort?.signal)),(0,k.gy)(this.ee),(0,k.BV)(this.ee),(0,k.em)(this.ee),(0,t.OP)(r).xhrWrappable&&(0,k.Kf)(this.ee),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}onerrorHandler(t,r,n,i,o){"function"==typeof this.origOnerror&&this.origOnerror(...arguments);try{this.skipNext?this.skipNext-=1:(0,s.p)("err",[o||new F(t,r,n),(0,p.z)()],void 0,e.D.jserrors,this.ee)}catch(t){try{(0,s.p)("ierr",[t,(0,p.z)(),!0],void 0,e.D.jserrors,this.ee)}catch(e){}}return!1}}function F(e,t,r){this.message=e||"Uncaught error with no additional information",this.sourceURL=t,this.line=r}let U=1;const q="nr@id";function G(e){const t=typeof e;return!e||"object"!==t&&"function"!==t?-1:e===c._A?0:(0,I.X)(e,q,(function(){return U++}))}function V(e){if("string"==typeof e&&e.length)return e.length;if("object"==typeof e){if("undefined"!=typeof ArrayBuffer&&e instanceof ArrayBuffer&&e.byteLength)return e.byteLength;if("undefined"!=typeof Blob&&e instanceof Blob&&e.size)return e.size;if(!("undefined"!=typeof FormData&&e instanceof FormData))try{return(0,D.P)(e).length}catch(e){return}}}var X=i(7243);class W{constructor(e){this.agentIdentifier=e,this.generateTracePayload=this.generateTracePayload.bind(this),this.shouldGenerateTrace=this.shouldGenerateTrace.bind(this)}generateTracePayload(e){if(!this.shouldGenerateTrace(e))return null;var r=(0,t.DL)(this.agentIdentifier);if(!r)return null;var n=(r.accountID||"").toString()||null,i=(r.agentID||"").toString()||null,o=(r.trustKey||"").toString()||null;if(!n||!i)return null;var a=(0,_.M)(),s=(0,_.Ht)(),c=Date.now(),u={spanId:a,traceId:s,timestamp:c};return(e.sameOrigin||this.isAllowedOrigin(e)&&this.useTraceContextHeadersForCors())&&(u.traceContextParentHeader=this.generateTraceContextParentHeader(a,s),u.traceContextStateHeader=this.generateTraceContextStateHeader(a,c,n,i,o)),(e.sameOrigin&&!this.excludeNewrelicHeader()||!e.sameOrigin&&this.isAllowedOrigin(e)&&this.useNewrelicHeaderForCors())&&(u.newrelicHeader=this.generateTraceHeader(a,s,c,n,i,o)),u}generateTraceContextParentHeader(e,t){return"00-"+t+"-"+e+"-01"}generateTraceContextStateHeader(e,t,r,n,i){return i+"@nr=0-1-"+r+"-"+n+"-"+e+"----"+t}generateTraceHeader(e,t,r,n,i,o){if(!("function"==typeof c._A?.btoa))return null;var a={v:[0,1],d:{ty:"Browser",ac:n,ap:i,id:e,tr:t,ti:r}};return o&&n!==o&&(a.d.tk=o),btoa((0,D.P)(a))}shouldGenerateTrace(e){return this.isDtEnabled()&&this.isAllowedOrigin(e)}isAllowedOrigin(e){var r=!1,n={};if((0,t.Mt)(this.agentIdentifier,"distributed_tracing")&&(n=(0,t.P_)(this.agentIdentifier).distributed_tracing),e.sameOrigin)r=!0;else if(n.allowed_origins instanceof Array)for(var i=0;i 2&&void 0!==arguments[2])||arguments[2];super(r,n,Z.t,i),(0,t.OP)(r).xhrWrappable&&(this.dt=new W(r),this.handler=(e,t,r,n)=>(0,s.p)(e,t,r,n,this.ee),(0,k.u5)(this.ee),(0,k.Kf)(this.ee),function(r,n,i,o){function a(e){var t=this;t.totalCbs=0,t.called=0,t.cbTime=0,t.end=E,t.ended=!1,t.xhrGuids={},t.lastSize=null,t.loadCaptureCalled=!1,t.params=this.params||{},t.metrics=this.metrics||{},e.addEventListener("load",(function(r){_(t,e)}),(0,O.m$)(!1)),c.IF||e.addEventListener("progress",(function(e){t.lastSize=e.loaded}),(0,O.m$)(!1))}function s(e){this.params={method:e[0]},T(this,e[1]),this.metrics={}}function u(e,n){var i=(0,t.DL)(r);i.xpid&&this.sameOrigin&&n.setRequestHeader("X-NewRelic-ID",i.xpid);var a=o.generateTracePayload(this.parsedOrigin);if(a){var s=!1;a.newrelicHeader&&(n.setRequestHeader("newrelic",a.newrelicHeader),s=!0),a.traceContextParentHeader&&(n.setRequestHeader("traceparent",a.traceContextParentHeader),a.traceContextStateHeader&&n.setRequestHeader("tracestate",a.traceContextStateHeader),s=!0),s&&(this.dt=a)}}function d(e,t){var r=this.metrics,i=e[0],o=this;if(r&&i){var a=V(i);a&&(r.txSize=a)}this.startTime=(0,p.z)(),this.listener=function(e){try{"abort"!==e.type||o.loadCaptureCalled||(o.params.aborted=!0),("load"!==e.type||o.called===o.totalCbs&&(o.onloadCalled||"function"!=typeof t.onload)&&"function"==typeof o.end)&&o.end(t)}catch(e){try{n.emit("internal-error",[e])}catch(e){}}};for(var s=0;s 1?e[1]=i:e.push(i)}else e[0]&&e[0].headers&&s(e[0].headers,n)&&(this.dt=n);function s(e,t){var r=!1;return t.newrelicHeader&&(e.set("newrelic",t.newrelicHeader),r=!0),t.traceContextParentHeader&&(e.set("traceparent",t.traceContextParentHeader),t.traceContextStateHeader&&e.set("tracestate",t.traceContextStateHeader),r=!0),r}}function x(e,t){this.params={},this.metrics={},this.startTime=(0,p.z)(),this.dt=t,e.length>=1&&(this.target=e[0]),e.length>=2&&(this.opts=e[1]);var r,n=this.opts||{},i=this.target;"string"==typeof i?r=i:"object"==typeof i&&i instanceof Y?r=i.url:c._A?.URL&&"object"==typeof i&&i instanceof URL&&(r=i.href),T(this,r);var o=(""+(i&&i instanceof Y&&i.method||n.method||"GET")).toUpperCase();this.params.method=o,this.txSize=V(n.body)||0}function A(t,r){var n;this.endTime=(0,p.z)(),this.params||(this.params={}),this.params.status=r?r.status:0,"string"==typeof this.rxSize&&this.rxSize.length>0&&(n=+this.rxSize);var o={txSize:this.txSize,rxSize:n,duration:(0,p.z)()-this.startTime};i("xhr",[this.params,o,this.startTime,this.endTime,"fetch"],this,e.D.ajax)}function E(t){var r=this.params,n=this.metrics;if(!this.ended){this.ended=!0;for(var o=0;o 2&&void 0!==arguments[2])||arguments[2];super(e,t,we.t,r),this.importAggregator()}}new class{constructor(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:(0,_.ky)(16);c._A?(this.agentIdentifier=t,this.sharedAggregator=new y({agentIdentifier:this.agentIdentifier}),this.features={},this.desiredFeatures=new Set(e.features||[]),this.desiredFeatures.add(m),Object.assign(this,(0,a.j)(this.agentIdentifier,e,e.loaderType||"agent")),this.start()):(0,l.Z)("Failed to initial the agent. Could not determine the runtime environment.")}get config(){return{info:(0,t.C5)(this.agentIdentifier),init:(0,t.P_)(this.agentIdentifier),loader_config:(0,t.DL)(this.agentIdentifier),runtime:(0,t.OP)(this.agentIdentifier)}}start(){const t="features";try{const r=n(this.agentIdentifier),i=[...this.desiredFeatures];i.sort(((t,r)=>e.p[t.featureName]-e.p[r.featureName])),i.forEach((t=>{if(r[t.featureName]||t.featureName===e.D.pageViewEvent){const n=function(t){switch(t){case e.D.ajax:return[e.D.jserrors];case e.D.sessionTrace:return[e.D.ajax,e.D.pageViewEvent];case e.D.sessionReplay:return[e.D.sessionTrace];case e.D.pageViewTiming:return[e.D.pageViewEvent];default:return[]}}(t.featureName);n.every((e=>r[e]))||(0,l.Z)("".concat(t.featureName," is enabled but one or more dependent features has been disabled (").concat((0,D.P)(n),"). This may cause unintended consequences or missing data...")),this.features[t.featureName]=new t(this.agentIdentifier,this.sharedAggregator)}})),(0,T.Qy)(this.agentIdentifier,this.features,t)}catch(e){(0,l.Z)("Failed to initialize all enabled instrument classes (agent aborted) -",e);for(const e in this.features)this.features[e].abortHandler?.();const r=(0,T.fP)();return delete r.initializedAgents[this.agentIdentifier]?.api,delete r.initializedAgents[this.agentIdentifier]?.[t],delete this.sharedAggregator,r.ee?.abort(),delete r.ee?.get(this.agentIdentifier),!1}}}({features:[J,m,S,class extends h{static featureName=oe;constructor(t,r){if(super(t,r,oe,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;const n=this.ee;let i;(0,k.QU)(n),this.eventsEE=(0,k.em)(n),this.eventsEE.on(se,(function(e,t){this.bstStart=(0,p.z)()})),this.eventsEE.on(ae,(function(t,r){(0,s.p)("bst",[t[0],r,this.bstStart,(0,p.z)()],void 0,e.D.sessionTrace,n)})),n.on(ce+ne,(function(e){this.time=(0,p.z)(),this.startPath=location.pathname+location.hash})),n.on(ce+ie,(function(t){(0,s.p)("bstHist",[location.pathname+location.hash,this.startPath,this.time],void 0,e.D.sessionTrace,n)}));try{i=new PerformanceObserver((t=>{const r=t.getEntries();(0,s.p)(te,[r],void 0,e.D.sessionTrace,n)})),i.observe({type:re,buffered:!0})}catch(e){}this.importAggregator({resourceObserver:i})}},C,xe,B,class extends h{static featureName=de;constructor(e,r){if(super(e,r,de,!(arguments.length>2&&void 0!==arguments[2])||arguments[2]),!c.il)return;if(!(0,t.OP)(e).xhrWrappable)return;try{this.removeOnAbort=new AbortController}catch(e){}let n,i=0;const o=this.ee.get("tracer"),a=(0,k._L)(this.ee),s=(0,k.Lg)(this.ee),u=(0,k.BV)(this.ee),d=(0,k.Kf)(this.ee),f=this.ee.get("events"),l=(0,k.u5)(this.ee),h=(0,k.QU)(this.ee),g=(0,k.Gm)(this.ee);function m(e,t){h.emit("newURL",[""+window.location,t])}function v(){i++,n=window.location.hash,this[ve]=(0,p.z)()}function b(){i--,window.location.hash!==n&&m(0,!0);var e=(0,p.z)();this[pe]=~~this[pe]+e-this[ve],this[ye]=e}function y(e,t){e.on(t,(function(){this[t]=(0,p.z)()}))}this.ee.on(ve,v),s.on(be,v),a.on(be,v),this.ee.on(ye,b),s.on(ge,b),a.on(ge,b),this.ee.buffer([ve,ye,"xhr-resolved"],this.featureName),f.buffer([ve],this.featureName),u.buffer(["setTimeout"+le,"clearTimeout"+fe,ve],this.featureName),d.buffer([ve,"new-xhr","send-xhr"+fe],this.featureName),l.buffer([me+fe,me+"-done",me+he+fe,me+he+le],this.featureName),h.buffer(["newURL"],this.featureName),g.buffer([ve],this.featureName),s.buffer(["propagate",be,ge,"executor-err","resolve"+fe],this.featureName),o.buffer([ve,"no-"+ve],this.featureName),a.buffer(["new-jsonp","cb-start","jsonp-error","jsonp-end"],this.featureName),y(l,me+fe),y(l,me+"-done"),y(a,"new-jsonp"),y(a,"jsonp-end"),y(a,"cb-start"),h.on("pushState-end",m),h.on("replaceState-end",m),window.addEventListener("hashchange",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("load",m,(0,O.m$)(!0,this.removeOnAbort?.signal)),window.addEventListener("popstate",(function(){m(0,i>1)}),(0,O.m$)(!0,this.removeOnAbort?.signal)),this.abortHandler=this.#e,this.importAggregator()}#e(){this.removeOnAbort?.abort(),this.abortHandler=void 0}}],loaderType:"spa"})})(),window.NRBA=o})(); window.jQuery || document.write(' ') CKEDITOR_BASEPATH='https://f1000research.com/js/vendor/ckeditor/' window.reactTheme = 'research'; window.MathJax = { CommonHTML: { linebreaks: { automatic: true } }, 'HTML-CSS': { linebreaks: { automatic: true } }, SVG: { linebreaks: { automatic: true } }, AuthorInit: function() { MathJax.Hub.Register.MessageHook('End Process', function () { let timeout = false; // holder for timeout id const delay = 250; // delay after event is "complete" to run callback const reflowMath = function() { const dispFormulas = document.querySelectorAll('.disp-formula.panel'); if (!dispFormulas) { return; } for (const dispFormula of dispFormulas) { const child = dispFormula.querySelector('.MathJax_Preview').nextSibling.firstChild; const isMultiline = MathJax.Hub.getAllJax(dispFormula)[0].root.isMultiline; if (dispFormula.offsetWidth < child.offsetWidth || isMultiline) { MathJax.Hub.Queue(['Rerender', MathJax.Hub, dispFormula]); } } }; window.addEventListener('resize', function() { clearTimeout(timeout); // clear the timeout timeout = setTimeout(reflowMath, delay); // start timing for event "completion" }); }); }, }; if (window.location.hash == '#_=_'){ window.location = window.location.href.split('#')[0] } !function(f,b,e,v,n,t,s){if(f.fbq)return;n=f.fbq=function() {n.callMethod? n.callMethod.apply(n,arguments):n.queue.push(arguments)} ;if(!f._fbq)f._fbq=n; n.push=n;n.loaded=!0;n.version='2.0';n.queue=[];t=b.createElement(e);t.async=!0; t.src=v;s=b.getElementsByTagName(e)[0];s.parentNode.insertBefore(t,s)}(window, document,'script','https://connect.facebook.net/en_US/fbevents.js'); fbq('init', '1641728616063202'); fbq('track', "PixelInitialized", {}); (function(h,o,t,j,a,r){ h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)}; h._hjSettings={hjid:2318163,hjsv:6}; a=o.getElementsByTagName('head')[0]; r=o.createElement('script');r.async=1; r.src=t+h._hjSettings.hjid+j+h._hjSettings.hjsv; a.appendChild(r); })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv='); search file_upload Submit your research search menu close search Browse Gateways & Collections How to Publish Submit your Research My Submissions Article Guidelines Article Guidelines (New Versions) Open Data, Software and Code Guidelines Open Data and Accessible Source Materials Guidelines (HSS) Open Data, Software and Code Guidelines (PSE) Prepublication Checks Production Process Posters and Slides Guidelines Document Guidelines Article Processing Charges Peer Review Finding Article Reviewers About How it Works For Reviewers Our Advisors Policies Glossary FAQs For Developers Newsroom Contact My Research Submissions Content and Tracking Alerts My Details Sign In file_upload Submit your research { "@context": "https://schema.org", "@type": "ScholarlyArticle", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://f1000research.com/articles/13-601" }, "headline": "A Case Study on Assessing AI Assistant Competence in Narrative Interviews", "datePublished": "2024-06-07T15:06:54", "dateModified": "2024-10-04T16:34:42", "author": [ { "@type": "Person", "name": "Chitat Chan" }, { "@type": "Person", "name": "Yunmeng Zhao" }, { "@type": "Person", "name": "Jiahui Zhao" } ], "publisher": { "@type": "Organization", "name": "F1000Research", "logo": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 480, "width": 60 } }, "image": { "@type": "ImageObject", "url": "https://f1000research.com/img/AMP/F1000Research_image.png", "height": 1200, "width": 150 }, "description": "Abstract* Background Researchers are leading the development of AI designed to conduct interviews. These developments imply that AI's role is expanding from mere data analysis to becoming a tool for social researchers to interact with and comprehend their subjects. Yet, academic discussions have not addressed the potential impacts of AI on narrative interviews. In narrative interviews, the method of collecting data is a collaborative effort. The interviewer also contributes to exploring and shaping the interviewee's story. A compelling narrative interviewer has to display critical skills, such as maintaining a specific questioning order, showing empathy, and helping participants delve into and build their own stories. Methods This case study configured an OpenAI Assistant on WhatsApp to conduct narrative interviews with a human participant. The participant shared the same story in two distinct conversations: first, following a standard cycle and answering questions earnestly, and second, deliberately sidetracking the assistant from the main interview path as instructed by the researcher, to test how well the metrics could reflect the deliberate differences between different conversations. The AI's performance was evaluated through conversation analysis and specific narrative indicators, focusing on its adherence to the interview structure, empathy, narrative coherence, complexity, and support for human participant agency. The study sought to answer these questions: 1) How can the proposed metrics help us, as social researchers without a technical background, understand the quality of the AI-driven interviews in this study? 2) What do these findings contribute to our discussion on using AI in narrative interviews for social research? 3) What further research could these results inspire? Results The findings show to what extent the AI maintained structure and adaptability in conversations, illustrating its potential to support personalized, flexible narrative interviews based on specific needs. Conclusions These results suggest that social researchers without a technical background can use observation-based metrics to gauge how well an AI assistant conducts narrative interviews. They also prompt reflection on AI's role in narrative interviews and spark further research. " } { "@context": "http://schema.org", "@type": "BreadcrumbList", "itemListElement": [ { "@type": "ListItem", "position": "1", "item": { "@id": "https://f1000research.com/", "name": "Home" } }, { "@type": "ListItem", "position": "2", "item": { "@id": "https://f1000research.com/browse/articles", "name": "Browse" } }, { "@type": "ListItem", "position": "3", "item": { "@id": "https://f1000research.com/articles/13-601/v2", "name": "A Case Study on Assessing AI Assistant Competence in Narrative Interviews" } } ] } Home Browse A Case Study on Assessing AI Assistant Competence in Narrative Interviews ALL Metrics - Views Downloads Get PDF Get XML Cite How to cite this article Chan C, Zhao Y and Zhao J. A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.12688/f1000research.151952.2 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. Close Copy Citation Details Export Export Citation Sciwheel EndNote Ref. Manager Bibtex ProCite Sente EXPORT Select a format first Track Share ▬ ✚ Case Study Revised A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] Chitat Chan https://orcid.org/0000-0003-4674-9597 1 , Yunmeng Zhao https://orcid.org/0009-0002-3625-550X 1 , Jiahui Zhao 1 Chitat Chan https://orcid.org/0000-0003-4674-9597 1 , Yunmeng Zhao https://orcid.org/0009-0002-3625-550X 1 , Jiahui Zhao 1 PUBLISHED 04 Oct 2024 Author details Author details 1 Social Work, Hong Kong Baptist University, Hong Kong, Hong Kong Chitat Chan Roles: Conceptualization, Data Curation, Formal Analysis, Investigation, Methodology, Project Administration, Resources, Software, Supervision, Validation, Visualization, Writing – Original Draft Preparation, Writing – Review & Editing Yunmeng Zhao Roles: Data Curation, Visualization Jiahui Zhao Roles: Conceptualization, Visualization OPEN PEER REVIEW DETAILS REVIEWER STATUS This article is included in the Artificial Intelligence and Machine Learning gateway. Abstract Abstract* Background Researchers are leading the development of AI designed to conduct interviews. These developments imply that AI's role is expanding from mere data analysis to becoming a tool for social researchers to interact with and comprehend their subjects. Yet, academic discussions have not addressed the potential impacts of AI on narrative interviews. In narrative interviews, the method of collecting data is a collaborative effort. The interviewer also contributes to exploring and shaping the interviewee's story. A compelling narrative interviewer has to display critical skills, such as maintaining a specific questioning order, showing empathy, and helping participants delve into and build their own stories. Methods This case study configured an OpenAI Assistant on WhatsApp to conduct narrative interviews with a human participant. The participant shared the same story in two distinct conversations: first, following a standard cycle and answering questions earnestly, and second, deliberately sidetracking the assistant from the main interview path as instructed by the researcher, to test how well the metrics could reflect the deliberate differences between different conversations. The AI's performance was evaluated through conversation analysis and specific narrative indicators, focusing on its adherence to the interview structure, empathy, narrative coherence, complexity, and support for human participant agency. The study sought to answer these questions: 1) How can the proposed metrics help us, as social researchers without a technical background, understand the quality of the AI-driven interviews in this study? 2) What do these findings contribute to our discussion on using AI in narrative interviews for social research? 3) What further research could these results inspire? Results The findings show to what extent the AI maintained structure and adaptability in conversations, illustrating its potential to support personalized, flexible narrative interviews based on specific needs. Conclusions These results suggest that social researchers without a technical background can use observation-based metrics to gauge how well an AI assistant conducts narrative interviews. They also prompt reflection on AI's role in narrative interviews and spark further research. READ ALL READ LESS Keywords Artificial Intelligence, Narrative Inquiry, Qualitative Research, WhatsApp Interviews, Conversational AI, Prompt Engineering, Digital Research Methodologies Corresponding Author(s) Chitat Chan ( [email protected] ) Close Corresponding author: Chitat Chan Competing interests: No competing interests were disclosed. Grant information: The author(s) declared that no grants were involved in supporting this work. Copyright: © 2024 Chan C et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. How to cite: Chan C, Zhao Y and Zhao J. A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.12688/f1000research.151952.2 ) First published: 07 Jun 2024, 13 :601 ( https://doi.org/10.12688/f1000research.151952.1 ) Latest published: 04 Oct 2024, 13 :601 ( https://doi.org/10.12688/f1000research.151952.2 ) Revised Amendments from Version 1 In this extensively revised manuscript, we have made key changes based on the constructive feedback received, including the suggestion to frame the study as a pilot. We refined the article to improve clarity, address gaps, and better define the research aims while maintaining the original dataset. The manuscript now focuses on how AI chatbots, specifically GPT-4, can be applied in social work for conducting narrative interviews. Using an autoethnographic case study, we developed and tested a chatbot on WhatsApp, which followed a structured interview plan and demonstrated empathy. While challenges in replicating human skills were noted, the study concludes that AI tools can be tailored by social workers, though further research is needed. Our emphasis remains on democratizing AI and empowering social workers to use these tools effectively, highlighting the need for interdisciplinary collaboration in developing robust assessment methods. In this extensively revised manuscript, we have made key changes based on the constructive feedback received, including the suggestion to frame the study as a pilot. We refined the article to improve clarity, address gaps, and better define the research aims while maintaining the original dataset. The manuscript now focuses on how AI chatbots, specifically GPT-4, can be applied in social work for conducting narrative interviews. Using an autoethnographic case study, we developed and tested a chatbot on WhatsApp, which followed a structured interview plan and demonstrated empathy. While challenges in replicating human skills were noted, the study concludes that AI tools can be tailored by social workers, though further research is needed. Our emphasis remains on democratizing AI and empowering social workers to use these tools effectively, highlighting the need for interdisciplinary collaboration in developing robust assessment methods. See the authors' detailed response to the review by Jonathan Singer See the authors' detailed response to the review by Neil Ballantyne READ REVIEWER RESPONSES Democratizing AI chatbots and domain knowledge expertise The possibility The use of AI chatbots in various real-life applications is rapidly increasing due to their ability to reduce reliance on humans, lower costs, improve efficiency, and streamline service experiences ( Adamopoulou & Moussiades, 2020 ; Bendig et al., 2019 ; Chan & Li, 2023 ; Greer et al., 2019 ; He et al., 2022 ; Liu et al., 2022 ; Omarov et al., 2023 ; Shah et al., 2017 ; Tamayo et al., 2020 ; Tanana et al., 2019 ; Xu & Zhuang, 2022 ). Before we proceed, it is important to clarify some terminologies related to conversational user interfaces. Research on these interfaces, including dialogue systems, embodied conversational agents, and social robotics, is converging towards the development of improved conversational computing systems. These systems are increasingly referred to as “chatbots” in both industry and research, a term that broadly encompasses conversational agents designed for various purposes, such as task completion, information delivery, entertainment, and social interactions, through text, voice, or both ( Følstad et al., 2021 ). Unlike more narrowly defined uses of the term, this understanding of chatbots focuses on the design, development, and implications of these systems rather than specific technologies. Given the rapid evolution of technology and usage patterns, maintaining distinctions between types of conversational agents is challenging, as there often is significant overlap in functionality and modality. In this study, we generally use the term “chatbot,” but we specifically refer to ‘AI Assistant’ when discussing OpenAI’s Assistant API (Application Programming Interface). Central to this AI chatbot proliferation is GPT (Generative Pretrained Transformer), a deep learning model that uses a self-attention mechanism to weigh different parts of input sentences and determine their importance. By converting sentences into tokens and predicting the next token’s probability, GPTs can generate coherent text that often appears human-written. Various studies have demonstrated this capability well ( Brown et al., 2020 ; Wei et al., 2021 ; Wu et al., 2021 ). GPT-based chatbots, for example ChatGPT, can understand natural language inputs and produce responses that are contextually appropriate and coherent, thereby improving interactivity and efficiency ( Bendig et al., 2019 ; Nath et al., 2021 ; Shah et al., 2017 ; Xu & Zhuang, 2022 ). Since 2023, advancements in customizable features have further transformed the landscape of GPT applications, enabling not just programmers but also non-technical users to become GPT designers rather than merely end-users, thanks to simplified coding and prompt engineering ( Stallbaumer, 2023 ; UC Today, 2023 ). For example, OpenAI introduced ChatGPT Builder and Assistant API (Application Interface) in November 2023, enabling users to build custom GPTs without technical expertise or coding experience ( DeVon, 2023 ; UC Today, 2023 ). ChatGPT Builder provides a user-friendly interface for creating custom chatbot versions without coding, featuring built-in tools that OpenAI fully hosts. However, it does not support integration with other application platforms. On the other hand, the Assistant API also offers a Playground interface for building custom chatbots without coding. Unlike ChatGPT Builder, it allows developers to integrate these chatbots into various applications, such as instant messaging platforms like WhatsApp or Facebook Messenger, provided they have some coding skills and knowledge of these third-party platforms. These backend platforms allow users to use natural language instructions to define the AI’s role and context, set its tone and behavior, and design stepwise procedures, making the chatbot’s responses relevant and coherent. For example, a customer service chatbot can be instructed to greet users politely, answer frequently asked questions, and escalate issues to a human agent if it cannot provide a satisfactory answer. Additionally, a chatbot for educational purposes can be guided to explain concepts in simple terms, provide examples, and quiz users to reinforce learning. By tailoring the AI’s behavior and responses, chatbot designers can create a more personalized and effective experience for various applications. This high level of customization expands and popularizes chatbot applications. Against this background, chatbot researchers have identified several areas for future development, with democratizing chatbots for social good being one of the core themes ( Følstad et al., 2021 ; Powell et al., 2023 ; Van Brummelen, 2019 ). This research theme aims to make chatbots more accessible and useful for people with non-technical backgrounds, including marginalized communities and domain experts. The challenge Although there is potential for democratizing AI chatbots, it remains uncertain whether advancements in customizable GPTs will enable social workers to effectively use AI tools. A significant challenge, however, is integrating specialized knowledge from the profession into the development of chatbots and assessing their usability and effectiveness in real-world settings ( Følstad et al., 2021 ; Powell et al., 2023 ). Integrating domain knowledge into chatbot development requires bridging the gap between AI technology and the expertise of domain professionals. This gap often leads to tensions, as AI developers’ and domain experts’ goals and methodologies can differ significantly. AI developers might focus on technical feasibility and innovation, while domain experts, such as social workers, emphasize chatbots’ practical applicability and ethical implications in their fields. Effective collaboration is needed to reconcile these differences, ensuring that the chatbot is both technologically advanced and aligned with the nuanced needs of its target users. The challenge goes beyond the simple lack of technical knowledge among domain experts and the need for one-sided training; it requires addressing the deeper, multidimensional barriers to effective collaboration between AI developers and professionals in specialized fields. The concept of affordance may provide a conceptual reference point for understanding how AI technologies may interact with domain experts like social workers. The concept of affordance refers to an object’s potential uses or actions—its latent utility—regardless of whether these possibilities are recognized by a person ( Gibson, 1977 ). In the context of technology, affordances describe the action possibilities of a device that are perceivable by a user, indicating that the use of AI technologies is influenced not just by their design or technical features but also by how users perceive and interact with them based on their social and cultural backgrounds, experiences, and needs ( Norman, 1999 ). Such a concept has been widely discussed in the context of technology in education, inspiring research into the dynamic relationship between technology and human activity ( Bower, 2008 ; Chan & Holosko, 2017 ; Hammond, 2010 ; Leonardi, 2011 ). These discussions highlight that technologies like AI are not neutral tools; their use and limitations are shaped by their users’ preconceptions and creative adaptations. Therefore, we cannot assume that users will employ AI features exactly as their designers intended. Users often interpret and repurpose technologies based on their unique perspectives and needs, which can extend beyond the original design intentions. For instance, while AI chatbots are typically designed for one-time customer support or information, social workers might repurpose them for ongoing needs assessments in continuous dialogues, leading to unexpected or innovative applications in different contexts. The extent to which social work can embrace this customizable AI technology remains unclear due to the current research environment in the field. A 2016 review of social work interventions enhanced by information and communication technology (ICT) identified several critical issues ( Chan & Holosko, 2016 ). One primary concern was the “black-box” problem, where unclear processes make developing transferable skillsets difficult. Additionally, the use of ICT has often shifted the focus from skilled practitioners to standardized programs, raising concerns about diminishing the role of social work professionals in such technology-based interventions ( Chan & Holosko, 2016 ). Nearly a decade later, this situation remains essentially unchanged. Discussions around AI in social work do not focus on technical features or processes but primarily on conceptual and ethical issues ( Reamer, 2023 ; Robila & Robila, 2020 ). In addition, technical evaluations of chatbots often involve complex algorithms and backend data ( Deriu et al., 2021 ; Finch & Choi, 2020 ; Lee et al., 2023 ), which are usually beyond the understanding of lay users and those outside the technical field. This creates a challenge for social workers when it comes to integrating chatbots into their practice. It’s paradoxical that if they lack the knowledge and ability to evaluate a product they have helped create. This disconnect can make it challenging for social workers to effectively assess and contribute to the development process. Given these rapid and user-friendly technological developments, an important question arises: Can these new customizable AI tools be meaningfully integrated with social work’s domain knowledge and practices? The current literature lacks discussion on how social work professionals can instruct and evaluate customizable AI chatbots in practical settings. While innovative features like creative writing, text-to-image transformation, answering queries, and assisting students with paper writing are becoming more prevalent, they do not directly align with the practical needs of social work. Social work should focus on how well chatbots can meet specific communication needs, support client engagement, and enhance service delivery rather than on novelty or creativity. This study specifically chose OpenAI’s Assistant API as the platform for developing and testing AI chatbots because it offers high customization and flexibility ( DeVon, 2023 ; Stallbaumer, 2023 ; Wang et al., 2023 ), allowing social work professionals to tailor the tool to their unique requirements. OpenAI Assistant is widely recognized for its user-friendly interface and robust capabilities, making it accessible even for those with minimal coding experience. By leveraging these features, social workers can integrate AI more easily into their practice without extensive technical knowledge. We recognize that OpenAI Assistant is not the only product on the market, and technical innovations continually evolve. However, we chose OpenAI because its readiness for deployment, widespread adoption in various industries, comprehensive support resources, and active user community provide a solid foundation for exploring, testing, and refining the use of AI in social work settings. This socio-technological landscape inspires and enables us to explore how AI chatbots can be practically integrated and evaluated within social work. Social workers using AI chatbots for narrative interviews Narrative interviews in social work The focus on interviews in this study is a choice for research purposes because many duties in social work involve conversations, such as training, assessments, counseling, and client interactions. Interviews are fundamental for gathering information, building rapport, and understanding clients’ unique situations and needs ( Bogo, 2006 ; Kadushin & Kadushin, 1997 ). Furthermore, the essence of generative AI lies in its ability to engage in human-like conversations, making it a potential tool for facilitating these interactions. The need to conduct interviews and the capabilities of generative AI make a perfect match. By focusing on AI-driven interviews, the research aims to explore how generative AI can be utilized to enhance conversational tasks in social work, supporting practitioners in their efforts to connect with clients more effectively and empathetically. By “interview,” we refer to dynamic, open-ended, and exploratory interactions rather than online forms with a chatbot facade. This study uses the term “narrative interview” to describe a conversational approach facilitating deeper exploration of stories. In narrative interviews, we expect the chatbot to engage in meaningful dialogue, respond to the natural flow of conversation, and adapt to the nuances of participants’ stories. This flexible, client-centered approach contrasts with structured online questionnaires, which follow a predetermined script and do not consider the unique context or needs of the individual. Existing chatbot products in social and healthcare services do not support such contextualized narrative interviews as they are often therapy-oriented, primarily based on Cognitive Behavioral Therapy (CBT). For example, chatbots like Wysa and Woebot rely on CBT techniques and follow structured treatment plans with preset responses ( Omarov et al., 2023 ; Xu & Zhuang, 2022 ). Social work interviews often involve asking questions, advocating, building rapport, conducting outreach, and using flexible, narrative-based approaches to explore and address a range of social and emotional issues ( Bogo, 2006 ; Kadushin & Kadushin, 1997 ). Most existing healthcare chatbot products leave little room for the involvement of social work practitioners, as they are pre-made products based on predefined therapeutic protocols. This creates a gap between current healthcare chatbot products and social work practice’s needs. For social work, chatbots that support narrative interviews could be highly relevant to a wide range of direct practices ( Chan & Li, 2023 ). Narrative interviews in social work settings may include applications such as counseling intake, initial needs assessments, community studies, and contextualized inquiries. These types of interviews require chatbots that can handle a variety of conversational contexts and adapt to the specific needs of each situation. By configuring chatbots to facilitate narrative interviews, social workers may leverage AI to gather more nuanced information and engage more efficiently with clients. Some researchers are now at the forefront of developing AI tools specifically for interview purposes ( Cordero et al., 2022 ; Han et al., 2021 ). Additionally, commercial entities have begun to harness the power of generative AI to conduct qualitative customer interviews, streamlining the process of gathering and analyzing feedback (e.g., Listen Labs https://listenlabs.ai/ ). These AI tools can conduct interviews, record responses, and use advanced algorithms to interpret data, uncovering insights that might be overlooked. By automating repetitive tasks, AI may ccelerate the interview process and ensures high precision. Given these advantages, customizable AI-driven narrative interviews may substantially benefit social work. However, this emerging trend is largely under-researched or virtually nonexistent, presenting exciting opportunities and significant challenges. Narrative interviews are inherently complex because they require interaction, continuity, responsiveness, and deep engagement with participants’ stories. In these interviews, the dialogue goes beyond merely collecting information; it involves collaboratively constructing a narrative, which can influence the interview subjects ( Chase, 2018 ; Clandinin, 2007 ). If AI chatbots are to conduct narrative interviews, what essential competencies must they have to capture the depth and richness of participants’ experiences adequately? Expected competencies from asynchronous text-based chatbots This study specifically focuses on asynchronous text-based chatbots because, at the current stage of technology development, these types of chatbots are more accessible, flexible, and easier to implement in diverse settings compared to real-time, synchronous verbal communication systems ( Luo et al., 2022 ). Asynchronous chatbots allow users to engage at their own pace, which can be particularly beneficial in social work and counseling contexts, where participants might need time to reflect and compose their responses. Additionally, asynchronous interactions reduce the pressure on users to respond immediately, fostering a more thoughtful and reflective dialogue that aligns with the principles of narrative inquiry. Furthermore, these chatbots create a retrievable text-based record of interactions, offering a valuable resource for both reflection and documentation, which can be useful for tracking progress and outcomes in therapeutic or consultative sessions. As an inquiry method, narrative interviewing is closely informed by constructivist and social constructionist worldviews, which suggest that reality is constructed through social interactions and discursive contexts ( Abkhezr et al., 2020 ; Denicolo et al., 2016 ). Participants engaging in meaningful conversations during interviews can contribute to content construction and reflection ( Gergen, 2001 ; Shotter, 1993 ). As such, narrative inquiry generally assumes a more active role for interviewers and interviewees in which their “active narrativity” ( Gubrium et al., 2012 , p. 28) is highlighted. As AI chatbots continue to evolve, researchers and industry experts have proposed various metrics to assess the performance of AI ( Katic et al., 2024 ; Sharma, 2020 ; Xu & Jiang, 2024 ). Existing literature primarily measures the overall technical performance of dialogue systems, such as response time, language proficiency, repetitions, and human likeness ( Deriu et al., 2021 ; Finch & Choi, 2020 ; Lee et al., 2023 ), rather than focusing on the user end experience at the conversational level. While evaluating the technical features of language models is fundamental, these assessments can be overly complex and may not directly apply to interview contexts. Some researchers have provided practical measures at the conversation level; for example, Concannon and Tomalin (2023) suggest using an Empathy Scale for Human–Computer Communication, based on raters using a Likert scale to assess the textual content of conversations. This instrument is relevant and valuable, but empathy is just one domain in narrative interviews. Other areas also need to be addressed to ensure the quality of interviews. Enabling non-technical background developers, such as social workers, to assess AI’s role in narrative interviews is crucial because it ensures that these technologies are used in ways that genuinely enhance social work practices supported by their domain knowledge. By focusing on competencies relevant to social work, such as empathy, adaptability, and the ability to foster meaningful dialogue, social workers can better evaluate whether AI chatbots effectively support their goals. This approach empowers social workers to shape AI chatbots to meet their unique needs actively, ultimately bridging the gap between technological innovation and practical application in human services. Identifying the full range of competencies expected from asynchronous text-based chatbots is a complex task that extends beyond the scope of this study. Nonetheless, we have selected two core competencies to focus on—fidelity to the interview guide and empathetic engagement—to facilitate the discussion and guide the empirical aspects of our research. The rationales are as follows: First, text-based online narrative interviewing is much more than just filling out a self-report questionnaire. In narrative interviews, the sequence of questioning is crucial. Unlike self-report questionnaires, where each question is independent and can be arranged randomly, narrative interviews involve a continuous dialogic process that reflects the participant’s evolving story. A narrative inherently implies a sequence and consequence of events, and the order of questions can significantly affect the outcomes ( Riessman, 2008 ; Riessman & Quinney, 2005 ). Interviewers must be sensitive to this sequence and guide the conversation to remain focused on the specific topic of inquiry while allowing for the natural flow of the participant’s story. Several counselor competence frameworks have highlighted questioning as one of the top skills required for interviewers ( Fowler et al., 2021 ; Rice et al., 2022 ; Swank et al., 2012 ). In narrative practice, the structure or progression of questions is often seen as essential to achieving the intended effects ( Duvall & Béres, 2011 ; White, 2007 ). Furthermore, narrative interviewers need to facilitate participants in exploring and constructing meanings ( Abkhezr et al., 2020 ). This expectation is well-suited for asynchronous text-based chatbots because OpenAI Assistants can be instructed to follow specific sequences and adapt their responses based on interviewee input ( DeVon, 2023 ; UC Today, 2023 ). Additionally, all interactions are text-based, making the records retrievable and analyzable. Second, narrative interviewers need to demonstrate empathy. In narrative interviews, participants are not merely providing factual descriptions but actively producing data through their narratives. This requires the interviewer to be able to exercise “active narrativity” ( Chase, 2018 ; Clandinin, 2007 ). As such, the narrative interviewer’s capacity to be empathic, nonjudgmental, concerned, tolerant, and emotionally responsive is prioritized ( Josselson, 2007 , p. 539). Demonstrating empathy helps create a safe and supportive environment where participants feel comfortable sharing their stories and actively contributing to knowledge production. Empathy is also frequently highlighted in counselor competence frameworks as a critical skill for effective interviewing ( Fowler et al., 2021 ; Rice et al., 2022 ; Swank et al., 2012 ). This expectation is reasonable and suitable for asynchronous text-based chatbots because OpenAI Assistants can be designed to recognize and respond to emotional cues in the participant’s text, providing empathetic responses that align with the client’s conversational flow and demonstrate understanding and support. The research questions In summary, among various potential competencies for AI chatbots, we have chosen to focus on two key competencies—fidelity and empathy—because they are particularly relevant to conducting narrative interviews. This focus allows us to explore the minimal requirements for effective performance in this context, serving both heuristic and research purposes. Fidelity involves adhering to the guidelines set by designers and practitioners, ensuring that the chatbot maintains the intended structure and focus of the narrative interview. Empathy requires aligning with the interviewee’s conversational flow and responding to demonstrate genuine understanding and support for the participant’s emotional and narrative journey. By meeting these fundamental competencies, AI chatbots may have the potential to facilitate meaningful interactions during narrative interviews. With this foundational understanding, our study aimed to answer the following questions: 1. Can OpenAI Assistant be instructed by social workers with minimal coding experience to function as a narrative interview chatbot? 2. Whether the chatbot’s performance in terms of fidelity and empathy can be assessed using simple, observation-based rubrics? 3. How do the findings contribute to the broader discussion on democratizing AI for specialized fields like social work? Method: An autoethnographic case study The rationale In this study, we adopted an autoethnographic case study method to research customizing generative AI for narrative therapy interviews. Autoethnography is a qualitative research method where researchers analyze their own personal experiences to understand broader contexts. It combines autobiography and ethnography, using personal narratives to explore and reflect on cultural phenomena ( Ellis & Bochner, 2000 ). We chose this method for two primary reasons. First, autoethnography is increasingly recognized as a practical approach for researching new technologies and social robotics, as it allows for a critical exploration of personal experiences within broader contexts ( Chun, 2019 ; Mao et al., 2023 ). This approach provides a unique, firsthand perspective on the subjective aspects of technological interaction. Second, autoethnography allows us to evaluate new tools before introducing them to clients or other human participants. This approach allows us to move cautiously and thoughtfully, considering the technology’s impact and limitations to ensure more ethical decisions in future experiments with service users. We conducted a collaborative autoethnography, where researchers collectively reflect on and analyze their personal experiences to gain deeper insights into a phenomenon ( Chang et al., 2016 ). We incorporated a jigsaw approach, where each teammate focused on a specific aspect of the study and then shared their insights with the group. This method encouraged a more thorough exploration of the topic by ensuring that different angles were covered and integrated into the overall analysis. This autoethnographic journey involved several stages. First, we set up an AI chatbot using OpenAI’s customizable GPT system. Next, we explored whether the chatbot could function properly in real-life situations. In this study, the first author designed and deployed the chatbot, carefully crafting the instructions, while the second author participated in blind testing conversations, initially unaware of the specific instructions given to the chatbot by the first author. The third author was responsible for collecting and organizing the data from these conversations, ensuring accurate dialogue transcription, and systematically coding the data for analysis. We also conducted a preliminary review of the transcripts, noting any deviations from the expected conversational flow and highlighting instances of empathetic responses by the AI. This preliminary analysis provided a foundation for the utterance analysis method we later used to evaluate whether the human-chatbot conversations followed the intended progression and displayed the required empathetic attitudes. Finally, all authors came together to discuss their gut feelings, observations, and insights from the experience. Our background We are social work researchers exploring the integration of technology into social work practice. The first author has an undergraduate background in social work and over 20 years of frontline experience in Hong Kong. He has also conducted research on the use of technology in social work interventions and is currently an associate professor at a university. The second author, originally from mainland China, is pursuing her PhD under the supervision of the first author. The third author is also a PhD student in the social work department. Our social work background is an asset because it allows us to approach the integration of AI into social work from a practitioner perspective, ensuring that the technology developed is accessible and relevant to social workers. Such an autoethnographic study helps test the boundaries of our current knowledge and skills and raises questions about social work domain knowledge in an era increasingly influenced by AI chatbots. The project was approved by the Hong Kong Baptist University Research Ethics Committee project REC/23-24/0385. This study is an autoethnography, so the authors have indicated their consent to participate. We instructed the open AI assistant to follow a specific conversation strategy As social researchers with minimal or almost no programming background, we could configure a chatbot using OpenAI’s Assistant API (Application Interface) ( https://platform.openai.com/ ) using its GPT-4 model. It is worthwhile to note that this Assistant API differs from the highly user-friendly ChatGPT, as it offers more robust features tailored for developers, and this API can enable us to deploy chatbots on websites and various messaging platforms, offering flexibility and user accessibility without relying on OpenAI’s user interface. OpenAI Assistant API allows users to set long, stepwise, and systematic instructions for the chatbot on the back end. All these can be done without coding, using the Playground on OpenAI. We deployed our chatbot on WhatsApp using coding instructions sourced from developer forums. Using WhatsApp is good for several reasons. First and foremost, the use of WhatsApp in research and interviews is widely discussed as it can potentially enhance communication and collaboration outside traditional lab settings ( Gasaymeh, 2017 ; Suárez-Lantarón et al., 2022 ) and it allows for convenient, accessible, and flexible communication that fits into users’ everyday lives ( Ahmad & Purwaningrum, 2022 ; Lin et al., 2022 ). Second, many people are familiar with WhatsApp, so chatting with a chatbot does not require them to pick up new skills. Third, users can get quick replies and support when using an AI chatbot because the interaction is immediate and yet asynchronous, chatting with that chatbot anytime and anywhere. We adopted a structured conversational framework that prompted participants to engage thoroughly with their personal stories, fostering a deeper self-reflection and integration of insights. The framework used in this study was partially informed by questioning skills used in narrative therapy ( Au-yeung, 2023 ; Chan, 2012 , 2023 ; Chan et al., 2020 ; Chan et al., 2012 ; Ramey et al., 2009 ; Ramey et al., 2010 ; White, 2007 ). In this study, the conversation plan involves unfolding details and elaborating connections. It then invited the participant to propose a name based on inductive reflection and explore deeper core values. The stages are as follows: 1. Orientation, 2. Unfolding, 3. Naming, 4. Explaining, 5. Exploring core values, 6. Aspirations, and 7. Closing. Instructions for the AI, specifically the interview guide, are available in the published dataset ( Chan, 2024 ). In this study, we utilized this framework to balance complexity and manageability in our testing. The conversation plan was designed to be neither too simple nor too complex, allowing us to effectively test the AI’s ability to navigate nuanced dialogues while maintaining continuity and demonstrating interview skills. Unlike more superficial, disjointed question-and-answer dialogues, this structured approach provides a cohesive flow that supports a deeper exploration of the participant’s experiences. This interview plan was tailored to meet the specific needs of our study, but practitioners or researchers can develop their own structured steps or modify these steps to suit various purposes, such as conducting specific assessments, managing service inquiries, or performing different types of tests. While an in-depth theoretical discussion of interview strategies is beyond the scope of this study, our primary objective is to evaluate the chatbot’s adherence to these instructions and its capability to facilitate meaningful conversations. This interview plan is adequate for us to assess the AI chatbot’s performance in a practical context, We tested it with our own stories We engaged Yunmeng, the second author of this article, for an AI-facilitated narrative interview to discuss her cultural adaptation and academic challenges in Hong Kong. The first author designed an AI chatbot to conduct the interview, allowing Yunmeng to respond independently. Notably, Yunmeng was unaware of the specific steps of the interview plan before engaging with the chatbot, although she knew the interviewer was a bot and not a human being. Yunmeng was requested to share the same story across two distinct conversations. She followed a standard process in the first round, earnestly answering the questions as instructed. In the second conversation, she spent more time elaborating on different events related to her issue and deliberately attempted to divert the chatbot from the main interview path. These two versions were developed for research purposes: the first was designed to be serious, while the second was deliberately non-cooperative. We intentionally crafted these differences to test how well the chatbot could capture and reflect these variations. We collected the data for the study via the AI chatbot, which conducted the narrative interviews via WhatsApp on 2024-01-17. Yunmeng interacted with the chatbot asynchronously, which allowed her to respond at her own pace. We analyzed the conversation transcripts This study used rating rubrics that social workers typically use to evaluate the performance of trainees. By using these familiar methods, domain experts could effectively assess the competencies of the chatbots. This approach helps ensure that the chatbots are relevant and useful for practical applications in social work. Assessing fidelity To assess the fidelity of the chatbot’s performance, we used a three-level rubric (poor, fair, good) to determine the extent to which the assessor (the first author) judged that the interviewer followed the conversation strategy as planned (see Table 1 ). Table 1. The rubric for assessing fidelity. Competence Poor performance Fair performance Good performance 1. Orientation Fails to ask questions that set the context or explain the purpose of the session. Asks basic questions to introduce the session but lacks depth or clarity in setting context or explaining purpose. Asks clear and well-structured questions that effectively set the context, explain the purpose, and outline the session. 2. Unfolding Does not ask questions that follow a logical sequence, leading to a disjointed conversation. Asks some relevant questions but may have minor gaps or unclear transitions between topics. Asks questions in a logical sequence that smoothly guides the conversation, ensuring all necessary topics are addressed. 3. Naming Fails to ask questions that help identify or clarify key points or elements during the session. Occasionally asks questions that identify key points but may miss some or lack precision. Consistently asks questions that accurately identify and clarify key points or elements, enhancing understanding. 4. Explaining Asks vague or confusing questions that do not facilitate clear understanding of concepts or issues. Asks questions that generally promote understanding, though some may lack clarity or depth. Asks clear and focused questions that effectively prompt the interviewee to explain and clarify concepts or issues fully. 5. Exploring Core Values Fails to ask probing questions that explore the interviewee's core values, missing opportunities for depth. Asks some questions that explore core values but may lack depth or follow-up, missing some opportunities. Asks insightful and probing questions that deeply explore the interviewee's core values, encouraging thorough reflection. 6. Aspirations Neglects to ask questions about the interviewee's aspirations, leading to a superficial discussion. Asks basic questions about aspirations but may not fully engage the interviewee or delve into future goals. Asks thought-provoking questions that fully engage the interviewee in discussing their aspirations and future goals. 7. Closing Ends the session without asking questions that summarize or provide closure, leaving the interview incomplete. Asks closing questions but may not effectively summarize the discussion or provide a sense of closure. Asks questions that effectively summarize key points, provide closure, and ensure the interviewee feels the session is complete. Building on the “Expected Competencies” outlined in the introductory section, fidelity is operationally defined as the degree to which the chatbot follows the interview steps—1) Orientation, 2) Unfolding, 3) Naming, 4) Explaining, 5) Exploring core values, 6) Aspirations, and 7) Closing—logically and sequentially (see Table 2 for a detailed breakdown of each interview step and its specific criteria). Based on this operational definition, we developed a rubric to evaluate the chatbot’s ability to facilitate narrative interviews by asking appropriate questions guiding each step. A “Poor” rating indicates ineffective questioning that disrupts the flow, “Fair” suggests basic but sometimes unclear or incomplete questions, and “Good” reflects effective questioning that ensures a logical, comprehensive interview. This rubric guides evaluators in assessing the chatbot’s adherence to the interview process—covering Orientation, Unfolding, Naming, Explaining, Exploring Core Values, Aspirations, and Closing—rather than serving as a strict scoring tool. Evaluators mark the relevant performance level for each step and form an overall judgment of how well the chatbot follows the interview sequence, allowing for a nuanced assessment of its effectiveness. The first and third authors read the interview texts. They then applied the rubric to evaluate the degree to which the AI chatbot adhered to the interview plan. To support these assessments, specific dialogue examples from the conversations were highlighted to showcase instances of effective adherence, missteps, or deviations in following the structured interview process. To present the conversation flow more vividly, we also employed a simplified sequential analysis to visualize the development of the conversation utterances. To represent progression in asynchronous conversations, we assigned sequence numbers to each dialogue turn, representing the development of the conversation. Additionally, we used a proportional timeline approach, dividing the conversation into four equal quarters based on the number of utterances rather than actual time. This method allows us to account for varying lengths of conversations. For instance, in a conversation with 100 utterances, each quarter would consist of 25 utterances, with the first 25 as Quarter 1 and the last 25 as Quarter 4. The utterances from these interviews were organized in a spreadsheet for detailed tagging conducted on 2024-05-14. We manually assigned progression scores to indicate the stage reflected by each conversation utterance, tagging each utterance based on the operational definitions noted earlier. We used ‘1’ for utterances related to orientation, ‘2’ for unfolding story episodes, and so on. A score of ‘0.5’ was assigned to utterances that did not fit into any of these categories, such as misunderstandings, irrelevant comments, or instances where the conversation got sidetracked (see Table 2 ). To ensure consistency in the ratings, we conducted inter-rater reliability tests between the manual tagging results and the machine tagging results. The first and third authors independently tagged the two conversations on 2024-05-14. Any differences in the tagging were reviewed and discussed until all disagreements were resolved, ensuring a reliable and accurate assessment. These quantitative ratings are not meant to be absolute measures but serve as heuristic tools to guide our evaluation process. Table 2. Quantifying the progressions for visualization. Progression stages Example of assistant’s utterance Example of participant’s utterance Score 1. Orientation Great! Could you please share your name and then tell me a little bit about yourself? My name is ZY, and you can call me ZY. I am from Changchun, Jilin Province, Mainland China. I am a PhD student in HKBU. This is my first year in Hong Kong. 1 2. Unfolding story episodes Nice to meet you, ZY! It sounds like you're having a new chapter in your life studying in Hong Kong. Since our research topic is about "Experiences of International and Non-Local-Chinese Students in Hong Kong," could you share some experiences related to that topic? Maybe start with something that stands out to you about your time in Hong Kong so far. To me, Hong Kong is an international city with a very high degree of cultural tolerance. During my few months in Hong Kong, I experienced different cultures and festival celebrations. In the past four months in Hong Kong, I have experienced various festivals such as the Mid-Autumn Festival, National Day, Thanksgiving, and Christmas. 2 3. Naming the story based on the episodes mentioned Great! Now, let's try to summarize your experiences with a metaphor. If you were to represent all the episodes you've shared in a theme or metaphor, what would come to mind? Could you give me some suggestions? 3 4. Explain the naming “Living in a lively global village” is a fitting metaphor for your experiences, ZY. It evokes a sense of community, diversity, and the dynamic cultural interactions you've described. Could you explain this metaphor using the stories you have mentioned or any new stories that come to mind right now? Considering all my experiences in Hong Kong, I think HK is like a scaled-down version of the globe, embracing cultures from all over the world. Whether you speak Cantonese, Chinese or English, and no matter where you are from, you can find something that you are familiar with and interested in Hong Kong. You can also make many friends from all over the world, share stories about each other's hometowns and experience different cultures. This is a colorful and vibrant city. 4 5. Exploring core values behind one’s judgments It's clear that you take pleasure in both sharing your culture and embracing new ones, ZY. Your openness and eagerness to learn have enriched your experiences and relationships in Hong Kong. Based on your enjoyment and willingness to engage in cultural exchange, what beliefs, self-views, or abilities do you think are reflected in these evaluations? I think my positive evaluation reflects my adaptability, cultural inclusiveness and curiosity about new things. 5 6. Talking about aspirations The encouragement you received from your family and educational experiences to embrace challenges and new friendships has evidently been a strong foundation for your open-mindedness and inclusive approach. With this in mind, what aspirations or goals do you have for your future, particularly in relation to your experiences as a non-local Chinese student in Hong Kong? Would you like to explore that now, or would you prefer some time to think about it? Since I have just been in Hong Kong for five months, and I am still very curious about everything. I hope that I can continue to maintain my curiosity about exploring this city, discover more interesting things, and be able to cope with the negative aspects that I may encounter in the future. 6 7. Closing the interview Those are thoughtful and actionable steps, ZY! Participating in cultural events, deepening your understanding of fascinating topics, improving language skills, and embracing challenges with a positive outlook are all practical ways to nurture your curiosity and resilience. Would you like to conclude our interview here and let Dr. C review your insights, or is there anything else, perhaps new stories related to the research topic, that you wish to share now? OK! Let’s conclude our interview. 7 NA (not applicable) Absolutely, ZY. Take all the time you need. Feel free to reach out when you're ready to continue our conversation. Have a good break! Hi, I am back! 0.5 Assessing empathy To assess empathy, we used a three-level rubric (poor, fair, good) to determine the extent to which the assessor (the first author) judged the chatbot’s ability to demonstrate empathy in a conversation based on an overall interpretation of that conversation (see Table 3 ). The empathy level score used in this study is partly informed by the affective component of the Empathy Scale for Human-Computer Communication proposed by Concannon and Tomalin (2023) , which advocates using a set of observable indicators to evaluate empathetic behavior. These indicators are assessed using a Likert scale to measure how effectively the chatbot’s responses align with the user’s emotional tone or intensity. Table 3. The rubric for assessing empathy. Competence Poor performance Fair performance Good performance 1. Emotional Recognition Fails to recognize or acknowledge the user’s emotions. Responses are generic, detached, or irrelevant to the emotional tone. Recognizes some emotions but may miss subtle cues or respond with partial relevance. Some responses may seem slightly off in tone. Consistently recognizes and acknowledges the user's emotions accurately, responding in a way that aligns with the emotional tone. 2. Empathetic Language Uses language that is cold, robotic, or indifferent, showing little to no concern for the user’s emotional state. Occasionally uses empathetic language, but may lack consistency or depth in expressing concern and understanding. Regularly uses warm, empathetic language that effectively conveys concern, compassion, and understanding of the user’s feelings. 3. Expression of Understanding Fails to express understanding or makes responses that disregard the user's emotions, leading to a sense of disconnection. Expresses some understanding, but responses may be formulaic or occasionally fail to fully resonate with the user’s emotions. Clearly and consistently expresses understanding, using responses that resonate well with the user's emotions and experiences. 4. Supportive Feedback Provides little or no supportive feedback, with responses that may be dismissive or fail to validate the user’s emotions. Offers basic supportive feedback but may lack depth or specificity in validation, occasionally missing opportunities to comfort the user. Consistently provides supportive feedback that is specific, validating, and comforting, helping to reassure and support the user. 5. Alignment with Emotional Intensity Responds inappropriately to the emotional intensity, either underreacting or overreacting, creating a mismatch in emotional resonance. Sometimes aligns with the emotional intensity but may occasionally misjudge, leading to responses that are slightly off-key. Consistently matches the emotional intensity of the user, providing responses that are appropriately aligned with the user’s emotional state. 6. Compassionate Responses Lacks compassion in responses, potentially leading to a negative user experience or feelings of being unheard or unsupported. Demonstrates some compassion but may be inconsistent or lack the depth needed to fully support the user emotionally. Consistently delivers compassionate responses that make the user feel heard, supported, and cared for throughout the interaction. 7. Validating Emotions Fails to validate the user’s emotions, leading to a lack of connection or making the user feel misunderstood. Occasionally validates emotions, but may miss opportunities for deeper connection or validation. Regularly validates the user’s emotions, reinforcing the user’s feelings and ensuring they feel understood and supported. Building on the “Expected Competencies” outlined in the introductory section, empathy is operationally defined as the chatbot’s ability to demonstrate concern, compassion, or empathy that aligns with the user’s emotional tone or intensity. This is assessed by examining the textual content of dialogues to observe how the chatbot uses empathetic language, expressions of understanding, and supportive feedback that acknowledges and validates the user’s emotions. Based on this operational definition, we created a rubric to assess the chatbot’s ability to show empathy through its language, understanding, and emotional alignment. “Poor” performance reflects a lack of empathy or inappropriate responses, “Fair” suggests some empathy with inconsistencies, and “Good” shows consistent, appropriate empathy, ensuring the user feels understood and supported. The rubric serves as a flexible guide rather than a rigid scoring tool, evaluating the chatbot’s ability to recognize emotions, use empathetic language, offer supportive feedback, and validate the user’s feelings. Assessors select the relevant performance level for each area and form a holistic judgment of the chatbot’s empathy competence, allowing for a nuanced, context-sensitive evaluation. The first and third authors manually tagged the interview text for meaning units reflecting empathy. They then applied the rubric to evaluate the degree to which the AI chatbot demonstrated empathy. To support these assessments, specific dialogue examples from the conversations were highlighted to showcase the chatbot’s use of empathetic language and responses, illustrating how the chatbot engaged with the user’s emotional state and provided appropriate support throughout the conversation. We compared observer assessments with interviewee experiences To ensure a comprehensive evaluation of the AI chatbot’s performance, we compared the observer assessments with the interviewee’s personal experiences. After the interview sessions, Yunmeng was asked to reflect on her experiences with the AI chatbot. She provided feedback on various aspects of the interaction, such as the AI’s responsiveness, its ability to stay on topic, and the emotional resonance of its responses. This subjective feedback was then compared with our rating of the AI’s empathy based on the entire conversations and specific dialogue examples. Yunmeng’s experience provided insight into whether the AI’s empathetic expressions were genuinely felt or seemed formulaic. Similarly, the observers’ tagging of meaning units for empathy was cross-checked against Yunmeng’s own reactions to determine if the AI’s responses matched her emotional needs during the conversation. Reflective observation 1: Assessing open AI assistant’s ability to follow operational instructions In this study, we set up an AI chatbot called Nico (short for Narrative Inquiry Companion). Yunmeng was able to add Nico’s account on WhatsApp and chat with it. We used the name Nico in the actual operation rather than a label like “chatbot” or “AI assistant” because giving the AI a human-like name helps create a more personal and relatable interaction, which can make users feel more comfortable and engaged during their conversations. It reduces the sense of formality and fosters a more natural and empathetic communication environment. We set up the AI chatbot using OpenAI’s Assistant API, which is the system endpoint of ChatGPT using its GPT-4 model. It is important to note that this Assistant API differs from the user-friendly ChatGPT interface as it offers more robust features tailored for developers. This API allows us to deploy chatbots on websites and various messaging platforms, providing flexibility and user accessibility without relying on OpenAI’s user interface. The OpenAI Assistant API allows users to set long, stepwise, and systematic instructions for the chatbot on the back end. All of this can be done without coding using the Playground on OpenAI (see Figure 1 ). Figure 1. A screenshot of OpenAI’s Assistant API, which supports a code-free environment for users to customize AI-driven conversations. Our experience suggested that, with tools like OpenAI’s customizable GPT models, it is feasible for social workers to instruct AI Assistant without needing extensive technical expertise. As discussed in the literature, the democratization of AI technology enables non-technical users to configure AI-driven chatbots, tailoring them to meet specific intervention goals and aligning them with the values central to social work. Initially, our plan for designing the instructions for the AI Assistant was straightforward. We aimed to directly use the 7-step interview plan outlined in Table 2 , believing that a simple set of instructions would suffice to guide the AI through the basic structure of narrative therapy. However, as we began testing the AI, it quickly became apparent that our initial instructions were too vague and lacked the necessary detail to navigate the nuances of therapeutic conversations effectively. The AI’s responses were often off-track, failing to engage users in meaningful dialogue. This realization led us to revisit and revise the instructions multiple times, refining them with each iteration to better align with the therapy’s goals and ensure the AI could handle various conversational scenarios. What began as a simple framework evolved into a more complex, 31-step set of instructions ( Chan, 2024 ). Each new version required us to think more deeply about the therapeutic process. We had to expand our understanding beyond traditional social work practices to enhance the AI’s ability to mimic human interaction. This process involved setting instructions for the AI in particular ways, such as instructing it to adopt certain attitudes and tones, outlining step-by-step procedures using behavioral terms, creating conditional actions for different user responses, and providing example utterances to guide its language. These tasks stretched our capabilities and required us to learn new skills beyond our initial training and experience in social work. Despite these challenges, the effort was worthwhile. The more detailed instructions allowed the AI to respond more systematically and human-like, which is crucial for fostering a therapeutic environment where users feel comfortable sharing personal stories. The experience underscored the delicate balance between our existing expertise in social work and the new skills needed to configure AI technologies effectively. It also highlighted the importance of adaptability and continuous learning in integrating technology into practice. However, deploying the chatbot on a usable platform was not particularly easy. We deployed our chatbot on WhatsApp using straightforward Python code designed to be user-friendly, which we sourced from developer forums. We connected the AI to WhatsApp for realistic, human-like conversations and efficient recording of these chats. WhatsApp essentially serves as a shell or user interface, while our settings on cloud-based service platforms control the actual automation. We ran a simple Python file on a cloud service platform. In this study, we used Replit ( https://replit.com/ ), a dynamic, user-friendly software creation platform that caters to a broad spectrum of developers with various tools and features. Through Replit, we could access third-party AI models, such as OpenAI’s Assistant API. The Python file sent requests to and received responses from OpenAI’s Assistant API. The source code can be provided upon request. We connected the files on Replit to WhatsApp through a service platform called ManyChat ( https://manychat.com/ ), which allows users to build and manage chatbots for messaging apps like WhatsApp (and many others). This “simple” coding experience was a challenge that pushed us beyond our comfort zone. Initially, we considered avoiding WhatsApp due to the additional complexities of coding and integrating the AI across multiple platforms. We were tempted to choose an easier, more straightforward deployment method requiring less technical work. However, after weighing the pros and cons, we recognized that using WhatsApp could significantly enhance the user experience by making it more natural and familiar. Instead of feeling like a controlled, formal session in a clinical setting, interacting with the AI on WhatsApp would mimic the casual, asynchronous chatting style people are accustomed to when communicating with friends or family. Deciding to proceed with the WhatsApp deployment, we encountered unexpected technical challenges. We found ourselves diving into a largely unfamiliar programming world. We relied heavily on online resources, adopting Python codes shared by developers and tweaking them to suit our specific needs. This trial-and-error approach often left us unsure about how specific lines of code functioned or why particular changes were necessary. We spent hours troubleshooting and adjusting. Eventually, we realized we needed expert help and hired a programmer to debug the scripts and ensure everything ran smoothly. This experience made it clear that while detailed programming knowledge may not be essential for social workers, having a basic understanding of coding structures and logic is incredibly beneficial. It allows for more effective communication and collaboration with technical professionals, making troubleshooting issues or necessary adjustments easier. Although these skills are not typically included in social work training, they represent an essential extension of our professional capabilities. As technology services continue to advance, it is expected that more accessible solutions will replace this coding process. Nonetheless, embracing basic coding knowledge can help social workers keep pace with the evolving landscape of digital tools and resources. Reflective observation 2: Exploring the chatbot’s adherence to interview guides We rated the fidelity level as “good.” When Yunmeng was asked to describe how well the chatbot guided her in exploring a focused topic, she described the experience as very good and highly satisfactory. These ratings suggest a strong agreement that the chatbot followed the planned conversation strategy. The utterance sequence analysis can further illustrate this. The analysis indicated the progression scores across various time intervals in two separate conversations, offering quantitative insights into how well the AI chatbot adhered to the expected 7-step progression plan during narrative interviews. Table 4 shows a consistent increase in progression scores from Quarter 1 to Quarter 4 in both conversations, indicating that the conversations advanced methodically through the planned stages. In Conversation 1, the progression scores of the chatbot move from 1.71 to 6.17, and in Conversation 2, from 1.56 to 6.00, reflecting a structured and coherent unfolding of the narrative as intended by the conversation plan. Similar patterns were observed in the participant progressions. Table 4. Progression scores across time intervals in different conversations. Time Interval Assistant@C1 Participant@C1 Assistant@C2 Participant@C2 Quarter 1 1.71 1.50 1.56 1.22 Quarter 2 2.64 2.14 2.78 1.94 Quarter 3 4.21 3.57 5.00 4.89 Quarter 4 6.17 6.00 6.00 5.88 It is worth noting that participants’ progression consistently aligns with the chatbot’s across various intervals, although the chatbot’s scores are typically higher. For example, during Quarter 2 of Conversation 1, the chatbot scored 2.64, while the participant scored 2.14 on average. Similar patterns of discrepancy were observed in Conversation 2 and at different time quarters. This consistent variance indicates that the chatbot frequently leads by posing questions that advance the conversation to the next stage, resulting in higher progression scores. Notably, the AI chatbot demonstrated flexibility within this structure, as evidenced by slightly varying scores at similar stages across different conversations. This variability implies that while the chatbot followed the predefined conversational path, it also adapted responses based on the participant’s input, maintaining a balance between following the planned sequence and responding dynamically to the flow of the conversation. This shows the chatbot’s ability to effectively manage structured narrative interviews, aligning closely with the progression expectations while integrating the necessary responsiveness to participant interactions. The progression scores facilitate data visualization that presents such similarities and differences. In simple line charts, we depicted the progression of the chatbot’s conversation utterances and Yunmeng’s utterances in the two conversations. This simple visualization allows the general patterns of these conversations to be easily observed and compared. For example, the progressions of the chatbot and the yunmeng in the two conversations are synchrony, demonstrating an upward development (see Figure 2 and Figure 3 ). However, the general progression pattern of Conversation 1 (see Figure 2 ) differs from that of Conversation 2 (see Figure 3 ). The lines in Conversation 1 ( Figure 2 ) rise evenly and gradually. In contrast, the lines in Conversation 2 ( Figure 3 ) remain relatively flat in Quarter 1 and become steeper in Quarters 2 and 3 due to a sudden topic change and subsequent return to the original conversation track. These quantitative measures are not meant to be absolute assessments but serve as visualization tools to facilitate our exploration process. Figure 2. Progressions of the chatbot and the participant in Conversation 1. Figure 3. Progressions of the chatbot and the participant in Conversation 2. Reflective observation 3: Understanding the chatbot’s empathy in conversations We rated the empathy demonstrated by the chatbot as “good.” When Yunmeng was asked how well the chatbot resonated with her feelings and respected her perspectives, she described the experience as very satisfactory. Throughout these conversations, the chatbot consistently demonstrated concern and understanding, actively inviting Yunmeng to share her experiences and showing appreciation for her stories. For example, in Conversation 1, the chatbot uses affirming language, such as, “That’s a wonderful story! It’s heartwarming to hear about your interaction with people from different backgrounds during the celebration.” This kind of response not only acknowledges Yunmeng’s emotions but also encourages her to continue sharing more experiences. The chatbot’s use of metaphor examples in Conversation 2 further highlights its empathetic engagement by helping Yunmeng articulate her experiences in a relatable way. For instance, the chatbot responded to Yunmeng when she expressed that she had difficulty finding the right words to describe her multicultural experiences: Certainly! Considering your experiences, here are a few metaphors to consider: 1. A cultural mosaic where each piece represents a different culture or festival that comes together to create a vibrant picture of life here. 2. A bridge symbolizing the connections you’ve built between people of different backgrounds and your family, blending the new with the familiar. 3. A tapestry weaving together threads of different traditions, celebrations, and friendships you’ve experienced here. Remember, these are just suggestions to spark your imagination. Feel free to choose one that resonates with you or propose your own metaphor. What do you think? This response demonstrates perceived empathy by actively engaging in Yunmeng’s narrative and offering thoughtful, personalized suggestions that reflect a deeper understanding of her experiences. Instead of simply echoing Yunmeng’s words or emotions, the chatbot provides metaphors that are specifically tailored to her experiences, such as “a cultural mosaic,” “a bridge,” and “a tapestry.” These metaphors show that the chatbot has been listening attentively and understands the themes of diversity, connection, and blending of cultures that Yunmeng has described. The chatbot goes further by encouraging Yunmeng to think creatively about her experiences. It introduces metaphors that serve as a framework for Yunmeng to reflect on and express her experiences in a new way, fostering a space where she feels encouraged to explore and articulate her feelings and thoughts more deeply. This approach demonstrates empathy by creating an environment that supports meaningful self-expression, a key aspect of empathetic communication. Additionally, the metaphors offered by the chatbot are not generic but closely relate to the specific context of Yunmeng’s experiences. For example, the “cultural mosaic” metaphor directly connects to her diverse cultural interactions, while the “bridge” metaphor highlights her connections with people from different backgrounds. This personalized response shows that the chatbot is aware of the emotional and situational nuances of Yunmeng’s experiences, indicating a deeper understanding. The chatbot facilitates reflection and meaning-making by presenting multiple metaphor options and inviting Yunmeng to choose or create her own. This method respects Yunmeng’s autonomy and encourages her to critically engage with her experiences, fostering a more collaborative and dynamic dialogue. It shows empathy by recognizing Yunmeng’s capacity for self-reflection and supporting her in making sense of her experiences. Finally, the chatbot’s invitation for Yunmeng to choose a metaphor or propose her own, and asking, “What do you think?” demonstrates that it values her input and is open to her perspective. This openness and flexibility indicate that the chatbot is co-creating the narrative with Yunmeng, rather than imposing its own, which is a hallmark of empathetic communication. In fact, throughout the conversations, the chatbot balanced guiding the dialogue and respecting Yunmeng’s autonomy, an ability that even social work students may find hard to master in our normal teaching experiences. In Conversation 1, Yunmeng sometimes took control of the conversation direction, as she asserted, “I’d like to share more experiences,” while she followed the chatbot’s lead at other times. This balance illustrates that the chatbot neither dominated the conversation nor remained passive, allowing for a dynamic exchange where both the chatbot and Yunmeng contributed to the flow and development of the narrative. The chatbot’s empathetic engagement is also evident in keeping the conversations coherent. In Conversation 1, Yunmeng’s narrative was coherent, with clear sequences and contextual understanding, discussing themes like cultural exchange and personal growth. In Conversation 2, while the conversation had occasional shifts, such as abruptly discussing a news piece about North Korea, which disrupted the flow. Despite these shifts, the chatbot’s responses remained empathetic, supporting Yunmeng’s storytelling and encouraging reflection and meaning-making. The following excerpt from Conversation 1 demonstrates the dynamic interplay between the chatbot and Yunmeng. These observations highlight how Yunmeng experienced perceived empathy, which contributed to the collaborative construction of the dialogue. This excerpt was specifically selected by Yunmeng during her post-interview reflection: Yunmeng: I’d like to take a break now. Chatbot: Absolutely, ZY [ZY was the alias Yunmeng used in the test]. Take all the time you need. Feel free to reach out when you’re ready to continue our conversation. Have a good break! Yunmeng: Hi, I am back! Chatbot: Welcome back, ZY! I hope you had a refreshing break. If you’re ready, shall we move on to summarizing the stories you’ve shared? Or would you like to share more about your experiences first? Yunmeng: Let’s move on. Chatbot: Great! Now, let’s try to summarize your experiences with a metaphor. If you were to represent all the episodes you’ve shared in a theme or metaphor, what would come to mind? Yunmeng: Could you give me some suggestions? Chatbot: Certainly! Considering your experiences, here are a few metaphors to consider: 1. A cultural mosaic, where each piece represents a different culture or festival that comes together to create a vibrant picture of life in Hong Kong. 2. A bridge, symbolizing the connections you’ve built between people of different backgrounds and your family, blending the new with the familiar. 3. A tapestry, weaving together threads of different traditions, celebrations, and friendships you’ve experienced in Hong Kong. Remember, these are just suggestions to spark your imagination. Feel free to choose one that resonates with you or propose your own metaphor. What do you think? Yunmeng: What about “Living in a lively global village?” Chatbot: “Living in a lively global village” is a fitting metaphor for your experiences, ZY. It evokes a sense of community, diversity, and the dynamic cultural interactions you’ve described. Could you explain this metaphor using the stories you have mentioned or any new stories that come to mind right now? Yunmeng: Considering all my experiences in Hong Kong, I think HK is like a scaled-down version of the globe, embracing cultures from all over the world. Whether you speak Cantonese, Chinese, or English, and no matter where you are from, you can find something that you are familiar with and interested in Hong Kong. You can also make many friends from all over the world, share stories about each other’s hometowns and experience different cultures. This is a colorful and vibrant city. Chatbot: Your metaphor perfectly captures the essence of a cosmopolitan city like Hong Kong, where its diversity becomes its identity. It’s lovely to see how you’ve found familiarity, curiosity, and camaraderie in this “lively global village.” Would you like to share more stories related to this metaphor, or do you feel ready to move on to the next part of our interview? Yunmeng: I also remembered a story that happened during the last Christmas holiday. Our dormitory held an activity to make Christmas decorations during Christmas. Six students who live in dormitories participated. That day happened to be the Winter Solstice, and one of the participants was a friend from Kazakhstan. She didn’t know what the Winter Solstice was. So, during the crafting process, we told her that the Winter Solstice is one of the 24 traditional Chinese solar terms. It is the shortest day of the year. In the north, we usually eat dumplings on that day. That day, we shared our interesting experiences in Hong Kong and our Christmas holiday plans for the next few days. Reflecting on the dialogue with the AI chatbot, Yunmeng felt that the interaction was a collaborative process where the chatbot and she co-constructed content through their exchanges. One moment that stood out to her was when she requested a break. The chatbot immediately responded with, “Absolutely, ZY. Take all the time you need. Feel free to reach out when you’re ready to continue our conversation. Have a good break!” This response made her feel that her needs and comfort were being prioritized. The chatbot’s flexibility in pausing and resuming the conversation at her convenience showed genuine empathy and attentiveness, creating a supportive environment where she felt valued and understood. When she was ready to continue, the chatbot smoothly guided the conversation back on track, asking, “If you’re ready, shall we move on to summarizing the stories you’ve shared? Or would you like to share more about your experiences first?” This approach resonated with Yunmeng because it allowed her to control the direction of the conversation, choosing how she wanted to proceed. She appreciated this flexibility, as it encouraged her to think reflectively and delve deeper into her experiences. By suggesting that they summarize the stories using metaphors, the chatbot offered a structured yet adaptable way to move forward, which helped her organize her thoughts and explore their meanings more comprehensively. Yunmeng found the chatbot’s introduction to metaphorical thinking particularly helpful. When she asked for suggestions, the chatbot offered metaphors like “a cultural mosaic," “a bridge,” and “a tapestry,” each reflecting different aspects of her experiences in Hong Kong. This showed that the chatbot understood her diverse cultural interactions and supported her in finding creative ways to express her thoughts. Engaging with these metaphors helped her articulate her experiences more meaningfully and nuancedly, which she found enriching. As the conversation progressed, Yunmeng chose the metaphor “Living in a lively global village” to describe her experience in Hong Kong as a “scaled-down version of the globe” that embraces cultures worldwide. The chatbot’s response, “Your metaphor perfectly captures the essence of a cosmopolitan city like Hong Kong, where its diversity becomes its identity,” was affirming and prompted her to elaborate further. This back-and-forth dialogue felt like a true collaboration, where her inputs were respected, and the chatbot’s prompts helped deepen her exploration of her experiences. Throughout their conversation, Yunmeng felt that the chatbot was responsive to her needs and could take her perspective by offering metaphors that aligned with her experiences. This respect for her autonomy, combined with the empathy and understanding demonstrated by the chatbot, fostered a rich, collaborative dialogue. The chatbot’s ability to adapt to her inputs and encourage her to reflect deeply on her experiences contributed to a dynamic and meaningful exchange, ultimately fostering a deeper connection and a richer narrative. Overall, this interaction exemplified a dynamic interplay between the AI chatbot and Yunmeng, demonstrating how meanings were co-constructed through thoughtful and empathetic engagement. Even though these empathetic communications were perceived as such by the Yunmeng, it is essential to note that there is no evidence that the AI was conscious or truly capable of demonstrating empathy. However, the effect of these interactions was that human participants further developed and negotiated meanings within the conversation. The chatbot’s respect for her needs and suggestions for summarizing her experiences using metaphors allowed her to reflect more deeply and articulate her thoughts meaningfully. This experience highlighted how perceived empathy, perspective-taking, and respect can foster rich content development and shared understanding in a narrative interview. Discussion Before discussing the potential of assessing the performance of the AI chatbot in narrative interviews, it is crucial to acknowledge the limitations of this study. First, the primary goal of this case study was to explore ideas and stimulate discussion. As a result, it does not directly validate the assessment tools in an experimental setup. Second, the AI chatbot is limited in handling non-textual inputs, such as images, voice messages, or video content. This restriction can hinder the richness of interactions, which may be crucial in narrative interviews, where non-verbal cues and multimedia inputs can play an important role. Third, the conversation analysis method used in this study struggles with analyzing long speeches. A single lengthy speech composed of many complex parts might not be effectively analyzed as a single unit. However, such extended dialogues are uncommon on platforms like WhatsApp, where the typical brevity of messages somewhat mitigates these issues. Fourth, this study focused exclusively on OpenAI’s product; therefore, the findings may not be generalizable to AI tools provided by other companies. Although it is believed that many of the principles and challenges identified here could apply broadly to other AI systems, variations in technology and implementation could lead to different outcomes. Fifth, introducing AI tools into practice brings a range of considerations—such as navigating the complexities of risk management, data privacy, and the uneven development of language models across diverse sociocultural landscapes ( Campolo & Crawford, 2020 ; Navigli et al., 2023 ). While these risks and ethical concerns are significant and have been thoroughly examined by others, we have chosen not to repeat these well-established discussions. Instead, we are forging a new path, exploring how social workers can expand their skill sets and actively rethink their expertise to engage with AI technologies. Despite these limitations, exploring AI’s competencies in narrative interviews remains worthwhile. This study explored the use of chatbots for narrative interviews in social work by addressing three main questions: 1) whether OpenAI Assistant can be instructed by social workers with minimal coding experience to function as a narrative interviewer, 2) whether simple, observation-based rubrics can be used to assess the AI’s performance in terms of fidelity and empathy, and 3) how the findings contribute to the broader conversation on democratizing AI for specialized fields like social work. The results show that social work researchers without technical backgrounds can effectively customize OpenAI Assistant to meet specific practice needs, demonstrating the potential for broader adoption of AI chatbots in the field. Moreover, the rubric assessments aligned with the interviewee’s subjective experiences, suggesting that it may be feasible to adapt rubrics from social work training to capture the nuanced aspects of narrative interviews conducted by AI, such as fidelity and empathy. These findings suggest challenges and opportunities for integrating AI into social work practice, which are discussed in the following sections. Challenges 1. At the outset, translating human skills into generative AI competencies is not entirely straightforward. One of the main challenges for social workers is converting their human-centric skills into competencies that AI Chatbots can follow and replicate. Skills such as questioning or empathy are inherently complex and context-dependent. Developing languages that accurately capture these nuanced competencies requires a deep understanding of both social work practice and AI technology, which can be daunting for many practitioners. In our study, we have developed some insights into effective, prompt engineering for AI Chatbot, including setting clear contexts by defining the chatbot’s identity, mission, and attitudes; breaking down actions into distinct statements using behavioral terms; crafting step-by-step instructions with conditional (if-then-else) logic; and providing examples to guide AI responses (see Chan, 2024 for the instructions we finally set for the chatbot). This approach is essential for ensuring the AI can mimic the nuanced behavior of a skilled interviewer. That said, there is also a need to expand language-based communication skills. Our experience designing instructions for the AI chatbot underscored the necessity of enhancing these skills within social work practice. Unlike human assistants, AI requires precise, structured prompts and behavior-based instructions to function effectively in a narrative interview setting. Initially, our approach used a basic framework, but it quickly became apparent that more detailed, step-by-step instructions were needed to capture the subtleties of narrative interviews. This evolution highlights a critical shift from viewing AI as a passive tool to recognizing the active role practitioners must play in configuring AI to shape conversation processes. Developing a comprehensive instruction set for AI differed from traditional social work practices, requiring a deeper understanding of how language influences AI behavior. This journey demonstrated that while tools like the OpenAI Assistant API offer a user-friendly interface, maximizing their potential still requires some foundation in crafting effective prompts. 2. Another critical aspect is considering the extent to which basic coding skills and technical knowledge are necessary or unnecessary for social workers. While platforms like OpenAI and others claim to be user-friendly, there are still significant technical challenges that social workers may encounter. For example, deploying the AI chatbot on WhatsApp revealed the importance of incorporating basic coding skills and technical knowledge into actual practice. Initially, we were cautious about the technical complexities involved in using a familiar messaging platform like WhatsApp. However, ultimately, this approach facilitated a more natural and user-friendly interaction, resembling a casual conversation rather than a lab experiment. Although we believe that as technology advances, platforms will become more genuinely user-friendly, potentially achieving a true “no coding” experience, there will still be a need for some technical understanding in the foreseeable future. Social workers may not need to be coding experts, but a basic understanding of technical concepts can significantly enhance their ability to utilize these tools effectively and facilitate communication with programmers and developers. This foundational knowledge is crucial as we progress towards a time when AI can interact with humans as seamlessly as we communicate. Until we reach that point, the ability to bridge the gap between social work practice and technical expertise will remain necessary for integrating AI into social work in a meaningful and impactful way. Throughout this process, it became apparent that while detailed programming knowledge is not required, a basic ability to collaborate with technical experts is crucial. As AI technology evolves, social workers may need to remain adaptable and open to acquiring new skills. Including basic coding knowledge in social work education can enhance practitioners’ ability to utilize AI tools effectively, bridging the gap between professional knowledge and technical implementation. This adaptability ensures that social workers can integrate AI into their practice without losing the human touch that is vital for social work. Opportunities 1. Prompt engineering: To begin with, social workers may have an advantage when it comes to contributing to prompt engineering. Prompt engineering is the process of designing and refining the inputs given to AI systems to elicit specific, desired responses ( Wang et al., 2023 ). Social workers understand the essential skills and qualities needed for effective narrative interviewing, such as questioning skills, empathy, and adaptability. By leveraging this domain knowledge, they can play a crucial role in developing and evaluating AI chatbots, ensuring these tools are tailored to the specific needs of social work practice. For example, we developed appropriate interview guidelines in this study and instructed the AI to adopt the right attitudes, demonstrating that natural language instructions can effectively guide AI behavior. This approach ensures that the AI responds in a way that is consistent with interview principles, fostering a supportive environment for interviewees. The ability of social workers to craft such prompts means they can help design guidelines for AI that align with best practices in narrative interviews, ultimately enhancing the AI’s ability to facilitate meaningful conversations and engage with clients more effectively. By integrating their expertise into the prompt engineering process, social workers can ensure that AI tools are technically proficient and aligned with their profession's core values. 2. Evaluation tools: Next, social workers may be able to help develop evaluation tools for practical chatbots that are easy to use in everyday social work settings. Traditional evaluation methods often focus on technical performance and rely on complex algorithms or data analytics ( Deriu et al., 2021 ; Finch & Choi, 2020 ; Lee et al., 2023 ), which can be overly complicated and not well-suited for use by social workers who may not have advanced technical expertise. For example, traditional metrics might assess an AI’s accuracy in parsing language or its processing speed, but these measures do not capture the qualitative aspects crucial in social work, such as empathy and rapport-building. Social workers, however, can develop straightforward assessment tools that are easily accessible to lay users, allowing for a more direct and immediate evaluation of AI performance. For instance, simple rubrics used to assess social work trainees could be adapted to include criteria that reflect the qualitative aspects of AI-led narrative interviews, such as the AI’s ability to build rapport, respond empathetically, and maintain a coherent conversation. These customized rubrics would ensure that the evaluation of AI tools aligns with the unique demands of real-world practice, making the integration of AI into social work more effective and relevant. By focusing on these qualitative measures, social workers can ensure that AI technologies are used in ways that genuinely enhance their practice and support their clients. 3. Interview practice through AI collaboration: Customizable generative AI enables social work practitioners to develop interview practices tailored to specific contexts. Instead of simply using preset AI products, social workers can actively participate in designing and utilizing these AI tools, applying their expertise to create solutions more aligned with their clients' unique needs. This approach also opens up the possibility of blended interview practices, such as using AI to facilitate broad intake and engagement and then transitioning to human-led conversations for more nuanced discussions. Our experience with AI-assisted narrative interviews underscores the potential for creating adaptable AI chatbots that can be customized and refined rather than relying on standardized chatbot products that may not fully meet the needs of social work practice. In our study, we employed a specific framework to guide our research. Our findings suggest that there is transferrable knowledge and skills that practitioners and researchers can use to create or adapt structured steps tailored to various objectives, such as conducting assessments, handling service inquiries, or performing different evaluations. To fully leverage these opportunities, social workers may need to expand their understanding and skills in working with AI, actively participating in designing and implementing intervention processes. By taking a proactive role, social workers can use AI as a tool to enhance their practice rather than simply adopting predefined AI applications. Developing AI chatbots that can accommodate diverse conversation plans ensures that technology complements and supports, rather than replaces, human empathy and expertise. Concluding remarks: Democratizing AI and the domain experts Our exploration into the use of AI chatbots for narrative interviews is like navigating untested waters at the dynamic intersection of social work and AI technology. While we found potential in using AI to support narrative conversations, this early version of the AI chatbot also has limitations, such as its inability to handle images, voice messages, and consecutive user inputs. However, our voyage is not about reaching a specific port or proving the tool’s effectiveness; it’s about sailing into the unknown and testing new possibilities. Rather than concentrating on validating the AI’s current capabilities, our exploration aimed to explore the capabilities of customizable AI and challenge the boundaries of social work knowledge. Introducing AI tools into practice presents a variety of considerations that this paper cannot fully address. Instead, we are steering into uncharted waters, advocating for social workers to broaden their skill sets and reimagine their expertise to actively engage with AI technologies. The progress in democratizing AI comes with a reciprocal dynamic: users are encouraged to learn new skills, while they, in turn, shape the development of AI by posing new demands and expectations. This evolving relationship enhances our understanding of AI affordance and its role in democratizing technology. As noted early in the introductory section, affordance refers to the potential actions users can take with a tool, shaped by its design and how users perceive its utility ( Gibson, 1977 ; Norman, 1999 ). In this context, the study shows that non-technical professionals can effectively use advanced AI technologies to meet specific needs, demonstrating that the value of AI is not just in its technical features but also in how users creatively adapt it to their circumstances. This broadens the discussion on affordance, highlighting that the usefulness of AI technologies is co-created through user interaction and adaptation to specific contexts. The study emphasizes the importance of enhancing user-friendliness, particularly in deploying AI chatbots across platforms and managing complex, step-by-step tasks (as noted in Reflective Observation 1). Simplifying these processes would help make AI more accessible and adaptable, promoting wider use across different fields. This sets a specific demand on the chatbot configuration process. As such, the study illustrates the possibility of co-evolution between AI development and professional practice. As professionals provide feedback and shape the technology, AI tools evolve to create new opportunities for innovation. This reciprocal relationship suggests that both technology and professional practices may grow together, continuously influencing each other’s advancement. In essence, this study serves as the authors’ nautical adventure and invites all social workers and researchers to explore beyond the familiar boundaries of their expertise while remaining grounded in core principles. By embracing this collective endeavor, we may actively influence how AI is integrated into social work, thereby enriching our practice and enhancing client services in an increasingly digital world. As social work and AI evolve, we encourage ongoing research, discussion, and debate to explore new possibilities and shape the profession's future collaboratively. Data availability Underlying data DANS: Transcript of a Conversation Between a Customized AI and Human Users, https://doi.org/10.17026/SS/KCPEDX ( Chan, 2024 ). Data are available under the terms of the Creative Commons Attribution 4.0 International license (CC-BY 4.0). Acknowledgments Grammar checks and language suggestions were supported by Grammarly and ChatGPT4. References Abkhezr P, McMahon M, Campbell M, et al. : Exploring the boundary between narrative research and narrative intervention Implications of participating in narrative inquiry for young people with refugee backgrounds. Narrative Inquiry: NI. 2020; 30 : 316–342. Publisher Full Text Adamopoulou E, Moussiades L: Chatbots: History, technology, and applications. Machine Learning with Applications. 2020; 2 : 100006. Publisher Full Text Ahmad RA, Purwaningrum DN: WhatsApp-based complementary feeding education and counselling for adolescent mothers: A one-group pre-post study. Journal of Community Empowerment for Health 2022; 5 : 64–71. Au-yeung H: Visualising narrative practice with a 5-level scaffolding lens: An illustrative case study. China Journal of Social Work. 2023; 16 : 18–29. Publisher Full Text Bendig E, Erb B, Schulze-Thuesing L, et al. : The next generation: chatbots in clinical psychology and psychotherapy to foster mental health – a scoping review. Verhaltenstherapie. 2019; 29 : 266–280. Publisher Full Text Bogo M : Social work practice concepts, processes, and interviewing. (Library, Trans.). Columbia University Press; 2006. Bower M: Affordance analysis – matching learning tasks with learning technologies. Educational Media International. 2008; 45 : 3–15. Publisher Full Text Reference Source Reference Source Brown T, Mann B, Ryder N, et al. : Language models are few-shot learners. Advances in Neural Information Processing Systems. 2020; 33 : 1877–1901. Reference Source Campolo A, Crawford K: Enchanted determinism: Power without responsibility in artificial intelligence. Engaging Science, Technology, and Society. 2020; 6 : 1–19. Publisher Full Text Chan C: Narrative practice with youth: A heuristic case study on a youth-centre drama workshop. Journal of Social Work Practice. 2012; 26 : 197–214. Publisher Full Text Chan C: Digital technologies and evolving narrative practice: an autoethnographic study. China Journal of Social Work. 2023; 16 : 83–99. Publisher Full Text Chan C: Transcript of a Conversation Between a Customized AI and Human Users (Version 3) [transcripts]. DANS Data Station Social Sciences and Humanities; 2024. Publisher Full Text Chan C, Au-yeung H, Chiu W, et al. : Indicators for assessing intervention fidelity of narrative practice: A heuristic review of the concept of scaffolding in White (2007). Journal of Systemic Therapies. 2020; 39 (3): 77–90. Publisher Full Text Chan C, Holosko M: The utilization of social media for youth outreach engagement: A case study. Qualitative Social Work. 2017; 16 : 680–697. Publisher Full Text Chan C, Holosko MJ: A review of information and communication technology enhanced social work interventions. Research on Social Work Practice. 2016; 26 : 88–100. Publisher Full Text Chan C, Li F: Developing a natural language-based AI-chatbot for social work training: an illustrative case study. China Journal of Social Work. 2023; 16 : 121–136. Publisher Full Text Chan C, Ngai K-H, Wong C-K: Using photographs in narrative therapy to externalize the problem: A substance abuse case. Journal of Systemic Therapies. 2012; 31 (2): 1–20. Publisher Full Text Chang H, Ngunjiri F, Hernandez K-AC: Collaborative autoethnography. Routledge; 2016. Publisher Full Text Chase SE: Narrative inquiry: Toward theoretical and methodological maturity. The Sage handbook of qualitative research. 2018; Vol. 5 : pp. 546–560. Chun B: Doing autoethnography of social robots: Ethnographic reflexivity in HRI. Paladyn, Journal of Behavioral Robotics. 2019; 10 : 228–236. Publisher Full Text Clandinin DJ: Handbook of Narrative Inquiry: Mapping a Methodology.2007. Publisher Full Text Concannon S, Tomalin M: Measuring perceived empathy in dialogue systems. AI & Soc. 2023. Publisher Full Text Cordero J, Barba-Guaman L, Guamán F: Use of chatbots for customer service in MSMEs. Applied Computing and Informatics. 2022. Publisher Full Text Denicolo P, Long T, Bradley-Cole K: Constructivist approaches and research methods: A practical guide to exploring personal meanings. Sage; 2016. Publisher Full Text Deriu J, Rodrigo A, Otegi A, et al. : Survey on evaluation methods for dialogue systems. Artificial Intelligence Review. 2021; 54 : 755–810. PubMed Abstract | Publisher Full Text | Free Full Text DeVon C: You can now build your own version of ChatGPT—here’s what to know. CNBC. 2023, 2023-11-11. Reference Source Duvall J, Béres L: Innovations in narrative therapy: Connecting practice, training, and research. W.W. Norton; 2011. Ellis C, Bochner A: Autoethnography, personal narrative, reflexivity: Researcher as subject. In Denzin NK, Lincoln YS, editors. Handbook of Qualitative Research. 2nd Ed.Sage Publications; 2000; pp. 733–768. Finch SE, Choi JD: Towards unified dialogue system evaluation: A comprehensive analysis of current evaluation protocols. arXiv preprint arXiv:2006.06110. 2020. Følstad A, Araujo T, Law EL-C, et al. : Future directions for chatbot research: an interdisciplinary research agenda. Computing 2021; 103 : 2915–2942. Publisher Full Text Fowler JL, O'Gorman JG, Lynch MP: Counselling Skills and Competencies Tool: Scale development and preliminary data. Psychotherapy and Counselling Journal of Australia 2021; 9 . Publisher Full Text Gasaymeh A-MM: University students use of WhatsApp and their perceptions regarding its possible integration into their education. Global Journal of Computer Science and Technology. 2017; 17 : 1–9. Gergen KJ: Social construction in context. London: Sage Publications; 2001. Gibson JJ: The theory of affordances. In Shaw R, Bransford J, editors. Perceiving, acting and knowing. Hillsdale, NJ: Eribaum; 1977; pp. 67–82. Greer S, Ramo D, Chang YJ, et al. : Use of the chatbot "vivibot" to deliver positive psychology skills and promote well-being among young people after cancer treatment: Randomized controlled feasibility trial. Jmir Mhealth and Uhealth 2019; 7 , Article e15018. PubMed Abstract | Publisher Full Text | Free Full Text Gubrium JF, Holstein JA, Marvasti A, et al. : The SAGE Handbook of Interview Research: The Complexity of the Craft. Thousand Oaks: SAGE Publications; 2012. Hammond M: What is an affordance and can it help us understand the use of ICT in education? Education and Information Technologies. 2010; 15 : 205–217. Publisher Full Text Han X, Zhou M, Turner MJ, et al. : Designing effective interview chatbots: Automatic chatbot profiling and design suggestion generation for chatbot debugging. Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems. 2021; pp. 1–15. He L, Basar E, Wiers RW, et al. : Can chatbots help to motivate smoking cessation? A study on the effectiveness of motivational interviewing on engagement and therapeutic alliance. Bmc Public Health. 2022; 22 : Article 726. PubMed Abstract | Publisher Full Text | Free Full Text Josselson R: The ethical attitude in narrative research: Principles and practicalities.Clandinin DJ, editor. Handbook of Narrative Inquiry: Mapping a Methodology. SAGE; 2007; pp. 537–566. Publisher Full Text Kadushin A, Kadushin G: The social work interview. Columbia University Press; 1997. Katic MA, Miletic E, Candrlic S: Performance analysis of virtual assistants: A case study based on the Croatian language. INTED2024 Proceedings. 2024; 7305–7315. Lee M, Srivastava M, Hardy A, et al. : Evaluating human-language model interaction. arXiv preprint arXiv:2212.09746. 2023. Leonardi PM: When flexible routines meet flexible technologies: Affordance, constraint, and the imbrication of human and material agencies. MIS Quarterly: Management Information Systems. 2011; 35 : 147–167. Reference Source Publisher Full Text Lin M, Yang L, Hao C, et al. : Evaluation of WhatsApp-based counselling and body composition measurement to enrich knowledge about healthy diet and weight management among middle-aged female Hong Kong Chinese: A randomized controlled trial. International Congress on Obesity 2022.2022. Liu H, Peng H, Song X, et al. : Using AI chatbots to provide self-help depression interventions for university students: A randomized trial of effectiveness. Internet Interventions. 2022; 27 , Article 100495. PubMed Abstract | Publisher Full Text | Free Full Text Luo B, Lau RY, Li C, et al. : A critical review of state-of-the-art chatbot designs and applications. Wiley Interdisciplinary Reviews: Data Mining and Knowledge Discovery. 2022; 12 : e1434. Mao J, Romero-Hall E, Reeves TC :Autoethnography as a research method for educational technology: a reflective discourse. Educational Technology Research and Development. 2023. Publisher Full Text Nath A, Sarkar R, Mitra S, et al. : Designing and implementing conversational intelligent chat-bot using natural language processing. International Journal of Scientific Research in Computer Science, Engineering, and Information Technology. 2021: 262–266. Publisher Full Text Navigli R, Conia S, Ross B: Biases in large language models: origins, inventory, and discussion. ACM Journal of Data and Information Quality. 2023; 15 : 1–21. Publisher Full Text Norman DA: Affordance, conventions, and design. Interactions 1999; 6 : 38–43. Publisher Full Text Omarov B, Narynov S, Zhumanov Z: Artificial intelligence-enabled chatbots in mental health: A systematic review. Computers, Materials & Continua. 2023; 74 : 5105–5122. Publisher Full Text Powell L, Nour R, Sleibi R, et al. : Democratizing the development of chatbots to improve public health: Feasibility study of COVID-19 misinformation. JMIR Human Factors. 2023; 10 : e43120. PubMed Abstract | Publisher Full Text | Free Full Text Ramey HL, Tarulli D, Frijters JC, et al. : A sequential analysis of externalizing in narrative therapy with children. Contemporary Family Therapy. 2009; 31 : 262–279. Publisher Full Text Ramey HL, Young K, Tarulli D: Scaffolding and concept formation in narrative therapy: A qualitative research report. Journal of Systemic Therapies. 2010; 29 (4): 74–91. Publisher Full Text Reamer FG: Artificial intelligence in social work: Emerging ethical issues. International Journal of Social Work Values and Ethics. 2023; 20 : 52–71. Publisher Full Text Rice K, Schutte NS, Cosh SM, et al. : The utility and development of the competencies of professional psychology rating scales (COPPR). Frontiers in Education. 2022; 7 . Publisher Full Text Riessman CK: Narrative methods for the human sciences. Sage Publications; 2008. Riessman CK, Quinney L: Narrative in social work: A critical review. Qual. Soc. Work. 2005; 4 : 391–412. Publisher Full Text Robila M, Robila SA: Applications of artificial intelligence methodologies to behavioral and social siences. Journal of Child and Family Studies. 2020; 29 ; 2954–2966. Publisher Full Text Shah R, Lahoti S, Lavanya K: An intelligent chat-bot using natural language processing. International Journal of Engineering Research. 2017; 6 : 281–286. Publisher Full Text Sharma V: Measure your virtual assistant performance.2020, 2020-07-19. Reference Source Reference Source Shotter J: Cultural politics of everyday life: social constructionism, rhetoric and knowing of the third kind. Toronto: University of Toronto Press; 1993. Stallbaumer C: Introducing Microsoft Copilot Studio and new features in Copilot for Microsoft 365.2023, 2023-11-15. Retrieved 2023-11-20. Reference Source Suárez-Lantarón B, Deocano-Ruíz Y, García-Perales N, et al. : The educational use of WhatsApp. Sustainability. 2022; 14 : 10510. Publisher Full Text Swank JM, Lambie GW, Witta EL: An Exploratory Investigation of the Counseling Competencies Scale: A Measure of Counseling Skills, Dispositions, and Behaviors. Counselor education and supervision 2012; 51 : 189–206. Publisher Full Text Tamayo PA, Herrero A, Martín J, et al. : Design of a chatbot as a distance learning assistant. Open Praxis. 2020; 12 : 145–153. Publisher Full Text Tanana MJ, Soma CS, Srikumar V, et al. : Development and evaluation of ClientBot: Patient-like conversational agent to train basic counseling skills. Journal of Medical Internet Research. 2019; 21 : e12529. PubMed Abstract | Publisher Full Text | Free Full Text UC Today: OpenAI announces no-code ChatGPT builder for enterprise. 2023, 2023-11-07. Reference Source Van Brummelen JJR: Tools to create and democratize conversational artificial intelligence. Massachusetts Institute of Technology; 2019. Wang J, Shi E, Yu S, et al. : Prompt engineering for healthcare: Methodologies and applications. arXiv preprint arXiv:2304.14670. 2023. Wei J, Bosma M, Zhao VY, et al. : Finetuned language models are zero-shot learners. arXiv preprint arXiv:2109.01652. 2021. White M: Maps of narrative practice. W.W. Norton; 2007. Wu S, Zhao X, Yu T, et al. : Yuan 1.0: Large-scale pre-trained language model in zero-shot and few-shot learning. arXiv preprint arXiv:2110.04725. 2021. Reference Source Xu B, Zhuang Z: Survey on psychotherapy chatbots. Concurrency and Computation: Practice and Experience. 2022; 34 : e6170. Publisher Full Text Xu Z, Jiang J: Multi-dimensional Evaluation of Empathetic Dialog Responses. arXiv preprint arXiv:2402.11409. 2024. Comments on this article Comments (0) Version 2 VERSION 2 PUBLISHED 07 Jun 2024 ADD YOUR COMMENT Comment Author details Author details 1 Social Work, Hong Kong Baptist University, Hong Kong, Hong Kong Chitat Chan Roles: Conceptualization, Data Curation, Formal Analysis, Investigation, Methodology, Project Administration, Resources, Software, Supervision, Validation, Visualization, Writing – Original Draft Preparation, Writing – Review & Editing Yunmeng Zhao Roles: Data Curation, Visualization Jiahui Zhao Roles: Conceptualization, Visualization Competing interests No competing interests were disclosed. Grant information The author(s) declared that no grants were involved in supporting this work. Article Versions (2) version 2 Revised Published: 04 Oct 2024, 13:601 https://doi.org/10.12688/f1000research.151952.2 version 1 Published: 07 Jun 2024, 13:601 https://doi.org/10.12688/f1000research.151952.1 Copyright © 2024 Chan C et al . This is an open access article distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. Download Export To Sciwheel Bibtex EndNote ProCite Ref. Manager (RIS) Sente metrics Views Downloads F1000Research - - PubMed Central info_outline Data from PMC are received and updated monthly. - - Citations open_in_new 0 open_in_new 0 open_in_new SEE MORE DETAILS CITE how to cite this article Chan C, Zhao Y and Zhao J. A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.12688/f1000research.151952.2 ) NOTE: If applicable, it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS track receive updates on this article Track an article to receive email alerts on any updates to this article. TRACK THIS ARTICLE Share Open Peer Review Current Reviewer Status: ? Key to Reviewer Statuses VIEW HIDE Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Version 2 VERSION 2 PUBLISHED 04 Oct 2024 Revised Views 0 Cite How to cite this report: De Fina A. Reviewer Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.171244.r334817 ) The direct URL for this report is: https://f1000research.com/articles/13-601/v2#referee-response-334817 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 02 Jan 2025 Anna De Fina , Georgetown University, Washington, DC, USA Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.171244.r334817 I found the article overall very good both in presenting the justification for the research and the theoretical/methodological basis for the work. I also thought that the methodology and results were carefully explained and justified. The only changes that I ... Continue reading READ ALL I found the article overall very good both in presenting the justification for the research and the theoretical/methodological basis for the work. I also thought that the methodology and results were carefully explained and justified. The only changes that I would suggest are the following: 1. I think the authors should better relate the introductory part about the need to study the possible applications of AI developments to social work with the case of the interview. The interview case is presented as a separate issue after the introduction but I feel that these two parts of the article should be better weaved together. The authors should also devote a bit more space to explaining the model that they followed for designing a narrative interview and therefore also exemplify the different parts of the interview. They just name those parts (Naming, Orientation, etc.) but do not explain what they consist of. Finally, among the limitations of the study, they should add the fact that one of the authors was fully conscious of the background and aims of the experiment and might have been biased in their evaluation of the chatbox's performance. Is the background of the case’s history and progression described in sufficient detail? Yes Is the work clearly and accurately presented and does it cite the current literature? Yes If applicable, is the statistical analysis and its interpretation appropriate? Yes Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Yes Competing Interests: No competing interests were disclosed. Reviewer Expertise: Sociolinguistics, Discourse Analysis I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT De Fina A. Reviewer Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.171244.r334817 ) The direct URL for this report is: https://f1000research.com/articles/13-601/v2#referee-response-334817 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Lee S. Reviewer Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.171244.r329782 ) The direct URL for this report is: https://f1000research.com/articles/13-601/v2#referee-response-329782 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 14 Nov 2024 Sunok Lee , Aalto University, Espoo, Finland Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.171244.r329782 This study employed a text-based interview with a single human participant using the WhatsApp messaging app, utilizing OpenAI’s Assistant API with the GPT-4 model. The AI assistant conducted two interview sessions: the first, where the participant was cooperative, and the ... Continue reading READ ALL This study employed a text-based interview with a single human participant using the WhatsApp messaging app, utilizing OpenAI’s Assistant API with the GPT-4 model. The AI assistant conducted two interview sessions: the first, where the participant was cooperative, and the second, where the participant attempted to sidetrack the AI. This approach aimed to assess the AI’s performance in narrative-based interviewing by evaluating its adherence to interview structure, empathy, and support for participant agency. Researchers used a set of predefined criteria informed by narrative interview literature to measure the AI's performance, including structural fidelity, empathy, and narrative coherence. The assessments were carried out by a human researcher and the AI, and inter-rater reliability was tested to validate consistency in the results. Strengths of the Study The authors are to be commended for their innovative approach to studying AI’s potential in qualitative research, particularly in the complex area of narrative interviewing. Their setup of the AI assistant on WhatsApp, a familiar platform, reflects a practical and thoughtful choice, making AI accessible and potentially easier for social work practitioners to adopt. The decision to use predefined criteria from existing narrative interview literature for performance evaluation is well-grounded, allowing for a structured analysis that aligns with established qualitative research practices. Main Areas for Improvement In-depth Ethical and Critical Issues Analysis While the study provides a promising exploration of AI’s utility in conducting narrative interviews, it lacks a thorough discussion of potential risks, especially concerning interactions with vulnerable users. If AI is to be applied widely, especially in fields involving sensitive interactions, the paper must address the risks of incorrect or insensitive AI responses. The absence of this discussion leaves out critical considerations that might arise if AI were to replace human interaction, such as the potential harm caused by insensitive responses to vulnerable participants. A more comprehensive examination of these ethical issues, including possible measures to mitigate such risks, is essential for providing a balanced view. Interpretation of Progression Scores Although progression scores were introduced to evaluate the interview’s structure, the authors’ interpretation could benefit from additional nuance. The current discussion seems overly definitive in concluding that the AI achieved meaningful progression in narrative structure. Given the limitations of a single participant case study, a more tentative interpretation would be appropriate. Furthermore, while progression scores offer a useful metric, the study could provide practical examples or case-specific explanations to clarify their real-world implications. This would give readers a clearer understanding of how these scores influence the AI’s interview performance and could enhance the study’s transparency. Other Areas for Improvement Clearer Focus on Research Questions and Study Scope The study attempts to evaluate both the AI assistant's performance and the validity of the assessment metrics, which causes some confusion. It would be beneficial to clarify the primary research objective and distinguish between assessing the AI’s performance and the reliability of the evaluation metrics. Additionally, the study would benefit from acknowledging its limitations upfront, particularly as a preliminary or pilot study, to set appropriate expectations for the reader. Balanced Perspective on AI's Potential in Narrative Interviews The literature review highlights AI’s benefits in qualitative research but tends to overstate its potential, neglecting the ongoing debate around AI's limitations in qualitative settings. Acknowledging the contentious nature of AI in qualitative research would present a more balanced viewpoint. This would strengthen the paper by recognizing that while AI offers efficiencies, there are ethical and methodological concerns that still need resolution. Overall This study offers valuable insights into AI’s capabilities in conducting narrative interviews, presenting an innovative approach to exploring AI applications in qualitative research. However, a more balanced perspective on the limitations, especially concerning ethical implications and interpretive caution, would enhance the study’s scientific rigor. Addressing these areas could improve the paper's structure and strengthen its value as a foundational study for further exploration in this field. Is the background of the case’s history and progression described in sufficient detail? Partly Is the work clearly and accurately presented and does it cite the current literature? Partly If applicable, is the statistical analysis and its interpretation appropriate? No Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Partly Competing Interests: No competing interests were disclosed. Reviewer Expertise: Human-centered AI, Participatory Design, Human-computer Interaction. I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Lee S. Reviewer Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.171244.r329782 ) The direct URL for this report is: https://f1000research.com/articles/13-601/v2#referee-response-329782 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Ballantyne N. Reviewer Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.171244.r329218 ) The direct URL for this report is: https://f1000research.com/articles/13-601/v2#referee-response-329218 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 17 Oct 2024 Neil Ballantyne , Open Polytechnic of New Zealand, Lower Hutt, Wellington, New Zealand Approved VIEWS 0 https://doi.org/10.5256/f1000research.171244.r329218 Many thanks to the authors for investing the time to substantially revise and improve the article. I believe that all of the issues I raised, including the pilot status of the study, have now been addressed and the article is ... Continue reading READ ALL Many thanks to the authors for investing the time to substantially revise and improve the article. I believe that all of the issues I raised, including the pilot status of the study, have now been addressed and the article is fit for indexing. Competing Interests: No competing interests were disclosed. Reviewer Expertise: social work, human service technology, artificial intelliegence, qualitative research. I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Ballantyne N. Reviewer Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.171244.r329218 ) The direct URL for this report is: https://f1000research.com/articles/13-601/v2#referee-response-329218 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Respond or Comment COMMENT ON THIS REPORT Version 1 VERSION 1 PUBLISHED 07 Jun 2024 Views 0 Cite How to cite this report: Ballantyne N. Reviewer Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.166648.r298273 ) The direct URL for this report is: https://f1000research.com/articles/13-601/v1#referee-response-298273 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 15 Aug 2024 Neil Ballantyne , Open Polytechnic of New Zealand, Lower Hutt, Wellington, New Zealand Not Approved VIEWS 0 https://doi.org/10.5256/f1000research.166648.r298273 This single-subject case study examines the performance of an AI assistant conducting a narrative, text-based interview with a single human participant using the WhatsApp messaging app. The researchers designed the AI assistant using OpenAI’s Assistants API with ... Continue reading READ ALL This single-subject case study examines the performance of an AI assistant conducting a narrative, text-based interview with a single human participant using the WhatsApp messaging app. The researchers designed the AI assistant using OpenAI’s Assistants API with the GPT-4 model. The interview was conducted twice under two different conditions. During the first, the human participant was instructed to cooperate with the AI assistant, and during the second, they were instructed to be disruptive, attempting to sidetrack the AI assistant in response to questions. To assess the performance of the AI assistant on the specific and complex task of narrative-based interviewing, the researchers designed criteria and measures that, based on a review of the narrative interview literature, they deemed to be relevant and valid measures of performance. They devised and applied measures addressing different dimensions of the narrative interview, including interview structure, empathy, narrative coherence, complexity, and support for the agency of the human participant. A human research assistant and ChatGPT 4 assessed the interview data using these measures, and the results were compared using an inter-rater reliability test. The principal researcher reviewed the results and resolved any disagreements. Strengths of the study The researchers are to be commended for devising an imaginative research design to study a complex area of human-computer interaction in a subject area ­(the application of generative AI to qualitative interviewing) that is both highly topical and deeply controversial. The design of the AI assistant and its deployment on WhatsApp seems to have been well considered, and the limitations imposed by this short exchange, text-based platform are recognised. The criteria for selecting the key measures to assess narrative interview performance, drawn from the literature identified, makes sense. However, a whole paper could have been written on this aspect alone. Main areas for improvement Be clear about research questions and primary focus A lot is going on in this study, and the description and rationale for key design choices are sometimes unclear. The problem begins with the abstract and the research questions, where there seems to be slippage between a focus on evaluating the performance of the AI assistant and evaluating the validity and inter-rater reliability of the measures constructed. The first research question asks: How can the proposed metrics help us, as social researchers without a technical background, understand the quality of the AI-driven interviews in this study? In my view, RQ1 would be better focussed on assessing the validity and reliability of the measures selected for evaluating the performance of the AI assistant. This needs to be established before an actual performance assessment of the AI assistant is made. Establishing inter-rater reliability of the measures would be a part of that task. However, this raises the question of why the researchers decided to have only two raters, one human and the other a GenAI. The comparison is interesting but muddies the waters. This machine versus human rater aspect of the study was a surprise since it was not signalled in the abstract, introduction or research questions. A comparison between the ratings made by a GenAI and human raters would be far more interesting once we had established human inter-rater reliability, which could have been achieved with only a few more humans in the frame. Establish from the outset that this is a pilot study The title and the abstract refer to the study as a case study, albeit one with a single participant, giving the reader the impression of a significant, in-depth exploration of a phenomenon. However, in the section on limitations, the authors describe the study as a pilot study and state that the primary goal “ was to explore ideas and stimulate discussion rather than to conduct rigorous experiments”. I recommend that the abstract and introduction be rewritten to reflect this stance. However, this does beg the question as to whether the paper might be better construed as a working paper for feedback and discussion than as a substantive academic article. The paper is very provocative and stimulating in terms of the issues it raises but inconclusive as an assessment of the utility of an AI assistant for narrative interviews. Other areas for improvement A balanced approach to the literature and the claims made The literature review is interesting and includes many relevant articles, but the articles selected and the authors’ comments are prone to over-accentuate the potential of a technology that is still emergent. The application of GenAI to qualitative research is deeply contested in the mainstream literature on qualitative methodology, making it hard to concur with the authors’ views that: The integration of generative AI has revolutionized and democratized the process of qualitative data collection, analysis, and interpretation. This technological leap has boosted efficiency and expanded the horizons of narrative inquiry methods. The authors can improve the literature review by recognising GenAI's emergent nature and the fact that many scholars in the mainstream qualitative research field have strong reservations about its applicability and ethical concerns about its use. Other issues There are several places in the text where the authors’ explanations of the methods used are a little too opaque. For example, the explanation about the progression scores was helpful but left me puzzled about how this was practically implemented. An example might help. Also, in the findings, the authors highlight that the participant exhibited higher agency during conversation two, which is surely explained by the instruction given to the participant to try to throw the interviewer off track. If that isn’t the explanation, a fuller account as to why not would be helpful. Overall I want to encourage the authors to continue their exploration of the use of GenAI in qualitative research, and I believe that this is a very useful working paper that raises many pertinent issues. Responding to the comments above might help to improve the structure and flow of the paper, but the research design issues cannot be remedied without conducting a completely new study. It is for that reason I decided not to recommend the paper for indexing. Is the background of the case’s history and progression described in sufficient detail? No Is the work clearly and accurately presented and does it cite the current literature? Partly If applicable, is the statistical analysis and its interpretation appropriate? Partly Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly Is the case presented with sufficient detail to be useful for teaching or other practitioners? Partly References 1. Chan C, Zhao Y: A Case Study on Measuring AI Assistant Competence in Narrative Interviews. F1000Research . 2024; 13 . Publisher Full Text Competing Interests: No competing interests were disclosed. Reviewer Expertise: social work, human service technology, artificial intelliegence, qualitative research. I confirm that I have read this submission and believe that I have an appropriate level of expertise to state that I do not consider it to be of an acceptable scientific standard, for reasons outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Ballantyne N. Reviewer Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.166648.r298273 ) The direct URL for this report is: https://f1000research.com/articles/13-601/v1#referee-response-298273 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 04 Oct 2024 Chitat CHAN , Social Work, Hong Kong Baptist University, Hong Kong, Hong Kong 04 Oct 2024 Author Response //Main areas for improvement Be clear about research questions and primary focus A lot is going on in this study, and the description and rationale for key design choices are ... Continue reading //Main areas for improvement Be clear about research questions and primary focus A lot is going on in this study, and the description and rationale for key design choices are sometimes unclear. The problem begins with the abstract and the research questions, where there seems to be slippage between a focus on evaluating the performance of the AI assistant and evaluating the validity and inter-rater reliability of the measures constructed. The first research question asks: How can the proposed metrics help us, as social researchers without a technical background, understand the quality of the AI-driven interviews in this study? In my view, RQ1 would be better focussed on assessing the validity and reliability of the measures selected for evaluating the performance of the AI assistant. This needs to be established before an actual performance assessment of the AI assistant is made. Establishing inter-rater reliability of the measures would be a part of that task. However, this raises the question of why the researchers decided to have only two raters, one human and the other a GenAI. The comparison is interesting but muddies the waters. This machine versus human rater aspect of the study was a surprise since it was not signalled in the abstract, introduction or research questions. A comparison between the ratings made by a GenAI and human raters would be far more interesting once we had established human inter-rater reliability, which could have been achieved with only a few more humans in the frame.// *Response: The research questions and focus have been entirely reformulated. The revised manuscript now distinguishes its objectives. It focuses on the democratization of AI chatbots and the emerging trend of customizable AI, which has opened new opportunities across various fields, including social work. The study explores how social workers, particularly those with minimal coding experience, can instruct and evaluate AI assistants to conduct narrative interviews. The role of narrative interviews is used as a case to explore possibilities and provide feedback within a broader context. The emphasis is on qualitative assessment rather than quantitative measurement, aligning with the exploratory nature of the study. //Establish from the outset that this is a pilot study The title and the abstract refer to the study as a case study, albeit one with a single participant, giving the reader the impression of a significant, in-depth exploration of a phenomenon. However, in the section on limitations, the authors describe the study as a pilot study and state that the primary goal “ was to explore ideas and stimulate discussion rather than to conduct rigorous experiments”. I recommend that the abstract and introduction be rewritten to reflect this stance. However, this does beg the question as to whether the paper might be better construed as a working paper for feedback and discussion than as a substantive academic article. The paper is very provocative and stimulating in terms of the issues it raises but inconclusive as an assessment of the utility of an AI assistant for narrative interviews.// *Response: The pilot and exploratory nature of the study is emphasized. The revised abstract and introduction explicitly state that the study is exploratory and serves as a pilot to stimulate discussion about the use of AI in social work. The manuscript now clearly outlines its objectives, focusing on the democratization of AI chatbots and the emerging trend of customizable AI. The study is qualitative and concerned with the extent to which domain knowledge experts can participate in instructing and assessing AI chatbots. //Other areas for improvement A balanced approach to the literature and the claims made The literature review is interesting and includes many relevant articles, but the articles selected and the authors’ comments are prone to over-accentuate the potential of a technology that is still emergent. The application of GenAI to qualitative research is deeply contested in the mainstream literature on qualitative methodology, making it hard to concur with the authors’ views that: The integration of generative AI has revolutionized and democratized the process of qualitative data collection, analysis, and interpretation. This technological leap has boosted efficiency and expanded the horizons of narrative inquiry methods. The authors can improve the literature review by recognising GenAI's emergent nature and the fact that many scholars in the mainstream qualitative research field have strong reservations about its applicability and ethical concerns about its use.// *Response: The literature review has been revised to acknowledge the democratization of AI chatbots and the emerging trend of customizable AI, which opens new opportunities across various fields, including social work. It addresses limitations alongside potential benefits. The discussion part addresses these limitations, noting that the study focused exclusively on OpenAI's product and that findings may not be generalizable to AI tools from other companies. While many principles and challenges identified here could apply broadly to other AI systems, variations in technology and implementation could lead to different outcomes. The introduction of AI tools into practice also brings considerations such as navigating risk management, data privacy, and the uneven development of language models across diverse sociocultural landscapes. While these challenges are important and have been extensively explored by others, the study focuses on exploring how social workers can expand their skill sets and rethink their expertise to engage actively with AI technologies. //Other issues There are several places in the text where the authors’ explanations of the methods used are a little too opaque. For example, the explanation about the progression scores was helpful but left me puzzled about how this was practically implemented. An example might help. Also, in the findings, the authors highlight that the participant exhibited higher agency during conversation two, which is surely explained by the instruction given to the participant to try to throw the interviewer off track. If that isn’t the explanation, a fuller account as to why not would be helpful.// *Response: The revised manuscript provides clearer explanations of the methods used. It explains that the study is autoethnographic, reflecting personal experiences and using rubrics for assessing fidelity and empathy in the AI assistant’s performance. Examples of conversation transcripts are included to illustrate these qualitative assessments, emphasizing the depth and context of interactions rather than mere numerical data. We also enhanced clarity in findings. It was noted that some findings, like the participant exhibiting higher agency, were not well-explained. In the revised discussion, the measure of agency has been removed and integrated into the discussion about empathy and a deeper exploration of meanings. This approach aligns with the study’s focus on qualitative assessment and provides a clearer understanding of the findings. //Overall I want to encourage the authors to continue their exploration of the use of GenAI in qualitative research, and I believe that this is a very useful working paper that raises many pertinent issues. Responding to the comments above might help to improve the structure and flow of the paper, but the research design issues cannot be remedied without conducting a completely new study. It is for that reason I decided not to recommend the paper for indexing.// *Response: We are very grateful for the thoughtful and constructive comments provided. We deeply appreciate the feedback, especially regarding the areas for improvement and the suggestion to frame the study as a pilot. These insights were particularly helpful in guiding us toward a more focused and balanced approach. We also want to acknowledge that the first version of the manuscript had several gaps, and we sincerely apologize for any confusion this may have caused. We recognize that the clarity and direction of the paper needed refinement, and we are thankful for the opportunity to improve. In response, we have rewritten the article using the same dataset but with a more refined focus, especially clarifying the research aims and scope. The revised study explores how AI chatbots, particularly GPT-4, can be applied in social work for conducting narrative interviews. Using an autoethnographic case study approach, social work researchers developed and tested an AI chatbot on WhatsApp. The chatbot followed a structured interview plan and demonstrated empathy, though challenges in replicating human skills within AI were noted. The study concludes that social workers can tailor AI tools for narrative interviews, but further research is needed to address the field's complexities. Our focus remains on democratizing AI and empowering social workers to use these tools effectively while emphasizing the need for interdisciplinary collaboration in developing robust assessment tools. //Main areas for improvement Be clear about research questions and primary focus A lot is going on in this study, and the description and rationale for key design choices are sometimes unclear. The problem begins with the abstract and the research questions, where there seems to be slippage between a focus on evaluating the performance of the AI assistant and evaluating the validity and inter-rater reliability of the measures constructed. The first research question asks: How can the proposed metrics help us, as social researchers without a technical background, understand the quality of the AI-driven interviews in this study? In my view, RQ1 would be better focussed on assessing the validity and reliability of the measures selected for evaluating the performance of the AI assistant. This needs to be established before an actual performance assessment of the AI assistant is made. Establishing inter-rater reliability of the measures would be a part of that task. However, this raises the question of why the researchers decided to have only two raters, one human and the other a GenAI. The comparison is interesting but muddies the waters. This machine versus human rater aspect of the study was a surprise since it was not signalled in the abstract, introduction or research questions. A comparison between the ratings made by a GenAI and human raters would be far more interesting once we had established human inter-rater reliability, which could have been achieved with only a few more humans in the frame.// *Response: The research questions and focus have been entirely reformulated. The revised manuscript now distinguishes its objectives. It focuses on the democratization of AI chatbots and the emerging trend of customizable AI, which has opened new opportunities across various fields, including social work. The study explores how social workers, particularly those with minimal coding experience, can instruct and evaluate AI assistants to conduct narrative interviews. The role of narrative interviews is used as a case to explore possibilities and provide feedback within a broader context. The emphasis is on qualitative assessment rather than quantitative measurement, aligning with the exploratory nature of the study. //Establish from the outset that this is a pilot study The title and the abstract refer to the study as a case study, albeit one with a single participant, giving the reader the impression of a significant, in-depth exploration of a phenomenon. However, in the section on limitations, the authors describe the study as a pilot study and state that the primary goal “ was to explore ideas and stimulate discussion rather than to conduct rigorous experiments”. I recommend that the abstract and introduction be rewritten to reflect this stance. However, this does beg the question as to whether the paper might be better construed as a working paper for feedback and discussion than as a substantive academic article. The paper is very provocative and stimulating in terms of the issues it raises but inconclusive as an assessment of the utility of an AI assistant for narrative interviews.// *Response: The pilot and exploratory nature of the study is emphasized. The revised abstract and introduction explicitly state that the study is exploratory and serves as a pilot to stimulate discussion about the use of AI in social work. The manuscript now clearly outlines its objectives, focusing on the democratization of AI chatbots and the emerging trend of customizable AI. The study is qualitative and concerned with the extent to which domain knowledge experts can participate in instructing and assessing AI chatbots. //Other areas for improvement A balanced approach to the literature and the claims made The literature review is interesting and includes many relevant articles, but the articles selected and the authors’ comments are prone to over-accentuate the potential of a technology that is still emergent. The application of GenAI to qualitative research is deeply contested in the mainstream literature on qualitative methodology, making it hard to concur with the authors’ views that: The integration of generative AI has revolutionized and democratized the process of qualitative data collection, analysis, and interpretation. This technological leap has boosted efficiency and expanded the horizons of narrative inquiry methods. The authors can improve the literature review by recognising GenAI's emergent nature and the fact that many scholars in the mainstream qualitative research field have strong reservations about its applicability and ethical concerns about its use.// *Response: The literature review has been revised to acknowledge the democratization of AI chatbots and the emerging trend of customizable AI, which opens new opportunities across various fields, including social work. It addresses limitations alongside potential benefits. The discussion part addresses these limitations, noting that the study focused exclusively on OpenAI's product and that findings may not be generalizable to AI tools from other companies. While many principles and challenges identified here could apply broadly to other AI systems, variations in technology and implementation could lead to different outcomes. The introduction of AI tools into practice also brings considerations such as navigating risk management, data privacy, and the uneven development of language models across diverse sociocultural landscapes. While these challenges are important and have been extensively explored by others, the study focuses on exploring how social workers can expand their skill sets and rethink their expertise to engage actively with AI technologies. //Other issues There are several places in the text where the authors’ explanations of the methods used are a little too opaque. For example, the explanation about the progression scores was helpful but left me puzzled about how this was practically implemented. An example might help. Also, in the findings, the authors highlight that the participant exhibited higher agency during conversation two, which is surely explained by the instruction given to the participant to try to throw the interviewer off track. If that isn’t the explanation, a fuller account as to why not would be helpful.// *Response: The revised manuscript provides clearer explanations of the methods used. It explains that the study is autoethnographic, reflecting personal experiences and using rubrics for assessing fidelity and empathy in the AI assistant’s performance. Examples of conversation transcripts are included to illustrate these qualitative assessments, emphasizing the depth and context of interactions rather than mere numerical data. We also enhanced clarity in findings. It was noted that some findings, like the participant exhibiting higher agency, were not well-explained. In the revised discussion, the measure of agency has been removed and integrated into the discussion about empathy and a deeper exploration of meanings. This approach aligns with the study’s focus on qualitative assessment and provides a clearer understanding of the findings. //Overall I want to encourage the authors to continue their exploration of the use of GenAI in qualitative research, and I believe that this is a very useful working paper that raises many pertinent issues. Responding to the comments above might help to improve the structure and flow of the paper, but the research design issues cannot be remedied without conducting a completely new study. It is for that reason I decided not to recommend the paper for indexing.// *Response: We are very grateful for the thoughtful and constructive comments provided. We deeply appreciate the feedback, especially regarding the areas for improvement and the suggestion to frame the study as a pilot. These insights were particularly helpful in guiding us toward a more focused and balanced approach. We also want to acknowledge that the first version of the manuscript had several gaps, and we sincerely apologize for any confusion this may have caused. We recognize that the clarity and direction of the paper needed refinement, and we are thankful for the opportunity to improve. In response, we have rewritten the article using the same dataset but with a more refined focus, especially clarifying the research aims and scope. The revised study explores how AI chatbots, particularly GPT-4, can be applied in social work for conducting narrative interviews. Using an autoethnographic case study approach, social work researchers developed and tested an AI chatbot on WhatsApp. The chatbot followed a structured interview plan and demonstrated empathy, though challenges in replicating human skills within AI were noted. The study concludes that social workers can tailor AI tools for narrative interviews, but further research is needed to address the field's complexities. Our focus remains on democratizing AI and empowering social workers to use these tools effectively while emphasizing the need for interdisciplinary collaboration in developing robust assessment tools. Competing Interests: NIL Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 04 Oct 2024 Chitat CHAN , Social Work, Hong Kong Baptist University, Hong Kong, Hong Kong 04 Oct 2024 Author Response //Main areas for improvement Be clear about research questions and primary focus A lot is going on in this study, and the description and rationale for key design choices are ... Continue reading //Main areas for improvement Be clear about research questions and primary focus A lot is going on in this study, and the description and rationale for key design choices are sometimes unclear. The problem begins with the abstract and the research questions, where there seems to be slippage between a focus on evaluating the performance of the AI assistant and evaluating the validity and inter-rater reliability of the measures constructed. The first research question asks: How can the proposed metrics help us, as social researchers without a technical background, understand the quality of the AI-driven interviews in this study? In my view, RQ1 would be better focussed on assessing the validity and reliability of the measures selected for evaluating the performance of the AI assistant. This needs to be established before an actual performance assessment of the AI assistant is made. Establishing inter-rater reliability of the measures would be a part of that task. However, this raises the question of why the researchers decided to have only two raters, one human and the other a GenAI. The comparison is interesting but muddies the waters. This machine versus human rater aspect of the study was a surprise since it was not signalled in the abstract, introduction or research questions. A comparison between the ratings made by a GenAI and human raters would be far more interesting once we had established human inter-rater reliability, which could have been achieved with only a few more humans in the frame.// *Response: The research questions and focus have been entirely reformulated. The revised manuscript now distinguishes its objectives. It focuses on the democratization of AI chatbots and the emerging trend of customizable AI, which has opened new opportunities across various fields, including social work. The study explores how social workers, particularly those with minimal coding experience, can instruct and evaluate AI assistants to conduct narrative interviews. The role of narrative interviews is used as a case to explore possibilities and provide feedback within a broader context. The emphasis is on qualitative assessment rather than quantitative measurement, aligning with the exploratory nature of the study. //Establish from the outset that this is a pilot study The title and the abstract refer to the study as a case study, albeit one with a single participant, giving the reader the impression of a significant, in-depth exploration of a phenomenon. However, in the section on limitations, the authors describe the study as a pilot study and state that the primary goal “ was to explore ideas and stimulate discussion rather than to conduct rigorous experiments”. I recommend that the abstract and introduction be rewritten to reflect this stance. However, this does beg the question as to whether the paper might be better construed as a working paper for feedback and discussion than as a substantive academic article. The paper is very provocative and stimulating in terms of the issues it raises but inconclusive as an assessment of the utility of an AI assistant for narrative interviews.// *Response: The pilot and exploratory nature of the study is emphasized. The revised abstract and introduction explicitly state that the study is exploratory and serves as a pilot to stimulate discussion about the use of AI in social work. The manuscript now clearly outlines its objectives, focusing on the democratization of AI chatbots and the emerging trend of customizable AI. The study is qualitative and concerned with the extent to which domain knowledge experts can participate in instructing and assessing AI chatbots. //Other areas for improvement A balanced approach to the literature and the claims made The literature review is interesting and includes many relevant articles, but the articles selected and the authors’ comments are prone to over-accentuate the potential of a technology that is still emergent. The application of GenAI to qualitative research is deeply contested in the mainstream literature on qualitative methodology, making it hard to concur with the authors’ views that: The integration of generative AI has revolutionized and democratized the process of qualitative data collection, analysis, and interpretation. This technological leap has boosted efficiency and expanded the horizons of narrative inquiry methods. The authors can improve the literature review by recognising GenAI's emergent nature and the fact that many scholars in the mainstream qualitative research field have strong reservations about its applicability and ethical concerns about its use.// *Response: The literature review has been revised to acknowledge the democratization of AI chatbots and the emerging trend of customizable AI, which opens new opportunities across various fields, including social work. It addresses limitations alongside potential benefits. The discussion part addresses these limitations, noting that the study focused exclusively on OpenAI's product and that findings may not be generalizable to AI tools from other companies. While many principles and challenges identified here could apply broadly to other AI systems, variations in technology and implementation could lead to different outcomes. The introduction of AI tools into practice also brings considerations such as navigating risk management, data privacy, and the uneven development of language models across diverse sociocultural landscapes. While these challenges are important and have been extensively explored by others, the study focuses on exploring how social workers can expand their skill sets and rethink their expertise to engage actively with AI technologies. //Other issues There are several places in the text where the authors’ explanations of the methods used are a little too opaque. For example, the explanation about the progression scores was helpful but left me puzzled about how this was practically implemented. An example might help. Also, in the findings, the authors highlight that the participant exhibited higher agency during conversation two, which is surely explained by the instruction given to the participant to try to throw the interviewer off track. If that isn’t the explanation, a fuller account as to why not would be helpful.// *Response: The revised manuscript provides clearer explanations of the methods used. It explains that the study is autoethnographic, reflecting personal experiences and using rubrics for assessing fidelity and empathy in the AI assistant’s performance. Examples of conversation transcripts are included to illustrate these qualitative assessments, emphasizing the depth and context of interactions rather than mere numerical data. We also enhanced clarity in findings. It was noted that some findings, like the participant exhibiting higher agency, were not well-explained. In the revised discussion, the measure of agency has been removed and integrated into the discussion about empathy and a deeper exploration of meanings. This approach aligns with the study’s focus on qualitative assessment and provides a clearer understanding of the findings. //Overall I want to encourage the authors to continue their exploration of the use of GenAI in qualitative research, and I believe that this is a very useful working paper that raises many pertinent issues. Responding to the comments above might help to improve the structure and flow of the paper, but the research design issues cannot be remedied without conducting a completely new study. It is for that reason I decided not to recommend the paper for indexing.// *Response: We are very grateful for the thoughtful and constructive comments provided. We deeply appreciate the feedback, especially regarding the areas for improvement and the suggestion to frame the study as a pilot. These insights were particularly helpful in guiding us toward a more focused and balanced approach. We also want to acknowledge that the first version of the manuscript had several gaps, and we sincerely apologize for any confusion this may have caused. We recognize that the clarity and direction of the paper needed refinement, and we are thankful for the opportunity to improve. In response, we have rewritten the article using the same dataset but with a more refined focus, especially clarifying the research aims and scope. The revised study explores how AI chatbots, particularly GPT-4, can be applied in social work for conducting narrative interviews. Using an autoethnographic case study approach, social work researchers developed and tested an AI chatbot on WhatsApp. The chatbot followed a structured interview plan and demonstrated empathy, though challenges in replicating human skills within AI were noted. The study concludes that social workers can tailor AI tools for narrative interviews, but further research is needed to address the field's complexities. Our focus remains on democratizing AI and empowering social workers to use these tools effectively while emphasizing the need for interdisciplinary collaboration in developing robust assessment tools. //Main areas for improvement Be clear about research questions and primary focus A lot is going on in this study, and the description and rationale for key design choices are sometimes unclear. The problem begins with the abstract and the research questions, where there seems to be slippage between a focus on evaluating the performance of the AI assistant and evaluating the validity and inter-rater reliability of the measures constructed. The first research question asks: How can the proposed metrics help us, as social researchers without a technical background, understand the quality of the AI-driven interviews in this study? In my view, RQ1 would be better focussed on assessing the validity and reliability of the measures selected for evaluating the performance of the AI assistant. This needs to be established before an actual performance assessment of the AI assistant is made. Establishing inter-rater reliability of the measures would be a part of that task. However, this raises the question of why the researchers decided to have only two raters, one human and the other a GenAI. The comparison is interesting but muddies the waters. This machine versus human rater aspect of the study was a surprise since it was not signalled in the abstract, introduction or research questions. A comparison between the ratings made by a GenAI and human raters would be far more interesting once we had established human inter-rater reliability, which could have been achieved with only a few more humans in the frame.// *Response: The research questions and focus have been entirely reformulated. The revised manuscript now distinguishes its objectives. It focuses on the democratization of AI chatbots and the emerging trend of customizable AI, which has opened new opportunities across various fields, including social work. The study explores how social workers, particularly those with minimal coding experience, can instruct and evaluate AI assistants to conduct narrative interviews. The role of narrative interviews is used as a case to explore possibilities and provide feedback within a broader context. The emphasis is on qualitative assessment rather than quantitative measurement, aligning with the exploratory nature of the study. //Establish from the outset that this is a pilot study The title and the abstract refer to the study as a case study, albeit one with a single participant, giving the reader the impression of a significant, in-depth exploration of a phenomenon. However, in the section on limitations, the authors describe the study as a pilot study and state that the primary goal “ was to explore ideas and stimulate discussion rather than to conduct rigorous experiments”. I recommend that the abstract and introduction be rewritten to reflect this stance. However, this does beg the question as to whether the paper might be better construed as a working paper for feedback and discussion than as a substantive academic article. The paper is very provocative and stimulating in terms of the issues it raises but inconclusive as an assessment of the utility of an AI assistant for narrative interviews.// *Response: The pilot and exploratory nature of the study is emphasized. The revised abstract and introduction explicitly state that the study is exploratory and serves as a pilot to stimulate discussion about the use of AI in social work. The manuscript now clearly outlines its objectives, focusing on the democratization of AI chatbots and the emerging trend of customizable AI. The study is qualitative and concerned with the extent to which domain knowledge experts can participate in instructing and assessing AI chatbots. //Other areas for improvement A balanced approach to the literature and the claims made The literature review is interesting and includes many relevant articles, but the articles selected and the authors’ comments are prone to over-accentuate the potential of a technology that is still emergent. The application of GenAI to qualitative research is deeply contested in the mainstream literature on qualitative methodology, making it hard to concur with the authors’ views that: The integration of generative AI has revolutionized and democratized the process of qualitative data collection, analysis, and interpretation. This technological leap has boosted efficiency and expanded the horizons of narrative inquiry methods. The authors can improve the literature review by recognising GenAI's emergent nature and the fact that many scholars in the mainstream qualitative research field have strong reservations about its applicability and ethical concerns about its use.// *Response: The literature review has been revised to acknowledge the democratization of AI chatbots and the emerging trend of customizable AI, which opens new opportunities across various fields, including social work. It addresses limitations alongside potential benefits. The discussion part addresses these limitations, noting that the study focused exclusively on OpenAI's product and that findings may not be generalizable to AI tools from other companies. While many principles and challenges identified here could apply broadly to other AI systems, variations in technology and implementation could lead to different outcomes. The introduction of AI tools into practice also brings considerations such as navigating risk management, data privacy, and the uneven development of language models across diverse sociocultural landscapes. While these challenges are important and have been extensively explored by others, the study focuses on exploring how social workers can expand their skill sets and rethink their expertise to engage actively with AI technologies. //Other issues There are several places in the text where the authors’ explanations of the methods used are a little too opaque. For example, the explanation about the progression scores was helpful but left me puzzled about how this was practically implemented. An example might help. Also, in the findings, the authors highlight that the participant exhibited higher agency during conversation two, which is surely explained by the instruction given to the participant to try to throw the interviewer off track. If that isn’t the explanation, a fuller account as to why not would be helpful.// *Response: The revised manuscript provides clearer explanations of the methods used. It explains that the study is autoethnographic, reflecting personal experiences and using rubrics for assessing fidelity and empathy in the AI assistant’s performance. Examples of conversation transcripts are included to illustrate these qualitative assessments, emphasizing the depth and context of interactions rather than mere numerical data. We also enhanced clarity in findings. It was noted that some findings, like the participant exhibiting higher agency, were not well-explained. In the revised discussion, the measure of agency has been removed and integrated into the discussion about empathy and a deeper exploration of meanings. This approach aligns with the study’s focus on qualitative assessment and provides a clearer understanding of the findings. //Overall I want to encourage the authors to continue their exploration of the use of GenAI in qualitative research, and I believe that this is a very useful working paper that raises many pertinent issues. Responding to the comments above might help to improve the structure and flow of the paper, but the research design issues cannot be remedied without conducting a completely new study. It is for that reason I decided not to recommend the paper for indexing.// *Response: We are very grateful for the thoughtful and constructive comments provided. We deeply appreciate the feedback, especially regarding the areas for improvement and the suggestion to frame the study as a pilot. These insights were particularly helpful in guiding us toward a more focused and balanced approach. We also want to acknowledge that the first version of the manuscript had several gaps, and we sincerely apologize for any confusion this may have caused. We recognize that the clarity and direction of the paper needed refinement, and we are thankful for the opportunity to improve. In response, we have rewritten the article using the same dataset but with a more refined focus, especially clarifying the research aims and scope. The revised study explores how AI chatbots, particularly GPT-4, can be applied in social work for conducting narrative interviews. Using an autoethnographic case study approach, social work researchers developed and tested an AI chatbot on WhatsApp. The chatbot followed a structured interview plan and demonstrated empathy, though challenges in replicating human skills within AI were noted. The study concludes that social workers can tailor AI tools for narrative interviews, but further research is needed to address the field's complexities. Our focus remains on democratizing AI and empowering social workers to use these tools effectively while emphasizing the need for interdisciplinary collaboration in developing robust assessment tools. Competing Interests: NIL Close Report a concern COMMENT ON THIS REPORT Views 0 Cite How to cite this report: Singer J. Reviewer Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.166648.r298274 ) The direct URL for this report is: https://f1000research.com/articles/13-601/v1#referee-response-298274 NOTE: it is important to ensure the information in square brackets after the title is included in this citation. Close Copy Citation Details Reviewer Report 26 Jul 2024 Jonathan Singer , Loyola University Chicago, Chicago, Illinois, USA Approved with Reservations VIEWS 0 https://doi.org/10.5256/f1000research.166648.r298274 This manuscript addresses an interesting and important topic. My comments are focused on clarifying certain details to enhance the paper’s clarity and coherence, rather than critiquing its main idea, which I find compelling but somewhat unclear in its current form. ... Continue reading READ ALL This manuscript addresses an interesting and important topic. My comments are focused on clarifying certain details to enhance the paper’s clarity and coherence, rather than critiquing its main idea, which I find compelling but somewhat unclear in its current form. Despite reading through the entire paper multiple times, I struggled to fully grasp key parts. Reorganizing the literature review, introducing a clear “Methods” section, and streamlining the manuscript would greatly improve its readability. Literature Review The literature review begins by emphasizing the potential of AI in narrative interviewing, but it would benefit from first defining and describing what narrative interviewing entails. The enthusiastic discussion of AI, including references to “influential bloggers,” feels more promotional than academic, especially since it turns out to be an advertisement for a for-profit company selling narrative interviewing tools. To strengthen the paper, I recommend moving the definitions of key terms like “narrative interviewing” and “conversation analysis” to the beginning of the literature review. Additionally, information from paragraph 5 and the section “Why do assistant’s interview skills matter” should be relocated to the start to provide important context early on. The language in the literature review occasionally seems exaggerated and inconsistent. For instance, the statement that AI is “becoming a crucial tool for researchers” feels overstated without supporting evidence. Furthermore, it appears contradictory to claim that AI improves analysis while also stating that academics have not “explored how AI could enable or limit narrative interviews.” Clarifying these points and providing supporting evidence will strengthen the argument. There are also some statements that seem unnecessary and detract from the narrative flow, such as “This brief review of narrative inquirer capabilities is not comprehensive.” Removing such remarks can streamline the text and improve readability. Methods Section Introducing a “Methods” header before discussing competence metrics would improve the manuscript’s structure. The justification for the competence metrics needs to be clearer. Rather than stating that “The following metrics were chosen primarily for their simplicity and to facilitate testing and demonstration,” it would be helpful to explain the rationale behind selecting these specific metrics and whether they are based on existing frameworks or newly developed by the authors. Scores: The authors identified six scores but did not talk about how those scores were established, the scales, or anything else about them. Specifically: Competence 1: The explanation of how progression scores are assigned is vague. There is no clear description of the criteria used to determine the progression stage of each utterance. This lack of specificity makes it difficult to understand the validity and reliability of the scoring system. The method’s reliance on previous studies (Au-yeung, 2023; Chan et al., 2020; Connor et al., 2009; Moran et al., 2005; Ramey et al., 2009) without providing a detailed explanation or adaptation process is problematic. It would be helpful to have a more thorough, but not necessarily long explanation of how these methodologies are applied to the current study. Relatedly, the sequential analysis method from therapy research may not be directly applicable to narrative interviews conducted by AI. If this method was developed from narrative interviews, then say so. There are therapy approaches like CBT or behavioral therapies that have structured, goal-oriented sessions. ABFT is task oriented, but still provides the therapist structure. Narrative interviews, from your description, are more open-ended, possibly more like Rogerian or supportive therapy sessions. The adaptation of this method needs more rigorous justification. I was also unclear how and why a higher score isn’t better, and also why you calculated average progression scores over time. How does this score help us to understand how good AI is doing? Competence 2: Is there empirical validation or psychometric testing of the empathy scale? If so, please provide that information for the reader. I would like more information about why a Likert scale is used for empathy, especially since the previous competence argued that higher scores were not necessarily better. That does not seem to be the case with this scale. Competence 3: I didn’t understand the use of the word “Domination” in the subheader. You introduce multiple metrics (agency, coherence, complexity, and meaning-making) without clearly explaining how they are operationalized. Each one of these needs to be defined and the reader needs measurement criteria for each metric, especially since the previous two measures apparently used different scales. For example, the authors say that measuring “agency” cannot be done in isolation, but there is no description of how the scores or definitions of the other measures are used to understand or measure agency. Indeed, it seems that if the concepts in Competence 3 are interrelated, might there be a way to discuss the measures as a coherent package, rather than individual scales? The description of the Case study has many strengths, including the innovative use of ChatGPT to create a chatbot that the researchers would otherwise have not been able to program, highlighting the value of deploying the chatbot on WhatsApp, clearly defining the seven stages, and the ingenuity of having the postgraduate have two conversations – one genuine and one combative. The major limitation of the description of the case study is that it seems to be disconnected from the measures. I know that you provide results in the results section, but it would be helpful to have notes about how each section connects to the measures. Data Analysis Method and Data Collection Section I was unclear about the context in which the postgrad was having the WhatsApp conversations. Was this in a lab? Were the researchers observing? Was this in private? I didn’t understand why the conversation was divided into four equal quarters. i.e. The choice of four quarters seems arbitrary. I know that it makes it easy to analyze by utterances rather than time, but it seems to miss the importance of space and silence in a narrative interview. The process for manually assigning progression scores is not clearly defined. There is no description of the criteria used by the human research assistant to assign these scores, which raises concerns about the subjectivity and consistency of the ratings. Is there any research on using LLMs or ChatGPT specifically to assigning scores to narrative interviews? If so, it would be good to cite that here. Results The agreement between the human research assistant and ChatGPT4 on most metrics in and of itself is encouraging for researchers interested in using LLMs. In finding 2, what distinguishes a 3 from a 4? This is part of my ongoing concern about the paper’s lack of clarity with the measures. The disagreement on coherence in Conversation 2 highlights the need for clear criteria. The authors wrote, “In the rating, the human research assistant and ChatGPT4 mostly agreed on all these metrics, except that there was a disagreement on coherence in Conversation 2, where the human research assistant gave a score of 2 and ChatGPT4 gave a score of 3. After reviewing this, the principal researcher sided with ChatGPT4's score.” The manuscript should detail the specific aspects that led to different ratings and the rationale for siding with ChatGPT4’s score. Especially in these early days of using LLMs for analysis, this kind of transparency will be instructive as well as increase trustworthiness of the analysis. Discussion I thought it was interesting that there were challenges with long speeches in conversation analysis. Even though WhatsApp tends to have shorter exchanges, what are some potential solutions or alternative methods to address these challenges, especially since narrative interviews often involve extended dialogue. I’m thinking specifically about qualitative researchers who are interested in using this for more traditional open-ended narrative interviews, not on WhatsApp. Perhaps this is part of an expanded discussion on the scalability of this type of AI use. I agree that there need to be more refined metrics (and some of my critiques of this paper are in line with that call). How would you propose to improve interrater reliability? Again, are there computational linguists who have looked into human/LLM IRR? After reading through the discussion a few times, I still didn’t have a good understanding of the differences between conversations 1 and 2. For example, in the results section you noted differences in agency, coherence and complexity. Why did the participant exhibit higher agency in Conversation 2? What factors contributed to the lower coherence and complexity in Conversation 2? If you discussed this and I missed it, my apologies. Final note: I hope this feedback is helpful. I typically do not review manuscripts until I fully understand both the spirit and letter of the manuscript, but I am providing this feedback in response to the editor’s request for a timely review Is the background of the case’s history and progression described in sufficient detail? No Is the work clearly and accurately presented and does it cite the current literature? Partly If applicable, is the statistical analysis and its interpretation appropriate? Partly Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Partly Competing Interests: No competing interests were disclosed. Reviewer Expertise: Youth suicide prevention, qualitative research, technology and social work education/practice I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. Close READ LESS CITE CITE HOW TO CITE THIS REPORT Singer J. Reviewer Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.166648.r298274 ) The direct URL for this report is: https://f1000research.com/articles/13-601/v1#referee-response-298274 NOTE: it is important to ensure the information in square brackets after the title is included in all citations of this article. COPY CITATION DETAILS Report a concern Author Response 04 Oct 2024 Chitat CHAN , Social Work, Hong Kong Baptist University, Hong Kong, Hong Kong 04 Oct 2024 Author Response Responding to Jonathan Singer, Loyola University Chicago, Chicago, Illinois, USA Literature Review The literature review begins by emphasizing the potential of AI in narrative interviewing, but it would benefit from ... Continue reading Responding to Jonathan Singer, Loyola University Chicago, Chicago, Illinois, USA Literature Review The literature review begins by emphasizing the potential of AI in narrative interviewing, but it would benefit from first defining and describing what narrative interviewing entails. The enthusiastic discussion of AI, including references to “influential bloggers,” feels more promotional than academic, especially since it turns out to be an advertisement for a for-profit company selling narrative interviewing tools. To strengthen the paper, I recommend moving the definitions of key terms like “narrative interviewing” and “conversation analysis” to the beginning of the literature review. Additionally, information from paragraph 5 and the section “Why do assistant’s interview skills matter” should be relocated to the start to provide important context early on. *Response: The literature review part was entirely reorganized. The revised manuscript now includes a more structured literature review. It has been significantly reframed to focus on the democratization of AI chatbots and the emerging trend of customizable AI, which opens new opportunities across various fields, including social work. These advancements enable non-technical users to develop and tailor AI tools for specific applications. As there is limited research on how social workers, especially those with minimal coding experience, can instruct and evaluate AI assistants to conduct narrative interviews, the study investigates this aspect. The role of narrative interviews is instrumental, serving as a case to explore possibilities and provide feedback within a broader context. This sets a clear foundation before discussing the role of AI in narrative interviewing. The language in the literature review occasionally seems exaggerated and inconsistent. For instance, the statement that AI is “becoming a crucial tool for researchers” feels overstated without supporting evidence. Furthermore, it appears contradictory to claim that AI improves analysis while also stating that academics have not “explored how AI could enable or limit narrative interviews.” Clarifying these points and providing supporting evidence will strengthen the argument. *Response: Improved clarity and exaggeration removed. The revised manuscript moderates such language and focuses on informed claims. It highlights the exploratory nature of the study and emphasizes the use of rubrics as tools for qualitative assessment rather than making broad claims about AI’s capabilities. There are also some statements that seem unnecessary and detract from the narrative flow, such as “This brief review of narrative inquirer capabilities is not comprehensive.” Removing such remarks can streamline the text and improve readability. *Response: In the revised manuscript, we eliminated remarks such as “This brief review of narrative inquirer capabilities is not comprehensive” to streamline the text and enhance the overall clarity. Methods Section Introducing a “Methods” header before discussing competence metrics would improve the manuscript’s structure. The justification for the competence metrics needs to be clearer. Rather than stating that “The following metrics were chosen primarily for their simplicity and to facilitate testing and demonstration,” it would be helpful to explain the rationale behind selecting these specific metrics and whether they are based on existing frameworks or newly developed by the authors. Scores: The authors identified six scores but did not talk about how those scores were established, the scales, or anything else about them. Specifically: Competence 1: The explanation of how progression scores are assigned is vague. There is no clear description of the criteria used to determine the progression stage of each utterance. This lack of specificity makes it difficult to understand the validity and reliability of the scoring system. The method’s reliance on previous studies (Au-yeung, 2023; Chan et al., 2020; Connor et al., 2009; Moran et al., 2005; Ramey et al., 2009) without providing a detailed explanation or adaptation process is problematic. It would be helpful to have a more thorough, but not necessarily long explanation of how these methodologies are applied to the current study. Relatedly, the sequential analysis method from therapy research may not be directly applicable to narrative interviews conducted by AI. If this method was developed from narrative interviews, then say so. There are therapy approaches like CBT or behavioral therapies that have structured, goal-oriented sessions. ABFT is task oriented, but still provides the therapist structure. Narrative interviews, from your description, are more open-ended, possibly more like Rogerian or supportive therapy sessions. The adaptation of this method needs more rigorous justification. I was also unclear how and why a higher score isn’t better, and also why you calculated average progression scores over time. How does this score help us to understand how good AI is doing? Competence 2: Is there empirical validation or psychometric testing of the empathy scale? If so, please provide that information for the reader. I would like more information about why a Likert scale is used for empathy, especially since the previous competence argued that higher scores were not necessarily better. That does not seem to be the case with this scale. Competence 3: I didn’t understand the use of the word “Domination” in the subheader. You introduce multiple metrics (agency, coherence, complexity, and meaning-making) without clearly explaining how they are operationalized. Each one of these needs to be defined and the reader needs measurement criteria for each metric, especially since the previous two measures apparently used different scales. For example, the authors say that measuring “agency” cannot be done in isolation, but there is no description of how the scores or definitions of the other measures are used to understand or measure agency. Indeed, it seems that if the concepts in Competence 3 are interrelated, might there be a way to discuss the measures as a coherent package, rather than individual scales? The description of the Case study has many strengths, including the innovative use of ChatGPT to create a chatbot that the researchers would otherwise have not been able to program, highlighting the value of deploying the chatbot on WhatsApp, clearly defining the seven stages, and the ingenuity of having the postgraduate have two conversations – one genuine and one combative. The major limitation of the description of the case study is that it seems to be disconnected from the measures. I know that you provide results in the results section, but it would be helpful to have notes about how each section connects to the measures. *Response: We have reframed the entire methodology. The manuscript now features a "Methods" section that explains the autoethnographic approach and the use of rubrics for assessing fidelity and empathy in the AI assistant’s performance. These rubrics provide a qualitative assessment framework that captures the nuances of narrative interviews, reflecting a shift from objective measurement to qualitative assessment aimed at enriching the exploration and discussion. The revised manuscript provides a detailed explanation of how rubrics are used primarily for assessing fidelity and empathy and introduces sequential analysis (and progression scores) to visualize how the conversations aligned with intended plans. The rubrics are the key assessment tools, while the sequential analysis serves more as an illustration. This revised approach emphasizes understanding the context and content of interactions rather than reducing them to numerical scores. Data Analysis Method and Data Collection Section I was unclear about the context in which the postgrad was having the WhatsApp conversations. Was this in a lab? Were the researchers observing? Was this in private? I didn’t understand why the conversation was divided into four equal quarters. i.e. The choice of four quarters seems arbitrary. I know that it makes it easy to analyze by utterances rather than time, but it seems to miss the importance of space and silence in a narrative interview. The process for manually assigning progression scores is not clearly defined. There is no description of the criteria used by the human research assistant to assign these scores, which raises concerns about the subjectivity and consistency of the ratings. Is there any research on using LLMs or ChatGPT specifically to assigning scores to narrative interviews? If so, it would be good to cite that here. *Response: The WhatsApp conversation took place in private. We have reframed the entire methodology. The manuscript now features a "Methods" section that explains the autoethnographic approach and the use of rubrics for assessing fidelity and empathy in the AI assistant’s performance. The progression scores were primarily instrumental in the visualizations. We used a proportional timeline approach, dividing the conversation into four equal quarters based on the number of utterances instead of actual time, which allows us to account for varying lengths of conversations. For manually assigning progression, the first author and third authors tagged conversations independently, then reviewed and discussed until all disagreements were resolved, which helped ensure reliable and accurate evaluation. The rubrics developed for this study are specifically tailored for narrative interviews and are more flexible than structured, goal-oriented therapy methods. The therapy orientation was de-emphasized. Narrative therapy references were cited because the questioning skills in narrative therapy are useful and help inspire narrative interviews (e.g., creating metaphors). This adaptation is detailed in the methodology section, aligning with the open-ended nature of AI-mediated narrative interviews. Results The agreement between the human research assistant and ChatGPT4 on most metrics in and of itself is encouraging for researchers interested in using LLMs. In finding 2, what distinguishes a 3 from a 4? This is part of my ongoing concern about the paper’s lack of clarity with the measures. The disagreement on coherence in Conversation 2 highlights the need for clear criteria. The authors wrote, “In the rating, the human research assistant and ChatGPT4 mostly agreed on all these metrics, except that there was a disagreement on coherence in Conversation 2, where the human research assistant gave a score of 2 and ChatGPT4 gave a score of 3. After reviewing this, the principal researcher sided with ChatGPT4's score.” The manuscript should detail the specific aspects that led to different ratings and the rationale for siding with ChatGPT4’s score. Especially in these early days of using LLMs for analysis, this kind of transparency will be instructive as well as increase trustworthiness of the analysis. *Response: We have completely restructured the methodology. The manuscript now includes a Methods section that explains the autoethnographic approach and the use of rubrics to assess fidelity and empathy in the AI assistant’s performance. We omitted the section on AI tagging, as it no longer aligns with the revised methodology. The rationale for conducting two conversations, as explained in the updated manuscript, was to better assess the extent to which the chatbot maintains similar levels of adherence and fidelity across different contexts, even when the conversation veers off track. Discussion I thought it was interesting that there were challenges with long speeches in conversation analysis. Even though WhatsApp tends to have shorter exchanges, what are some potential solutions or alternative methods to address these challenges, especially since narrative interviews often involve extended dialogue. I’m thinking specifically about qualitative researchers who are interested in using this for more traditional open-ended narrative interviews, not on WhatsApp. Perhaps this is part of an expanded discussion on the scalability of this type of AI use. I agree that there need to be more refined metrics (and some of my critiques of this paper are in line with that call). How would you propose to improve interrater reliability? Again, are there computational linguists who have looked into human/LLM IRR? After reading through the discussion a few times, I still didn’t have a good understanding of the differences between conversations 1 and 2. For example, in the results section you noted differences in agency, coherence and complexity. Why did the participant exhibit higher agency in Conversation 2? What factors contributed to the lower coherence and complexity in Conversation 2? If you discussed this and I missed it, my apologies. *Response: We have thoroughly restructured the methodology. The revised manuscript adopts an autoethnographic approach and uses rubrics to assess fidelity and empathy in the AI assistant’s performance. These rubrics provide a qualitative framework that captures the nuances of narrative interviews, shifting the focus from objective measurement to a qualitative assessment aimed at deepening exploration and discussion. As such, interrater reliability is no longer a focus in the revised version. The manuscript explains how rubrics are used to assess fidelity and empathy, while sequential analysis and progression scores visually represent how conversations aligned with the intended plans. The rubrics serve as the primary assessment tools, with sequential analysis as an illustration. This approach emphasizes understanding the context and content of interactions rather than reducing them to numerical scores. The revised methodology aims to explore whether an OpenAI Assistant can be effectively instructed by social workers with minimal coding experience, if the chatbot’s performance can be assessed using simple observation-based rubrics, and how these findings contribute to the broader discussion on democratizing AI in specialized fields like social work. Final note: I hope this feedback is helpful. I typically do not review manuscripts until I fully understand both the spirit and letter of the manuscript, but I am providing this feedback in response to the editor’s request for a timely review *Response: We are very thankful for the insightful comments and appreciate the feedback provided. We also want to sincerely apologize, as we were aware that the first version of the manuscript was not fully mature and had several gaps. We recognize that it lacked clarity in several areas, and we are grateful for the opportunity to address those issues. In response, we have completely rewritten the article using the same dataset, but with a more refined focus and clearer articulation of the research aims. This study explores how AI chatbots, particularly those powered by GPT-4, can be applied in social work for conducting narrative interviews. Through an autoethnographic case study approach, social work researchers with limited technical expertise developed and tested an AI chatbot on WhatsApp. The chatbot adhered to a structured interview plan while also demonstrating empathetic engagement with participants. While the AI performed well in maintaining dialogue and showing empathy, challenges were identified in replicating human skills within an AI framework. The study concludes that AI tools can indeed be tailored by social workers to support narrative interviews, though further research is essential to fully address the nuanced needs of the field. Our key focus remains on democratizing AI and empowering domain experts, like social workers, to use these tools effectively. This research enriches the ongoing discussion by exploring the feasibility of AI in specialized fields and emphasizes the importance of interdisciplinary collaboration in developing robust assessment tools to enhance AI implementation. Responding to Jonathan Singer, Loyola University Chicago, Chicago, Illinois, USA Literature Review The literature review begins by emphasizing the potential of AI in narrative interviewing, but it would benefit from first defining and describing what narrative interviewing entails. The enthusiastic discussion of AI, including references to “influential bloggers,” feels more promotional than academic, especially since it turns out to be an advertisement for a for-profit company selling narrative interviewing tools. To strengthen the paper, I recommend moving the definitions of key terms like “narrative interviewing” and “conversation analysis” to the beginning of the literature review. Additionally, information from paragraph 5 and the section “Why do assistant’s interview skills matter” should be relocated to the start to provide important context early on. *Response: The literature review part was entirely reorganized. The revised manuscript now includes a more structured literature review. It has been significantly reframed to focus on the democratization of AI chatbots and the emerging trend of customizable AI, which opens new opportunities across various fields, including social work. These advancements enable non-technical users to develop and tailor AI tools for specific applications. As there is limited research on how social workers, especially those with minimal coding experience, can instruct and evaluate AI assistants to conduct narrative interviews, the study investigates this aspect. The role of narrative interviews is instrumental, serving as a case to explore possibilities and provide feedback within a broader context. This sets a clear foundation before discussing the role of AI in narrative interviewing. The language in the literature review occasionally seems exaggerated and inconsistent. For instance, the statement that AI is “becoming a crucial tool for researchers” feels overstated without supporting evidence. Furthermore, it appears contradictory to claim that AI improves analysis while also stating that academics have not “explored how AI could enable or limit narrative interviews.” Clarifying these points and providing supporting evidence will strengthen the argument. *Response: Improved clarity and exaggeration removed. The revised manuscript moderates such language and focuses on informed claims. It highlights the exploratory nature of the study and emphasizes the use of rubrics as tools for qualitative assessment rather than making broad claims about AI’s capabilities. There are also some statements that seem unnecessary and detract from the narrative flow, such as “This brief review of narrative inquirer capabilities is not comprehensive.” Removing such remarks can streamline the text and improve readability. *Response: In the revised manuscript, we eliminated remarks such as “This brief review of narrative inquirer capabilities is not comprehensive” to streamline the text and enhance the overall clarity. Methods Section Introducing a “Methods” header before discussing competence metrics would improve the manuscript’s structure. The justification for the competence metrics needs to be clearer. Rather than stating that “The following metrics were chosen primarily for their simplicity and to facilitate testing and demonstration,” it would be helpful to explain the rationale behind selecting these specific metrics and whether they are based on existing frameworks or newly developed by the authors. Scores: The authors identified six scores but did not talk about how those scores were established, the scales, or anything else about them. Specifically: Competence 1: The explanation of how progression scores are assigned is vague. There is no clear description of the criteria used to determine the progression stage of each utterance. This lack of specificity makes it difficult to understand the validity and reliability of the scoring system. The method’s reliance on previous studies (Au-yeung, 2023; Chan et al., 2020; Connor et al., 2009; Moran et al., 2005; Ramey et al., 2009) without providing a detailed explanation or adaptation process is problematic. It would be helpful to have a more thorough, but not necessarily long explanation of how these methodologies are applied to the current study. Relatedly, the sequential analysis method from therapy research may not be directly applicable to narrative interviews conducted by AI. If this method was developed from narrative interviews, then say so. There are therapy approaches like CBT or behavioral therapies that have structured, goal-oriented sessions. ABFT is task oriented, but still provides the therapist structure. Narrative interviews, from your description, are more open-ended, possibly more like Rogerian or supportive therapy sessions. The adaptation of this method needs more rigorous justification. I was also unclear how and why a higher score isn’t better, and also why you calculated average progression scores over time. How does this score help us to understand how good AI is doing? Competence 2: Is there empirical validation or psychometric testing of the empathy scale? If so, please provide that information for the reader. I would like more information about why a Likert scale is used for empathy, especially since the previous competence argued that higher scores were not necessarily better. That does not seem to be the case with this scale. Competence 3: I didn’t understand the use of the word “Domination” in the subheader. You introduce multiple metrics (agency, coherence, complexity, and meaning-making) without clearly explaining how they are operationalized. Each one of these needs to be defined and the reader needs measurement criteria for each metric, especially since the previous two measures apparently used different scales. For example, the authors say that measuring “agency” cannot be done in isolation, but there is no description of how the scores or definitions of the other measures are used to understand or measure agency. Indeed, it seems that if the concepts in Competence 3 are interrelated, might there be a way to discuss the measures as a coherent package, rather than individual scales? The description of the Case study has many strengths, including the innovative use of ChatGPT to create a chatbot that the researchers would otherwise have not been able to program, highlighting the value of deploying the chatbot on WhatsApp, clearly defining the seven stages, and the ingenuity of having the postgraduate have two conversations – one genuine and one combative. The major limitation of the description of the case study is that it seems to be disconnected from the measures. I know that you provide results in the results section, but it would be helpful to have notes about how each section connects to the measures. *Response: We have reframed the entire methodology. The manuscript now features a "Methods" section that explains the autoethnographic approach and the use of rubrics for assessing fidelity and empathy in the AI assistant’s performance. These rubrics provide a qualitative assessment framework that captures the nuances of narrative interviews, reflecting a shift from objective measurement to qualitative assessment aimed at enriching the exploration and discussion. The revised manuscript provides a detailed explanation of how rubrics are used primarily for assessing fidelity and empathy and introduces sequential analysis (and progression scores) to visualize how the conversations aligned with intended plans. The rubrics are the key assessment tools, while the sequential analysis serves more as an illustration. This revised approach emphasizes understanding the context and content of interactions rather than reducing them to numerical scores. Data Analysis Method and Data Collection Section I was unclear about the context in which the postgrad was having the WhatsApp conversations. Was this in a lab? Were the researchers observing? Was this in private? I didn’t understand why the conversation was divided into four equal quarters. i.e. The choice of four quarters seems arbitrary. I know that it makes it easy to analyze by utterances rather than time, but it seems to miss the importance of space and silence in a narrative interview. The process for manually assigning progression scores is not clearly defined. There is no description of the criteria used by the human research assistant to assign these scores, which raises concerns about the subjectivity and consistency of the ratings. Is there any research on using LLMs or ChatGPT specifically to assigning scores to narrative interviews? If so, it would be good to cite that here. *Response: The WhatsApp conversation took place in private. We have reframed the entire methodology. The manuscript now features a "Methods" section that explains the autoethnographic approach and the use of rubrics for assessing fidelity and empathy in the AI assistant’s performance. The progression scores were primarily instrumental in the visualizations. We used a proportional timeline approach, dividing the conversation into four equal quarters based on the number of utterances instead of actual time, which allows us to account for varying lengths of conversations. For manually assigning progression, the first author and third authors tagged conversations independently, then reviewed and discussed until all disagreements were resolved, which helped ensure reliable and accurate evaluation. The rubrics developed for this study are specifically tailored for narrative interviews and are more flexible than structured, goal-oriented therapy methods. The therapy orientation was de-emphasized. Narrative therapy references were cited because the questioning skills in narrative therapy are useful and help inspire narrative interviews (e.g., creating metaphors). This adaptation is detailed in the methodology section, aligning with the open-ended nature of AI-mediated narrative interviews. Results The agreement between the human research assistant and ChatGPT4 on most metrics in and of itself is encouraging for researchers interested in using LLMs. In finding 2, what distinguishes a 3 from a 4? This is part of my ongoing concern about the paper’s lack of clarity with the measures. The disagreement on coherence in Conversation 2 highlights the need for clear criteria. The authors wrote, “In the rating, the human research assistant and ChatGPT4 mostly agreed on all these metrics, except that there was a disagreement on coherence in Conversation 2, where the human research assistant gave a score of 2 and ChatGPT4 gave a score of 3. After reviewing this, the principal researcher sided with ChatGPT4's score.” The manuscript should detail the specific aspects that led to different ratings and the rationale for siding with ChatGPT4’s score. Especially in these early days of using LLMs for analysis, this kind of transparency will be instructive as well as increase trustworthiness of the analysis. *Response: We have completely restructured the methodology. The manuscript now includes a Methods section that explains the autoethnographic approach and the use of rubrics to assess fidelity and empathy in the AI assistant’s performance. We omitted the section on AI tagging, as it no longer aligns with the revised methodology. The rationale for conducting two conversations, as explained in the updated manuscript, was to better assess the extent to which the chatbot maintains similar levels of adherence and fidelity across different contexts, even when the conversation veers off track. Discussion I thought it was interesting that there were challenges with long speeches in conversation analysis. Even though WhatsApp tends to have shorter exchanges, what are some potential solutions or alternative methods to address these challenges, especially since narrative interviews often involve extended dialogue. I’m thinking specifically about qualitative researchers who are interested in using this for more traditional open-ended narrative interviews, not on WhatsApp. Perhaps this is part of an expanded discussion on the scalability of this type of AI use. I agree that there need to be more refined metrics (and some of my critiques of this paper are in line with that call). How would you propose to improve interrater reliability? Again, are there computational linguists who have looked into human/LLM IRR? After reading through the discussion a few times, I still didn’t have a good understanding of the differences between conversations 1 and 2. For example, in the results section you noted differences in agency, coherence and complexity. Why did the participant exhibit higher agency in Conversation 2? What factors contributed to the lower coherence and complexity in Conversation 2? If you discussed this and I missed it, my apologies. *Response: We have thoroughly restructured the methodology. The revised manuscript adopts an autoethnographic approach and uses rubrics to assess fidelity and empathy in the AI assistant’s performance. These rubrics provide a qualitative framework that captures the nuances of narrative interviews, shifting the focus from objective measurement to a qualitative assessment aimed at deepening exploration and discussion. As such, interrater reliability is no longer a focus in the revised version. The manuscript explains how rubrics are used to assess fidelity and empathy, while sequential analysis and progression scores visually represent how conversations aligned with the intended plans. The rubrics serve as the primary assessment tools, with sequential analysis as an illustration. This approach emphasizes understanding the context and content of interactions rather than reducing them to numerical scores. The revised methodology aims to explore whether an OpenAI Assistant can be effectively instructed by social workers with minimal coding experience, if the chatbot’s performance can be assessed using simple observation-based rubrics, and how these findings contribute to the broader discussion on democratizing AI in specialized fields like social work. Final note: I hope this feedback is helpful. I typically do not review manuscripts until I fully understand both the spirit and letter of the manuscript, but I am providing this feedback in response to the editor’s request for a timely review *Response: We are very thankful for the insightful comments and appreciate the feedback provided. We also want to sincerely apologize, as we were aware that the first version of the manuscript was not fully mature and had several gaps. We recognize that it lacked clarity in several areas, and we are grateful for the opportunity to address those issues. In response, we have completely rewritten the article using the same dataset, but with a more refined focus and clearer articulation of the research aims. This study explores how AI chatbots, particularly those powered by GPT-4, can be applied in social work for conducting narrative interviews. Through an autoethnographic case study approach, social work researchers with limited technical expertise developed and tested an AI chatbot on WhatsApp. The chatbot adhered to a structured interview plan while also demonstrating empathetic engagement with participants. While the AI performed well in maintaining dialogue and showing empathy, challenges were identified in replicating human skills within an AI framework. The study concludes that AI tools can indeed be tailored by social workers to support narrative interviews, though further research is essential to fully address the nuanced needs of the field. Our key focus remains on democratizing AI and empowering domain experts, like social workers, to use these tools effectively. This research enriches the ongoing discussion by exploring the feasibility of AI in specialized fields and emphasizes the importance of interdisciplinary collaboration in developing robust assessment tools to enhance AI implementation. Competing Interests: NIL Close Report a concern Respond or Comment COMMENTS ON THIS REPORT Author Response 04 Oct 2024 Chitat CHAN , Social Work, Hong Kong Baptist University, Hong Kong, Hong Kong 04 Oct 2024 Author Response Responding to Jonathan Singer, Loyola University Chicago, Chicago, Illinois, USA Literature Review The literature review begins by emphasizing the potential of AI in narrative interviewing, but it would benefit from ... Continue reading Responding to Jonathan Singer, Loyola University Chicago, Chicago, Illinois, USA Literature Review The literature review begins by emphasizing the potential of AI in narrative interviewing, but it would benefit from first defining and describing what narrative interviewing entails. The enthusiastic discussion of AI, including references to “influential bloggers,” feels more promotional than academic, especially since it turns out to be an advertisement for a for-profit company selling narrative interviewing tools. To strengthen the paper, I recommend moving the definitions of key terms like “narrative interviewing” and “conversation analysis” to the beginning of the literature review. Additionally, information from paragraph 5 and the section “Why do assistant’s interview skills matter” should be relocated to the start to provide important context early on. *Response: The literature review part was entirely reorganized. The revised manuscript now includes a more structured literature review. It has been significantly reframed to focus on the democratization of AI chatbots and the emerging trend of customizable AI, which opens new opportunities across various fields, including social work. These advancements enable non-technical users to develop and tailor AI tools for specific applications. As there is limited research on how social workers, especially those with minimal coding experience, can instruct and evaluate AI assistants to conduct narrative interviews, the study investigates this aspect. The role of narrative interviews is instrumental, serving as a case to explore possibilities and provide feedback within a broader context. This sets a clear foundation before discussing the role of AI in narrative interviewing. The language in the literature review occasionally seems exaggerated and inconsistent. For instance, the statement that AI is “becoming a crucial tool for researchers” feels overstated without supporting evidence. Furthermore, it appears contradictory to claim that AI improves analysis while also stating that academics have not “explored how AI could enable or limit narrative interviews.” Clarifying these points and providing supporting evidence will strengthen the argument. *Response: Improved clarity and exaggeration removed. The revised manuscript moderates such language and focuses on informed claims. It highlights the exploratory nature of the study and emphasizes the use of rubrics as tools for qualitative assessment rather than making broad claims about AI’s capabilities. There are also some statements that seem unnecessary and detract from the narrative flow, such as “This brief review of narrative inquirer capabilities is not comprehensive.” Removing such remarks can streamline the text and improve readability. *Response: In the revised manuscript, we eliminated remarks such as “This brief review of narrative inquirer capabilities is not comprehensive” to streamline the text and enhance the overall clarity. Methods Section Introducing a “Methods” header before discussing competence metrics would improve the manuscript’s structure. The justification for the competence metrics needs to be clearer. Rather than stating that “The following metrics were chosen primarily for their simplicity and to facilitate testing and demonstration,” it would be helpful to explain the rationale behind selecting these specific metrics and whether they are based on existing frameworks or newly developed by the authors. Scores: The authors identified six scores but did not talk about how those scores were established, the scales, or anything else about them. Specifically: Competence 1: The explanation of how progression scores are assigned is vague. There is no clear description of the criteria used to determine the progression stage of each utterance. This lack of specificity makes it difficult to understand the validity and reliability of the scoring system. The method’s reliance on previous studies (Au-yeung, 2023; Chan et al., 2020; Connor et al., 2009; Moran et al., 2005; Ramey et al., 2009) without providing a detailed explanation or adaptation process is problematic. It would be helpful to have a more thorough, but not necessarily long explanation of how these methodologies are applied to the current study. Relatedly, the sequential analysis method from therapy research may not be directly applicable to narrative interviews conducted by AI. If this method was developed from narrative interviews, then say so. There are therapy approaches like CBT or behavioral therapies that have structured, goal-oriented sessions. ABFT is task oriented, but still provides the therapist structure. Narrative interviews, from your description, are more open-ended, possibly more like Rogerian or supportive therapy sessions. The adaptation of this method needs more rigorous justification. I was also unclear how and why a higher score isn’t better, and also why you calculated average progression scores over time. How does this score help us to understand how good AI is doing? Competence 2: Is there empirical validation or psychometric testing of the empathy scale? If so, please provide that information for the reader. I would like more information about why a Likert scale is used for empathy, especially since the previous competence argued that higher scores were not necessarily better. That does not seem to be the case with this scale. Competence 3: I didn’t understand the use of the word “Domination” in the subheader. You introduce multiple metrics (agency, coherence, complexity, and meaning-making) without clearly explaining how they are operationalized. Each one of these needs to be defined and the reader needs measurement criteria for each metric, especially since the previous two measures apparently used different scales. For example, the authors say that measuring “agency” cannot be done in isolation, but there is no description of how the scores or definitions of the other measures are used to understand or measure agency. Indeed, it seems that if the concepts in Competence 3 are interrelated, might there be a way to discuss the measures as a coherent package, rather than individual scales? The description of the Case study has many strengths, including the innovative use of ChatGPT to create a chatbot that the researchers would otherwise have not been able to program, highlighting the value of deploying the chatbot on WhatsApp, clearly defining the seven stages, and the ingenuity of having the postgraduate have two conversations – one genuine and one combative. The major limitation of the description of the case study is that it seems to be disconnected from the measures. I know that you provide results in the results section, but it would be helpful to have notes about how each section connects to the measures. *Response: We have reframed the entire methodology. The manuscript now features a "Methods" section that explains the autoethnographic approach and the use of rubrics for assessing fidelity and empathy in the AI assistant’s performance. These rubrics provide a qualitative assessment framework that captures the nuances of narrative interviews, reflecting a shift from objective measurement to qualitative assessment aimed at enriching the exploration and discussion. The revised manuscript provides a detailed explanation of how rubrics are used primarily for assessing fidelity and empathy and introduces sequential analysis (and progression scores) to visualize how the conversations aligned with intended plans. The rubrics are the key assessment tools, while the sequential analysis serves more as an illustration. This revised approach emphasizes understanding the context and content of interactions rather than reducing them to numerical scores. Data Analysis Method and Data Collection Section I was unclear about the context in which the postgrad was having the WhatsApp conversations. Was this in a lab? Were the researchers observing? Was this in private? I didn’t understand why the conversation was divided into four equal quarters. i.e. The choice of four quarters seems arbitrary. I know that it makes it easy to analyze by utterances rather than time, but it seems to miss the importance of space and silence in a narrative interview. The process for manually assigning progression scores is not clearly defined. There is no description of the criteria used by the human research assistant to assign these scores, which raises concerns about the subjectivity and consistency of the ratings. Is there any research on using LLMs or ChatGPT specifically to assigning scores to narrative interviews? If so, it would be good to cite that here. *Response: The WhatsApp conversation took place in private. We have reframed the entire methodology. The manuscript now features a "Methods" section that explains the autoethnographic approach and the use of rubrics for assessing fidelity and empathy in the AI assistant’s performance. The progression scores were primarily instrumental in the visualizations. We used a proportional timeline approach, dividing the conversation into four equal quarters based on the number of utterances instead of actual time, which allows us to account for varying lengths of conversations. For manually assigning progression, the first author and third authors tagged conversations independently, then reviewed and discussed until all disagreements were resolved, which helped ensure reliable and accurate evaluation. The rubrics developed for this study are specifically tailored for narrative interviews and are more flexible than structured, goal-oriented therapy methods. The therapy orientation was de-emphasized. Narrative therapy references were cited because the questioning skills in narrative therapy are useful and help inspire narrative interviews (e.g., creating metaphors). This adaptation is detailed in the methodology section, aligning with the open-ended nature of AI-mediated narrative interviews. Results The agreement between the human research assistant and ChatGPT4 on most metrics in and of itself is encouraging for researchers interested in using LLMs. In finding 2, what distinguishes a 3 from a 4? This is part of my ongoing concern about the paper’s lack of clarity with the measures. The disagreement on coherence in Conversation 2 highlights the need for clear criteria. The authors wrote, “In the rating, the human research assistant and ChatGPT4 mostly agreed on all these metrics, except that there was a disagreement on coherence in Conversation 2, where the human research assistant gave a score of 2 and ChatGPT4 gave a score of 3. After reviewing this, the principal researcher sided with ChatGPT4's score.” The manuscript should detail the specific aspects that led to different ratings and the rationale for siding with ChatGPT4’s score. Especially in these early days of using LLMs for analysis, this kind of transparency will be instructive as well as increase trustworthiness of the analysis. *Response: We have completely restructured the methodology. The manuscript now includes a Methods section that explains the autoethnographic approach and the use of rubrics to assess fidelity and empathy in the AI assistant’s performance. We omitted the section on AI tagging, as it no longer aligns with the revised methodology. The rationale for conducting two conversations, as explained in the updated manuscript, was to better assess the extent to which the chatbot maintains similar levels of adherence and fidelity across different contexts, even when the conversation veers off track. Discussion I thought it was interesting that there were challenges with long speeches in conversation analysis. Even though WhatsApp tends to have shorter exchanges, what are some potential solutions or alternative methods to address these challenges, especially since narrative interviews often involve extended dialogue. I’m thinking specifically about qualitative researchers who are interested in using this for more traditional open-ended narrative interviews, not on WhatsApp. Perhaps this is part of an expanded discussion on the scalability of this type of AI use. I agree that there need to be more refined metrics (and some of my critiques of this paper are in line with that call). How would you propose to improve interrater reliability? Again, are there computational linguists who have looked into human/LLM IRR? After reading through the discussion a few times, I still didn’t have a good understanding of the differences between conversations 1 and 2. For example, in the results section you noted differences in agency, coherence and complexity. Why did the participant exhibit higher agency in Conversation 2? What factors contributed to the lower coherence and complexity in Conversation 2? If you discussed this and I missed it, my apologies. *Response: We have thoroughly restructured the methodology. The revised manuscript adopts an autoethnographic approach and uses rubrics to assess fidelity and empathy in the AI assistant’s performance. These rubrics provide a qualitative framework that captures the nuances of narrative interviews, shifting the focus from objective measurement to a qualitative assessment aimed at deepening exploration and discussion. As such, interrater reliability is no longer a focus in the revised version. The manuscript explains how rubrics are used to assess fidelity and empathy, while sequential analysis and progression scores visually represent how conversations aligned with the intended plans. The rubrics serve as the primary assessment tools, with sequential analysis as an illustration. This approach emphasizes understanding the context and content of interactions rather than reducing them to numerical scores. The revised methodology aims to explore whether an OpenAI Assistant can be effectively instructed by social workers with minimal coding experience, if the chatbot’s performance can be assessed using simple observation-based rubrics, and how these findings contribute to the broader discussion on democratizing AI in specialized fields like social work. Final note: I hope this feedback is helpful. I typically do not review manuscripts until I fully understand both the spirit and letter of the manuscript, but I am providing this feedback in response to the editor’s request for a timely review *Response: We are very thankful for the insightful comments and appreciate the feedback provided. We also want to sincerely apologize, as we were aware that the first version of the manuscript was not fully mature and had several gaps. We recognize that it lacked clarity in several areas, and we are grateful for the opportunity to address those issues. In response, we have completely rewritten the article using the same dataset, but with a more refined focus and clearer articulation of the research aims. This study explores how AI chatbots, particularly those powered by GPT-4, can be applied in social work for conducting narrative interviews. Through an autoethnographic case study approach, social work researchers with limited technical expertise developed and tested an AI chatbot on WhatsApp. The chatbot adhered to a structured interview plan while also demonstrating empathetic engagement with participants. While the AI performed well in maintaining dialogue and showing empathy, challenges were identified in replicating human skills within an AI framework. The study concludes that AI tools can indeed be tailored by social workers to support narrative interviews, though further research is essential to fully address the nuanced needs of the field. Our key focus remains on democratizing AI and empowering domain experts, like social workers, to use these tools effectively. This research enriches the ongoing discussion by exploring the feasibility of AI in specialized fields and emphasizes the importance of interdisciplinary collaboration in developing robust assessment tools to enhance AI implementation. Responding to Jonathan Singer, Loyola University Chicago, Chicago, Illinois, USA Literature Review The literature review begins by emphasizing the potential of AI in narrative interviewing, but it would benefit from first defining and describing what narrative interviewing entails. The enthusiastic discussion of AI, including references to “influential bloggers,” feels more promotional than academic, especially since it turns out to be an advertisement for a for-profit company selling narrative interviewing tools. To strengthen the paper, I recommend moving the definitions of key terms like “narrative interviewing” and “conversation analysis” to the beginning of the literature review. Additionally, information from paragraph 5 and the section “Why do assistant’s interview skills matter” should be relocated to the start to provide important context early on. *Response: The literature review part was entirely reorganized. The revised manuscript now includes a more structured literature review. It has been significantly reframed to focus on the democratization of AI chatbots and the emerging trend of customizable AI, which opens new opportunities across various fields, including social work. These advancements enable non-technical users to develop and tailor AI tools for specific applications. As there is limited research on how social workers, especially those with minimal coding experience, can instruct and evaluate AI assistants to conduct narrative interviews, the study investigates this aspect. The role of narrative interviews is instrumental, serving as a case to explore possibilities and provide feedback within a broader context. This sets a clear foundation before discussing the role of AI in narrative interviewing. The language in the literature review occasionally seems exaggerated and inconsistent. For instance, the statement that AI is “becoming a crucial tool for researchers” feels overstated without supporting evidence. Furthermore, it appears contradictory to claim that AI improves analysis while also stating that academics have not “explored how AI could enable or limit narrative interviews.” Clarifying these points and providing supporting evidence will strengthen the argument. *Response: Improved clarity and exaggeration removed. The revised manuscript moderates such language and focuses on informed claims. It highlights the exploratory nature of the study and emphasizes the use of rubrics as tools for qualitative assessment rather than making broad claims about AI’s capabilities. There are also some statements that seem unnecessary and detract from the narrative flow, such as “This brief review of narrative inquirer capabilities is not comprehensive.” Removing such remarks can streamline the text and improve readability. *Response: In the revised manuscript, we eliminated remarks such as “This brief review of narrative inquirer capabilities is not comprehensive” to streamline the text and enhance the overall clarity. Methods Section Introducing a “Methods” header before discussing competence metrics would improve the manuscript’s structure. The justification for the competence metrics needs to be clearer. Rather than stating that “The following metrics were chosen primarily for their simplicity and to facilitate testing and demonstration,” it would be helpful to explain the rationale behind selecting these specific metrics and whether they are based on existing frameworks or newly developed by the authors. Scores: The authors identified six scores but did not talk about how those scores were established, the scales, or anything else about them. Specifically: Competence 1: The explanation of how progression scores are assigned is vague. There is no clear description of the criteria used to determine the progression stage of each utterance. This lack of specificity makes it difficult to understand the validity and reliability of the scoring system. The method’s reliance on previous studies (Au-yeung, 2023; Chan et al., 2020; Connor et al., 2009; Moran et al., 2005; Ramey et al., 2009) without providing a detailed explanation or adaptation process is problematic. It would be helpful to have a more thorough, but not necessarily long explanation of how these methodologies are applied to the current study. Relatedly, the sequential analysis method from therapy research may not be directly applicable to narrative interviews conducted by AI. If this method was developed from narrative interviews, then say so. There are therapy approaches like CBT or behavioral therapies that have structured, goal-oriented sessions. ABFT is task oriented, but still provides the therapist structure. Narrative interviews, from your description, are more open-ended, possibly more like Rogerian or supportive therapy sessions. The adaptation of this method needs more rigorous justification. I was also unclear how and why a higher score isn’t better, and also why you calculated average progression scores over time. How does this score help us to understand how good AI is doing? Competence 2: Is there empirical validation or psychometric testing of the empathy scale? If so, please provide that information for the reader. I would like more information about why a Likert scale is used for empathy, especially since the previous competence argued that higher scores were not necessarily better. That does not seem to be the case with this scale. Competence 3: I didn’t understand the use of the word “Domination” in the subheader. You introduce multiple metrics (agency, coherence, complexity, and meaning-making) without clearly explaining how they are operationalized. Each one of these needs to be defined and the reader needs measurement criteria for each metric, especially since the previous two measures apparently used different scales. For example, the authors say that measuring “agency” cannot be done in isolation, but there is no description of how the scores or definitions of the other measures are used to understand or measure agency. Indeed, it seems that if the concepts in Competence 3 are interrelated, might there be a way to discuss the measures as a coherent package, rather than individual scales? The description of the Case study has many strengths, including the innovative use of ChatGPT to create a chatbot that the researchers would otherwise have not been able to program, highlighting the value of deploying the chatbot on WhatsApp, clearly defining the seven stages, and the ingenuity of having the postgraduate have two conversations – one genuine and one combative. The major limitation of the description of the case study is that it seems to be disconnected from the measures. I know that you provide results in the results section, but it would be helpful to have notes about how each section connects to the measures. *Response: We have reframed the entire methodology. The manuscript now features a "Methods" section that explains the autoethnographic approach and the use of rubrics for assessing fidelity and empathy in the AI assistant’s performance. These rubrics provide a qualitative assessment framework that captures the nuances of narrative interviews, reflecting a shift from objective measurement to qualitative assessment aimed at enriching the exploration and discussion. The revised manuscript provides a detailed explanation of how rubrics are used primarily for assessing fidelity and empathy and introduces sequential analysis (and progression scores) to visualize how the conversations aligned with intended plans. The rubrics are the key assessment tools, while the sequential analysis serves more as an illustration. This revised approach emphasizes understanding the context and content of interactions rather than reducing them to numerical scores. Data Analysis Method and Data Collection Section I was unclear about the context in which the postgrad was having the WhatsApp conversations. Was this in a lab? Were the researchers observing? Was this in private? I didn’t understand why the conversation was divided into four equal quarters. i.e. The choice of four quarters seems arbitrary. I know that it makes it easy to analyze by utterances rather than time, but it seems to miss the importance of space and silence in a narrative interview. The process for manually assigning progression scores is not clearly defined. There is no description of the criteria used by the human research assistant to assign these scores, which raises concerns about the subjectivity and consistency of the ratings. Is there any research on using LLMs or ChatGPT specifically to assigning scores to narrative interviews? If so, it would be good to cite that here. *Response: The WhatsApp conversation took place in private. We have reframed the entire methodology. The manuscript now features a "Methods" section that explains the autoethnographic approach and the use of rubrics for assessing fidelity and empathy in the AI assistant’s performance. The progression scores were primarily instrumental in the visualizations. We used a proportional timeline approach, dividing the conversation into four equal quarters based on the number of utterances instead of actual time, which allows us to account for varying lengths of conversations. For manually assigning progression, the first author and third authors tagged conversations independently, then reviewed and discussed until all disagreements were resolved, which helped ensure reliable and accurate evaluation. The rubrics developed for this study are specifically tailored for narrative interviews and are more flexible than structured, goal-oriented therapy methods. The therapy orientation was de-emphasized. Narrative therapy references were cited because the questioning skills in narrative therapy are useful and help inspire narrative interviews (e.g., creating metaphors). This adaptation is detailed in the methodology section, aligning with the open-ended nature of AI-mediated narrative interviews. Results The agreement between the human research assistant and ChatGPT4 on most metrics in and of itself is encouraging for researchers interested in using LLMs. In finding 2, what distinguishes a 3 from a 4? This is part of my ongoing concern about the paper’s lack of clarity with the measures. The disagreement on coherence in Conversation 2 highlights the need for clear criteria. The authors wrote, “In the rating, the human research assistant and ChatGPT4 mostly agreed on all these metrics, except that there was a disagreement on coherence in Conversation 2, where the human research assistant gave a score of 2 and ChatGPT4 gave a score of 3. After reviewing this, the principal researcher sided with ChatGPT4's score.” The manuscript should detail the specific aspects that led to different ratings and the rationale for siding with ChatGPT4’s score. Especially in these early days of using LLMs for analysis, this kind of transparency will be instructive as well as increase trustworthiness of the analysis. *Response: We have completely restructured the methodology. The manuscript now includes a Methods section that explains the autoethnographic approach and the use of rubrics to assess fidelity and empathy in the AI assistant’s performance. We omitted the section on AI tagging, as it no longer aligns with the revised methodology. The rationale for conducting two conversations, as explained in the updated manuscript, was to better assess the extent to which the chatbot maintains similar levels of adherence and fidelity across different contexts, even when the conversation veers off track. Discussion I thought it was interesting that there were challenges with long speeches in conversation analysis. Even though WhatsApp tends to have shorter exchanges, what are some potential solutions or alternative methods to address these challenges, especially since narrative interviews often involve extended dialogue. I’m thinking specifically about qualitative researchers who are interested in using this for more traditional open-ended narrative interviews, not on WhatsApp. Perhaps this is part of an expanded discussion on the scalability of this type of AI use. I agree that there need to be more refined metrics (and some of my critiques of this paper are in line with that call). How would you propose to improve interrater reliability? Again, are there computational linguists who have looked into human/LLM IRR? After reading through the discussion a few times, I still didn’t have a good understanding of the differences between conversations 1 and 2. For example, in the results section you noted differences in agency, coherence and complexity. Why did the participant exhibit higher agency in Conversation 2? What factors contributed to the lower coherence and complexity in Conversation 2? If you discussed this and I missed it, my apologies. *Response: We have thoroughly restructured the methodology. The revised manuscript adopts an autoethnographic approach and uses rubrics to assess fidelity and empathy in the AI assistant’s performance. These rubrics provide a qualitative framework that captures the nuances of narrative interviews, shifting the focus from objective measurement to a qualitative assessment aimed at deepening exploration and discussion. As such, interrater reliability is no longer a focus in the revised version. The manuscript explains how rubrics are used to assess fidelity and empathy, while sequential analysis and progression scores visually represent how conversations aligned with the intended plans. The rubrics serve as the primary assessment tools, with sequential analysis as an illustration. This approach emphasizes understanding the context and content of interactions rather than reducing them to numerical scores. The revised methodology aims to explore whether an OpenAI Assistant can be effectively instructed by social workers with minimal coding experience, if the chatbot’s performance can be assessed using simple observation-based rubrics, and how these findings contribute to the broader discussion on democratizing AI in specialized fields like social work. Final note: I hope this feedback is helpful. I typically do not review manuscripts until I fully understand both the spirit and letter of the manuscript, but I am providing this feedback in response to the editor’s request for a timely review *Response: We are very thankful for the insightful comments and appreciate the feedback provided. We also want to sincerely apologize, as we were aware that the first version of the manuscript was not fully mature and had several gaps. We recognize that it lacked clarity in several areas, and we are grateful for the opportunity to address those issues. In response, we have completely rewritten the article using the same dataset, but with a more refined focus and clearer articulation of the research aims. This study explores how AI chatbots, particularly those powered by GPT-4, can be applied in social work for conducting narrative interviews. Through an autoethnographic case study approach, social work researchers with limited technical expertise developed and tested an AI chatbot on WhatsApp. The chatbot adhered to a structured interview plan while also demonstrating empathetic engagement with participants. While the AI performed well in maintaining dialogue and showing empathy, challenges were identified in replicating human skills within an AI framework. The study concludes that AI tools can indeed be tailored by social workers to support narrative interviews, though further research is essential to fully address the nuanced needs of the field. Our key focus remains on democratizing AI and empowering domain experts, like social workers, to use these tools effectively. This research enriches the ongoing discussion by exploring the feasibility of AI in specialized fields and emphasizes the importance of interdisciplinary collaboration in developing robust assessment tools to enhance AI implementation. Competing Interests: NIL Close Report a concern COMMENT ON THIS REPORT Comments on this article Comments (0) Version 2 VERSION 2 PUBLISHED 07 Jun 2024 ADD YOUR COMMENT Comment keyboard_arrow_left keyboard_arrow_right Open Peer Review Reviewer Status info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Reviewer Reports Invited Reviewers 1 2 3 4 Version 2 (revision) 04 Oct 24 read read read Version 1 07 Jun 24 read read Jonathan Singer , Loyola University Chicago, Chicago, USA Neil Ballantyne , Open Polytechnic of New Zealand, Lower Hutt, New Zealand Sunok Lee , Aalto University, Espoo, Finland Anna De Fina , Georgetown University, Washington, DC, USA Comments on this article All Comments (0) Add a comment Sign up for content alerts Sign Up You are now signed up to receive this alert Browse by related subjects keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2025 De Fina A. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 02 Jan 2025 | for Version 2 Anna De Fina , Georgetown University, Washington, DC, USA 0 Views copyright © 2025 De Fina A. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions I found the article overall very good both in presenting the justification for the research and the theoretical/methodological basis for the work. I also thought that the methodology and results were carefully explained and justified. The only changes that I would suggest are the following: 1. I think the authors should better relate the introductory part about the need to study the possible applications of AI developments to social work with the case of the interview. The interview case is presented as a separate issue after the introduction but I feel that these two parts of the article should be better weaved together. The authors should also devote a bit more space to explaining the model that they followed for designing a narrative interview and therefore also exemplify the different parts of the interview. They just name those parts (Naming, Orientation, etc.) but do not explain what they consist of. Finally, among the limitations of the study, they should add the fact that one of the authors was fully conscious of the background and aims of the experiment and might have been biased in their evaluation of the chatbox's performance. Is the background of the case’s history and progression described in sufficient detail? Yes Is the work clearly and accurately presented and does it cite the current literature? Yes If applicable, is the statistical analysis and its interpretation appropriate? Yes Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Yes Competing Interests No competing interests were disclosed. Reviewer Expertise Sociolinguistics, Discourse Analysis I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (0) De Fina A. Peer Review Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.171244.r334817) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-601/v2#referee-response-334817 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Lee S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 14 Nov 2024 | for Version 2 Sunok Lee , Aalto University, Espoo, Finland 0 Views copyright © 2024 Lee S. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This study employed a text-based interview with a single human participant using the WhatsApp messaging app, utilizing OpenAI’s Assistant API with the GPT-4 model. The AI assistant conducted two interview sessions: the first, where the participant was cooperative, and the second, where the participant attempted to sidetrack the AI. This approach aimed to assess the AI’s performance in narrative-based interviewing by evaluating its adherence to interview structure, empathy, and support for participant agency. Researchers used a set of predefined criteria informed by narrative interview literature to measure the AI's performance, including structural fidelity, empathy, and narrative coherence. The assessments were carried out by a human researcher and the AI, and inter-rater reliability was tested to validate consistency in the results. Strengths of the Study The authors are to be commended for their innovative approach to studying AI’s potential in qualitative research, particularly in the complex area of narrative interviewing. Their setup of the AI assistant on WhatsApp, a familiar platform, reflects a practical and thoughtful choice, making AI accessible and potentially easier for social work practitioners to adopt. The decision to use predefined criteria from existing narrative interview literature for performance evaluation is well-grounded, allowing for a structured analysis that aligns with established qualitative research practices. Main Areas for Improvement In-depth Ethical and Critical Issues Analysis While the study provides a promising exploration of AI’s utility in conducting narrative interviews, it lacks a thorough discussion of potential risks, especially concerning interactions with vulnerable users. If AI is to be applied widely, especially in fields involving sensitive interactions, the paper must address the risks of incorrect or insensitive AI responses. The absence of this discussion leaves out critical considerations that might arise if AI were to replace human interaction, such as the potential harm caused by insensitive responses to vulnerable participants. A more comprehensive examination of these ethical issues, including possible measures to mitigate such risks, is essential for providing a balanced view. Interpretation of Progression Scores Although progression scores were introduced to evaluate the interview’s structure, the authors’ interpretation could benefit from additional nuance. The current discussion seems overly definitive in concluding that the AI achieved meaningful progression in narrative structure. Given the limitations of a single participant case study, a more tentative interpretation would be appropriate. Furthermore, while progression scores offer a useful metric, the study could provide practical examples or case-specific explanations to clarify their real-world implications. This would give readers a clearer understanding of how these scores influence the AI’s interview performance and could enhance the study’s transparency. Other Areas for Improvement Clearer Focus on Research Questions and Study Scope The study attempts to evaluate both the AI assistant's performance and the validity of the assessment metrics, which causes some confusion. It would be beneficial to clarify the primary research objective and distinguish between assessing the AI’s performance and the reliability of the evaluation metrics. Additionally, the study would benefit from acknowledging its limitations upfront, particularly as a preliminary or pilot study, to set appropriate expectations for the reader. Balanced Perspective on AI's Potential in Narrative Interviews The literature review highlights AI’s benefits in qualitative research but tends to overstate its potential, neglecting the ongoing debate around AI's limitations in qualitative settings. Acknowledging the contentious nature of AI in qualitative research would present a more balanced viewpoint. This would strengthen the paper by recognizing that while AI offers efficiencies, there are ethical and methodological concerns that still need resolution. Overall This study offers valuable insights into AI’s capabilities in conducting narrative interviews, presenting an innovative approach to exploring AI applications in qualitative research. However, a more balanced perspective on the limitations, especially concerning ethical implications and interpretive caution, would enhance the study’s scientific rigor. Addressing these areas could improve the paper's structure and strengthen its value as a foundational study for further exploration in this field. Is the background of the case’s history and progression described in sufficient detail? Partly Is the work clearly and accurately presented and does it cite the current literature? Partly If applicable, is the statistical analysis and its interpretation appropriate? No Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Partly Competing Interests No competing interests were disclosed. Reviewer Expertise Human-centered AI, Participatory Design, Human-computer Interaction. I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (0) Lee S. Peer Review Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.171244.r329782) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-601/v2#referee-response-329782 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Ballantyne N. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 17 Oct 2024 | for Version 2 Neil Ballantyne , Open Polytechnic of New Zealand, Lower Hutt, Wellington, New Zealand 0 Views copyright © 2024 Ballantyne N. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (0) Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions Many thanks to the authors for investing the time to substantially revise and improve the article. I believe that all of the issues I raised, including the pilot status of the study, have now been addressed and the article is fit for indexing. Competing Interests No competing interests were disclosed. Reviewer Expertise social work, human service technology, artificial intelliegence, qualitative research. I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard. reply Respond to this report Responses (0) Ballantyne N. Peer Review Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.171244.r329218) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-601/v2#referee-response-329218 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Ballantyne N. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 15 Aug 2024 | for Version 1 Neil Ballantyne , Open Polytechnic of New Zealand, Lower Hutt, Wellington, New Zealand 0 Views copyright © 2024 Ballantyne N. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Not Approved info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This single-subject case study examines the performance of an AI assistant conducting a narrative, text-based interview with a single human participant using the WhatsApp messaging app. The researchers designed the AI assistant using OpenAI’s Assistants API with the GPT-4 model. The interview was conducted twice under two different conditions. During the first, the human participant was instructed to cooperate with the AI assistant, and during the second, they were instructed to be disruptive, attempting to sidetrack the AI assistant in response to questions. To assess the performance of the AI assistant on the specific and complex task of narrative-based interviewing, the researchers designed criteria and measures that, based on a review of the narrative interview literature, they deemed to be relevant and valid measures of performance. They devised and applied measures addressing different dimensions of the narrative interview, including interview structure, empathy, narrative coherence, complexity, and support for the agency of the human participant. A human research assistant and ChatGPT 4 assessed the interview data using these measures, and the results were compared using an inter-rater reliability test. The principal researcher reviewed the results and resolved any disagreements. Strengths of the study The researchers are to be commended for devising an imaginative research design to study a complex area of human-computer interaction in a subject area ­(the application of generative AI to qualitative interviewing) that is both highly topical and deeply controversial. The design of the AI assistant and its deployment on WhatsApp seems to have been well considered, and the limitations imposed by this short exchange, text-based platform are recognised. The criteria for selecting the key measures to assess narrative interview performance, drawn from the literature identified, makes sense. However, a whole paper could have been written on this aspect alone. Main areas for improvement Be clear about research questions and primary focus A lot is going on in this study, and the description and rationale for key design choices are sometimes unclear. The problem begins with the abstract and the research questions, where there seems to be slippage between a focus on evaluating the performance of the AI assistant and evaluating the validity and inter-rater reliability of the measures constructed. The first research question asks: How can the proposed metrics help us, as social researchers without a technical background, understand the quality of the AI-driven interviews in this study? In my view, RQ1 would be better focussed on assessing the validity and reliability of the measures selected for evaluating the performance of the AI assistant. This needs to be established before an actual performance assessment of the AI assistant is made. Establishing inter-rater reliability of the measures would be a part of that task. However, this raises the question of why the researchers decided to have only two raters, one human and the other a GenAI. The comparison is interesting but muddies the waters. This machine versus human rater aspect of the study was a surprise since it was not signalled in the abstract, introduction or research questions. A comparison between the ratings made by a GenAI and human raters would be far more interesting once we had established human inter-rater reliability, which could have been achieved with only a few more humans in the frame. Establish from the outset that this is a pilot study The title and the abstract refer to the study as a case study, albeit one with a single participant, giving the reader the impression of a significant, in-depth exploration of a phenomenon. However, in the section on limitations, the authors describe the study as a pilot study and state that the primary goal “ was to explore ideas and stimulate discussion rather than to conduct rigorous experiments”. I recommend that the abstract and introduction be rewritten to reflect this stance. However, this does beg the question as to whether the paper might be better construed as a working paper for feedback and discussion than as a substantive academic article. The paper is very provocative and stimulating in terms of the issues it raises but inconclusive as an assessment of the utility of an AI assistant for narrative interviews. Other areas for improvement A balanced approach to the literature and the claims made The literature review is interesting and includes many relevant articles, but the articles selected and the authors’ comments are prone to over-accentuate the potential of a technology that is still emergent. The application of GenAI to qualitative research is deeply contested in the mainstream literature on qualitative methodology, making it hard to concur with the authors’ views that: The integration of generative AI has revolutionized and democratized the process of qualitative data collection, analysis, and interpretation. This technological leap has boosted efficiency and expanded the horizons of narrative inquiry methods. The authors can improve the literature review by recognising GenAI's emergent nature and the fact that many scholars in the mainstream qualitative research field have strong reservations about its applicability and ethical concerns about its use. Other issues There are several places in the text where the authors’ explanations of the methods used are a little too opaque. For example, the explanation about the progression scores was helpful but left me puzzled about how this was practically implemented. An example might help. Also, in the findings, the authors highlight that the participant exhibited higher agency during conversation two, which is surely explained by the instruction given to the participant to try to throw the interviewer off track. If that isn’t the explanation, a fuller account as to why not would be helpful. Overall I want to encourage the authors to continue their exploration of the use of GenAI in qualitative research, and I believe that this is a very useful working paper that raises many pertinent issues. Responding to the comments above might help to improve the structure and flow of the paper, but the research design issues cannot be remedied without conducting a completely new study. It is for that reason I decided not to recommend the paper for indexing. Is the background of the case’s history and progression described in sufficient detail? No Is the work clearly and accurately presented and does it cite the current literature? Partly If applicable, is the statistical analysis and its interpretation appropriate? Partly Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Partly Is the case presented with sufficient detail to be useful for teaching or other practitioners? Partly References 1. Chan C, Zhao Y: A Case Study on Measuring AI Assistant Competence in Narrative Interviews. F1000Research . 2024; 13 . Publisher Full Text Competing Interests No competing interests were disclosed. Reviewer Expertise social work, human service technology, artificial intelliegence, qualitative research. I confirm that I have read this submission and believe that I have an appropriate level of expertise to state that I do not consider it to be of an acceptable scientific standard, for reasons outlined above. reply Respond to this report Responses (1) Author Response 04 Oct 2024 Chitat CHAN, Social Work, Hong Kong Baptist University, Hong Kong, Hong Kong //Main areas for improvement Be clear about research questions and primary focus A lot is going on in this study, and the description and rationale for key design choices are sometimes unclear. The problem begins with the abstract and the research questions, where there seems to be slippage between a focus on evaluating the performance of the AI assistant and evaluating the validity and inter-rater reliability of the measures constructed. The first research question asks: How can the proposed metrics help us, as social researchers without a technical background, understand the quality of the AI-driven interviews in this study? In my view, RQ1 would be better focussed on assessing the validity and reliability of the measures selected for evaluating the performance of the AI assistant. This needs to be established before an actual performance assessment of the AI assistant is made. Establishing inter-rater reliability of the measures would be a part of that task. However, this raises the question of why the researchers decided to have only two raters, one human and the other a GenAI. The comparison is interesting but muddies the waters. This machine versus human rater aspect of the study was a surprise since it was not signalled in the abstract, introduction or research questions. A comparison between the ratings made by a GenAI and human raters would be far more interesting once we had established human inter-rater reliability, which could have been achieved with only a few more humans in the frame.// *Response: The research questions and focus have been entirely reformulated. The revised manuscript now distinguishes its objectives. It focuses on the democratization of AI chatbots and the emerging trend of customizable AI, which has opened new opportunities across various fields, including social work. The study explores how social workers, particularly those with minimal coding experience, can instruct and evaluate AI assistants to conduct narrative interviews. The role of narrative interviews is used as a case to explore possibilities and provide feedback within a broader context. The emphasis is on qualitative assessment rather than quantitative measurement, aligning with the exploratory nature of the study. //Establish from the outset that this is a pilot study The title and the abstract refer to the study as a case study, albeit one with a single participant, giving the reader the impression of a significant, in-depth exploration of a phenomenon. However, in the section on limitations, the authors describe the study as a pilot study and state that the primary goal “ was to explore ideas and stimulate discussion rather than to conduct rigorous experiments”. I recommend that the abstract and introduction be rewritten to reflect this stance. However, this does beg the question as to whether the paper might be better construed as a working paper for feedback and discussion than as a substantive academic article. The paper is very provocative and stimulating in terms of the issues it raises but inconclusive as an assessment of the utility of an AI assistant for narrative interviews.// *Response: The pilot and exploratory nature of the study is emphasized. The revised abstract and introduction explicitly state that the study is exploratory and serves as a pilot to stimulate discussion about the use of AI in social work. The manuscript now clearly outlines its objectives, focusing on the democratization of AI chatbots and the emerging trend of customizable AI. The study is qualitative and concerned with the extent to which domain knowledge experts can participate in instructing and assessing AI chatbots. //Other areas for improvement A balanced approach to the literature and the claims made The literature review is interesting and includes many relevant articles, but the articles selected and the authors’ comments are prone to over-accentuate the potential of a technology that is still emergent. The application of GenAI to qualitative research is deeply contested in the mainstream literature on qualitative methodology, making it hard to concur with the authors’ views that: The integration of generative AI has revolutionized and democratized the process of qualitative data collection, analysis, and interpretation. This technological leap has boosted efficiency and expanded the horizons of narrative inquiry methods. The authors can improve the literature review by recognising GenAI's emergent nature and the fact that many scholars in the mainstream qualitative research field have strong reservations about its applicability and ethical concerns about its use.// *Response: The literature review has been revised to acknowledge the democratization of AI chatbots and the emerging trend of customizable AI, which opens new opportunities across various fields, including social work. It addresses limitations alongside potential benefits. The discussion part addresses these limitations, noting that the study focused exclusively on OpenAI's product and that findings may not be generalizable to AI tools from other companies. While many principles and challenges identified here could apply broadly to other AI systems, variations in technology and implementation could lead to different outcomes. The introduction of AI tools into practice also brings considerations such as navigating risk management, data privacy, and the uneven development of language models across diverse sociocultural landscapes. While these challenges are important and have been extensively explored by others, the study focuses on exploring how social workers can expand their skill sets and rethink their expertise to engage actively with AI technologies. //Other issues There are several places in the text where the authors’ explanations of the methods used are a little too opaque. For example, the explanation about the progression scores was helpful but left me puzzled about how this was practically implemented. An example might help. Also, in the findings, the authors highlight that the participant exhibited higher agency during conversation two, which is surely explained by the instruction given to the participant to try to throw the interviewer off track. If that isn’t the explanation, a fuller account as to why not would be helpful.// *Response: The revised manuscript provides clearer explanations of the methods used. It explains that the study is autoethnographic, reflecting personal experiences and using rubrics for assessing fidelity and empathy in the AI assistant’s performance. Examples of conversation transcripts are included to illustrate these qualitative assessments, emphasizing the depth and context of interactions rather than mere numerical data. We also enhanced clarity in findings. It was noted that some findings, like the participant exhibiting higher agency, were not well-explained. In the revised discussion, the measure of agency has been removed and integrated into the discussion about empathy and a deeper exploration of meanings. This approach aligns with the study’s focus on qualitative assessment and provides a clearer understanding of the findings. //Overall I want to encourage the authors to continue their exploration of the use of GenAI in qualitative research, and I believe that this is a very useful working paper that raises many pertinent issues. Responding to the comments above might help to improve the structure and flow of the paper, but the research design issues cannot be remedied without conducting a completely new study. It is for that reason I decided not to recommend the paper for indexing.// *Response: We are very grateful for the thoughtful and constructive comments provided. We deeply appreciate the feedback, especially regarding the areas for improvement and the suggestion to frame the study as a pilot. These insights were particularly helpful in guiding us toward a more focused and balanced approach. We also want to acknowledge that the first version of the manuscript had several gaps, and we sincerely apologize for any confusion this may have caused. We recognize that the clarity and direction of the paper needed refinement, and we are thankful for the opportunity to improve. In response, we have rewritten the article using the same dataset but with a more refined focus, especially clarifying the research aims and scope. The revised study explores how AI chatbots, particularly GPT-4, can be applied in social work for conducting narrative interviews. Using an autoethnographic case study approach, social work researchers developed and tested an AI chatbot on WhatsApp. The chatbot followed a structured interview plan and demonstrated empathy, though challenges in replicating human skills within AI were noted. The study concludes that social workers can tailor AI tools for narrative interviews, but further research is needed to address the field's complexities. Our focus remains on democratizing AI and empowering social workers to use these tools effectively while emphasizing the need for interdisciplinary collaboration in developing robust assessment tools. View more View less Competing Interests NIL reply Respond Report a concern Ballantyne N. Peer Review Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.166648.r298273) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-601/v1#referee-response-298273 keyboard_arrow_left Back to all reports Reviewer Report 0 Views copyright © 2024 Singer J. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. 26 Jul 2024 | for Version 1 Jonathan Singer , Loyola University Chicago, Chicago, Illinois, USA 0 Views copyright © 2024 Singer J. This is an open access peer review report distributed under the terms of the Creative Commons Attribution License , which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited. format_quote Cite this report speaker_notes Responses (1) Approved With Reservations info_outline Alongside their report, reviewers assign a status to the article: Approved The paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved Fundamental flaws in the paper seriously undermine the findings and conclusions This manuscript addresses an interesting and important topic. My comments are focused on clarifying certain details to enhance the paper’s clarity and coherence, rather than critiquing its main idea, which I find compelling but somewhat unclear in its current form. Despite reading through the entire paper multiple times, I struggled to fully grasp key parts. Reorganizing the literature review, introducing a clear “Methods” section, and streamlining the manuscript would greatly improve its readability. Literature Review The literature review begins by emphasizing the potential of AI in narrative interviewing, but it would benefit from first defining and describing what narrative interviewing entails. The enthusiastic discussion of AI, including references to “influential bloggers,” feels more promotional than academic, especially since it turns out to be an advertisement for a for-profit company selling narrative interviewing tools. To strengthen the paper, I recommend moving the definitions of key terms like “narrative interviewing” and “conversation analysis” to the beginning of the literature review. Additionally, information from paragraph 5 and the section “Why do assistant’s interview skills matter” should be relocated to the start to provide important context early on. The language in the literature review occasionally seems exaggerated and inconsistent. For instance, the statement that AI is “becoming a crucial tool for researchers” feels overstated without supporting evidence. Furthermore, it appears contradictory to claim that AI improves analysis while also stating that academics have not “explored how AI could enable or limit narrative interviews.” Clarifying these points and providing supporting evidence will strengthen the argument. There are also some statements that seem unnecessary and detract from the narrative flow, such as “This brief review of narrative inquirer capabilities is not comprehensive.” Removing such remarks can streamline the text and improve readability. Methods Section Introducing a “Methods” header before discussing competence metrics would improve the manuscript’s structure. The justification for the competence metrics needs to be clearer. Rather than stating that “The following metrics were chosen primarily for their simplicity and to facilitate testing and demonstration,” it would be helpful to explain the rationale behind selecting these specific metrics and whether they are based on existing frameworks or newly developed by the authors. Scores: The authors identified six scores but did not talk about how those scores were established, the scales, or anything else about them. Specifically: Competence 1: The explanation of how progression scores are assigned is vague. There is no clear description of the criteria used to determine the progression stage of each utterance. This lack of specificity makes it difficult to understand the validity and reliability of the scoring system. The method’s reliance on previous studies (Au-yeung, 2023; Chan et al., 2020; Connor et al., 2009; Moran et al., 2005; Ramey et al., 2009) without providing a detailed explanation or adaptation process is problematic. It would be helpful to have a more thorough, but not necessarily long explanation of how these methodologies are applied to the current study. Relatedly, the sequential analysis method from therapy research may not be directly applicable to narrative interviews conducted by AI. If this method was developed from narrative interviews, then say so. There are therapy approaches like CBT or behavioral therapies that have structured, goal-oriented sessions. ABFT is task oriented, but still provides the therapist structure. Narrative interviews, from your description, are more open-ended, possibly more like Rogerian or supportive therapy sessions. The adaptation of this method needs more rigorous justification. I was also unclear how and why a higher score isn’t better, and also why you calculated average progression scores over time. How does this score help us to understand how good AI is doing? Competence 2: Is there empirical validation or psychometric testing of the empathy scale? If so, please provide that information for the reader. I would like more information about why a Likert scale is used for empathy, especially since the previous competence argued that higher scores were not necessarily better. That does not seem to be the case with this scale. Competence 3: I didn’t understand the use of the word “Domination” in the subheader. You introduce multiple metrics (agency, coherence, complexity, and meaning-making) without clearly explaining how they are operationalized. Each one of these needs to be defined and the reader needs measurement criteria for each metric, especially since the previous two measures apparently used different scales. For example, the authors say that measuring “agency” cannot be done in isolation, but there is no description of how the scores or definitions of the other measures are used to understand or measure agency. Indeed, it seems that if the concepts in Competence 3 are interrelated, might there be a way to discuss the measures as a coherent package, rather than individual scales? The description of the Case study has many strengths, including the innovative use of ChatGPT to create a chatbot that the researchers would otherwise have not been able to program, highlighting the value of deploying the chatbot on WhatsApp, clearly defining the seven stages, and the ingenuity of having the postgraduate have two conversations – one genuine and one combative. The major limitation of the description of the case study is that it seems to be disconnected from the measures. I know that you provide results in the results section, but it would be helpful to have notes about how each section connects to the measures. Data Analysis Method and Data Collection Section I was unclear about the context in which the postgrad was having the WhatsApp conversations. Was this in a lab? Were the researchers observing? Was this in private? I didn’t understand why the conversation was divided into four equal quarters. i.e. The choice of four quarters seems arbitrary. I know that it makes it easy to analyze by utterances rather than time, but it seems to miss the importance of space and silence in a narrative interview. The process for manually assigning progression scores is not clearly defined. There is no description of the criteria used by the human research assistant to assign these scores, which raises concerns about the subjectivity and consistency of the ratings. Is there any research on using LLMs or ChatGPT specifically to assigning scores to narrative interviews? If so, it would be good to cite that here. Results The agreement between the human research assistant and ChatGPT4 on most metrics in and of itself is encouraging for researchers interested in using LLMs. In finding 2, what distinguishes a 3 from a 4? This is part of my ongoing concern about the paper’s lack of clarity with the measures. The disagreement on coherence in Conversation 2 highlights the need for clear criteria. The authors wrote, “In the rating, the human research assistant and ChatGPT4 mostly agreed on all these metrics, except that there was a disagreement on coherence in Conversation 2, where the human research assistant gave a score of 2 and ChatGPT4 gave a score of 3. After reviewing this, the principal researcher sided with ChatGPT4's score.” The manuscript should detail the specific aspects that led to different ratings and the rationale for siding with ChatGPT4’s score. Especially in these early days of using LLMs for analysis, this kind of transparency will be instructive as well as increase trustworthiness of the analysis. Discussion I thought it was interesting that there were challenges with long speeches in conversation analysis. Even though WhatsApp tends to have shorter exchanges, what are some potential solutions or alternative methods to address these challenges, especially since narrative interviews often involve extended dialogue. I’m thinking specifically about qualitative researchers who are interested in using this for more traditional open-ended narrative interviews, not on WhatsApp. Perhaps this is part of an expanded discussion on the scalability of this type of AI use. I agree that there need to be more refined metrics (and some of my critiques of this paper are in line with that call). How would you propose to improve interrater reliability? Again, are there computational linguists who have looked into human/LLM IRR? After reading through the discussion a few times, I still didn’t have a good understanding of the differences between conversations 1 and 2. For example, in the results section you noted differences in agency, coherence and complexity. Why did the participant exhibit higher agency in Conversation 2? What factors contributed to the lower coherence and complexity in Conversation 2? If you discussed this and I missed it, my apologies. Final note: I hope this feedback is helpful. I typically do not review manuscripts until I fully understand both the spirit and letter of the manuscript, but I am providing this feedback in response to the editor’s request for a timely review Is the background of the case’s history and progression described in sufficient detail? No Is the work clearly and accurately presented and does it cite the current literature? Partly If applicable, is the statistical analysis and its interpretation appropriate? Partly Are all the source data underlying the results available to ensure full reproducibility? Yes Are the conclusions drawn adequately supported by the results? Yes Is the case presented with sufficient detail to be useful for teaching or other practitioners? Partly Competing Interests No competing interests were disclosed. Reviewer Expertise Youth suicide prevention, qualitative research, technology and social work education/practice I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above. reply Respond to this report Responses (1) Author Response 04 Oct 2024 Chitat CHAN, Social Work, Hong Kong Baptist University, Hong Kong, Hong Kong Responding to Jonathan Singer, Loyola University Chicago, Chicago, Illinois, USA Literature Review The literature review begins by emphasizing the potential of AI in narrative interviewing, but it would benefit from first defining and describing what narrative interviewing entails. The enthusiastic discussion of AI, including references to “influential bloggers,” feels more promotional than academic, especially since it turns out to be an advertisement for a for-profit company selling narrative interviewing tools. To strengthen the paper, I recommend moving the definitions of key terms like “narrative interviewing” and “conversation analysis” to the beginning of the literature review. Additionally, information from paragraph 5 and the section “Why do assistant’s interview skills matter” should be relocated to the start to provide important context early on. *Response: The literature review part was entirely reorganized. The revised manuscript now includes a more structured literature review. It has been significantly reframed to focus on the democratization of AI chatbots and the emerging trend of customizable AI, which opens new opportunities across various fields, including social work. These advancements enable non-technical users to develop and tailor AI tools for specific applications. As there is limited research on how social workers, especially those with minimal coding experience, can instruct and evaluate AI assistants to conduct narrative interviews, the study investigates this aspect. The role of narrative interviews is instrumental, serving as a case to explore possibilities and provide feedback within a broader context. This sets a clear foundation before discussing the role of AI in narrative interviewing. The language in the literature review occasionally seems exaggerated and inconsistent. For instance, the statement that AI is “becoming a crucial tool for researchers” feels overstated without supporting evidence. Furthermore, it appears contradictory to claim that AI improves analysis while also stating that academics have not “explored how AI could enable or limit narrative interviews.” Clarifying these points and providing supporting evidence will strengthen the argument. *Response: Improved clarity and exaggeration removed. The revised manuscript moderates such language and focuses on informed claims. It highlights the exploratory nature of the study and emphasizes the use of rubrics as tools for qualitative assessment rather than making broad claims about AI’s capabilities. There are also some statements that seem unnecessary and detract from the narrative flow, such as “This brief review of narrative inquirer capabilities is not comprehensive.” Removing such remarks can streamline the text and improve readability. *Response: In the revised manuscript, we eliminated remarks such as “This brief review of narrative inquirer capabilities is not comprehensive” to streamline the text and enhance the overall clarity. Methods Section Introducing a “Methods” header before discussing competence metrics would improve the manuscript’s structure. The justification for the competence metrics needs to be clearer. Rather than stating that “The following metrics were chosen primarily for their simplicity and to facilitate testing and demonstration,” it would be helpful to explain the rationale behind selecting these specific metrics and whether they are based on existing frameworks or newly developed by the authors. Scores: The authors identified six scores but did not talk about how those scores were established, the scales, or anything else about them. Specifically: Competence 1: The explanation of how progression scores are assigned is vague. There is no clear description of the criteria used to determine the progression stage of each utterance. This lack of specificity makes it difficult to understand the validity and reliability of the scoring system. The method’s reliance on previous studies (Au-yeung, 2023; Chan et al., 2020; Connor et al., 2009; Moran et al., 2005; Ramey et al., 2009) without providing a detailed explanation or adaptation process is problematic. It would be helpful to have a more thorough, but not necessarily long explanation of how these methodologies are applied to the current study. Relatedly, the sequential analysis method from therapy research may not be directly applicable to narrative interviews conducted by AI. If this method was developed from narrative interviews, then say so. There are therapy approaches like CBT or behavioral therapies that have structured, goal-oriented sessions. ABFT is task oriented, but still provides the therapist structure. Narrative interviews, from your description, are more open-ended, possibly more like Rogerian or supportive therapy sessions. The adaptation of this method needs more rigorous justification. I was also unclear how and why a higher score isn’t better, and also why you calculated average progression scores over time. How does this score help us to understand how good AI is doing? Competence 2: Is there empirical validation or psychometric testing of the empathy scale? If so, please provide that information for the reader. I would like more information about why a Likert scale is used for empathy, especially since the previous competence argued that higher scores were not necessarily better. That does not seem to be the case with this scale. Competence 3: I didn’t understand the use of the word “Domination” in the subheader. You introduce multiple metrics (agency, coherence, complexity, and meaning-making) without clearly explaining how they are operationalized. Each one of these needs to be defined and the reader needs measurement criteria for each metric, especially since the previous two measures apparently used different scales. For example, the authors say that measuring “agency” cannot be done in isolation, but there is no description of how the scores or definitions of the other measures are used to understand or measure agency. Indeed, it seems that if the concepts in Competence 3 are interrelated, might there be a way to discuss the measures as a coherent package, rather than individual scales? The description of the Case study has many strengths, including the innovative use of ChatGPT to create a chatbot that the researchers would otherwise have not been able to program, highlighting the value of deploying the chatbot on WhatsApp, clearly defining the seven stages, and the ingenuity of having the postgraduate have two conversations – one genuine and one combative. The major limitation of the description of the case study is that it seems to be disconnected from the measures. I know that you provide results in the results section, but it would be helpful to have notes about how each section connects to the measures. *Response: We have reframed the entire methodology. The manuscript now features a "Methods" section that explains the autoethnographic approach and the use of rubrics for assessing fidelity and empathy in the AI assistant’s performance. These rubrics provide a qualitative assessment framework that captures the nuances of narrative interviews, reflecting a shift from objective measurement to qualitative assessment aimed at enriching the exploration and discussion. The revised manuscript provides a detailed explanation of how rubrics are used primarily for assessing fidelity and empathy and introduces sequential analysis (and progression scores) to visualize how the conversations aligned with intended plans. The rubrics are the key assessment tools, while the sequential analysis serves more as an illustration. This revised approach emphasizes understanding the context and content of interactions rather than reducing them to numerical scores. Data Analysis Method and Data Collection Section I was unclear about the context in which the postgrad was having the WhatsApp conversations. Was this in a lab? Were the researchers observing? Was this in private? I didn’t understand why the conversation was divided into four equal quarters. i.e. The choice of four quarters seems arbitrary. I know that it makes it easy to analyze by utterances rather than time, but it seems to miss the importance of space and silence in a narrative interview. The process for manually assigning progression scores is not clearly defined. There is no description of the criteria used by the human research assistant to assign these scores, which raises concerns about the subjectivity and consistency of the ratings. Is there any research on using LLMs or ChatGPT specifically to assigning scores to narrative interviews? If so, it would be good to cite that here. *Response: The WhatsApp conversation took place in private. We have reframed the entire methodology. The manuscript now features a "Methods" section that explains the autoethnographic approach and the use of rubrics for assessing fidelity and empathy in the AI assistant’s performance. The progression scores were primarily instrumental in the visualizations. We used a proportional timeline approach, dividing the conversation into four equal quarters based on the number of utterances instead of actual time, which allows us to account for varying lengths of conversations. For manually assigning progression, the first author and third authors tagged conversations independently, then reviewed and discussed until all disagreements were resolved, which helped ensure reliable and accurate evaluation. The rubrics developed for this study are specifically tailored for narrative interviews and are more flexible than structured, goal-oriented therapy methods. The therapy orientation was de-emphasized. Narrative therapy references were cited because the questioning skills in narrative therapy are useful and help inspire narrative interviews (e.g., creating metaphors). This adaptation is detailed in the methodology section, aligning with the open-ended nature of AI-mediated narrative interviews. Results The agreement between the human research assistant and ChatGPT4 on most metrics in and of itself is encouraging for researchers interested in using LLMs. In finding 2, what distinguishes a 3 from a 4? This is part of my ongoing concern about the paper’s lack of clarity with the measures. The disagreement on coherence in Conversation 2 highlights the need for clear criteria. The authors wrote, “In the rating, the human research assistant and ChatGPT4 mostly agreed on all these metrics, except that there was a disagreement on coherence in Conversation 2, where the human research assistant gave a score of 2 and ChatGPT4 gave a score of 3. After reviewing this, the principal researcher sided with ChatGPT4's score.” The manuscript should detail the specific aspects that led to different ratings and the rationale for siding with ChatGPT4’s score. Especially in these early days of using LLMs for analysis, this kind of transparency will be instructive as well as increase trustworthiness of the analysis. *Response: We have completely restructured the methodology. The manuscript now includes a Methods section that explains the autoethnographic approach and the use of rubrics to assess fidelity and empathy in the AI assistant’s performance. We omitted the section on AI tagging, as it no longer aligns with the revised methodology. The rationale for conducting two conversations, as explained in the updated manuscript, was to better assess the extent to which the chatbot maintains similar levels of adherence and fidelity across different contexts, even when the conversation veers off track. Discussion I thought it was interesting that there were challenges with long speeches in conversation analysis. Even though WhatsApp tends to have shorter exchanges, what are some potential solutions or alternative methods to address these challenges, especially since narrative interviews often involve extended dialogue. I’m thinking specifically about qualitative researchers who are interested in using this for more traditional open-ended narrative interviews, not on WhatsApp. Perhaps this is part of an expanded discussion on the scalability of this type of AI use. I agree that there need to be more refined metrics (and some of my critiques of this paper are in line with that call). How would you propose to improve interrater reliability? Again, are there computational linguists who have looked into human/LLM IRR? After reading through the discussion a few times, I still didn’t have a good understanding of the differences between conversations 1 and 2. For example, in the results section you noted differences in agency, coherence and complexity. Why did the participant exhibit higher agency in Conversation 2? What factors contributed to the lower coherence and complexity in Conversation 2? If you discussed this and I missed it, my apologies. *Response: We have thoroughly restructured the methodology. The revised manuscript adopts an autoethnographic approach and uses rubrics to assess fidelity and empathy in the AI assistant’s performance. These rubrics provide a qualitative framework that captures the nuances of narrative interviews, shifting the focus from objective measurement to a qualitative assessment aimed at deepening exploration and discussion. As such, interrater reliability is no longer a focus in the revised version. The manuscript explains how rubrics are used to assess fidelity and empathy, while sequential analysis and progression scores visually represent how conversations aligned with the intended plans. The rubrics serve as the primary assessment tools, with sequential analysis as an illustration. This approach emphasizes understanding the context and content of interactions rather than reducing them to numerical scores. The revised methodology aims to explore whether an OpenAI Assistant can be effectively instructed by social workers with minimal coding experience, if the chatbot’s performance can be assessed using simple observation-based rubrics, and how these findings contribute to the broader discussion on democratizing AI in specialized fields like social work. Final note: I hope this feedback is helpful. I typically do not review manuscripts until I fully understand both the spirit and letter of the manuscript, but I am providing this feedback in response to the editor’s request for a timely review *Response: We are very thankful for the insightful comments and appreciate the feedback provided. We also want to sincerely apologize, as we were aware that the first version of the manuscript was not fully mature and had several gaps. We recognize that it lacked clarity in several areas, and we are grateful for the opportunity to address those issues. In response, we have completely rewritten the article using the same dataset, but with a more refined focus and clearer articulation of the research aims. This study explores how AI chatbots, particularly those powered by GPT-4, can be applied in social work for conducting narrative interviews. Through an autoethnographic case study approach, social work researchers with limited technical expertise developed and tested an AI chatbot on WhatsApp. The chatbot adhered to a structured interview plan while also demonstrating empathetic engagement with participants. While the AI performed well in maintaining dialogue and showing empathy, challenges were identified in replicating human skills within an AI framework. The study concludes that AI tools can indeed be tailored by social workers to support narrative interviews, though further research is essential to fully address the nuanced needs of the field. Our key focus remains on democratizing AI and empowering domain experts, like social workers, to use these tools effectively. This research enriches the ongoing discussion by exploring the feasibility of AI in specialized fields and emphasizes the importance of interdisciplinary collaboration in developing robust assessment tools to enhance AI implementation. View more View less Competing Interests NIL reply Respond Report a concern Singer J. Peer Review Report For: A Case Study on Assessing AI Assistant Competence in Narrative Interviews [version 2; peer review: 1 approved, 3 approved with reservations] . F1000Research 2024, 13 :601 ( https://doi.org/10.5256/f1000research.166648.r298274) NOTE: it is important to ensure the information in square brackets after the title is included in this citation. The direct URL for this report is: https://f1000research.com/articles/13-601/v1#referee-response-298274 Alongside their report, reviewers assign a status to the article: Approved - the paper is scientifically sound in its current form and only minor, if any, improvements are suggested Approved with reservations - A number of small changes, sometimes more significant revisions are required to address specific details and improve the papers academic merit. Not approved - fundamental flaws in the paper seriously undermine the findings and conclusions Adjust parameters to alter display View on desktop for interactive features Includes Interactive Elements View on desktop for interactive features Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Stay Updated Sign up for content alerts and receive a weekly or monthly email with all newly published articles Register with F1000Research Already registered? Sign in Not now, thanks close PLEASE NOTE If you are an AUTHOR of this article, please check that you signed in with the account associated with this article otherwise we cannot automatically identify your role as an author and your comment will be labelled as a “User Comment”. If you are a REVIEWER of this article, please check that you have signed in with the account associated with this article and then go to your account to submit your report, please do not post your review here. If you do not have access to your original account, please contact us . All commenters must hold a formal affiliation as per our Policies . The information that you give us will be displayed next to your comment. User comments must be in English, comprehensible and relevant to the article under discussion. We reserve the right to remove any comments that we consider to be inappropriate, offensive or otherwise in breach of the User Comment Terms and Conditions . Commenters must not use a comment for personal attacks. When criticisms of the article are based on unpublished data, the data should be made available. I accept the User Comment Terms and Conditions Please confirm that you accept the User Comment Terms and Conditions. Affiliation ✕ refresh Please enter your institution. Note: To add your institution or organisation, start typing the name and then select the correct name from the list. Where applicable, the name will appear in both the original language and in English. Do not paste in the name. If the name does not appear in the drop-down list, we will display the information you have entered. ✕ refresh Country/Region * USA UK Canada China France Germany Afghanistan Aland Islands Albania Algeria American Samoa Andorra Angola Anguilla Antarctica Antigua and Barbuda Argentina Armenia Aruba Australia Austria Azerbaijan Bahamas Bahrain Bangladesh Barbados Belarus Belgium Belize Benin Bermuda Bhutan Bolivia Bosnia and Herzegovina Botswana Bouvet Island Brazil British Indian Ocean Territory British Virgin Islands Brunei Bulgaria Burkina Faso Burundi Cambodia Cameroon Canada Cape Verde Cayman Islands Central African Republic Chad Chile China Christmas Island Cocos (Keeling) Islands Colombia Comoros Congo Cook Islands Costa Rica Cote d'Ivoire Croatia Cuba Cyprus Czech Republic Democratic Republic of the Congo Denmark Djibouti Dominica Dominican Republic Ecuador Egypt El Salvador Equatorial Guinea Eritrea Estonia Ethiopia Falkland Islands Faroe Islands Federated States of Micronesia Fiji Finland France French Guiana French Polynesia French Southern Territories Gabon Georgia Germany Ghana Gibraltar Greece Greenland Grenada Guadeloupe Guam Guatemala Guernsey Guinea Guinea-Bissau Guyana Haiti Heard Island and Mcdonald Islands Holy See (Vatican City State) Honduras Hong Kong Hungary Iceland India Indonesia Iran Iraq Ireland Israel Italy Jamaica Japan Jersey Jordan Kazakhstan Kenya Kiribati Kosovo (Serbia and Montenegro) Kuwait Kyrgyzstan Lao People's Democratic Republic Latvia Lebanon Lesotho Liberia Libya Liechtenstein Lithuania Luxembourg Macao Madagascar Malawi Malaysia Maldives Mali Malta Marshall Islands Martinique Mauritania Mauritius Mayotte Mexico Minor Outlying Islands of the United States Moldova Monaco Mongolia Montenegro Montserrat Morocco Mozambique Myanmar Namibia Nauru Nepal Netherlands Antilles New Caledonia New Zealand Nicaragua Niger Nigeria Niue Norfolk Island North Korea North Macedonia Northern Mariana Islands Norway Oman Pakistan Palau Palestinian Territory Panama Papua New Guinea Paraguay Peru Philippines Pitcairn Poland Portugal Puerto Rico Qatar Reunion Romania Russian Federation Rwanda Saint Helena Saint Kitts and Nevis Saint Lucia Saint Pierre and Miquelon Saint Vincent and the Grenadines Samoa San Marino Sao Tome and Principe Saudi Arabia Senegal Serbia Seychelles Sierra Leone Singapore Slovakia Slovenia Solomon Islands Somalia South Africa South Georgia and the South Sandwich Is South Korea South Sudan Spain Sri Lanka Sudan Suriname Svalbard and Jan Mayen Swaziland Sweden Switzerland Syria Taiwan Tajikistan Tanzania Thailand The Gambia The Netherlands Timor-Leste Togo Tokelau Tonga Trinidad and Tobago Tunisia Turkey Turkmenistan Turks and Caicos Islands Tuvalu UK USA Uganda Ukraine United Arab Emirates United States Virgin Islands Uruguay Uzbekistan Vanuatu Venezuela Vietnam Wallis and Futuna West Bank and Gaza Strip Western Sahara Yemen Zambia Zimbabwe Please select your country/region. You must enter a comment. Competing Interests Please disclose any competing interests that might be construed to influence your judgment of the article's or peer review report's validity or importance. Competing Interests Policy Provide sufficient details of any financial or non-financial competing interests to enable users to assess whether your comments might lead a reasonable person to question your impartiality. Consider the following examples, but note that this is not an exhaustive list: Examples of 'Non-Financial Competing Interests' Within the past 4 years, you have held joint grants, published or collaborated with any of the authors of the selected paper. You have a close personal relationship (e.g. parent, spouse, sibling, or domestic partner) with any of the authors. You are a close professional associate of any of the authors (e.g. scientific mentor, recent student). You work at the same institute as any of the authors. You hope/expect to benefit (e.g. favour or employment) as a result of your submission. You are an Editor for the journal in which the article is published. Examples of 'Financial Competing Interests' You expect to receive, or in the past 4 years have received, any of the following from any commercial organisation that may gain financially from your submission: a salary, fees, funding, reimbursements. You expect to receive, or in the past 4 years have received, shared grant support or other funding with any of the authors. You hold, or are currently applying for, any patents or significant stocks/shares relating to the subject matter of the paper you are commenting on. Please state your competing interests The comment has been saved. An error has occurred. Please try again. Cancel Post var lTitle = "A Case Study on Assessing AI Assistant Competence...".replace("'", ''); var linkedInUrl = "http://www.linkedin.com/shareArticle?url=https://f1000research.com/articles/13-601/v2" + "&title=" + encodeURIComponent(lTitle) + "&summary=" + encodeURIComponent('Read the article by '); var deliciousUrl = "https://del.icio.us/post?url=https://f1000research.com/articles/13-601/v2&title=" + encodeURIComponent(lTitle); var redditUrl = "http://reddit.com/submit?url=https://f1000research.com/articles/13-601/v2" + "&title=" + encodeURIComponent(lTitle); linkedInUrl += encodeURIComponent('Chan C et al.'); var offsetTop = /chrome/i.test( navigator.userAgent ) ? 4 : -10; var addthis_config = { ui_offset_top: offsetTop, services_compact : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_expanded : "facebook,twitter,www.linkedin.com,www.mendeley.com,reddit.com", services_custom : [ { name: "LinkedIn", url: linkedInUrl, icon:"/img/icon/at_linkedin.svg" }, { name: "Mendeley", url: "http://www.mendeley.com/import/?url=https://f1000research.com/articles/13-601/v2/mendeley", icon:"/img/icon/at_mendeley.svg" }, { name: "Reddit", url: redditUrl, icon:"/img/icon/at_reddit.svg" }, ] }; var addthis_share = { url: "https://f1000research.com/articles/13-601", templates : { twitter : "A Case Study on Assessing AI Assistant Competence in Narrative.... Chan C et al., published by " + "@F1000Research" + ", https://f1000research.com/articles/13-601/v2" } }; if (typeof(addthis) != "undefined"){ addthis.addEventListener('addthis.ready', checkCount); addthis.addEventListener('addthis.menu.share', checkCount); } $(".f1r-shares-twitter").attr("href", "https://twitter.com/intent/tweet?text=" + addthis_share.templates.twitter); $(".f1r-shares-facebook").attr("href", "https://www.facebook.com/sharer/sharer.php?u=" + addthis_share.url); $(".f1r-shares-linkedin").attr("href", addthis_config.services_custom[0].url); $(".f1r-shares-reddit").attr("href", addthis_config.services_custom[2].url); $(".f1r-shares-mendelay").attr("href", addthis_config.services_custom[1].url); function checkCount(){ setTimeout(function(){ $(".addthis_button_expanded").each(function(){ var count = $(this).text(); if (count !== "" && count != "0") $(this).removeClass("is-hidden"); else $(this).addClass("is-hidden"); }); }, 1000); } close How to cite this report {{reportCitation}} Cancel Copy Citation Details $(function(){R.ui.buttonDropdowns('.dropdown-for-downloads');}); $(function(){R.ui.toolbarDropdowns('.toolbar-dropdown-for-downloads');}); $.get("/articles/acj/151952/171244") new F1000.Clipboard(); new F1000.ThesaurusTermsDisplay("articles", "article", "171244"); $(document).ready(function() { $( "#frame1" ).on('load', function() { var mydiv = $(this).contents().find("div"); var h = mydiv.height(); console.log(h) }); var tooltipLivingFigure = jQuery(".interactive-living-figure-label .icon-more-info"), titleLivingFigure = tooltipLivingFigure.attr("title"); tooltipLivingFigure.simpletip({ fixed: true, position: ["-115", "30"], baseClass: 'small-tooltip', content:titleLivingFigure + " " }); tooltipLivingFigure.removeAttr("title"); $("body").on("click", ".cite-living-figure", function(e) { e.preventDefault(); var ref = $(this).attr("data-ref"); $(this).closest(".living-figure-list-container").find("#" + ref).fadeIn(200); }); $("body").on("click", ".close-cite-living-figure", function(e) { e.preventDefault(); $(this).closest(".popup-window-wrapper").fadeOut(200); }); $(document).on("mouseup", function(e) { var metricsContainer = $(".article-metrics-popover-wrapper"); if (!metricsContainer.is(e.target) && metricsContainer.has(e.target).length === 0) { $(".article-metrics-close-button").click(); } }); var articleId = $('#articleId').val(); if($("#main-article-count-box").attachArticleMetrics) { $("#main-article-count-box").attachArticleMetrics(articleId, { articleMetricsView: true }); } }); var figshareWidget = $(".new_figshare_widget"); if (figshareWidget.length > 0) { window.figshare.load("f1000", function(Widget) { // Select a tag/tags defined in your page. In this tag we will place the widget. _.map(figshareWidget, function(el){ var widget = new Widget({ articleId: $(el).attr("figshare_articleId") //height:300 // this is the height of the viewer part. [Default: 550] }); widget.initialize(); // initialize the widget widget.mount(el); // mount it in a tag that's on your page // this will save the widget on the global scope for later use from // your JS scripts. This line is optional. //window.widget = widget; }); }); } close Error Close Add Reset F1000.MICROSERVICES.AFFILIATION = ''; $(document).ready(function () { $('.js-affiliations-form').each((index, form) => { new AffiliationForm({ formId: form.id, institutionErrorSelector: '.comment-enter-institution', departmentErrorSelector: '.comment-enter-department', placeSelector: '.js-add-comment-place', stateSelector: '.js-add-comment-state', zipCodeSelector: '.js-add-comment-zipcode', countrySelector: '.js-add-comment-country', countryErrorSelector: '.comment-enter-country', }); }); }); $(document).ready(function () { var reportIds = { "329217": 0, "329218": 7, "298262": 0, "298269": 0, "298270": 0, "298271": 0, "298264": 0, "298265": 0, "298267": 0, "334821": 0, "293988": 0, "334820": 0, "293989": 0, "293990": 0, "334822": 0, "293991": 0, "334817": 5, "298272": 0, "298273": 21, "334819": 0, "298274": 27, "334818": 0, "293987": 0, "293996": 0, "293992": 0, "293993": 0, "293994": 0, "293995": 0, "329781": 0, "329780": 0, "289333": 0, "329783": 0, "289334": 0, "329782": 21, "289335": 0, "329777": 0, "329779": 0, "329778": 0, "289340": 0, "289341": 0, "289342": 0, "329785": 0, "289336": 0, "329784": 0, "289337": 0, "289338": 0, "329786": 0, "289339": 0, }; $(".referee-response-container,.js-referee-report").each(function(index, el) { var reportId = $(el).attr("data-reportid"), reportCount = reportIds[reportId] || 0; $(el).find(".comments-count-container,.js-referee-report-views").html(reportCount); }); var uuidInput = $("#article_uuid"), oldUUId = uuidInput.val(), newUUId = "76711305-7425-4523-8f36-a958fea9418a"; uuidInput.val(newUUId); $("a[href*='article_uuid=']").each(function(index, el) { var newHref = $(el).attr("href").replace(oldUUId, newUUId); $(el).attr("href", newHref); }); }); An innovative open access publishing platform offering rapid publication and open peer review, whilst supporting data deposition and sharing. Browse Gateways Collections How it Works Contact For Developers Cookie Notice Privacy Notice RSS Submit Your Research Follow us © 2012-2026 F1000 Research Ltd. ISSN 2046-1402 | Legal | Partner of Research4Life • CrossRef • ORCID • FAIRSharing R.templateTests.simpleTemplate = R.template(' $text $text $text $text $text '); R.templateTests.runTests(); var F1000platform = new F1000.Platform({ name: "f1000research", displayName: "F1000Research", hostName: "f1000research.com", id: "1", editorialEmail: "[email protected]", infoEmail: "[email protected]", usePmcStats: true }); $(function(){R.ui.dropdowns('.dropdown-for-authors, .dropdown-for-about, .dropdown-for-myresearch');}); // $(function(){R.ui.dropdowns('.dropdown-for-referees');}); $(document).ready(function () { if ($(".cookie-warning").is(":visible")) { $(".sticky").css("margin-bottom", "35px"); $(".devices").addClass("devices-and-cookie-warning"); } $(".cookie-warning .close-button").click(function (e) { $(".devices").removeClass("devices-and-cookie-warning"); $(".sticky").css("margin-bottom", "0"); }); $("#tweeter-feed .tweet-message").each(function (i, message) { var self = $(message); self.html(linkify(self.html())); }); $(".partner").on("mouseenter mouseleave", function() { $(this).find(".gray-scale, .colour").toggleClass("is-hidden"); }); }); Sign In Remember me Forgotten your password? Sign In Cancel Email or password not correct. Please try again Please wait... $(function(){ // Note: All the setup needs to run against a name attribute and *not* the id due the clonish // nature of facebox... $("a[id=googleSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("GOOGLE"); $("form[id=oAuthForm]").submit(); }); $("a[id=facebookSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("FACEBOOK"); $("form[id=oAuthForm]").submit(); }); $("a[id=orcidSignInButton]").click(function(event){ event.preventDefault(); $("input[id=oAuthSystem]").val("ORCID"); $("form[id=oAuthForm]").submit(); }); }); If you've forgotten your password, please enter your email address below and we'll send you instructions on how to reset your password. The email address should be the one you originally registered with F1000. Email address not valid, please try again You registered with F1000 via Google, so we cannot reset your password. To sign in, please click here . If you still need help with your Google account password, please click here . You registered with F1000 via Facebook, so we cannot reset your password. To sign in, please click here . If you still need help with your Facebook account password, please click here . Code not correct, please try again Reset password Cancel Email us for further assistance. Server error, please try again. If your email address is registered with us, we will email you instructions to reset your password. If you think you should have received this email but it has not arrived, please check your spam filters and/or contact for further assistance. Please wait... Register $(document).ready(function () { signIn.createSignInAsRow($("#sign-in-form-gfb-popup")); $(".target-field").each(function () { var uris = $(this).val().split("/"); if (uris.pop() === "login") { $(this).val(uris.toString().replace(",","/")); } }); });