The feedback loop: A systematic review of how evaluation practices inform conversational agent design

The feedback loop: A systematic review of how evaluation practices inform conversational agent design | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article The feedback loop: A systematic review of how evaluation practices inform conversational agent design Neşe Baz Aktaş, Adem Akbıyık This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-7235054/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 02 Mar, 2026 Read the published version in Universal Access in the Information Society → Version 1 posted 9 You are reading this latest preprint version Abstract Conversational Agents (CAs) have become increasingly integrated in our daily interactions. Despite this growing adoption, research on CA design remains fragmented, often lacking clear connections between design and its effects on user experience. Bridging perspectives from Human-Computer Interaction (HCI) and Information Systems (IS), this study provides a structured, design element-level synthesis of CA design practices and their effect on user experience. To do this, we systematically reviewed 97 empirical studies published in leading HCI and IS venues, following a rigorous three-stage methodology. Building on this, we proposed a CA design framework that identifies design dimensions and maps them with user outcomes to provide actionable guidance for practitioners. The resulting structured framework offers conceptual clarity and practical guidance, laying the groundwork for future empirical research and evidence-based design strategies in CA development. conversational agents chatbot user experience human-computer interaction Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Published Journal Publication published 02 Mar, 2026 Read the published version in Universal Access in the Information Society → Version 1 posted Editorial decision: Revision requested 25 Nov, 2025 Reviews received at journal 21 Nov, 2025 Reviewers agreed at journal 19 Nov, 2025 Reviews received at journal 19 Sep, 2025 Reviewers agreed at journal 17 Sep, 2025 Reviewers invited by journal 11 Sep, 2025 Editor assigned by journal 11 Sep, 2025 Submission checks completed at journal 07 Aug, 2025 First submitted to journal 28 Jul, 2025 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-7235054","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":518310815,"identity":"7c56dfa7-7321-4c4f-b085-41bfc5074206","order_by":0,"name":"Neşe Baz Aktaş","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABLUlEQVRIie2QP2uDQBxAfyLoYnE9F/0KB0JLlvarKIJdHAqBYmlpFeFcJM75GNk7GAK62Ga9UGjNoksGS6GkJZQqUggYS8cO9+Dg3R2P+wPAYPxfjGZwftFNhP3V4QQB8AHuJ8mviUjQn5ITMVwXcGPfyvGcuPXnM2hhar5dkVOQRQfzr/e9ZBTlOobUQYiaZDWdjAHn9kJ5IBYo0QZDUvYSTB0BgeAioBx5OooMwHDuKT7h260m6d8Mv1TVFr5cpC3nZNwmWlwFHz65g7OhhMIxcMRBODEJL22bF1M7bU5ZAEYDSe7oyJzYyoyagTL1DAnT0h55j5mE8vIiyQ8kWbau63dLVpet7AxVi2195V1eq3JozQr3wC+3GD/CEZA6EToZCPbZ9YTBYDAY8A3x0nPLFz6exgAAAABJRU5ErkJggg==","orcid":"","institution":"Sakarya University","correspondingAuthor":true,"prefix":"","firstName":"Neşe","middleName":"Baz","lastName":"Aktaş","suffix":""},{"id":518310816,"identity":"5d945436-e207-4032-b73f-4a3e3c020329","order_by":1,"name":"Adem Akbıyık","email":"","orcid":"","institution":"Sakarya University","correspondingAuthor":false,"prefix":"","firstName":"Adem","middleName":"","lastName":"Akbıyık","suffix":""}],"badges":[],"createdAt":"2025-07-28 14:38:26","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-7235054/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-7235054/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1007/s10209-026-01311-0","type":"published","date":"2026-03-02T15:57:51+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":92413835,"identity":"daf70531-0b84-478b-9f46-a274a97f2ca8","added_by":"auto","created_at":"2025-09-29 13:07:47","extension":"pdf","order_by":0,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":800982,"visible":true,"origin":"","legend":"","description":"","filename":"UAISManuscriptSLR2.pdf","url":"https://assets-eu.researchsquare.com/files/rs-7235054/v1/609cb4bfff7806c7227589c0.pdf"},{"id":92413834,"identity":"2f88e88d-5e8e-49ae-8a65-25212d58dc0a","added_by":"auto","created_at":"2025-09-29 13:07:47","extension":"json","order_by":1,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":3775,"visible":true,"origin":"","legend":"","description":"","filename":"c3089cd7d84345c78a8a08e5b0efe788.json","url":"https://assets-eu.researchsquare.com/files/rs-7235054/v1/133b86e7b69fa07a02bd8ad1.json"},{"id":104250643,"identity":"da1c15ec-308c-4fa3-aaf2-e3c46126a615","added_by":"auto","created_at":"2026-03-09 16:03:21","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":722885,"visible":true,"origin":"","legend":"","description":"","filename":"UAISManuscriptSLR2.pdf","url":"https://assets-eu.researchsquare.com/files/rs-7235054/v1_covered_3fb57319-875c-4d03-9c92-0dcd36f56189.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"The feedback loop: A systematic review of how evaluation practices inform conversational agent design","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"universal-access-in-the-information-society","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"uais","sideBox":"Learn more about [Universal Access in the Information Society](http://link.springer.com/journal/10209)","snPcode":"10209","submissionUrl":"https://submission.nature.com/new-submission/10209/3","title":"Universal Access in the Information Society","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"conversational agents, chatbot, user experience, human-computer interaction","lastPublishedDoi":"10.21203/rs.3.rs-7235054/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-7235054/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Conversational Agents (CAs) have become increasingly integrated in our daily interactions. Despite this growing adoption, research on CA design remains fragmented, often lacking clear connections between design and its effects on user experience. Bridging perspectives from Human-Computer Interaction (HCI) and Information Systems (IS), this study provides a structured, design element-level synthesis of CA design practices and their effect on user experience. To do this, we systematically reviewed 97 empirical studies published in leading HCI and IS venues, following a rigorous three-stage methodology. Building on this, we proposed a CA design framework that identifies design dimensions and maps them with user outcomes to provide actionable guidance for practitioners. The resulting structured framework offers conceptual clarity and practical guidance, laying the groundwork for future empirical research and evidence-based design strategies in CA development.","manuscriptTitle":"The feedback loop: A systematic review of how evaluation practices inform conversational agent design","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-09-29 13:07:42","doi":"10.21203/rs.3.rs-7235054/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2025-11-25T14:58:56+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-11-21T14:16:31+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"71511166775036580820151885786336714296","date":"2025-11-19T12:16:43+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-09-19T09:28:12+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"330490496670894833960436488370955796676","date":"2025-09-17T12:38:29+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2025-09-11T16:57:28+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2025-09-11T16:47:31+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2025-08-07T11:02:27+00:00","index":"","fulltext":""},{"type":"submitted","content":"Universal Access in the Information Society","date":"2025-07-28T14:31:44+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"universal-access-in-the-information-society","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"uais","sideBox":"Learn more about [Universal Access in the Information Society](http://link.springer.com/journal/10209)","snPcode":"10209","submissionUrl":"https://submission.nature.com/new-submission/10209/3","title":"Universal Access in the Information Society","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"69b8ce92-ae45-4344-8902-5cf97515f7e9","owner":[],"postedDate":"September 29th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[],"tags":[],"updatedAt":"2026-03-09T16:01:13+00:00","versionOfRecord":{"articleIdentity":"rs-7235054","link":"https://doi.org/10.1007/s10209-026-01311-0","journal":{"identity":"universal-access-in-the-information-society","isVorOnly":false,"title":"Universal Access in the Information Society"},"publishedOn":"2026-03-02 15:57:51","publishedOnDateReadable":"March 2nd, 2026"},"versionCreatedAt":"2025-09-29 13:07:42","video":"","vorDoi":"10.1007/s10209-026-01311-0","vorDoiUrl":"https://doi.org/10.1007/s10209-026-01311-0","workflowStages":[]},"version":"v1","identity":"rs-7235054","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-7235054","identity":"rs-7235054","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}