Using Large Language Models for Process Analysis: Identifying Deviations by Comparing Process Models - A Test Report

Using Large Language Models for Process Analysis: Identifying Deviations by Comparing Process Models - A Test Report | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Using Large Language Models for Process Analysis: Identifying Deviations by Comparing Process Models - A Test Report Christian Bennoit, Sasha Zamani, Tobias Greff This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-7754037/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 9 You are reading this latest preprint version Abstract Large Language Models are becoming a promising tool in Business Process Management for analyzing and validating process models. In this study, LLMs were evaluated by comparing correct reference models with systematically modified variants that contained either typical modeling mistakes or harmless variations such as layout or wording changes. The results show that LLMs can reliably distinguish between structural or semantic errors and acceptable variations, demonstrating strong potential for automated model validation. Their performance is shaped by factors such as model complexity, the type and encoding of deviations, and the structure of the comparison setting. Reliability improves when models are provided in clear formats and guided with precise instructions, while extended or continuous interactions across multiple comparisons reduce consistency. At the same time, language differences have little impact, and even multiple models can be processed effectively when context is provided in a controlled way. Taken together, these findings suggest that LLMs can provide meaningful support in process model validation, offering efficiency and abstraction beyond manual comparison, while still requiring careful setup and interaction design to ensure dependable outcomes. Business Process Management (BPM) Large Language Models (LLMs) Process Analysis Model Comparison Error Detection Test Report Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Under Review Version 1 posted Editorial decision: Revision requested 17 Nov, 2025 Reviews received at journal 17 Nov, 2025 Reviews received at journal 15 Nov, 2025 Reviewers agreed at journal 17 Oct, 2025 Reviewers agreed at journal 05 Oct, 2025 Reviewers invited by journal 03 Oct, 2025 Editor assigned by journal 03 Oct, 2025 Submission checks completed at journal 02 Oct, 2025 First submitted to journal 30 Sep, 2025 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-7754037","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":529674223,"identity":"5b5dc328-1fa7-496f-be9a-d3e6720362e9","order_by":0,"name":"Christian Bennoit","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABSElEQVRIie2RsUrDUBSGTyiky0m73lBLXyGlkHaI5FXupZBJBHFwKwmFdipdWxT6CgEXnUwIJEsfoKKDUnByUAS5UhBv01RsYp0d7jdccv/k4z+HAEgk/xECEKQnYHpHKLtZDkr6tDnzCv2hIAZZriqeC/RXBUT+rQASmr1S118Xler56DH84J1GG7RweXJlHdj68knn0DluzPrDpcahPs6V3CdGhJQ0r91KtzWdO4g1x6whkFMjVrw+UmhNd2sM4kAkdlH8AM2aNoiEQs075ZMwX90ozA8KSsgpsYXSXqWKnrzfciBsNsiUm6ISiMHYuqWUKgTNhRiMufG2JbfLQgyGDun6UaUrBhO7zI/OVmvFj5k3vXBIa7LbUp04pVdu9Q79ZBS+aQPLLg+TyyaHHpv1k4eXZ8uqj/M/ZktpT0725BKJRCL5gy9K3m9DiW/IcQAAAABJRU5ErkJggg==","orcid":"","institution":"August-Wilhelm Scheer Institute for Digitized Products and Processes gGmbH","correspondingAuthor":true,"prefix":"","firstName":"Christian","middleName":"","lastName":"Bennoit","suffix":""},{"id":529674224,"identity":"5305ee41-0ba6-46d5-a3a9-3bd996584a43","order_by":1,"name":"Sasha Zamani","email":"","orcid":"","institution":"August-Wilhelm Scheer Institute for Digitized Products and Processes gGmbH","correspondingAuthor":false,"prefix":"","firstName":"Sasha","middleName":"","lastName":"Zamani","suffix":""},{"id":529674225,"identity":"546b4685-f35a-4124-9d9c-5c6a9c0ac5ff","order_by":2,"name":"Tobias Greff","email":"","orcid":"","institution":"August-Wilhelm Scheer Institute for Digitized Products and Processes gGmbH","correspondingAuthor":false,"prefix":"","firstName":"Tobias","middleName":"","lastName":"Greff","suffix":""}],"badges":[],"createdAt":"2025-09-30 17:53:23","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-7754037/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-7754037/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":93689036,"identity":"01ddcc8f-994a-49da-85f5-18ae39f26a32","added_by":"auto","created_at":"2025-10-16 13:49:16","extension":"json","order_by":0,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":5240,"visible":true,"origin":"","legend":"","description":"","filename":"8f7959f6a75d4358981de2644ec53e10.json","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/134e0502005cf3da91efabf3.json"},{"id":93689037,"identity":"4e8080d2-4146-44f7-b5d4-b625337401bf","added_by":"auto","created_at":"2025-10-16 13:49:16","extension":"xml","order_by":1,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":90088,"visible":true,"origin":"","legend":"","description":"","filename":"8f7959f6a75d4358981de2644ec53e101enriched.xml","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/5e59bd4e86da2b4c72b98b43.xml"},{"id":93689039,"identity":"7235e370-0e9c-4a10-a468-ea7c24ca4c94","added_by":"auto","created_at":"2025-10-16 13:49:16","extension":"pdf","order_by":2,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":327969,"visible":true,"origin":"","legend":"","description":"","filename":"BennoitZamaniGreffUsingLLMs....pdf","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/f5b24a9da4b89d701b2c700b.pdf"},{"id":93689043,"identity":"800b0e05-3469-4979-b210-675ac00d4e84","added_by":"auto","created_at":"2025-10-16 13:49:16","extension":"png","order_by":3,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":163069,"visible":true,"origin":"","legend":"","description":"","filename":"combihard02.png","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/76b80505888f85fcfcb6fccf.png"},{"id":93689038,"identity":"dc8cd376-572e-4d8a-9232-f22e5b97522a","added_by":"auto","created_at":"2025-10-16 13:49:16","extension":"pdf","order_by":4,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":56669,"visible":true,"origin":"","legend":"","description":"","filename":"coverletter.pdf","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/aefe49317e116915288b6d17.pdf"},{"id":93689042,"identity":"39fe6e62-d8a1-4ec4-85ac-af20927f1106","added_by":"auto","created_at":"2025-10-16 13:49:16","extension":"png","order_by":5,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":31372,"visible":true,"origin":"","legend":"","description":"","filename":"easy.png","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/280ad72ab7e0db9c1d66da49.png"},{"id":93689046,"identity":"1c5825b0-d269-403b-8912-9ada50e13c38","added_by":"auto","created_at":"2025-10-16 13:49:16","extension":"png","order_by":6,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":103520,"visible":true,"origin":"","legend":"","description":"","filename":"hard.png","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/bd1fa25bdbe18d47117909d2.png"},{"id":93689052,"identity":"6522c5e3-8a84-435d-bbf3-3cf1d753933d","added_by":"auto","created_at":"2025-10-16 13:49:16","extension":"bst","order_by":7,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":146013,"visible":true,"origin":"","legend":"","description":"","filename":"snapacite.bst","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/7f43405bca3604014366b073.bst"},{"id":93690371,"identity":"0fe3ecb0-72d8-4a02-a015-e8f027ab6546","added_by":"auto","created_at":"2025-10-16 13:57:16","extension":"bst","order_by":8,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":29828,"visible":true,"origin":"","legend":"","description":"","filename":"snaps.bst","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/8fc8ad7f94a4a84665d5fe4f.bst"},{"id":93691128,"identity":"04bf7f49-a4aa-45cf-b253-d0d233852671","added_by":"auto","created_at":"2025-10-16 14:05:16","extension":"bst","order_by":9,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":35515,"visible":true,"origin":"","legend":"","description":"","filename":"snbasic.bst","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/659100b5f4314361fd1f1759.bst"},{"id":93690374,"identity":"a4df84d8-d27b-4f88-be6a-30c94789a840","added_by":"auto","created_at":"2025-10-16 13:57:16","extension":"bst","order_by":10,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":33968,"visible":true,"origin":"","legend":"","description":"","filename":"snchicago.bst","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/6e5bf4fb6ec9b8bbeabdc5da.bst"},{"id":93691127,"identity":"b5abfdf2-9c34-4bf0-91c1-125acdac5948","added_by":"auto","created_at":"2025-10-16 14:05:16","extension":"cls","order_by":11,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":55857,"visible":true,"origin":"","legend":"","description":"","filename":"snjnl.cls","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/90fa2f27c7b6d6183ea51c38.cls"},{"id":93690373,"identity":"51988d65-8810-4a17-a9f9-a5e7aa680b7b","added_by":"auto","created_at":"2025-10-16 13:57:16","extension":"bst","order_by":12,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":64023,"visible":true,"origin":"","legend":"","description":"","filename":"snmathphysay.bst","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/5b3d3bacc8f93f825fff408f.bst"},{"id":93690376,"identity":"1cf0b193-0e28-472e-888d-93b3142e5ad7","added_by":"auto","created_at":"2025-10-16 13:57:16","extension":"bst","order_by":13,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":64166,"visible":true,"origin":"","legend":"","description":"","filename":"snmathphysnum.bst","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/8c2a2d9a3eff21bb1c605da5.bst"},{"id":93689049,"identity":"bde1bfe0-6ca9-4d06-ad01-f6d80b47b7f0","added_by":"auto","created_at":"2025-10-16 13:49:16","extension":"bst","order_by":14,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":37333,"visible":true,"origin":"","legend":"","description":"","filename":"snnature.bst","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/055a64956df3280ae6330e69.bst"},{"id":93690378,"identity":"bf47945a-9e2b-4ee5-b4b7-8700018981b9","added_by":"auto","created_at":"2025-10-16 13:57:16","extension":"bst","order_by":15,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":39951,"visible":true,"origin":"","legend":"","description":"","filename":"snvancouveray.bst","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/14df558c2ea6fc925d494e0d.bst"},{"id":93691129,"identity":"6bd44862-4af3-4339-83ea-8409b837bd29","added_by":"auto","created_at":"2025-10-16 14:05:16","extension":"bst","order_by":16,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":40758,"visible":true,"origin":"","legend":"","description":"","filename":"snvancouvernum.bst","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/6460eb8da147b133ebc2f20b.bst"},{"id":93690377,"identity":"be826fd8-63e8-4efe-b410-d57302b0dc52","added_by":"auto","created_at":"2025-10-16 13:57:16","extension":"png","order_by":17,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":96127,"visible":true,"origin":"","legend":"","description":"","filename":"Onlinecombihard02.png","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/d02cbe649500b1eb8e3b7d0e.png"},{"id":93689054,"identity":"32f7989d-043e-425e-b65e-3833a0455fa6","added_by":"auto","created_at":"2025-10-16 13:49:16","extension":"xml","order_by":20,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":103593,"visible":true,"origin":"","legend":"","description":"","filename":"8f7959f6a75d4358981de2644ec53e101structuring.xml","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/6e7d3070fe8ace88adedb379.xml"},{"id":93689051,"identity":"2b0a047f-0bc7-41f1-930b-062835ca4b1a","added_by":"auto","created_at":"2025-10-16 13:49:16","extension":"html","order_by":21,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":98821,"visible":true,"origin":"","legend":"","description":"","filename":"earlyproof.html","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1/0c0bc29b7a110085576ecab6.html"},{"id":93692574,"identity":"d123d561-2548-4305-acef-d86e3a1d2038","added_by":"auto","created_at":"2025-10-16 14:13:16","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":418229,"visible":true,"origin":"","legend":"","description":"","filename":"BennoitZamaniGreffUsingLLMs....pdf","url":"https://assets-eu.researchsquare.com/files/rs-7754037/v1_covered_5b6626c9-dfc9-4c6f-8a40-215fc62c7e46.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Using Large Language Models for Process Analysis: Identifying Deviations by Comparing Process Models - A Test Report","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"process-science","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"","sideBox":"Learn more about [Process Science](https://link.springer.com/journal/44311)","snPcode":"44311","submissionUrl":"https://submission.springernature.com/new-submission/44311","title":"Process Science","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Springer Open","inReviewEnabled":true,"inReviewRevisionsEnabled":true},"keywords":"Business Process Management (BPM), Large Language Models (LLMs), Process Analysis, Model Comparison, Error Detection, Test Report","lastPublishedDoi":"10.21203/rs.3.rs-7754037/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-7754037/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eLarge Language Models are becoming a promising tool in Business Process Management for analyzing and validating process models. In this study, LLMs were evaluated by comparing correct reference models with systematically modified variants that contained either typical modeling mistakes or harmless variations such as layout or wording changes. The results show that LLMs can reliably distinguish between structural or semantic errors and acceptable variations, demonstrating strong potential for automated model validation. Their performance is shaped by factors such as model complexity, the type and encoding of deviations, and the structure of the comparison setting. Reliability improves when models are provided in clear formats and guided with precise instructions, while extended or continuous interactions across multiple comparisons reduce consistency. At the same time, language differences have little impact, and even multiple models can be processed effectively when context is provided in a controlled way. Taken together, these findings suggest that LLMs can provide meaningful support in process model validation, offering efficiency and abstraction beyond manual comparison, while still requiring careful setup and interaction design to ensure dependable outcomes.\u003c/p\u003e","manuscriptTitle":"Using Large Language Models for Process Analysis: Identifying Deviations by Comparing Process Models - A Test Report","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-10-16 13:49:11","doi":"10.21203/rs.3.rs-7754037/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2025-11-17T13:46:58+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-11-17T12:47:47+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-11-15T19:14:35+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"48383505406728809831966921243439343399","date":"2025-10-17T16:31:39+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"117088511571170529706507565292198090102","date":"2025-10-05T15:04:02+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2025-10-03T10:11:50+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2025-10-03T10:00:56+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2025-10-02T07:37:09+00:00","index":"","fulltext":""},{"type":"submitted","content":"Process Science","date":"2025-09-30T17:48:14+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"process-science","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"","sideBox":"Learn more about [Process Science](https://link.springer.com/journal/44311)","snPcode":"44311","submissionUrl":"https://submission.springernature.com/new-submission/44311","title":"Process Science","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Springer Open","inReviewEnabled":true,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"710c7e5c-af8c-4027-a23e-a27c87c0c7a2","owner":[],"postedDate":"October 16th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[],"tags":[],"updatedAt":"2026-04-09T09:54:57+00:00","versionOfRecord":[],"versionCreatedAt":"2025-10-16 13:49:11","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-7754037","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-7754037","identity":"rs-7754037","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}