The Quality Assurance Machine: A System-Level Software Quality Assurance Architecture for ML-Enabled Systems

The Quality Assurance Machine: A System-Level Software Quality Assurance Architecture for ML-Enabled Systems | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article The Quality Assurance Machine: A System-Level Software Quality Assurance Architecture for ML-Enabled Systems Shane Downing This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-9227068/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 6 You are reading this latest preprint version Abstract This paper evaluates whether a reusable assurance architecture, the Quality Assurance Machine (QAM), can provide effective product and process quality assurance for ML-enabled software platforms. The QAM is a system-level SQA architecture that turns plans and policies into versioned configurations, executes them in controlled environments, and produces preserved run evidence that supports traceability, auditability, and controlled change. The study follows Design Science Research and evaluates the instantiated artifact using eight assurance requirements (AR1-AR8) synthesized from standards-based guidance, including IEEE 730 and ISO/IEC/IEEE 15026. A four-year longitudinal evaluation combines two methods. First, operational evidence from routine regression and release-validation runs, defect linkages, environment governance artifacts, and preserved logs is analyzed to assess product assurance outcomes and process-quality monitoring outcomes (reported as documented, artifact-grounded examples). Second, Design of Experiments is used to characterize infrastructure-driven variation and dominant performance factors, which are then translated into Performance Analysis engine (PAe) baselines. Once deployed, the PAe adds detection-focused monitoring via sigma-gated checks. The results show that the QAM supports the management of assurance work by standardizing SQA workflows. SQA ML-Enabled Systems Machine Learning Design Science Research Quality Assurance Machine (QAM) Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Under Review Version 1 posted Reviewers agreed at journal 13 May, 2026 Reviewers agreed at journal 12 May, 2026 Reviewers invited by journal 05 Apr, 2026 Editor assigned by journal 04 Apr, 2026 Submission checks completed at journal 25 Mar, 2026 First submitted to journal 25 Mar, 2026 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-9227068","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":619417895,"identity":"16e30d78-b91b-4dd1-bc9e-076880e73f17","order_by":0,"name":"Shane Downing","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABQUlEQVRIie2QvWrDMBCAz7jYiyCrjNrkFQSGlGKIX8VC0CwaMnbIkMle3GZ16HuUji4GZzF0dXGHmICzZiopLaEy+YE49V6oPjg4nfRxdwJQKP4kngwK0DEnJ0X9kOpN4ahYYdxQ0O51i1JbuXdSbld6AU/JaPQ+mM1WFd6Mn7rXQbCA8tlxXfP1ZQF3DpucKjSrbklEKz4lom+FaWFfZhkFlg1ZiLhOQSZNBYt+gWjCrUdhEDAKFmEBwPzEQ8ANrPlJU+lFe4W+zSsCW6n0VotacVFnaW607ZkC+U4Z0Bz6RPPrLvIDpaKFmBugTc4UucvwSyqeFQr75v6hsDESNGa+3CVf2thLh3ZzsIAnNvpO3I45L/PNR9HFMik/fcc1p6xcr8fOVXOwPXX3C3w4xce69/vzGleGvm6/VygUiv/MD4WRd1RNq2eRAAAAAElFTkSuQmCC","orcid":"","institution":"Indiana State University","correspondingAuthor":true,"prefix":"","firstName":"Shane","middleName":"","lastName":"Downing","suffix":""}],"badges":[],"createdAt":"2026-03-25 21:08:18","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-9227068/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-9227068/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":106726113,"identity":"6820ca86-d10f-4161-ab73-ce2e4c48b695","added_by":"auto","created_at":"2026-04-12 18:35:23","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":1069192,"visible":true,"origin":"","legend":"","description":"","filename":"QAMforMLenabledsystems.pdf","url":"https://assets-eu.researchsquare.com/files/rs-9227068/v1_covered_05fe373d-abf9-4a61-b816-1e66fa228374.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"The Quality Assurance Machine: A System-Level Software Quality Assurance Architecture for ML-Enabled Systems","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"software-quality-journal","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"sqjo","sideBox":"Learn more about [Software Quality Journal](http://link.springer.com/journal/11219)","snPcode":"11219","submissionUrl":"https://submission.nature.com/new-submission/11219/3","title":"Software Quality Journal","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"SQA, ML-Enabled Systems, Machine Learning, Design Science Research, Quality Assurance Machine (QAM)","lastPublishedDoi":"10.21203/rs.3.rs-9227068/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-9227068/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eThis paper evaluates whether a reusable assurance architecture, the Quality Assurance Machine (QAM), can provide effective product and process quality assurance for ML-enabled software platforms. The QAM is a system-level SQA architecture that turns plans and policies into versioned configurations, executes them in controlled environments, and produces preserved run evidence that supports traceability, auditability, and controlled change. The study follows Design Science Research and evaluates the instantiated artifact using eight assurance requirements (AR1-AR8) synthesized from standards-based guidance, including IEEE 730 and ISO/IEC/IEEE 15026. A four-year longitudinal evaluation combines two methods. First, operational evidence from routine regression and release-validation runs, defect linkages, environment governance artifacts, and preserved logs is analyzed to assess product assurance outcomes and process-quality monitoring outcomes (reported as documented, artifact-grounded examples). Second, Design of Experiments is used to characterize infrastructure-driven variation and dominant performance factors, which are then translated into Performance Analysis engine (PAe) baselines. Once deployed, the PAe adds detection-focused monitoring via sigma-gated checks. The results show that the QAM supports the management of assurance work by standardizing SQA workflows.\u003c/p\u003e \u003cp\u003e \u003c/p\u003e","manuscriptTitle":"The Quality Assurance Machine: A System-Level Software Quality Assurance Architecture for ML-Enabled Systems","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2026-04-10 15:06:22","doi":"10.21203/rs.3.rs-9227068/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"reviewerAgreed","content":"311185599926958013129564861930957391454","date":"2026-05-13T05:38:51+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"55977500396150716574094165626205235","date":"2026-05-12T19:53:43+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2026-04-05T23:27:07+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2026-04-04T08:21:24+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2026-03-26T01:46:51+00:00","index":"","fulltext":""},{"type":"submitted","content":"Software Quality Journal","date":"2026-03-25T20:57:04+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"software-quality-journal","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"sqjo","sideBox":"Learn more about [Software Quality Journal](http://link.springer.com/journal/11219)","snPcode":"11219","submissionUrl":"https://submission.nature.com/new-submission/11219/3","title":"Software Quality Journal","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"2a0a1676-7609-491b-a567-c7d72cc936bf","owner":[],"postedDate":"April 10th, 2026","published":true,"recentEditorialEvents":[{"type":"reviewerAgreed","content":"311185599926958013129564861930957391454","date":"2026-05-13T05:38:51+00:00","index":23,"fulltext":""},{"type":"reviewerAgreed","content":"55977500396150716574094165626205235","date":"2026-05-12T19:53:43+00:00","index":22,"fulltext":""}],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[],"tags":[],"updatedAt":"2026-04-10T15:06:22+00:00","versionOfRecord":[],"versionCreatedAt":"2026-04-10 15:06:22","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-9227068","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-9227068","identity":"rs-9227068","version":["v1"]},"buildId":"XKTyCvWXoU3ODBz1xrDgd","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}