Creation, Evaluation and Self-Validation of Simulation Models with Large Language Models

Creation, Evaluation and Self-Validation of Simulation Models with Large Language Models | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Creation, Evaluation and Self-Validation of Simulation Models with Large Language Models Tobias Möltner, Peter Manzl, Michael Pieber, Johannes Gerstmayr This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-6566994/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 13 Nov, 2025 Read the published version in Neurocomputing → Version 1 posted You are reading this latest preprint version Abstract Engineering tasks are significantly underrepresented in current large language model (LLM) datasets and research, despite their complexity and practical importance. These tasks often demand a deep mathematical and involve a combination of textual descriptions, visual representations, and numerical data. Moreover, engineering frequently relies on accepted approximations and models rather than exact values. Therefore, the present paper advances the integration of LLMs into mechanical engineering by introducing a comprehensive framework for automated simulation model generation and validation. The framework is designed as a benchmark and focuses on mechanical engineering problems in dynamics in particular on multibody dynamics simulation models in Python. It allows for the creation of a large number of test cases due to its use of parametrized models with ground truth solutions, allowing evaluation for executability and correctness. Lastly, LLM-agents are employed to generate simulation models and perform self-evaluation through a predefined set of validation methods, assessing models for parametrization errors. Evaluation results using classical F-score metrics demonstrate that most tested LLMs identify a majority of incorrect models, while the best-performing model achieves high accuracy in differing between correct and wrong simulation models. Mechanical Engineering Artificial Intelligence and Machine Learning multibody dynamics mechanical system simulation large language models model validation AI agents Full Text Additional Declarations The authors declare no competing interests. Cite Share Download PDF Status: Published Journal Publication published 13 Nov, 2025 Read the published version in Neurocomputing → Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-6566994","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":450352031,"identity":"5d0460f7-0402-458b-9a61-b67cf6827473","order_by":0,"name":"Tobias Möltner","email":"","orcid":"https://orcid.org/0009-0004-2471-9439","institution":"University of Innsbruck","correspondingAuthor":false,"prefix":"","firstName":"Tobias","middleName":"","lastName":"Möltner","suffix":""},{"id":450352032,"identity":"62d5d8a4-7ddb-48dd-9263-92c0ae143a80","order_by":1,"name":"Peter Manzl","email":"","orcid":"https://orcid.org/0000-0002-7707-5524","institution":"University of Innsbruck","correspondingAuthor":false,"prefix":"","firstName":"Peter","middleName":"","lastName":"Manzl","suffix":""},{"id":450352033,"identity":"b6fa8cc5-b18c-482f-8189-2a279b24b4b1","order_by":2,"name":"Michael Pieber","email":"","orcid":"https://orcid.org/0000-0002-3745-3439","institution":"University of Innsbruck","correspondingAuthor":false,"prefix":"","firstName":"Michael","middleName":"","lastName":"Pieber","suffix":""},{"id":450352034,"identity":"f5a779f3-8d84-4639-931f-43f08c2fc37a","order_by":3,"name":"Johannes Gerstmayr","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABNUlEQVRIie3RMUvDQBTA8RcOkuVicTsINH6ElEApWM1XuSOQKSJuHTNdl+rc4pfIVDJGArocnSM3FcHJQSlIioNei6lCY3AUvP90HO/HOxIAne4PZiW4PuX1HcqBJfRHYuwI3gxtB036RfI2QqAm2FPXLWR8ebO6yIYBEHT7sq7mrifFK1lmEXTGubF6biCTRejMRMQSYobXmMpeujibEyZiIIIi0rRlGnuOzQsK5MlHQKWRClsRPgIo1Wubif9m83f1sPOVUVEZpAI/bolbAqoayOE07qstuZGQGKmvJpkipiIxeCWYTVscLPrHNg8Zx5GPcCTDmTD9ARMR7gnGB2KfHFgTX9r8NOhYxYNRDeXJlUDL+3UWdrt3RVGO9skusz4cfb4Fb35aC/iW+8s5nU6n+z99APliajSet8Z+AAAAAElFTkSuQmCC","orcid":"https://orcid.org/0000-0003-3576-1830","institution":"University of Innsbruck","correspondingAuthor":true,"prefix":"","firstName":"Johannes","middleName":"","lastName":"Gerstmayr","suffix":""}],"badges":[],"createdAt":"2025-04-30 17:23:54","currentVersionCode":1,"declarations":{"humanSubjects":false,"vertebrateSubjects":false,"conflictsOfInterestStatement":false,"humanSubjectEthicalGuidelines":false,"humanSubjectConsent":false,"humanSubjectClinicalTrial":false,"humanSubjectCaseReport":false,"vertebrateSubjectEthicalGuidelines":false},"doi":"10.21203/rs.3.rs-6566994/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-6566994/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1016/j.neucom.2025.132030","type":"published","date":"2025-11-14T00:00:00+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":96393123,"identity":"16db8341-bd19-46b9-848d-8da97a028b4e","added_by":"auto","created_at":"2025-11-20 14:47:48","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":1844095,"visible":true,"origin":"","legend":"","description":"","filename":"LLMPaperVirtualLabssubmitted.pdf","url":"https://assets-eu.researchsquare.com/files/rs-6566994/v1_covered_f5f7a678-a71f-42b2-a658-55d5c482acd9.pdf"}],"financialInterests":"The authors declare no competing interests.","formattedTitle":"\u003cp\u003eCreation, Evaluation and Self-Validation of Simulation Models with Large Language Models\u003c/p\u003e","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":true,"hideJournal":false,"highlight":"","institution":"Universität Innsbruck","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"multibody dynamics, mechanical system simulation, large language models, model validation, AI agents","lastPublishedDoi":"10.21203/rs.3.rs-6566994/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-6566994/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eEngineering tasks are significantly underrepresented in current large language model (LLM) datasets and research, despite their complexity and practical importance. These tasks often demand a deep mathematical and involve a combination of textual descriptions, visual representations, and numerical data. Moreover, engineering frequently relies on accepted approximations and models rather than exact values. Therefore, the present paper advances the integration of LLMs into mechanical engineering by introducing a comprehensive framework for automated simulation model generation and validation. The framework is designed as a benchmark and focuses on mechanical engineering problems in dynamics in particular on multibody dynamics simulation models in Python. It allows for the creation of a large number of test cases due to its use of parametrized models with ground truth solutions, allowing evaluation for executability and correctness. Lastly, LLM-agents are employed to generate simulation models and perform self-evaluation through a predefined set of validation methods, assessing models for parametrization errors. Evaluation results using classical F-score metrics demonstrate that most tested LLMs identify a majority of incorrect models, while the best-performing model achieves high accuracy in differing between correct and wrong simulation models.\u003c/p\u003e","manuscriptTitle":"Creation, Evaluation and Self-Validation of Simulation Models with Large Language Models","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-05-02 02:33:03","doi":"10.21203/rs.3.rs-6566994/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"c12c0a11-db57-49d2-94a1-73c1e74f69e1","owner":[],"postedDate":"May 2nd, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[{"id":47925035,"name":"Mechanical Engineering"},{"id":47925036,"name":"Artificial Intelligence and Machine Learning"}],"tags":[],"updatedAt":"2025-11-20T14:47:39+00:00","versionOfRecord":{"articleIdentity":"rs-6566994","link":"https://doi.org/10.1016/j.neucom.2025.132030","journal":{"identity":"neurocomputing","isVorOnly":true,"title":"Neurocomputing"},"publishedOn":"2025-11-14 00:00:00","publishedOnDateReadable":"November 14th, 2025"},"versionCreatedAt":"2025-05-02 02:33:03","video":"","vorDoi":"10.1016/j.neucom.2025.132030","vorDoiUrl":"https://doi.org/10.1016/j.neucom.2025.132030","workflowStages":[]},"version":"v1","identity":"rs-6566994","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-6566994","identity":"rs-6566994","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}