Multi-Sallm: A Multilingual Security Assessment of Generated Code

Multi-Sallm: A Multilingual Security Assessment of Generated Code | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Multi-Sallm: A Multilingual Security Assessment of Generated Code Mohammed Latif Siddiq, Noshin Ulfat, Nishat Raihan, Joanna C. S. Santos, and 1 more This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-7745381/v1 This work is licensed under a CC BY 4.0 License Status: Under Revision Version 1 posted 11 You are reading this latest preprint version Abstract As Large Language Models (LLMs) become increasingly integrated into software engineers' daily workflows, it is critical to ensure the code they generate is not just functionally correct but also secure. While LLMs can boost developer productivity, prior empirical studies have shown that they often produce insecure code. This issue stems from two key factors. First, the datasets commonly used to evaluate LLMs don't accurately reflect real-world software engineering tasks where security is a concern. Instead, they tend to focus on competitive programming problems or classroom-style exercises, which lack the complexity and security risks of production code integrated into larger systems. Second, current evaluation metrics mostly emphasize functional correctness and overlook security aspects altogether. To address these gaps, we introduce Multi-Sallm, a benchmarking framework designed to systematically evaluate LLMs' ability to generate secure code. The framework includes three main components: (1) a novel dataset of security-focused Python prompts translated into 23 natural languages, (2) configurable assessment techniques for analyzing generated code, and (3) new metrics that assess models from the perspective of secure code generation. security evaluation large language models pre-trained transformer model metrics multilingual Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Under Revision Version 1 posted Editorial decision: Revision requested 04 Jan, 2026 Reviews received at journal 26 Dec, 2025 Reviews received at journal 06 Dec, 2025 Reviewers agreed at journal 05 Nov, 2025 Reviewers agreed at journal 02 Nov, 2025 Reviews received at journal 01 Nov, 2025 Reviewers agreed at journal 13 Oct, 2025 Reviewers invited by journal 11 Oct, 2025 Editor assigned by journal 06 Oct, 2025 Submission checks completed at journal 04 Oct, 2025 First submitted to journal 29 Sep, 2025 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-7745381","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":556291619,"identity":"961f13d5-d4d5-49a5-a6fb-616be8480d64","order_by":0,"name":"Mohammed Latif Siddiq","email":"","orcid":"","institution":"University of Notre Dame","correspondingAuthor":false,"prefix":"","firstName":"Mohammed","middleName":"Latif","lastName":"Siddiq","suffix":""},{"id":556291621,"identity":"43832505-b7e4-4bb4-89c5-25b204fe6cc2","order_by":1,"name":"Noshin Ulfat","email":"","orcid":"","institution":"IQVIA Inc","correspondingAuthor":false,"prefix":"","firstName":"Noshin","middleName":"","lastName":"Ulfat","suffix":""},{"id":556291623,"identity":"a0352986-9a6f-436e-b189-9073e87d6176","order_by":2,"name":"Nishat Raihan","email":"","orcid":"","institution":"George Mason University","correspondingAuthor":false,"prefix":"","firstName":"Nishat","middleName":"","lastName":"Raihan","suffix":""},{"id":556291624,"identity":"cf693db3-2620-427a-9ae0-ba201cdb9a79","order_by":3,"name":"Joanna C. S. Santos","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAvUlEQVRIiWNgGAWjYBACxgYYi72BQYKBgZkULTwHiNSCABIJRGphbm9+9rigxs5efubbgzcYKqwTGwhpYew5Zm4841hy4obbeckWDGfSidAyI8FMmoftQIKBdI6ZBGPbYWK0pH+T5vl3AOiwM0At/4jSkmMmzdt2gLHhBg9QSwMxWnrOlEnz9gH9cibH2CLhWLoxQS2G7e3bpHm+AUOs/YzhjQ811rKEtaCoSCCkHATkiVE0CkbBKBgFIxwAAHkKOs2upeXcAAAAAElFTkSuQmCC","orcid":"","institution":"University of Notre Dame","correspondingAuthor":true,"prefix":"","firstName":"Joanna","middleName":"C. S.","lastName":"Santos","suffix":""},{"id":556291627,"identity":"fce34b87-b8f0-498a-878e-be2aae35f120","order_by":4,"name":"Marcos Zampieri","email":"","orcid":"","institution":"George Mason University","correspondingAuthor":false,"prefix":"","firstName":"Marcos","middleName":"","lastName":"Zampieri","suffix":""}],"badges":[],"createdAt":"2025-09-29 21:38:15","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-7745381/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-7745381/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":98313181,"identity":"35331e53-af70-4ced-8874-bcc73511fed4","added_by":"auto","created_at":"2025-12-16 12:43:43","extension":"json","order_by":0,"title":"","display":"","copyAsset":false,"role":"acdc-reference","size":6639,"visible":true,"origin":"","legend":"","description":"","filename":"8038048390a34129ae884147532bb4a4.json","url":"https://assets-eu.researchsquare.com/files/rs-7745381/v1/48e6d960a725172bc7f7c0d8.json"},{"id":98437987,"identity":"4a1961bc-eef5-4a4a-8a1d-45a7e105c094","added_by":"auto","created_at":"2025-12-17 16:58:22","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":7424515,"visible":true,"origin":"","legend":"","description":"","filename":"ASUJASYDE25multiSALLMAFrameworkforEvaluatingLargeLearningModels.pdf","url":"https://assets-eu.researchsquare.com/files/rs-7745381/v1_covered_b993c6f1-45c0-4fd3-8a73-532b3a7de2b0.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Multi-Sallm: A Multilingual Security Assessment of Generated Code","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":true,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"automated-software-engineering","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"ause","sideBox":"Learn more about [Automated Software Engineering](http://link.springer.com/journal/10515)","snPcode":"10515","submissionUrl":"https://submission.nature.com/new-submission/10515/3","title":"Automated Software Engineering","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"security evaluation, large language models, pre-trained transformer model, metrics, multilingual","lastPublishedDoi":"10.21203/rs.3.rs-7745381/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-7745381/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"As Large Language Models (LLMs) become increasingly integrated into software engineers' daily workflows, it is critical to ensure the code they generate is not just functionally correct but also secure. While LLMs can boost developer productivity, prior empirical studies have shown that they often produce insecure code. This issue stems from two key factors. First, the datasets commonly used to evaluate LLMs don't accurately reflect real-world software engineering tasks where security is a concern. Instead, they tend to focus on competitive programming problems or classroom-style exercises, which lack the complexity and security risks of production code integrated into larger systems. Second, current evaluation metrics mostly emphasize functional correctness and overlook security aspects altogether. To address these gaps, we introduce Multi-Sallm, a benchmarking framework designed to systematically evaluate LLMs' ability to generate secure code. The framework includes three main components: (1) a novel dataset of security-focused Python prompts translated into 23 natural languages, (2) configurable assessment techniques for analyzing generated code, and (3) new metrics that assess models from the perspective of secure code generation.","manuscriptTitle":"Multi-Sallm: A Multilingual Security Assessment of Generated Code","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-12-16 12:43:38","doi":"10.21203/rs.3.rs-7745381/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2026-01-04T20:30:28+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-12-26T20:17:14+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-12-06T16:56:07+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"134251155713133254122250673778796329023","date":"2025-11-06T02:05:21+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"33085063875093089137569507064073953046","date":"2025-11-02T19:33:25+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-11-01T22:35:50+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"228509936419457097045241038086745720709","date":"2025-10-13T08:42:16+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2025-10-11T10:00:24+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2025-10-06T09:09:34+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2025-10-04T05:27:03+00:00","index":"","fulltext":""},{"type":"submitted","content":"Automated Software Engineering","date":"2025-09-29T21:27:45+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"automated-software-engineering","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"ause","sideBox":"Learn more about [Automated Software Engineering](http://link.springer.com/journal/10515)","snPcode":"10515","submissionUrl":"https://submission.nature.com/new-submission/10515/3","title":"Automated Software Engineering","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"095d3ccd-7f13-40a7-bf9d-724be1c227c9","owner":[],"postedDate":"December 16th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"in-revision","subjectAreas":[],"tags":[],"updatedAt":"2026-05-20T21:39:07+00:00","versionOfRecord":[],"versionCreatedAt":"2025-12-16 12:43:38","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-7745381","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-7745381","identity":"rs-7745381","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}