An Evaluation of the Safety of ChatGPT with Malicious Prompt Injection

An Evaluation of the Safety of ChatGPT with Malicious Prompt Injection | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article An Evaluation of the Safety of ChatGPT with Malicious Prompt Injection Jiang Han, Mingming Guo This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-4487194/v1 This work is licensed under a CC BY 4.0 License Status: Posted Version 1 posted You are reading this latest preprint version Abstract Artificial intelligence systems, particularly those involving sophisticated neural network architectures like ChatGPT, have demonstrated remarkable capabilities in generating human-like text. However, the susceptibility of these systems to malicious prompt injections poses significant risks, necessitating comprehensive evaluations of their safety and robustness. The study presents a novel automated framework for systematically injecting and analyzing malicious prompts to assess the vulnerabilities of ChatGPT. Results indicate substantial rates of harmful responses across various scenarios, highlighting critical areas for improvement in model defenses. The findings underscore the importance of advanced adversarial training, real-time monitoring, and interdisciplinary collaboration to enhance the ethical deployment of AI systems. Recommendations for future research emphasize the need for robust safety mechanisms and transparent model operations to mitigate the risks associated with adversarial inputs. Artificial Intelligence and Machine Learning AI safety adversarial attacks ethical AI model robustness prompt injection Full Text Additional Declarations The authors declare no competing interests. Cite Share Download PDF Status: Posted Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-4487194","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":307435438,"identity":"afc83d18-f45d-4798-ab5a-63e62a15bdce","order_by":0,"name":"Jiang Han","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA6UlEQVRIiWNgGAWjYBACAzDJIwEkmI+B2WzsxGthS2NgSABSzERpgegzA2thIKTFnP3sAQYGGQt5c/413x58/LFNno+ZgfHDxxzcWix78hJADjPcOePtdsMZCbcN25gZmCVnbsPjsAM5BiAtjBtunN0mzZNwmxGohY2ZF5+W82/AWuw33DjzDKTFnrCWGxBbEjec72EDaUkkQgvEluQNN9jMJGek3U5uY2Zsxu+X80BbGHvqbDecP/xM4oPNbdv57c0HP3zEowUI2H//7QFSEgkwAcYGvOoh4AcQ8x8gQuEoGAWjYBSMSAAA1zZJso89i90AAAAASUVORK5CYII=","orcid":"https://orcid.org/0009-0006-9872-4294","institution":"Mingri Software Co., Ltd","correspondingAuthor":true,"prefix":"","firstName":"Jiang","middleName":"","lastName":"Han","suffix":""},{"id":307435439,"identity":"af44c5c1-2a12-432e-9001-3d859b4bdb9d","order_by":1,"name":"Mingming Guo","email":"","orcid":"https://orcid.org/0009-0000-9006-892X","institution":"Mingri Software Co., Ltd","correspondingAuthor":false,"prefix":"","firstName":"Mingming","middleName":"","lastName":"Guo","suffix":""}],"badges":[],"createdAt":"2024-05-27 23:06:14","currentVersionCode":1,"declarations":{"humanSubjects":false,"vertebrateSubjects":false,"conflictsOfInterestStatement":false,"humanSubjectEthicalGuidelines":false,"humanSubjectConsent":false,"humanSubjectClinicalTrial":false,"humanSubjectCaseReport":false,"vertebrateSubjectEthicalGuidelines":false},"doi":"10.21203/rs.3.rs-4487194/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-4487194/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":57386391,"identity":"ac368cc1-19ac-4d07-9123-b0bfa58fb483","added_by":"auto","created_at":"2024-05-30 03:59:44","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":193016,"visible":true,"origin":"","legend":"","description":"","filename":"preprint.pdf","url":"https://assets-eu.researchsquare.com/files/rs-4487194/v1_covered_7e79010b-4419-4292-97ec-64032cbc262e.pdf"}],"financialInterests":"The authors declare no competing interests.","formattedTitle":"\u003cp\u003eAn Evaluation of the Safety of ChatGPT with Malicious Prompt Injection\u003c/p\u003e","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":true,"hideJournal":true,"highlight":"","institution":"Mingri Software Co., Ltd","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"AI safety, adversarial attacks, ethical AI, model robustness, prompt injection","lastPublishedDoi":"10.21203/rs.3.rs-4487194/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-4487194/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eArtificial intelligence systems, particularly those involving sophisticated neural network architectures like ChatGPT, have demonstrated remarkable capabilities in generating human-like text. However, the susceptibility of these systems to malicious prompt injections poses significant risks, necessitating comprehensive evaluations of their safety and robustness. The study presents a novel automated framework for systematically injecting and analyzing malicious prompts to assess the vulnerabilities of ChatGPT. Results indicate substantial rates of harmful responses across various scenarios, highlighting critical areas for improvement in model defenses. The findings underscore the importance of advanced adversarial training, real-time monitoring, and interdisciplinary collaboration to enhance the ethical deployment of AI systems. Recommendations for future research emphasize the need for robust safety mechanisms and transparent model operations to mitigate the risks associated with adversarial inputs.\u003c/p\u003e","manuscriptTitle":"An Evaluation of the Safety of ChatGPT with Malicious Prompt Injection","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-05-30 03:51:37","doi":"10.21203/rs.3.rs-4487194/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"d035cd03-dd41-41de-ba69-8ef71514cfa0","owner":[],"postedDate":"May 30th, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"posted","subjectAreas":[{"id":32471104,"name":"Artificial Intelligence and Machine Learning"}],"tags":[],"updatedAt":"2024-05-30T03:51:37+00:00","versionOfRecord":[],"versionCreatedAt":"2024-05-30 03:51:37","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-4487194","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-4487194","identity":"rs-4487194","version":["v1"]},"buildId":"qtupq5eGEP_6zYnWcrvyt","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}