Optimizing XGBoost for Intrusion Detection Using a Hybrid Firefly-PSO Algorithm 

Optimizing XGBoost for Intrusion Detection Using a Hybrid Firefly-PSO Algorithm | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Optimizing XGBoost for Intrusion Detection Using a Hybrid Firefly-PSO Algorithm Paul Mensah This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-5784518/v1 This work is licensed under a CC BY 4.0 License Status: Posted Version 1 posted You are reading this latest preprint version Abstract Intrusion detection systems (IDS) play a critical role in safeguarding networks against cyberattacks. Machine learning algorithms, particularly XGBoost, have been widely adopted in IDS for their robustness and efficiency. However, the performance of XGBoost can be significantly improved through hyperparameter optimization. This study proposes a hybrid Firefly-Particle Swarm Optimization (Firefly-PSO) algorithm for tuning XGBoost's hyperparameters to enhance intrusion detection performance. The hybrid algorithm combines the global search ability of PSO and the local search efficiency of the Firefly Algorithm. The proposed model is evaluated on three benchmark datasets: NSL-KDD, CICIDS2017, and UNSW-NB15. Experimental results show that the Firefly-PSO XGBoost model outperforms traditional optimization techniques such as Grid Search and Random Search in terms of accuracy, precision, recall, F1-score, and computational efficiency. Additionally, the model demonstrates excellent generalization capability and minimal false positive and negative rates, making it a robust solution for real-world IDS deployment. The findings of this study highlight the potential of hybrid optimization algorithms for improving machine learning-based IDS. Intrusion Detection XGBoost Firefly Algorithm Particle Swarm Optimization Hyperparameter Optimization Machine Learning Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Posted Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-5784518","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":419194112,"identity":"5e60bf32-ed62-45ef-882a-77dc8be238e4","order_by":0,"name":"Paul Mensah","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABAElEQVRIiWNgGAWjYPACCwY2Bh7GA0CWHIh74AFhLRIgLQwgLcZgLQnEaGGAaklsAPHxadFt7z324GebRD4f+9kDBz622aTPDzv8EGiLnZxuA3YtZmfOpRv2tklYtvHkJRyc2ZaWu/F2mgFQS7Kx2QEcWm7kmEnwtkkYsEnwGBzmOXM4d+PsBJCWA4nbcGm5/8ZM8i9Cy/90w9npH/BrucFjJo2wpeJAgrx0DgFbzuSYG8ucA2rhyTE4OKMi2XCDdE7BgQQDPH45fsbs4ZsyGwP59jOGDz4Y2MnLz07f/OFDhZ0cLi1AwIbKNQCrNMCpHIsW+Qa8qkfBKBgFo2AEAgCSbV7a2ypyYgAAAABJRU5ErkJggg==","orcid":"","institution":"","correspondingAuthor":true,"prefix":"","firstName":"Paul","middleName":"","lastName":"Mensah","suffix":""}],"badges":[],"createdAt":"2025-01-07 23:53:10","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-5784518/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-5784518/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":77279549,"identity":"453eb742-fe96-47f9-89a2-b5549b5bb86d","added_by":"auto","created_at":"2025-02-27 03:40:10","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":173267,"visible":true,"origin":"","legend":"","description":"","filename":"Manuscript.pdf","url":"https://assets-eu.researchsquare.com/files/rs-5784518/v1_covered_b08aaff2-488d-43b1-8564-3cc6e2741bbc.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Optimizing XGBoost for Intrusion Detection Using a Hybrid Firefly-PSO Algorithm ","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":true,"hideJournal":true,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"Intrusion Detection, XGBoost, Firefly Algorithm, Particle Swarm Optimization, Hyperparameter Optimization, Machine Learning","lastPublishedDoi":"10.21203/rs.3.rs-5784518/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-5784518/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Intrusion detection systems (IDS) play a critical role in safeguarding networks against cyberattacks. Machine learning algorithms, particularly XGBoost, have been widely adopted in IDS for their robustness and efficiency. However, the performance of XGBoost can be significantly improved through hyperparameter optimization. This study proposes a hybrid Firefly-Particle Swarm Optimization (Firefly-PSO) algorithm for tuning XGBoost's hyperparameters to enhance intrusion detection performance. The hybrid algorithm combines the global search ability of PSO and the local search efficiency of the Firefly Algorithm. The proposed model is evaluated on three benchmark datasets: NSL-KDD, CICIDS2017, and UNSW-NB15. Experimental results show that the Firefly-PSO XGBoost model outperforms traditional optimization techniques such as Grid Search and Random Search in terms of accuracy, precision, recall, F1-score, and computational efficiency. Additionally, the model demonstrates excellent generalization capability and minimal false positive and negative rates, making it a robust solution for real-world IDS deployment. The findings of this study highlight the potential of hybrid optimization algorithms for improving machine learning-based IDS.","manuscriptTitle":"Optimizing XGBoost for Intrusion Detection Using a Hybrid Firefly-PSO Algorithm ","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-02-27 03:40:01","doi":"10.21203/rs.3.rs-5784518/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"e8104660-8220-435a-9c8a-8e63537f50d9","owner":[],"postedDate":"February 27th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"posted","subjectAreas":[],"tags":[],"updatedAt":"2025-02-27T03:40:01+00:00","versionOfRecord":[],"versionCreatedAt":"2025-02-27 03:40:01","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-5784518","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-5784518","identity":"rs-5784518","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}