Systematic Evaluation of Label Noise Effects on Accuracy and Calibration in Deep Neural Networks

Systematic Evaluation of Label Noise Effects on Accuracy and Calibration in Deep Neural Networks | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Systematic Evaluation of Label Noise Effects on Accuracy and Calibration in Deep Neural Networks Christopher Boseak This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-7197053/v1 This work is licensed under a CC BY 4.0 License Status: Posted Version 1 posted You are reading this latest preprint version Abstract Label noise is a pervasive issue in real-world datasets that can degrade both the accuracy and calibration of deep neural networks. In this study, we systematically examine how symmetric (random) and asymmetric (class-dependent) label noise influence model accuracy and confidence calibration in image classification using the CIFAR-10 dataset and a ResNet-18 architecture. We apply five levels of label noise (0%, 10%, 20%, 40%, 60%) and evaluate their effects using metrics such as test accuracy, Expected Calibration Error (ECE), and predictive entropy. Our findings show that increasing noise levels significantly degrade classification accuracy and impair model calibration. In particular, asymmetric noise at a 60% corruption level causes test accuracy to drop to approximately 38.7% while ECE surges above 35%, indicating extreme overconfidence in incorrect predictions. By contrast, symmetric noise at the same noise level yields higher predictive entropy (uncertainty) and a comparatively modest miscalibration (ECE ∼9%). These results highlight the importance of distinguishing noise types when assessing model robustness and reliability. All experiments are reproducible, with code and data publicly available to facilitate further investigation. Artificial Intelligence and Machine Learning machine learning deep neural networks artificial intelligence Full Text Additional Declarations The authors declare no competing interests. Cite Share Download PDF Status: Posted Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-7197053","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":489805455,"identity":"b6acfa51-765a-4005-91bf-01ddcbabb4e5","order_by":0,"name":"Christopher Boseak","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAt0lEQVRIiWNgGAWjYDACCTCZIAciDzwgRYsxWEsCKVoSG8AUMTr4Zzcf/vChJi19ftjhh0Bb7OR0GwhZcudYmuSMYzm5G2+nGQC1JBubHSCgxUAix4yZh60id+PsBJCWA4nbiNBi/PnPv4p0w9npH4jWYiDN2JaTIC+dQ6QtYL/09qUZbpDOKTiQYECEX8Ah9uNbsrz87PTNHz5U2MkR1IJwIVilAbHKQUC+gRTVo2AUjIJRMKIAAJ0kRrItEaPDAAAAAElFTkSuQmCC","orcid":"https://orcid.org/0009-0004-4323-8016","institution":"","correspondingAuthor":true,"prefix":"","firstName":"Christopher","middleName":"","lastName":"Boseak","suffix":""}],"badges":[],"createdAt":"2025-07-23 13:52:35","currentVersionCode":1,"declarations":{"humanSubjects":false,"vertebrateSubjects":false,"conflictsOfInterestStatement":false,"humanSubjectEthicalGuidelines":false,"humanSubjectConsent":false,"humanSubjectClinicalTrial":false,"humanSubjectCaseReport":false,"vertebrateSubjectEthicalGuidelines":false},"doi":"10.21203/rs.3.rs-7197053/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-7197053/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":87475007,"identity":"708b9e8c-6330-4d2b-b266-8a7800c4d823","added_by":"auto","created_at":"2025-07-24 09:02:42","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":856536,"visible":true,"origin":"","legend":"","description":"","filename":"labelnoiseboseak.pdf","url":"https://assets-eu.researchsquare.com/files/rs-7197053/v1_covered_8829f870-4b87-47a5-89b0-6b84fac0598f.pdf"}],"financialInterests":"The authors declare no competing interests.","formattedTitle":"\u003cp\u003eSystematic Evaluation of Label Noise Effects on Accuracy and Calibration in Deep Neural Networks\u003c/p\u003e","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":true,"hideJournal":true,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"machine learning, deep neural networks, artificial intelligence","lastPublishedDoi":"10.21203/rs.3.rs-7197053/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-7197053/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eLabel noise is a pervasive issue in real-world datasets that can degrade both the accuracy and calibration of deep neural networks. In this study, we systematically examine how symmetric (random) and asymmetric (class-dependent) label noise influence model accuracy and confidence calibration in image classification using the CIFAR-10 dataset and a ResNet-18 architecture. We apply five levels of label noise (0%, 10%, 20%, 40%, 60%) and evaluate their effects using metrics such as test accuracy, Expected Calibration Error (ECE), and predictive entropy. Our findings show that increasing noise levels significantly degrade classification accuracy and impair model calibration. In particular, asymmetric noise at a 60% corruption level causes test accuracy to drop to approximately 38.7% while ECE surges above 35%, indicating extreme overconfidence in incorrect predictions. By contrast, symmetric noise at the same noise level yields higher predictive entropy (uncertainty) and a comparatively modest miscalibration (ECE ∼9%). These results highlight the importance of distinguishing noise types when assessing model robustness and reliability. All experiments are reproducible, with code and data publicly available to facilitate further investigation.\u003c/p\u003e","manuscriptTitle":"Systematic Evaluation of Label Noise Effects on Accuracy and Calibration in Deep Neural Networks","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-07-24 08:54:34","doi":"10.21203/rs.3.rs-7197053/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"dab817c1-5043-4f12-a9e8-d8817308186c","owner":[],"postedDate":"July 24th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"posted","subjectAreas":[{"id":52002522,"name":"Artificial Intelligence and Machine Learning"}],"tags":[],"updatedAt":"2025-07-24T08:54:34+00:00","versionOfRecord":[],"versionCreatedAt":"2025-07-24 08:54:34","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-7197053","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-7197053","identity":"rs-7197053","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}