Symmetry Breaking in Neural Network Optimization: Insights from Input Dimension Expansion

Symmetry Breaking in Neural Network Optimization: Insights from Input Dimension Expansion | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Article Symmetry Breaking in Neural Network Optimization: Insights from Input Dimension Expansion Deyu Meng, Jun-Jie Zhang, Nan Cheng, Fu-Peng Li, Xiu-Cheng Wang, and 2 more This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-5768541/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 9 You are reading this latest preprint version Abstract Understanding how neural networks learn and optimize remains a central point in machine learning, with implications for designing better models. While techniques like dropout and batch normalization are widely used, the underlying principles driving their success—such as symmetry breaking, a concept in physics—are underexplored. We propose the symmetry breaking hypothesis, showing that breaking symmetries during training (e.g., via input expansion) substantially improves performance across tasks. We develop a metric to quantify symmetry breaking in networks, revealing its role in common optimization methods and its connection to properties like equivariance. This metric offers a practical tool to evaluate architectures without exhaustive training or full datasets, enabling more efficient design choices. Our work positions symmetry breaking as a unifying principle behind optimization techniques, bridging theoretical gaps and providing actionable insights for improving model efficiency. Physical sciences/Mathematics and computing Physical sciences/Physics/Statistical physics thermodynamics and nonlinear dynamics/Complex networks Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Under Review Version 1 posted Reviews received at journal 14 May, 2025 Reviews received at journal 28 Apr, 2025 Reviewers agreed at journal 22 Apr, 2025 Reviews received at journal 21 Apr, 2025 Reviewers agreed at journal 21 Apr, 2025 Reviewers agreed at journal 21 Apr, 2025 Reviewers invited by journal 20 Apr, 2025 Submission checks completed at journal 16 Apr, 2025 First submitted to journal 07 Apr, 2025 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-5768541","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Article","associatedPublications":[],"authors":[{"id":445829170,"identity":"2151bdf4-62eb-4979-9606-2048d4e23826","order_by":0,"name":"Deyu Meng","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABBElEQVRIiWNgGAWjYDCCA2DShoGBmbmBgYEHzDMgRksaUAsjSIsB0VoOAzFICwMRWvhuZCc+Lvh1Ppq/nbGBmUfmT2IDe/M2CYaaOzi1SN7I3Ww8s+927ozDjA2MM3gMEht4jpVJMBx7hlOLwY3cbdK8PbdzG4BaGD6AtEjkmEkwNhzGp2X7b96ec7nzQVoSQFrk3xDUso2Z58eB3A0IW3jwa5E883azNG9Dcu5GoJaDM3iMjdt40ootEo7h1sJ3PHfjZ54/drnzzh8++Ji3R062n/3wxhsfanBrYRBIAMZIG4R9gLGHgYENxErArYGBgf8AkPgD4/3Ap3QUjIJRMApGKgAAY3papiBLMzwAAAAASUVORK5CYII=","orcid":"","institution":"Xi'an Jiaotong University","correspondingAuthor":true,"prefix":"","firstName":"Deyu","middleName":"","lastName":"Meng","suffix":""},{"id":445829174,"identity":"e4b819ce-5544-4eeb-b0b2-f9a318dd2e19","order_by":1,"name":"Jun-Jie Zhang","email":"","orcid":"","institution":"Northwest Institute of Nuclear Technology","correspondingAuthor":false,"prefix":"","firstName":"Jun-Jie","middleName":"","lastName":"Zhang","suffix":""},{"id":445829177,"identity":"61609595-e250-4743-8f60-f7681926f52b","order_by":2,"name":"Nan Cheng","email":"","orcid":"","institution":"Xidian University","correspondingAuthor":false,"prefix":"","firstName":"Nan","middleName":"","lastName":"Cheng","suffix":""},{"id":445829180,"identity":"3da66f74-6e91-4276-b0bf-351a0690906d","order_by":3,"name":"Fu-Peng Li","email":"","orcid":"","institution":"Central China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Fu-Peng","middleName":"","lastName":"Li","suffix":""},{"id":445829181,"identity":"05fd3158-26d4-44c0-960d-6f17ab56e3c3","order_by":4,"name":"Xiu-Cheng Wang","email":"","orcid":"","institution":"Xidian University","correspondingAuthor":false,"prefix":"","firstName":"Xiu-Cheng","middleName":"","lastName":"Wang","suffix":""},{"id":445829183,"identity":"69ebcb11-7dda-410c-a00f-a2dcf46215ff","order_by":5,"name":"Jian-Nan Chen","email":"","orcid":"","institution":"Northwest Institute of Nuclear Technology","correspondingAuthor":false,"prefix":"","firstName":"Jian-Nan","middleName":"","lastName":"Chen","suffix":""},{"id":445829191,"identity":"a08983a3-e3cf-48a5-a85f-211f04fd3899","order_by":6,"name":"Long-Gang Pang","email":"","orcid":"","institution":"Central China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Long-Gang","middleName":"","lastName":"Pang","suffix":""}],"badges":[],"createdAt":"2025-01-05 15:38:03","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-5768541/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-5768541/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":81183939,"identity":"1dc991c9-9336-4100-a5f5-6a899896f36f","added_by":"auto","created_at":"2025-04-23 08:03:53","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":976464,"visible":true,"origin":"","legend":"","description":"","filename":"texfile.pdf","url":"https://assets-eu.researchsquare.com/files/rs-5768541/v1_covered_8190744e-2635-43c8-931d-8070bd5820c4.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Symmetry Breaking in Neural Network Optimization: Insights from Input Dimension Expansion","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"npj-artificial-intelligence","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"","sideBox":"Learn more about [npj Artificial Intelligence](https://www.nature.com/npjai)","snPcode":"443878","submissionUrl":"https://submission.springernature.com/new-submission/443878/3","title":"npj Artificial Intelligence","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"NPJ","inReviewEnabled":true,"inReviewRevisionsEnabled":true},"keywords":"","lastPublishedDoi":"10.21203/rs.3.rs-5768541/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-5768541/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Understanding how neural networks learn and optimize remains a central point in machine learning, with implications for designing better models. While techniques like dropout and batch normalization are widely used, the underlying principles driving their success—such as symmetry breaking, a concept in physics—are underexplored. We propose the symmetry breaking hypothesis, showing that breaking symmetries during training (e.g., via input expansion) substantially improves performance across tasks. We develop a metric to quantify symmetry breaking in networks, revealing its role in common optimization methods and its connection to properties like equivariance. This metric offers a practical tool to evaluate architectures without exhaustive training or full datasets, enabling more efficient design choices. Our work positions symmetry breaking as a unifying principle behind optimization techniques, bridging theoretical gaps and providing actionable insights for improving model efficiency.","manuscriptTitle":"Symmetry Breaking in Neural Network Optimization: Insights from Input Dimension Expansion","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-04-23 07:39:42","doi":"10.21203/rs.3.rs-5768541/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"editorInvitedReview","content":"","date":"2025-05-14T23:08:01+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-04-29T01:45:42+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"284927559509120309929175841933818940306","date":"2025-04-22T11:51:20+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-04-21T14:54:57+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"299432149817282024494622098736889425329","date":"2025-04-21T14:26:47+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"26477536238542384558068984751600573865","date":"2025-04-21T13:51:00+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2025-04-20T11:03:31+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2025-04-16T15:03:04+00:00","index":"","fulltext":""},{"type":"submitted","content":"npj Artificial Intelligence","date":"2025-04-07T12:58:26+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"npj-artificial-intelligence","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"","sideBox":"Learn more about [npj Artificial Intelligence](https://www.nature.com/npjai)","snPcode":"443878","submissionUrl":"https://submission.springernature.com/new-submission/443878/3","title":"npj Artificial Intelligence","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"NPJ","inReviewEnabled":true,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"05f9b1e9-75d9-4ec0-878c-461b8b0e9b6c","owner":[],"postedDate":"April 23rd, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[{"id":47456532,"name":"Physical sciences/Mathematics and computing"},{"id":47456533,"name":"Physical sciences/Physics/Statistical physics thermodynamics and nonlinear dynamics/Complex networks"}],"tags":[],"updatedAt":"2025-05-15T07:53:12+00:00","versionOfRecord":[],"versionCreatedAt":"2025-04-23 07:39:42","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-5768541","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-5768541","identity":"rs-5768541","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}