Sparse Friendly Distillation Using Feature Decoupling

Sparse Friendly Distillation Using Feature Decoupling | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Sparse Friendly Distillation Using Feature Decoupling WeiHong He, YuLi Fu, Youjun Xiang This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-3811616/v1 This work is licensed under a CC BY 4.0 License Status: Posted Version 1 posted You are reading this latest preprint version Abstract In our paper, we introduce the sparse-friendly distillation framework as an effective training strategy for knowledge distillation. While model sparsity techniques have been widely adopted to reduce training overhead, sparse student models often struggle to achieve good performance in knowledge distillation. To address this issue, our framework leverages the observation that sparse student models exhibit different behaviors in foreground and background features. We separate these features using different pooling techniques and apply separate mean squared error (MSE) feature distillation. Furthermore, we dynamically adjust the weights of the two loss components to optimize performance. Experimental results on CIFAR-10 and CIFAR-100 benchmarks demonstrate significant performance improvements, validating the effectiveness of our methodology. Additionally, we provide a comprehensive analysis of our experimental results, further validating the effectiveness of our approach. Sparse Training Knowledge Distillation Feature Decoupling Model Compression Machine Learning Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Posted Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-3811616","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":264045356,"identity":"1b56cb92-3810-421e-8a5e-d2000ff325a9","order_by":0,"name":"WeiHong He","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA5klEQVRIiWNgGAWjYBACAxCR2AAkmNkPGHwwsLEjQQs7T0LhjIK0ZOK0MIK08DMYfOb5cAjMxgvM2XsMHzzcYZMn78yQuNnG4AAzA/vhoxvwabHsOWNskHgmrdjwMONh4xyDO3wMPGlpN/A67EbuNonEtsOJG5sZ0oBanjEzSPCY4ddy/+32H1At5r8tDA4zNhDUcoN3GwNIy3xmBgNjBqK0nMn/DHRYWuIGZp4Ewx6DtGQ2gn45fizx4882m8T5/ccPGPz4Y2PHz374GF4tCL0HoAw2opSDgHwD0UpHwSgYBaNgpAEAiulQMXK2KT4AAAAASUVORK5CYII=","orcid":"","institution":"South China University of Technology","correspondingAuthor":true,"prefix":"","firstName":"WeiHong","middleName":"","lastName":"He","suffix":""},{"id":264045357,"identity":"19d08bdc-f042-4526-9080-b2f5b806a319","order_by":1,"name":"YuLi Fu","email":"","orcid":"","institution":"South China University of Technology","correspondingAuthor":false,"prefix":"","firstName":"YuLi","middleName":"","lastName":"Fu","suffix":""},{"id":264045358,"identity":"caab0306-a03d-4f74-a175-fb17803bb1a3","order_by":2,"name":"Youjun Xiang","email":"","orcid":"","institution":"South China University of Technology","correspondingAuthor":false,"prefix":"","firstName":"Youjun","middleName":"","lastName":"Xiang","suffix":""}],"badges":[],"createdAt":"2023-12-27 09:45:22","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-3811616/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-3811616/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":55263993,"identity":"6c6f9881-94b2-49b4-9846-72d750268b9a","added_by":"auto","created_at":"2024-04-25 01:32:08","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":284219,"visible":true,"origin":"","legend":"","description":"","filename":"SparseFriendlyDistillationUsingFeatureDecoupling.pdf","url":"https://assets-eu.researchsquare.com/files/rs-3811616/v1_covered_749a712c-a476-4dd0-b288-29303ae853cb.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Sparse Friendly Distillation Using Feature Decoupling","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":true,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"Sparse Training, Knowledge Distillation, Feature Decoupling, Model Compression, Machine Learning","lastPublishedDoi":"10.21203/rs.3.rs-3811616/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-3811616/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"In our paper, we introduce the sparse-friendly distillation framework as an effective training strategy for knowledge distillation. While model sparsity techniques have been widely adopted to reduce training overhead, sparse student models often struggle to achieve good performance in knowledge distillation. To address this issue, our framework leverages the observation that sparse student models exhibit different behaviors in foreground and background features. We separate these features using different pooling techniques and apply separate mean squared error (MSE) feature distillation. Furthermore, we dynamically adjust the weights of the two loss components to optimize performance. Experimental results on CIFAR-10 and CIFAR-100 benchmarks demonstrate significant performance improvements, validating the effectiveness of our methodology. Additionally, we provide a comprehensive analysis of our experimental results, further validating the effectiveness of our approach.","manuscriptTitle":"Sparse Friendly Distillation Using Feature Decoupling","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-01-01 07:07:18","doi":"10.21203/rs.3.rs-3811616/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"36fcec58-4b3f-4971-84bc-8da11cdfb1c2","owner":[],"postedDate":"January 1st, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"posted","subjectAreas":[],"tags":[],"updatedAt":"2024-04-20T11:04:38+00:00","versionOfRecord":[],"versionCreatedAt":"2024-01-01 07:07:18","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-3811616","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-3811616","identity":"rs-3811616","version":["v1"]},"buildId":"qtupq5eGEP_6zYnWcrvyt","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}