AET:Auxiliary and Embedded Teacher for Online Mutual Knowledge Distillation for Image Classification

AET:Auxiliary and Embedded Teacher for Online Mutual Knowledge Distillation for Image Classification | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article AET:Auxiliary and Embedded Teacher for Online Mutual Knowledge Distillation for Image Classification Wen Liu, Xingzhu Liang, Feilong Bi, Yu-e Lin This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-4478667/v1 This work is licensed under a CC BY 4.0 License Status: Posted Version 1 posted You are reading this latest preprint version Abstract Distillation (OKD) has emerged as a powerful technique for model compression, eliminating the need for pre-trained teachers in traditional methods. While recent advancements in feature fusion have further improved OKD's capabilities, existing approaches solely focus on final-layer fusion, potentially hindering the effectiveness of the fused classifier. In this work, we propose a novel Auxiliary and Embedded Teacher (AET) approach to tackle these challenges. AET addresses the critical issues of feature fusion position selection and potential performance degradation after fusion. We introduce embedded teachers, formed by combining multiple mid-level sub-networks, to promote mutual learning among student networks. Additionally, auxiliary teachers provide enriched information and guide the fusion classifier, ultimately enhancing overall performance. Extensive evaluations on four benchmark datasets (CIFAR-10/100, CINIC-10, and ImageNet2012) demonstrate the superiority of the proposed AET approach. Code is available: https://github.com/JSJ515-Group/AET knowledge distillation auxiliary learning feature fusion embedded module Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Posted Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-4478667","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":308143392,"identity":"c8087f60-6605-4d1d-a938-dfb02d8e637e","order_by":0,"name":"Wen Liu","email":"","orcid":"","institution":"Anhui University of Science and Technology","correspondingAuthor":false,"prefix":"","firstName":"Wen","middleName":"","lastName":"Liu","suffix":""},{"id":308143393,"identity":"65bfe4bf-be96-4e1a-9f84-2140f423ab99","order_by":1,"name":"Xingzhu Liang","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA3ElEQVRIie3PIQ+CQBTAcW5uWHBUCeBXgBn5Mke6AmaDsme5pLPyNWzGY7dBwVGJmCy6YTToBNR6XnTz/uHu3fZ+4TRNpfrJBsDa0/m8ZAjqybSfpEl3BiBN3DJdsdt+SbYJOdXa3A9geGBCYkEK6abIo6QKPdAKEoAxw0JiohWwEc0iqEIEiPIAxoYrJPoAQXqnGZlU5AjoIUFMHQEf0QV2K+xBO38n1rpdsynzdsXZS3BGptQIxcQtOb9eaDxxclI3zcK3t8NCTN7x14W738nst8WSeyqVSvWXPQEaQ0ybP6/G2wAAAABJRU5ErkJggg==","orcid":"","institution":"Anhui University of Science and Technology","correspondingAuthor":true,"prefix":"","firstName":"Xingzhu","middleName":"","lastName":"Liang","suffix":""},{"id":308143394,"identity":"1dcf3d74-1778-4a15-95bf-e8907f6ec910","order_by":2,"name":"Feilong Bi","email":"","orcid":"","institution":"Anhui University of Science and Technology","correspondingAuthor":false,"prefix":"","firstName":"Feilong","middleName":"","lastName":"Bi","suffix":""},{"id":308143395,"identity":"f0f753a3-ae3d-4ea7-ab07-e1df19769f21","order_by":3,"name":"Yu-e Lin","email":"","orcid":"","institution":"Anhui University of Science and Technology","correspondingAuthor":false,"prefix":"","firstName":"Yu-e","middleName":"","lastName":"Lin","suffix":""}],"badges":[],"createdAt":"2024-05-26 05:08:16","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-4478667/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-4478667/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":71110768,"identity":"593e395b-6612-4d7a-a49f-b2ae9a3dbaee","added_by":"auto","created_at":"2024-12-11 08:48:22","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":865012,"visible":true,"origin":"","legend":"","description":"","filename":"reviseAET.pdf","url":"https://assets-eu.researchsquare.com/files/rs-4478667/v1_covered_bc86376c-b754-451d-9d69-590ebecfb3b5.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"AET:Auxiliary and Embedded Teacher for Online Mutual Knowledge Distillation for Image Classification","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":true,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"knowledge distillation, auxiliary learning, feature fusion, embedded module","lastPublishedDoi":"10.21203/rs.3.rs-4478667/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-4478667/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Distillation (OKD) has emerged as a powerful technique for model compression, eliminating the need for pre-trained teachers in traditional methods. While recent advancements in feature fusion have further improved OKD's capabilities, existing approaches solely focus on final-layer fusion, potentially hindering the effectiveness of the fused classifier. In this work, we propose a novel Auxiliary and Embedded Teacher (AET) approach to tackle these challenges. AET addresses the critical issues of feature fusion position selection and potential performance degradation after fusion. We introduce embedded teachers, formed by combining multiple mid-level sub-networks, to promote mutual learning among student networks. Additionally, auxiliary teachers provide enriched information and guide the fusion classifier, ultimately enhancing overall performance. Extensive evaluations on four benchmark datasets (CIFAR-10/100, CINIC-10, and ImageNet2012) demonstrate the superiority of the proposed AET approach. Code is available: https://github.com/JSJ515-Group/AET","manuscriptTitle":"AET:Auxiliary and Embedded Teacher for Online Mutual Knowledge Distillation for Image Classification","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-06-06 16:23:28","doi":"10.21203/rs.3.rs-4478667/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"073fe195-391e-4ec6-95be-3d8bf8698e55","owner":[],"postedDate":"June 6th, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"posted","subjectAreas":[],"tags":[],"updatedAt":"2024-12-11T08:40:01+00:00","versionOfRecord":[],"versionCreatedAt":"2024-06-06 16:23:28","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-4478667","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-4478667","identity":"rs-4478667","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}