Lightweight Self-Supervised Representation Learning with Knowledge Distillation on Compact Datasets

Lightweight Self-Supervised Representation Learning with Knowledge Distillation on Compact Datasets | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Lightweight Self-Supervised Representation Learning with Knowledge Distillation on Compact Datasets Khawla Hussein ِAli This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-6866114/v1 This work is licensed under a CC BY 4.0 License Status: Posted Version 1 posted You are reading this latest preprint version Abstract Recent studies have demonstrated that self-supervised learning techniques exhibit strong performance in visual representation tasks, particularly in scenarios where labeled data is limited. However, it remains challenging to train deep models when data is limited due to overfitting and a lack of generalization. This paper proposes a novel approach to leveraging knowledge distillation to enhance self-supervised representation learning in resource-constrained settings. The method we use trains an EfficientNet-B0 student model using a MobileNetV2 teacher, which is trained on the STL-10 dataset. We incorporate gradual alpha scheduling and early stopping to ensure the training remains stable and knowledge is preserved. We have observed that our approach, which utilizes different sample sizes, outperforms the student model alone. Our method, Self-Supervised with Knowledge Distillation (SS-KD), achieves 72.71% accuracy on 2500 samples, outperforming several state-of-the-art self-supervised and distillation approaches. When scaled to 5000 samples, our model reaches 83.00% , demonstrating strong scalability with limited data. Available Code at: Self-Supervised-using-Knowledge-Distillation- Self-Supervised Learning Knowledge Distillation Compact Datasets Representation Learning Lightweight Models Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Posted Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-6866114","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":475456423,"identity":"e969f7f4-913b-4501-91e1-06e1e7461877","order_by":0,"name":"Khawla Hussein ِAli","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABHElEQVRIiWNgGAWjYBACCQkg8QCI+ZiZDzAwNoDEeICYDYxwa0kAqWFmSyBVCwOPAaoWXEBydvMziYQaBnk2dp5vEh932Nmbt589wPCh7DADH3sCVi3SMsfMJBKOMRi2MfNuk5x5Jjlxzpm8BMYZ5w4zsPE8wKpFTiLB7EYCGwMjSIs0bxtzggRDjgEzbxtQiwR2W+Qk0r/dSPjHYN/GzPMMqKXeXoL/jQHzXzxapCVyzG4ktjEkArWwAbUcZpwhAbSFEY8WyRk55T8S+ySS25jZjC1nnjmeOEPijcHBnnPpPLj8InEjfbPBh282tv38hx/e+LijGuiwHMMHP8qs5eTbsdsC04nKPcAAjh28WrADMrSMglEwCkbBcAQAXrRTXZNfJr8AAAAASUVORK5CYII=","orcid":"","institution":"University of Basrah","correspondingAuthor":true,"prefix":"","firstName":"Khawla","middleName":"Hussein","lastName":"ِAli","suffix":""}],"badges":[],"createdAt":"2025-06-10 20:38:20","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-6866114/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-6866114/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":86985852,"identity":"b3ea3c44-d98c-4340-86e7-8d59faeeafa5","added_by":"auto","created_at":"2025-07-18 03:16:39","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":615531,"visible":true,"origin":"","legend":"","description":"","filename":"KnowledgeDistilationKhawla.pdf","url":"https://assets-eu.researchsquare.com/files/rs-6866114/v1_covered_2ea83d0f-559c-4d20-969f-57b69f638f05.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Lightweight Self-Supervised Representation Learning with Knowledge Distillation on Compact Datasets","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":true,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"Self-Supervised Learning, Knowledge Distillation, Compact Datasets, Representation Learning, Lightweight Models","lastPublishedDoi":"10.21203/rs.3.rs-6866114/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-6866114/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eRecent studies have demonstrated that self-supervised learning techniques exhibit strong performance in visual representation tasks, particularly in scenarios where labeled data is limited. However, it remains challenging to train deep models when data is limited due to overfitting and a lack of generalization. This paper proposes a novel approach to leveraging knowledge distillation to enhance self-supervised representation learning in resource-constrained settings. The method we use trains an EfficientNet-B0 student model using a MobileNetV2 teacher, which is trained on the STL-10 dataset. We incorporate gradual alpha scheduling and early stopping to ensure the training remains stable and knowledge is preserved. We have observed that our approach, which utilizes different sample sizes, outperforms the student model alone. Our method, Self-Supervised with Knowledge Distillation (SS-KD), achieves \u003cstrong\u003e72.71%\u003c/strong\u003e accuracy on 2500 samples, outperforming several state-of-the-art self-supervised and distillation approaches. When scaled to 5000 samples, our model reaches \u003cstrong\u003e83.00%\u003c/strong\u003e, demonstrating strong scalability with limited data.\u003c/p\u003e\n\u003cp\u003e\u003cstrong\u003eAvailable Code at: Self-Supervised-using-Knowledge-Distillation-\u003c/strong\u003e\u003c/p\u003e","manuscriptTitle":"Lightweight Self-Supervised Representation Learning with Knowledge Distillation on Compact Datasets","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-06-25 11:16:18","doi":"10.21203/rs.3.rs-6866114/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"f90eeb26-e5b7-443d-a58a-0e572191618d","owner":[],"postedDate":"June 25th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"posted","subjectAreas":[],"tags":[],"updatedAt":"2025-07-18T03:08:23+00:00","versionOfRecord":[],"versionCreatedAt":"2025-06-25 11:16:18","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-6866114","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-6866114","identity":"rs-6866114","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}