Smartformer: An Intelligent Transformer Compression Framework for Time-Series Modeling

Smartformer: An Intelligent Transformer Compression Framework for Time-Series Modeling | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Smartformer: An Intelligent Transformer Compression Framework for Time-Series Modeling Xiaojian Wang, Yinan Wang, Jin Yang, Ying Chen This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-1780688/v2 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 09 Jul, 2024 Read the published version in IISE Transactions → Version 2 posted You are reading this latest preprint version Show more versions Abstract Transformer, as one of the cutting-edge deep neural networks (DNNs), has achieved outstanding performance in time-series data analysis. However, this model usually requires massive amounts of parameters to fit. Over-parameterization not only brings storage challenges in a resource-limited setting but also inevitably results in the model over-fitting. Even though literature works introduced several ways to reduce the parameter size of Transformers, none of them addressed this over-parameterized issue by concurrently achieving the following three objectives: preserving the model architecture, maintaining the model performance, and reducing the model complexity (number of parameters). In this study, we propose an intelligent model compression framework, Smartformer, by incorporating reinforcement learning and CP-decomposition techniques to satisfy the aforementioned three objectives. In the experiment, we apply Smartformer and five baseline methods to two existing time-series Transformer models for model compression. The results demonstrate that our proposed Smartformer is the only method that consistently generates the compressed model on various scenarios by satisfying the three objectives. In particular, the Smartformer can mitigate the overfitting issue and thus improve the accuracy of the existing time-series models in all scenarios. Artificial Intelligence and Machine Learning Intelligent model compression Transformer CP-decomposition Reinforcement learning Deep learning Full Text Additional Declarations The authors declare no competing interests. Cite Share Download PDF Status: Published Journal Publication published 09 Jul, 2024 Read the published version in IISE Transactions → Version 2 posted You are reading this latest preprint version Show more versions Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-1780688","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":115926338,"identity":"d87b92e7-39c8-40b9-bfb0-d634ddb26d3d","order_by":0,"name":"Xiaojian Wang","email":"","orcid":"","institution":"Harbin Institute of Technology","correspondingAuthor":false,"prefix":"","firstName":"Xiaojian","middleName":"","lastName":"Wang","suffix":""},{"id":115926339,"identity":"7596242c-da47-415d-a259-f5f6b952745f","order_by":1,"name":"Yinan Wang","email":"","orcid":"","institution":"Rensselaer Polytechnic Institute","correspondingAuthor":false,"prefix":"","firstName":"Yinan","middleName":"","lastName":"Wang","suffix":""},{"id":115926340,"identity":"e31b7aab-9533-415d-93b7-c833c88b3e72","order_by":2,"name":"Jin Yang","email":"","orcid":"","institution":"Harbin Institute of Technology","correspondingAuthor":false,"prefix":"","firstName":"Jin","middleName":"","lastName":"Yang","suffix":""},{"id":115926341,"identity":"bfa34166-198c-4d3a-b555-7badc441b60f","order_by":3,"name":"Ying Chen","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA10lEQVRIiWNgGAWjYDACZiBmbGBg4JcAkWAusVokZzA2NhCnhQGqxeAGmCJCi8Fx3sMvfu6wyTO+3dz+gKHCOrGB/ewB/FoO86VZ9p5JKza7cxDosDPpiQ08eQkEtPCYGTO2HU7cdiOxsQHEaJDgMSBGy//EzTNAWv4Rp8X4MWPbgcQNEiAtDURokQTawtjblpw4A+iwGQnH0o3beHLwa+E7f8b4w882u8T+GekPPnyosZbtZz+DX4vCAQY2CTgvAYjZ8KoHAvkGBuYPhBSNglEwCkbBCAcADj9LbFhWhSQAAAAASUVORK5CYII=","orcid":"https://orcid.org/0000-0002-0366-131X","institution":"Harbin Institute of Technology","correspondingAuthor":true,"prefix":"","firstName":"Ying","middleName":"","lastName":"Chen","suffix":""}],"badges":[],"createdAt":"2022-06-21 12:35:54","currentVersionCode":2,"declarations":{"humanSubjects":false,"vertebrateSubjects":false,"conflictsOfInterestStatement":false,"humanSubjectEthicalGuidelines":false,"humanSubjectConsent":false,"humanSubjectClinicalTrial":false,"humanSubjectCaseReport":false,"vertebrateSubjectEthicalGuidelines":false},"doi":"10.21203/rs.3.rs-1780688/v2","doiUrl":"https://doi.org/10.21203/rs.3.rs-1780688/v2","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1080/24725854.2024.2376645","type":"published","date":"2024-07-09T19:40:54+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[],"financialInterests":"The authors declare no competing interests.","formattedTitle":"\u003cp\u003eSmartformer: An Intelligent Transformer Compression Framework for Time-Series Modeling\u003c/p\u003e","fulltext":[{"header":"Full Text","content":"This preprint is available for \u003ca href='/article/rs-1780688/latest.pdf' target='_blank'\u003edownload as a PDF\u003c/a\u003e."}],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":true,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":false,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"Intelligent model compression, Transformer, CP-decomposition, Reinforcement learning, Deep learning","lastPublishedDoi":"10.21203/rs.3.rs-1780688/v2","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-1780688/v2","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eTransformer, as one of the cutting-edge deep neural networks (DNNs), has achieved outstanding performance in time-series data analysis. However, this model usually requires massive amounts of parameters to fit. Over-parameterization not only brings storage challenges in a resource-limited setting but also inevitably results in the model over-fitting. Even though literature works introduced several ways to reduce the parameter size of Transformers, none of them addressed this over-parameterized issue by concurrently achieving the following three objectives: preserving the model architecture, maintaining the model performance, and reducing the model complexity (number of parameters). In this study, we propose an intelligent model compression framework, Smartformer, by incorporating reinforcement learning and CP-decomposition techniques to satisfy the aforementioned three objectives. In the experiment, we apply Smartformer and five baseline methods to two existing time-series Transformer models for model compression. The results demonstrate that our proposed Smartformer is the only method that consistently generates the compressed model on various scenarios by satisfying the three objectives. In particular, the Smartformer can mitigate the overfitting issue and thus improve the accuracy of the existing time-series models in all scenarios.\u003c/p\u003e","manuscriptTitle":"Smartformer: An Intelligent Transformer Compression Framework for Time-Series Modeling","msid":"","msnumber":"","nonDraftVersions":[{"code":2,"date":"2024-07-15 19:37:43","doi":"10.21203/rs.3.rs-1780688/v2","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}},{"code":1,"date":"2022-06-23 18:07:14","doi":"10.21203/rs.3.rs-1780688/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"194c523e-ceb6-421b-8abc-af69857fa601","owner":[],"postedDate":"July 15th, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[{"id":34642509,"name":"Artificial Intelligence and Machine Learning"}],"tags":[],"updatedAt":"2024-07-15T19:40:54+00:00","versionOfRecord":{"articleIdentity":"rs-1780688","link":"https://doi.org/10.1080/24725854.2024.2376645","journal":{"identity":"iise-transactions","isVorOnly":true,"title":"IISE Transactions"},"publishedOn":"2024-07-09 19:40:54","publishedOnDateReadable":"July 9th, 2024"},"versionCreatedAt":"2024-07-15 19:37:43","video":"","vorDoi":"10.1080/24725854.2024.2376645","vorDoiUrl":"https://doi.org/10.1080/24725854.2024.2376645","workflowStages":[]},"version":"v2","identity":"rs-1780688","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-1780688","identity":"rs-1780688","version":["v2"]},"buildId":"qtupq5eGEP_6zYnWcrvyt","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}