Evaluating and Accelerating Vision Transformers on GPU-based Embedded Edge AI Systems

Evaluating and Accelerating Vision Transformers on GPU-based Embedded Edge AI Systems | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Evaluating and Accelerating Vision Transformers on GPU-based Embedded Edge AI Systems Ignacio Martin-Salinas, Jose M. Badia, Oscar Valls, German Leon, and 4 more This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-5083258/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 9 You are reading this latest preprint version Abstract Many current embedded systems comprise heterogeneous computing components including quite powerful GPUs, which enables their application across diverse sectors. This study demonstrates the efficient execution of a medium-sized Self-Supervised Audio Spectrogram Transformer (SSAST) model on a low-power System-on-Chip (SoC). Through comprehensive evaluation, including real-time inference scenarios, we show that GPUs outperform multi-core CPUs in inference processes. Optimization techniques such as adjusting batch size, model compilation with TensorRT, and reducing data precision significantly enhance inference time, energy consumption, and memory usage. In particular, negligible accuracy degradation is observed, with post-training quantization to 8-bit integers showing less than 1% loss. This research underscores the feasibility of deploying transformer neural networks on low-power embedded devices, ensuring efficiency in time, energy, and memory while maintaining the accuracy of the results. Vision Transformer GPU Low-power System-on-Chip Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Under Review Version 1 posted Editorial decision: Revision requested 12 Nov, 2024 Reviews received at journal 07 Nov, 2024 Reviews received at journal 07 Nov, 2024 Reviewers agreed at journal 10 Oct, 2024 Reviewers agreed at journal 10 Oct, 2024 Reviewers invited by journal 09 Oct, 2024 Editor assigned by journal 17 Sep, 2024 Submission checks completed at journal 16 Sep, 2024 First submitted to journal 13 Sep, 2024 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-5083258","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":377326356,"identity":"8d1c3b23-236e-4b69-bcbb-9f72169d9e3f","order_by":0,"name":"Ignacio Martin-Salinas","email":"","orcid":"","institution":"Carlos III University of Madrid","correspondingAuthor":false,"prefix":"","firstName":"Ignacio","middleName":"","lastName":"Martin-Salinas","suffix":""},{"id":377326357,"identity":"904658c7-f2f3-465c-8564-af674907338c","order_by":1,"name":"Jose M. Badia","email":"","orcid":"","institution":"Jaume I University","correspondingAuthor":false,"prefix":"","firstName":"Jose","middleName":"M.","lastName":"Badia","suffix":""},{"id":377326358,"identity":"f2e6da3c-b446-44e5-8336-e0941a3989e6","order_by":2,"name":"Oscar Valls","email":"","orcid":"","institution":"Universitat Politècnica de València","correspondingAuthor":false,"prefix":"","firstName":"Oscar","middleName":"","lastName":"Valls","suffix":""},{"id":377326359,"identity":"50ab96e4-08c3-4567-a5b2-2420717eaf41","order_by":3,"name":"German Leon","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAkklEQVRIiWNgGAWjYBACPhDxgYGNh3gtbEDMOINkLcwkqAdpYW8+9ti2jU+Ggb39AZFaeI6lG+e2AR3Gc8aASC0SOWbSYC0SOcQ6TP6NmbQlSIv8c2IdJsFjJs0ItoWBWIfxpKVJ9pxj42HjySFSCz/74WMSP8qO2fOzHyfSYVBwDBxBJIEaUjWMglEwCkbBSAIAoh0cA752iL4AAAAASUVORK5CYII=","orcid":"","institution":"Jaume I University","correspondingAuthor":true,"prefix":"","firstName":"German","middleName":"","lastName":"Leon","suffix":""},{"id":377326360,"identity":"f705c4c0-7c6d-448f-84e0-4932007e0dff","order_by":4,"name":"Rocio del Amor","email":"","orcid":"","institution":"Universitat Politècnica de València","correspondingAuthor":false,"prefix":"","firstName":"Rocio","middleName":"del","lastName":"Amor","suffix":""},{"id":377326361,"identity":"64f0ebb6-83c6-418a-a5b4-445431fa7474","order_by":5,"name":"Jose A. Belloch","email":"","orcid":"","institution":"Carlos III University of Madrid","correspondingAuthor":false,"prefix":"","firstName":"Jose","middleName":"A.","lastName":"Belloch","suffix":""},{"id":377326362,"identity":"5f603366-d3d5-4431-a434-d020bd0785a6","order_by":6,"name":"Adrian Amor-Martin","email":"","orcid":"","institution":"Carlos III University of Madrid","correspondingAuthor":false,"prefix":"","firstName":"Adrian","middleName":"","lastName":"Amor-Martin","suffix":""},{"id":377326363,"identity":"56bcaeba-dc99-4987-a141-8a8caa595ec8","order_by":7,"name":"Valery Naranjo","email":"","orcid":"","institution":"Universitat Politècnica de València","correspondingAuthor":false,"prefix":"","firstName":"Valery","middleName":"","lastName":"Naranjo","suffix":""}],"badges":[],"createdAt":"2024-09-13 10:59:15","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-5083258/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-5083258/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":70626641,"identity":"22f9041a-ddb8-411d-a2bd-6ee305f5d56e","added_by":"auto","created_at":"2024-12-05 04:52:48","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":523777,"visible":true,"origin":"","legend":"","description":"","filename":"JoSCMMSE23TRANSFORMERS2.pdf","url":"https://assets-eu.researchsquare.com/files/rs-5083258/v1_covered_7dcbed04-40c0-4c57-8c47-02fc11107836.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Evaluating and Accelerating Vision Transformers on GPU-based Embedded Edge AI Systems","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"the-journal-of-supercomputing","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"","sideBox":"Learn more about [The Journal of Supercomputing](https://www.springer.com/journal/11227)","snPcode":"11227","submissionUrl":"https://submission.nature.com/new-submission/11227/3","title":"The Journal of Supercomputing","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Vision Transformer, GPU, Low-power System-on-Chip","lastPublishedDoi":"10.21203/rs.3.rs-5083258/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-5083258/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Many current embedded systems comprise heterogeneous computing components including quite powerful GPUs, which enables their application across diverse sectors. This study demonstrates the efficient execution of a medium-sized Self-Supervised Audio Spectrogram Transformer (SSAST) model on a low-power System-on-Chip (SoC). Through comprehensive evaluation, including real-time inference scenarios, we show that GPUs outperform multi-core CPUs in inference processes. Optimization techniques such as adjusting batch size, model compilation with TensorRT, and reducing data precision significantly enhance inference time, energy consumption, and memory usage. In particular, negligible accuracy degradation is observed, with post-training quantization to 8-bit integers showing less than 1% loss. This research underscores the feasibility of deploying transformer neural networks on low-power embedded devices, ensuring efficiency in time, energy, and memory while maintaining the accuracy of the results.","manuscriptTitle":"Evaluating and Accelerating Vision Transformers on GPU-based Embedded Edge AI Systems","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-12-05 04:44:41","doi":"10.21203/rs.3.rs-5083258/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2024-11-12T18:12:04+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2024-11-07T18:47:28+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2024-11-07T16:59:17+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"305326303693642794484475479024292163790","date":"2024-10-10T18:41:54+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"248751046224550931226143091608116805427","date":"2024-10-10T06:30:56+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2024-10-09T11:42:24+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2024-09-17T22:24:59+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2024-09-17T03:00:10+00:00","index":"","fulltext":""},{"type":"submitted","content":"The Journal of Supercomputing","date":"2024-09-13T10:56:58+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"the-journal-of-supercomputing","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"","sideBox":"Learn more about [The Journal of Supercomputing](https://www.springer.com/journal/11227)","snPcode":"11227","submissionUrl":"https://submission.nature.com/new-submission/11227/3","title":"The Journal of Supercomputing","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"e1ad97e7-be0d-4881-940e-e2cac6269a19","owner":[],"postedDate":"December 5th, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[],"tags":[],"updatedAt":"2024-12-05T04:44:41+00:00","versionOfRecord":[],"versionCreatedAt":"2024-12-05 04:44:41","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-5083258","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-5083258","identity":"rs-5083258","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}