Balanced segmentation of CNNs for multi-TPU inference

Balanced segmentation of CNNs for multi-TPU inference | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Balanced segmentation of CNNs for multi-TPU inference Jorge Villarrubia, Luis Costero, Francisco D. Igual, Katzalin Olcoz This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-3095752/v1 This work is licensed under a CC BY 4.0 License Status: Posted Version 1 posted You are reading this latest preprint version Abstract In this paper, we propose different alternatives for CNN (Convolutional Neural Networks) segmentation, addressing inference processes on computing architectures composed by multiple Edge TPUs. Specifically, we compare the inference performance for a number of state-of-the-art CNN models taking as a reference inference times on one TPU and a compiler-based pipelined inference implementation as provided by the Google's Edge TPU compiler. Departing from a profiled-based segmentation strategy, we provide further refinements to balance the workload across multiple TPUs, leveraging their co-operative computing power, reducing work imbalance and alleviating the memory access bottleneck due to the limited amount of on-chip memory per TPU. The observed performance results compared with a single TPU yield super-linear speedups and accelerations up to 2.60x compared with the segmentation offered by the compiler targeting multiple TPUs. Domain-specific architectures Edge TPU deep learning model segmentation model inference Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Posted Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-3095752","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":213386298,"identity":"58d937f7-51b6-47e0-842c-9e87a6663d28","order_by":0,"name":"Jorge Villarrubia","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAsUlEQVRIiWNgGAWjYBACxgYgkVAgwcNPohYDGxnJBtLsMkizMThArGLm9t5nEg8MDvMY30h+uoGhoo4Ih/UcN5NIAGoxu5FmdoPhzGEitMxIYzaAaMlhu8HYRoTz4FqMZ4C0/CPGYTPSGB8kGKTxGEiAtDQwE6Gl5xhIiw2PxJlnZjcSjhHhF8P2NoaDPyok7Pnbk5/d+FBDhMMMG5B5CYQ1MDDIE6NoFIyCUTAKRjgAADMpNvc9fRrJAAAAAElFTkSuQmCC","orcid":"","institution":"Complutense University of Madrid","correspondingAuthor":true,"prefix":"","firstName":"Jorge","middleName":"","lastName":"Villarrubia","suffix":""},{"id":213386299,"identity":"f5e02e3a-cefb-47c7-b780-651252ed9f1a","order_by":1,"name":"Luis Costero","email":"","orcid":"","institution":"Complutense University of Madrid","correspondingAuthor":false,"prefix":"","firstName":"Luis","middleName":"","lastName":"Costero","suffix":""},{"id":213386300,"identity":"372f1af1-2985-45c7-8377-e1b874853a7d","order_by":2,"name":"Francisco D. Igual","email":"","orcid":"","institution":"Complutense University of Madrid","correspondingAuthor":false,"prefix":"","firstName":"Francisco","middleName":"D.","lastName":"Igual","suffix":""},{"id":213386301,"identity":"caed7dec-bac3-42f7-a268-36b937b0fbb1","order_by":3,"name":"Katzalin Olcoz","email":"","orcid":"","institution":"Complutense University of Madrid","correspondingAuthor":false,"prefix":"","firstName":"Katzalin","middleName":"","lastName":"Olcoz","suffix":""}],"badges":[],"createdAt":"2023-06-22 10:14:36","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-3095752/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-3095752/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":44612052,"identity":"5e6decd7-b408-45c2-b28f-910ab0ffc36b","added_by":"auto","created_at":"2023-10-14 01:07:32","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":652957,"visible":true,"origin":"","legend":"","description":"","filename":"BalancedsegmentationofCNNsformultiTPUinference.pdf","url":"https://assets-eu.researchsquare.com/files/rs-3095752/v1_covered_3f441d6c-3552-40a2-84f0-e17e231e810a.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Balanced segmentation of CNNs for multi-TPU inference","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":true,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"Domain-specific architectures, Edge TPU, deep learning, model segmentation, model inference","lastPublishedDoi":"10.21203/rs.3.rs-3095752/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-3095752/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"In this paper, we propose different alternatives for CNN (Convolutional Neural Networks) segmentation, addressing inference processes on computing architectures composed by multiple Edge TPUs. Specifically, we compare the inference performance for a number of state-of-the-art CNN models taking as a reference inference times on one TPU and a compiler-based pipelined inference implementation as provided by the Google's Edge TPU compiler. Departing from a profiled-based segmentation strategy, we provide further refinements to balance the workload across multiple TPUs, leveraging their co-operative computing power, reducing work imbalance and alleviating the memory access bottleneck due to the limited amount of on-chip memory per TPU. The observed performance results compared with a single TPU yield super-linear speedups and accelerations up to 2.60x compared with the segmentation offered by the compiler targeting multiple TPUs.","manuscriptTitle":"Balanced segmentation of CNNs for multi-TPU inference","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2023-06-29 06:31:10","doi":"10.21203/rs.3.rs-3095752/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"d9f9c076-3ea3-4612-b3a4-d00e0c4d869e","owner":[],"postedDate":"June 29th, 2023","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"posted","subjectAreas":[],"tags":[],"updatedAt":"2023-10-14T00:59:17+00:00","versionOfRecord":[],"versionCreatedAt":"2023-06-29 06:31:10","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-3095752","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-3095752","identity":"rs-3095752","version":["v1"]},"buildId":"J0_U0BvcaRcwD8yVFaRlm","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}