ITrans: Generative Image Inpainting with Transformers (ChinaMM)

ITrans: Generative Image Inpainting with Transformers (ChinaMM) | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article ITrans: Generative Image Inpainting with Transformers (ChinaMM) Wei Miao, Lijun Wang, Huchuan Lu, Kaining Huang, Xinchu Shi, Bocong Liu This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-3068126/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 17 Jan, 2024 Read the published version in Multimedia Systems → Version 1 posted 7 You are reading this latest preprint version Abstract Despite significant improvements, convolutional neural network (CNN) based methods are struggling with handling long-range global image dependencies due to their limited receptive fields, leading to an unsatisfactory inpainting performance under complicated scenarios. To address this issue, we propose the Inpainting Transformer (ITrans) network, which combines the power of both self-attention and convolution operations. The ITrans network augments convolutional encoder-decoder structure with two novel designs, \ie, the Global and Local Transformers. The Global Transformer aggregates high-level image context from the encoder in a global perspective, and propagates the encoded global representation to the decoder in a multi-scale manner. Meanwhile, the Local Transformer is intended to extract low-level image details inside the local neighborhood at a reduced computational overhead. By incorporating the above two Transformers, ITrans is capable of both global relationship modeling and local details encoding, which is essential for hallucinating perceptually realistic images. Extensive experiments demonstrate that the proposed ITrans network outperforms favorably against state-of-the-art inpainting methods both quantitatively and qualitatively. Convolutional Neural Network Image Inpainting Global Transformer Local Transformer Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Published Journal Publication published 17 Jan, 2024 Read the published version in Multimedia Systems → Version 1 posted Editorial decision: Accepted 07 Oct, 2023 Reviews received at journal 20 Jul, 2023 Reviewers agreed at journal 20 Jul, 2023 Reviewers invited by journal 17 Jul, 2023 Editor assigned by journal 27 Jun, 2023 Submission checks completed at journal 16 Jun, 2023 First submitted to journal 15 Jun, 2023 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-3068126","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":238461899,"identity":"4a98d9f8-97ef-41bd-a8d5-ac13491eb651","order_by":0,"name":"Wei Miao","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA50lEQVRIiWNgGAWjYFCCBMMHHyoYGNjYGRgkHkBECGoxNpxxBqiFGaglgcGAKC1m0rxtQJpoLbrtydskZ87bJs/HzMB4I7HtDwM/e44Bw88duLWYnXlWbPFx223DNmYGZovENgMGyZ43Boy9Z/BouZFjeHPmttuMQC1sEiAtBjdyDJgZ2/BqMZDmnXPbHq7FnggtRtK8DbcTEbZIENIC9IvhjGO3k9uYGZstEs4Z80iceVZwsBefluPJGx98qLltO7+9+eCND2VycvztQJGfeLQgAcYGEMkDIg4QpWEUjIJRMApGAU4AADuIT5YFXoPTAAAAAElFTkSuQmCC","orcid":"","institution":"Dalian University of Technology","correspondingAuthor":true,"prefix":"","firstName":"Wei","middleName":"","lastName":"Miao","suffix":""},{"id":238461903,"identity":"396f3431-5e95-4197-aa12-3366eb2c4ce3","order_by":1,"name":"Lijun Wang","email":"","orcid":"","institution":"Dalian University of Technology","correspondingAuthor":false,"prefix":"","firstName":"Lijun","middleName":"","lastName":"Wang","suffix":""},{"id":238461905,"identity":"9c5f1749-38ef-466c-b3a9-fb83a84fa7b3","order_by":2,"name":"Huchuan Lu","email":"","orcid":"","institution":"Dalian University of Technology","correspondingAuthor":false,"prefix":"","firstName":"Huchuan","middleName":"","lastName":"Lu","suffix":""},{"id":238461906,"identity":"74f0e55c-0de0-49b1-98ac-78d82a09bf14","order_by":3,"name":"Kaining Huang","email":"","orcid":"","institution":"Meituan Group","correspondingAuthor":false,"prefix":"","firstName":"Kaining","middleName":"","lastName":"Huang","suffix":""},{"id":238461908,"identity":"3c8f5cc3-9b4b-4ff4-8304-986b83987474","order_by":4,"name":"Xinchu Shi","email":"","orcid":"","institution":"Meituan Group","correspondingAuthor":false,"prefix":"","firstName":"Xinchu","middleName":"","lastName":"Shi","suffix":""},{"id":238461909,"identity":"d746603e-0f72-46b0-aac2-71ca168474c2","order_by":5,"name":"Bocong Liu","email":"","orcid":"","institution":"Meituan Group","correspondingAuthor":false,"prefix":"","firstName":"Bocong","middleName":"","lastName":"Liu","suffix":""}],"badges":[],"createdAt":"2023-06-15 13:44:23","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-3068126/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-3068126/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1007/s00530-023-01211-w","type":"published","date":"2024-01-17T15:01:28+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":49978910,"identity":"da54040f-4f73-4e50-b6e7-939b0601a145","added_by":"auto","created_at":"2024-01-22 15:10:03","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":2251174,"visible":true,"origin":"","legend":"","description":"","filename":"MultimediaSystemsChinaMM.pdf","url":"https://assets-eu.researchsquare.com/files/rs-3068126/v1_covered_e23ba3cf-a80b-4fd2-bf7d-6f54b0d8c8a5.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"ITrans: Generative Image Inpainting with Transformers (ChinaMM)","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"multimedia-systems","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"mmsj","sideBox":"Learn more about [Multimedia Systems](http://link.springer.com/journal/530)","snPcode":"530","submissionUrl":"https://submission.nature.com/new-submission/530/3","title":"Multimedia Systems","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Convolutional Neural Network, Image Inpainting, Global Transformer, Local Transformer","lastPublishedDoi":"10.21203/rs.3.rs-3068126/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-3068126/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Despite significant improvements, convolutional neural network (CNN) based methods are struggling with handling long-range global image dependencies due to their limited receptive fields, leading to an unsatisfactory inpainting performance under complicated scenarios. To address this issue, we propose the Inpainting Transformer (ITrans) network, which combines the power of both self-attention and convolution operations. The ITrans network augments convolutional encoder-decoder structure with two novel designs, \\ie, the Global and Local Transformers. The Global Transformer aggregates high-level image context from the encoder in a global perspective, and propagates the encoded global representation to the decoder in a multi-scale manner. Meanwhile, the Local Transformer is intended to extract low-level image details inside the local neighborhood at a reduced computational overhead. By incorporating the above two Transformers, ITrans is capable of both global relationship modeling and local details encoding, which is essential for hallucinating perceptually realistic images. Extensive experiments demonstrate that the proposed ITrans network outperforms favorably against state-of-the-art inpainting methods both quantitatively and qualitatively.","manuscriptTitle":"ITrans: Generative Image Inpainting with Transformers (ChinaMM)","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-01-01 06:17:22","doi":"10.21203/rs.3.rs-3068126/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Accepted","date":"2023-10-07T07:17:25+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2023-07-20T08:44:47+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"0fe3627a-b34f-4d65-9a12-c77c590ebcc4","date":"2023-07-20T08:42:15+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2023-07-17T07:56:29+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2023-06-28T03:38:04+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2023-06-16T07:52:35+00:00","index":"","fulltext":""},{"type":"submitted","content":"Multimedia Systems","date":"2023-06-15T13:33:47+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"multimedia-systems","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"mmsj","sideBox":"Learn more about [Multimedia Systems](http://link.springer.com/journal/530)","snPcode":"530","submissionUrl":"https://submission.nature.com/new-submission/530/3","title":"Multimedia Systems","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"966c2a01-0dbe-44e6-a99b-5c4bb21f2f72","owner":[],"postedDate":"January 1st, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[],"tags":[],"updatedAt":"2024-01-22T15:06:59+00:00","versionOfRecord":{"articleIdentity":"rs-3068126","link":"https://doi.org/10.1007/s00530-023-01211-w","journal":{"identity":"multimedia-systems","isVorOnly":false,"title":"Multimedia Systems"},"publishedOn":"2024-01-17 15:01:28","publishedOnDateReadable":"January 17th, 2024"},"versionCreatedAt":"2024-01-01 06:17:22","video":"","vorDoi":"10.1007/s00530-023-01211-w","vorDoiUrl":"https://doi.org/10.1007/s00530-023-01211-w","workflowStages":[]},"version":"v1","identity":"rs-3068126","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-3068126","identity":"rs-3068126","version":["v1"]},"buildId":"qtupq5eGEP_6zYnWcrvyt","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}