Multistage Attention Region Supplement Transformer for Fine-grained Visual Categorization

Multistage Attention Region Supplement Transformer for Fine-grained Visual Categorization | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Multistage Attention Region Supplement Transformer for Fine-grained Visual Categorization Aokun Mei, Hua Huo, Jiaxin Xu, Ningya Xu This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-3845719/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 16 Jun, 2024 Read the published version in The Visual Computer → Version 1 posted 7 You are reading this latest preprint version Abstract Fine-Grained Visual Categorization (FGVC) involves learning detailed features for images that are difficult to distinguish within the same subclass. This enables networks to differentiate between instances of different classes that share very similar visual content. Therefore, learning how to extract nuanced representations of selected object details is crucial. This paper introduces a novel fine-grained visual classification model with Vision Transformer (ViT) as the backbone, namely Multistage Attention Region Supplement Transformer (MARS-Trans). Our main contributions are as follows. First, we observed that in ViT's multi-head attention module, each Attention Head's softmaxed feature results are directly concatenated and multiplied by weights. Consequently, we propose a Multistage Attention Module (MAM) to grade the attention heads based on their weights. Additionally, we introduce a Region Supplement Module (RSM) to suppress non-critical regions and enhance edge information in key areas, further emphasizing the discriminative features. Finally, we use our proposed Approximate Adjust Method (AAM) to refine the final features and improve classification results. We conducted thorough experiments with MARS-Trans on four popular public fine-grained image datasets, validating the effectiveness of these modules. SOTA results are achieved on one dataset, and competitive performance is demonstrated on the other three datasets.The code is available at https://github.com/ArrikenMei/MARS-Trans . FGVC Vision Transformer Multi-Stage Attention Region Supplement Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Published Journal Publication published 16 Jun, 2024 Read the published version in The Visual Computer → Version 1 posted Editorial decision: Revision requested 10 Mar, 2024 Reviews received at journal 20 Jan, 2024 Reviewers agreed at journal 11 Jan, 2024 Reviewers invited by journal 11 Jan, 2024 Editor assigned by journal 10 Jan, 2024 Submission checks completed at journal 09 Jan, 2024 First submitted to journal 08 Jan, 2024 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-3845719","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":266326026,"identity":"b4c48edc-35a8-423a-a6e4-29670b37563d","order_by":0,"name":"Aokun Mei","email":"","orcid":"","institution":"Henan University of Science and Technology","correspondingAuthor":false,"prefix":"","firstName":"Aokun","middleName":"","lastName":"Mei","suffix":""},{"id":266326027,"identity":"7ef01172-2315-47e7-a43d-8a43c60e1fdf","order_by":1,"name":"Hua Huo","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA1ElEQVRIiWNgGAWjYJCCAxUGDAxAxPggocKGSC1nIFqYDR6cSSPSmjMMYC1skg/bDhFWbc5++OGBAwV2DOYSyccqEtgOMPC3dyfg1WLZk2Zw4IBBMoPljLS0Gwk8dxgkzpzdgFeLwYEchsMfDA4wGNzIMbuRIPGMwUAil4CW828YgLZAtBQkGBwmQsuNHIQWhoQEIrRYzngG8YvBmWfJEgkH0ngI+sWcP/nxhwN/7BgMjicf/Pjzn40cf3svAYdB6foGKIMHr3JkLaNgFIyCUTAKcAMABclPeBk+tmQAAAAASUVORK5CYII=","orcid":"","institution":"Henan University of Science and Technology","correspondingAuthor":true,"prefix":"","firstName":"Hua","middleName":"","lastName":"Huo","suffix":""},{"id":266326028,"identity":"7700a1b2-6441-48a8-8cdd-0c9eb18e320a","order_by":2,"name":"Jiaxin Xu","email":"","orcid":"","institution":"Henan University of Science and Technology","correspondingAuthor":false,"prefix":"","firstName":"Jiaxin","middleName":"","lastName":"Xu","suffix":""},{"id":266326029,"identity":"f0a15aa3-a7e7-4d91-8497-830524820b28","order_by":3,"name":"Ningya Xu","email":"","orcid":"","institution":"Henan University of Science and Technology","correspondingAuthor":false,"prefix":"","firstName":"Ningya","middleName":"","lastName":"Xu","suffix":""}],"badges":[],"createdAt":"2024-01-08 15:14:23","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-3845719/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-3845719/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1007/s00371-024-03502-3","type":"published","date":"2024-06-17T00:19:56+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":58530083,"identity":"d7be5aaf-1ab9-40db-91a4-190d995f10e0","added_by":"auto","created_at":"2024-06-18 00:20:05","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":1097972,"visible":true,"origin":"","legend":"","description":"","filename":"MultistageAttentionRegionSupplementTransformerforFinegrainedVisualCategorization.pdf","url":"https://assets-eu.researchsquare.com/files/rs-3845719/v1_covered_fd1931bb-9cf3-4ca4-bef1-9f7fbf404d67.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Multistage Attention Region Supplement Transformer for Fine-grained Visual Categorization","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"the-visual-computer","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"tvcj","sideBox":"Learn more about [The Visual Computer](http://link.springer.com/journal/371)","snPcode":"371","submissionUrl":"https://submission.nature.com/new-submission/371/3","title":"The Visual Computer","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"FGVC, Vision Transformer, Multi-Stage Attention, Region Supplement","lastPublishedDoi":"10.21203/rs.3.rs-3845719/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-3845719/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Fine-Grained Visual Categorization (FGVC) involves learning detailed features for images that are difficult to distinguish within the same subclass. This enables networks to differentiate between instances of different classes that share very similar visual content. Therefore, learning how to extract nuanced representations of selected object details is crucial. This paper introduces a novel fine-grained visual classification model with Vision Transformer (ViT) as the backbone, namely Multistage Attention Region Supplement Transformer (MARS-Trans). Our main contributions are as follows. First, we observed that in ViT's multi-head attention module, each Attention Head's softmaxed feature results are directly concatenated and multiplied by weights. Consequently, we propose a Multistage Attention Module (MAM) to grade the attention heads based on their weights. Additionally, we introduce a Region Supplement Module (RSM) to suppress non-critical regions and enhance edge information in key areas, further emphasizing the discriminative features. Finally, we use our proposed Approximate Adjust Method (AAM) to refine the final features and improve classification results. We conducted thorough experiments with MARS-Trans on four popular public fine-grained image datasets, validating the effectiveness of these modules. SOTA results are achieved on one dataset, and competitive performance is demonstrated on the other three datasets.The code is available at https://github.com/ArrikenMei/MARS-Trans.","manuscriptTitle":"Multistage Attention Region Supplement Transformer for Fine-grained Visual Categorization","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-01-11 04:07:02","doi":"10.21203/rs.3.rs-3845719/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2024-03-10T21:42:20+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2024-01-20T13:06:20+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"efa7dbff-9f66-4aa1-9c1b-e5a491de7899","date":"2024-01-11T14:13:26+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2024-01-11T13:19:37+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2024-01-10T05:40:04+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2024-01-09T16:43:28+00:00","index":"","fulltext":""},{"type":"submitted","content":"The Visual Computer","date":"2024-01-08T15:07:27+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"the-visual-computer","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"tvcj","sideBox":"Learn more about [The Visual Computer](http://link.springer.com/journal/371)","snPcode":"371","submissionUrl":"https://submission.nature.com/new-submission/371/3","title":"The Visual Computer","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"1f376a8b-c3d5-4c61-98d6-8ef7a0d06fd8","owner":[],"postedDate":"January 11th, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[],"tags":[],"updatedAt":"2024-06-18T00:19:56+00:00","versionOfRecord":{"articleIdentity":"rs-3845719","link":"https://doi.org/10.1007/s00371-024-03502-3","journal":{"identity":"the-visual-computer","isVorOnly":false,"title":"The Visual Computer"},"publishedOn":"2024-06-17 00:19:56","publishedOnDateReadable":"June 17th, 2024"},"versionCreatedAt":"2024-01-11 04:07:02","video":"","vorDoi":"10.1007/s00371-024-03502-3","vorDoiUrl":"https://doi.org/10.1007/s00371-024-03502-3","workflowStages":[]},"version":"v1","identity":"rs-3845719","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-3845719","identity":"rs-3845719","version":["v1"]},"buildId":"qtupq5eGEP_6zYnWcrvyt","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}