Modality-Aware Fusion and Selection for Robust Multispectral Pedestrian Detection

Modality-Aware Fusion and Selection for Robust Multispectral Pedestrian Detection | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Article Modality-Aware Fusion and Selection for Robust Multispectral Pedestrian Detection Yaoxiang Hu, Hua Jin This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-9256972/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 8 You are reading this latest preprint version Abstract Multispectral pedestrian detection, integrating visible and infrared modalities, is pivotal for robust perception in all-weather autonomous driving and surveillance systems. However, existing methods primarily rely on direct fusion mechanisms, which often fail to account for modality-specific noise (\eg, smoke in visible or thermal crossover in infrared). Consequently, when one modality is corrupted, ineffective fusion propagates ''zero-information'' noise into the feature space, leading to feature contamination and performance degradation. To address this challenge, we propose a synergizing fusion and competition detection framework that synergistically leverages complementary information while suppressing modal interference. Specifically, our architecture incorporates a dual-pathway encoding strategy: a multimodal fusion encoder to aggregate cross-modal contexts and a multimodal competition encoder to adaptively select optimal features based on saliency scores, thereby filtering out unreliable modalities. Furthermore, we introduce a multimodal visual refinement module within the decoder to align features and enhance high-precision localization. Extensive experiments on three benchmark datasets (LLVIP, FLIR-aligned, and $\mathrm{M^3FD}$) demonstrate that our method significantly outperforms state-of-the-art approaches. Notably, our model achieves substantial gains in high-stringency metrics (\ie, $mAP_{75}$), validating its effectiveness in generating robust and precise detections in complex dynamic scenarios. Physical sciences/Engineering Physical sciences/Mathematics and computing Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Under Review Version 1 posted Reviewers agreed at journal 15 May, 2026 Reviews received at journal 07 May, 2026 Reviewers agreed at journal 23 Apr, 2026 Reviewers invited by journal 18 Apr, 2026 Editor assigned by journal 18 Apr, 2026 Editor invited by journal 13 Apr, 2026 Submission checks completed at journal 10 Apr, 2026 First submitted to journal 10 Apr, 2026 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-9256972","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Article","associatedPublications":[],"authors":[{"id":628134468,"identity":"f8efa51e-f6d7-4866-8dea-cc64e15a6626","order_by":0,"name":"Yaoxiang Hu","email":"","orcid":"","institution":"Dalian Jiaotong University","correspondingAuthor":false,"prefix":"","firstName":"Yaoxiang","middleName":"","lastName":"Hu","suffix":""},{"id":628134471,"identity":"d84bc0be-2a97-49e4-b2f4-58c42e3a0c29","order_by":1,"name":"Hua Jin","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAwElEQVRIiWNgGAWjYDACCcYGBgkGBmZ+ZuaDD0jTItnOlmxApBYobXCex0yAKB3ys5tbN1juqGU3PsxgxsBQYxNNUAvjnINtNyTPHGc2O8yQ9oDhWFpuAyEtzBKJQC1tx0BajhswNhwmrIUNpsW4mbFNgigtPBAtNcwGzMxsxGmRgGg5wCxxmI3ZIIEYv8jPSH92W7KtLpm///zHBx9qbAhrAQFmCYbDyWBWAjHKQYDxA0OdHbGKR8EoGAWjYAQCADpIPLyH6RFSAAAAAElFTkSuQmCC","orcid":"","institution":"Dalian Jiaotong University","correspondingAuthor":true,"prefix":"","firstName":"Hua","middleName":"","lastName":"Jin","suffix":""}],"badges":[],"createdAt":"2026-03-29 07:08:44","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-9256972/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-9256972/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":107870244,"identity":"e98cc8ae-9066-442b-80d8-59b512529c5a","added_by":"auto","created_at":"2026-04-27 07:39:10","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":2529078,"visible":true,"origin":"","legend":"","description":"","filename":"TemplateforsubmissionstoScientificReports.pdf","url":"https://assets-eu.researchsquare.com/files/rs-9256972/v1_covered_ef4571a1-8dfc-4839-921b-d9792ab79982.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Modality-Aware Fusion and Selection for Robust Multispectral Pedestrian Detection","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"scientific-reports","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"scirep","sideBox":"Learn more about [Scientific Reports](http://www.nature.com/srep/)","snPcode":"","submissionUrl":"","title":"Scientific Reports","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Scientific Reports","inReviewEnabled":true,"inReviewRevisionsEnabled":true},"keywords":"","lastPublishedDoi":"10.21203/rs.3.rs-9256972/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-9256972/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Multispectral pedestrian detection, integrating visible and infrared modalities, is pivotal for robust perception in all-weather autonomous driving and surveillance systems. However, existing methods primarily rely on direct fusion mechanisms, which often fail to account for modality-specific noise (\\eg, smoke in visible or thermal crossover in infrared). Consequently, when one modality is corrupted, ineffective fusion propagates ''zero-information'' noise into the feature space, leading to feature contamination and performance degradation. To address this challenge, we propose a synergizing fusion and competition detection framework that synergistically leverages complementary information while suppressing modal interference. Specifically, our architecture incorporates a dual-pathway encoding strategy: a multimodal fusion encoder to aggregate cross-modal contexts and a multimodal competition encoder to adaptively select optimal features based on saliency scores, thereby filtering out unreliable modalities. Furthermore, we introduce a multimodal visual refinement module within the decoder to align features and enhance high-precision localization. Extensive experiments on three benchmark datasets (LLVIP, FLIR-aligned, and $\\mathrm{M^3FD}$) demonstrate that our method significantly outperforms state-of-the-art approaches. Notably, our model achieves substantial gains in high-stringency metrics (\\ie, $mAP_{75}$), validating its effectiveness in generating robust and precise detections in complex dynamic scenarios.","manuscriptTitle":"Modality-Aware Fusion and Selection for Robust Multispectral Pedestrian Detection","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2026-04-27 05:36:22","doi":"10.21203/rs.3.rs-9256972/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"reviewerAgreed","content":"1319158076895480009466113250762456264","date":"2026-05-15T07:10:29+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2026-05-07T05:20:55+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"123251791845554069851843333406187690415","date":"2026-04-23T05:23:16+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2026-04-18T14:32:15+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2026-04-18T14:31:14+00:00","index":"","fulltext":""},{"type":"editorInvited","content":"","date":"2026-04-13T12:18:45+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2026-04-10T12:07:20+00:00","index":"","fulltext":""},{"type":"submitted","content":"Scientific Reports","date":"2026-04-10T10:57:16+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"scientific-reports","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"scirep","sideBox":"Learn more about [Scientific Reports](http://www.nature.com/srep/)","snPcode":"","submissionUrl":"","title":"Scientific Reports","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Scientific Reports","inReviewEnabled":true,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"c5ee370b-0302-420c-899e-25b0a2486af1","owner":[],"postedDate":"April 27th, 2026","published":true,"recentEditorialEvents":[{"type":"reviewerAgreed","content":"1319158076895480009466113250762456264","date":"2026-05-15T07:10:29+00:00","index":78,"fulltext":""},{"type":"editorInvitedReview","content":"","date":"2026-05-07T05:20:55+00:00","index":64,"fulltext":""}],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[{"id":66856017,"name":"Physical sciences/Engineering"},{"id":66856018,"name":"Physical sciences/Mathematics and computing"}],"tags":[],"updatedAt":"2026-04-27T05:36:26+00:00","versionOfRecord":[],"versionCreatedAt":"2026-04-27 05:36:22","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-9256972","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-9256972","identity":"rs-9256972","version":["v1"]},"buildId":"XKTyCvWXoU3ODBz1xrDgd","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}