Multimodal Spatial-Temporal and Spectral Fusion for Audio-Visual Deepfake Detection

Multimodal Spatial-Temporal and Spectral Fusion for Audio-Visual Deepfake Detection | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Multimodal Spatial-Temporal and Spectral Fusion for Audio-Visual Deepfake Detection Abhishek Kumar, Gaurav Kumar This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-9514886/v1 This work is licensed under a CC BY 4.0 License Status: Posted Version 1 posted You are reading this latest preprint version Abstract Deepfakes are becoming very big problem for online security in today world. Early defense systems was quite good at catching the obvious visual mistakes in standalone pictures, but modern deepfake generators do not make those simple pixel level errors anymore. The situation is getting more complex today because attackers are mixing perfectly genuine video footage with algorithmically cloned audio. Most commercial security systems are only doing checking for one data stream at a time, so these cross-modal files easily slip through. To solve this problem, our paper is presenting a late-fusion multimodal architecture. We did separation of the detection process into two independent expert branches. The first branch is using an Xception-LSTM network to track the physical geometry of a face over a 10-frame sequence, hunting for unnatural temporal variance (micro-flickering). The second branch is handling the audio by converting 16 kHz sound waves into 2D Mel-Spectrograms, which allows a ResNet-18 model to visually spot the high-frequency drop-offs (between 8 kHz to 16 kHz) caused by synthetic voice generators. By concatenating these parameters in a final Multi Layer Perceptron (MLP), our model hit a 99.62% global accuracy on the FakeAVCeleb dataset. Most importantly, we successfully proved that a late-fusion methodology dramatically lowers the false positive rate to near-zero (1 out of 96 authentic videos), achieving an AUC score of 0.999 and beating several state-of the-art baselines. Artificial Intelligence and Machine Learning Deepfakes Multimodal Fusion Spatial Temporal Analysis Mel-Spectrogram Late Fusion Full Text Additional Declarations The authors declare no competing interests. Cite Share Download PDF Status: Posted Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-9514886","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":628869525,"identity":"59b0cd91-0e1d-46d7-8f5a-303c08208e2e","order_by":0,"name":"Abhishek Kumar","email":"","orcid":"","institution":"Gautam Buddha University","correspondingAuthor":false,"prefix":"","firstName":"Abhishek","middleName":"","lastName":"Kumar","suffix":""},{"id":628869526,"identity":"3846a636-99b3-4520-8b2b-c786a99efacb","order_by":1,"name":"Gaurav Kumar","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABBklEQVRIiWNgGAWjYDADNgYGA4aHDRJyIM6BB0RpYQNqSWyQMAZrSSDSGpAWEAICfFrk3c+YffjZxiDPJ9+88UHiDov0+WGHHwJtsZPTbcCuxfBMjvHM3jYGwzY2tmKDxDMSuRtvpxkAtSQbmx3AoaUhx5iB5wwDYxsbj5lEYhtQy+wEkJYDidtwael/Y8z45wyDPVCL+Q+glnTD2ekf8GqRl8gxZuapYEgE2QIkJRLkpXPw22Ig8ayYWaZCIrmNLa0Y5DDDDdI5BQcSDHD7Rb4/eTPjGwMb2/nNhzd++NhWJy8/O33zhw8VdnK4tBhAxCXQRQywKwfb0kBYZBSMglEwCkY6AACSZFtc8eAkGQAAAABJRU5ErkJggg==","orcid":"","institution":"Gautam Buddha University","correspondingAuthor":true,"prefix":"","firstName":"Gaurav","middleName":"","lastName":"Kumar","suffix":""}],"badges":[],"createdAt":"2026-04-24 09:31:42","currentVersionCode":1,"declarations":{"humanSubjects":true,"vertebrateSubjects":false,"conflictsOfInterestStatement":false,"humanSubjectEthicalGuidelines":true,"humanSubjectConsent":true,"humanSubjectClinicalTrial":false,"humanSubjectCaseReport":false,"vertebrateSubjectEthicalGuidelines":false},"doi":"10.21203/rs.3.rs-9514886/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-9514886/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":107870581,"identity":"39e05ca9-f141-4ebe-b36e-8396e2ac0a7b","added_by":"auto","created_at":"2026-04-27 07:39:57","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":427851,"visible":true,"origin":"","legend":"","description":"","filename":"MultimodalSpatialTemporalandSpectralFusion.pdf","url":"https://assets-eu.researchsquare.com/files/rs-9514886/v1_covered_f96acc6b-37d5-4704-b794-0abb4a86912c.pdf"}],"financialInterests":"The authors declare no competing interests.","formattedTitle":"\u003cp\u003eMultimodal Spatial-Temporal and Spectral Fusion for Audio-Visual Deepfake Detection\u003c/p\u003e","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":true,"hideJournal":true,"highlight":"","institution":"Gautam Buddha University","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"Deepfakes, Multimodal Fusion, Spatial Temporal Analysis, Mel-Spectrogram, Late Fusion","lastPublishedDoi":"10.21203/rs.3.rs-9514886/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-9514886/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eDeepfakes are becoming very big problem for online security in today world. Early defense systems was quite good at catching the obvious visual mistakes in standalone pictures, but modern deepfake generators do not make those simple pixel level errors anymore. The situation is getting more complex today because attackers are mixing perfectly genuine video footage with algorithmically cloned audio. Most commercial security systems are only doing checking for one data stream at a time, so these cross-modal files easily slip through. To solve this problem, our paper is presenting a late-fusion multimodal architecture. We did separation of the detection process into two independent expert branches. The first branch is using an Xception-LSTM network to track the physical geometry of a face over a 10-frame sequence, hunting for unnatural temporal variance (micro-flickering). The second branch is handling the audio by converting 16 kHz sound waves into 2D Mel-Spectrograms, which allows a ResNet-18 model to visually spot the high-frequency drop-offs (between 8 kHz to 16 kHz) caused by synthetic voice generators. By concatenating these parameters in a final Multi Layer Perceptron (MLP), our model hit a 99.62% global accuracy on the FakeAVCeleb dataset. Most importantly, we successfully proved that a late-fusion methodology dramatically lowers the false positive rate to near-zero (1 out of 96 authentic videos), achieving an AUC score of 0.999 and beating several state-of the-art baselines.\u003c/p\u003e","manuscriptTitle":"Multimodal Spatial-Temporal and Spectral Fusion for Audio-Visual Deepfake Detection","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2026-04-27 03:27:31","doi":"10.21203/rs.3.rs-9514886/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"a3393397-f68b-4ba6-a6bf-5966a43c86b4","owner":[],"postedDate":"April 27th, 2026","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"posted","subjectAreas":[{"id":66934978,"name":"Artificial Intelligence and Machine Learning"}],"tags":[],"updatedAt":"2026-04-27T03:27:32+00:00","versionOfRecord":[],"versionCreatedAt":"2026-04-27 03:27:31","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-9514886","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-9514886","identity":"rs-9514886","version":["v1"]},"buildId":"XKTyCvWXoU3ODBz1xrDgd","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}