Transformers with Sparse Attention for Granger Causality

Transformers with Sparse Attention for Granger Causality | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Transformers with Sparse Attention for Granger Causality Riya Mahesh, Rahul Vashisht, Chandrashekar Lakshminarayanan This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-7214786/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 11 You are reading this latest preprint version Abstract Temporal causal analysis means understanding the underlying causes behind observed variables over time. Deep learning based methods such as transformers are increasingly used to capture temporal dynamics and causal relationships beyond mere correlations. Recent works suggest self-attention weights of transformers as a useful indicator of causal links. We leverage this to propose a novel modification to the self-attention module to establish causal links between the variables of multivariate time-series data with varying lag dependencies. Our Sparse Attention Transformer captures causal relationships using a two-fold approach - performing temporal attention first followed by attention between the variables across the time steps masking them individually to compute Granger Causality indices. The key novelty in our approach is the ability of the model to assert importance and pick the most significant past time instances for its prediction task against manually feeding a fixed time lag value. We demonstrate the effectiveness of our approach via extensive experimentation on several synthetic benchmark datasets. Furthermore, we compare the performance of our model with the traditional Vector Autoregression based Granger Causality method that assumes fixed lag length. Multi-variate time series Granger Causality Stationarity Self-attention Sparse Attention Transformers Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Under Review Version 1 posted Reviews received at journal 07 Oct, 2025 Reviews received at journal 01 Oct, 2025 Reviewers agreed at journal 24 Sep, 2025 Reviewers agreed at journal 24 Sep, 2025 Reviewers agreed at journal 22 Sep, 2025 Reviewers agreed at journal 13 Sep, 2025 Reviewers agreed at journal 10 Sep, 2025 Reviewers invited by journal 04 Aug, 2025 Editor assigned by journal 29 Jul, 2025 Submission checks completed at journal 29 Jul, 2025 First submitted to journal 25 Jul, 2025 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-7214786","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":495368550,"identity":"f92519d5-7a7b-44cb-a7fe-7400545207f6","order_by":0,"name":"Riya Mahesh","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABBUlEQVRIiWNgGAWjYFAC5gOHGQwk5PhB7IQCBh42qDgPbi1siYcZCiyMJRtAWgyAWtgIauExPszwoSJxwwEQxwBkBgFn8bM3GBwuMJBg3Hz+dOKHBwYMMnzy3QkMP2oYZMxxaJHsOZBweIaBBLPZjdzNEhCH8W5g7DnGwGPZgF2LwY2EA4d5DCTYzG7wboBrYeBtYOAxOIBdi/39hw0gLTzG/Wc3/4Db8hePFqCTGEBaJAwYcrfBbWHGZ4vEmTQGkF8MJG7kbrNIAFrHxpa74bDMMQmcWvjbz3/+XPCnrr4f6LCbPyps7OWbz258+KbGxh6XFgxbweQBGGMUjIJRMApGAXkAABVMUkU7UoZCAAAAAElFTkSuQmCC","orcid":"","institution":"Indian Institute of Technology Madras","correspondingAuthor":true,"prefix":"","firstName":"Riya","middleName":"","lastName":"Mahesh","suffix":""},{"id":495368551,"identity":"e98493d9-1a7f-4053-83eb-1087adab8cff","order_by":1,"name":"Rahul Vashisht","email":"","orcid":"","institution":"Indian Institute of Technology Madras","correspondingAuthor":false,"prefix":"","firstName":"Rahul","middleName":"","lastName":"Vashisht","suffix":""},{"id":495368552,"identity":"0a7cb4b7-c9c6-4c3b-9678-a74c2cfcb490","order_by":2,"name":"Chandrashekar Lakshminarayanan","email":"","orcid":"","institution":"Indian Institute of Technology Madras","correspondingAuthor":false,"prefix":"","firstName":"Chandrashekar","middleName":"","lastName":"Lakshminarayanan","suffix":""}],"badges":[],"createdAt":"2025-07-25 13:38:10","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-7214786/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-7214786/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":88395786,"identity":"6d2a0c02-44aa-4627-a4e3-f15ddc1304cd","added_by":"auto","created_at":"2025-08-06 06:05:23","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":391746,"visible":true,"origin":"","legend":"","description":"","filename":"TransformersWithSparseAttentionForGrangerCausalityDiscoverDataRevised.pdf","url":"https://assets-eu.researchsquare.com/files/rs-7214786/v1_covered_215bc6ce-d0dc-4634-be29-aed17979b080.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Transformers with Sparse Attention for Granger Causality","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"discover-data","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"did","sideBox":"Learn more about [Discover Data](https://www.springer.com/44248)","snPcode":"44248","submissionUrl":"https://submission.nature.com/new-submission/44248/3","title":"Discover Data","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Discover Series","inReviewEnabled":true,"inReviewRevisionsEnabled":true},"keywords":"Multi-variate time series, Granger Causality, Stationarity, Self-attention, Sparse Attention, Transformers","lastPublishedDoi":"10.21203/rs.3.rs-7214786/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-7214786/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eTemporal causal analysis means understanding the underlying causes behind observed variables over time. Deep learning based methods such as transformers are increasingly used to capture temporal dynamics and causal relationships beyond mere correlations. Recent works suggest self-attention weights of transformers as a useful indicator of causal links. We leverage this to propose a novel modification to the self-attention module to establish causal links between the variables of multivariate time-series data with varying lag dependencies. Our Sparse Attention Transformer captures causal relationships using a two-fold approach - performing temporal attention first followed by attention between the variables across the time steps masking them individually to compute Granger Causality indices. The key novelty in our approach is the ability of the model to assert importance and pick the most significant past time instances for its prediction task against manually feeding a fixed time lag value. We demonstrate the effectiveness of our approach via extensive experimentation on several synthetic benchmark datasets. Furthermore, we compare the performance of our model with the traditional Vector Autoregression based Granger Causality method that assumes fixed lag length.\u003c/p\u003e","manuscriptTitle":"Transformers with Sparse Attention for Granger Causality","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-08-06 05:49:18","doi":"10.21203/rs.3.rs-7214786/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"editorInvitedReview","content":"","date":"2025-10-07T12:17:10+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-10-01T07:42:44+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"247091404092987665878612713500632566874","date":"2025-09-24T10:14:47+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"241742136703567347071253882540557383237","date":"2025-09-24T10:04:25+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"299765788664876115093374995301652703391","date":"2025-09-22T08:54:25+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"49315184188377110844011136445990822996","date":"2025-09-14T01:07:37+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"90276375011110019125128552810364425246","date":"2025-09-10T19:28:30+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2025-08-04T11:44:18+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2025-07-29T12:29:02+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2025-07-29T12:26:52+00:00","index":"","fulltext":""},{"type":"submitted","content":"Discover Data","date":"2025-07-25T13:23:09+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"discover-data","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"did","sideBox":"Learn more about [Discover Data](https://www.springer.com/44248)","snPcode":"44248","submissionUrl":"https://submission.nature.com/new-submission/44248/3","title":"Discover Data","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Discover Series","inReviewEnabled":true,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"cce7e46c-b62a-433f-ae6c-fc3822a2b724","owner":[],"postedDate":"August 6th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[],"tags":[],"updatedAt":"2025-10-26T03:26:30+00:00","versionOfRecord":[],"versionCreatedAt":"2025-08-06 05:49:18","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-7214786","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-7214786","identity":"rs-7214786","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}