Standard Attention as a Small-Angle Limit of Riemannian Geometric Algebra Transformers

Standard Attention as a Small-Angle Limit of Riemannian Geometric Algebra Transformers | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Physical Sciences - Article Standard Attention as a Small-Angle Limit of Riemannian Geometric Algebra Transformers Zachary Joseph This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-8811263/v1 This work is licensed under a CC BY 4.0 License Status: Posted Version 1 posted You are reading this latest preprint version Abstract Transformers implicitly assume a flat representational geome- try: similarity is Euclidean, composition is linear, and attention is a softmax of dot products. We show that these operations arise as a degenerate limit of a curved geometric operator family on the rotor manifold Spin(3), realized in the Clifford algebra Cl(3, 0). In the small-angle regime, geometric soft- max reduces to classical dot-product attention; yet, at depth, many small-angle steps accumulate curvature through Baker– Campbell–Hausdorff commutators, providing a precise sense in which standard Transformers approximate rotor dynamics. The paper focuses on this mathematical discovery and its struc- tural implications for interpretability; performance studies are a separate empirical program. Physical sciences/Mathematics and computing/Computational science Physical sciences/Mathematics and computing/Pure mathematics Physical sciences/Mathematics and computing/Applied mathematics Physical sciences/Physics/Information theory and computation Full Text Additional Declarations There is NO Competing Interest. Supplementary Files SupplementaryInformationRiemannianGeometricAlgebraTransformer.pdf Supplementary Information: Riemannian Geometric Algebra Transformers Cite Share Download PDF Status: Posted Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-8811263","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Physical Sciences - Article","associatedPublications":[],"authors":[{"id":588935504,"identity":"011839f8-5999-42f1-b77e-02e8680eb3db","order_by":0,"name":"Zachary Joseph","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA3UlEQVRIiWNgGAWjYHACZgYGHhs5fhDzAQODAbFa0owlGxgYGxKI18JwOHHDAWK1yLcffmzAIMOcuPl4j/mDhAobYwb2w0c34NPC2JNmnMDAw2a87cwZw4aEM2lmQFem3cDvqhzmw394eGS33cgxbEhsO2zDIMFjhlcLG/8b5gMMPBKMm2cQq4VHIocZ6DADxQ0SEC1mBLVISDwzNmDgSTCWOHOscAbQL8ZshPwi35/8WIKx578cf3vzhg8fKmwM+9kPH8OrBQwYe5B9R1A5GPwgTtkoGAWjYBSMUAAANs9ESX99i8oAAAAASUVORK5CYII=","orcid":"https://orcid.org/0009-0008-4177-958X","institution":"Individual","correspondingAuthor":true,"prefix":"","firstName":"Zachary","middleName":"","lastName":"Joseph","suffix":""}],"badges":[],"createdAt":"2026-02-06 23:50:15","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-8811263/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-8811263/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":102388234,"identity":"79a8ba0a-fbc8-4ec7-8c82-65616bbce90a","added_by":"auto","created_at":"2026-02-11 08:22:48","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":297976,"visible":true,"origin":"","legend":"Article File","description":"","filename":"RiemannianGeometricAlgebraTransformer.pdf","url":"https://assets-eu.researchsquare.com/files/rs-8811263/v1_covered_d563ed23-df76-4bd4-8407-5eb9ca90502d.pdf"},{"id":102388233,"identity":"83eb6bac-423c-4336-b36f-df5c762084ce","added_by":"auto","created_at":"2026-02-11 08:22:43","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"supplement","size":176802,"visible":true,"origin":"","legend":"Supplementary Information: Riemannian Geometric Algebra Transformers","description":"","filename":"SupplementaryInformationRiemannianGeometricAlgebraTransformer.pdf","url":"https://assets-eu.researchsquare.com/files/rs-8811263/v1/c7c29903e0e5f29a3f809207.pdf"}],"financialInterests":"There is \u003cb\u003eNO\u003c/b\u003e Competing Interest.","formattedTitle":"Standard Attention as a Small-Angle Limit of Riemannian Geometric Algebra Transformers","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":true,"hideJournal":true,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"","lastPublishedDoi":"10.21203/rs.3.rs-8811263/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-8811263/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Transformers implicitly assume a flat representational geome-\r\ntry: similarity is Euclidean, composition is linear, and attention\r\nis a softmax of dot products. We show that these operations\r\narise as a degenerate limit of a curved geometric operator\r\nfamily on the rotor manifold Spin(3), realized in the Clifford\r\nalgebra Cl(3, 0). In the small-angle regime, geometric soft-\r\nmax reduces to classical dot-product attention; yet, at depth,\r\nmany small-angle steps accumulate curvature through Baker–\r\nCampbell–Hausdorff commutators, providing a precise sense\r\nin which standard Transformers approximate rotor dynamics.\r\nThe paper focuses on this mathematical discovery and its struc-\r\ntural implications for interpretability; performance studies are\r\na separate empirical program.","manuscriptTitle":"Standard Attention as a Small-Angle Limit of Riemannian Geometric Algebra Transformers","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2026-02-11 08:22:38","doi":"10.21203/rs.3.rs-8811263/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"bdaaefb5-235b-47a0-89d9-075d596befeb","owner":[],"postedDate":"February 11th, 2026","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"posted","subjectAreas":[{"id":62659199,"name":"Physical sciences/Mathematics and computing/Computational science"},{"id":62659200,"name":"Physical sciences/Mathematics and computing/Pure mathematics"},{"id":62659201,"name":"Physical sciences/Mathematics and computing/Applied mathematics"},{"id":62659202,"name":"Physical sciences/Physics/Information theory and computation"}],"tags":[],"updatedAt":"2026-02-11T08:22:38+00:00","versionOfRecord":[],"versionCreatedAt":"2026-02-11 08:22:38","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-8811263","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-8811263","identity":"rs-8811263","version":["v1"]},"buildId":"XKTyCvWXoU3ODBz1xrDgd","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}