MatterChat: A Multi-Modal LLM for Material Science

MatterChat: A Multi-Modal LLM for Material Science | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Article MatterChat: A Multi-Modal LLM for Material Science Yingheng Tang, Wenbin Xu, Jie Cao, Jianzhu Ma, Weilu Gao, Steven Farrell, and 4 more This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-6373794/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 24 Apr, 2026 Read the published version in Nature Machine Intelligence → Version 1 posted You are reading this latest preprint version Abstract Understanding and predicting the properties of inorganic materials is crucial for accelerating advancements in materials science and driving applications in energy, electronics, and beyond. Integrating material structure data with language-based information through multi-modal large language models (LLMs) offers great potential to support these efforts by enhancing human–AI interaction. However, a key challenge lies in integrating atomic structures at full resolution into LLMs. In this work, we introduce MatterChat, a versatile structure-aware multi-modal LLM that unifies material structural data and textual inputs into a single cohesive model. MatterChat employs a bridging module to effectively align a pretrained universal machine learning interatomic potential with a pretrained LLM, reducing training costs and enhancing flexibility. Our results demonstrate that MatterChat significantly improves performance in material property prediction and human-AI interaction, surpassing general-purpose LLMs such as GPT-4. We also demonstrate its usefulness in applications such as more advanced scientific reasoning and step-by-step material synthesis. Physical sciences/Materials science Physical sciences/Materials science/Theory and computation/Computational methods Foundational model Large Language Model Multi-Modal Learning Inorganic Material Full Text Additional Declarations There is NO Competing Interest. Supplementary Files MatterChatSI.pdf SI MatterChatSIv2.pdf Supplementary Information Cite Share Download PDF Status: Published Journal Publication published 24 Apr, 2026 Read the published version in Nature Machine Intelligence → Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-6373794","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Article","associatedPublications":[],"authors":[{"id":457787166,"identity":"fb2c5d84-e008-4218-bae8-f6c18e6858ac","order_by":0,"name":"Yingheng Tang","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAuUlEQVRIiWNgGAWjYNACHhsZA3aiVbOBCJk0HgNm0rTYHCZBi8H95mcPv+Sc5zFnZmB+8bGNGC3H2MyNZc7c5rFsZmCznEmcFgYzacme2zwGhxnYjHnOEKWF/Zu05L9zJGnhMZP8wHMApIX5MU8FEVokj+WUSTPwJAP9wtjGOIMYLXyHj2+T/MFjJ2fO3nz4wwcDIrQoHGBgYOYBMxnbJIjQwMAg3wBU+wPCZv5AlJZRMApGwSgYcQAAtikwlVWjQZcAAAAASUVORK5CYII=","orcid":"https://orcid.org/0009-0001-5362-2546","institution":"Lawrence Berkeley National Laboratory","correspondingAuthor":true,"prefix":"","firstName":"Yingheng","middleName":"","lastName":"Tang","suffix":""},{"id":457787167,"identity":"ec9ab048-994c-4d56-815a-4e4bee06c552","order_by":1,"name":"Wenbin Xu","email":"","orcid":"","institution":"Lawrence Berkeley National Laboratory","correspondingAuthor":false,"prefix":"","firstName":"Wenbin","middleName":"","lastName":"Xu","suffix":""},{"id":457787168,"identity":"58424e7a-38ae-4588-8720-c20a6e6c29ba","order_by":2,"name":"Jie Cao","email":"","orcid":"","institution":"University of Colorado Boulder","correspondingAuthor":false,"prefix":"","firstName":"Jie","middleName":"","lastName":"Cao","suffix":""},{"id":457787169,"identity":"bcc62d49-6291-4d47-ae6f-bc9b5590230b","order_by":3,"name":"Jianzhu Ma","email":"","orcid":"https://orcid.org/0000-0002-8236-6609","institution":"Tsinghua University","correspondingAuthor":false,"prefix":"","firstName":"Jianzhu","middleName":"","lastName":"Ma","suffix":""},{"id":457787170,"identity":"7b762558-a023-44af-afd8-97fb88ddd4f4","order_by":4,"name":"Weilu Gao","email":"","orcid":"https://orcid.org/0000-0003-3139-034X","institution":"The University of Utah","correspondingAuthor":false,"prefix":"","firstName":"Weilu","middleName":"","lastName":"Gao","suffix":""},{"id":457787171,"identity":"8d9711c5-055d-46fb-ac96-b687aa7e4f57","order_by":5,"name":"Steven Farrell","email":"","orcid":"","institution":"Lawrence Berkeley National Lab","correspondingAuthor":false,"prefix":"","firstName":"Steven","middleName":"","lastName":"Farrell","suffix":""},{"id":457787172,"identity":"f32dcd8c-52db-4986-9e35-1bc7d7a98d27","order_by":6,"name":"N. Benjamin Erichson","email":"","orcid":"https://orcid.org/0000-0003-0667-3516","institution":"Lawrence Berkeley National Lab","correspondingAuthor":false,"prefix":"","firstName":"N.","middleName":"Benjamin","lastName":"Erichson","suffix":""},{"id":457787173,"identity":"bf8862bb-f5d5-476e-80a9-4d84befd0714","order_by":7,"name":"Michael Mahoney","email":"","orcid":"","institution":"UC Berkeley","correspondingAuthor":false,"prefix":"","firstName":"Michael","middleName":"","lastName":"Mahoney","suffix":""},{"id":457787174,"identity":"f58ed290-d192-4e8f-b4c8-af139cdf706e","order_by":8,"name":"Andy Nonaka","email":"","orcid":"","institution":"Lawrence Berkeley National Laboratory","correspondingAuthor":false,"prefix":"","firstName":"Andy","middleName":"","lastName":"Nonaka","suffix":""},{"id":457787175,"identity":"ebe30eeb-fcdb-4b14-aefe-8ad603f1d02f","order_by":9,"name":"Zhi Yao","email":"","orcid":"","institution":"Lawrence Berkeley National Laboratory","correspondingAuthor":false,"prefix":"","firstName":"Zhi","middleName":"","lastName":"Yao","suffix":""}],"badges":[],"createdAt":"2025-04-04 06:51:34","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-6373794/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-6373794/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1038/s42256-026-01214-y","type":"published","date":"2026-04-24T04:00:00+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":107775894,"identity":"262408c2-7c7d-45b2-834b-702444d1bda0","added_by":"auto","created_at":"2026-04-25 07:05:51","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":4139003,"visible":true,"origin":"","legend":"Article File","description":"","filename":"MatterChatmainv2.pdf","url":"https://assets-eu.researchsquare.com/files/rs-6373794/v1_covered_5f150a0d-1b10-4b57-9bb1-aa70cac7e575.pdf"},{"id":83100514,"identity":"ff68bc3b-2361-42cf-91c7-af941fd086c7","added_by":"auto","created_at":"2025-05-20 04:50:21","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"supplement","size":1151966,"visible":true,"origin":"","legend":"SI","description":"","filename":"MatterChatSI.pdf","url":"https://assets-eu.researchsquare.com/files/rs-6373794/v1/db22a3dc8977bd50f0e447fc.pdf"},{"id":83100515,"identity":"9bba9b46-4ffb-4ac9-9ee6-a2bac6ec134e","added_by":"auto","created_at":"2025-05-20 04:50:21","extension":"pdf","order_by":2,"title":"","display":"","copyAsset":false,"role":"supplement","size":1153845,"visible":true,"origin":"","legend":"Supplementary Information","description":"","filename":"MatterChatSIv2.pdf","url":"https://assets-eu.researchsquare.com/files/rs-6373794/v1/f15fa301967af188c9b7e0ba.pdf"}],"financialInterests":"There is \u003cb\u003eNO\u003c/b\u003e Competing Interest.","formattedTitle":"MatterChat: A Multi-Modal LLM for Material Science","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":true,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"nature-portfolio","isNatureJournal":true,"hasQc":false,"allowDirectSubmit":false,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"","title":"Nature Portfolio","twitterHandle":"","acdcEnabled":false,"dfaEnabled":false,"editorialSystem":"ejp","reportingPortfolio":"","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Foundational model, Large Language Model, Multi-Modal Learning, Inorganic Material","lastPublishedDoi":"10.21203/rs.3.rs-6373794/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-6373794/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Understanding and predicting the properties of inorganic materials is crucial for accelerating advancements in materials science and driving applications in energy, electronics, and beyond. Integrating material structure data with language-based information through multi-modal large language models (LLMs) offers great potential to support these efforts by enhancing human–AI interaction. However, a key challenge lies in integrating atomic structures at full resolution into LLMs. In this work, we introduce MatterChat, a versatile structure-aware multi-modal LLM that unifies material structural data and textual inputs into a single cohesive model. \r\nMatterChat employs a bridging module to effectively align a pretrained universal machine learning interatomic potential with a pretrained LLM, reducing training costs and enhancing flexibility. Our results demonstrate that MatterChat significantly improves performance in material property prediction and human-AI interaction, surpassing general-purpose LLMs such as GPT-4. We also demonstrate its usefulness in applications such as more advanced scientific reasoning and step-by-step material synthesis.","manuscriptTitle":"MatterChat: A Multi-Modal LLM for Material Science","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-05-20 04:50:16","doi":"10.21203/rs.3.rs-6373794/v1","editorialEvents":[],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"nature-machine-intelligence","isNatureJournal":true,"hasQc":false,"allowDirectSubmit":false,"externalIdentity":"natmachintell","sideBox":"Learn more about [Nature Machine Intelligence](http://www.nature.com/natmachintell/)","snPcode":"","submissionUrl":"","title":"Nature Machine Intelligence","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"ejp","reportingPortfolio":"Nature Research","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"38e4e349-5f3b-411a-b5e3-5789b1046afe","owner":[],"postedDate":"May 20th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[{"id":48654481,"name":"Physical sciences/Materials science"},{"id":48654482,"name":"Physical sciences/Materials science/Theory and computation/Computational methods"}],"tags":[],"updatedAt":"2026-04-25T07:05:34+00:00","versionOfRecord":{"articleIdentity":"rs-6373794","link":"https://doi.org/10.1038/s42256-026-01214-y","journal":{"identity":"nature-machine-intelligence","isVorOnly":false,"title":"Nature Machine Intelligence"},"publishedOn":"2026-04-24 04:00:00","publishedOnDateReadable":"April 24th, 2026"},"versionCreatedAt":"2025-05-20 04:50:16","video":"","vorDoi":"10.1038/s42256-026-01214-y","vorDoiUrl":"https://doi.org/10.1038/s42256-026-01214-y","workflowStages":[]},"version":"v1","identity":"rs-6373794","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-6373794","identity":"rs-6373794","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}