Beyond performance: How design choices shape chemical language models

Beyond performance: How design choices shape chemical language models | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Beyond performance: How design choices shape chemical language models Inken Fender, Jannik Andrian Gut, Thomas Lemmin This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-6732063/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 18 Nov, 2025 Read the published version in Journal of Cheminformatics → Version 1 posted 11 You are reading this latest preprint version Abstract Chemical language models (CLMs) have shown strong performance in molecular property prediction and generation tasks. However, the impact of design choices, such as molecular representation format, tokenization strategy, and model architecture, on both performance and chemical interpretability remains underexplored. In this study, we systematically evaluate how these factors influence CLM performance and chemical understanding. We evaluated models through fine-tuning on downstream tasks and probing the structure of their latent spaces using simple classifiers and dimensionality reduction techniques.Despite similar performance on downstream tasks across model configurations, we observed substantial differences in the structure and interpretability of their internal representations. SMILES molecular representation format with atomwise tokenization strategy consistently produced more chemically meaningful embeddings, while models based on BART and RoBERTa architectures yielded comparably interpretable representations. These findings highlight that design choices meaningfully shape how chemical information is represented, even when external metrics appear unchanged. This insight can inform future model development, encouraging more chemically grounded and interpretable CLMs. large language models chemical language models interpretability machine learning for chemistry explainable AI (XAI) SMILES SELFIES RoBERTa BART Full Text Additional Declarations No competing interests reported. Supplementary Files Supportinginformation.pdf Cite Share Download PDF Status: Published Journal Publication published 18 Nov, 2025 Read the published version in Journal of Cheminformatics → Version 1 posted Editorial decision: Revision requested 05 Aug, 2025 Reviews received at journal 24 Jul, 2025 Reviewers agreed at journal 23 Jul, 2025 Reviewers agreed at journal 22 Jul, 2025 Reviews received at journal 18 Jun, 2025 Reviewers agreed at journal 10 Jun, 2025 Reviewers agreed at journal 10 Jun, 2025 Reviewers invited by journal 10 Jun, 2025 Editor assigned by journal 29 May, 2025 Submission checks completed at journal 27 May, 2025 First submitted to journal 23 May, 2025 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-6732063","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":469699880,"identity":"4b28e032-3c53-4234-bf48-223205ac7d36","order_by":0,"name":"Inken Fender","email":"","orcid":"","institution":"University of Bern","correspondingAuthor":false,"prefix":"","firstName":"Inken","middleName":"","lastName":"Fender","suffix":""},{"id":469699881,"identity":"11a622a9-60ff-4b2c-bd31-0538c0046ad9","order_by":1,"name":"Jannik Andrian Gut","email":"","orcid":"","institution":"University of Bern","correspondingAuthor":false,"prefix":"","firstName":"Jannik","middleName":"Andrian","lastName":"Gut","suffix":""},{"id":469699882,"identity":"026b138f-3796-4813-89b5-95e52ff3be58","order_by":2,"name":"Thomas Lemmin","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA5klEQVRIiWNgGAWjYDACCQglAyIOMFQwMLCBRBgbcOvggWrhgWg5Q6oWBsY2qL34tNhLNx/d8OMPAw+/2OnEw4XzDufxSTcfYPi5A48tMsfSbva2MfBIzs7dcHjmtsPFbDLHEhh7z+BzWI7ZDd4GBh6D20AtvNsOJ7ZJ5Bgwg12IR8vNP0CH2YO1zAFpyf9AUMttHjagLdIgLQ1gWxjwa7mRlnZbtk2CRwJkC8+x9MQ2mWMGB3vxaGGfkXzs5ps/NnL8s3M3f+apsU6cP7v54YOfeLRAgQQq9wBBDaNgFIyCUTAK8AIAHq9RMk0lG8MAAAAASUVORK5CYII=","orcid":"","institution":"University of Bern","correspondingAuthor":true,"prefix":"","firstName":"Thomas","middleName":"","lastName":"Lemmin","suffix":""}],"badges":[],"createdAt":"2025-05-23 10:38:17","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-6732063/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-6732063/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1186/s13321-025-01099-w","type":"published","date":"2025-11-18T15:57:32+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":96650196,"identity":"af1b566e-3319-477d-8407-3408ce51ceb7","added_by":"auto","created_at":"2025-11-24 16:09:35","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":2532302,"visible":true,"origin":"","legend":"","description":"","filename":"SMILESorSELFIES.pdf","url":"https://assets-eu.researchsquare.com/files/rs-6732063/v1_covered_73a67544-d093-4bc6-a817-780f9850ea27.pdf"},{"id":84449178,"identity":"ef8802ba-8e00-4c33-a3db-faf231870091","added_by":"auto","created_at":"2025-06-12 06:28:06","extension":"pdf","order_by":0,"title":"","display":"","copyAsset":false,"role":"supplement","size":3914017,"visible":true,"origin":"","legend":"","description":"","filename":"Supportinginformation.pdf","url":"https://assets-eu.researchsquare.com/files/rs-6732063/v1/9c89b02490ee94a1526dbc3d.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Beyond performance: How design choices shape chemical language models","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"journal-of-cheminformatics","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"chin","sideBox":"Learn more about [Journal of Cheminformatics](https://jcheminf.biomedcentral.com/)","snPcode":"13321","submissionUrl":"https://submission.nature.com/new-submission/13321/3","title":"Journal of Cheminformatics","twitterHandle":"@jcheminf","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"BMC/SO AJ","inReviewEnabled":true,"inReviewRevisionsEnabled":true},"keywords":"large language models, chemical language models, interpretability, machine learning for chemistry, explainable AI (XAI), SMILES, SELFIES, RoBERTa, BART","lastPublishedDoi":"10.21203/rs.3.rs-6732063/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-6732063/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Chemical language models (CLMs) have shown strong performance in molecular property prediction and generation tasks. However, the impact of design choices, such as molecular representation format, tokenization strategy, and model architecture, on both performance and chemical interpretability remains underexplored. In this study, we systematically evaluate how these factors influence CLM performance and chemical understanding. We evaluated models through fine-tuning on downstream tasks and probing the structure of their latent spaces using simple classifiers and dimensionality reduction techniques.Despite similar performance on downstream tasks across model configurations, we observed substantial differences in the structure and interpretability of their internal representations. SMILES molecular representation format with atomwise tokenization strategy consistently produced more chemically meaningful embeddings, while models based on BART and RoBERTa architectures yielded comparably interpretable representations. These findings highlight that design choices meaningfully shape how chemical information is represented, even when external metrics appear unchanged. This insight can inform future model development, encouraging more chemically grounded and interpretable CLMs.","manuscriptTitle":"Beyond performance: How design choices shape chemical language models","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-06-12 06:28:01","doi":"10.21203/rs.3.rs-6732063/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2025-08-05T13:55:49+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-07-24T10:59:06+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"158015243621832508832723199777389525094","date":"2025-07-23T12:47:41+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"276223756797092432139176737399594760119","date":"2025-07-22T19:19:34+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-06-18T11:30:12+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"338577835380269615426471484423198468708","date":"2025-06-10T21:51:26+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"304092054108289116006837969989422029162","date":"2025-06-10T16:51:04+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2025-06-10T16:32:11+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2025-05-29T10:43:14+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2025-05-27T09:17:18+00:00","index":"","fulltext":""},{"type":"submitted","content":"Journal of Cheminformatics","date":"2025-05-23T10:32:20+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"journal-of-cheminformatics","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"chin","sideBox":"Learn more about [Journal of Cheminformatics](https://jcheminf.biomedcentral.com/)","snPcode":"13321","submissionUrl":"https://submission.nature.com/new-submission/13321/3","title":"Journal of Cheminformatics","twitterHandle":"@jcheminf","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"BMC/SO AJ","inReviewEnabled":true,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"6ef173c2-94c6-477c-95bf-53568ae831c9","owner":[],"postedDate":"June 12th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[],"tags":[],"updatedAt":"2025-11-24T16:03:24+00:00","versionOfRecord":{"articleIdentity":"rs-6732063","link":"https://doi.org/10.1186/s13321-025-01099-w","journal":{"identity":"journal-of-cheminformatics","isVorOnly":false,"title":"Journal of Cheminformatics"},"publishedOn":"2025-11-18 15:57:32","publishedOnDateReadable":"November 18th, 2025"},"versionCreatedAt":"2025-06-12 06:28:01","video":"","vorDoi":"10.1186/s13321-025-01099-w","vorDoiUrl":"https://doi.org/10.1186/s13321-025-01099-w","workflowStages":[]},"version":"v1","identity":"rs-6732063","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-6732063","identity":"rs-6732063","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}