Feasibility of decoding covert speech in ECoG with a Transformer trained on overt speech

Feasibility of decoding covert speech in ECoG with a Transformer trained on overt speech | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Article Feasibility of decoding covert speech in ECoG with a Transformer trained on overt speech Shuji Komeiji, Takumi Mitsuhashi, Yasushi Iimura, Hiroharu Suzuki, and 3 more This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-3927907/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 8 You are reading this latest preprint version Abstract Several attempts for speech brain–computer interfacing (BCI) have been made to decode phonemes, sub-words, words, or sentences using invasive measurements, such as the electrocorticogram (ECoG), during auditory speech perception, overt speech, or imagined (covert) speech. Decoding sentences from covert speech is a challenging task. Sixteen epilepsy patients with intracranially implanted electrodes participated in this study, and ECoGs were recorded during overt speech, covert speech, and passive listening of eight Japanese sentences, each consisting of three tokens. A Transformer neural network model was applied to decode text sentences from covert speech, which was trained using ECoGs obtained during overt speech. We first examined the proposed Transformer model using the same task for training and testing and then evaluated the model’s performance when trained with overt or perception tasks for decoding covert speech. The Transformer model trained on covert speech achieved an average token error rate (TER) of 46.6% for decoding covert speech, whereas the model trained on overt speech achieved a comparable TER of 46.3% (p > 0.05; d = 0.07). Therefore the challenge of collecting training data for covert speech can be addressed using overt speech. The performance of covert speech can improve by using large amounts of overt speech. Biological sciences/Neuroscience Biological sciences/Neuroscience/Computational neuroscience/Neural decoding Full Text Additional Declarations No competing interests reported. Supplementary Files ECoG2Text2komeijisupplementaryv2.pdf Cite Share Download PDF Status: Under Review Version 1 posted Editorial decision: Revision requested 15 Mar, 2024 Reviews received at journal 26 Feb, 2024 Reviewers agreed at journal 26 Feb, 2024 Reviewers invited by journal 26 Feb, 2024 Editor assigned by journal 25 Feb, 2024 Editor invited by journal 22 Feb, 2024 Submission checks completed at journal 22 Feb, 2024 First submitted to journal 04 Feb, 2024 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-3927907","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Article","associatedPublications":[],"authors":[{"id":274840738,"identity":"2ac40eca-7ee8-4b1c-a302-9e7ec765df18","order_by":0,"name":"Shuji Komeiji","email":"","orcid":"","institution":"Tokyo University of Agriculture and Technology","correspondingAuthor":false,"prefix":"","firstName":"Shuji","middleName":"","lastName":"Komeiji","suffix":""},{"id":274840739,"identity":"0215ea98-c077-4434-964a-20d85da38d22","order_by":1,"name":"Takumi Mitsuhashi","email":"","orcid":"","institution":"Juntendo University School of Medicine","correspondingAuthor":false,"prefix":"","firstName":"Takumi","middleName":"","lastName":"Mitsuhashi","suffix":""},{"id":274840740,"identity":"f22f4056-76e6-4ff8-b4d3-35585c89fdb5","order_by":2,"name":"Yasushi Iimura","email":"","orcid":"","institution":"Juntendo University School of Medicine","correspondingAuthor":false,"prefix":"","firstName":"Yasushi","middleName":"","lastName":"Iimura","suffix":""},{"id":274840741,"identity":"02108fad-b429-48de-9ae7-9cb5f67274d2","order_by":3,"name":"Hiroharu Suzuki","email":"","orcid":"","institution":"Juntendo University School of Medicine","correspondingAuthor":false,"prefix":"","firstName":"Hiroharu","middleName":"","lastName":"Suzuki","suffix":""},{"id":274840743,"identity":"fafcda66-9203-4391-bdb2-d96443eea0d3","order_by":4,"name":"Hidenori Sugano","email":"","orcid":"","institution":"Juntendo University School of Medicine","correspondingAuthor":false,"prefix":"","firstName":"Hidenori","middleName":"","lastName":"Sugano","suffix":""},{"id":274840745,"identity":"e90c1f0e-d2ad-4364-871a-529e76897f7c","order_by":5,"name":"Koichi Shinoda","email":"","orcid":"","institution":"Tokyo Institute of Technology","correspondingAuthor":false,"prefix":"","firstName":"Koichi","middleName":"","lastName":"Shinoda","suffix":""},{"id":274840747,"identity":"bf9158e5-468f-49b8-8f4e-e664d3340e9c","order_by":6,"name":"Toshihisa Tanaka","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA6ElEQVRIiWNgGAWjYJACxgYGBjkUEQN8ynkgWgyMoVoNiNeS2ICsBS+wZz978OOMmj/p/e0H2B/z1Pxh4G8/wFBcgM8WnrxkyQ3HDHJnnElgbOY5ZsAgcSaBwXgGXoflGEg+YDPI3SDBANTCBnTYDQYGYx58WvjfGP988M8g3QCs5Z8BgzxBLRI5ZpIb2wwSwFp42wwYDAhqufHGzHJmn7HhjDOJjTPn9hnzGJ5JbMDrF/b+HOObPd/k5PnbDx/48OabnJzc8cPHjPGFGBIARQs4ohjbjInTgQSYH5OsZRSMglEwCoYzAADyBkRO7IDMRQAAAABJRU5ErkJggg==","orcid":"","institution":"Tokyo University of Agriculture and Technology","correspondingAuthor":true,"prefix":"","firstName":"Toshihisa","middleName":"","lastName":"Tanaka","suffix":""}],"badges":[],"createdAt":"2024-02-04 14:31:29","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-3927907/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-3927907/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":51630997,"identity":"c6296a7a-23de-441e-aa74-d984285d8595","added_by":"auto","created_at":"2024-02-26 09:24:49","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":818832,"visible":true,"origin":"","legend":"","description":"","filename":"ECoG2Text2komeijiv2.pdf","url":"https://assets-eu.researchsquare.com/files/rs-3927907/v1_covered_757b46e3-7360-4081-83f2-2733d5569b24.pdf"},{"id":51630606,"identity":"db083541-3200-4955-898c-f33a7386e2e3","added_by":"auto","created_at":"2024-02-26 09:16:44","extension":"pdf","order_by":2,"title":"","display":"","copyAsset":false,"role":"supplement","size":9457477,"visible":true,"origin":"","legend":"","description":"","filename":"ECoG2Text2komeijisupplementaryv2.pdf","url":"https://assets-eu.researchsquare.com/files/rs-3927907/v1/41f8e96a575a420e84ae26da.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Feasibility of decoding covert speech in ECoG with a Transformer trained on overt speech","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"scientific-reports","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"scirep","sideBox":"Learn more about [Scientific Reports](http://www.nature.com/srep/)","snPcode":"","submissionUrl":"","title":"Scientific Reports","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Scientific Reports","inReviewEnabled":true,"inReviewRevisionsEnabled":true},"keywords":"","lastPublishedDoi":"10.21203/rs.3.rs-3927907/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-3927907/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Several attempts for speech brain–computer interfacing (BCI) have been made to decode phonemes, sub-words, words, or sentences using invasive measurements, such as the electrocorticogram (ECoG), during auditory speech perception, overt speech, or imagined (covert) speech. Decoding sentences from covert speech is a challenging task. Sixteen epilepsy patients with intracranially implanted electrodes participated in this study, and ECoGs were recorded during overt speech, covert speech, and passive listening of eight Japanese sentences, each consisting of three tokens. A Transformer neural network model was applied to decode text sentences from covert speech, which was trained using ECoGs obtained during overt speech. We first examined the proposed Transformer model using the same task for training and testing and then evaluated the model’s performance when trained with overt or perception tasks for decoding covert speech. The Transformer model trained on covert speech achieved an average token error rate (TER) of 46.6% for decoding covert speech, whereas the model trained on overt speech achieved a comparable TER of 46.3% (p \u003e 0.05; d = 0.07). Therefore the challenge of collecting training data for covert speech can be addressed using overt speech. The performance of covert speech can improve by using large amounts of overt speech.","manuscriptTitle":"Feasibility of decoding covert speech in ECoG with a Transformer trained on overt speech","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-02-26 09:16:39","doi":"10.21203/rs.3.rs-3927907/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2024-03-15T05:19:25+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2024-02-26T12:11:46+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"f2c803ef-8f70-4072-af5b-f13fcda735d2","date":"2024-02-26T09:23:35+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2024-02-26T08:45:43+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2024-02-25T23:11:17+00:00","index":"","fulltext":""},{"type":"editorInvited","content":"","date":"2024-02-22T15:55:25+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2024-02-22T15:50:09+00:00","index":"","fulltext":""},{"type":"submitted","content":"Scientific Reports","date":"2024-02-04T14:16:08+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"scientific-reports","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"scirep","sideBox":"Learn more about [Scientific Reports](http://www.nature.com/srep/)","snPcode":"","submissionUrl":"","title":"Scientific Reports","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Scientific Reports","inReviewEnabled":true,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"4dc1f0a7-807d-469e-b868-754e13e4a1b1","owner":[],"postedDate":"February 26th, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[{"id":28960162,"name":"Biological sciences/Neuroscience"},{"id":28960163,"name":"Biological sciences/Neuroscience/Computational neuroscience/Neural decoding"}],"tags":[],"updatedAt":"2024-05-15T02:38:22+00:00","versionOfRecord":[],"versionCreatedAt":"2024-02-26 09:16:39","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-3927907","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-3927907","identity":"rs-3927907","version":["v1"]},"buildId":"qtupq5eGEP_6zYnWcrvyt","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}