Decoding semantics from natural speech using human intracranial EEG

Decoding semantics from natural speech using human intracranial EEG | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Decoding semantics from natural speech using human intracranial EEG Camille R. C. Pescatore , Haoyu Zhang , Alex E. Hadjinicolaou , Angelique C. Paulk , John D. Rolston , R. Mark Richardson , Ziv M. Williams , View ORCID Profile Jing Cai , Sydney S. Cash doi: https://doi.org/10.1101/2025.02.10.637051 Camille R. C. Pescatore 1 Department of Neurology, Massachusetts General Hospital, Harvard Medical School , Boston, MA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Haoyu Zhang 1 Department of Neurology, Massachusetts General Hospital, Harvard Medical School , Boston, MA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Alex E. Hadjinicolaou 1 Department of Neurology, Massachusetts General Hospital, Harvard Medical School , Boston, MA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Angelique C. Paulk 1 Department of Neurology, Massachusetts General Hospital, Harvard Medical School , Boston, MA 2 Center for Neurotechnology and Neurorecovery, Department of Neurology, Massachusetts General Hospital , Boston, MA Find this author on Google Scholar Find this author on PubMed Search for this author on this site John D. Rolston 3 Department of Neurosurgery, Brigham and Women’s Hospital, Harvard Medical School , Boston, MA Find this author on Google Scholar Find this author on PubMed Search for this author on this site R. Mark Richardson 4 Department of Neurosurgery, Massachusetts General Hospital, Harvard Medical School , Boston, MA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ziv M. Williams 4 Department of Neurosurgery, Massachusetts General Hospital, Harvard Medical School , Boston, MA 5 Harvard-MIT Division of Health Sciences and Technology , Boston, MA 6 Harvard Medical School, Program in Neuroscience , Boston, MA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Jing Cai 4 Department of Neurosurgery, Massachusetts General Hospital, Harvard Medical School , Boston, MA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Jing Cai For correspondence: jcai5{at}mgh.harvard.edu Sydney S. Cash 1 Department of Neurology, Massachusetts General Hospital, Harvard Medical School , Boston, MA 2 Center for Neurotechnology and Neurorecovery, Department of Neurology, Massachusetts General Hospital , Boston, MA 5 Harvard-MIT Division of Health Sciences and Technology , Boston, MA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Supplementary material Preview PDF Abstract Brain-computer interfaces (BCIs) hold promise for restoring natural language production capabilities in patients with speech impairments, potentially enabling smooth conversation that conveys meaningful information via synthesized words. While considerable progress has been made in decoding phonetic features of speech, our ability to extract lexical semantic information (i.e. the meaning of individual words) from neural activity remains largely unexplored. Moreover, most existing BCI research has relied on controlled experimental paradigms rather than natural conversation, limiting our understanding of semantic decoding in ecological contexts. Here, we investigated the feasibility of decoding lexical semantic information from stereo-electroencephalography (sEEG) recordings in 14 participants during spontaneous conversation. Using multivariate pattern analysis, we were able to decode word level semantic features during language production with an average accuracy of 21% across all participants compared to a chance level of 10%. This semantic decoding remained robust across different semantic representations while maintaining specificity to semantic features. Further, we identified a distributed left-lateralized network spanning precentral gyrus, pars triangularis, and middle temporal cortex, with low-frequency oscillations showing stronger contributions. Together, our results establish the feasibility of extracting word meanings from neural activity during natural speech production and demonstrate the potential for decoding semantic content from unconstrained speech. Introduction Brain-Computer Interfaces (BCIs) represent a promising means for restoring communicative abilities in individuals with severe language production impairments ( 1 - 6 ). By circumventing the compromised neural-motor pathways, BCIs aim to directly translate the patient’s intended speech into meaningful output ( 7 , 8 ). Over the past decade, the majority of BCI approaches have employed both invasive and non-invasive neural recording techniques to reconstruct various phonetic features of speech, including acoustic waveforms and phoneme sequences ( 4 , 9 ). Such approaches have yielded increasingly fruitful results, achieving remarkable decoding accuracies in controlled experimental settings ( 1, 7, 9-12 ). In contrast, comparatively little research has focused on the reconstruction of semantics from neural signals. Semantic planning plays an important role, as the fundamental aspect of language production is to convey meaningful message between speakers ( 13 ), which allows for more robust and accurate communication even under challenging conditions like low signal quality or limited bandwidth ( 14 ). Semantic decoding, therefore, has the potential to provide natural, rapid and effective speech restoration. Previous studies have demonstrated the feasibility of decoding semantic meanings from neural signals during both language comprehension and production using imaging methods, and have achieved notable accuracy in reconstructing the meaning of sentences ( 15 , 16 ). However, these investigations primarily employed blocked experimental designs and lack the scenarios of natural language production. Natural language production during dialogue involves a diverse combination of words to convey a vast range of meanings, with temporal dynamics that can vary significantly based on the interactions between speakers ( 17 , 18 ). As a result, decoding semantic features from natural language production in such contexts poses a considerably greater challenge ( 19 ). Due to these complexities, our understanding of how effectively lexical-semantic features can be decoded from language production in natural conversations remains limited. Recent neuroimaging studies have revealed that semantic representation relies on a distributed network of interconnected brain regions ( 20 - 25 ). This semantic network includes regions in the temporal, parietal, frontal lobes, and subcortical structures, including activations in the anterior temporal lobe (ATL) and posterior middle temporal gyrus (pMTG) during various semantic tasks ( 26 , 27 ). Further, the involvement of modality-specific sensory and motor areas suggests that semantic representations are partially grounded in the neural circuits that support language perception and production ( 28 , 29 ). Together, this widespread, interconnected network underlies the complex nature of semantic processing in the human brain. In this work, we employed a new approach by utilizing intracranial stereo-electroencephalography (sEEG) recordings during natural conversations to investigate the extent to which lexical semantic features can be decoded in language production ( 30 ). This approach allowed us to capture neural activity associated with spontaneous language production, providing a more naturalistic context. Using multiclass decoding of neural activity during natural speech production, we successfully predicted lexical semantic features across all participants, demonstrating the feasibility of reliably decoding lexical semantic information from neural activity during natural language production. Key brain regions and oscillations in specific frequency bands were also identified. Together, our findings pave the way for future research into the neural basis of semantic processing in real-world communication contexts. Results Neural recording during natural language production In this work, we utilized semi-chronically implanted depth sEEG recordings from 14 participants undergoing epilepsy monitoring as part of their clinical care (6 females and 8 males, average age of 34, ranging between 16 to 59, Figure 1A ) ( 30 ). Recording channels were distributed across the frontotemporal lobes, covering major brain areas known for language production planning, including superior and middle temporal cortices, precentral and pars triangularis ( Figure 1B ). Voltages recorded from these channels were filtered to six frequency bands, including theta (4-8 Hz), alpha (8-12 Hz), beta (12-30 Hz), low-gamma (30-55 Hz), mid-gamma (70-110 Hz) and high-gamma (130-170 Hz), and the envelopes of each band were calculated. Download figure Open in new tab Figure 1. Natural speech production and channel distribution. A . Participants engaged in natural conversations while their neural activity was recorded by stereo-electroencephalography. Neural activity was bipolar-referenced and filtered to various frequency bands, and the envelope of each band was extracted. Meanwhile, the words the participants produced were transcribed and temporally aligned with the neural recordings. The lexical semantic content of participants’ speech was encoded using word2vec embeddings and subsequently categorized into 10 distinct semantic clusters to capture different word meanings. B . The recording included a total of 1908 channels, which were primarily distributed in the frontal, temporal and mesial areas in both hemispheres. C . The total number of words and unique words are plotted by each participant. On average, each participant produced 2,696 words, with 621 unique words among them. D . To categorize the meaning of the words, we performed spectral clustering on the word2vec embeddings into 10 categories. To visualize this semantic organization, we projected selected words from one participant onto their first two t-SNE dimensions, illustrating the natural clustering of meaning in this reduced dimensional space. To allow for the most ecological speech production, participants engaged in natural conversations with an experimenter, which lasted between 16 to 92 minutes. From these conversations, we transcribed words in sentences during speech production. In total, each participant spoke an average of 2,696 ± 1,786 (Mean ± s.t.d) words, including 621 ± 278 unique, unrepeated words ( Figure 1C ). The sentences cover a wide range of topics, such as personal experience, movies, video games, career, family and travel plans ( Table S1 ). Lexical semantic categorization In natural conversation, lexical semantic meaning can be highly diverse, so we first represented these semantic features using vectorized representations to enable more accurate decoding of neural activity. Word2vec represents a powerful approach for encoding lexical semantics through dense vector representations ( 31 ). By learning from extensive text corpora, word2vec captures semantic relationships between words based on their contextual co-occurrence patterns. This methodology enables words with similar semantic content to cluster together in a high-dimensional vector space. The ability to encode lexical semantic information in a vectorized format makes word2vec an ideal tool for examining whether neural activity reflects underlying semantic structures. In our analysis, we extracted 300-dimensional word2vec embeddings for each word uttered by participants, and excluded words not present in the pre-trained vocabulary. Given the open-ended nature of our dataset, which allowed participants to freely produce language without lexical constraints, comprehensively characterizing the semantic content of the thousands of recorded words presents a significant analytical challenge. To mitigate this complexity, we implemented a dimensionality reduction approach by categorizing words into 10 semantic clusters using word2vec embedding representations. This number of clusters ensures that each category captures a meaningful portion of the vocabulary, with most containing between 2% and 20% of the produced words. The resulting clusters were comprised of words with similar meanings. For example, ‘sisters and ‘family’ were in the same cluster, whereas ‘scary’ and ‘travel’ were in separate clusters ( Figure 1D ). In addition, this clustering procedure produced generally stable results across multiple runs. On average, 68.8% of the words were consistently grouped into the same clusters across different clustering iterations, regardless of the initial random seed used to start the clustering process. This high level of consistency suggests that the clustering method is reliable. The exact words that were consistent across different iterations were shown in Table S2 . In this way, we categorized words reliably based on their lexical semantic representation. Lexical semantic decoding from local field potentials To investigate to what extent the semantic category of individual words can be readout from neural activity during natural sentence production, we developed a multivariate decoding model to predict the identity of 10 clusters of lexical semantics. This approach allowed us to test whether patterns of neural activity contained sufficient information to predict the semantic feature of each word. Since we had 10 possible clusters, chance performance would be 10% if the model made predictions randomly. However, using a 5-fold cross-validation framework where the model was trained on 80% of words and tested on held-out words, we found that our decoding model reached a decoding accuracy of 20.9 ± 3.4% (mean ± s.t.d) across 14 participants ( Figure 2A ). Notably, some participants achieved a decoding accuracy as high as 26.50%, which was 2.65 times higher than what can be explained by chance. The decoding accuracy across different semantic clusters also displayed variations (Standard deviation across clusters ranges from 7% to 19%, with a mean of 13%, Figure S1 ). Next, we compared the performance against a permutation control where we randomly shuffled the word labels relative to the neural data, thereby destroying any true relationship between neural patterns and word meanings while preserving the distribution of words across clusters. This permutation analysis yielded an average accuracy of 10.0%, confirming that our observed decoding performance significantly exceeded chance levels (p = 0.0001 for all participants) and could not be explained by any imbalance in the number of words per semantic cluster. Download figure Open in new tab Figure 2. Lexical semantic decoding results and control analysis. A . Left , example sentence with decoding results for each word. The color labels the cluster of the word, whereas the transparency labels whether the word was predicted correctly or not. Right , single-participant decoding accuracies (blue) are shown with the performance derived from randomized permutation controls (gray). The theoretical chance level was determined by the reciprocal of the total number of clusters. B . The percentage improvement in decoding accuracy compared to chance remained consistent across different numbers of clusters. C . Decoding accuracy for phoneme clusters remained at chance level, indicating that phonetic planning features did not drive the result in lexical semantic decoding. D . Comparable decoding accuracies were achieved using fastText (top) and GloVe (bottom) embeddings. Next, to ensure that the decoding capability of the neural activity was not limited only to the specific number of lexical semantic clusters, we performed the same modeling procedures, but using various number of word clusters of 5, 10, 20 and 30 clusters ( Figure 2B , Figure S2 ). Overall, the decoding accuracies were significantly higher than chance for all the numbers of cluster we tested (T-test, p ≤ 1 × 10 −7 ). Further, despite a decrease of absolute decoding accuracy with the increase of cluster number, the relative increase of accuracy compared to chance remains consistently higher than chance among 5, 10, 20 and 30 clusters, ranging from 99.1% to 156.0%. This suggests that the decoding feasibility is robust across varying levels of granularity in the semantic clustering. We further investigated the temporal consistency of lexical semantic decoding by examining whether the decoding remained stable across different neural activity time windows. Utilizing the same decoding methods, we analyzed neural activity averaged across window durations ranging from 100 to 900 milliseconds prior to word onset. The average decoding accuracy across participants consistently ranged from 19% to 23%, with no statistically significant differences between temporal windows (One-way ANOVA, p = 0.21, Figure S3 ). These findings suggest that lexical semantic neural representations are not confined to a specific temporal duration but rather exhibit a consistent pattern of neural encoding across multiple time intervals. To rule out the possibility that lexical semantic decoding results arise from neural activity related to the motor planning of word pronunciation, we conducted a control analysis to decode the phonetic features of each word. Specifically, we applied identical decoding procedures to seven consonant clusters for each word. We found, however, that the decoding accuracy for phonetic features (15.9% ± 3.0%) was statistically indistinguishable from chance levels (15.1% ± 1.8%, Permutation test, p = 0.22). These results demonstrate that our semantic decoding is not driven by neural representations of word pronunciation, thereby suggesting the specificity of semantic feature encoding in neural activity. Last, we examined whether the decoding capability was generalizable across different lexical semantic embeddings. Instead of using word2vec, we performed the decoding analysis on categories based on two independent word embeddings, FastText and GloVe, and observed similar decoding accuracies from these two embeddings (FastText: 23.3% ± 4.6%; GloVe: 20.7% ± 3.6%), with all participants showing significantly higher than chance decoding (Permutation test, p = 1 × 10 −5 ). Further, the decoding accuracies were similar across these three embeddings (ANOVA, p = 0.18), indicating that the prediction of lexical semantic was not dependent on the specific word embeddings. Lexical semantic computation across various brain areas and frequencies Next, we identified the specific brain regions and frequency bands contributing to the decoding of lexical semantic clusters to investigate the specific neural basis of the lexical semantic decoding. Based on the selection criteria of statistically significant responses to lexical semantic features using the Wilcoxon rank-sum test (p-value < 0.05, Bonferroni corrected for clusters), we found an average of 12.1% channels showing selective responses across participants ( Figure 3A ). This selectivity was significantly higher than those from chance in all participants (Chi square proportion test, p < 0.05), with majority of channels and frequencies displayed selective responses to only one semantic category (91% of responding channels, Figure S4 ). Together, these findings suggest that lexical semantic information is encoded through the coordinated activity of these selectively responding channels. Download figure Open in new tab Figure 3. Neural contributions to lexical semantic decoding across brain regions and frequency bands. A . Percentage of selected channels across participants. B . Distribution of selected channels across various brain areas. The transparency of each channel indicates how frequently this channel is selected across different clustering iterations. C . The percentage of selected channels across various brain areas. Among these areas, we identified that pars-triangularis, precentral and middle temporal cortex (bolded) showed significantly higher percentage of responding channels than other areas. D . The decoding accuracy using channels from these three areas. E . Distribution of neural signal contributions across frequency bands in the selected channels. Blue bars represent the percentage of channels within each frequency band, while orange bars indicate the relative importance of channels based on their feature importance in each frequency range. The channels that responded to lexical semantic were distributed broadly across multiple brain areas ( Figure 3B, C ). Not surprisingly, we found that the left hemisphere was more involved in semantic planning compared to the right hemisphere (T-test, p = 1.63x 10 −7 ). This lateralization effect is, as expected, consistent with expectations of language lateralization ( 32 ). Further, we focused on identifying the brain areas with a high number of recording channels across participants (> 30 channels), which resulted in 13 specific brain areas. To determine which of these high-coverage areas were significantly different from the rest, we performed a T-test comparing each area to the others. The result revealed that middle temporal cortex, precentral and pars triangularis showed significantly higher ratio of responding channels than the other brain areas (T-test, p < 0.05, Figure S5 ). Together, by examining neural response during language production planning, we found a specialized yet distributed nature of semantic representation in various brain areas. Based on the identification of semantic processing brain areas, we next investigated the extent to which lexical semantic features could be decoded using only these targeted regions. Decoding analysis using channels only from these three regions showed an accuracy of 16.4 ± 3.6% (mean ± s.t.d), which were significantly higher than those predicted by chance (Permutation test, p < 10 −5 , Figure 3D ). However, decoding accuracy from channels in these three areas were significantly lower compared to decoding accuracy using channels from all areas (Permutation test, p = 0.006). Together, we found that the semantic processing areas contributed significantly to lexical semantic decoding, though including broader area will increase the decoding accuracy. To further understand the neural mechanisms underlying lexical semantic decoding, we analyzed the distribution of neural responses across different frequency bands. When examining the selected neural responses, we found that all investigated frequency bands (theta, alpha, beta and gamma bands) were present but that there was greater representation in lower frequencies (ANOVA, p = 9 × 10 −78 , Figure 3E ). A similar trend was also observed when considering the feature importance of different frequency bands, revealing a strong negative correlation ( Figure 3E , Pearson correlation coefficient, R = -0.83, p = 0.043). This low-frequency bias was not an artifact of our time window selection method, as using smaller windows yielded consistent results ( Figure S6 ). Together, these findings collectively illustrate how a diverse population of neural channels and specific frequency bands collaboratively contribute to the planning of lexical semantic production, suggesting that lower frequency neural oscillations play a particularly significant role in semantic information processing. Discussion While recent advances in neural technology and computational methods have enabled the decoding of various aspects of language from neural signals ( 1 - 10 ), the ability to extract word-specific semantic content during natural speech production has remained elusive. Here, we demonstrate the successful decoding of lexical semantic information from sEEG recordings in 14 participants engaged in spontaneous language production during natural conversation ( 30 ). Our analyses revealed consistent above-chance decoding accuracy across participants, with robust performance across multiple semantic clustering and embedding models. Importantly, control analyses showed chance-level decoding of phonemic features, indicating that little phonetic features were encoded in these channels, therefore the semantic decoding results was not driven by these low-level acoustic properties. These findings provide compelling evidence for the neural encoding of lexical semantics while producing natural speech and establish the feasibility of extracting word meanings from human brain activity during unconstrained language production. Natural conversation represents a uniquely challenging context for studying language production mechanisms, as it involves potentially richer semantics and temporal dynamics compared to controlled experimental paradigms ( 17 , 18 ). During spontaneous discourse, speakers freely select topics, vocabulary, and semantic content while engaging in dynamic linguistic behaviors such as co-construction and backchanneling ( 33 ). These naturalistic elements introduce substantial variability that has traditionally complicated the systematic study of language processes. Nevertheless, our striking ability to achieve above-chance decoding accuracy of lexical semantic features under these unconstrained conditions demonstrates the robustness of semantic representations in neural activity during natural communicative interactions. While this level of decoding is insufficient for any clinically useful decoding system, this success in decoding semantic content from natural speech extends previous findings from more controlled experimental settings to ecologically valid contexts ( 16 ). It also suggests that more intentionally diverse and complete cortical coverage may permit even greater decoding accuracy. Another striking finding is that we showed the neural substrates supporting lexical semantic processing during natural language production comprise an extensive, interconnected network with left hemisphere lateralization ( 21 , 32 ). Across this distributed system, we identified regions that contributed disproportionately higher to semantic decoding including the precentral gyrus, pars triangularis, and middle temporal cortex. This suggests that semantic representations during speech production are predominantly, although not exclusively, encoded by areas known to process language and motor planning ( 34 - 38 ). However, we showed that semantic planning extends beyond these language areas: The decoding accuracy from all regions was significantly higher than those from the three major regions, suggesting that semantic information is distributed across multiple cortical regions rather than being confined to traditional language areas. The distributed nature of semantic processing suggests that optimal semantic BCI performance may need to incorporate information from a broader network of brain areas to capture intended meaning during speech production. Further, these results indicate that semantic information was encoded across a broad frequency range, from theta to gamma bands. Notably, lower frequencies yielded higher decoding accuracy gain compared to higher frequencies, potentially reflecting semantic retrieval processes ( 39 - 41 ). This preference to lower frequencies was not accounted for by the time window used to calculate neural activity, as smaller time windows resulted in similar trends. Although the type and design of recording electrodes may impact the contribution of frequency bands ( 42 ), these findings indicate that lexical semantic processing during natural speech production is supported by a distributed neural network across multiple frequency bands, with particularly strong contributions from lower-frequency oscillations. Together, our findings establish the feasibility of decoding lexical semantic information during natural language production and provide a detailed mapping of the neural substrates supporting this process. However, there are several limitations for this study: First, our study prioritized demonstrating the basic feasibility of semantic decoding over maximizing word-level accuracy. Future investigations employing larger word samples per participant may achieve higher decoding precision and enable more fine-grained semantic categorization. Further, we observed substantial inter-individual variability in decoding performance, and the contributions of electrode location variation versus genuine individual neural coding differences remain to be revealed. Despite these limitations, this work represents a crucial first step in decoding lexical semantic features during natural speech production and establishes a foundation for future research in this domain. Most importantly, we envision that this approach may be leveraged with other models of speech decoding, particularly those which are phoneme focused, to produce ever smoother, more accurate and rapid abilities to restore communication to those who have lost it due to severe neurological injury. Acknowledgement C.R.C.P is supported by Madame Stella Zorzet and the Zorzet legacy through the EPFL WISH Foundation. Z.M.W. is supported by NIH R01DC019653 and NIH U01NS123130. J.C is supported by the Mussallem Transformative Award and American Association of University Women. S.S.C. is supported by NIH U01NS098968. We utilized Anthropic Claude to refine the language of this manuscript. References 1. ↵ G. K. Anumanchipalli , J. Chartier , E. F. Chang , Speech synthesis from neural decoding of spoken sentences . Nature 568 , 493 – 498 ( 2019 ). OpenUrl CrossRef PubMed 2. F. R. Willett , D. T. Avansino , L. R. Hochberg , J. M. Henderson , K. V. Shenoy , High-performance brain-to-text communication via handwriting . Nature 593 , 249 – 254 ( 2021 ). OpenUrl CrossRef PubMed 3. J. R. Wolpaw , N. Birbaumer , D. J. McFarland , G. Pfurtscheller , T. M. Vaughan , Brain–computer interfaces for communication and control . Clinical neurophysiology 113 , 767 – 791 ( 2002 ). OpenUrl CrossRef PubMed Web of Science 4. ↵ A. B. Silva , K. T. Littlejohn , J. R. Liu , D. A. Moses , E. F. Chang , The speech neuroprosthesis . Nature Reviews Neuroscience , 1 – 20 ( 2024 ). 5. M. W. Mathis , A. P. Rotondo , E. F. Chang , A. S. Tolias , A. Mathis , Decoding the brain: From neural representations to mechanistic models . Cell 187 , 5814 – 5832 ( 2024 ). OpenUrl CrossRef PubMed 6. ↵ S. L. Metzger et al. , A high-performance neuroprosthesis for speech decoding and avatar control . Nature 620 , 1037 – 1046 ( 2023 ). OpenUrl CrossRef PubMed 7. ↵ F. R. Willett et al. , A high-performance speech neuroprosthesis . Nature 620 , 1031 – 1036 ( 2023 ). OpenUrl CrossRef PubMed 8. ↵ N. S. Card et al. , An accurate and rapidly calibrating speech neuroprosthesis . New England Journal of Medicine 391 , 609 – 618 ( 2024 ). OpenUrl CrossRef PubMed 9. ↵ J. Kohler et al. , Synthesizing speech from intracranial depth electrodes using an encoder-decoder framework . arXiv preprint arxiv: 2111.01457 , ( 2021 ). 10. ↵ G. H. Wilson et al. , Decoding spoken English from intracortical electrode arrays in dorsal precentral gyrus . Journal of neural engineering 17 , 066007 ( 2020 ). OpenUrl CrossRef PubMed 11. S. D. Stavisky et al. , Neural ensemble dynamics in dorsal motor cortex during speech in people with paralysis . Elife 8 , e46015 ( 2019 ). OpenUrl CrossRef PubMed 12. T. Milekovic et al. , Stable long-term BCI-enabled communication in ALS and locked-in syndrome using LFP signals . Journal of neurophysiology 120 , 343 – 360 ( 2018 ). OpenUrl CrossRef PubMed 13. ↵ H. Grice , Logic and conversation . Syntax and semantics 3 , ( 1975 ). 14. ↵ X. Luo , H.-H. Chen , Q. Guo , Semantic communications: Overview, open issues, and future research directions . IEEE Wireless Communications 29 , 210 – 219 ( 2022 ). OpenUrl CrossRef 15. ↵ S. L. Frisby , A. D. Halai , C. R. Cox , M. A. L. Ralph , T. T. Rogers , Decoding semantic representations in mind and brain . Trends in cognitive sciences 27 , 258 – 281 ( 2023 ). OpenUrl CrossRef PubMed 16. ↵ J. Tang , A. LeBel , S. Jain , A. G. Huth , Semantic reconstruction of continuous language from noninvasive brain recordings . Nature Neuroscience 26 , 858 – 866 ( 2023 ). OpenUrl CrossRef PubMed 17. ↵ E. A. Maguire , Studying the freely-behaving brain with fMRI . Neuroimage 62 , 1170 – 1176 ( 2012 ). OpenUrl CrossRef PubMed Web of Science 18. ↵ L. Schilbach et al. , Toward a second-person neuroscience1 . Behavioral and brain sciences 36 , 393 – 414 ( 2013 ). OpenUrl CrossRef PubMed 19. ↵ Z. Zada et al. , A shared model-based linguistic space for transmitting our thoughts from brain to brain in natural conversations . Neuron , ( 2024 ). 20. ↵ C. Herff , M. Verwoert , J. Amigó-Vega , M. C. Ottenhoff , in 2023 IEEE International Conference on Systems, Man, and Cybernetics (SMC) . ( IEEE , 2023 ), pp. 4764 – 4769 . 21. ↵ A. G. Huth , W. A. De Heer , T. L. Griffiths , F. E. Theunissen , J. L. Gallant , Natural speech reveals the semantic maps that tile human cerebral cortex . Nature 532 , 453 – 458 ( 2016 ). OpenUrl CrossRef PubMed 22. E. Fedorenko et al. , Neural correlate of the construction of sentence meaning . Proc Natl Acad Sci U S A 113 , E6256 – E6262 ( 2016 ). OpenUrl Abstract / FREE Full Text 23. M. J. Nelson et al. , Neurophysiological dynamics of phrase-structure building during sentence processing . Proc Natl Acad Sci U S A 114 , E3669 – E3678 ( 2017 ). OpenUrl Abstract / FREE Full Text 24. E. Fedorenko , A. Nieto-Castanon , N. Kanwisher , Lexical and syntactic representations in the brain: an fMRI investigation with multi-voxel pattern analyses . Neuropsychologia 50 , 499 – 513 ( 2012 ). OpenUrl CrossRef PubMed Web of Science 25. ↵ C. Humphries , J. R. Binder , D. A. Medler , E. Liebenthal , Syntactic and semantic modulation of neural activity during auditory sentence comprehension . J Cogn Neurosci 18 , 665 – 679 ( 2006 ). OpenUrl CrossRef PubMed Web of Science 26. ↵ E. Murphy et al. , The spatiotemporal dynamics of semantic integration in the human brain . Nature Communications 14 , 6336 ( 2023 ). OpenUrl CrossRef PubMed 27. ↵ M. Jamali et al. , Semantic encoding during language comprehension at single-cell resolution . Nature 631 , 610 – 616 ( 2024 ). OpenUrl CrossRef PubMed 28. ↵ G. A. Castellucci , C. K. Kovach , M. A. Howard , 3rd . , J. D. W. Greenlee , M. A. Long , A speech planning network for interactive language use . Nature 602 , 117 – 122 ( 2022 ). OpenUrl CrossRef PubMed 29. ↵ S. Bögels , L. Magyari , S. C. Levinson , Neural signatures of response planning occur midway through an incoming question in conversation . Scientific reports 5 , 1 – 11 ( 2015 ). OpenUrl 30. ↵ J. Cai , A. E. Hadjinicolaou , A. C. Paulk , Z. M. Williams , S. S. Cash , Natural language processing models reveal neural dynamics of human conversation . bioRxiv , 2023.2003. 2010.531095 ( 2023 ). 31. ↵ T. Mikolov , Efficient estimation of word representations in vector space . arXiv preprint arxiv: 1301.3781 , ( 2013 ). 32. ↵ E. Fedorenko , A. A. Ivanova , T. I. Regev , The language network as a natural kind within the broader landscape of the human brain . Nature Reviews Neuroscience , 1 – 24 ( 2024 ). 33. ↵ C. Goodwin , J. Heritage , Conversation analysis . Annual review of anthropology 19 , 283 – 307 ( 1990 ). OpenUrl CrossRef Web of Science 34. ↵ L. J. Silbert , C. J. Honey , E. Simony , D. Poeppel , U. Hasson , Coupled neural systems underlie the production and comprehension of naturalistic narrative speech . Proc Natl Acad Sci U S A 111 , E4687 – 4696 ( 2014 ). OpenUrl Abstract / FREE Full Text 35. S. Heim , A. D. Friederici , Phonological processing in language production: time course of brain activity . Neuroreport 14 , 2031 – 2033 ( 2003 ). OpenUrl CrossRef PubMed Web of Science 36. G. Jang et al. , Everyday conversation requires cognitive inference: neural bases of comprehending implicated meanings in conversations . Neuroimage 81 , 61 – 72 ( 2013 ). OpenUrl CrossRef PubMed 37. W. Feng et al. , Effects of contextual relevance on pragmatic inference during conversation: An fMRI study . Brain Lang 171 , 52 – 61 ( 2017 ). OpenUrl CrossRef PubMed 38. ↵ M. Schrimpf et al. , Artificial neural networks accurately predict language processing in the brain . BioRxiv , 2020.2006. 2026.174482 ( 2020 ). 39. ↵ M. C. Bastiaansen , R. Oostenveld , O. Jensen , P. Hagoort , I see what you mean: theta power increases are involved in the retrieval of lexical semantic information . Brain and language 106 , 15 – 28 ( 2008 ). OpenUrl CrossRef PubMed 40. T. Proix et al. , Imagined speech can be decoded from low-and cross-frequency intracranial EEG features . Nature communications 13 , 48 ( 2022 ). OpenUrl CrossRef PubMed 41. ↵ J. M. Schneider , S. Poudel , A. D. Abel , M. J. Maguire , Age and vocabulary knowledge differentially influence the N400 and theta responses during semantic retrieval . Developmental Cognitive Neuroscience 61 , 101251 ( 2023 ). OpenUrl CrossRef PubMed 42. ↵ D. M. Ackermann , E. L. Foldes , N. Bhadra , K. L. Kilgore , Effect of bipolar cuff electrode design on block thresholds in high-frequency electrical neural conduction block . IEEE Transactions on Neural Systems and Rehabilitation Engineering 17 , 469 – 477 ( 2009 ). OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted February 11, 2025. Download PDF Supplementary Material Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Decoding semantics from natural speech using human intracranial EEG Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Decoding semantics from natural speech using human intracranial EEG Camille R. C. Pescatore , Haoyu Zhang , Alex E. Hadjinicolaou , Angelique C. Paulk , John D. Rolston , R. Mark Richardson , Ziv M. Williams , Jing Cai , Sydney S. Cash bioRxiv 2025.02.10.637051; doi: https://doi.org/10.1101/2025.02.10.637051 Share This Article: Copy Citation Tools Decoding semantics from natural speech using human intracranial EEG Camille R. C. Pescatore , Haoyu Zhang , Alex E. Hadjinicolaou , Angelique C. Paulk , John D. Rolston , R. Mark Richardson , Ziv M. Williams , Jing Cai , Sydney S. Cash bioRxiv 2025.02.10.637051; doi: https://doi.org/10.1101/2025.02.10.637051 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Neuroscience Subject Areas All Articles Animal Behavior and Cognition (7636) Biochemistry (17704) Bioengineering (13898) Bioinformatics (41967) Biophysics (21460) Cancer Biology (18599) Cell Biology (25525) Clinical Trials (138) Developmental Biology (13384) Ecology (19909) Epidemiology (2067) Evolutionary Biology (24326) Genetics (15613) Genomics (22512) Immunology (17740) Microbiology (40423) Molecular Biology (17191) Neuroscience (88645) Paleontology (667) Pathology (2835) Pharmacology and Toxicology (4825) Physiology (7646) Plant Biology (15158) Scientific Communication and Education (2046) Synthetic Biology (4302) Systems Biology (9825) Zoology (2271)