Decoding the neural stages from action and object recognition to mentalizing

Decoding the neural stages from action and object recognition to mentalizing | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Decoding the neural stages from action and object recognition to mentalizing View ORCID Profile Moritz F. Wurm , View ORCID Profile Seoyoung Lee doi: https://doi.org/10.1101/2025.11.25.690412 Moritz F. Wurm 1 CIMeC – Center for Mind/Brain Sciences, University of Trento , 36068 Rovereto, Italy Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Moritz F. Wurm For correspondence: moritz.wurm{at}unitn.it Seoyoung Lee 1 CIMeC – Center for Mind/Brain Sciences, University of Trento , 36068 Rovereto, Italy 2 Department of Psychology, University of Chicago , Chicago, IL 60637, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Seoyoung Lee Abstract Full Text Info/History Metrics Preview PDF Abstract Higher-level action interpretation, such as inferring underlying intentions and predicting future actions, requires the integration of conceptual action information (e.g. "opening") with semantic knowledge about persons and objects (e.g. "my friend Anna", "pizza box"). However, how the neural systems for action and object recognition and memory interact with each other to form the basis for inferring higher-level mental states remains unclear. Here we use fMRI-based crossmodal multiple regression representational similarity analysis in human female and male participants to elucidate the processing stages from basic action and object recognition to mentalizing. We show that inferring intentions from observed actions or written sentences involves a modality-general network of lateral and medial frontoparietal and temporal brain regions associated with conceptual action and object representation and mentalizing. The representational profiles in these regions are explained by models capturing different types of conceptual information, revealing distinct but partially overlapping networks for action, object, and mental state representation. There was no strict separation of networks for action, object, and mental state representations, arguing against a sequential bottom-up hierarchy from action and object understanding pathways to the mentalizing network. Rather, left-hemispheric regions, specifically ventrolateral prefrontal, inferior parietal and anterior lateral occipitotemporal cortex, showed strong representational overlap, pointing towards a core network for making meaning of action-object structures at a conceptual level. We argue that this core network represents a distributional semantic hub between classic networks for action and object understanding and the mentalizing network. Significance Statement How does the human brain integrate information from actions, e.g., "open a pizza box", to understand the actions’ underlying intentions? To do so, the brain needs to combine information from different neural networks—for action and object recognition—and pass them to the mentalizing network for inferring intentions, such as "satisfying hunger". We characterize the interplay of networks using fMRI-based crossmodal multivariate analyses and find that a left-lateralized core network in inferior frontal and parietal cortex and lateral occipitotemporal cortex represents all critical ingredients—conceptual action and object information as well as higher-level mental state representation simultaneously in an overlapping manner. This suggests that this core network is essential for semantic interpretation and functions as bridge between recognition pathways and the mentalizing system. Introduction Understanding others’ actions is central to mentalizing—the ability to infer hidden mental states such as desires, feelings, and intentions. In naturalistic settings, recognizing an action alone (e.g., opening ) is insufficient; it must be contextualized by integrating semantic information about the people and objects involved. For instance, recognizing someone opening a pizza box allows us to infer that she is hungry and wants to eat. How does the brain combine such diverse information to support these inferences? Neural systems for recognizing actions, objects, and for mentalizing have typically been studied separately. Action recognition involves the lateral occipitotemporal cortex (LOTC), anterior inferior parietal lobe (aIPL), and premotor cortex (PMC) ( Van Overwalle and Baetens, 2009 ; Caspers et al., 2010 ). Within this action observation network (AON), left anterior LOTC represents the meaning of actions at a conceptual level ( Lingnau and Downing, 2015 ; Wurm and Caramazza, 2022 ). Object recognition primarily involves the ventral "what" stream in ventral temporal cortex (VTC) (Goodale and Milner, 1992; Kravitz et al., 2013). While the ventral stream partially overlaps with LOTC, conceptual knowledge about object categories and properties is more prominent in medial temporal cortex ( Fairhall and Caramazza, 2013 ; Clarke and Tyler, 2014 ; Martin et al., 2018 ; Wurm and Caramazza, 2022 ). Conceptual object representations are also found in angular gyrus (AG), lateral and dorsomedial prefrontal cortex (latPFC, dmPFC), and precuneus (PC) ( Fairhall and Caramazza, 2013 ; Leonardelli and Fairhall, 2022 ). Mentalizing engages a network comprising the temporoparietal junction (TPJ), dorsal and ventral medial prefrontal cortex (dmPFC, vmPFC), PC, and anterior temporal cortex (ATC) ( Frith and Frith, 2006 ; Van Overwalle, 2009 ; Thornton and Mitchell, 2018 ). Notably, the mentalizing network (MN) overlaps with conceptual object representations in PC, dmPFC, and AG/TPJ, but not with the AON. In addition, the MN and the AON are differentially activated when people observe actions or read phrases describing them (e.g., “she is highlighting a textbook”), depending on whether the task involves thinking about why rather than how the action is performed ( Spunt and Lieberman, 2012 ). For mental state inference, action and object knowledge should be accessed at the conceptual level, regardless of whether the stimulus is visual or verbal. This raises the question of how conceptual action and object representations interact to activate associated representations of mental states in the MN ( Fig. 1A ). One possibility is an intermediate stage that integrates conceptual action and object knowledge, bridging recognition systems with the MN. A potential substrate for this integrational stage is lateral mid superior temporal cortex (lmSTC), which is sensitive to agent-recipient pairings in action sentences (e.g. "dog chases man" vs. "man chasing dog") ( Frankland and Greene, 2015 ). However, evidence of lmSTC integrating agent/recipient information with actions is lacking. Another candidate is left ventrolateral prefrontal cortex (vlPFC), implicated in semantic control and integration of actions and scenes ( Wurm and Schubotz, 2012 ) and object knowledge ( Badre and Wagner, 2007 ). AG, also associated with semantic processing ( Binder et al., 2009 ; Frankland and Greene, 2020a), might provide the entry point for mental state inference via adjacent TPJ (Carter and Huettel, 2013). Alternatively, action and object information might be transmitted directly to the MN, without a distinct integrational stage encoding a "neutral" combination of actions and objects. In that case, potential entry points into the MN include ATC, left vlPFC, and left TPJ. Download figure Open in new tab Figure 1. (A) Model for the integration of action and object recognition pathways and the mentalizing network. (B) Experimental design. In two separate fMRI sessions, participants observed videos of actions and corresponding sentences while performing a mentalizing task that requires participants to think about why the action is carried out. (C) Representational dissimilarity models (RDMs) used for multiple regression RSA. The present study examined the pathway from action and object recognition to higher-level mental state inference. Using fMRI-based crossmodal multiple regression RSA ( Wurm and Caramazza, 2019 ), we localized modality-general representations of actions, objects, mental states, and a putative integrational stage. We found that left inferior frontal and parietal cortex and LOTC represent conceptual action and object information as well as higher-level mental state representation in an overlapping manner. These regions may form a network essential for semantic interpretation, serving as a bridge between recognition pathways and the mentalizing system. Results To investigate the processing pathway from basic action and object recognition to conceptual action and object representation and higher-level mental state representations, we used a crossmodal fMRI paradigm in combination with a mentalizing task that required participants to access all these stages of processing. In two separate fMRI sessions, participants observed videos of actions and corresponding action sentences (e.g. she is opening a pizza box; Fig 1B ) while thinking about why the action is carried out (e.g. because she is hungry and wants to eat; see Supplementary Material for behavioral results). Importantly, actions were crossed with objects, to allow the construction of independent similarity models for action and object information, an interaction model that treats each action-object combination as a distinct representation, and a mental states model that captures why the action is carried out ( Fig. 1C ). A reaction times model was included to regress out more general effects of cognitive load associated with the mentalizing task. Modality-general representations during action-related mentalizing In previous studies, modality-general action representations were found in left anterior LOTC only ( Wurm and Caramazza, 2019 ). The lack of evidence for the involvement of regions in the MN might suggest that representations of associated mental states, such as intentions, were not automatically activated. This may be due to the experimental task, which required action understanding but not higher-level functions such as mentalizing. Hence, we first tested whether action videos and sentences could be decoded beyond the AON, specifically in regions associated with object recognition and memory and mentalizing, when participants engaged in reasoning about the actions’ underlying mental states. We decoded the nine actions both within modality (videos, sentences) and across modality—that is, we trained a classifier to discriminate the activation patterns associated with the observed actions and tested the classifier on its ability to discriminate the activation patterns associated with the corresponding action sentences. Both the decoding of action videos and action sentences as well as decoding across action videos and sentences ( Fig. 2 , Tab. S2) revealed above chance decoding accuracy not only in brain regions associated with conceptual action representation (left anterior LOTC) ( Wurm and Caramazza, 2019 ), but also in regions associated with conceptual object representation (left posterior middle/inferior temporal gyrus (pMTG/ITG), VTC, latPFC, dMPFC, PC) ( Fairhall and Caramazza, 2013 ; Leonardelli and Fairhall, 2022 ) and mentalizing (dmPFC, TPJ, ATC, PC, vlPFC) ( Spunt and Lieberman, 2012 ). Additional above-chance decoding was observed in bilateral aIPL. These results show that not only the left anterior LOTC but a much larger network of brain regions represents modality-general information if the task requires higher-level action interpretation beyond action recognition—that is, mental state inference. Download figure Open in new tab Figure 2. Crossmodal (A), within-video (B), and within-sentence (C) decoding of the nine action-object combinations (decoding accuracy at chance = 11.11%). Mean accuracy maps were thresholded using Monte Carlo correction for multiple comparisons (voxel threshold p=0.001, corrected cluster threshold p=0.05). Crossmodal representation of actions, objects, action-object integration, and mental states Principally, the crossmodal decoding in the obtained regions may be driven by various levels of conceptual representation: actions, objects, the integration of actions and objects, or mental states. To characterize the level of representation with regard to these four stages of processing, we carried out a crossmodal multiple regression RSA: We first extracted from different ROIs the pairwise classifications of the crossmodal decoding described above and converted them into representational dissimilarity matrices (RDMs; see Methods for details). We tested three sets of ROIs associated with (1) modality-general action representation ( Wurm and Caramazza, 2019 ), (2) modality-general object representation ( Fairhall and Caramazza, 2013 ), and (3) mentalizing during action observation / sentence reading ( Spunt and Lieberman, 2012 ). These ROIs also covered candidate regions for the hypothesized integrational stage (left AG, left vlPFC, and left ATC2; see Fig. 3 , top right panel). The RDMs extracted from the ROIs were entered into a multiple regression with the four models of interest and the control model for reaction times ( Fig. 1C ). This allowed testing of whether the model RDMs explain the representational variance in the neural RDMs over and above each other. Download figure Open in new tab Figure 3. Crossmodal multiple regression RSA ROI analysis. ROIs were based on peak coordinates (upper right panel) for conceptual representation of actions (blue), objects (orange), and mental states (pink; see Methods for details). Hypothesized regions for integrational representations are marked in green. All action ROIs reveal modality-general action-specific representations, all object ROIs reveal modality-general object-specific representations, and seven of ten mental states ROIs reveal modality-general mental state representations. No ROI showed evidence for an integrational stage of processing. Asterisks indicate FDR-corrected significant decoding accuracies above chance (* p<0.05, ** p<0.01, *** p<0.001, **** p<0.0001). Error bars indicate SEM. We found that each of the three sets of ROIs exhibited the corresponding type of representation: In all action ROIs, the representational content could be explained by the action model; in all object ROIs, the representational content could be explained by the object model; and in seven of the ten mentalizing ROIs (except left and right ATC and vmPFC), the representational content could be explained by the mentalizing model ( Fig. 3 ). The integration model could not explain the representational content in any of the ROIs. The Reaction Times model explained variance in left AG, left dmPFC, PC, and left and right ATC—all regions belonging to the MN. A repeated-measures ANOVA indicated that the representational content differed between ROIs (Interaction ROI x MODEL: F(76,1672)=1.55, p=0.002). Interestingly, most ROIs were representationally heterogenous, that is, the representational variance in the ROIs could be explained by two or three of the tested models. This representational heterogeneity can be described by the following pattern: The action ROIs—left pMTG and pSTS—were not only sensitive to conceptual action information but also to object information in the videos and sentences. Similarly, all ROIs sensitive to mental states (except dmPFC3) were also sensitive to object information. Thus, almost all ROIs sensitive to objects were also sensitive to either actions or mental states, or both. Only VTC and vmPFC showed significant effects for the object model only. In left pSTS, left pMTG/IFG, and left dmPFC, all three types of representation—actions, objects, and mental states—were found. We also repeated the ROI analysis using ROIs based on cluster peak coordinates from the crossmodal decoding map ( Fig. S1A ), which revealed largely converging results and additionally found left aIPL to represent actions, objects, and mental states ( Fig. S1B ). To test whether there were additional regions sensitive to modality-general action, object, integration, and mental state representations, we conducted a searchlight analysis ( Fig. 4 ). This analysis revealed results that were consistent with the ROI analysis. First, modality-general action representations were found in left pMTG/pSTS as well as in bilateral aIPL––another key region of the AON––and in left IFG. Second, modality-general object representations were found in all regions previously reported for conceptual object representation ( Fairhall and Caramazza, 2013 ; Leonardelli and Fairhall, 2022 ), as well as in the left anterior vlPFC and bilateral aIPL. Third, modality-general mental state representations were found in left vlPFC, left premotor cortex, posterior TPJ, and pSTS extending into middle/inferior temporal cortex. The integration model did not reveal clusters surviving multiple comparison correction. The Reaction Times model revealed clusters in left AG and PC, overlapping with the object model. Again, these results demonstrate partially overlapping representations: First, the strongest overlap––between actions, objects, and mental states—was observed in left LOTC. Second, actions overlapped with objects (but not with mental states) in bilateral aIPL. Third, action clusters overlapped with mental states (but not with objects) in left vlPFC. Download figure Open in new tab Figure 4. Crossmodal multiple regression searchlight RSA. Maps for action, object, and mental state models revealed overlapping effects in left anterior vlPFC and left anterior MTG and ITG. In addition, overlapping effects for the action and object models were found in left LOTC and bilateral aIPL. No significant effects were found for the integration model. The Reaction Times model revealed clusters in left AG and PC. T -maps were thresholded using Monte Carlo correction for multiple comparisons (voxel threshold p=0.001, corrected cluster threshold p=0.05). Interestingly, the mentalizing cluster in left pSTS/temporal cortex was anterior of the action and object clusters in LOTC, pointing towards a posterior-anterior gradient from action and object representations to a mentalizing-related stage of processing. To characterize this gradient in more detail and show how the representational content changes from posterior occipital to anterior temporal cortex, we performed a path-of-ROI analysis. This was done by plotting the beta values of the five models as a function of position along a vector from posterior middle occipital to anterior middle temporal cortex ( Fig. 5 ). This analysis revealed a significant change from representational content (ANOVA interaction ROI x MODEL: F(156,3432) = 4.15, p < 0.0001). Specifically, conceptual object representations peaked at the level of posterior LOTC, followed by conceptual action representations peaking in anterior LOTC. The peak of mental state representations was located even more anteriorly in the posterior/mid temporal cortex. Finally, we observed a peak for the Reaction Times model slightly anterior of the mental states peak. In line with a recent proposal for an action recognition pathway in left LOTC ( Wurm and Caramazza, 2022 ), these results demonstrate a partially overlapping progression from object to action to mental state representation from lateral occipital to temporal cortex. Download figure Open in new tab Figure 5. Path-of-ROI analysis. Mean beta values from the crossmodal multiple regression searchlight RSA were plotted as a function of position along the posterior-anterior axis (MNI y-coordinates). The x-axis color bar corresponds to the color coding of the path-of-ROI shown in the brain inset. Informational connectivity between action, object, and mental state regions The multiple regression RSA revealed a complex pattern of partially overlapping modality-general action, object, and mental state representations in the action, object, and mentalizing networks. How do the brain regions of these networks exchange information and thereby transform their representational content? To investigate how the representations in the tested brain regions are related to each other, we conducted an informational connectivity analysis ( Coutanche and Thompson-Schill, 2013 ): We correlated the crossmodal RDMs of the action, object, and mentalizing ROIs with each other, which reflects how distant the representational contents in the different regions are to each other ( Fig. 6A ). Then, we used multidimensional scaling of the ROI-to-ROI similarities to visualize the representational distances between ROIs ( Fig. 6B ). In line with the multiple regression RSA, we found that the representations in the action, object, and mentalizing networks partially overlap with each other rather than forming clearly separable networks. This is informative because it allows us to assess where in the brain the networks representationally overlap and where they do not, which might indicate the flow of information between these networks during mental state inference. We observed that all left lateral regions formed a prominent cluster with high representational similarity, distinct from the medial ROIs (PC, dmPFC, vmPFC) and right ATC, which were representationally more diverse. The left-hemispheric cluster groups together nodes from all three networks, including three nodes from the MN: vlPFC, TPJ-pSTS, and ATC. This suggests that information from the action and object networks flows to the MN via these left-hemispheric regions and then onward to other nodes of the MN (i.e., to the midline regions PC, dmPFC, and vmPFC). Due to the high similarity within the left lateral cluster, it is difficult to assess further distinctions, specifically between vlPFC, TPJ-pSTS, and ATC––the three brain regions hypothesized to be likely entrance nodes to the MN. The high similarity between these three MN regions and the action and object nodes (VTC, pMTG, pSTS) suggests that these regions form a common network during action intention inference rather than gradual pathways from one network to the other as illustrated in Fig. 1A . A similar clustering was observed using ROIs based on the crossmodal decoding, demonstrating high similarity between left pMTG, vlFPC, PMC, aIPL, and left VTC as opposed to the representationally more distinct MN regions (left and right TPJ, PC, dmPFC) and right VTC ( Fig. S1C and D ). Download figure Open in new tab Figure 6. Crossmodal informational connectivity analysis. (A) Correlation matrix from pairwise correlations between group-averaged neural RDMs extracted from the crossmodal decoding. (B) Multidimensional scaling (MDS) of the representational distances between ROIs. Dot colors indicate associated networks for conceptual representation of actions (blue), objects (orange), and mental states (pink). Notably, the high similarity between the three MN nodes (vlPFC, TPJ-pSTS, and ATC) and action and object nodes (pMTG, pSTS, and VTC) is unlikely due to anatomical proximity as the regions are relatively distant. To test this more directly, we computed pairwise Euclidean distances between the ROIs using their MNI coordinates and displayed the distances using multidimensional scaling ( Fig. S2A ). No clustering of the aforementioned regions was observed. We also correlated the Euclidean distance matrix with the correlation matrix shown in Fig. 6A . The matrices were correlated with each other (r(188)=0.49, p<0.0001), but this effect is likely driven by some highly adjacent ROIs (e.g. VTC1-3, TPJ-pSTS and AG, PC and PC-PCC). Indeed, when we repeated the analysis using decoding-based ROIs ( Fig. S2B ), no correlation between distance and correlation matrices was observed (r(64)=0.09, p=0.48). Discussion Understanding the hidden motives of others’ actions requires not only recognizing the action as such (e.g. open ) but also integrating it with conceptual object knowledge (e.g. people have different motives to open a present or a pizza box). This study aimed to characterize brain regions that represent the stages of conceptual processing during mental state inference from actions. We revealed three main findings: (1) When people infer mental states from observed actions or written sentences, conceptual representations are found in a largely left-lateralized network of frontal, parietal, and temporal regions that are typically associated with action and object semantics, mentalizing ( Spunt and Lieberman, 2012 ; Fairhall and Caramazza, 2013 ; Wurm and Caramazza, 2019 ), and semantic processing more generally ( Binder et al., 2009 ; Jackson, 2021 ). (2) The representational profiles in these regions can be explained by models capturing different types of conceptual information, revealing distinct but partially overlapping networks for action and object representation (e.g. open and pizza box , respectively) and mentalizing (e.g. satisfy hunger ). We found no evidence for an integrational stage, and there was no strict segregation of networks for action, object, and mental state representations, arguing against a sequential bottom-up hierarchy from action and object recognition pathways to the mentalizing network. Rather, left lateral frontal, parietal, and occipitotemporal regions showed strong representational overlap, suggesting that they form a core network for deriving meaning from conceptual action-object structures. (3) Mental state information was found not only in classic MN regions but most prominently in this core network (particularly in left vlPFC, PMC, and anterior LOTC). In LOTC, information is organized along a posterior-anterior gradient from objects to actions to mental states. This might indicate that conceptual action and object information is already transformed to higher-order intention-related information in left LOTC, in interplay with left frontal and parietal regions. How the neural systems for action recognition and mentalizing relate to and interact with each other has been studied intensively ( de Lange et al., 2008 ; Van Overwalle and Baetens, 2009 ; Spunt et al., 2010 ; Becchio et al., 2012 ; Catmur, 2015 ). fMRI studies contrasting task demands (e.g. "why" vs. "what" questions) ( Spunt et al., 2010 ; Spunt and Lieberman, 2012 ) and stimulus conditions (e.g. typically vs. atypically performed actions) ( de Lange et al., 2008 ) typically reveal activation of either the AON or the MN, which can create the impression that these systems are usually complementarily active. However, the hierarchical organization of actions and their underlying intentions presupposes that understanding intentions first builds on action recognition ( Kilner et al., 2007 ; Catmur, 2014 ), rather than either process being active independently. Our decoding analysis shows that indeed, both the AON and MN are simultaneously active and represent specific modality-general information about the actions if the task requires participants to infer mental states from the actions. By contrast, for tasks that require only action recognition but not mental state inference, decoding of modality-general action representations is largely restricted to left LOTC ( Wurm and Caramazza, 2019 ; Karakose-Akbiyik et al., 2023 ). Strikingly, representational content overlaps in several AON and MN nodes: Left anterior LOTC (pSTS and MTG), left PMC, and left IPL, which are regions typically associated with action recognition, also represent mental state information, and the MN regions—left TPJ and precuneus—represent action information. In addition, almost all MN regions (except right TPJ) represent conceptual object information, which is in line with previous findings ( Fairhall and Caramazza, 2013 ; Leonardelli and Fairhall, 2022 ). This complex picture of representational overlap is not straightforward to interpret, but one interim conclusion is that a clear distinction between the AON and the MN (and large parts of the semantic object network) is not supported by the present findings. While the RSA revealed that the mental states model could successfully capture information in seven out of ten MN regions (and in five out of six decoding-based MN ROIs, except right TPJ), the searchlight RSA showed the strongest effects in regions that differ from the classical MN—namely in left vlPFC/PMC and left anterior LOTC (comprising pSTS, MTG, and ITG), which overlap closely with the semantic control network ( Jackson, 2021 ). This discrepancy might suggest that inferring mental states from actions does not primarily rely on the classical MN, which is often localized using theory-of-mind tasks, such as understanding false belief vs. physical stories ( Saxe and Kanwisher, 2003 ; Van Overwalle, 2009 ; Van Overwalle and Baetens, 2009 ; Spunt and Adolphs, 2014 ). Indeed, MN regions that showed no effects in the present study, such as right TPJ and vmPFC, are associated with differentiating one’s own beliefs from others’, social decision making, and moral reasoning ( Van Overwalle, 2009 )—functions that may be less involved in inferring intentions from actions. By contrast, left vlPFC/PMC and anterior LOTC might be involved in retrieving semantic information about actions and objects to infer associated intentions, a function that is usually less required in classic theory-of-mind tasks. Notably, there is often no clear-cut distinction between intentions, understood as abstract, hidden motives for carrying out an action (e.g. satisfying hunger ), and more concrete, potentially observable goals (e.g. wanting to eat pizza ) ( Csibra and Gergely, 2007 ; Uithol et al., 2011 ; Thompson et al., 2019 ). Accordingly, intention inference may involve the prediction of possible future actions (e.g. that opening pizza box is followed by eating ) ( Csibra and Gergely, 2007 ). It is therefore possible that the representational content captured by the mentalizing model reflects processes important for inferring/predicting future actions. In line with this interpretation, watching videos or reading sentences of predictable vs. unpredictable actions is associated with left vlPFC and MTG ( Wurm et al., 2014 ; Belluzzi and Fairhall, 2025 ). Thus, we argue that the identified left-lateralized lateral "core network" consisting of anterior LOTC, vlPFC/PMC, and aIPL is critically involved in inferring possible future actions, as opposed to the MN regions medial cortex (dmPFC, vmPFC, and PC) and TPJ, which might be more important for theory-of-mind related processes such as belief representation and perspective taking ( Schurz et al., 2014 ; Jamali et al., 2021 ). This interpretation is supported by the information connectivity analysis, which revealed a representational clustering of left LOTC regions (pSTS, MTG, ITG), left vlPFC, and left aIPL that were representationally distinct from medial MN regions and right TPJ. The left lateral network representing action, object, and mental state information comprised not only LOTC and vlPFC, but also aIPL. The aIPL is usually associated with action recognition ( Van Overwalle and Baetens, 2009 ), action goal representation ( Hamilton and Grafton, 2006 ; Liu et al., 2024 ), and functional knowledge about tools and other manipulable objects ( Goldenberg and Spatt, 2009 ; Leshinskaya and Caramazza, 2015 ). aIPL has also been found in conjunction with left vlPFC and anterior LOTC for predicting action sentences ( Belluzzi and Fairhall, 2025 ). Thus, aIPL might play a role in the semantic retrieval of action/function knowledge associated with an object (e.g. bottle is not only associated with the observed opening action, but also with, e.g., drinking or pouring ), which is critical for inferring mental states from object-directed actions. Inferring mental states from complex object-directed actions requires the integration of action and object knowledge at the conceptual level. Thus, we used a conservative crossmodal decoding approach to isolate action, object, and mental state information at a strictly conceptual level. However, this approach required using a relatively small set of nine action-object combinations that crossed three actions with three objects. This limited the heterogeneity and complexity of the actions’ underlying mental states. Larger stimulus sets may allow for a better isolation and characterization of mental state representations. For example, recent work showed that mental state information in the MN is organized along rationality, valence, and social impact ( Tamir et al., 2016 )—dimensions that were not substantially varied in the present study. Moreover, inferring an action’s underlying mental states may involve not only the integration of object knowledge but also of knowledge about involved persons, such as age, status, and personality traits ( Tamir and Thornton, 2018 ). For example, we likely infer different intentions from opening a bottle of cleaning detergent if the action is carried out by an adult or a toddler. How person knowledge (and other contextual information) is integrated with action and object information to interpret an action scene in its full complexity is an exciting question for future research. In conclusion, the processing from action and object understanding to inferred mental states relies on a left lateralized network of frontal, inferior parietal, and occipitotemporal regions. Each node in this network does not encode unitary information but instead represents multiple types of information simultaneously. This suggests that the nodes largely work in parallel rather than following a strict hierarchical processing from action and object information to inferred mental states. This invites a reconsideration of the strict segregation of recognition pathways (most importantly the AON) and the MN in the context of inferring mental state from actions. Methods Participants 27 native Italian speakers (15 females; mean age, 24.2 years; age range, 19-41 years) participated in the fMRI experiment. All participants were right-handed and had normal or corrected-to-normal vision. Four participants were excluded due to bad performance in the task (number of responses during regular trials were less than 2 standard deviations from the mean). The final sample size was therefore N = 23. The study was approved by the Ethics Committee for research involving human participants at the University of Trento, Italy. Stimuli Nine action-object combinations were generated by crossing three actions (open, close, squash) with three objects (pizza box, present, water bottle). For these action-object combinations, video and sentence stimuli were created ( Fig. 1B ). For the videos, we filmed each action-object combination performed by two actors (female, male), from two different perspectives (front, side), and using two object exemplars (large, small). In addition, videos were left-right flipped resulting in a total of 16 exemplars per action-object combination (144 action videos in total). Different exemplars for each action-object combination were created to increase the perceptual variance of the stimuli. All 144 videos were edited using iMovie (Apple) and Matlab (Mathworks). The final versions had a length of 2 s, were in gray scale, and a resolution of 400 x 712 pixels. The sentence stimuli were written descriptions of the action-object combinations. All sentences followed a subject-verb-object structure. To create nine action-object combinations, the verbs close, open, and squash were crossed with the objects water bottle, pizza box, and present box. To increase stimulus variance, the verb-object pairs were crossed with six subjects: she, he, the girl, the boy, the woman, the man. This created six exemplars per action-object combination (54 sentences in total). Each sentence was presented on a light gray background (400 x 712 pixels) in three consecutive chunks (subject, verb, object), with each chunk shown for 666 ms (2 s per sentence). Different font types (Comic Sans MS, Verdana, MV Boli, Times New Roman, Calibri Light, MS UI Gothic, Arial) and font sizes (32-42) were used to increase the perceptual variance of the sentence stimuli. Additionally, we created question trials that consisted of questions targeting the possible mental states that could be attributed to the action portrayed in the videos and sentences. These question trials were presented interleaved with the experimental trials (14% of all trials). Each action-object combination was associated with two questions: one designed to elicit an ’agree’ response while the other a ’disagree’ response (e.g. for the video/sentence portraying ’she opens a pizza box’, we expected participants to agree with the question ’does she want to satisfy hunger?’, but disagree with the question ’does she want to be healthy?’). This was to ensure that the participants were paying attention and constantly thinking about the mental states that could be associated with the actions. For each action-object combination and agree/disagree question trial, there were two variants: One focused on intentions (e.g. ’does she want to satisfy hunger?’), whereas the other focused on feelings (’is she hungry?’). Each question was presented on a light gray background (400 x 712 pixels) in one chunk for 2 s using the same font type (Arial) and font size (40). In the scanner, the stimuli were back-projected onto the center of an acrylic screen via a projector (EPSON EMP-7900) and viewed through a mirror on the head coil (video and sentence presentation 7.5° x 2.4° visual angle). Stimulus presentation, response collection, and synchronization with the scanner were controlled by Matlab with Psychtoolbox extensions ( Brainard, 1997 ). Experimental Design All participants took part in two sessions: a video session and a sentence session. For both sessions, stimuli were presented in a mixed event-related design. Each 2 s video/sentence trial was followed by a 3 s fixation period, which displayed ‘why?’ on the screen. Participants were instructed to think about the hidden motives of the action, similar to the task by Spunt and Lieberman (2012) , which revealed activation of the MN when the ‘why’ task was contrasted with the ’how’ task. However, to further motivate the participants to engage in this task, they were asked to press the button on the response box with their right index finger when they had a potential mental state in mind regarding why someone is performing a certain action (e.g. the woman is opening a water bottle because she is thirsty). Participants could respond during the video/sentence or during the ’why?’ fixation period after the video/sentence. During the fixation period, a fixation cross was displayed on the screen for the remainder of the trial, which disappeared as soon as the button was pressed. Question trials were presented on the screen for 5 s after the presentation of a video/sentence trial. Participants were asked to press one of 4 buttons—from strongly agree to strongly disagree—on the response box. Specifically, they were asked to use the right index finger if they strongly agree with the mental state that was being attributed to the immediate prior action video/sentence, the right middle finger if they partially agree, the right ring finger if they partially disagree, and the right little finger if they strongly disagree. To ensure participants understood the task, a short practice run was completed prior to entering the scanner. Trials were presented in blocks of eighteen video/sentence trials, with each action-object combination being shown twice per block and three question trials (21 trials per block in total). Three blocks were presented per run, separated by 10 s fixation periods. Each run began with a 2 s fixation period after five dummy volumes and ended with a 16 s fixation period. Video and sentence runs were presented in separate sessions, with the video session consisting of four functional scans and the sentence session consisting of five functional scans. There were more functional runs for sentences than videos to increase the power for the sentence stimuli, since previous crossmodal decoding studies found that decoding tends to be weaker for sentences compared to videos ( Wurm & Caramazza, 2019 ). The order of the sessions was counterbalanced across participants. Data Acquisition Functional and structural data were collected at the Centre for Mind/Brain Sciences at the University of Trento using a Siemens Prisma 3T MR scanner with a 64-channel phased array head coil. The structural scan was collected between the video and the sentence session. Functional images were acquired using a T2*-weighted gradient echo-planar imaging (EPI) sequence with fat suppression (235 images per functional run; repetition time (TR) of 1500 ms; echo time (TE) of 28 ms; flip angle (FA) of 70°; field of view (FOV) of 200 mm; matrix size of 66 × 66; voxel resolution of 3 × 3 × 3 mm; and 45 transverse slices with a slice thickness of 3 mm in ascending interleaved order). Ten radiofrequency pulses at the beginning of each functional run were sent to avoid T1 saturation. Structural T1-weighted images were obtained using an MPRAGE sequence (176 sagittal slices, TR = 2530 ms, inversion time = 1100 ms, FA = 7°, FOV = 256 × 256 mm, 1 × 1 × 1 mm voxel resolution). Preprocessing Data were preprocessed using BrainVoyager QX 2.8 (BrainInnovation) in combination with the NeuroElf Toolbox, and custom Matlab code. The first volume of the first functional run was aligned to the high-resolution anatomy (six parameters). Data were 3D motion corrected using trilinear interpolation with the first volume of the first run for each participant being used as reference. This was followed by slice time correction. Spatial smoothing was applied with a Gaussian kernel of 8 mm full width at half maximum (FWHM) for univariate analysis and 3 mm FWHM for MVPA. Multivariate pattern decoding Decoding analyses were carried out to identify the brain regions where action-object combinations could be discriminated from each other. For each participant, functional session, and run, a general linear model was computed using design matrices containing predictors for action-object combinations (9 combinations * 2 = 18; two predictors per action-object combination were created by using the first and second half of each run), question trials, the six parameters from 3D motion correction (x, y, z translation and rotation), and 6 temporal drift predictors for high-pass filtering. Each trial was modelled according to the onset and offset of the 2 s video/sentence and these models were then used as reference when fitting the signal time course of each voxel. This procedure created two beta maps per run for each of the nine action-object combinations. Action-object classification was performed for each participant in volume space using searchlight spheres with a radius of 12 mm and a linear discriminant analysis (LDA) classifier as implemented in the CoSMoMVPA toolbox ( Oosterhof et al., 2016 ). Each voxel was demeaned by subtracting the mean of all betas. Demeaning was executed to remove activations patterns that are common to all action-object combinations. All classification analyses (crossmodal, within-video, within-sentence) utilized a ’one-against-one’ multiclass decoding approach where each of the nine action-object combinations were discriminated from the remaining eight action-object combinations in a pairwise manner. In the crossmodal action-object classification, a classifier was trained to discriminate the action-object combinations using the data of the video session and its accuracy at discriminating the action-object combinations was tested using the data of the sentence session. This process was repeated conversely (e.g. training the classifier on sentence data and testing on video data) and the resulting accuracies from both classification directions were averaged. For the within-modality (within-video and within-sentence) action-object classification, a classifier was trained to discriminate the nine action-object combinations using a leave-one-beta-out cross-validation: Seven beta patterns from a total of eight per action-object combination for the within-video classification and nine beta patterns from a total of ten per action-object combination for the within-sentence, were used for training. The accuracy of the classifier at discriminating the action-object combinations was then tested using the held-out data. This procedure was repeated to exhaust all possible training and testing combinations (eight iterations for within-video and ten iterations for within-sentence). The resulting classification accuracies from all possible training and testing combinations were then averaged to produce one classification accuracy per searchlight sphere. For all decoding schemes, a one-tailed one sample t-test was performed on the accuracy maps to identify voxels in which the decoding accuracy is significantly above chance (11.11%). The resulting t-maps were corrected for multiple comparisons using Monte Carlo correction as implemented in CosmoMVPA ( Oosterhof et al., 2016 ), with an initial threshold of p=0.001 at the voxel level, 10000 Monte Carlo simulations, and a one-tailed corrected cluster threshold of p = 0.05 (z = 1.65). The maps were projected on a cortex-based surface for visualization. For the multiple regression RSA, the pairwise classifications of the nine action-object combinations were used to create a confusion matrix, which was then converted into a representational dissimilarity matrix (RDM; see below). Representational Dissimilarity Models Action, object, and integration representational dissimilarity models (RDMs) were created by comparing action-object combinations in a pairwise manner and assigning 1 or 0, with 1 representing similarity and 0 representing dissimilarity. For the action model, action-object combinations were similar if they contain the same action irrespective of the object involved. For the object model, action-object combinations were similar if they contained the same object irrespective of the action involved. For the integration model, action-object combinations were similar if they contained both the same action and object (or, in other words, the interaction between action and object model; Fig. 1C ). To generate a mental states model that reflects the similarity of mental states associated with the nine action-object combinations, an independent group of participants (N = 13) took part in a behavioral ratings experiment. In this experiment, sixteen different questions that reflect the potential mental states of the nine action-object combinations were asked for each action-object combination ( Supplementary Table 1 ). The questions targeted the possible intentions and feelings associated with the action-object combinations ( Tamir et al., 2016 ). Participants had to respond to every question using a 6-point Likert scale. The mental states model was constructed by computing pairwise correlations between group-averaged responses to the 16 ratings questions for each of the nine action-object combinations. The resulting 9 × 9 matrix was then subtracted from one to create a dissimilarity matrix ( Fig. 1C ). Finally, we included a model for reaction times in the multiple regression RSA. This model aimed at controlling for potential differences in cognitive load to infer mental states to the actions. Errors and RTs of correct responses to action trials were averaged across participants. To construct the model, we first computed the mean reaction times across participants for the nine action-object combinations separately for the video and sentence session. The mean reaction times were then z-scored for the two sessions to eliminate session-related differences between video and sentence reaction times. For the crossmodal RSA, we computed the pairwise Euclidean distances between the nine reaction times of the sentence session and the nine accuracies of the video session. The model thus reflects how similar the nine action-object combinations are in terms of speed to press a button (broadly reflecting the cognitive load to infer associated mental states) across the two sessions. For the within-session RSA, we computed the pairwise Euclidean distances between the reaction times of the nine action-object combinations for each session separately. The neural RDMs were created by extracting the pairwise classifications (the classification matrix) from each ROI or searchlight sphere of each participant’s crossmodal decoding map, symmetrizing the matrix across the main diagonal, and converting the classifications into dissimilarity measures by subtracting 100 – accuracy. To reduce noise in the neural RDMs, we spatially smoothed them by computing the mean RDM of each voxel and of the surrounding voxels within a radius of three voxels. RSA usually uses the pairwise similarities of the neural and model RDMs (the off-diagonal values of the RDM), whereas the similarity of each condition with itself (the on-diagonal values of the RDM) is not included. However, for the interaction model, which distinguishes each action-object combination from each other, all pairwise comparisons have the same value (0=dissimilar). Thus, the interaction model would be singular and produce meaningless results in the RSA. A solution to this problem is to include the on-diagonal values of model and neural RDMs in the RSA ( Walther et al., 2016 ). Importantly, because we used cross-validation for the multiple regression RSA, the on-diagonal values have interpretable zero-points and can therefore be included in the RSA without biasing ( Walther et al., 2016 ). To test for potential collinearity between the models, we computed the condition indices (CI), variance decomposition proportions (VDP), and variance inflation factors (VIFs) using the colldiag function in Matlab. The results of these tests (all CI < 4, all VDP < 0.77, all VIF < 2.9) did not indicate potential estimation problems. Multiple regression RSA To characterize the representational content isolated by the crossmodal decoding, an ROI-based multiple regression RSA using the crossmodal classification accuracies was performed. Twenty ROIs ( Supplementary Table 1 ) were defined based on three fMRI studies that aimed at identifying modality-general representations of actions (left pMTG, left pSTS) ( Wurm and Caramazza, 2019 ), objects (left VTC1, left VTC2, left VTC3, left pMTG-ITG, left AG, left latPFC, left dmPFC, PC) ( Fairhall and Caramazza, 2013 ), and action intentions (left dmPFC1, left dmPFC2, left dmPFC3, left ATC1, left ATC2, left vlPFC, PC-PCC, left vmPFC, TPJ-pSTS, right ATC) ( Spunt and Lieberman, 2012 ). It is less straightforward to define ROIs for the integrational stage. However, the aforementioned ROIs also included the hypothesized regions for action-object integration, i.e., left ATC2 (anatomical distance to lmSTC in Frankland & Greene, 2015 : 12.8 mm), left vlPFC, and left AG. Therefore, we did not define separate ROIs for the integrational stage. All pairwise classifications of the action-object combinations were extracted to construct neural RDMs for each voxel instead of a single classification accuracy for each voxel. Thus, for each ROI voxel, a 9 × 9 confusion matrix was created, averaged across voxels, made symmetrical across the main diagonal, and transformed into model RDMs by subtracting the classification rate (i.e. the number of predictions for an action-object combination divided by the number of predictions for all action-object combinations) from one. This resulted in one RDM per ROI and participant. The off-diagonals (i.e. the lower triangle of the matrix) and the on-diagonals of the models were vectorized, z-scored, and entered as dependent variables into a multiple regression. The correlation coefficients were Fisher transformed and entered into one-tailed one-sample t-tests, which were False Discovery Rate (FDR) corrected for the number of ROIs included in the regression. Searchlight multiple regression RSA A similar procedure as the ROI-based multiple regression RSA was conducted for the crossmodal searchlight decoding multiple regression RSA. Instead of ROIs, the pairwise classification of the crossmodal action-object decoding was extracted from each voxel in the searchlight sphere, resulting in 9 × 9 confusion matrix for each voxel in the sphere-of-interest and one RDM per searchlight sphere and participant. The RSA procedure described was done for each voxel and the resulting beta values were assigned to the center voxel of each searchlight sphere. The searchlight multiple regression RSA was restricted to voxels that yielded significant above chance accuracies in the decoding analysis, which indicates reliably discriminable information of the action-object combinations ( Ritchie et al., 2017 ). Resulting beta maps were entered into one-tailed one-sample t-tests and corrected for multiple comparisons using Monte Carlo Cluster (MCC) correction as described in the Multivariate pattern decoding subsection. For completeness, we also carried out within-modality (within-video and within-sentence) multiple regression searchlight RSA that were based on the neural RDMs derived from the within-video and within-sentence decoding (see Supplementary Figures S3 and S4 ). However, because the within-video and within-sentence decoding were not specifically sensitive to modality-general (conceptual) information, they are not well suited to detect the conceptual action, object, integration, and mentalizing stages targeted in this study. The decoding generally exploits the information that most reliably discriminates the nine action-object combinations, which is not necessarily conceptual information. In particular, for the within-video decoding, the nine action-object combinations are discriminable based on various perceptual and kinematic dissimilarities. Thus, the integration model is expected to pick up the specific perceptual and kinematic differences between the nine action-object combinations. By contrast, the crossmodal RDMs reflect the dissimilarities of the action-object combinations at a modality-general level only. Therefore, the crossmodal and within-modality RDMs are likely to reflect different kinds of information, which limits the interpretability of the within-modality multiple regression RSA, in particular for the integration model, and the comparability of the crossmodal and the within-modality multiple regression RSA. Another limitation is that the within-video and within-sentence decoding are each based on 50% of the data, which reduces the power of the within-modality multiple regression RSA. Path-of-ROI analysis To investigate the change of representational content along the posterior-anterior axis from occipital to anterior temporal cortex, we used a path-of-ROI analysis ( Konkle and Caramazza, 2013 ; Wurm et al., 2017 ; Wurm and Caramazza, 2019 ). Anchor points were based on the posterior and anterior points of occipital and temporal poles, respectively, which were then connected with a vector on the flattened surface along the middle temporal gyrus. Along this vector, a series of 40 partially overlapping ROIs (12 mm radius, centers spaced 3 mm) was defined. From each ROI, beta values were extracted from the searchlight maps of the crossmodal multiple regression RSA, averaged across voxels, and plotted as a function of position on the posterior-anterior axis. Informational connectivity analysis To test how similar the representational content of the ROIs was to each other, we used informational connectivity analysis ( Coutanche and Thompson-Schill, 2013 ). First, we extracted the neural RDMs of the ROIs from the crossmodal decoding map and correlated them with each other. We only used the lower triangle of the RDMs (off-diagonal pairwise distances between actions) because the on-diagonal correct classifications would confound the informational connectivity analysis with the general strength of decoding in the ROIs (i.e., regions with high crossmodal decoding accuracies show stronger similarity with each other than with regions with lower decoding accuracy). We then converted the resulting correlations between ROIs into distances by subtracting 1 – r, and visualized the pairwise distances between ROIs and decoding schemes using multidimensional scaling (metric stress). The authors declare no competing financial interests. Supplemental Material Behavioral Results For each trial of both action video and sentence fMRI sessions, participants were instructed to press a button as soon as they could think of a possible underlying intention of the action (maximally 5 seconds after video/sentence onset). Four of the 27 participants responded in less than 50% of the trials (and less than 2 standard deviations from the mean) and were therefore excluded from the sample. The remaining 23 participants showed high response rates for all nine action-object combinations of both video and sentence sessions: In the video session, participants responded to 93.2% of trials (0.8 SEM) with a mean reaction time of 2.62 seconds (0.021 SEM). In the sentence session, participants responded to 93.5% of trials (0.4 SEM) with a mean reaction time of 2.86 seconds (0.017 SEM). 14% of trials were followed by question trials (e.g., ’satisfy thirst?’) that either matched the previous action or not. Participants could respond from 1 = strongly agree to 4 = strongly disagree. To estimate the general accuracy of the participants’ responses, we collapsed responses 1 and 2 (generally agree) and responses 3 and 4 (generally disagree). In the video session, responses were correct in 83.3 % of trials (1.4 SEM); in the sentence session, responses were correct in 79.6% of trials (1.6 SEM). These results demonstrate that the participants sufficiently inferred mental states from both action videos and sentences. View this table: View inline View popup Download powerpoint Supplementary Table 1: ROI coordinates in MNI space for action ( Wurm and Caramazza, 2019 ), object ( Fairhall and Caramazza, 2013 ), and mentalizing networks ( Spunt and Lieberman, 2012 ) View this table: View inline View popup Download powerpoint Supplementary Table 2: Peak coordinates from main clusters of the crossmodal decoding in MNI space Download figure Open in new tab Supplementary Figure S1. Results for functionally defined ROIs. (A) Peak coordinates were extracted from the crossmodal decoding map ( Fig. 2 of the main article). (B) Crossmodal multiple regression RSA. Asterisks indicate FDR-corrected significant decoding accuracies above chance (* p<0.05, ** p<0.01, *** p<0.001, **** p<0.0001). Error bars indicate SEM. (C) Correlation matrix from pairwise correlations between group-averaged neural RDMs extracted from the crossmodal decoding. (D) Multidimensional scaling (MDS). Colors indicate associated networks for conceptual representation of actions (blue), objects (orange), and mental states (pink). Download figure Open in new tab Supplementary Figure S2. Multidimensional scaling based on Euclidean distance between ROIs. Euclidean distances between MNI coordinates of the literature-based ROIs (A) and decoding-based ROIs (B) were computed, and the resulting distance matrices were entered into multidimensional scaling as reported for the multidimensional scaling based on neural RDMs (see section Informational connectivity analysis in the Methods section of the main text). Colors indicate associated networks for conceptual representation of actions (blue), objects (orange), and mental states (pink). Download figure Open in new tab Supplementary Figure S3. Within-video multiple regression RSA. (A) ROI analysis. Asterisks indicate FDR-corrected significant decoding accuracies above chance (* p<0.05, ** p<0.01, *** p<0.001, **** p<0.0001). Error bars indicate SEM. (B) Searchlight RSA maps. T -maps were thresholded using Monte Carlo correction for multiple comparisons (voxel threshold p=0.001, corrected cluster threshold p=0.05). The Reaction Times model did not reveal clusters surviving multiple comparison correction. Download figure Open in new tab Supplementary Figure S4 Within-sentence multiple regression RSA. (A) ROI analysis. Asterisks indicate FDR-corrected significant decoding accuracies above chance (* p<0.05, ** p<0.01, *** p<0.001, **** p<0.0001). Error bars indicate SEM. (B) Searchlight RSA maps. T -maps were thresholded using Monte Carlo correction for multiple comparisons (voxel threshold p=0.001, corrected cluster threshold p=0.05). The Integration model and the Reaction Times model did not reveal clusters surviving multiple comparison correction Acknowledgements We thank Viola Pegoretti and Leonora Sasso for assistance in preparing the video stimuli. This research was supported by the Caritro Foundation, Italy, and the Fondo Italiano per la Scienza. Funder Information Declared Fondo Italiano per la Scienza , FIS00003032 References 1. ↵ Badre D , Wagner AD ( 2007 ) Left ventrolateral prefrontal cortex and the cognitive control of memory . Neuropsychologia 45 : 2883 – 2901 . OpenUrl CrossRef PubMed Web of Science 2. ↵ Becchio C , Manera V , Sartori L , Cavallo A , Castiello U ( 2012 ) Grasping intentions: from thought experiments to empirical evidence . Front Hum Neurosci 6 : 117 . OpenUrl CrossRef PubMed 3. ↵ Belluzzi A , Fairhall SL ( 2025 ) Semantic control regions actively generate rule-based semantic predictions . Neuroimage 317 : 121379 . OpenUrl PubMed 4. ↵ Binder JR , Desai RH , Graves WW , Conant LL ( 2009 ) Where is the semantic system? A critical review and meta-analysis of 120 functional neuroimaging studies . Cereb Cortex 19 : 2767 – 2796 . OpenUrl CrossRef PubMed Web of Science 5. ↵ Brainard DH ( 1997 ) The Psychophysics Toolbox . Spatial vision 10 : 433 – 436 . OpenUrl CrossRef PubMed Web of Science 6. ↵ Caspers S , Zilles K , Laird AR , Eickhoff SB ( 2010 ) ALE meta-analysis of action observation and imitation in the human brain . Neuroimage 50 : 1148 – 1167 . OpenUrl CrossRef PubMed Web of Science 7. ↵ Catmur C ( 2014 ) Unconvincing support for role of mirror neurons in "action understanding": commentary on Michael et al. (2014). Front Hum Neurosci 8 : 553 . OpenUrl PubMed 8. ↵ Catmur C ( 2015 ) Understanding intentions from actions: Direct perception, inference, and the roles of mirror and mentalizing systems . Conscious Cogn 36 : 426 – 433 . OpenUrl CrossRef PubMed 9. ↵ Clarke A , Tyler LK ( 2014 ) Object-specific semantic coding in human perirhinal cortex . J Neurosci 34 : 4766 – 4775 . OpenUrl Abstract / FREE Full Text 10. ↵ Coutanche MN , Thompson-Schill SL ( 2013 ) Informational connectivity: identifying synchronized discriminability of multi-voxel patterns across the brain . Frontiers in Human Neuroscience 7 . 11. ↵ Csibra G , Gergely G ( 2007 ) ’Obsessed with goals’: functions and mechanisms of teleological interpretation of actions in humans . Acta Psychol (Amst ) 124 : 60 – 78 . OpenUrl CrossRef PubMed Web of Science 12. ↵ de Lange FP , Spronk M , Willems RM , Toni I , Bekkering H ( 2008 ) Complementary systems for understanding action intentions . Curr Biol 18 : 454 – 457 . OpenUrl CrossRef PubMed Web of Science 13. ↵ Fairhall SL , Caramazza A ( 2013 ) Brain regions that represent amodal conceptual knowledge . J Neurosci 33 : 10552 – 10558 . OpenUrl Abstract / FREE Full Text 14. ↵ Frankland SM , Greene JD ( 2015 ) An architecture for encoding sentence meaning in left mid-superior temporal cortex . Proc Natl Acad Sci U S A 112 : 11732 – 11737 . OpenUrl Abstract / FREE Full Text 15. ↵ Frith C , Frith U ( 2006 ) The neural basis of mentalizing . Neuron 50 : 531 – 534 . OpenUrl CrossRef PubMed Web of Science 16. ↵ Goldenberg G , Spatt J ( 2009 ) The neural basis of tool use . Brain 132 : 1645 – 1655 . OpenUrl CrossRef PubMed Web of Science 17. ↵ Hamilton AF , Grafton ST ( 2006 ) Goal representation in human anterior intraparietal sulcus . J Neurosci 26 : 1133 – 1137 . OpenUrl Abstract / FREE Full Text 18. ↵ Jackson RL ( 2021 ) The neural correlates of semantic control revisited . Neuroimage 224 : 117444 . OpenUrl CrossRef PubMed 19. ↵ Jamali M , Grannan BL , Fedorenko E , Saxe R , Baez-Mendoza R , Williams ZM ( 2021 ) Single-neuronal predictions of others’ beliefs in humans . Nature 591 : 610 – 614 . OpenUrl CrossRef PubMed 20. ↵ Karakose-Akbiyik S , Caramazza A , Wurm MF ( 2023 ) A shared neural code for the physics of actions and object events . Nat Commun 14 : 3316 . OpenUrl CrossRef PubMed 21. ↵ Kilner JM , Friston KJ , Frith CD ( 2007 ) Predictive coding: an account of the mirror neuron system . Cogn Process 8 : 159 – 166 . OpenUrl CrossRef PubMed 22. ↵ Konkle T , Caramazza A ( 2013 ) Tripartite organization of the ventral stream by animacy and object size . J Neurosci 33 : 10235 – 10242 . OpenUrl Abstract / FREE Full Text 23. ↵ Leonardelli E , Fairhall SL ( 2022 ) Similarity-based fMRI-MEG fusion reveals hierarchical organisation within the brain’s semantic system . Neuroimage 259 . 24. ↵ Leshinskaya A , Caramazza A ( 2015 ) Abstract categories of functions in anterior parietal lobe . Neuropsychologia 76 : 27 – 40 . OpenUrl CrossRef PubMed 25. ↵ Lingnau A , Downing PE ( 2015 ) The lateral occipitotemporal cortex in action . Trends Cogn Sci 19 : 268 – 277 . OpenUrl CrossRef PubMed 26. ↵ Liu S , Wurm MF , Caramazza A ( 2024 ) Dissociating goal from outcome during action observation . Cereb Cortex 34 . 27. ↵ Martin CB , Douglas D , Newsome RN , Man LL , Barense MD ( 2018 ) Integrative and distinctive coding of visual and conceptual object features in the ventral visual stream . Elife 7 . 28. ↵ Oosterhof NN , Connolly AC , Haxby JV ( 2016 ) CoSMoMVPA: Multi-Modal Multivariate Pattern Analysis of Neuroimaging Data in Matlab/GNU Octave . Front Neuroinform 10 : 27 . OpenUrl PubMed 29. ↵ Ritchie JB , Bracci S , Op de Beeck H ( 2017 ) Avoiding illusory effects in representational similarity analysis: What (not) to do with the diagonal . Neuroimage 148 : 197 – 200 . OpenUrl CrossRef PubMed 30. ↵ Saxe R , Kanwisher N ( 2003 ) People thinking about thinking people. The role of the temporo-parietal junction in "theory of mind" . Neuroimage 19 : 1835 – 1842 . OpenUrl CrossRef PubMed Web of Science 31. ↵ Schurz M , Radua J , Aichhorn M , Richlan F , Perner J ( 2014 ) Fractionating theory of mind: a meta-analysis of functional brain imaging studies . Neurosci Biobehav Rev 42 : 9 – 34 . OpenUrl CrossRef PubMed 32. ↵ Spunt RP , Lieberman MD ( 2012 ) Dissociating modality-specific and supramodal neural systems for action understanding . J Neurosci 32 : 3575 – 3583 . OpenUrl Abstract / FREE Full Text 33. ↵ Spunt RP , Adolphs R ( 2014 ) Validating the Why/How contrast for functional MRI studies of Theory of Mind . Neuroimage 99 : 301 – 311 . OpenUrl CrossRef PubMed 34. ↵ Spunt RP , Satpute AB , Lieberman MD ( 2010 ) Identifying the what, why, and how of an observed action: an fMRI study of mentalizing and mechanizing during action observation . J Cogn Neurosci 23 : 63 – 74 . OpenUrl Web of Science 35. ↵ Tamir DI , Thornton MA ( 2018 ) Modeling the Predictive Social Mind . Trends Cogn Sci 22 : 201 – 212 . OpenUrl CrossRef PubMed 36. ↵ Tamir DI , Thornton MA , Contreras JM , Mitchell JP ( 2016 ) Neural evidence that three dimensions organize mental state representation: Rationality, social impact, and valence . Proc Natl Acad Sci U S A 113 : 194 – 199 . OpenUrl Abstract / FREE Full Text 37. ↵ Thompson EL , Bird G , Catmur C ( 2019 ) Conceptualizing and testing action understanding . Neurosci Biobehav Rev 105 : 106 – 114 . OpenUrl CrossRef PubMed 38. ↵ Thornton MA , Mitchell JP ( 2018 ) Theories of Person Perception Predict Patterns of Neural Activity During Mentalizing . Cereb Cortex 28 : 3505 – 3520 . OpenUrl CrossRef PubMed 39. ↵ Uithol S , van Rooij I , Bekkering H , Haselager P ( 2011 ) Understanding motor resonance . Soc Neurosci : 1 – 10 . 40. ↵ Van Overwalle F ( 2009 ) Social cognition and the brain: a meta-analysis . Hum Brain Mapp 30 : 829 – 858 . OpenUrl CrossRef PubMed Web of Science 41. ↵ Van Overwalle F , Baetens K ( 2009 ) Understanding others’ actions and goals by mirror and mentalizing systems: a meta-analysis . Neuroimage 48 : 564 – 584 . OpenUrl CrossRef PubMed Web of Science 42. ↵ Walther A , Nili H , Ejaz N , Alink A , Kriegeskorte N , Diedrichsen J ( 2016 ) Reliability of dissimilarity measures for multi-voxel pattern analysis . Neuroimage 137 : 188 – 200 . OpenUrl CrossRef PubMed 43. ↵ Wurm MF , Schubotz RI ( 2012 ) Squeezing lemons in the bathroom: contextual information modulates action recognition . Neuroimage 59 : 1551 – 1559 . OpenUrl CrossRef PubMed 44. ↵ Wurm MF , Caramazza A ( 2019 ) Distinct roles of temporal and frontoparietal cortex in representing actions across vision and language . Nat Commun 10 : 289 . OpenUrl CrossRef PubMed 45. ↵ Wurm MF , Caramazza A ( 2022 ) Two ’what’ pathways for action and object recognition . Trends Cogn Sci 26 : 103 – 116 . OpenUrl CrossRef PubMed 46. ↵ Wurm MF , Caramazza A , Lingnau A ( 2017 ) Action Categories in Lateral Occipitotemporal Cortex Are Organized Along Sociality and Transitivity . J Neurosci 37 : 562 – 575 . OpenUrl Abstract / FREE Full Text 47. ↵ Wurm MF , Hrkać M , Morikawa Y , Schubotz RI ( 2014 ) Predicting goals in action episodes attenuates BOLD response in inferior frontal and occipitotemporal cortex . Behavioural brain research 274 : 108 – 117 . OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted November 26, 2025. Download PDF Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Decoding the neural stages from action and object recognition to mentalizing Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Decoding the neural stages from action and object recognition to mentalizing Moritz F. Wurm , Seoyoung Lee bioRxiv 2025.11.25.690412; doi: https://doi.org/10.1101/2025.11.25.690412 Share This Article: Copy Citation Tools Decoding the neural stages from action and object recognition to mentalizing Moritz F. Wurm , Seoyoung Lee bioRxiv 2025.11.25.690412; doi: https://doi.org/10.1101/2025.11.25.690412 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Neuroscience Subject Areas All Articles Animal Behavior and Cognition (7618) Biochemistry (17633) Bioengineering (13856) Bioinformatics (41841) Biophysics (21399) Cancer Biology (18529) Cell Biology (25422) Clinical Trials (138) Developmental Biology (13352) Ecology (19860) Epidemiology (2067) Evolutionary Biology (24282) Genetics (15582) Genomics (22462) Immunology (17700) Microbiology (40295) Molecular Biology (17140) Neuroscience (88419) Paleontology (666) Pathology (2823) Pharmacology and Toxicology (4813) Physiology (7632) Plant Biology (15107) Scientific Communication and Education (2042) Synthetic Biology (4284) Systems Biology (9808) Zoology (2267)