Movement sonification during haptic exploration shifts emotional outcome without altering texture perception

Movement sonification during haptic exploration shifts emotional outcome without altering texture perception | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Movement sonification during haptic exploration shifts emotional outcome without altering texture perception View ORCID Profile Laurence Mouchnino , View ORCID Profile Pierre-Henri Cornuault , View ORCID Profile Jenny Faucheu , View ORCID Profile Arnaud Witt , Chloé Sutter , View ORCID Profile Benjamin Weiland , View ORCID Profile Jean Blouin , View ORCID Profile Francesco Massi , View ORCID Profile Eric Chatelet , View ORCID Profile Jérémy Danna doi: https://doi.org/10.1101/2025.05.28.656569 Laurence Mouchnino 1 Aix-Marseille Université CNRS , CRPN Marseille, France 2 Institut Universitaire de France , Paris, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Laurence Mouchnino For correspondence: laurence.mouchnino{at}univ-amu.fr Pierre-Henri Cornuault 3 Université Marie et Louis Pasteur, UFR-ST / institut FEMTO ST , F-25000 Besançon, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Pierre-Henri Cornuault Jenny Faucheu 4 Ecole des Mines de Saint-Etienne Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Jenny Faucheu Arnaud Witt 5 Université de Bourgogne/ Laboratoire d’Etude de l’Apprentissage et du Développement , Dijon, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Arnaud Witt Chloé Sutter 1 Aix-Marseille Université CNRS , CRPN Marseille, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site Benjamin Weiland 6 Université de Bourgogne Europe, ISAT , Laboratoire DRIVE, Auxerre, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Benjamin Weiland Jean Blouin 1 Aix-Marseille Université CNRS , CRPN Marseille, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Jean Blouin Francesco Massi 7 Department of Mechanical and Aerospace Engineering, Sapienza University of Rome , Rome, Italy Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Francesco Massi Eric Chatelet 8 Univ. Lyon, INSA Lyon, CNRS , LAMCOS, UMR5259, 69621 Villeurbanne, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Eric Chatelet Jérémy Danna 9 CLLE, Université de Toulouse, CNRS , Toulouse, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Jérémy Danna Abstract Full Text Info/History Metrics Preview PDF Abstract Adding movement sonification to haptic exploration can change the perceptual outcome of a textured surface through multisensory processing. We hypothesized that auditory-evoked emotions would influence the appraisal of textured surfaces, with corresponding changes reflected in cortical excitability. Participants actively rubbed two different textured surfaces (slippery and rough) either without movement sonification, or with pleasant or disagreeable movement sonification. We found that sounds, whether agreeable or disagreeable, did not change the texture appraisal. However, the less pleasant surface was associated with a stronger negative hedonic valence, particularly when paired with disagreeable movement sonification. Time frequency analyses of EEG activities revealed a significant reduction in beta-band power [15-25 Hz] within the source-estimated sensorimotor and superior posterior parietal cortices when contrasting both pleasant and unpleasant sound with the silent touch. This suggests that the primary somatosensory cortices together with the superior parietal regions participated in the audio-tactile binding, with both pleasant and unpleasant sounds. In addition, we observed a significant increase in beta-band power in medial visual areas, specifically when disagreeable movement sonification was paired with tactile exploration. This may reflect a disengagement of visual cortical processing, potentially amplifying auditory-driven emotional responses and intensifying the perceived unpleasantness of the explored surfaces. Introduction Haptic exploration for extracting information about objects is generally considered a silent process. When vision is absent, the texture of an object is thus perceived through multisensory processing of tactile and proprioceptive information (i.e., haptic information). Interestingly, when haptic exploration becomes audible, the perception of texture can change. One of the most compelling demonstrations of tactile-auditory integration altering texture perception is the so-called parchment-skin illusion ( Champoux et al. 2011 ; Jousmäki & Hari, 1998 ). Specifically, people experiencing this illusion perceive the palms of their hands as becoming either dry or moist, depending on the spectral content of the hand-friction sound played while rubbing hands together. Thus, adding movement-related sounds, a technique called movement sonification, can significantly alter the perceived texture, even though the tactile and proprioceptive inputs evoked during surface exploration remain unchanged. The primary objective of this technique, which is conceptually like the Foley effect in cinema, is to convey movement-related information and enhance both the perception and control of movement ( Effenberg, 2005 ). Applied to pen movements, this technique has been successfully used to improve handwriting performance (Veron-Delor et al., 2020; for a review, see Danna & Velay, 2017 ). In texture discrimination tasks, haptic exploration has been shown to change in the presence of textured sounds compared to non-textured sounds, notably by increasing movement velocity ( Landelle et al., 2021 ). A fundamental property of sound is its capacity to evoke a wide range of emotions ( Mithen 2009 ; Dubé & Le Bel 2003 ). Despite this, little is known about how auditory-evoked emotions influence the appraisal of textured surfaces scanned by a finger. This gap persists although there is well-established evidence that our appreciation of objects is strongly influenced by the tactile sensations and emotions elicited when touching them. The present study aims to determine whether movement sonification during tactile exploration of a surface alters both the perceptual appraisal of the texture and, its emotional valence (ranging from “positive, pleasure” to “negative, displeasure”). Changes in the perceptual and emotional appraisal of a texture can result from bottom-up and top-down mechanisms. From a bottom-up perspective, the Bayesian approach proposes that the central nervous system integrates information from different sensory modalities based on the estimated probability that it originates from a common or independent source ( Körding et al., 2007 ). The mechanisms underlying cross-modal interactions in the human brain remain largely unknown. This is particularly the case for audio-tactile interactions, despite their frequent occurrence in everyday life, for example, when hearing footsteps on a gravel path. This example complies with the ‘‘temporal principle’’ of multisensory integration stating that an interaction is most likely to be maximally effective when stimuli overlap in time ( Meredith et al., 1987 ). Sound and touch are likely to respond to this principle as they are deeply entangled. Mechanical forces generated by physical contact not only activate somatosensory mechanoreceptors but also generate acoustic waves detectable by the auditory system (Békésy 1959). To date, neurons responsive to both audio and tactile stimuli have been found in primates in the ventral premotor cortex, the ventral intraparietal region and the superior temporal sulcus (e.g., Graziano et al., 1999 ; Schlack et al., 2005 ). Further evidence indicates that audio-tactile interactions occur already at early stages of sensory processing. Notably, the somatosensory (S1) and auditory (A1) cortical regions, presumptive unimodal sensory areas, exhibit enhanced responses to bimodal stimuli relative to unimodal inputs (e.g., Landelle et al., 2023 ). This has been evidenced in animal studies by Lakatos et al. (2007) showing that somatosensory inputs reset the phase of ongoing neuronal oscillations, so that accompanying auditory inputs arrive during an ideal, high-excitability phase, and produce amplified neuronal responses. In contrast, responses to auditory inputs arriving during the opposing low-excitability phase tend to be suppressed. Zhang et al. (2020) extended these findings from primary cortical areas to secondary somatosensory cortice (S2). They showed in the mouse neocortex that, pairing sound with whisker stimulation modulates tactile responses in both S1 and S2. Sound responses of many sound-selective neurons in S2 neurons were spatially colocalized with S2 touch-selective neurons. Complementing these bottom-up mechanisms, other approaches propose that the perception of a texture under our skin is not driven solely by somatosensory stimulation (see Engel et al., 2001 , for a review). Rather, somatosensory processing would be under the influence of top-down control, which actively modulates, and routes sensory inputs based on expectation, attention, and the perceptual task ( Talsma et al., 2010 ; Gilbert et al. 2007 for review). For example, pleasure induced by listening to pleasant music has been shown to require higher levels of attention compared to neutral music, thereby reducing the attentional resources available for other tasks ( Nemati et al., 2019 ). This relationship between attention and the experience of pleasurable sounds can influence how sensory information is processed and integrated, as attentional states have been shown to modulate somatosensory processing ( Wiesman & Wilson, 2020 ). Through top-down and bottom-up mechanisms, movement sonification may then modulate the hedonic valence of textures during haptic exploration, particularly when the associated sounds convey emotional attributes. In the present study, we hypothesized that texture exploration producing pleasant sounds though movement sonification will be rated as more pleasant, while textures associated with unpleasant sounds will be perceived as less pleasant. These perceptual effects of movement sonification may be linked to changes in beta power within cortical areas involved in processing somatosensory inputs (e.g., S1, PPC) as well as in areas engaged in evaluating emotional valence, such as the cingulate cortex. Methods Participants Twelve right-handed adults (6 women, 6 men, mean age 26 ± 2), without any known neurological or motor disorders, participated in the experiment. All participants gave their written informed consent to take part in this study, which conformed to the standards set in the Declaration of Helsinki. This study was approved by the local Ethics Committee (CERSTAPS: IRB00012476-2021-09-12-140). Task The participants were asked to stroke their finger across the sample surface in a disto-proximal direction several times (∼40 times over 4 recording sessions of 15 seconds), lifting the finger between each stroke. They used their dominant hand and had their eyes closed. Sliding velocity, contact load and finger/surface angle were freely chosen by the participants. After completing the series of strokes, participants indicated whether they felt the texture they just explored as slippery/vibrating/rough (perceptual outcome) and rated their liking of the texture using a 4-levels hedonic scale (emotional outcome): I like it a lot / I like it / I don’t like it / I really don’t like it. Two textured panels made of polyurethane resin (P1 and P2, 60 × 26 mm) were selected according to the results by Faucheu et al. (2019) . In their study, among the 51 explored textures, P1 was clearly qualified as SLIPPERY and P2 as ROUGH. In addition, the hedonic rating of the panels was consensual across the 43 participants: P1 was rated as likeable, and P2 as disagreeable. For both panels, the texture consisted of a flat surface patterned with cylindrical dots, evenly arranged in a hexagonal array. The panels varied in height, diameter and spacing between the cylindrical dots (see Table 1 and Fig. 1A , left panel). View this table: View inline View popup Download powerpoint Table 1. Topographic characteristics of P1 and P2 panels Download figure Open in new tab Figure 1. Microscale topographic views of the slippery and rough surfaces (left panel) and example of vertical acceleration recorded during a single sliding finger movement across the P2 rough surface (right panel) ( A ) and behavioral data: movement duration, normal and tangential forces exerted on the surfaces (respectively left, middle and right panels) (B ). Movement sonification Real time sonification of the finger movement was ensured by an optical motion tracking system fixed to fingernail (sampling frequency: 100 Hz), using a Max/MSP® program. The musical sonification was based on a mapping between the finger’s positions and musical chords with or without the presence of false notes (e.g. audio recordings of the false notes are available at: https://www.gdr.tact.uha.fr/groupes-de-travail/comprendre-et-caracteriser/ ). The sonified movement was revealed by loudspeakers placed in front of the finger as it moves across the surface. The musical chord was tested on some fifteen individuals prior to the experiment. The sonification without false notes was deemed agreeable, while the sonification with false notes was considered disagreeable by most participants. Recordings and analyses To record normal and tangential (shear) force data from the finger/surface interaction, the textured surface is mounted on the top of a home-made multi-sensors box comprising three resistive normal load sensors and one piezoelectric force sensor (Kistler 9217A). Data are acquired through a National Instrument data acquisition system (NI PCIe-6321) using a 10 kHz sampling frequency, and recorded with Labview software. The vertical acceleration of the right index finger was recorded at 1000 Hz with a small and light (5 mm x 12 mm x 7 mm, 0.8 g) accelerometer (PCB 352A24, PCB Piezotronics, Inc., Depew, USA) secured on the finger’s nail with wax. With a high sensitivity of 100 mV/g and a measurement range of ± 50 g pk, this accelerometer can detect very low levels of acceleration. The onset and offset of the finger acceleration were used to define as the onset and offset of the stroking movement, respectively ( Fig1A , right panel). Electroencephalography (EEG) activity was continuously recorded from 64 Ag/AgCl surface electrodes embedded in an elastic cap (BioSemi ActiveTwo system: BioSemi, Netherlands). Specific to the BioSemi system, “ground” electrodes were replaced by Common Mode Sense active and Driven Right Leg passive electrodes. The signals were pre-amplified at the electrode sites, post amplified with DC amplifiers, and digitized at a sampling rate of 1024 Hz (Actiview acquisition program). The signals of each electrode were referenced to the mean signal of all electrodes. The continuous EEG signal was segmented into epochs aligned with the onset of the monotonic increase in shear forces generated by the finger on the surface. The average amplitude computed 100 ms prior to the force onset served as baseline. A mean of 35 trials for P1 and 32 trials for P2 were included in the analyses; the number of trials did not significantly differ between the 3 conditions of sonification (F 2,18 = 3.1; p > 0.05). Cortical sources Neural sources were estimated with the dynamical Statistical Parametric Mapping (dSPM, Dale et al., 2000) implemented in the Brainstorm software. A boundary element model (BEM) with three realistic layers (scalp, inner skull, and outer skull) was used to compute the forward model on the anatomical MRI brain template from the Montreal Neurological Institute (MNI Colin27). Using a realistic model has been shown to provide more accurate solution than a simple three concentric spheres model ( Sohrabpour et al., 2015 ). We used of a high number of vertices (i.e., 15 002 vertices) to enhance the spatial resolution of the brain template. Such EEG source reconstruction has proved to be suited for investigating the activity of outer and inner cortical surfaces with 64 sensors ( Chand and Dhamala, 2017 ; Ponz et al., 2014 ). Measuring and modelling the noise contaminating the data is beneficial to source estimation. Noise covariance matrices were computed using the trials recorded while the index finger stayed stationary on the surface. Beta band (15-25 Hz) event-related synchronization / desynchronization (ERS/ERD) was computed relative to a baseline window from - 3000 to -500 ms prior to finger movements and then averaged over 1000 ms interval starting at the onset of the strokes. This approach ensured that our ERD/ERS analyses captured most of the exploratory movement, encompassing approximately two-thirds of its total duration. Statistics The normal distribution of data sets was assessed using the Kolmogorov–Smirnov test to validate the use of parametric tests. For the behavioral data, analyses of variance (ANOVAs) were conducted on movement duration, normal force and tangential force using the following design: Surface (2 types of surfaces: P1 and P2) × Sonification (3 modalities: silent, agreeable, and disagreeable). For the categorical data on perceptual and emotional outcomes, Pearson chi-square (χ 2 ) tests were conducted to compare the two texture panels (P1 and P2) and the three sonification conditions. The EEG data was submitted to paired t -tests implemented in Brainstorm software. Results Behaviour Non-parametric (χ 2 ) tests revealed that the interactions between Surface and Sonification were not significant for any of the behavioral variables: stroke duration (F 2,22 = 0.39; p = 0.67); normal forces (F 2,22 = 1.80; p = 0.18) and tangential forces (F 2,22 = 1.37; p = 0.27) ( Fig.1B ). For simplicity and consistency, subsequent analyses will be conducted separately for the factors Surface and Sonification. Although no specific instructions were given regarding the finger movement, stroke durations were slightly but significantly shorter on the slippery surface (P1: 1512 ± 527 ms) compared to the rough surface (P2: 1668 ± 586 ms), as confirmed by a significant surface effect (F 1,11 = 13.18; p <0.01). The shorter stroke duration observed on the slippery surface cannot be attributed to smaller shear (tangential) forces as these forces did not differ significantly between both surfaces (F 1,11 = 0.55; p = 0.47). On the other hand, the normal force exerted by the finger was unexpectedly higher on the P1 slippery surface (0.18 ± 0.3N) than on P2 rough surface (0.15 ± 0.3N) (F 1,11 = 21.73; p < 0.01). The forces (normal or tangential) exerted by the finger while rubbing the surfaces were not significantly affected by the movement sonification (F 2,22 = 3.18; p = 0.061 and F 2,22 = 3.13; p = 0.063, for the normal and tangential forces, respectively). Perceptual outcome Participants categorized P1 and P2 surfaces differently (χ 2 = 168,61; p < 0.001). P1 was overwhelmingly perceived as slippery (84% on average), whereas P2 was primarily identified as rough, though with less consensus (63% on average) ( Fig.2A ). Additionally, P2 was also identified as vibrating in about one-third of the responses (34% on average). These surface categorizations were consistent with those reported by Faucheu et al. (2019) using the same materials. The sonification, either agreeable or disagreeable, did not change the texture appraisal (i.e., perceptual outcome) as suggested by the absence of significant effect of the Pearson chi-square on P1 (χ 2 =5.61; p = 0.23) nor on P2 (χ 2 = 2.73; p = 0.60). Download figure Open in new tab Figure 2. Texture perceptual appraisal ( A ) and. emotional appraisal ( B ) of the P1 slippery and P2 rough surfaces, without stroke sonification, or with either agreeable (P1A, P2A) or disagreeable (P1D, P2D) sonification. (*, p < 0.05 and ***, p < 0.001). Emotional outcome To examine the relationship between movement sonification conditions and surface hedonic judgement, emotional appraisal was evaluated using a Likert scale. For the slippery surface P1, adding either a pleasant or unpleasant sound did not change the hedonic perception of the surface ( p = 0.67). A striking finding is that only the rough, and less pleasant P2 surface showed a change in its hedonic valence depending on the sonification conditions (χ 2 = 6.06; p = 0.04, Fig.2B ). The P2 surface was perceived as more unpleasant with disageeable sonification compared to when it was stroked without movement sonification ( p = 0.01). The pleasant sonification did not change the emotional output ( p < 0.05) ( Fig.2B ). Brain activity Given that the power of beta-band neural oscillations is inversely correlated with cortical processing ( Cheyne et al. 2003 ; Pfurtscheller and Lopes da Dilva, 1999), contrasting the topographical distribution of beta power between both sonification conditions and the silent stroke condition can provide reliable estimate of the effect of movement sonification on cortical processing. For the P2 rough surface, the contrasts revealed a significant reduction in beta-band power within the source-estimated sensorimotor and superior parietal cortices during stroking under both sonification conditions, relative to the silent stroke condition ( Fig. 3A , cold blue color). This decreased beta-band power (ERD) suggests amplified neuronal responses when the somatosensory inputs, particularly of tactile origin from the stroking movement, are paired with auditory inputs. However, in other cortical regions, beta-band power varied depending on the valence of the sonification. Download figure Open in new tab Figure 3. Topographical distribution of beta band power resulting from the contrasts: P2 rough surface with agreeable (left panel) and disagreeable (right panel) sonifications vs P2 with silent stroke (A) , P1: slippery surface with agreeable (left panel) and disagreeable (right panel) sonifications vs P1 silent stroke (B) . The agreeable movement sonification elicited an ERD largely circumscribed to the posterior cingulate cortices ( Fig. 3A , left panel). In contrast, the disagreeable sonification induced an increased beta-band power (ERS) over both the right and left primary visual cortices relative to the silent condition (indicated by the warm orange color in Fig. 3A , right panel). The valence of the movement sonification had no effect on beta-band power when stroking the P1 slippery surface ( Fig. 3B ). Overall, pairing an auditory stimulus with finger movement significantly altered cortical processing when participants stroked the rough, unpleasant P2 surface, but had no observable effect during contact with the smooth, pleasant P1 surface. Discussion The aim of this study was to investigate whether movement sonification alters the perceptual evaluation and hedonic valence of textured surfaces, as assessed through behavioural, psychophysical, and neurophysiological measures. The results show that texture appraisal is impervious to sonification conditions, whereas their emotional valence is not. This effect was observed with the P2 rough surface, which was classified as disagreeable and perceived as even more unpleasant when paired with disagreeable movement sonification. However, the brain’s response to the audio-tactile integration was altered by both the agreeable and the disagreeable movement sonifications. In both conditions where stroking movement evoked auditory inputs, we observed a marked decrease in beta-band oscillatory power over the sensorimotor cortex and the superior parietal cortex as compared to silent touch ( Fig. 3A , ERD blue/cold color). Such decreases in beta-band power are commonly interpreted as reflecting increased cortical excitability and enhanced sensory processing ( Haegens et al. 2011 ; Romei et al. 2008 ; Sauseng et al. 2009; Kilavik 2012 for review). It is also in line with the fMRI study by Landelle et al.’s (2023), which demonstrated the sensitivity of the primary somatosensory cortex to tactile input from the hand. Together, these findings suggest that multisensory interactions are not confined to higher-order brain regions but also take place in early somatosensory areas including primary somatosensory cortices and superior parietal regions. This suggestion is supported by neuroanatomical studies showing direct thalamocortical projections to the superior parietal areas in macaques ( Pearson et al. 1978 ; Pons and Kaas 1985 ; Padberg et al. 2009 ; Impieri et al. 2018 ). The early multisensory interaction is thought to be mediated by means of phase resetting. The presentation of a stimulus to one sensory modality (e.g., auditory) resets the phase of ongoing oscillations in another modality (e.g., tactile) such that processing in the latter modality is modulated ( Lakatos et al., 2007 , 2009 ). While these data speak to the importance of primary somatosensory regions for audio-tactile information integration, evaluating the hedonic valence of movement sonification likely engages higher-order brain areas. One candidate region is the posterior cingulate cortices (PCC), which exhibited a significant increase in beta band ERD (15-25 Hz) during agreeable sonification compared to silent touch. However, this change in beta band power was not accompanied by a corresponding change in the emotional rating of the scanned surface. It is possible that altering the hedonic valence of the touched surface would require a sound that is not merely free of dissonance but is explicitly pleasant. The fact that PCC beta-band power remained unchanged during disagreeable sonification suggests that this medial cortical region may be specifically involved in processing pleasantness. In this light, our findings are consistent with studies showing that the PCC is engaged in evaluating the emotional valence ( Maddock et al., 2003 ) and context of sensory information via pathways from somatosensory regions (superior and inferior parietal areas, Vogt, 2014 ), auditory regions ( Vogt & Laureys, 2009 ) and adjacent cingulate areas such as the midcingulate cortex ( Vogt 2016 ; Rolls, 2014; Grabenhorst & Rolls, 2011 ). Due to these extensive connections, the PCC is considered to exhibit the highest degree of cross-network interaction ( Betti et al., 2021 for review). When false notes were present, as during the disagreeable movement sonification of P2 rough surface, a bilateral decrease in brain activity was observed in the early visual cortex compared to silent touch. This was evidenced by an increase in event related synchronization (ERS) (red/warm color, Fig. 3A , right panel) in occipital regions (BA17). This observation raises the question as to how and why the earliest visual cortical area, known to receive direct retinal inputs from the thalamic lateral geniculate nucleus, is modulated by auditory signals. Signatures of auditory inputs in primary visual cortex were previously observed by Vetter et al.’s (2014) fMRI study in absence of visual inputs (see also Manson et al. 2019 ). Their results showed that the early visual cortex contributes, via its corticocortical connections, to the encoding of complex natural sounds as abstract, higher-level internal representations ( Petro et al., 2017 for a review), further highlighting the existence of multiple cortical representations of sounds. The disengagement of visual cortical areas during disagreeable movement sonification (reflected by increased beta-band ERS) may have played a crucial role in enhancing auditory-driven emotional responses by reducing the influence of representations unrelated to auditory pleasantness. This mechanism could explain why participants rated the rough surface as even more unpleasant when their haptic exploration was paired with dissonant sonification (see Fig. 2A ). Our results may be interpreted within the framework of the Bayesian causal inference model that postulates that multisensory integration is a consequence of an optimal probabilistic inference of each sensory modality ( Kayser & Shams, 2015 ). Accordingly, both sensory sources, auditory and tactile, are integrated in the judgment task but with a different weight allocated to each sensory input, which depends on the level of relevance and accuracy of the processed information. For instance, for the surface P2, the addition of disagreeable sounds shifts the emotional response towards the unpleasant, as the emotional categorization of the tactile modality is less stabilized as shown by the emotional rating distribution. The lack of effect of sound conditions on P1 slippery agreeable surface on neural excitability ( Fig. 3B ) suggests that the tactile modality has a greater weight than the auditory modality in this texture perception task at both perceptual and emotional levels. Indeed, adding an unpleasant sound is not decisive when the tactile experience is considered agreeable. This aligns with findings in young adults, who were able to segregate irrelevant auditory information to maintain their haptic discrimination performance ( Lederman, 1979 ; Landelle et al., 2021 ). Funding This work was supported by the project COMTACT (ANR 2020-CE28-0010-03), funded by the french “Agence Nationale de la Recherche” (ANR). Acknowledgments We thank Gilles Marivier and Pauline Couta, society Weydo Music, for creating the sonification set up. Funder Information Declared ANR 2020-CE28-0010-03 Bibliography ↵ Betti , V. , Della Penna , S. , de Pasquale , F. , & Corbetta , M. ( 2021 ). Spontaneous beta band rhythms in the predictive coding of natural stimuli . The Neuroscientist , 27 ( 2 ), 184 – 201 . OpenUrl CrossRef PubMed ↵ Champoux , F. , Collignon , O. , Bacon , B. A. , Lepore , F. , Zatorre , R. J. , & Théoret , H. ( 2011 ). Early-and late-onset blindness both curb audiotactile integration on the parchment-skin illusion . Psychological Science , 22 ( 1 ), 19 – 25 . OpenUrl CrossRef PubMed ↵ Chand , G. B. , & Dhamala , M. ( 2017 ). Interactions between the anterior cingulate-insula network and the fronto-parietal network during perceptual decision-making . Neuroimage , 152 , 381 – 389 . OpenUrl CrossRef PubMed ↵ Cheyne , D. , Gaetz , W. , Garnero , L. , Lachaux , J. P. , Ducorps , A. , Schwartz , D. , & Varela , F. J. ( 2003 ). Neuromagnetic imaging of cortical oscillations accompanying tactile stimulation . Cognitive brain research , 17 ( 3 ), 599 – 611 . OpenUrl CrossRef PubMed ↵ Danna , J. , & Velay , J. L. ( 2017 ). On the auditory-proprioception substitution hypothesis: movement sonification in two deafferented subjects learning to write new characters . Frontiers in neuroscience , 11 , 137 . OpenUrl PubMed ↵ Dubé , L. , & Le Bel , J. ( 2003 ). The content and structure of laypeople’s concept of pleasure . Cognition and emotion , 17 ( 2 ), 263 – 295 . OpenUrl CrossRef ↵ Effenberg , A. O. ( 2005 ). Movement sonification: Effects on perception and action . IEEE multimedia , 12 ( 2 ), 53 – 59 . OpenUrl CrossRef ↵ Engel , A. K. , Fries , P. , & Singer , W. ( 2001 ). Dynamic predictions: oscillations and synchrony in top–down processing . Nature Reviews Neuroscience , 2 ( 10 ), 704 – 716 . OpenUrl ↵ Faucheu , J. , Weiland , B. , Juganaru-Mathieu , M. , Witt , A. , & Cornuault , P. H. ( 2019 ). Tactile aesthetics: Textures that we like or hate to touch . Acta Psychologica , 201 , 102950 . OpenUrl CrossRef PubMed Gilbert , C. D. , & Sigman , M. ( 2007 ). Brain states: top-down influences in sensory processing . Neuron , 54 ( 5 ), 677 – 696 . OpenUrl CrossRef PubMed Web of Science ↵ Graziano , M. S. , Reiss , L. A. , & Gross , C. G. ( 1999 ). A neuronal representation of the location of nearby sounds . Nature , 397 ( 6718 ), 428 – 430 . OpenUrl CrossRef PubMed Web of Science ↵ Haegens , S. , Händel , B. F. , & Jensen , O. ( 2011 ). Top-down controlled alpha band activity in somatosensory areas determines behavioral performance in a discrimination task . Journal of neuroscience , 31 ( 14 ), 5197 – 5204 . OpenUrl Abstract / FREE Full Text ↵ Impieri D , Gamberini M , Passarelli L , Rosa MGP , Galletti C. ( 2018 ). Thalamo-cortical projections to the macaque superior parietal lobule areas PEc and PE . J Comp Neurol . 526 : 1041 – 1056 . OpenUrl CrossRef PubMed ↵ Jousmäki , V. , & Hari , R. ( 1998 ). Parchment-skin illusion: sound-biased touch . Current biology , 8 ( 6 ), R190 – R191 . OpenUrl CrossRef PubMed Web of Science ↵ Kayser , C. , & Shams , L. ( 2015 ). Multisensory causal inference in the brain . PLoS biology , 13 ( 2 ), e1002075 . OpenUrl CrossRef PubMed Kilavik , B. E. , Zaepffel , M. , Brovelli , A. , MacKay , W. A. , & Riehle , A. ( 2013 ). The ups and downs of beta oscillations in sensorimotor cortex . Experimental neurology , 245 , 15 – 26 . OpenUrl CrossRef PubMed ↵ Körding , K. P. , Beierholm , U. , Ma , W. J. , Quartz , S. , Tenenbaum , J. B. , & Shams , L. ( 2007 ). Causal inference in multisensory perception . PLoS one , 2 ( 9 ), e943 . OpenUrl CrossRef PubMed ↵ Lakatos , P. , Chen , C. M. , O’Connell , M. N. , Mills , A. , & Schroeder , C. E. ( 2007 ). Neuronal oscillations and multisensory interaction in primary auditory cortex . Neuron , 53 ( 2 ), 279 – 292 . OpenUrl CrossRef PubMed Web of Science ↵ Lakatos , P. , O’Connell , M. N. , Barczak , A. , Mills , A. , Javitt , D. C. , & Schroeder , C. E. ( 2009 ). The leading sense: supramodal control of neurophysiological context by attention . Neuron , 64 ( 3 ), 419 – 430 . OpenUrl CrossRef PubMed Web of Science ↵ Landelle , C. , Danna , J. , Nazarian , B. , Amberg , M. , Giraud , F. , Pruvost , L. , Kronland-Martinet , R. , Ystad , S. , Aramaki , M. , & Kavounoudias , A. ( 2021 ). The impact of movement sonification in haptic perception changes with aging . Scientific Reports , 11 : 5124 . doi : 10.1038/s41598-021-84581-3 OpenUrl CrossRef PubMed ↵ Landelle , C. , Caron-Guyon , J. , Nazarian , B. , Anton , J.L. , Sein , J. , Pruvost , L. , Amberg , M. , Giraux , F. , Félician , O. , Danna , J. , & Kavounoudias , A. ( 2023 ). Beyond Sense-Specific processing: Decoding Texture in the Brain from Touch and Sonified Movement . iScience 26 ( 10 ), 107965 . doi: 10.1016/j.isci.2023.107965 OpenUrl CrossRef PubMed ↵ Lederman , S. J. Auditory texture perception . Perception 8 , 93 – 103 ( 1979 ) OpenUrl CrossRef PubMed Web of Science ↵ Grabenhorst , F. , & Rolls , E. T. ( 2011 ). tValue, pleasure and choice in the ventral prefrontal cortex . Trends in cognitive sciences , 15 ( 2 ), 56 – 67 . OpenUrl CrossRef PubMed Web of Science ↵ Maddock , R. J. , Garrett , A. S. , & Buonocore , M. H. ( 2003 ). Posterior cingulate cortex activation by emotional words: fMRI evidence from a valence decision task . Human brain mapping , 18 ( 1 ), 30 – 41 . OpenUrl CrossRef PubMed Web of Science ↵ Manson GA , Tremblay L , Lebar N , de Grosbois J , Mouchnino L , Blouin J ( 2019 ) Auditory cues for somatosensory targets invoke visuomotor transformations: Behavioral and electrophysiological evidence . Plos One 14 ( 5 ): e0215518 . OpenUrl CrossRef PubMed ↵ Meredith , M. A. , Nemitz , J. W. , & Stein , B. E. ( 1987 ). Determinants of multisensory integration in superior colliculus neurons . I. Temporal factors. Journal of Neuroscience , 7 ( 10 ), 3215 – 3229 . OpenUrl PubMed ↵ Mithen , S. ( 2009 ). The music instinct: the evolutionary basis of musicality . Annals of the New York Academy of Sciences , 1169 ( 1 ), 3 – 12 . OpenUrl CrossRef PubMed ↵ Nemati , S. , Akrami , H. , Salehi , S. , Esteky , H. , & Moghimi , S. ( 2019 ). Lost in music: Neural signature of pleasure and its role in modulating attentional resources . Brain Research , 1711 , 7 – 15 . OpenUrl CrossRef PubMed ↵ Padberg J , Cerkevich C , Engle J , Rajan AT , Recanzone G , Kaas J , Krubitzer L. ( 2009 ). Thalamocortical connections of parietal somatosensory cortical fields in macaque monkeys are highly divergent and convergent . Cereb Cortex . 19 : 2038 – 2064 . OpenUrl CrossRef PubMed Web of Science ↵ Pearson RC , Brodal P , Powell TP . ( 1978 ). The projection of the thalamus upon the parietal lobe in the monkey . Brain Res . 144 : 143 – 148 . OpenUrl CrossRef PubMed Web of Science Pfurtscheller , G. , Lopes da Silva , F. H. ( 1999 ). Event-related EEG/MEG synchronization and desynchronization : Basic principles . Clinical Neurophysiology , 110 ( 11 ). ↵ Petro , L. S. , Paton , A. T. , & Muckli , L. ( 2017 ). Contextual modulation of primary visual cortex by auditory signals . Philosophical Transactions of the Royal Society B: Biological Sciences , 372 ( 1714 ), 20160104 . OpenUrl CrossRef PubMed ↵ Pons TP , Kaas JH . ( 1985 ). Connections of area 2 of somatosensory cortex with the anterior pulvinar and subdivisions of the ventroposterior complex in macaque monkeys . J Comp Neurol . 240 : 16 – 36 . OpenUrl CrossRef PubMed Web of Science ↵ Ponz , A. , Montant , M. , Liegeois-Chauvel , C. , Silva , C. , Braun , M. , Jacobs , A. M. , Ziegler , J. C. ( 2014 ). Emotion processing in words: A test of the neural re-use hypothesis using surface and intracranial EEG . Social Cognitive and Affective Neuroscience , 9 ( 5 ), 619 – 627 . OpenUrl CrossRef PubMed Rolls , E. T. ( 2019 ). The cingulate cortex and limbic systems for emotion, action, and memory . Brain Structure and Function , 224 ( 9 ), 3001 – 3018 . OpenUrl CrossRef ↵ Romei , V. , Brodbeck , V. , Michel , C. , Amedi , A. , Pascual-Leone , A. , & Thut , G. ( 2008 ). Spontaneous fluctuations in posterior α-band EEG activity reflect variability in excitability of human visual areas . Cerebral cortex , 18 ( 9 ), 2010 – 2018 . OpenUrl CrossRef PubMed Web of Science ↵ Schlack , A. , Sterbing-D’Angelo , S. J. , Hartung , K. , Hoffmann , K. P. , & Bremmer , F. ( 2005 ). Multisensory space representations in the macaque ventral intraparietal area . Journal of Neuroscience , 25 ( 18 ), 4616 – 4625 . OpenUrl Abstract / FREE Full Text ↵ Sohrabpour , A. , Lu , Y. , Kankirawatana , P. , Blount , J. , Kim , H. , & He , B. ( 2015 ). Effect of EEG electrode number on epileptic source localization in pediatric patients . Clinical Neurophysiology , 126 ( 3 ), 472 – 480 . OpenUrl CrossRef PubMed ↵ Talsma , D. , Senkowski , D. , Soto-Faraco , S. , & Woldorff , M. G. ( 2010 ). The multifaceted interplay between attention and multisensory integration . Trends in cognitive sciences , 14 ( 9 ), 400 – 410 . OpenUrl CrossRef PubMed Web of Science Véron-Delor , L. , Pinto , S. , Eusebio , A. , Azulay , J. P. , Witjas , T. , Velay , J. L. , & Danna , J. ( 2020 ). Musical sonification improves motor control in Parkinson’s disease: a proof of concept with handwriting . Annals of the New York Academy of Sciences , 1465 ( 1 ), 132 – 145 . OpenUrl CrossRef PubMed ↵ Vogt , B. A. ( 2014 ). Submodalities of emotion in the context of cingulate subregions . Cortex , 59 ( 197-202 ), 10 – 1016 . OpenUrl ↵ Vogt , B. A. ( 2016 ). Midcingulate cortex: structure, connections, homologies, functions and diseases . Journal of chemical neuroanatomy , 74 , 28 – 46 . OpenUrl CrossRef PubMed ↵ Vogt , B. A. , & Laureys , S. ( 2009 ). The primate posterior cingulate gyrus: connections, sensorimotor orientation, gateway to limbic processing . Cingulate neurobiology and disease , 275 – 308 . Von Békésy , G. ( 1959 ). Neural funneling along the skin and between the inner and outer hair cells of the cochlea . The Journal of the Acoustical Society of America , 31 ( 9 ), 1236 – 1249 . OpenUrl CrossRef Web of Science ↵ Wiesman , A. I. , & Wilson , T. W. ( 2020 ). Attention modulates the gating of primary somatosensory oscillations . Neuroimage , 211 , 116610 . OpenUrl CrossRef PubMed ↵ Zhang , M. , Kwon , S. E. , Ben-Johny , M. , O’Connor , D. H. , & Issa , J. B. ( 2020 ). Spectral hallmark of auditory-tactile interactions in the mouse somatosensory cortex . Communications biology , 3 ( 1 ), 64 . OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted June 01, 2025. Download PDF Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Movement sonification during haptic exploration shifts emotional outcome without altering texture perception Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Movement sonification during haptic exploration shifts emotional outcome without altering texture perception Laurence Mouchnino , Pierre-Henri Cornuault , Jenny Faucheu , Arnaud Witt , Chloé Sutter , Benjamin Weiland , Jean Blouin , Francesco Massi , Eric Chatelet , Jérémy Danna bioRxiv 2025.05.28.656569; doi: https://doi.org/10.1101/2025.05.28.656569 Share This Article: Copy Citation Tools Movement sonification during haptic exploration shifts emotional outcome without altering texture perception Laurence Mouchnino , Pierre-Henri Cornuault , Jenny Faucheu , Arnaud Witt , Chloé Sutter , Benjamin Weiland , Jean Blouin , Francesco Massi , Eric Chatelet , Jérémy Danna bioRxiv 2025.05.28.656569; doi: https://doi.org/10.1101/2025.05.28.656569 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Neuroscience Subject Areas All Articles Animal Behavior and Cognition (7618) Biochemistry (17635) Bioengineering (13859) Bioinformatics (41846) Biophysics (21401) Cancer Biology (18534) Cell Biology (25423) Clinical Trials (138) Developmental Biology (13352) Ecology (19860) Epidemiology (2067) Evolutionary Biology (24286) Genetics (15582) Genomics (22463) Immunology (17700) Microbiology (40298) Molecular Biology (17141) Neuroscience (88429) Paleontology (666) Pathology (2825) Pharmacology and Toxicology (4813) Physiology (7633) Plant Biology (15107) Scientific Communication and Education (2042) Synthetic Biology (4284) Systems Biology (9808) Zoology (2267)