Framed RSA: Representational comparisons that honor both geometry and population-mean response preferences

Framed RSA: Representational comparisons that honor both geometry and population-mean response preferences | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Framed RSA: Representational comparisons that honor both geometry and population-mean response preferences JohnMark Taylor , View ORCID Profile Nikolaus Kriegeskorte doi: https://doi.org/10.1101/2025.07.10.664257 JohnMark Taylor 1 Columbia University Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: johnmarkedwardtaylor{at}gmail.com Nikolaus Kriegeskorte 1 Columbia University Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Nikolaus Kriegeskorte Abstract Full Text Info/History Metrics Preview PDF Abstract Representational similarity analysis (RSA) characterizes the geometry of neural activity patterns elicited by different stimuli while discarding information about neural response preferences, regional population-mean activity and the absolute location and orientation of the patterns in the multivariate response space. Analyses of the tuning of individual neurons or voxels and of the population-mean activation are often performed separately from RSA and thought of as serving complementary purposes. When evaluating alternative representational models, invariance to certain aspects of the neural code is desirable because systems might use superficially different encodings to implement the same computations. However, neural preferences and regional-mean activation are arguably physiologically and mechanistically important, and so we may want for our models to predict them correctly. Here we introduce a novel analysis technique, framed RSA, which honors both the geometry and the population-mean preferences in evaluating model-predicted representations. To achieve this, we augment the set of patterns that define the geometry by two reference patterns: the zero-point (origin) and a uniform constant pattern in the multivariate response space, enabling RSA to incorporate information about the global location, orientation, and mean activation of neural population codes. First we present the mathematical and methodological underpinnings of framed RSA, including how it interacts with different RSA analysis choices, such as the use of cross-validated dissimilarity estimates and whitened RDM comparators. Second, we show empirically that framed RSA generally improves accuracy for both brain-region identification (using fMRI data from the Natural Scenes Dataset and the Things Ventral Stream Spiking Dataset) and deep-neural-network-layer identification relative to existing RSA approaches. By incorporating neural population preferences into model evaluation, framed RSA enables more mechanistically meaningful model comparisons and benefits from improved power for model-comparative inference. Introduction Biological and artificial neural networks represent different aspects of the world via distributed population codes, in which the activation levels of a collection of units signify the contents of perceptual or cognitive representations. A central concern in cognitive neuroscience, computational neuroscience, and AI is measuring the similarity of these population codes across different brain regions or deep neural network (DNN) layers, 18 which is essential for goals such as comparing the function of brain regions within a species, 19 evaluating inter-species similarity in neural function, 11 and assessing DNNs as models of brain activity. 5 , 7 , 21 , 22 However,the systems to be compared differ in many ways. Some of these differences (such as which neuron implements which tuning function) may be idiosyncratic to the individual animal or model instance 13 and inessential to the algorithm, while others (such as what information is explicitly represented) may be indicative of the algorithm implemented by each system. 8 , 10 A key question, then, is which aspects of the neural code should inform the comparison of two systems. One common approach to comparing brain regions or DNN layers is representational similarity analysis 9 (RSA), which involves measuring the pairwise similarity between the neural representations of different stimuli or conditions, and comparing the overall profile of similarity, or representational geometry , between different systems. In neuroscience, researchers often use the correlation between representational dissimilarity matrices (RDMs) as the representational comparator, 9 , 14 , 17 whereas in machine learning the linear centered kernel alignment is the most widely used comparator 6 (for the relationships among these approaches see 3 , 20 ). The representational geometry determines the decodable information and aspects of the format in which the information is encoded (up to an orthogonal linear transform), 8 thus capturing a computationally crucial aspect of the neural code. Comparing systems in terms of their representational geometry disregards other aspects of the neural population code, such as the number of units in the population, the global location or orientation of the set of stimulus activation vectors in multivariate response space, the tuning of the individual neurons and the mean activation evoked by each stimulus. While this abstraction is often useful, some of this discarded information may be biologically or computationally important. Neural responses are nonnegative in both brains and DNNs. For biological neurons there is also a limit on the maximum spiking rate. This restricts the neural population response space to a hypercube, providing a frame that all responses patterns are confined within. The RDM defines the pairwise distances among the patterns, thus rigidly relating the patterns to each other. However, it does not define where the ensemble of patterns is placed and how it is oriented within the hypercube. The location and orientation determine the tuning of individual neurons, the connections needed to encode and decode particular content, and the energy requirements of the code. The location and orientation of the ensemble of patterns thus is not irrelevant to the computational function of the system. The mean activation of a region, in particular, reflects the metabolic costs of a given coding scheme, as well as determining the efficiency of downstream readout. Extending RSA to incorporate this information would enable better distinguishing between systems that vary with respect to these factors in systems for which they are relevant. Here we introduce framed RSA , a new method that allows RSA to incorporate information about the global location and orientation of patterns in multivariate response space, as well as their mean activation. This is achieved through augmenting the set of stimulus patterns with two “framing patterns,” consisting of an all-zeros vector and an all-c vector, where c is some fixed value ( Figure 1 ). The distances between the stimulus patterns and these framing patterns determine the global location and relative mean activations of the stimulus patterns, in addition to greatly reducing the rotational invariance of RSA, such that only rotations that preserve mean activation are disregarded. This extension of RSA also serves to integrate it with the influential, yet traditionally separate method of comparing the mean activation of a brain region to different stimulus categories, 2 , 4 allowing both representational geometry and mean activation to be incorporated into a single functional fingerprint for a brain region. Download figure Open in new tab Figure 1: Motivation and schematic illustration of Framed RSA. (A) Stimuli can elicit distinct neural patterns that nonetheless have the same regional mean activation (left panel), and two populations with identical representational geometry can have reversed regional mean preferences (right panel). (B) Framed RSA involves adding the all-zeros (origin) and all-c patterns to the geometry, creating a “reference frame” that removes the translational invariance and much of the rotational invariance that is present in standard RSA. The distances from the stimuli to the all-zeros (blue) can be added to the initial RDM D to form D 0 , and the distances from the stimulus patterns to the all-c (red) can be further added to form D 0 c . We first describe the mathematics underlying framed RSA, as well as how it interacts with various methodological considerations present in traditional RSA. We then present the results of several brain region and DNN layer identification experiments with a known ground truth to validate the method and explore different methodological choices, showing that it offers substantially improved power for model-comparative inference over both traditional RSA and mean activation approaches. Finally, we make recommendations for best practices when using framed RSA. Framed RSA has been incorporated into the open-source RSA toolbox, 17 making it easily adaptable into existing analysis pipelines. 1 Methods 1.1 Overview The activity patterns elicited by stimuli in a given system can be represented as points in multivariate response space, whose axes correspond to the activations of different units (e.g., individual neurons or fMRI voxels). Standard RSA 9 , 14 , 17 involves computing the pairwise dissimilarities between these activation patterns to construct a representational dissimilarity matrix (RDM) for each system under consideration, followed by applying a comparator function to measure the similarity of RDMs across systems. RSA discards several kinds of information present in multivariate response space. First, the RDM is unaffected by global, uniform translations and rotations of the full set of activity patterns, since these operations are isometric, leaving the distances between patterns unchanged. A consequence of the rotational invariance is that RSA discards information about the tuning of individual units, such as whether they are tuned to a single stimulus property, or whether they show mixed selectivity 16 to multiple such properties. Of particular relevance for present purposes, rotational invariance also entails that RSA is invariant to the regional mean activation preferences for a set of stimuli, such as a higher overall activation to images of faces than to images of houses, since certain rotations can invert the mean response preferences of a region while leaving the distances between patterns unchanged ( Fig. 1 ). RSA is also invariant to the number of units in a population, as long as there are enough units to preserve all pairwise distances. Finally, RSA is invariant to global rescaling of patterns (i.e., multiplying all channels by the same constant); while rescaling the patterns would also rescale the distances between them by the same factor, the RDM comparator functions typically used in RSA—such as correlation or cosine—are invariant to such rescaling. The resulting abstraction after removing this information and only keeping the pairwise dissimilarities between patterns is called the representational geometry of a system. This abstraction is highly computationally informative, in that it fully determines the set of linear readout operations that can be performed by a downstream decoder with access to all units in the population. 8 In Framed RSA ( Fig. 1 ), we augment the set of patterns defining the geometry with two additional patterns: the origin or all-zeros vector 0 , and a vector c consisting of some fixed positive constant c: These two patterns define a hypercube in multivariate response space that constitutes a reference frame for determining the global location and orientation of the stimulus patterns. To aid intuition and emphasize the special role of these patterns, in visualizing the resulting RDM we set 0 as the first pattern and c as the last pattern, such that the entries corresponding to the distances to these patterns form a square “frame” around the RDM, further motivating the name of the method. We use D to denote the RDM containing the stimulus patterns alone, D 0 for the RDM with the stimulus patterns and 0 , and D 0c for the RDM containing the stimulus patterns, 0 , and c . Introducing 0 and c to the geometry endows the RDM with additional information about the location and orientation of the stimulus patterns, removing several kinds of invariance present in standard RSA. The RDM becomes sensitive to all global translations of the stimulus patterns, since these shift the distances to 0 in the RDM. The RDM additionally becomes sensitive to all rotations of the stimulus patterns, with the important exception of rotations about c , since such rotations preserve the distances to these patterns. For a population of n units, the resulting number of degrees of rotational freedom—that is, the number of ways the geometry can rotate without changing the RDM—is , since there are n-1 orthogonal axes all of which are orthogonal to c , and any given pair of these axes defines a unique plane of rotation (e.g., with three units there is one remaining degree of rotational freedom, and with 100 units there are 4,851 remaining degrees of rotational freedom). While other choices of reference patterns would also provide information about the location and orientation of the stimulus patterns, the positions of patterns relative to 0 and c are of special computational relevance for several reasons. First, distances to 0 uniquely correspond to the norms of the patterns. Second, the hypercube defined by 0 and c corresponds to the non-negative sector of multivariate response space. This sector often has a privileged status for biological and artificial neural networks, since firing rates in biological neurons are necessarily non-negative, and since the nonlinear activation functions in artificial neural networks often treat positive and negative values differently (e.g., a ReLU unit will clamp all negative activations to zero). Finally, and crucially, including the distances to 0 and c in the RDM allows the regional mean activation elicited by each stimulus to be recovered (up to a scaling factor) via linear operations on RDM entries when Euclidean distance (or certain variants thereof, such as Mahalanobis or crossnobis distance) is used as the dissimilarity metric. This can be shown by expanding the formula for the squared Euclidean distances among 0, c , and a stimulus pattern s : The final equation can be rearranged to solve for c T s , which corresponds to the regional mean activation times a constant. 1.2 Using noise-normalized dissimilarity metrics in Framed RSA To account for varying levels of noise in different channels, or correlated noise between channels, it is often desirable to use Mahalanobis distance as a dissimilarity metric, in either its standard form or its cross-validated variant, the crossnobis distance. 17 Crossnobis distance is defined as where A and B refer to different measurement folds (e.g., fMRI runs); if A=B this reduces to standard Mahalanobis distance. In the context of Framed RSA, if we follow the same steps as for Euclidean distance we can use cross 2 ( 0, c ), cross 2 ( 0, s ), and cross 2 ( s, c ) to solve for c T Σ − 1 s , which reflects not the raw regional mean as with Euclidean distance, but rather the noise-normalized regional mean, which reflects the extent to which the regional mean is statistically discriminable from zero. If only univariate noise normalization is applied this corresponds to rescaling each channel by its noise variance before computing the mean, and if multivariate noise normalization is applied this corresponds to first rescaling the activations based on the principle components of the noise covariance prior to taking the mean. 1.3 Choosing the value of c As noted earlier, two further invariances present in standard RSA are invariance to the number of units, and invariance to uniform rescaling of the patterns. These both remain desirable in Framed RSA, since we often wish to compare systems with differing numbers of units, or systems whose activations are measured on different scales with no principled choice of conversion factor (e.g., between BOLD activity and DNN unit activation). The constant factor c must therefore be chosen in a way that respects these constraints. To ensure this, we choose c such that the norm of c is equal to the mean norm of the stimulus patterns times a fixed constant k . Thus, while the value of the constant c may vary across the systems being compared, the ratio of the norm of c to the mean norm of the other patterns remains fixed. As a pragmatic default choice we set k = 1 (i.e., the norm of c equals the mean norm of the stimulus patterns), but in the region identification analyses presented later we also explore how results depend on the value of k . When Mahalanobis or crossnobis distance is used, there is one additional complication to consider: should the norm of c be chosen based on the mean norm of the stimulus patterns before or after whitening? Consider two systems whose geometries are distinct before whitening, but become identical after whitening. Applying standard RSA with Mahalanbis/crossnobis as the dissimilarity metric would regard these systems as the same; if we wish for the same to be true for Framed RSA, the norm of c must be tuned based on the norms of the patterns after whitening. 1.4 Using whitened RDM comparators in Framed RSA A simple way of comparing RDMs is to use functions such as cosine or correlation. However, these have the limitation that they do not account for the covariance structure among RDM entries. 3 For instance, the distances x→y and x→z share the same pattern x and its associated noise, inducing correlations between the RDM entries associated with these distances. Whitened variants of correlation and cosine can be used to correct for this covariance structure, which has been shown to improve model-comparative power in some cases. 17 These whitened variants require estimating and inverting V , the matrix of covariances between RDM entries (i.e., with as many rows and columns as there are off-diagonal entries of the corresponding RDM). In past work, V has been estimated under the simplifying assumption that the true distances between patterns are zero. 3 However, this approach is unsuitable for Framed RSA for two reasons. First, the c and 0 vectors used in Framed RSA are both noiseless; if we further assume the distance between them to be zero, their corresponding entries in V will be identical, resulting in a singular matrix. Second, since Framed RSA involves choosing a scaling factor for c , it is desirable for V to take into account the magnitude of this scaling factor, which is impossible if we compute V under the assumption that the true distances are zero. The requirement of using noisy estimates of the underlying distances to compute V also raises potential numerical challenges, since the size of V scales with the fourth power of the number of patterns (i.e., the number of pairs-of-pairs of stimuli), and imprecise estimates of the underlying distances may render the inversion of V numerically unstable. We therefore make several adjustments to the use of whitened RDM comparators in the case of Framed RSA. 1.5 Region Identification Experiments As discussed above, introducing 0 and c to the geometry allows the regional mean activations evoked by each stimulus (either the raw mean if Euclidean distance is used as the dissimilarity metric, or the noise-normalized mean if Mahalanobis or crossnobis distance is used) to be linearly recovered from the RDM entries. The mean stimulus preferences of a region are of theoretical value in their own right. A further practical question is whether Framed RSA might enable more powerful adjudication among computational models of brain function relative to standard RSA. Since we typically do not have access to the ground-truth model for a given system (e.g., V1), as a proxy metric we use the approach of region identification . This approach adopts the pragmatic assumption that the best model of a given system will be another instance of the same system. Thus, one way to assess the effectiveness of a given method for model adjudication is to measure how frequently it categorizes different instances of the same system (e.g., V1 of two different subjects) as more similar than instances of different systems (e.g., V1 in one subject and V2 in a different subject). For simplicity, we use the phrase “region identification” to encompass the application of this approach to DNNs as well, treating their layers as analogous to regions of the brain. We applied this approach across three datasets. The first dataset is the Natural Scenes Dataset 1 (NSD), a massive open-source 7T fMRI dataset with responses to thousands of images from eight subjects, including responses to 1000 shared images that were presented to all subjects. The second dataset is the Things Ventral Stream Spiking Dataset 15 (TVSD), consisting of intracranial neural recording data from V1, V4, and IT of two macaque monkeys. The third dataset is a set of ten instances of AlexNet 12 trained for image classification on ImageNet from different random seeds. These datasets were chosen based on the following criteria. First, the region identification paradigm requires that multivariate pattern data can be collected from multiple clearly demarcated regions (layers in the case of DNNs) of multiple subjects (instances in the case of DNNs), with every region present in every subject. This requirement excludes modalities such as EEG or MEG, since these methods lack the spatial resolution to collect fine-grained response patterns from specific brain regions. The requirement that data from all regions be present in all subjects also limits the viability of using human intracranial data, since such data is typically collected opportunistically, with the location of electrodes and quantity of data being constrained by clinical factors. Second, the datasets are complementary in several useful respects. The NSD and TVSD involve different species and different measurement modalities with varying levels of noise, with fMRI being much noisier than intracranial recordings. Since the regional mean activation is plausibly robust to noise, it is of interest to examine how the potential benefits of Framed RSA generalize across different noise levels. The inclusion of a DNN dataset offers the benefit of having access to the ground-truth stimulus patterns, and being able to vary the level of noise as desired. The analysis pipeline for all datasets followed the same general approach, with relevant adjustments being made for each dataset as needed; for simplicity, we use the term “region” to encompass both brain regions and DNN layers, and the term “subject” to encompass both individual humans/monkeys and DNN instances. A given subject was designated as the test subject, and a given region was designated as the test region. An RDM was computed for this test region, and for all the regions in the left-out training subjects. The RDMs for the training subjects were then averaged to generate a single training RDM for each region. The similarity between the test RDM and each training RDM was computed, with a trial being tallied as “correct” if the test RDM was more similar to the training RDM of the corresponding region than it was to the training RDMs for any other region. This was repeated for all possible test subjects and test regions. This entire procedure was performed for different random draws of stimuli, varying the number of stimuli, and for different choices of analysis method. The manner in which stimuli were sampled varied across the three datasets in order to account for their varying structure. For all datasets, the methods being compared were 1) standard RSA, 2) Framed RSA including the stimulus patterns and 0 , 3) Framed RSA including the stimulus patterns, 0 , and c , and 4) the profile of mean activations. For all RSA methods, crossnobis was used as the dissimilarity estimator, with univariate noise normalization being applied. To examine how the benefits of Framed RSA might depend on the choice of RDM comparator, #1-3 were run using correlation, whitened correlation, cosine, or whitened cosine. For the whitened RDM comparators, an adjusted formula for V , the matrix of covariances among RDM entries, was used. For most analyses, c was set to have norm equal to the mean norm of the stimulus patterns, with the exception of an analysis in the NSD that compares region identification accuracy for different lengths of c . Region identification accuracy was then compared between methods within each comparator and number of stimuli (e.g., within 10 stimuli for whitened correlation), with this comparison method varying across datasets. The performance of standard RSA and Framed RSA (including 0 and c ) were also compared to the profile of mean activations, using the comparator that yielded the highest overall performance for these two methods. 1.5.1 Natural Scenes Dataset Region Identification The Natural Scenes Dataset 1 (NSD) is a 7T fMRI dataset with neural responses to natural scene images collected from eight subjects, with 30-40 scanning sessions per subject. Each subject viewed up to three repetitions of 10000 images, with 1000 of these images shared across subjects. Since the region identification paradigm requires common stimuli across subjects, analysis was restricted to these shared stimuli. These stimuli were further restricted to only those images that were viewed at least twice by all eight subjects (since computing crossnobis distance requires at least two trials per stimulus), yielding a final set of 766 stimuli. Draws of 10, 20, 50, 100, or 150 stimuli were used, using as many disjoint samples as possible (e.g., seven samples of 100, or 5 samples of 150). As regions of interest (ROIs), we used 14 retinotopic and category-selective regions: retinotopic regions V1, V2, V3, and hV4; the face-selective regions FFA and OFA; the body-selective regions EBA and FBA; the word-selective regions OWFA, VWFA, and mfs-words; and the place-selective regions OPA, PPA, and RSC. Activations from these regions were extracted using the volumetric masks provided in the NSD, with no further voxel selection performed. The NSD provides single-trial beta values computed using three different methods, of which we used the ‘b3’ variant, which allows the shape of the hemodynamic response function (HRF) to vary across voxels, and uses ridge regression and the GLMdenoise technique to improve the estimation of beta values. The cross-validated Mahalanobis (crossnobis) dissimilarity metric takes into account varying levels of noise between voxels if univariate noise normalization is applied, and further takes into account correlated noise between voxels if multivariate noise normalization is applied. Univariate noise normalization was applied for these analyses. The voxel noise covariance matrix was computed using beta values from the images that were unique to each subject (up to a maximum of 9000). We followed the approach of taking the differences between the single-trial beta values for each stimulus presentation and the mean beta across repetitions of that stimulus and using these differences to compute the covariance between voxels, rather than the alternative approach of using the timeseries residuals from the GLM. For the whitened RDM comparators, the dissimilarity covariance matrix V was computed using data from the held-out test subject. For the NSD, V was estimated using a slightly different procedure than for the other two datasets in order to account for noisy estimation of the stimulus patterns. Accuracy was compared between methods using matched-pairs t-tests within each number of stimuli and For the NSD, a further analysis was performed to examine the effects of varying the length of c . Specifically, c was set to have norm .5x, 1x, or 2x the mean norm of the stimulus patterns, with accuracy being compared within each RDM comparator and number of stimuli as with the other analyses. 1.5.2 Things Ventral Stream Spiking Dataset Region Identification The Things Ventral Stream Spiking Dataset (TVSD) contains neural recording data from V1, V4, and IT cortex of two macaque monkeys. 15 This dataset includes 20000 images that were each shown once to both macaques, and 100 images that were shown 30 times to each macaque. Due to the small number of images, for this dataset we used a bootstrap resampling procedure to estimate the variability of the accuracy estimates. Specifically, for each bootstrap sample we made 30 random draws with replacement from the 30 trials for each stimulus, stipulating that draws from first 15 trials were included in the first crossvalidation fold in computing the crossnobis distance, and draws from the second 15 trials were included in the second crossvalidation fold. Two hundred samples each of 3, 5, 10, 20, or 50 images (allowing for image repeats across samples, but not within samples) were then drawn, the pipeline was performed for each draw, and the accuracy was averaged across draws within each bin (i.e., number of stimuli and analysis variant). One hundred bootstrap samples were drawn in this manner. The channel noise covariance matrix was computed using the differences between the beta value for each trial and the mean beta across repetitions of the image for that trial, using all 30 trials from the set of 100 repeated images. As with the NSD, univariate noise normalization was applied. An important difference between the TVSD and the other two datasets is that there are only two “subjects” (versus eight for the NSD, and ten for the DNN dataset), such that there is no meaningful distinction between “training” and “test” subjects for each trial. To account for this, two adjustments were made to the analysis pipeline. First, accuracy was tallied in both possible “directions” on each trial. That is, whereas for the other two datasets accuracy was tallied as the proportion of trials where the held-out test RDM was more similar to the correct training RDM than to the other training RDMs (i.e., with the training and test subjects being treated asymmetrically), for the TVSD both directions were tallied. Thus, for each stimulus draw, a maximum of six “points” were possible: one point if Monkey 1’s V1 was more similar to Monkey 2’s V1 than to Monkey 2’s V4 and IT, another point if Monkey 2’s V1 was more similar to Monkey 1’s V1 than to Monkey 1’s V4 and IT (the opposite direction), and so on for the other possible comparisons. Second, for the whitened RDM comparators, instead of using the data from the “test” subject to compute V (the matrix of covariances among RDM entries) as with the other two datasets, V was computed for both monkeys, the RDM similarity was separately computed using the two estimates of V, and the two resulting similarity scores were averaged to derive a final score. Matched-pairs t-tests were then used to compare accuracy between conditions, where the standard error was the standard deviation of the 100 bootstrap samples. 1.5.3 Deep Neural Network Layer Identification In the third dataset, we extend the region identification paradigm to deep neural network layers. The analysis follows the same logic, replacing brain regions with specific DNN layers, and replacing separate subjects with DNNs trained from different random seeds. Beyond being of interest in its own right, DNNs offer several further benefits. First, they are “glass-box” systems whose ground-truth activations can be known with full precision. Second, while naturally noise-free, varying levels of noise can be simulated at the experimenter’s discretion, allowing the performance of Framed RSA at varying levels of noise to be characterized. Ten instances of AlexNet trained on ImageNet from different random seeds were used, with activations extracted from all layers from the second pooling layer onward, for 13 layers in total. Disjoint sets of images were randomly drawn from MSCOCO, with a sample size of 3, 5, 10, 20, or 50 stimuli. Noise was added to the activations from each layer using the following procedure. In order to roughly equate the relative noise magnitude across layers, for each stimulus sample the grand standard deviation across stimuli and channels was computed, and isotropic noise with standard deviation set to either 1x or 10x the activation standard deviation was added to all units. Two “trials” per image were drawn in this manner. Matched-pairs t-test were used to compare accuracy between analysis methods. 2 Results A region identification paradigm was used as a proxy task to evaluate how powerfully Framed RSA could adjudicate between different computational models. Since Framed RSA includes information about both the representational geometry and regional mean activations of a neural population, its performance was also compared to standard RSA and to the profile of mean activations, in order to characterize how it might perform relative to either kind of information in isolation. This approach was applied across three datasets. 2.1 Natural Scenes Dataset Region identification results for the Natural Scenes Dataset are shown in Fig. 2 . Framed RSA significantly outperforms standard RSA at all stimulus sample sizes when whitened correlation or whitened cosine were used as RDM comparators, with weaker or mixed results when non-whitened correlation or cosine were used as RDM comparators. Framed RSA outperformed the profile of mean activations for small numbers of stimuli, but was significantly outperformed at higher numbers of stimuli. Download figure Open in new tab Figure 2: Region identification results for the Natural Scenes Dataset. (A) Region identification accuracy for Framed RSA, standard RSA, and profile of mean activations for different RDM comparators. Significant pairwise contrasts (matched-pairs t-tests; p < .05) are denoted using the icons above each datapoint. To compare RSA variants to the profile of mean activation, the results for Framed and Standard RSA with their highest-performing comparator are re-plotted on the mean profile panel. (B) Region identification performance results when the length of c is varied. Next, we examined the effects of varying the ratio of the norm of c relative to the mean norm of the stimulus vectors. We found that a ratio of .5 or 1 outperformed a ratio of 2 across all RDM comparators, with little difference between .5 or 1. 2.2 Things Ventral Stream Spiking Dataset Region identification results for the Things Ventral Stream Spiking Dataset are shown in Fig. 3 . Framed RSA yields higher region identification performance than standard RSA at low numbers of stimuli across all RDM comparators. For high numbers of stimuli, Framed RSA performs similarly to standard RSA when unwhitened correlation or cosine are used as RDM comparators, but lower performance when their whitened variants are used. Framed RSA outperforms the profile of mean activations for all numbers of stimuli. Download figure Open in new tab Figure 3: Region identification results for the TVSD. Significant pairwise comparisons (matched pairs t-tests, p < .05) are denoted with the icons above each datapoint. To compare the accuracy for the profile of mean activations to standard and Framed RSA, these analyses are re-plotted on the Mean Profile panel, using the RDM comparator that yielded the highest overall accuracy. 2.3 DNN Layer Identification DNN layer identification results are shown in Fig. 4 , at low ( A ) and high ( B ) levels of noise. At low levels of noise, Framed RSA significantly outperforms standard RSA and the profile of mean activations at all numbers of stimuli, and for all RDM comparators. At high levels of noise, Framed RSA substantially outperforms standard RSA across all comparators and numbers of stimuli. However, while it outperforms the profile of mean activations when few stimuli are used, at higher numbers of stimuli the profile of mean activations yields higher accuracy. Download figure Open in new tab Figure 4: DNN layer identification performance results for low (A) and high (B) levels of noise. Significant pairwise comparisons (matched pairs t-tests, p < .05) are denoted with the icons above each datapoint. To compare the accuracy for the profile of mean activations to standard and Framed RSA, these analyses are re-plotted on the Mean Profile panel, using the RDM comparator that yielded the highest overall accuracy. 3 Discussion Comparing computational systems requires a choice of which aspects of the system to compare and which to disregard. Here we introduce Framed RSA, which combines information about both the representational geometry and the regional mean stimulus preferences of a neural population through the simple approach of adding the origin and a constant pattern to the geometry, providing a “reference frame” for the global location and orientation of the stimulus patterns. A region identification paradigm was used to compare the model adjudication performance of Framed RSA relative to standard RSA or the profile of mean activations across three datasets. For the Natural Scenes fMRI dataset, Framed RSA either matched or outperformed Standard RSA across all RDM comparators or numbers of stimuli. Interestingly, however, when many stimuli were used the profile of mean activations performed best of all. One interpretation of these results is that the regional mean is relatively robust to noise, since averaging across voxels will tend to cancel out the effects of noise. We additionally found that Framed RSA is not entirely robust to the choice for the norm of c : making c too long relative to the stimulus vectors tended to greatly reduce performance. For a macaque neural recording dataset (the TVSD), Framed RSA only offered improved performance relative to standard RSA for low numbers of stimuli. Given the lower levels of noise in neural recording data relative to fMRI, it is possible that the regional mean activation adds little extra information relative to the fairly fine-grained information already present in the neural population patterns. It was outperformed by standard RSA for higher numbers of stimuli when whitened correlation or whitened cosine were used as RDM comparators, the only such case among any of the three datasets. For the DNN dataset, Framed RSA outperformed standard RSA at both low and high levels of noise, and outperformed the profile of mean activations at low levels of noise, but not at high levels of noise. These results corroborate the interpretation that the regional mean response profile may be especially diagnostic at higher levels of noise. In conclusion, Framed RSA offers a simple approach for incorporating both representational geometry and regional mean information into model comparison, offering potential benefits for more powerful model comparison, especially for measurement modalities in which high levels of noise are present. 4 References [1]. ↵ Emily J Allen et al. “ A massive 7T fMRI dataset to bridge cognitive neuroscience and artificial intel-ligence ”. In: Nature neuroscience 25 . 1 ( 2022 ), pp. 116 – 126 . OpenUrl [2]. ↵ Marc N Coutanche . “ Distinguishing multi-voxel patterns and mean activation: why, how, and what does it tell us? ” In: Cognitive, Affective, & Behavioral Neuroscience 13 ( 2013 ), pp. 667 – 673 . OpenUrl [3]. ↵ Jörn Diedrichsen et al. “ Comparing representational geometries using the unbiased distance correlation ”. In: arXiv preprint arxiv: 2007.02789 ( 2020 ). [4]. ↵ Nancy Kanwisher and Galit Yovel . “ The fusiform face area: a cortical region specialized for the perception of faces ”. In: Philosophical Transactions of the Royal Society B: Biological Sciences 361 . 1476 ( 2006 ), pp. 2109 – 2128 . OpenUrl [5]. ↵ Seyed-Mahdi Khaligh-Razavi and Nikolaus Kriegeskorte . “ Deep supervised, but not unsupervised, models may explain IT cortical representation ”. In: PLoS computational biology 10 . 11 ( 2014 ), e1003915 . OpenUrl [6]. ↵ Simon Kornblith et al. “ Similarity of neural network representations revisited ”. In: International conference on machine learning. PMLR . 2019 , pp. 3519 – 3529 . [7]. ↵ Nikolaus Kriegeskorte . “ Deep neural networks: a new framework for modeling biological vision and brain information processing ”. In: Annual review of vision science 1 . 1 ( 2015 ), pp. 417 – 446 . OpenUrl CrossRef [8]. ↵ Nikolaus Kriegeskorte and Jörn Diedrichsen . “ Peeling the onion of brain representations ”. In: Annual review of neuroscience 42 . 1 ( 2019 ), pp. 407 – 432 . OpenUrl [9]. ↵ Nikolaus Kriegeskorte , Marieke Mur , and Peter A Bandettini . “ Representational similarity analysisconnecting the branches of systems neuroscience ”. In: Frontiers in systems neuroscience 2 ( 2008 ), p. 4 . OpenUrl [10]. ↵ Nikolaus Kriegeskorte and Xue-Xin Wei . “ Neural tuning and representational geometry ”. In: Nature Reviews Neuroscience 22 . 11 ( 2021 ), pp. 703 – 718 . OpenUrl [11]. ↵ Nikolaus Kriegeskorte et al. “ Matching categorical object representations in inferior temporal cortex of man and monkey ”. In: Neuron 60 . 6 ( 2008 ), pp. 1126 – 1141 . OpenUrl [12]. ↵ Alex Krizhevsky , Ilya Sutskever , and Geoffrey E Hinton . “ Imagenet classification with deep convolutional neural networks ”. In: Advances in neural information processing systems 25 ( 2012 ), pp. 1097 – 1105 . OpenUrl [13]. ↵ Johannes Mehrer et al. “ Individual differences among deep neural network models ”. In: Nature communications 11 . 1 ( 2020 ), p. 5725 . OpenUrl [14]. ↵ Hamed Nili et al. “ A toolbox for representational similarity analysis ”. In: PLoS computational biology 10 . 4 ( 2014 ), e1003553 . OpenUrl [15]. ↵ Paolo Papale et al. “ An extensive dataset of spiking activity to reveal the syntax of the ventral stream ”. In: Neuron ( 2025 ). [16]. ↵ Mattia Rigotti et al. “ The importance of mixed selectivity in complex cognitive tasks ”. In: Nature 497.7451 ( 2013 ), pp. 585 – 590 . [17]. ↵ Heiko H Schütt et al. “ Statistical inference on representational geometries ”. In: Elife 12 ( 2023 ), e82566 . OpenUrl [18]. ↵ Ilia Sucholutsky et al. “ Getting aligned on representational alignment ”. In: arXiv preprint arxiv: 2310.13018 ( 2023 ). [19]. ↵ Maryam Vaziri-Pashkam and Yaoda Xu . “ An information-driven 2-pathway characterization of occipitotemporal and posterior parietal visual object representations ”. In: Cerebral Cortex 29 . 5 ( 2019 ), pp. 2034 – 2050 . OpenUrl [20]. ↵ Alex H Williams . “ Equivalence between representational similarity analysis, centered kernel alignment, and canonical correlations analysis ”. In: bioRxiv ( 2024 ), pp. 2024 – 10 . [21]. ↵ Daniel LK Yamins and James J DiCarlo . “ Using goal-driven deep learning models to understand sensory cortex ”. In: Nature neuroscience 19 . 3 ( 2016 ), pp. 356 – 365 . OpenUrl CrossRef [22]. ↵ Daniel LK Yamins et al. “ Performance-optimized hierarchical models predict neural responses in higher visual cortex ”. In: Proceedings of the national academy of sciences 111 . 23 ( 2014 ), pp. 8619 – 8624 . OpenUrl View the discussion thread. Back to top Previous Next Posted July 16, 2025. Download PDF Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Framed RSA: Representational comparisons that honor both geometry and population-mean response preferences Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Framed RSA: Representational comparisons that honor both geometry and population-mean response preferences JohnMark Taylor , Nikolaus Kriegeskorte bioRxiv 2025.07.10.664257; doi: https://doi.org/10.1101/2025.07.10.664257 Share This Article: Copy Citation Tools Framed RSA: Representational comparisons that honor both geometry and population-mean response preferences JohnMark Taylor , Nikolaus Kriegeskorte bioRxiv 2025.07.10.664257; doi: https://doi.org/10.1101/2025.07.10.664257 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Neuroscience Subject Areas All Articles Animal Behavior and Cognition (7636) Biochemistry (17703) Bioengineering (13897) Bioinformatics (41960) Biophysics (21458) Cancer Biology (18597) Cell Biology (25524) Clinical Trials (138) Developmental Biology (13381) Ecology (19905) Epidemiology (2067) Evolutionary Biology (24325) Genetics (15612) Genomics (22512) Immunology (17738) Microbiology (40421) Molecular Biology (17187) Neuroscience (88627) Paleontology (667) Pathology (2834) Pharmacology and Toxicology (4825) Physiology (7645) Plant Biology (15158) Scientific Communication and Education (2046) Synthetic Biology (4302) Systems Biology (9825) Zoology (2271)