Robustness in Hopfield neural networks with biased memory patterns

Robustness in Hopfield neural networks with biased memory patterns | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Robustness in Hopfield neural networks with biased memory patterns Ali Montazeri , View ORCID Profile Robert Schmidt doi: https://doi.org/10.1101/2024.10.31.621250 Ali Montazeri 1 Independent researcher Find this author on Google Scholar Find this author on PubMed Search for this author on this site Robert Schmidt 2 Institute for Neural Computation, Faculty of Computer Science, Ruhr-Universität Bochum Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Robert Schmidt For correspondence: robert.schmidt{at}ruhr-uni-bochum.de Abstract Full Text Info/History Metrics Preview PDF Abstract Biological neural networks are able to store and retrieve patterns in the presence of different types of noise. Hopfield neural networks have been inspired by biological neural networks and provide a model for auto-associative memory patterns. An important parameter in these networks is the pattern bias, i.e. the mean activity level of the network, which is closely related to the degree of sparseness in the coding scheme. Here we studied the relation between robustness against different types of biologically-motivated noise and pattern bias. To do so, we developed performance and robustness measures, which are applicable to varying degrees of sparseness of the memory representations, using analytically-optimized thresholds and corruption tolerances adjusted by mutual information. We then applied these tools in numerical simulations and found that, for different types of noise, the pattern load, i.e. the number of patterns that the network has to store, determined which pattern bias is most robust. Across different types of noise, the higher the pattern load was, the more biased was the most robust performing pattern representation. Given the variation in the sparseness level in different brain regions, our findings suggest that memory pattern encoding schemes (i.e. degree of sparseness) in different brain regions might be adapted to the expected memory load in order to best mitigate the adverse effects of disruptions. 1 Introduction One characteristic of memory in biological organisms is the robust performance under noisy conditions ( Rolls & Deco, 2010 ). Hopfield neural networks have been inspired by biological neural networks and provide a model for auto-associative memories ( Hopfield, 1982 ). The effect of noise on retrieval and capacity in Hopfield networks has been of high interest already in the early seminal studies of the Hopfield network properties ( Amit et al., 1987 ), and is relevant as a computational model of memory systems in biological organisms. Furthermore, understanding robustness against noise is relevant for neurological disorders such as Alzheimer’s disease, in which, during the preclinical phase, memories seem to endure pathological alterations ( Long & Holtzman, 2019 ). Noise can be introduced in Hopfield networks on different levels. In analogy to statistical mechanics of physical systems, a common model for noise is to consider thermal fluctuations of the activation function, whose strength is described by a temperature parameter. As a model for auto-associative memories in the brain, the temperature parameter, or an equivalent stochastic threshold ( Hertz et al., 1991 ), captures some aspects of the random variability in neural activity. However, in biological neural networks there are also other types and sources of noise ( Faisal et al., 2008 ). For example, structural noise includes random variation of the connection weights (e.g. due to variations in post-synaptic potentials; Ribrault et al., 2011 ), but also random loss of connections or whole neurons (e.g. due to injury or neurodegenerative diseases). In addition, the initial pattern presented to the network for retrieval can be subject to noise, so that for a random subset of units the states are flipped when a stored pattern is presented. Optogenetic methods in neuroscience ( Boyden et al., 2005 ) also motivate to consider a variation of this type of noise, so that a random set of units is just collectively activated or deactivated, irrespective of the actual pattern. From a biological perspective, Hopfield networks, as a model for auto-associative memories in the brain, should exhibit robustness against the biologically-motivated types of noise. In addition to noise, also the coding schemes employed by Hopfield networks have been studied especially in the context of memory capacity. In the classic Hopfield network memory patterns are unbiased, i.e. that a given pattern activates 50% of the network nodes. Subsequent work demonstrated the ability of Hopfield networks to also store biased patterns, i.e. patterns that activate less than 50% of the network nodes ( Amit et al., 1987 ). Biased, or sparse, patterns are also common in biological neural networks, and it turns out that also Hopfield networks can increase their storage capacity for biased patterns ( Tsodyks & Feigel’man, 1988 ). However, dealing with biased patterns requires a modification of the Hopfield network learning rule as biased patterns otherwise increase spurious states and thereby degrade performance. Another aspect of network performance that goes beyond storage capacity is robustness. The robustness of a Hopfield network relates to the dynamical properties of the network. In the energy landscape the desired stable states form basins of attraction. The relative size and energy levels of the basins of attraction of the desired stable states in comparison to those of the spurious attractors determine the chances for a successful memory retrieval ( Rolls & Deco, 2010 ). This is schematically illustrated in Figure 1A&B , where the storage capacities are similar but the attractors associated with the memory states in A have larger and deeper basins of attraction than in B. Therefore, even when starting from the desired stable states, the behaviour of the two networks will be different in the presence of noise. Download figure Open in new tab Figure 1: Energy landscapes and pattern bias. (A, B) Schematic illustration of the energy landscape of two networks with differences in the size and depth of the basins of attraction. The memory states’ attractors are shown by dots. When starting in a stable state, noise might push the network in (B) into a local minimum, while network in (A) would remain in the desired state. (C) Examples of memory pattern configurations from very sparse (left) to unbiased (right) representations. The network has 900 (30×30) nodes. The white and black squares represent 1 and 0 states, respectively. Here we propose and examine a measure of robustness in Hopfield networks to store biased and unbiased patterns. We found for different types of noise that the most robust pattern bias depended on the pattern load. This indicates that there is a trade-off between memory load and robustness so that the memory system should employ a coding scheme, i.e. a level of pattern bias, that depends on the expected pattern load. We propose that this might be relevant for biological networks, so that coding schemes employed in different brain regions might be tuned towards optimal performance based on the expected memory load in those regions. 2 Methods 2.1 The Hopfield network The attractor neural network proposed by Hopfield ( Hopfield, 1982 ) accounts for several characteristics of auto-associative memories including pattern completion and memory retrieval from an incomplete or flawed clue. In this network, the N nodes are McCulloch-Pitts neurons, which are either in a (quiescent) 0 state or in an (active) 1 state. For each neuron i , given the symmetric connection strengths W ij from neurons j to i , the state variable V i is updated by the following dynamical rule where h i is the neuron’s local field, and H is the Heaviside step function with the value of 0 for negative arguments and 1 for positive arguments. In the classic model the update is carried out asynchronously, so that units are updated one at a time. The goal is to store a set of M binary input patterns {ξ µ , 1 ≤ µ ≤ M } via adjustments to the connection weights {W ij , 1 ≤ i, j ≤ N } , such that the desired memories become the network’s stable states. The weights are determined based on Hebbian correlation learning, i.e. In standard Hopfield networks there are symmetric recurrent connections between all nodes (but no self-coupling, i.e. W ii = 0). For unbiased patterns, each is a (fixed) random and independent Bernoulli bit with a mean level of activity p of 50%, i.e. The randomness and independence of bits bring mathematical convenience for studying the network’s characteristics, for instance in the signal-to-noise analysis of the model. An important contribution of Hopfield (1982) was to introduce the notion of an energy function into neural networks: The activity update dynamics of the system ensure that the energy function always decreases or remains constant. The stable states or attractors of the network are the local minima of the energy surface, and the “basins of attraction” correspond to the valleys around each minimum ( Hertz et al. 1991 ; Figure 1A&B ). For the stored patterns to be retrieved successfully from an incomplete or distorted stimulus, they must coincide with the network’s attractors (i.e. the local minima of the energy function). Thereby, an initial network configuration, with bit states being different from the original pattern, is updated step by step towards the stored pattern, where no further changes take place. However, the energy landscape has also local minima corresponding to “spurious” states, that can interfere with the memory recall. The storage capacity of the network is the maximal number of stored patterns M , for which a retrieval is possible. The storage capacity of a Hopfield network can be expressed using the load parameter α = M/N ( MacKay, 2003 ) that takes into account the network size N . Small deviations between the stored and retrieved pattern are considered acceptable and increase the storage capacity considerably ( MacKay, 2003 ). To measure the distance between a stored and a retrieved pattern we use the (normalized) Hamming distance, which calculates the ratio of bits that are different between two binary configurations ([0, 1]). 2.2 The Hopfield model for biased patterns The original Hopfield model works well for uncorrelated patterns when the Bernoulli probability p in Eq. 3 is 50%. Memory patterns are called biased if the mean neural activity p in Eq. 3 deviates from 0.5, and we refer to them here as sparse if p ≪ 0.5. Schematic examples of memory patterns with different levels of bias are shown in Figure 1C . In order to store biased patterns, the Hopfield model must be modified to avoid spurious states from becoming dominant. The learning rule is adjusted as follows ( Amit et al., 1987 ): In addition, the Heaviside step function in the dynamical rule ( Eq. 1 ) is replaced by an optimized activation threshold (the same for all nodes), which keeps the capacity in the order of as p → 0. ( Gardner, 1988 ; Tsodyks & Feigel’man, 1988 ; Vicente & Amit, 1989 ). For a given p , the optimized activation threshold can be obtained by a signal-to-noise analysis of the network ( Vicente & Amit, 1989 ). Given that the network state is a desired memory pattern υ , i.e. , the local field h i can be decomposed into a signal part, which reinforces the desired memory, and a remaining quasi-random cross-talk noise due to interference from other stored patterns. The derivation of the signal and noise parts is provided in Appendix A (see also Buhmann et al., 1988). Assuming a uniform activation threshold θ , the derivation results in the local field h i to have the signal parts: The cross-talk noise has a zero mean with the variance: Hence, the optimal threshold corresponds to the average of the expressions (6) and (7) being zero, which yields: The signal and noise parts are shown schematically in the Figure 2A as mean values and standard deviations of the normally-distributed local fields, respectively. The optimal threshold shifts the two distributions, so that the absolute value of their means is identical (see example in Figure 2B ). The optimal threshold has a concave shape as a function of p and there are upper and lower bounds for θ to keep the averages for and on their current side of the x-axis ( Figure 2C ). Download figure Open in new tab Figure 2: Optimal thresholds and tolerance for different pattern bias. (A, B) The probability density functions for the local field h i are given by the signal part (the mean of the Gaussians; dashed black lines) and a cross-talk noise part (the standard deviation), when the bit of the stored pattern is either 0 or 1. Without an activation threshold (A) the mean lies below 0 for , but due to the noise part the distribution extends to positive values, which would lead to an erroneous activation according to Eq. 1 . Introducing an optimal activation threshold θ shifts the means of the distributions so that the unit is active only for and not for (B). (C) The optimal threshold varies for different levels of mean activity. The upper and lower bounds of the thresholds ensure that the means of the distributions (B) do not have the same sign. (D) For different levels of pattern bias, tolerances, i.e. corruption levels (in Hamming distance) that corresponded to 88% normalized mutual information, were determined. The capacity of the original Hopfield network increases by a factor of 2 if the so-called S-variables ∈ {± 1 } instead of V-variables ∈ { 0, 1 } are used ( Hopfield, 1982 ). Given the linear transformation between the two representations, i.e. S i = 2 V i − 1, the S-model is equivalent to the V-model with the random thresholds Similarly, for the biased memory patterns, if the “random local fields” p Σ j W ij are added to each node i of the V-model, the variance of the cross-talk noise improves from αp 3 (1− p ) 2 to αp 3 (1 − p ) 3 (see appendix B for details). The application of this random local threshold is equivalent to the optimized model from a continuous family of models discussed by Vicente and Amit (1989) . In summary, the resulting optimal signal-to-noise ratio with both a uniform external field θ opt which optimizes the signal, and the random local fields p Σ j W ij which correlate with the patterns and reduce the cross-talk noise is Thus, as expected, the recalling behavior improves for sparse patterns. The modified dynamics of our model are with the following energy function ( Vicente & Amit, 1989 ) 2.3 Noise in the network The conventional way to add noise to the neural network (or temperature to the Ising model) is to make the update rule for units stochastic by introducing the logistic function f β , so that an activation state V is taken with probability P : is the total field equal to h i − θ − p Σ j≠i W ij for the dynamics in Eq. 10 , and the parameter β relates to the pseudo-temperature T by The parameter T controls the steepness of the sigmoid function near 0. The above-mentioned noise is equivalent to adding a stochastic threshold θ stoch , drawn from the probability density , to the deterministic model ( Hertz et al., 1991 ). The ability of Hopfield networks to robustly retrieve stored patterns and work as a memory device depends on the regime in the T- α plane. For a given temperature, networks with small load parameters have attractors associated with the stored memory patterns. In this regime spurious states still exist, and can take the form of mixture states (Amit et al., 1985a) or states uncorrelated to to any of the stored patterns. If the load parameter or temperature is increased, those states start to dominate the network. While there is a temperature range in which the efficiency of the system is improved (Amit et al., 1985a), the network becomes no longer useful as a memory device if, given a temperature T , a critical load parameter α c ( T ) is exceeded, or if the temperature is increased to an overly high value ( Hertz et al., 1991 ). The critical load parameter in the noiseless original Hopfield network α c (0) was found to be almost 0.138 (Amit et al., 1985b), and increases superlinearly for a model with biased patterns and the dynamics of Eq. (10) ( Vicente & Amit, 1989 ). While the noise in the form of the logistic function and temperature as in Eq. (12) is relevant in statistical mechanics, there are other types and sources of noise in biological neural networks ( Faisal et al., 2008 ). Therefore, we applied the following different types of noise separately to the network to investigate the model’s robustness against them: Stochastic threshold. A stochastic threshold was added to the model as described above. The pseudo-temperature in the logistic function of Eq. 12 ranged from 0 to 1 / 8, which corresponds to a range of 0 to 1 using S-variables. Weight noise. Weight noise was implemented by adding Gaussian noise to the weight matrix. This additive noise was generated from the normal distribution with a standard deviation between 0 (no weight noise) and 0.002. For comparison, the highest standard deviation that was used corresponds almost to the standard deviation in the weight matrix of the model with unbiased patterns, which was the highest among the different levels of p . The self-coupling terms W ii were still set to 0. Connection dilution. Connection dilution was incorporated in the model by setting a random set of weights in the weight matrix to 0. The probability of diluting an individual, recurrent connection ranged from 0% to 100%. Node dilution. For node dilution a random set of nodes was completely removed from the model. The fraction of deleted nodes ranged from 0% to 90%. Thereby, in few cases a pattern may even be completely removed, or no longer distinguishable from another one, so its retrieval failed automatically. States turning active. To implement this new type of noise a random set of states of a stored memory was set to 1, irrespective of whether they were 0 or 1 in the stored pattern, to produce the initial configuration for the simulation. A range of 0 to N states are chosen for this distortion. States turning quiescent. In correspondence to E) this noise type was implemented by setting a random set of states of a stored memory 0 to produce the initial configuration for the simulation. A range of 0 to N states were chosen for this distortion. State flipping noise. Finally, this commonly used type of noise was implemented by flipping a random set of states of a stored memory from 0 to 1 and vice versa to produce the initial configuration for the simulation. A range of 0 to N states are chosen for this distortion. Since n flips of the states of a memory pattern results exactly in n/N Hamming distance, this form of distortion is widely used in the literature. 2.4 Patterns The number of nodes N , or pattern size, was set to 5000. A set of M random patterns with independent Bernoulli bits was generated, each with a mean level of activity p determining the pattern bias: The pattern bias, or sparseness level, given by p ranged as follows (with 50% corresponding to unbiased patterns and 1% to extremely sparse patterns): The number of patterns to store was also systematically varied with the load parameter α ranging from 5% to 50%. 2.5 Simulation procedure For each choice of p and α and for each experiment, M random patterns were generated, and the weight matrix was determined according to the modified Hebbian learning rule ( Eq. 5 ). The optimal uniform threshold θ opt was set according to Eq. 8 and the random local fields p Σ j W ij were then obtained accordingly. Next, each type of noise was investigated separately. For noise that affected the network dynamics (types A-D, see above), the network was initialized with one of the memory patterns at a time. For the other noise types (E-G) the network was instead initialized using the distorted versions of the memory patterns. In all cases synchronous dynamics were used, so that all units were updated simultaneously, as there is no significant change of results in comparison to the asynchronous updating ( Hertz et al., 1991 ), but an enormous computational advantage. The maximum number of update steps was set to 50, which was sufficient to reach a stable state. Afterwards, the Hamming distance between the original pattern and the settled network’s state was calculated. For the case of node dilution, the Hamming distance was calculated based on the remaining nodes. For each type and magnitude of noise, 100 experiments were performed. To keep the runtime of the relatively large networks feasible, PyTorch Tensor computing with a graphics processing unit (GPU) acceleration was used. 2.6 Performance and Robustness measures The performance of Hopfield networks can be assessed based on the critical value α c ( T ), for which there is a discontinuous jump in Hamming distance between stored and retrieved patterns. For a network just below this critical value and with unbiased patterns, the Hamming distance between a desired memory and its associated attractor is 1.6% ( MacKay, 2003 ). This value is called “corruption tolerance” for a successful memory retrieval, and depends on the pattern bias. To obtain a comparable tolerance level for the different values of the pattern bias, the normalized mutual information between memory patterns and their corrupted versions was used. This is necessary for the comparison because the total information is maximized when p = 50% and reduces as the pattern representation becomes sparser. The normalized mutual information between patterns X and Y determines the fraction of information in X that is contained in Y ( Cover & Thomas, 1991 ), and is given by: where H ( X ) is the entropy and H ( X | Y ) the conditional entropy. In our case X and Y can be considered as two random variables, which assume the value 0 or 1. Thus, there are four possible events X = i, Y = j with probabilities p ij that sum to 1. The two marginal distributions are given for X by p X =0 = p 00 + p 01 and p X =1 = p 10 + p 11 , and for Y by p Y =0 = p 00 + p 10 and p Y =1 = p 01 + p 11 . Hence, the general definitions of H ( X ) and H ( X | Y ) reduce to To find the relationship between the level of corruption (in Hamming distance) and the normalized mutual information, numerical simulations were used. For unbiased patterns, the mutual information associated with around 1.6% error was 0.88 bits. Therefore, we used this value for calibration purposes to find tolerances yielding comparable corruption levels for biased representations ( Figure 2D ). These tolerances were then used to measure the network’s performance as the “hit ratio”, i.e. the ratio of retrieved patterns with a smaller Hamming distance than the tolerance level. The robustness measure was based on the relation between noise levels and hit ratio. In general, increasing noise levels eventually lead to a break down of the network performance and the hit ratio rapidly drops to zero. For each type of noise, the robustness was therefore measured by calculating the area under the curve (AUC) of the hit ratio versus the entire noise range, normalized by the maximum possible area. The normalized AUC is a non-parametric measure which aggregates the performance of the network against a certain noise type. 3 Results The goal of our study was to determine the relation between the robustness of the Hopfield network and pattern bias for varying memory loads. To do so we first confirmed the optimality of the analytically-derived thresholds, which then were used in the subsequent simulations. Next, we examined the networks’ load capacity in terms of the hit ratio and the Hamming distance, which form the basis for our robustness measure. Finally, we applied our robustness measure to test how different types of noise affect the network performance depending on the pattern bias. 3.1 Optimal activation thresholds To calculate the optimal activation thresholds we used the signal-to-noise ratio analysis ( Eq. 8 ; see also Figure 2 ). We verified this approach numerically by running simulations for different values of thresholds θ , for each level of pattern bias. We found that the analytically-derived optimal thresholds indeed coincided with the maximum hit ratio of 1 in all cases, as exemplified in Figure 3A for α = 0.1 (i.e. 500 patterns). The range of thresholds for which the network showed highest performance decreased for sparse patterns, as expected from the decreasing distance between the upper and lower bounds ( Figure 2C ). Download figure Open in new tab Figure 3: Network performance and capacity. (A) For different levels of pattern bias (y-axis), the performance of different thresholds was tested in numerical simulations (here α = 0.1). The black lines indicate the range of thresholds for which the hit ratio was 1 (outside this range the hit ratio quickly degraded to 0). The red vertical lines indicate the analytically-derived optimal thresholds for comparison, which in all cases was well within the range of optimal performance. (B) Hit ratios for different load parameters, depicted for various levels of p (see legend). All models show optimal performance with a hit ratio of 1 for low values of α , but undergo a fast transition to a non-retrieval network at different, higher load values. (C) Histograms of Hamming distances for p = 5% (left panel) and p = 50% (right panel) and for five different load parameters around the transition from a fully functioning to a non-retrieval network. (D) 90% percentile of Hamming distance for different load parameters and for p = 5% (left) and p = 50% (right). The red dashed lines show the distances and load parameters of the points with maximum curvature. 3.2 Network capacity To determine the capacity of the networks we examined the hit ratio as a function of the load parameter for different levels of pattern bias. We found that as the pattern bias decreased, the transition point from a hit ratio of 1 to 0 occurred for higher values of the load parameter, meaning that the network’s capacity increased ( Figure 3B ). The transition from high to low hit ratios also became slower for the sparser patterns (see e.g. p = 2%). Only the 1% sparsely coded networks could consistently store more patterns than the number of nodes, i.e. α > 1 still yielded hit ratios of 1. The capacity of the network for unbiased patterns was between 0.13 and 0.15, in agreement with the results from the replica theory (Amit et al., 1985b). The higher load capacity for sparse patterns, as confirmed by the numerical simulations, points to the question whether this higher capacity also translates to a higher robustness of biased patterns more generally. Before looking at robustness, we first studied the behavior of networks at the edge of their breakdown in performance in more detail. To do so, we examined the distributions of Hamming distances between the desired memory patterns and the settled stable states for the cases of p = 5% and p = 50% ( Figure 3C ). For load parameters considerably below the critical value α c , all simulations produced zero or negligible Hamming distances, indicating successful retrievals. However, when surpassing the network’s capacity, the stored patterns were not anymore the point attractors of the network and large distances between the memory patterns and the settled states emerged. The shifts between these two modes of the histograms were very rapid (compare e.g. α = 0.11 and α = 0.17 in Figure 3C ). To capture the behavior of the network as the loading exceeds the capacity, we took the 90-th percentile of Hamming distances for a varying load α for p = 5% and p = 50% as examples ( Figure 3D ). For each we found an abrupt jump in the Hamming distances, confirming a sudden breakdown in performance. The Hamming distance from which a quick departure took place was considerably higher for unbiased patterns. This indicates some corruption of the pattern retrieval occurring already before the overall performance breakdown, potentially related to a difference in robustness between unbiased and bias patterns. 3.3 Measuring performance and robustness To determine how different types of noise affected the load capacity and hit ratio depending on the pattern bias, we used a robustness measure that incorporated a sparseness-dependent tolerance based on mutual information (see Methods). The robustness measure thereby provided an aggregated measure of the network’s behavior, which we evaluated for each noise type, as the normalized AUC of the hit ratios ( Figure 4 ). Download figure Open in new tab Figure 4: Robustness of the Hopfield networks against various kinds of disruptions. (A-G) Each panel shows the robustness measure for different pattern bias and load parameters (colored lines) against the indicated noise type. (H) Aggregated results indicating the most robust pattern bias for each pattern load and noise type. In the simulations with a stochastic threshold, equivalent to noise from the temperature parameter, unbiased patterns ( p = 50%) had the highest robustness for settings with a low pattern load ( α = 0.05; Figure 4A ). In contrast, for high pattern loads ( α ≥ 0.4), networks with both biased and unbiased patterns exhibited very low robustness. However, at intermediate levels of pattern load, networks using intermediate levels of pattern bias were most robust. This indicates that the pattern load that a networks receives determines which pattern representation is most suitable for retrieval of memory patterns in light of noise. The simulations using weight noise yielded very similar results ( Figure 4B ), with the only difference being a slight shift in the robustness values, while the relation between pattern load and robustness of the pattern bias remained the same. Therefore, both types of noise, unbiased patterns were only robust for low pattern loads, while biased patterns were more robust with increasing pattern loads. The noise types involving connection and node dilution yielded overall qualitatively similar results, even though sparser networks were more robust here in general, irrespective of the pattern load ( Figure 4C&D ). Therefore, there was a more abrupt shift, so that a slight increase in pattern load lead to the sparse patterns being most robust. For the three noise types that involved changes in the patterns presented during retrieval (state flips, states turning active, and state turning quiescent), sparse patterns again exhibited a robustness advantage for higher loads ( Figure 4E-G ). With a gradual increase in pattern load, there was a gradual increase in the pattern bias that was most robust. To summarize the relation between the pattern load of a network with the pattern bias that performs most robust against different types of noise, we determined for each pattern load the pattern bias p that was associated with the highest robustness. For example, for the stochastic threshold, α = 0.15 had the peak robustness for a mean activity p = 20%. The aggregated results illustrated that with increasing pattern loads, an increase in patterns bias ensures the most robust performance ( Figure 4H ). For connection and node dilution, there is an abrupt change in more robust performance of biased patterns. For the other noise types the change is more gradual, so that intermediate levels of pattern bias perform most robust for intermediate pattern loads. We conclude that sparse patterns can be more robust against different types of noise compared to unbiased patterns in Hopfield networks. Importantly, for most robust performance, the pattern bias should be chosen according to the expected pattern load. 4 Discussion We studied the robustness of pattern retrieval in Hopfield networks for seven different types of biologically-motivated noise. To do so, we first analytically determined optimal thresholds and tolerance levels for different levels of pattern bias. Then we studied the retrieval performance as a function of the pattern load and the pattern bias to see what happens in the regime where the retrieval performance breaks down. Finally, we applied a robustness measure, which aggregates the hit ratio over a range of noise strengths, and found that different pattern loads required different levels of pattern bias for most robust performance. Overall, the higher the pattern load, the more biased the most robust pattern representation had to be. To measure the pattern retrieval performance, we defined the hit ratio based on a corruption tolerance. This is more appropriate than using e.g. the mean Hamming distance because the Hamming distance of a failed retrieval from the stored pattern depends on the pattern bias. For example, in the case of connection dilution, after deleting many connections, an entirely quiescent pattern becomes the network’s stable state. This non-retrieval state has on average a Hamming distance p to the stored patterns. In contrast, the expected distance between two random patterns in the network is 2 p (1− p ), meaning that the retrieval of a mistaken memory results in lower Hamming distances for biased patterns. To mitigate this issue for comparing performance of different levels of pattern bias, we derived the tolerance levels based on mutual information. This approach was further supported by our finding that the Hamming distances before the abrupt degradation in the retrieval performance ( Figure 3D ) agreed with our differentiated choice of tolerance values. While we found that different types of noise exhibit a similar relation between the pattern load and the most robust pattern bias ( Figure 4H ), the underlying mechanisms and dynamics are likely to be more complex and heterogeneous across the different noise types. For the stochastic threshold noise type, sparse patterns had very low robustness even for low pattern loads ( Figure 4A ). Intuitively, this is plausible because for sparse patterns a stochastic threshold means that the activation of the few active nodes is at risk of being changed and therefore the pattern might not be retrieved. More formally, less biased (i.e. more distributed) patterns have a higher information load with the total entropy of the stored patterns being proportional to the Shannon function h ( p ) = − p log 2 ( p ) − (1 − p ) log 2 (1 − p ) ( Frolov et al., 1997 ), which is low for sparse patterns and maximal for the unbiased patterns. For Hopfield-like memory networks in the brain this suggests that activity fluctuations of single neurons that are part of sparse patterns are detrimental for accurate pattern retrieval. Possible ways to reduce the risk for this detrimental effect in the brain include a low, stable baseline activity in single neurons of these networks. Some evidence supporting this possibility comes from the observation that neurons in the medial temporal lobe with less selective responses to sensory stimuli have higher baseline firing rates than neurons with more selective responses ( Mormann et al., 2008 ). Furthermore, medium-spiny neurons in the striatum, a region associated with sparse coding of actions, exhibit very low baseline activity (Berke et al., 2004; Markowitz et al., 2018 ). For our simulations with weight noise, sparser representations slightly improved in their robustness compared to the stochastic threshold noise type ( Figure 4B ). Thus, the mean activity level of the most robust networks against weight noise was lower than that in the stochastic model ( Figure 4H ). However, as the range of the noise values was chosen to cover the network’s behavior from a successful retrieval to a breakdown, the robustness values for different noise types are not directly comparable. Instead they indicate within a given noise type the relative performance of different pattern representations and pattern loads. For weight noise, perturbations in the connection strengths cause the weight matrix to lose its symmetry and thereby a sufficient condition for the existence of the energy function, even though such disturbed networks can still operate as attractor systems ( Rolls & Deco, 2010 ). After the introduction of asymmetry in the weight matrix, the mean field theory can be applied for the case of M ≪ N , and it was shown that asymmetry is then analogous to increasing the temperature in the network ( Hertz et al., 1986 ) and can make spurious states unstable, potentially improving the network’s performance ( Hertz et al., 1991 ). Possibly, this connection underlies the qualitatively similar retrieval for the two noise types and accounts for their similar robustness profiles ( Figure 4A&B ). One limitation is that we only considered static weight noise here, so that a single post-learning perturbation took place in the connections. In general, the brain is exposed to more dynamic noise, e.g. for weight noise through synaptic strengths ( Ribrault et al., 2011 ), which affects the network during learning and retrieval with varying perturbations. Similar to weight noise, also connection dilution introduces an asymmetry in the weight matrix. Interestingly, this had a different effect though on the robustness in relation to pattern bias ( Figure 4C ). Overall, biased patterns had high robustness with peak performance for sparse patterns and α > 0.1. It seems that here, next to the information load (see above), a second, opposing dynamic exists which favors sparsely-coded patterns and makes them less vulnerable to spurious minima, increasing their load capacity. This dynamic could be related to the reduction of spurious states in asymmetric networks ( Hertz et al., 1991 ). However, in our simulations the asymmetry was introduced by a random, post-learning dilution of the connections, which is different from a network without all-to-all connectivity, or a network with a learning rule for asymmetric connection weights. In the brain, connection dilution (or synaptic loss) strong correlates with cognitive decline ( Scheff et al., 2006 ). Horn et al. (1993) used a similar Hopfield network to investigate the deterioration of memory retrieval due to synaptic loss in Alzheimer’s disease. The random deletion of connections by the probability d corresponds to multiplying the connection strengths W ij by the Bernoulli variables d ij , which are 1 with the probability (1 − d ), and 0 otherwise. Considering the signal-to-noise ratio, this means that the two signal values (the mean values of the nodes’ local fields for the 0 and 1 states), before introducing an activation threshold, decrease proportional to (1 − d ). Thereby, the threshold is no longer optimal, i.e. keeping the local fields in a situation similar to Figure 2A instead of Figure 2B . Thus, the signal-to-noise ratio reduces and performances declines, providing a potential explanation for some of the cognitive deficits in Alzheimer’s disease. The results from simulations involving node dilution showed a strong similarity to connection dilution ( Figure 4C&D ). For load parameters smaller than capacity, the robustness levels were far from zero in both cases, indicating that a memory breakdown happened only after a high number of deletions. Based on the higher capacity of sparse pattern representations in Hopfield networks, one could expect that they are in general also more robust. However, our results demonstrate that this is not always the case. This can be seen e.g. in the effect of node dilution on the signal-to-noise ratio. The total number of nodes is reduced by the node loss, but the number of patterns remains constant. This change is different from simply an increase in the load parameter, since the disruption takes place after the learning phase. Using the optimal uniform threshold θ opt from Eq. 8 , the signal-to-noise ratio for one bit of a stored pattern is given by Eq. 9 . For a deletion ratio d , the signal and cross-talk noise terms in Appendix B must be adjusted such that the number of summed terms and thus both signals (before threshold) and cross-talk noise are reduced by the factors (1 − d ) and , respectively. Therefore, the signal-to-noise ratio for the model with threshold θ opt and deletion ratio d is given by: Therefore, the deterioration in the signal-to-noise ratio after deletion of nodes is stronger for sparser patterns, especially for higher values of d , which indicates that the higher information load in more distributed patterns can contribute to robustness. This is in line with the view that ‘local codes’, i.e. very sparse patterns here, are sensitive to error and noise, and the redundancy in more distributed representations improves fault tolerance ( Spanne & Jörntell, 2015 ). Nevertheless, in our simulations, for increased pattern loads, the robustness advantage of sparse patterns relative to more distributed patterns was more pronounced for connection and node loss in comparison to stochastic thresholds and weight noise. This indicates that sparse patterns in Hopfield-like networks in the brain might be useful to reduce their vulnerability to structural loss in brain regions such as hippocampus and thereby postpone the onset of cognitive decline e.g. in Alzheimer’s disease. The other types of noise we examined manifested as distorted versions of the stored patterns presented during the retrieval phase, i.e. states were made active, quiescent, or flipped their values ( Figure 4E-G ). The results between these types of noise were qualitatively similar. However, the advantage of sparse patterns was strongest for the case of states turning quiescent ( Figure 4G ), since there was a smaller probability of the few active units to be affected by that change. The high robustness values for states turning active or quiescent point to successful retrieval even for a large number of activated or silenced units ( Figure 4F&G ). In addition the robustness profiles exhibited pronounced peaks for each pattern load, providing a specific level of pattern bias that performed most robust. This points to another potential advantage of sparse pattern representations in the brain, so that additional activation of neurons that do not belong to the stored pattern, does not strongly impair the pattern retrieval. For example, a population pattern in the hippocampus represented by sparse place field activity would be robust against conflicting activation of random place cells. This type of conflicting activation (or deactivation) might also mimic the application of optogenetics in neuroscience in which neuronal activity of a targeted part of a neural circuit can either be activated or silenced by using light with a millisecond temporal resolution ( Boyden et al., 2005 ). In contrast, state flips as a type of noise when presenting a pattern for retrieval entails knowing the actual memory pattern and then distorting it, which may not have an obvious biological correspondence. Our results on the relation between pattern bias and robustness are relevant for the ongoing discussion about coding schemes and pattern representation in the brain. Different brain regions seem to employ different coding schemes in the range from sparse to distributed representations (Quiroga & Kreiman, 2010). This might reflect different functional specializations as coding schemes relate to the ability to generalize ( Spanne & Jörntell, 2015 ). For example, neural coding in the hippocampus, with the very localized activity of place cells, is considered to be sparse, while cortical regions are often associated with more distributed representations (Quiroga & Kreiman, 2010). Based on our results the question of which coding scheme is optimal for a given brain region, should also consider the expected pattern load. Intuitively, a given memory network in the brain should have a capacity that is as high as possible. However, there are many constraints on memory systems such those imposed by sparseness on memory lifetimes ( Leibold & Kempter, 2007 ) and the ability to store sequences ( Leibold & Kempter, 2006 ), which are arguably just as important as capacity. Furthermore, Hopfield-like networks in the brain might be used to store patterns more generally, i.e. not necessarily only patterns related to semantic or episodic memories, but also non-declarative memories. On an evolutionary time scale, these networks might have been optimized to a specific level of sparse or distributed representations that is the result of a trade-off between capacity, robustness, and the expected pattern load. For example, a brain network that is typically only used for a low pattern load, might be able to afford a rather distributed pattern representation, providing high robustness against various types of noise ( Figure 4H ). In contrast, networks that are used to store a high number of patterns might have evolved to use a more sparse pattern representation, again providing high robustness against various types of noise. This view is in line with the proposal that sparse representations are used by memory systems to achieve a high capacity, while distributed representations are used by sensory systems ( Rolls & Treves, 2011 ). However, our results emphasize that not only the extremes of the coding schemes, i.e. sparse or fully-distributed, should be considered, as for certain intermediate pattern loads, also intermediate levels of pattern bias might be advantageous. It has been pointed out several times that the current use of the terminology related to pattern representations and neural coding is inconsistent across different subfields (Quiroga & Kreiman, 2010; Spanne & Jörntell, 2015 ; Beyeler et al., 2019 ). We used the term unbiased here to refer to patterns with p = 0.5, which would correspond to a fully distributed ( Rolls & Treves, 2011 ) or dense ( Spanne & Jörntell, 2015 ) representation. Biased patterns (here p < 0.5) are also referred to as sparse distributed representations ( Rolls & Treves, 2011 ), while we used the term sparse to indicate that p << 0.5 and the term sparseness to refer to the degree to which the representation deviates from p = 0.5. While Hopfield networks are an established model for memory networks in the brain, they make strong abstractions of the underlying biological networks and their dynamics. For our simulations these abstractions are important to consider for the effects of noise on different pattern representations. Firstly, the patterns we examined here were uncorrelated, i.e. for each pattern the active and inactive units were randomly selected. In contrast, neural activation patterns are often correlated, e.g. due to the spatial activity patterns of hippocampal place cells, which is problematic for Hopfield-like storage of memory patterns e.g. in CA3 ( Papp et al., 2007 ; Neher et al., 2015 ). Secondly, the symmetric all-to-all connectivity in Hopfield networks does not match connectivity rules of biological neural networks or cell-type specific connectivity. For these reasons hippocampal models using Hopfield networks are often modified to better account for neurophysiological characteristics ( Rolls, 2013 ). While our results on the robustness consider here only a general type of Hopfield network, it remains to be shown how neurophysiological characteristics of specific brain regions and networks affect robustness. Finally, although we used a relatively large network ( N = 5000), the properties of sparse and distributed codes, including robustness, might be affected by the network size ( Spanne & Jörntell, 2015 ). Despite these limitations our results suggest that the performance of Hopfield networks should not be judged entirely based on capacity considerations, but also based on the robustness against different types of noise, which are relevant for biological neural networks. Our findings indicate that the coding schemes employed by different brain regions are adapted so that their vulnerability to disruptions is mitigated. Code availability The code for the simulations done in this manuscript and their visualization is available here: https://gitlab.ruhr-uni-bochum.de/smidtrc8/robustness-in-hopfield-networks . Acknowledgments We thank Heinz-Jürgen Schmidt for interesting discussions and helpful comments on a previous version of this manuscript. Appendix A) Signal-to-noise ratio analysis in the modified Hopfield network Assumed is a uniform activation threshold θ . Starting from the update rule for the state of node i , we have where H is the Heaviside step function. Using the Eq. (5) , we expand the weights in Eq. (18) : When the network is in the stored pattern υ , it means that , thus Since the expression 21.1 is equal to Due to the independence of the memory bits, we have Hence, On the other hand, the expression 21.2 consists of ( M − 1)( N − 1) terms, each with the following possible outcomes and probabilities: Hence, We express results in probabilistic terms, since there is a large number of elements drawn from the same distribution. In reality, the results are deterministic and based on the realizations of the Bernoulli variables. If the expressions 21.1 and 21.2 are independent and by ignoring the smaller variance in 21.1, we can approximate the total terms inside Eq. (21) as the distributions summarized in Table 2 , based on the ξ -pattern state value .The optimal threshold θ opt is equalizes the absolute values of the two means, as shown in the Figure 2B . View this table: View inline View popup Download powerpoint Table 1: Probability table of the elements in the expression 21.2 View this table: View inline View popup Download powerpoint Table 2: Signals and cross-talk noises for different states of a memory pattern View this table: View inline View popup Download powerpoint Table 3: Probability table of the elements in the expression 28.2 B. The reduced cross-talk noise by adding correlated random local fields By a transformation in form of V j → V j − p , Eq. (21) changes to where the expression 28.1 is equal to: Using Eq. (23) , we have The mean and variance expressions above show that the signal is unchanged. Regarding the expression 28.2, there are again ( M − 1)( N − 1) terms, each with the following possible outcomes and probabilities: Hence, The decreased variance of 28.2 in comparison to 21.2 means that the transformation V j → V j − p or equivalently adding the random thresholds p Σ j ≠i W ij decreases the noise term. References Amari , S. I. , & Maginu , K. ( 1988 ). Statistical neurodynamics of associative memory . Neural Networks , 1 ( 1 ), 63 – 73 . OpenUrl CrossRef Web of Science Amari , S. I. ( 1989 ). Characteristics of sparsely encoded associative memory . Neural Networks , 2 ( 6 ), 451 – 457 . OpenUrl CrossRef Web of Science Amit , D. J. , Gutfreund , H. , & Sompolinsky , H. ( 1985 ). Spin-glass models of neural networks . Physical Review A , 32 ( 2 ), 1007 . OpenUrl CrossRef PubMed Amit , D. J. , Gutfreund , H. , & Sompolinsky , H. ( 1985 ). Storing infinite numbers of patterns in a spin-glass model of neural networks . Physical Review Letters , 55 ( 14 ), 1530 . OpenUrl CrossRef PubMed Web of Science ↵ Amit , D. J. , Gutfreund , H. , & Sompolinsky , H. ( 1987 ). Information storage in neural networks with low levels of activity . Physical Review A , 35 ( 5 ), 2293 . OpenUrl CrossRef PubMed Web of Science Attwell , D. , & Laughlin , S. B. ( 2001 ). An energy budget for signaling in the grey matter of the brain . Journal of Cerebral Blood Flow & Metabolism , 21 ( 10 ), 1133 – 1145 . OpenUrl CrossRef PubMed Barnes , C. A. , McNaughton , B. L. , Mizumori , S. J. , Leonard , B. W. , & Lin , L. H. ( 1990 ). Chapter Comparison of spatial and temporal characteristics of neuronal activity in sequential stages of hippocampal processing . Progress in brain research , 83 , 287 – 300 . OpenUrl CrossRef PubMed Web of Science Bellucci , A. , Mercuri , N. B. , Venneri , A. , Faustini , G. , Longhena , F. , Pizzi , M. , … & Spano , P. ( 2016 ). Parkinson’s disease: from synaptic loss to connectome dysfunction . Neuropathology and applied neurobiology , 42 ( 1 ), 77 – 94 . OpenUrl CrossRef PubMed Berke , J. , Okatan , M. , Skurski , J. & Eichenbaum , H. Oscillatory Entrainment of Striatal Neurons in Freely Moving Rats . Neuron , 43 , 883 – 896 . ↵ Beyeler , Michael and Rounds , Emily L. and Carlson , Kristofor D. and Dutt , Nikil and Krichmar , Jeffrey L. ( 2019 ). Neural correlates of sparse coding and dimensionality reduction . PLOS Computational Biology . 15 , 1 – 33 (2019,6) , doi: 10.1371/journal.pcbi.1006908 OpenUrl CrossRef ↵ Bowers , J. S. ( 2009 ). On the biological plausibility of grandmother cells: implications for neural network theories in psychology and neuroscience . Psychological Review , 116 ( 1 ), 220 . OpenUrl CrossRef PubMed Web of Science ↵ Boyden , E. S. , Zhang , F. , Bamberg , E. , Nagel , G. , & Deisseroth , K. ( 2005 ). Millisecond-timescale, genetically targeted optical control of neural activity . Nature Neuroscience , 8 ( 9 ), 1263 – 1268 . OpenUrl CrossRef PubMed Web of Science Buhmann , J. , Divko , R. , & Schulten , K. ( 1989 ). Associative memory with high information content . Physical Review A , 39 ( 5 ), 2689 . OpenUrl CrossRef PubMed Coolen , A. C. C. , & Sherrington , D. ( 1993 ). Dynamics of fully connected attractor neural networks near saturation . Physical Review Letters , 71 ( 23 ), 3886 . OpenUrl CrossRef PubMed ↵ Cover , T. M. , & Thomas , J. A. ( 1991 ). Elements of Information Theory . New York : Wiley DiCarlo , J. J. , & Cox , D. D. ( 2007 ). Untangling invariant object recognition . Trends in Cognitive Sciences , 11 ( 8 ), 333 – 341 . OpenUrl CrossRef PubMed Web of Science ↵ Faisal , A. A. , Selen , L. P. , & Wolpert , D. M. ( 2008 ). Noise in the nervous system . Nature Reviews Neuroscience , 9 ( 4 ), 292 – 303 . OpenUrl CrossRef PubMed Web of Science Foldiak , P. ( 2003 ). Sparse coding in the primate cortex . The handbook of brain theory and neural networks . ↵ Frolov , A. A. , Husek , D. , & Muraviev , I. P. ( 1997 ). Informational capacity and recall quality in sparsely encoded Hopfield-like neural network: Analytical approaches and computer simulation . Neural Networks , 10 ( 5 ), 845 – 855 . OpenUrl CrossRef PubMed Frolov , A. A. , Husek , D. , & Rachkovskii , D. A. ( 2006 ). Time of searching for similar binary vectors in associative memory . Cybernetics and Systems Analysis , 42 ( 5 ), 615 – 623 . OpenUrl CrossRef Gardner , E. ( 1987 ). Maximum storage capacity in neural networks . Europhysics Letters , 4 ( 4 ), 481 . OpenUrl CrossRef Gardner , E. , Derrida , B. , & Mottishaw , P. ( 1987 ). Zero temperature parallel dynamics for infinite range spin glasses and neural networks . Journal de Physique , 48 ( 5 ), 741 – 755 . OpenUrl CrossRef ↵ Gardner , E. ( 1988 ). The space of interactions in neural network models . Journal of Physics A: Mathematical and General , 21 ( 1 ), 257 . OpenUrl CrossRef Web of Science ↵ Hertz , J. A. , Grinstein , G. , & Solla , S. A. ( 1986 , August ). Memory networks with asymmetric bonds . In AIP Conference Proceedings (Vol. 151 , No. 1 , pp. 212 – 218 ). American Institute of Physics . OpenUrl CrossRef ↵ Hertz , J. , Krogh , A. , Palmer , R. G. , & Horner , H. ( 1991 ). Introduction to the theory of neural computation . Redwood City CA : Addison-Wesley . ↵ Hopfield , J. J. ( 1982 ). Neural networks and physical systems with emergent collective computational abilities . Proceedings of the National Academy of Sciences , 79 ( 8 ), 2554 – 2558 . OpenUrl Abstract / FREE Full Text ↵ Horn , D. , Ruppin , E. , Usher , M. , & Herrmann , M. ( 1993 ). Neural network modeling of memory deterioration in Alzheimer’s disease . Neural Computation , 5 ( 5 ), 736 – 749 . OpenUrl CrossRef Lawrence , M. , Trappenberg , T. , & Fine , A. ( 2006 ). Rapid learning and robust recall of long sequences in modular associator networks . Neurocomputing , 69 ( 7-9 ), 634 – 641 . OpenUrl CrossRef ↵ Leibold , C. & Kempter , R. Memory Capacity for Sequences in a Recurrent Network with Biological Constraints . Neural Computation . 18 , 904 – 941 ( 2006 ,4). OpenUrl CrossRef PubMed Web of Science ↵ Leibold , C. & Kempter , R. Sparseness Constrains the Prolongation of Memory Lifetime via Synaptic Metaplasticity . Cerebral Cortex . 18 , 67 – 77 ( 2007 ,5). OpenUrl PubMed Lennie , P. ( 2003 ). The cost of cortical computation . Current Biology , 13 ( 6 ), 493 – 497 . OpenUrl CrossRef PubMed Web of Science Linster , C. , & Hasselmo , M. E. ( 2001 ). Neuromodulation and the functional dynamics of piriform cortex . Chemical Senses , 26 ( 5 ), 585 – 594 . OpenUrl CrossRef PubMed Web of Science ↵ Long , J. M. , & Holtzman , D. M. ( 2019 ). Alzheimer disease: an update on pathobiology and treatment strategies . Cell , 179 ( 2 ), 312 – 339 . OpenUrl CrossRef PubMed ↵ MacKay , D. J. ( 2003 ). Information theory, inference and learning algorithms . Cambridge university press . ↵ Markowitz , J. , Gillis , W. , Beron , C. , Neufeld , S. , Robertson , K. , Bhagat , N. , Peterson , R. , Peterson , E. , Hyun , M. , Linderman , S. , Sabatini , B. & Datta , S. The striatum organizes 3D behavior via moment-to-moment action selection . Cell . 174 , 44 – 58 .e17 ( 2018 ,6), https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6026065/ OpenUrl CrossRef PubMed ↵ Mormann , F. , Kornblith , S. , Quiroga , R. Q. , Kraskov , A. , Cerf , M. , Fried , I. , & Koch , C. ( 2008 ). Latency and selectivity of single neurons indicate hierarchical processing in the human medial temporal lobe . Journal of Neuroscience , 28 ( 36 ), 8865 – 8872 . OpenUrl Abstract / FREE Full Text ↵ Neher , T. , Cheng , S. & Wiskott , L. Memory Storage Fidelity in the Hippocampal Circuit: The Role of Subregions and Input Statistics . PLOS Computational Biology . 11 , e1004250 ( 2015 ,5). OpenUrl CrossRef PubMed Okada , M. ( 1995 ). A hierarchy of macrodynamical equations for associative memory . Neural Networks , 8 ( 6 ), 833 – 838 . OpenUrl CrossRef Web of Science Okada , M. ( 1996 ). Notions of associative memory and sparse coding . Neural Networks , 9 ( 8 ), 1429 – 1458 . OpenUrl CrossRef PubMed Web of Science Olshausen , B. A. ( 2003 ). Principles of image representation in visual cortex . The Visual Neurosciences , 2 , 1603 – 1615 . OpenUrl Olshausen , B. A. , & Field , D. J. ( 2004 ). Sparse coding of sensory inputs . Current Opinion in Neurobiology , 14 ( 4 ), 481 – 487 . OpenUrl CrossRef PubMed Web of Science Ozeki , T. , & Nishimori , H. ( 1994 ). Noise distributions in retrieval dynamics of the Hopfield model . Journal of Physics A: Mathematical and General , 27 ( 21 ), 7061 . OpenUrl CrossRef ↵ Papp , G. , Witter , M. & Treves , A. The CA3 network as a memory store for spatial representations . Learning & Memory . 14 , 732 – 744 ( 2007 ,11). OpenUrl Abstract / FREE Full Text Purves , Augustine , George J. , Fitzpatrick , David , Hall , William C. , LaMantia , Anthony-Samuel , Mooney , Richard D. , Platt , Michael L. , & White , Leonard E. ( 2018 ). Neuroscience (International sixth edition .). Oxford University Press Quiroga , R. Q. , Reddy , L. , Koch , C. , & Fried , I. ( 2007 ). Decoding visual inputs from multiple neurons in the human temporal lobe . Journal of Neurophysiology , 98 ( 4 ), 1997 – 2007 . OpenUrl CrossRef PubMed Web of Science Quian Quiroga , R. , & Kreiman , G. ( 2010 ). Measuring sparseness in the brain: comment on Bowers (2009) . Psychol Rev , 117 ( 1 ), 291 – 297 OpenUrl CrossRef PubMed Ramsauer , H. , Schaefl , B. , Lehner , J. , Seidl , P. , Widrich , M. , Adler , T. , … & Hochreiter , S. ( 2020 ). Hopfield networks is all you need . arXiv preprint arxiv: 2008.02217v3 . ↵ Ribrault , C. , Sekimoto , K. & Triller , A. From the stochasticity of molecular processes to the variability of synaptic transmission . Nature Reviews Neuroscience , 12 , 375 – 387 ( 2011 ) OpenUrl CrossRef PubMed ↵ Rolls , E. T. , & Deco , G. ( 2010 ). The noisy brain: stochastic dynamics as a principle of brain function (Vol. 34 ). Oxford : Oxford university press . ↵ Rolls , E. A quantitative theory of the functions of the hippocampal CA3 network in memory . Frontiers In Cellular Neuroscience . 7 ( 2013 ,6). Rolls , E. T. , & Tovee , M. J. ( 1995 ). Sparseness of the neuronal representation of stimuli in the primate temporal visual cortex . Journal of Neurophysiology , 73 ( 2 ), 713 – 726 . OpenUrl CrossRef PubMed Web of Science ↵ Rolls , E.T. , & Treves , A. ( 2011 ) The neuronal encoding of information in the brain . Progress In Neurobiology . 95 , 448 – 490 . OpenUrl CrossRef PubMed Scheff , S. W. , & Price , D. A. ( 2006 ). Alzheimer’s disease-related alterations in synaptic density: neocortex and hippocampus . Journal of Alzheimer’s Disease , 9 ( s3 ), 101 – 115 . OpenUrl ↵ Scheff , S. W. , Price , D. A. , Schmitt , F. A. , & Mufson , E. J. ( 2006 ). Hippocampal synaptic loss in early Alzheimer’s disease and mild cognitive impairment . Neurobiology of Aging , 27 ( 10 ), 1372 – 1384 . OpenUrl CrossRef PubMed Web of Science ↵ Spanne , A. , & Jörntell , H. ( 2015 ). Questioning the role of sparse coding in the brain . Trends in Neurosciences , 38 ( 7 ), 417 – 427 . OpenUrl CrossRef PubMed Trappenberg , T. ( 2009 ). Fundamentals of computational neuroscience . OUP Oxford . Treves , A. , & Rolls , E. T. ( 1991 ). What determines the capacity of autoassociative memories in the brain? . Network: Computation in Neural Systems , 2 ( 4 ), 371 . OpenUrl CrossRef ↵ Tsodyks , M. V. , & Feigel’man , M. V. ( 1988 ). The enhanced storage capacity in neural networks with low activity level . Europhysics Letters , 6 ( 2 ), 101 . OpenUrl CrossRef Web of Science ↵ Vicente , C. P. , & Amit , D. J. ( 1989 ). Optimised network for sparsely coded patterns . Journal of Physics A: Mathematical and General , 22 ( 5 ), 559 . OpenUrl CrossRef Willmore , B. , & Tolhurst , D. J. ( 2001 ). Characterizing the sparseness of neural codes . Network: Computation in Neural Systems , 12 ( 3 ), 255 . OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted November 04, 2024. Download PDF Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Robustness in Hopfield neural networks with biased memory patterns Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Robustness in Hopfield neural networks with biased memory patterns Ali Montazeri , Robert Schmidt bioRxiv 2024.10.31.621250; doi: https://doi.org/10.1101/2024.10.31.621250 Share This Article: Copy Citation Tools Robustness in Hopfield neural networks with biased memory patterns Ali Montazeri , Robert Schmidt bioRxiv 2024.10.31.621250; doi: https://doi.org/10.1101/2024.10.31.621250 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Neuroscience Subject Areas All Articles Animal Behavior and Cognition (7644) Biochemistry (17725) Bioengineering (13915) Bioinformatics (42031) Biophysics (21486) Cancer Biology (18635) Cell Biology (25548) Clinical Trials (138) Developmental Biology (13397) Ecology (19937) Epidemiology (2067) Evolutionary Biology (24361) Genetics (15619) Genomics (22538) Immunology (17763) Microbiology (40468) Molecular Biology (17206) Neuroscience (88733) Paleontology (667) Pathology (2842) Pharmacology and Toxicology (4834) Physiology (7657) Plant Biology (15172) Scientific Communication and Education (2046) Synthetic Biology (4304) Systems Biology (9831) Zoology (2272)