Ensemble Post-hoc Explainable AI in Multivariate Time Series: Identifying Medical Features Driving Disease Prediction

Ensemble Post-hoc Explainable AI for Multilead ECG: Identifying Disease-Relevant Features in Single-Lead Interpretations | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Ensemble Post-hoc Explainable AI for Multilead ECG: Identifying Disease-Relevant Features in Single-Lead Interpretations Jacqueline Michelle Metsch , Philip Hempel , Miriam Cindy Maurer , Kristin Elisabeth Steinhaus , Nicolai Spicher , View ORCID Profile Anne-Christin Hauschild doi: https://doi.org/10.1101/2025.02.14.638219 Jacqueline Michelle Metsch 1 Institute for Predictive Deep Learning for Medicine and Healthcare, Justus-Liebig University , Gießen Campus Institute Data Science Gießen, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: jacqueline.metsch{at}unigiessen.de Philip Hempel 2 Department of Medical Informatics University Medical Center Göttingen, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site Miriam Cindy Maurer 3 Department of Medical Informatics University Medical Center Campus Institut Data Science Göttingen , Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site Kristin Elisabeth Steinhaus 4 Department of Cardiology and Pneumology University Medical Center Göttingen , Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site Nicolai Spicher 5 Department of Health Technology Technical University of Denmark Kongens Lyngby , Denmark Find this author on Google Scholar Find this author on PubMed Search for this author on this site Anne-Christin Hauschild 1 Institute for Predictive Deep Learning for Medicine and Healthcare, Justus-Liebig University , Gießen Campus Institute Data Science Gießen, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Anne-Christin Hauschild Abstract Full Text Info/History Metrics Preview PDF Abstract Background Despite the growing success of deep learning (DL) in multivariate time-series classification, such as 12-lead electro-cardiography (ECG), widespread integration into clinical practice has yet to be achieved. The limited transparency of DL hinders clinical adoption, where understanding model decisions is crucial for trust and compliance with regulations such as the General Data Protection Regulation (GDPR) or the EU AI Act. Results To tackle this challenge, we implemented a widely used 1D-ResNet in Pytorch that was trained on the large-scale Brazilian CODE dataset to classify six different ECG abnormalities. We employed the model on the German PTB XL dataset, and evaluated its decision-making processes using 16 post-hoc explainable AI (XAI) methods. To assess the clinical relevance of the model’s attributions, we conducted a Wilcoxon signed-rank test to identify features with significantly higher relevance for each XAI method. We used an ensemble majority vote approach to validate whether the model has learned clinically meaningful features for each abnormality. Additionally, a Mann–Whitney U test was employed to detect significant differences in relevance attributions between correctly and incorrectly classified ECGs. Overall, the model achieved sensitivity scores above 0.9 for most abnormalities in the PTB XL dataset. However, our XAI analysis showed that the model struggled to capture clinically relevant features for some diseases. Certain XAI methods, including DeepLift, DeepLiftShap, and Occlusion, consistently highlighted clinically meaningful features across abnormalities, while others, such as LIME, KernelShap, and LRP, failed to do so. Moreover, some XAI methods demonstrated significant differences in attributions between correctly and incorrectly classified ECGs, highlighting their potential for enhancing model robustness and interpretability. Conclusion Our findings underscore the importance of selecting suitable XAI methods tailored to specific model architectures and data types to ensure transparency and reliability. By identifying effective XAI techniques, this study contributes to closing the gap between DL advancements and their clinical implementation, paving the way for more trustworthy AI-driven healthcare solutions. 1 Background Deep learning (DL) models have demonstrated remarkable performance across various domains, including healthcare [ 28 ], finance [ 27 ], and autonomous systems [ 53 ]. However, due to their inherent complexity they are considered “black-box” models, offering little to no transparency regarding their decision-making processes. This lack of explainability is particularly concerning in high-risk applications such as medical diagnosis, where understanding a model’s reasoning is critical for gaining the users’ trust. In particular for medical data, there are multiple examples where DL models learned shortcuts in the data, leading to classifications unrelated to diagnostic features and resulting in misdiagnosis [ 10 ]. The recently proposed EU AI Act [ 1 ] explicitly underscores the need for transparency and accountability in AI development and application, enforcing the importance of interpretability. To address these concerns, researchers have increasingly turned to Explainable AI (XAI) methods to analyse whether a model learned the correct features from input data [ 7 , 26 , 30 ]. These techniques aim to provide insights into the internal workings of DL models, helping to validate their predictions and detect biases or spurious correlations. Their effectiveness has already been demonstrated in image classification tasks, where explanations are typically visualized as heatmaps that indicate whether the model focuses on the correct regions of the image [ 13 , 44 ]. A range of XAI methods, including gradient-based approaches, perturbation-based methods, and model-agnostic techniques, can be employed to assess the decision-making process of deep networks [ 17 ]. However, the choice of the XAI method significantly impacts the interpretations derived from a model. [ 4 , 15 , 22 , 25 ] found that different XAI techniques can produce varying relevance attributions for the same model and dataset, leading to potential inconsistencies in explainability results. Metsch et al. [ 25 ] for example found that IntegratedGradients, DeepLift, DeepShap and GradientShap perform similarly well on different datatypes, while Deconvolution, Guided Backpropagation, and LRP- α 1- β 0 differ from other methods. This variability gives rise to concerns about the reliability and biases of XAI-based model validation, underscoring the necessity of selecting appropriate interpretability techniques tailored to the specific application. It is, therefore, crucial to identify these discrepancies and address them accordingly in order to ensure that explainability efforts genuinely enhance trust and model accountability rather than introducing further ambiguity. In feature selection, the issue of method-specific bias has been tackled using ensemble approaches that compute a quantitative, consensus-based relevance score [ 6 , 31 ]. A similar strategy has emerged in XAI, where ensemble techniques typically aggregate heatmaps produced by multiple explanation methods to mitigate individual biases [ 12 , 57 ]. Additional feature selection approaches, such as multi-step feature selection [ 50 ], were not explored in this study due to time constraints. In this study, we address the issue of transparency and reliability by developing a comprehensive processing pipeline that integrates three steps for evaluating the applicability of XAI Methods for time series in medicine and proposes a novel ensemble approach. We implement the following steps: (1) a widely used 1D-ResNet was trained on the large-scale CODE dataset, (2) we evaluated its prediction performance on the German PTB XL dataset, (3) benchmarked the decision-making processes using 16 post-hoc explainable AI (XAI) methods, and (4) used an ensemble majority vote approach to validate whether the model has learned clinically meaningful features. Figure 1 summarizes the pipeline. Download figure Open in new tab Figure 1: Processing pipeline employed in this paper. First, we train a widely used 1D-ResNet on a large-scale ECG dataset from Brasil. Afterward, the trained model is used to generate predictions for data stemming from the German PTB XL dataset. Subsequently, we apply 16 different post-hoc XAI methods to generate relevance attributions for the predicted samples. Lastly, we developed a processing pipeline that averages the ECGs as well as their relevance attributions, then focuses on the important lead V1 and performs two types of statistical tests for quantitative analysis of the results. 2 Methods ECG is a measurement of the electrical activity of the human heart and standard of care for assessment of cardiovascular health. Typically, 10 cables are connected to the human body, resulting in 12 signals recorded in parallel – entitled “leads” – making ECG signals multivariate time series. We first train a widely used 1D-ResNet model [ 34 ] on the large-scale Brazilian CODE dataset [ 34 – 36 ] to classify six different abnormalities in 12-lead ECG data and then generating explanations using 16 different XAI methods based on the models predictive performance on the German PTB XL dataset [ 49 ]. We use a majority vote ensemble approach on the XAI methods to more robustly identify whether the model learned clinically relevant features to predict the different abnormalities. In the realm of feature selection methods, ensemble approaches are already being employed to reduce biases of specific methods [ 6 , 31 ]. In the field of XAI, ensemble approaches are also gaining popularity, though they often focus on aggregating heatmaps generated by multiple XAI methods [ 12 , 57 ]. Additionally, we employ a quantitative analysis of all XAI methods to identify which methods are appropriate for our specific classification task, as well as their capabilities in discerning between true positive and false positive classified samples. See Figure 1 for details. 2.1 CODE Data The Brazilian CODE dataset [ 34 – 36 ] comprises multiple 12-lead ECGs from 1, 676, 384 unique patients (mean age: 51.2; 60.3% female), recorded through the Telehealth Network of Minas Gerais (TNMG). Telehealth refers to a remote healthcare model in which specialists (e.g., cardiologists) virtually review ECGs sent from primary care facilities, ensuring high-quality diagnoses despite geographical distance. The ECGs were acquired between 2010 and 2016 in 811 counties, with each 7–10-second trace is annotated for six key abnormalities: (1) 1st degree AV block (1dAVb), (2) Complete Right Bundle Branch Block (CRBBB), (3) Complete Left Bundle Branch Block (CLBBB), (4) Sinus Bradycardia (SB), (5) Atrial Fibrillation (AF), and (6) Sinus Tachycardia (ST). These specialist-reviewed labels provide a robust ground truth while making CODE one of the largest and most comprehensive labeled 12-lead ECG repositories available for developing and evaluating AI models. The CODE Study was approved by the Research Ethics Committee of the Universidade Federal de Minas Gerais (protocol 49368496317.7.0000.5149) 2.2 PTB XL Data PTB XL [ 49 ] contains 21799 clinical 12 lead ECG recordings of 10 seconds each from 18869 patients. Each ECG recording was annotated by cardiologists with multiple diagnostic statements, resulting in a total of 71 different ECG statements. These statements are further grouped into five high-level diagnostic categories: Norm (NORM), Myocardial Infarction (MI), ST/T Change (STTC), Conduction Disorder (CD) and Hypertrophy (HYP). In addition to the ECG waveforms, the dataset provides extensive metadata. The ECG signals are available in both high-resolution (500 Hz) and low-resolution (100 Hz) formats. 2.3 Training and Evaluation The CODE dataset is used for model training following the original implementation, while the PTB XL dataset serves as an independent external dataset for evaluating both model performance and XAI behavior under domain shift. The use of PTB XL as an external dataset allows us to evaluate XAI behavior under domain shift, reflecting more realistic deployment conditions compared to in-distribution analysis on the training dataset. We replicated the 12-lead training pipeline proposed by A.H. Ribeiro et al . [ 34 ] using a PyTorch implementation of their 1D-ResNet architecture adapted from [ 19 ]. Specifically, we applied their preprocessing steps (resampling to 400 Hz, zero-padding each 10 s ECG to 4,096 samples) and adopted the same hyperparameter settings as in the original work. By closely following the original training protocol and changing only the deep learning framework, we ensured minimal deviations from the methods validated in [ 34 ] while demonstrating reproducibility in PyTorch, enabling the application of Captum to XAI methods [ 17 ]. In line with common practices for large-scale data mining, we used raw ECG signals without additional filtering. Our experiments revealed lower performance when filters were applied during training or prediction, corroborating evidence from prior publications that DL models can inherently handle noisy inputs [ 8 , 14 , 34 ]. Thus, the 1D-ResNet effectively learns diagnostic features directly from the unprocessed waveforms, leveraging its robust architecture to mitigate noise disturbances while retaining clinically valuable signal characteristics. 2.4 XAI Methods We used the BenchXAI package [ 25 ] to generate relevance attributions with 15 different local post-hoc XAI methods. We applied all of these methods to the samples from the PTB XL dataset to generate relevance attributions for each sample (local) after the model was trained (post-hoc). In addition, we used the Guided Grad-CAM [ 38 ] XAI method which only works with convolutional network layers. The following XAI methods were used in this study. A shorter summary of the methods can be found in Appendix Section A.2. 2.4.1 Backpropagation-based methods Backpropagation-based XAI methods utilize the backpropagation mechanism of neural networks to propagate either the gradient of the network with respect to the input or other values, such as the network’s output. This allows them to assign attributions throughout the network, ultimately reaching the input layer. Simonyan et al. introduced the Input × Gradient method in the context of image classification to generate saliency maps as explanations [ 44 ]. In this approach, they relied on the absolute values of the gradients. The use of gradients is intuitive, as gradients indicate how a change in each input dimension affects the model’s output. By multiplying the gradient by the corresponding input value, the method provides an attribution score that reflects both the input’s influence on the output and its magnitude within the input space. This method is computationally efficient, as the gradient is inherently calculated during backpropagation. However, it has limitations, as relying on gradients alone can be simplistic and may lead to misleading results by pointing to local optima of the model’s output. Similar to Input × Gradient, the Deconvolution method [ 56 ] calculates the gradient of the model output with respect to the input. The primary difference between the two methods is that Deconvolution applies a rectified linear unit (ReLU) activation function to the relevance attributions during backpropagation, but only if a ReLU activation was used in the corresponding layer during the forward pass through the network. The Guided Backpropagation method [ 45 ] merges the Input × Gradient [ 43 ] and Deconvolution [ 56 ] methods. It achieves this by backpropagating relevance attributions only when both the activation during the forward pass and the gradient during the backward pass are positive. The Guided Grad-CAM method combines the Grad-CAM [ 38 ] and guided backpropagation methods to create more detailed and localized visual explanations. Grad-CAM generates a coarse heatmap by computing gradients of the output with respect to a specific convolutional layer. Guided backpropagation refines this heatmap by incorporating fine-grained feature information, producing higher resolution explanations that combine spatial importance with detailed gradients. Sundararajan et al. [ 47 ] introduced Integrated Gradients , a method that calculates the average gradient along a straight-line path from a baseline to the input. It attributes importance to each feature by integrating these gradients, providing a more stable and theoretically grounded explanation compared to simpler gradient methods. Bach et al. [ 5 ] introduced Layerwise Relevance Propagation (LRP) , a method that calculates relevance attributions by decomposing a neural network’s output layer by layer, propagating relevance backward through the network until it reaches the input. The decomposition ensures that, for any layer, the relevance attributions for all neurons sum up to the network’s output. To stabilize the method and avoid unbounded attributions caused by very small relevance attributions, they proposed adding a small stabilizing term, ϵ > 0, referred to as LRP - ϵ . Another stabilization method proposed in [ 5 ] is LRP - α 1- β 0, which separates positive and negative pre-activations, propagating only the positive ones. Additionally, LRP - γ , introduced in [ 29 ], assigns greater importance to positive relevance attributions. DeepLIFT (Deep Learning Important FeaTures), introduced by Shrikumar et al. [ 42 ], operates similarly to the Integrated Gradients method. While Integrated Gradients calculates the average partial derivative of each feature along a straight-line path from the baseline to the input, DeepLIFT simplifies this process by approximating the same quantity in a single step, replacing the gradient at each non-linearity with its average gradient. 2.4.2 Perturbation-based methods Perturbation-based XAI methods modify an input sample to observe how these changes impact the neural network’s output. This is often done by replacing specific features or regions, such as pixels in an image, with a baseline value, typically set to zero In the context of image classification, Occlusion was introduced by Zeiler et al. [ 56 ]. This method involves masking one or more features/pixels in the input, performing a forward pass through the network, and computing the difference in outputs as the relevance attribution. For image or signal data, occlusion can be applied to individual pixels or by sliding a feature mask across the image, setting all pixels within the mask to a baseline value, and calculating the resulting output difference. Shapley Value Sampling was introduced by Strumbelj et al. [ 46 ] as a faster sampling based approach to calculate Shapley Values. Shapley values [ 39 , 40 ] come from the field of cooperative game theory and calculate the average marginal contribution of a feature value across all possible subsets of features. Lundberg and Lee [ 21 ] introduced SHAP (SHapley Additive ex-Planations) values, which represent the Shapley values of a conditional expectation function of the original model. GradientSHAP [ 21 ] approximates SHAP values by computing expectations of gradients using random sampling from a distribution of baseline values, serving as an approximation of the Integrated Gradients method. KernelSHAP [ 21 ], another SHAP-based method, uses the LIME [ 37 ] approach to approximate SHAP values. Finally, DeepLiftSHAP [ 21 ] employs the DeepLIFT method [ 42 ] to approximate SHAP values by rewriting the DeepLIFT multipliers in terms of SHAP values (see [ 21 ] for detailed formulas). 2.4.3 Surrogate methods Surrogate XAI methods involve training interpretable models, such as linear regression, to approximate the behavior of a neural network. Due to the complexity of neural networks, these surrogate models can only approximate the network locally around a specific input sample, providing a sufficiently accurate approximation and making the surrogate model’s parameters interpretable. Ribeiro et al. [ 37 ] introduced the LIME (Local Interpretable Model-agnostic Explanations) method, which trains an interpretable model locally around a specific prediction. The core concept of LIME is the fidelity-interpretability trade-off: the surrogate model must faithfully approximate the behavior of the black-box model in the local region around the input while maintaining low complexity to remain human-interpretable. As a model-agnostic approach, LIME can be applied to any AI model. 2.5 XAI Evaluation 2.5.1 Preprocessing Since analyzing individual ECG recordings is not only time-consuming but also does not allow for generalization, we decided to perform a quantitative analysis. First, we calculated the average heartbeats for each ECG from the PTB XL dataset by using the widely used XQRS() function of the Python WFDB package [ 54 ] for R-Peak detection and then performed a majority vote to ensure that they were detected in multiple leads filtering false detected peaks robustly. Afterward, we removed all samples with more than 10 overlapping heartbeats. The heartbeats have a length of 650ms, 250ms before the R-Peak, and 400ms after the R-Peak. For all remaining samples, the average heartbeat is then calculated as the mean overall validated heartbeats for each lead. Within each average heartbeat, we analysed four clinically relevant regions: the P-wave, QRS complex, R-peak, and T-wave. Since the actual position of these waves varies between individuals and with heart rate, clinical definitions rely on morphological rather than fixed temporal criteria [ 32 ]. For a reproducible quantitative analysis across many samples, however, fixed intervals are required. We therefore defined the following intervals based on a cardiological assessment of representative ECGs from the PTB-XL dataset, chosen to reliably contain the respective waveform in the majority of cases: P-wave from 210ms to 70ms before the R-peak (140ms), QRS complex from 60ms before to 90ms after the R-peak (150ms), R-peak region ±20ms around the R-peak, and T-wave from 180ms to 350ms after the R-peak (170ms). Figure 2 shows an example of an average heartbeat with relevant regions being highlighted. Download figure Open in new tab Figure 2: Example of average heartbeats of a patient with Complete Right Bundle Branch Block (CRBBB) with clinically relevant P-wave, QRS complex, R-peak region (red), and T-wave regions highlighted. The corresponding relevances for all heartbeats are then identically averaged to create average relevance attributions for each average heartbeat. Since the ranges of relevance attributions vary greatly across different post-hoc XAI methods, we decided to normalize each average ECG relevance attribution using the same rank normalization employed in [ 25 ]. This will normalize all relevance attributions into the range [−1, 1] while keeping the correct order of relevance attributions within each averaged ECG. After normalizing the relevance attributions, we decided to focus our further evaluation on lead V1 instead of on all twelve leads. From a clinical standpoint, Lead V1 is often emphasized because of its anatomic orientation typically captures a strong signal from the right atrium and right ventricle. This proximity makes V1 especially sensitive to conduction and rate abnormalities, enabling clearer detection of the characteristic waveforms and timing changes pertinent to the six target arrhythmias [ 48 ]. Focusing on a single lead further enables a more concise and interpretable analysis of the XAI methods. However, we acknowledge that this choice may limit the generalizability of our observations across all leads, and extending the analysis to additional leads represents an important direction for future work. To evaluate the normalized XAI relevance attributions for the four clinically relevant regions, we first plotted them using box-plots to gauge their overall distribution for each XAI method and for all correctly classified samples for each disease. 2.5.2 Statistical Analysis Afterward, we performed a one-sided Wilcoxon signed-rank test [ 41 , 52 ] to analyse whether the mean relevance of a clinically relevant region was significantly larger than that of each of the other regions. Let X be the mean relevance of one region of the ECG (e.g., P-Wave) and let Y be the mean relevance of another region of the same ECG (e.g., QRS-Complex). The two-sided null hypothesis can be written as H 0 : „median ( X ) = median ( Y ) “meaning there is no difference between the medians of the two distributions with H 1 : „median ( X ) ≠ median ( Y )”. The one-sided test tests whether one distribution is significantly larger than the other. We then plotted the p-values of these tests in a heatmap to show which region had significantly larger attributions than another region. A significant p-value (<0.05) indicates the rejection of the null hypotheses for a specific XAI method. Because of repeated testing for each clinically relevant region, XAI method, and disease, all p-values were corrected using the Bonferroni correction [ 11 ] to reduce the likelihood of making a Type-1-error (incorrectly rejecting H 0 ). Lastly, we performed a one-sided Mann–Whitney U test [ 41 , 52 ] to test for significant differences in relevance attributions of the clinically relevant regions between correctly and incorrectly classified ECGs, more specifically between true positives and false positives. Let X and Y be the relevances of two different ECGs but for the same region (e.g., QRS-Complex). A general formulation of the two-sided null hypothesis can be expressed as H 0 : „median ( X ) = median ( Y ) “meaning the median of the relevances X represents equals the median of the relevances Y represents with an alternative hypothesis H 1 : „median ( X ) ≠ median ( Y )”. The one-sided version of this test tests whether one sample distribution is significantly larger or smaller than the other. A significant p-value (<0.05) indicates the rejection of the null hypotheses for a specific region and XAI method. Because of repeated testing for each clinically relevant region, XAI method, and disease, all p-values were corrected using the Bonferroni correction [ 11 ]. 3 Results First, we analyzed the performance of our trained model on the PTB XL dataset. As our XAI analysis focuses on correctly identified disease cases, sensitivity is the primary performance metric of interest, complemented by TP and FP counts to provide additional context on model behavior. Overall, our trained model had high sensitivity scores of > 0.9 for most diseases, except 1AVB and SBRAD, while AFIB and SBRAD had the fewest false positives (see Table 1 ). View this table: View inline View popup Download powerpoint Table 1: Table with information on PTB-XL dataset after preprocessing according to Section 2.5.1 . It summarizes descriptive performance metrics across disease classes. When looking at the overall distributions of normalized relevance attributions for CRBBB (see Figure 3 ) we found that the LIME method produced mainly zero attributions with only a few out-liers, while the LRP methods (LRP, LRP- ϵ , LRP- γ , and LRP- α 1- β 0) had median attributions of zero, with a larger interquartile range. Moreover, KernelShap, LIME, and the LRP methods showed almost no difference in median attributions between all four clinically relevant regions (P-wave, QRS complex, R-Peak region, T-wave). We found that this was also true for the other five diseases (see Appendix Section A.3). Since these six XAI methods were not able to sufficiently discern between the relevant regions for all diseases, we have decided to disregard them for the remainder of the analysis. Download figure Open in new tab Figure 3: Boxplots showing the distribution of normalised relevance attributions for all true positive CRBBB samples for all 16 post-hoc XAI methods and all four clinically relevant regions. To validate how well or even if our model learned clinically relevant features for each disease we used a one-sided wilcoxon signed-rank test to observe whether the clinically relevant features had significantly higher relevance attributions then the other three features (p-value < 0.05, see Section 2.5.2 ). We then performed a majority vote over the results for all remaining ten XAI methods to get a more robust decision on whether the model learned the correct features for each disease. Heatmaps with p-values for each disease, feature, and XAI method can be found in Appendix Section A.4. For 1AVB, AFIB the P-wave is the most relevant feature, while for CRBBB and CLBBB, the QRS complex is the most relevant feature. We did not consider the diseases SBRAD and STACH for this analysis, since they are rhythm based diagnoses, which are typically diagnosed across multiple heartbeats (and we are currently looking at an averaged heartbeat). Conduction and rate abnormalities – such as 1AVB, AFIB – primarily originating in the atria affect the morphology or timing of the P-wave, while conduction blocks located in the ventricles like CRBBB and CLBBB predominantly alter the QRS complex. These ECG features align with the standard interpretation guidelines [ 48 ] as key diagnostic signatures for each respective arrhythmia. In Table 2 , we can see that based on the majority vote of all ten XAI methods, our model was able to correctly learn clinically relevant features for AFIB, CRBBB. For these diseases, the relevant features had significantly larger relevance attributions more than 50% of the time. Additionally, the results from Table 2 clearly show that the XAI methods DeepLift, DeepLiftShap, and Occlusion were the only ones able to correctly identify the clinically relevant features over all four diseases, while Guided Backpropagation and Integrated Gradients only failed to do so for CLBBB and AFIB respectively. View this table: View inline View popup Download powerpoint Table 2: Table showing counts on how often each relevant feature had significantly larger relevance attributions than the other features (p-value < 0.05) for 1AVB, AFIB, CRBBB, and CLBBB and all XAI methods. Since there are three other features in total and ten XAI methods, a total of 30 significantly larger p-values could have been achieved for each disease. The percentage value used for the majority vote is calculated by dividing the total count for each disease by 30. A positive majority vote (>50%) is highlighted. Lastly, we wanted to analyze whether there are significant differences in relevance attributions between true positive and false positive classified samples for each disease. To test this, we performed a one-sided Mann–Whitney U test (see Section 2.5.2 ). When looking at the heatmaps in Figure 4 , we would have expected significant differences (p-value < 0.05) in P-wave attributions for 1AVB, AFIB. However, there was only a majority (6 out of 10 XAI methods) of significant differences for AFIB. For SBRAD and STACH most significant differences were found in the P-Wave region, even though we did not expect to find any significant differences for these diseases since they are clinically diagnosed over multiple hearbeats and not an average heartbeat, which the XAI methods are looking at. However, since the underlying AI model is trained on the full length ECG, it could indicate that the model learned the distances between P-Waves of multiple heartbeats. For SBRAD, it should be noted that there were only 28 false positive samples (see Table 1 ), which could explain the lack of significant p-values. For CLBBB and CRBBB, there should be a majority of significant differences in the QRS complex, but in our case, this is only true for CRBBB (8 out of 10 XAI methods). The underlying ECG signals (see Figure 4 top) show representative examples for all six diseases. In 1AVB, the attributions highlight the P-wave where the electric impulse is delayed. For AFIB, the absence of the P-wave is indicated by the attributions. In the bundle branch blocks (CLBBB and CRBBB), the broad QRS is visible and shows high attributions. For the rhythm disorders (SBRAD, STACH) the attributions are evenly spaced across the averaged heartbeat, because these disorders are typically not diagnosed over an averaged heartbeat, but over multiple heartbeats. Download figure Open in new tab Figure 4: Top: Exemplary averaged heartbeats from lead V1 for each disease class. The P-wave, QRS complex, R-peak region (red), and T-wave are indicated as physiologically relevant segments. Integrated Gradients attributions are overlaid as area plots to visualize feature relevance across the cardiac cycle. Bottom: Heatmaps of statistical significance (one-sided Mann–Whitney U test) comparing relevance attributions between true positive and false positive samples across features, XAI methods, and disease classes. Significant results indicate rejection of the null hypothesis that both distributions are identical in favor of higher relevance in true positives. P-values are Bonferroni-corrected for multiple comparisons. To improve readability across very small values, p-values are grouped into magnitude bins and displayed using the notation 0*, 0**, and 0*** corresponding to p < 0.05, p < 1e-20, and p < 1e-50, respectively. 4 Discussion In this study, we demonstrated the potential of gaining more robust explanations on the decision-making process of a DL model by using an ensemble approach of post-hoc XAI methods. Additionally, we showed the importance of selecting appropriate XAI methods for a specific classification task, as well as their capabilities to identify false positive samples. Rather than providing a clinical decision-support tool, our work is intended to support the informed selection and interpretation of XAI methods when applied to ECG classification models. We found that LRP-type XAI methods require careful parameter tuning to produce stable and interpretable relevance attributions, as well as KernelShap and LIME. A possible explanation for this may lie in the structural differences between image time-series data. While these methods have been widely validated in computer vision tasks, ECG signals have strong temporal dependencies that do not match well with the patch- or pixel-based approaches used by many of these methods. In particular, perturbation-based methods such as LIME and KernelSHAP may generate out-of-distribution samples when applied to temporal data, potentially leading to un-stable or physiologically implausible attributions [ 9 , 18 , 23 , 51 ]. Improvements of these methods in this domain is an active field of research, with many adaptations being made to the algorithm [ 2 , 23 , 24 ]. For the LRP methods, this behavior is also noticed by other researchers [ 16 , 25 ]. As suggested by Kohlbrenner et al. [ 16 ], using different LRP methods for different neural network layers in combination could improve their performance. Our ensemble majority vote approach using the XAI methods Deconvolution, DeepLift, DeepLiftShap, GradientShap, Guided Backpropagation, GuidedGradCam, Input×Gradient, Integrated Gradients, Occlusion, and ShapleyValueSampling, suggest that the model captures features that are consistent with clinically relevant patterns for AFIB, CRBBB. This majority vote approach gives us more robust explanations since post-hoc XAI methods by themselves have been shown to be very sensitive to small changes in the data or model [ 20 , 55 ]. When looking at the XAI methods by themselves, only DeepLift, DeepLiftShap, and Occlusion showed that our model might have learned clinically relevant features for all diseases. Additionally, our ensemble approach showed that for our model and dataset, the 10 remaining XAI methods could potentially be employed to discern between true positive and false positive classified samples for diseases AFIB and CRBBB. This capability highlights the potential of XAI methods as diagnostic tools for model inspection and error analysis, particularly in distinguishing false positive from true positive predictions. These differences can not only improve our understanding of the model’s strengths and weaknesses but also support the development of more transparent, reliable, and robust AI systems. We note that the number of false positive samples in the SBRAD class is limited (n = 28), which may reduce the statistical power and increase uncertainty in subgroup-specific statistical comparisons. Additionally, the use of fixed temporal intervals represents a limitation of our approach, and future work will explore morphology-based segmentation to better capture inter-individual variability. Especially for rhythm diagnoses such as SBRAD and STACH, which are defined by R–R interval across multiple heartbeats, restricting the analysis to an averaged heartbeat may limit the ability of XAI methods to identify meaningful relevance. The comparison to other studies demonstrates that the performance of XAI methods might differ between various DL models since methods like GradCam, which underperforms in our experimental setting, are successfully employed with multiple datasets and DL models in other ECG classification tasks [ 3 , 33 ]. While our comprehensive evaluation is based on one of the largest ECG datasets, multiple diseases, and a variety of XAI methods, the lack of additional datasets and DL architectures demonstrates a limitation of our current study. Therefore, evaluating the proposed ensemble post-hoc XAI framework in more external datasets and the integration of additional DL architectures is an avenue for future work. Lastly, we acknowledge that the several gradient-based XAI methods in our ensemble may produce correlated attributions. However, the ensemble majority-vote approach is intended as a robustness strategy rather than a statistically independent aggregation, and exploring inter-method similarity remains an important direction for future work. Future work could also explore alternative ensemble aggregation strategies beyond the majority-vote threshold (≥50%), which was used in this study as a simple and interpretable criterion ensuring that only features supported by more than half of the XAI methods are considered robust. In summary, we show the importance of using ensemble XAI approaches to enhance the interpretability and reliability of DL models, supporting the development of more transparent and interpretable AI systems for healthcare applications. Our results indicate that no single XAI method consistently outperforms others across all settings; instead, ensemble-based interpretation or careful task-dependent selection may provide more reliable insights. 5 Conclusion This study demonstrates the effectiveness of ensemble post-hoc XAI methods in improving the interpretability, robustness, and clinical relevance of deep learning models for ECG classification. By aggregating outputs from multiple explanation techniques, we were able to identify consistent, meaningful patterns associated with different heart abnormalities. The results emphasize the importance of method selection and parameter tuning in XAI applications, and show that ensemble-based strategies can mitigate the limitations of individual explanation methods. In particular, the ability of our ensemble approach to detect false positives offers promising opportunities for model debugging and refinement. Future research should focus on applying this ensemble XAI framework to different datasets and model architectures to evaluate its generalizability. Expanding the diversity of test conditions will help determine the reliability and practical applicability of ensemble explanations in real-world clinical settings. Ultimately, our findings support the integration of ensemble XAI techniques as a key component in the development of trustworthy and transparent AI systems for healthcare. 6 Declarations 6.1 Ethics approval and consent to participate Not applicable. 6.2 Consent for publication Not applicable. 6.3 Availability of data and materials The data that support the findings of this study are available from https://doi.org/10.17044/scilifelab.15169716.v1 but restrictions apply to the availability of these data, which were used under license for the current study, and so are not publicly available. Data are however available from the authors upon reasonable request and with permission of https://doi.org/10.17044/scilifelab.15169716.v1 . The code for our analysis is available under https://github.com/HauschildLab/EnsembleXAIForTime . 6.4 Competing interests The authors declare that they have no competing interests. 6.5 Funding This work is supported in part by funds from the German Ministry of Education and Research (BMBF) under grant agreement No. 01KD2208A and No. 01KD2414A (project FAIrPaCT), by the Innovation Committee at the Federal Joint Committee NO. 01VSF20014 (project KI-Thrust) and by the Deutsche Forschungsgemeinschaft (DFG) under project number 567400630. In addition, this work was partially funded by the Lower Saxony “Vorab” of the Volkswagen Foundation and the Ministry for Science and Culture of Lower Saxony, Germany under Grant 76211-12-1/21. 6.6 Authors’ contributions A-C.H., N.S., P.H. and J.M.M conceptualised the project. M.C.M. was responsible for the data acquisition. P.H. implemented the training of the AI model. J.M.M. performed all the XAI analysis. J.M.M. and P.H. generated the figures and prepared the results. P.H. and K.E.S. validated the results clinically. J.M.M., P.H., M.C.M., and K.E.S. wrote the manuscript. All authors performed review and editing. 6.7 Acknowledgements This work used the Scientific Compute Cluster at GWDG, the joint data centre of Max Planck Society for the Advancement of Science (MPG) and University of Göttingen. A Appendix A.1 Data View this table: View inline View popup Download powerpoint Table 3: Comparison of cohort statistics and ECG abnormalities between CODE and PTB-XL. Values are n (%) unless otherwise specified. A.2 XAI Methods View this table: View inline View popup Download powerpoint Table 4: Overview of the 16 XAI methods used in this study, grouped by methodological category. A.3 Boxplots Download figure Open in new tab Figure 5: Boxplots showing distribution of normalised relevance attributions for all true positive 1AVB samples for all 16 post-hoc XAI methods and all four clinically relevant regions. Download figure Open in new tab Figure 6: Boxplots showing distribution of normalised relevance attributions for all true positive AFIB samples for all 16 post-hoc XAI methods and all four clinically relevant regions. Download figure Open in new tab Figure 7: Boxplots showing distribution of normalised relevance attributions for all true positive LBBB samples for all 16 post-hoc XAI methods and all four clinically relevant regions. Download figure Open in new tab Figure 8: Boxplots showing distribution of normalised relevance attributions for all true positive SBRAD samples for all 16 post-hoc XAI methods and all four clinically relevant regions. Download figure Open in new tab Figure 9: Boxplots showing distribution of normalised relevance attributions for all true positive STACH samples for all 16 post-hoc XAI methods and all four clinically relevant regions. A.4 Wilcoxon Download figure Open in new tab Figure 10: Heatmaps showing significant p-values (<0.05) when performing a one sided Wilcoxon test between all features for all ten XAI methods on all true positive 1AVB samples. A significant p-value indicates the rejection of H 0 : median(mean(Relevance region 1) - mean(Relevance region 2)) = 0 in favor of H 1 : median(mean(Relevance region 1) - mean(Relevance region 2)) > 0, indicating that feature 1 is significantly larger than feature 2. P-values were corrected using Bonferroni correction to account for multiple testing. 0***: p-value < 1e-50, 0**: p-value < 1e-20, 0*: p-value < 0.05. Download figure Open in new tab Figure 11: Heatmaps showing significant p-values (<0.05) when performing a one sided Wilcoxon test between all features for all ten XAI methods on all true positive AFIB samples. A significant p-value indicates the rejection of H 0 : median(mean(Relevance region 1) - mean(Relevance region 2)) = 0 in favor of H 1 : median(mean(Relevance region 1) - mean(Relevance region 2)) > 0, indicating that feature 1 is significantly larger than feature 2. P-values were corrected using Bonferroni correction to account for multiple testing. 0***: p-value < 1e-50, 0**: p-value < 1e-20, 0*: p-value < 0.05. Download figure Open in new tab Figure 12: Heatmaps showing significant p-values (<0.05) when performing a one sided Wilcoxon test between all features for all ten XAI methods on all true positive LBBB samples. A significant p-value indicates the rejection of H 0 : median(mean(Relevance region 1) - mean(Relevance region 2)) = 0 in favor of H 1 : median(mean(Relevance region 1) - mean(Relevance region 2)) > 0, indicating that feature 1 is significantly larger than feature 2. P-values were corrected using Bonferroni correction to account for multiple testing. 0***: p-value < 1e-50, 0**: p-value < 1e-20, 0*: p-value < 0.05. Download figure Open in new tab Figure 13: Heatmaps showing significant p-values (<0.05) when performing a one sided Wilcoxon test between all features for all ten XAI methods on all true positive RBBB samples. A significant p-value indicates the rejection of H 0 : median(mean(Relevance region 1) - mean(Relevance region 2)) = 0 in favor of H 1 : median(mean(Relevance region 1) - mean(Relevance region 2)) > 0, indicating that feature 1 is significantly larger than feature 2. P-values were corrected using Bonferroni correction to account for multiple testing. 0***: p-value < 1e-50, 0**: p-value < 1e-20, 0*: p-value < 0.05. Footnotes Major revisions to the manuscript. Title changed, clinical evaluation added, discussion updated. References [1]. ↵ Eu ai act . https://artificialintelligenceact.eu/ . Accessed: 2025-03-03 . [2]. ↵ Talal AA Abdullah , Mohd Soperi Mohd Zahid , Waleed Ali , and Shahab Ul Hassan . B-lime: An improvement of lime for interpretable deep learning classification of cardiac arrhythmia from ecg signals . Processes , 11 ( 2 ): 595 , 2023 . OpenUrl [3]. ↵ C Apama , P Rohini , and V Pandiyarasan . Interpretation of resnet50 model for mi related cardiac events using explainable grad-cam approach . In Current Directions in Biomedical Engineering , volume 8 , pages 723 – 726 . De Gruyter , 2022 . OpenUrl [4]. ↵ Bauke K O Arends , Wouter A C van Amsterdam , Pim van der Harst , Maarten van Smeden , René van Es , and Rutger R van de Leur . Signal or noise? evaluating commonly used attribution methods for explaining deep neural networks in electrocardiogram classification . European Heart Journal - Digital Health , 7 ( 3 ): ztag038 , 04 2026 . OpenUrl [5]. ↵ Sebastian Bach , Alexander Binder , Grégoire Montavon , Frederick Klauschen , Klaus-Robert Müller , and Wojciech Samek . On pixel-wise explanations for non-linear classifier decisions by layer-wise relevance propagation . PLOS ONE , 10 ( 7 ): 1 – 46 , 07 2015 . OpenUrl CrossRef PubMed [6]. ↵ Jacqueline Michelle Beinecke , Patrick Anders , Tino Schurrat , Dominik Heider , Markus Luster , Damiano Librizzi , and Anne-Christin Hauschild . Evaluation of machine learning strategies for imaging confirmed prostate cancer recurrence prediction on electronic health records . Computers in Biology and Medicine , 143 : 105263 , 2022 . OpenUrl CrossRef PubMed [7]. ↵ Theresa Bender , Jacqueline M Beinecke , Dagmar Krefting , Carolin Müller , Henning Dathe , Tim Seidler , Nicolai Spicher , and Anne-Christin Hauschild . Analysis of a deep learning model for 12-lead ecg classification reveals learned features similar to diagnostic criteria . IEEE Journal of Biomedical and Health Informatics , 28 ( 4 ): 1848 – 1859 , 2023 . OpenUrl [8]. ↵ Theresa Bender , Philip Hempel (formerly Gemke), Ennio Idrobo-Ávila , Henning Dathe , Dagmar Krefting , and Nicolai Spicher . Benchmarking the impact of noise on deep learning-based classification of atrial fibrillation in 12-lead ecg . Studies in Health Technology and Informatics , 302 : 977 – 981 , 05 2023 . OpenUrl PubMed [9]. ↵ Yu-Neng Chuang , Guanchu Wang , Fan Yang , Quan Zhou , Pushkar Tripathi , Xu-anting Cai , and Xia Hu . Cortx: Contrastive framework for real-time explanation . arXiv preprint arXiv:2303. 02794 , 2023 . [10]. ↵ Alex J DeGrave , Joseph D Janizek , and Su-In Lee . Ai for radiographic covid-19 detection selects shortcuts over signal . Nature Machine Intelligence , 3 ( 7 ): 610 – 619 , 2021 . OpenUrl [11]. ↵ Olive Jean Dunn . Multiple comparisons among means . Journal of the American statistical association , 56 ( 293 ): 52 – 64 , 1961 . OpenUrl CrossRef Web of Science [12]. ↵ Rudresh Dwivedi , Pranay Kothari , Deepak Chopra , Manjot Singh , and Ritesh Kumar . An efficient ensemble explainable ai (xai) approach for morphed face detection . Pattern Recognition Letters , 184 : 197 – 204 , 2024 . OpenUrl [13]. ↵ Rokas Gipiškis , Chun-Wei Tsai , and Olga Kurasova . Explainable ai (xai) in image segmentation in medicine, industry, and beyond: A survey . ICT Express , 10 ( 6 ): 1331 – 1354 , 2024 . OpenUrl [14]. ↵ Philip Hempel , Antônio H Ribeiro , Marcus Vollmer , Theresa Bender , Marcus Dörr , Dagmar Krefting , and Nicolai Spicher . Explainable ai associates ecg aging effects with increased cardiovascular risk in a longitudinal population study . npj Digital Medicine , 8 ( 1 ): 25 , 2025 . OpenUrl PubMed [15]. ↵ Lukas Klein , Carsten T. Lüth , Udo Schlegel , Till J. Bungert , Mennatallah El-Assady , and Paul F. Jäger . Navigating the Maze of Explainable AI: A Systematic Approach to Evaluating Methods and Metrics , 2024 . Version Number: 3. [16]. ↵ Maximilian Kohlbrenner , Alexander Bauer , Shinichi Nakajima , Alexander Binder , Wojciech Samek , and Sebastian Lapuschkin . Towards best practice in explaining neural network decisions with lrp . 2020 International Joint Conference on Neural Networks (IJCNN) , pages 1 – 7 , 2019 . [17]. ↵ Narine Kokhlikyan , Vivek Miglani , Miguel Martin , Edward Wang , Bilal Alsallakh , Jonathan Reynolds , Alexander Melnikov , Natalia Kliushkina , Carlos Araya , Siqi Yan , and Orion Reblitz-Richardson . Captum: A unified and generic model interpretability library for pytorch , 2020 . [18]. ↵ Kin Kwan Leung , Clayton Rooke , Jonathan Smith , Saba Zuberi , and Maksims Volkovs . Temporal dependencies in feature importance for time series predictions . arXiv preprint arXiv:2107. 14317 , 2021 . [19]. ↵ Emilly M Lima , Antônio H Ribeiro , Gabriela MM Paixão Manoel Horta Ribeiro , Marcelo M Pinto-Filho , Paulo R Gomes , Derick M Oliveira , Ester C Sabino , Bruce B Duncan , Luana Giatti , et al. Deep neural network-estimated electro-cardiographic age as a mortality predictor . Nature communications , 12 ( 1 ): 5117 , 2021 . OpenUrl PubMed [20]. ↵ Luca Longo , Mario Brcic , Federico Cabitza , Jaesik Choi , Roberto Confalonieri , Javier Del Ser , Riccardo Guidotti , Yoichi Hayashi , Francisco Herrera , Andreas Holzinger , et al. Explainable artificial intelligence (xai) 2.0: A manifesto of open challenges and interdisciplinary research directions . Information Fusion , 106 : 102301 , 2024 . [21]. ↵ I. Guyon , U. Von Luxburg , S. Bengio , H. Wallach , R. Fergus , S. Vishwanathan , and R. Garnett Scott M Lundberg and Su-In Lee . A unified approach to interpreting model predictions . In I. Guyon , U. Von Luxburg , S. Bengio , H. Wallach , R. Fergus , S. Vishwanathan , and R. Garnett , editors, Advances in Neural Information Processing Systems , volume 30 . Curran Associates, Inc ., 2017 . [22]. ↵ Miriam Cindy Maurer , Jacqueline Michelle Metsch , Philip Hempel , Theresa Bender , Nicolai Spicher , and Anne-Christin Hauschild . Explainable Artificial Intelligence on Biosignals for Clinical Decision Support . In Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining, KDD ‘24 , pages 6597 – 6604 , New York, NY, USA , August 2024 . Association for Computing Machinery . [23]. ↵ Han Meng , Christian Wagner , and Isaac Triguero . An initial step towards stable explanations for multivariate time series classifiers with lime . In 2023 IEEE International Conference on Fuzzy Systems (FUZZ) , pages 1 – 6 . IEEE , 2023 . [24]. ↵ Han Meng , Christian Wagner , and Isaac Triguero . Segal time series classification—stable explanations using a generative model and an adaptive weighting method for lime . Neural Networks , 176 : 106345 , 2024 . [25]. ↵ Jacqueline Michelle Metsch and Anne-Christin Hauschild . Benchxai: Comprehensive benchmarking of post-hoc explainable ai methods on multi-modal biomedical data . bioRxiv , 2024 . [26]. ↵ Jacqueline Michelle Metsch , Anna Saranti , Alessa Angerschmid , Bastian Pfeifer , Vanessa Klemt , Andreas Holzinger , and Anne-Christin Hauschild . Clarus: An interactive explainable ai platform for manual counterfactuals in graph neural networks . Journal of Biomedical Informatics , 150 : 104600 , 2024 . [27]. ↵ Ebikella Mienye , Nobert Jere , George Obaido , Ibomoiye Domor Mienye , and Kehinde Aruleba . Deep learning in finance: A survey of applications and techniques. AI , 5 ( 4 ): 2066 , 2024 . OpenUrl [28]. ↵ Riccardo Miotto , Fei Wang , Shuang Wang , Xiaoqian Jiang , and Joel T Dudley . Deep learning for healthcare: review, opportunities and challenges . Briefings in bioinformatics , 19 ( 6 ): 1236 – 1246 , 2018 . OpenUrl CrossRef PubMed [29]. ↵ Grégoire Montavon , Alexander Binder , Sebastian Lapuschkin , Wojciech Samek , and Klaus-Robert Müller . Layer-Wise Relevance Propagation: An Overview , pages 193 – 209 . Springer International Publishing , Cham , 2019 . [30]. ↵ Paul Vincent Naser , Miriam Cindy Maurer , Maximilian Fischer , Kianush Karimian-Jazi , Chiraz Ben-Salah , Awais Akbar Bajwa , Martin Jakobs , Christine Jungk , Jessica Jesser , Martin Bendszus , Klaus Maier-Hein , Sandro M. Krieg , Peter Neher , and Jan-Oliver Neumann . Deep learning aided preoperative diagnosis of primary central nervous system lymphoma . iScience , 27 ( 2 ): 109023 , February 2024 . OpenUrl PubMed [31]. ↵ Ursula Neumann , Nikita Genze , and Dominik Heider . Efs: an ensemble feature selection tool implemented as r-package and web-application . BioData mining , 10 : 1 – 9 , 2017 . OpenUrl PubMed [32]. ↵ H. K. Walker , W. D. Hall , and J. W. Hurst R. J. Noble , J. S. Hillis , and D. A. Rothbaum . Electrocardiography . In H. K. Walker , W. D. Hall , and J. W. Hurst , editors, Clinical Methods: The History, Physical, and Laboratory Examinations, chapter 33 . Butterworths , Boston , 3 edition, 1990 . [33]. ↵ Jaya Ojha , Hårek Haugerud , Anis Yazidi , and Pedro G Lind . Exploring interpretable ai methods for ecg data classification . In Proceedings of the 5th ACM Workshop on Intelligent Cross-Data Analysis and Retrieval , pages 11 – 18 , 2024 . [34]. ↵ Antônio H. Ribeiro , Manoel Horta Ribeiro Gabriela M. M. Paixão , Derick M. Oliveira , Paulo R. Gomes , Jéssica A. Canazart , Milton P. S. Ferreira , Carl R. Andersson , Peter W. Macfarlane , Wagner Meira Jr . ., Thomas B. Schön , and Antonio Luiz P. Ribeiro . Automatic diagnosis of the 12-lead ECG using a deep neural network . Nature Communications , 11 ( 1 ): 1760 , 2020 . OpenUrl PubMed [35]. Antonio H. Ribeiro , Manoel Horta Ribeiro Gabriela M. Paixão , Derick M. Oliveira , Paulo R. Gomes , Jéssica A. Canazart , Milton P. Ferreira , Carl R. Andersson , Peter W. Macfarlane , Wagner Meira Jr . ., Thomas B. Schön , and Antonio Luiz P. Ribeiro . CODE dataset . Dataset , 11 2021 . [36]. ↵ Antonio Luiz P Ribeiro , Gabriela MM Paixao , Paulo R Gomes , Manoel Horta Ribeiro , Antonio H Ribeiro , Jessica A Canazart , Derick M Oliveira , Milton P Ferreira , Emilly M Lima , Jermana Lopes de Moraes , et al. Tele-electrocardiography and bigdata: the code (clinical outcomes in digital electrocardiography) study . Journal of electrocardiology , 57 : S75 – S78 , 2019 . OpenUrl CrossRef [37]. ↵ Marco Ribeiro , Sameer Singh , and Carlos Guestrin . “why should I trust you?”: Explaining the predictions of any classifier . In Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Demonstrations , pages 97 – 101 , San Diego, California , June 2016 . Association for Computational Linguistics . [38]. ↵ Ramprasaath R. Selvaraju , Michael Cogswell , Abhishek Das , Ramakrishna Vedantam , Devi Parikh , and Dhruv Batra . Grad-cam: Visual explanations from deep networks via gradient-based localization . In 2017 IEEE International Conference on Computer Vision (ICCV) , pages 618 – 626 , 2017 . [39]. ↵ L. S. Shapley and Martin Shubik . A method for evaluating the distribution of power in a committee system . American Political Science Review , 48 ( 3 ): 787 – 792 , 1954 . OpenUrl CrossRef [40]. ↵ Lloyd S. Shapley . A Value for N-Person Games . RAND Corporation , Santa Monica, CA , 1952 . [41]. ↵ David J Sheskin . Handbook of parametric and nonparametric statistical procedures . Chapman and hall/CRC , 2003 . [42]. ↵ Avanti Shrikumar , Peyton Greenside , and Anshul Kundaje . Learning important features through propagating activation differences . In Proceedings of the 34th International Conference on Machine Learning - Volume 70 , ICML’17, page 3145 – 3153 . JMLR.org , 2017 . OpenUrl [43]. ↵ Avanti Shrikumar , Peyton Greenside , Anna Shcherbina , and Anshul Kundaje . Not just a black box: Learning important features through propagating activation differences , 2017 . [44]. ↵ Karen Simonyan , Andrea Vedaldi , and Andrew Zisserman . Deep inside convolutional networks: Visualising image classification models and saliency maps , 2014 . [45]. ↵ J.T. Springenberg , A. Dosovitskiy , T. Brox , and M. Riedmiller . Striving for simplicity: The all convolutional net . In ICLR (workshop track) , 2015 . [46]. ↵ Erik Strumbelj and Igor Kononenko . An efficient explanation of individual classifications using game theory . J. Mach. Learn. Res ., 11 : 1 – 18 , 2010 . OpenUrl [47]. ↵ Mukund Sundararajan , Ankur Taly , and Qiqi Yan . Axiomatic attribution for deep networks , 2017 . [48]. ↵ Borys Surawicz , Rory Childers , Barbara J Deal , and Leonard S Gettes . Aha/accf/hrs recommendations for the standardization and interpretation of the electrocardiogram: part iii: intraventricular conduction disturbances: a scientific statement from the american heart association electrocardiography and arrhythmias committee, council on clinical cardiology; the american college of cardiology foundation; and the heart rhythm society: endorsed by the international society for computerized electrocardiology . Circulation , 119 ( 10 ): e235 – e240 , 2009 . OpenUrl FREE Full Text [49]. ↵ Patrick Wagner , Nils Strodthoff , Ralf-Dieter Bousseljot , Dieter Kreiseler , Fatima I. Lunze , Wojciech Samek , and Tobias Schaeffter . PTB-XL, a large publicly available electrocardiography dataset . Scientific Data , 7 ( 1 ): 154 , May 2020 . OpenUrl PubMed [50]. ↵ Hongnian Wang , Mingyang Zhang , Liyi Mai , Xin Li , Abdelouahab Bellou , and Lijuan Wu . An effective multi-step feature selection framework for clinical outcome prediction using electronic medical records . BMC Medical Informatics and Decision Making , 25 ( 1 ): 84 , 2025 . OpenUrl [51]. ↵ Jiazhan Wang , Ruifeng Zhang , and Qiang Li . Tf-lime: Interpretation method for time-series models based on time–frequency features . Sensors , 25 ( 9 ): 2845 , 2025 . OpenUrl PubMed [52]. ↵ Frank Wilcoxon . Individual comparisons by ranking methods . Biometrics bulletin , 1 ( 6 ): 80 – 83 , 1945 . OpenUrl CrossRef Web of Science [53]. ↵ Di Wu , Sheng Liu , Wei Wei , and Yu Sui . A new satellite-ship autonomous communication system with an integrated deep learning anomaly detection algorithm . Multimedia Tools and Applications , pages 1 – 26 , 2024 . [54]. ↵ Chen Xie , Lucas McCullum , Alistair Johnson , Tom Pollard , Brian Gow , and Benjamin Moody . Waveform database software package (wfdb) for python . PhysioNet , 2022 . [55]. ↵ Chih-Kuan Yeh , Cheng-Yu Hsieh , Arun Suggala , David I Inouye , and Pradeep K Ravikumar . On the (in) fidelity and sensitivity of explanations . Advances in neural information processing systems , 32 , 2019 . [56]. ↵ David Fleet , Tomas Pajdla , Bernt Schiele , and Tinne Tuytelaars Matthew D. Zeiler and Rob Fergus . Visualizing and understanding convolutional networks . In David Fleet , Tomas Pajdla , Bernt Schiele , and Tinne Tuytelaars , editors, Computer Vision – ECCV 2014 , pages 818 – 833 , Cham , 2014 . Springer International Publishing . [57]. ↵ Lin Zou , Han Leong Goh , Charlene Jin Yee Liew , Jessica Lishan Quah , Gary Tianyu Gu , Jun Jie Chew , Mukundaram Prem Kumar , Christine Gia Lee Ang , and Andy Wee An Ta . Ensemble image explainable ai (xai) algorithm for severe community-acquired pneumonia and covid-19 respiratory infections . IEEE Transactions on Artificial Intelligence , 4 ( 2 ): 242 – 254 , 2022 . OpenUrl View the discussion thread. Back to top Previous Next Posted May 18, 2026. Download PDF Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Ensemble Post-hoc Explainable AI for Multilead ECG: Identifying Disease-Relevant Features in Single-Lead Interpretations Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Ensemble Post-hoc Explainable AI for Multilead ECG: Identifying Disease-Relevant Features in Single-Lead Interpretations Jacqueline Michelle Metsch , Philip Hempel , Miriam Cindy Maurer , Kristin Elisabeth Steinhaus , Nicolai Spicher , Anne-Christin Hauschild bioRxiv 2025.02.14.638219; doi: https://doi.org/10.1101/2025.02.14.638219 Share This Article: Copy Citation Tools Ensemble Post-hoc Explainable AI for Multilead ECG: Identifying Disease-Relevant Features in Single-Lead Interpretations Jacqueline Michelle Metsch , Philip Hempel , Miriam Cindy Maurer , Kristin Elisabeth Steinhaus , Nicolai Spicher , Anne-Christin Hauschild bioRxiv 2025.02.14.638219; doi: https://doi.org/10.1101/2025.02.14.638219 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Bioinformatics Subject Areas All Articles Animal Behavior and Cognition (7622) Biochemistry (17648) Bioengineering (13871) Bioinformatics (41880) Biophysics (21423) Cancer Biology (18558) Cell Biology (25460) Clinical Trials (138) Developmental Biology (13364) Ecology (19866) Epidemiology (2067) Evolutionary Biology (24290) Genetics (15589) Genomics (22475) Immunology (17711) Microbiology (40327) Molecular Biology (17145) Neuroscience (88473) Paleontology (666) Pathology (2827) Pharmacology and Toxicology (4816) Physiology (7635) Plant Biology (15114) Scientific Communication and Education (2044) Synthetic Biology (4286) Systems Biology (9815) Zoology (2268)