A Hybrid Deep Learning Ensemble for Accurate Skin Cancer Classification

A Hybrid Deep Learning Ensemble for Accurate Skin Cancer Classification | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search A Hybrid Deep Learning Ensemble for Accurate Skin Cancer Classification View ORCID Profile Alireza Rahi doi: https://doi.org/10.1101/2025.09.03.25335044 Alireza Rahi 1 Tehran , Iran Roles: Independent Researcher Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Alireza Rahi Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Skin cancer is one of the most common types of cancer worldwide, and early detection is crucial for improving patient survival rates. In this study, we propose a hybrid deep learning ensemble model for the automatic classification of dermoscopic images into benign and malignant categories. The framework integrates multiple deep learning architectures and combines their predictive strengths through a meta-learning approach. Experimental evaluations on a benchmark dataset demonstrated that the proposed ensemble achieved a classification accuracy of 91.7% and a ROC-AUC score of 0.974 , outperforming individual models. These results highlight the potential of hybrid ensemble methods as reliable computer- aided diagnostic tools for dermatology, contributing to early and accurate skin cancer detection. Introduction Skin cancer is among the most commonly diagnosed cancers worldwide, and melanoma remains its most aggressive and life-threatening type. Early detection and accurate diagnosis significantly improve survival rates. However, traditional diagnostic approaches, including dermoscopic examination, are highly dependent on clinician expertise and often yield moderate accuracy [ 2 ]. The rapid progress of deep learning, particularly convolutional neural networks (CNNs) and hybrid architectures, has revolutionized medical image analysis by enabling automated feature extraction and classification with high performance [ 2 ], [ 3 ]. Despite these advances, individual models often face challenges such as overfitting, limited generalization, and reduced robustness when applied to real-world clinical data [ 5 ]. To address these issues, ensemble and hybrid deep learning strategies have been introduced. For example, recent works have integrated CNNs, recurrent networks, and machine learning classifiers, achieving significant performance gains on public skin cancer datasets such as HAM10000 and ISIC [ 3 ], [ 4 ], [ 5 ]. These ensemble methods leverage the complementary strengths of diverse architectures, improving classification accuracy and reliability in computer-aided dermatology. In this study, we propose a hybrid deep learning ensemble model trained and evaluated on the publicly available Kaggle dataset containing 10,000 dermoscopic images of benign and malignant lesions [ 1 ]. By integrating multiple base learners with a meta-learning approach, our method achieves robust classification performance, reaching 91.7% accuracy and a ROC-AUC of 0.974 , outperforming individual models. The results highlight the effectiveness of hybrid ensemble frameworks as reliable tools for early and accurate skin cancer diagnosis. Related Work Several studies have explored the application of deep learning in skin cancer diagnosis. Hosny et al. [ 2 ] provided a comprehensive review of deep learning methods for melanoma detection, highlighting their advantages over traditional approaches while also noting limitations such as overfitting and dataset imbalance. Mahbod et al. [ 3 ] introduced a hybrid deep learning framework that combined segmentation and classification models to improve melanoma diagnosis. Their method demonstrated the potential of integrating multiple neural architectures for enhanced diagnostic accuracy. Similarly, Abbas et al. [ 4 ] employed a hybrid feature extraction approach, using DenseNet201, Xception, and MobileNet to generate deep representations, which were then classified with machine learning ensembles. This approach achieved promising results on benchmark datasets, underscoring the effectiveness of hybrid pipelines. In addition, Ramesh et al. [ 5 ] proposed an ensemble learning strategy that integrated convolutional neural networks (CNNs) and vision transformer models for multiclass skin lesion classification. Their ensemble framework improved generalization and robustness, addressing challenges inherent in medical image analysis. While these methods achieved impressive results on public datasets such as HAM10000 and ISIC, there remains a need for models that balance accuracy, robustness, and computational efficiency. In this context, the present study builds upon prior work by introducing a hybrid ensemble model trained on a large-scale Kaggle dataset [ 1 ], aiming to improve classification performance in distinguishing between benign and malignant skin lesions. Methodology The proposed framework employs a hybrid deep learning ensemble for binary classification of skin lesions into benign and malignant categories. The methodology consists of the following main components: 1. Dataset and Preprocessing The experiments were conducted using the publicly available Kaggle dataset containing 10,000 dermoscopic images of skin lesions [ 1 ]. Images were resized to 224×224224 \times 224224×224 pixels and normalized to the [0,1][0,1][0,1] range. The dataset was divided into training, validation, and test sets with stratified sampling to preserve class balance. Data augmentation, including random flipping, brightness adjustment, and contrast variation, was applied to enhance generalization and mitigate overfitting. 2. CNN+LSTM Model A custom CNN-LSTM architecture was designed to capture both spatial and sequential dependencies in dermoscopic images. Convolutional layers extracted local features, followed by reshaping into sequences processed by an LSTM unit. The final dense layer with a softmax activation performed classification into benign and malignant classes. 3. DenseNet-like Model Inspired by DenseNet, a second convolutional model was developed with residual connections and dense feature propagation. This architecture improved feature reuse, mitigated vanishing gradient issues, and produced robust high-level representations. Global average pooling and a dense layer with softmax activation completed the classification pipeline. 4. Meta-Learner (Ensemble Strategy) To exploit the complementary strengths of the CNN+LSTM and DenseNet models, their softmax prediction vectors were concatenated and used as input to a Gradient Boosting Classifier serving as the meta-learner. This ensemble approach aimed to improve overall predictive accuracy and robustness by combining heterogeneous feature representations. 5. Training Strategy Both CNN+LSTM and DenseNet models were trained with Adam optimizer (learning rate = 1e-4) and categorical cross-entropy loss. Early stopping and learning rate scheduling were employed to prevent overfitting and accelerate convergence. The meta-learner was trained on the concatenated prediction features extracted from the training set. 6. Evaluation Metrics Performance was assessed using accuracy, ROC-AUC, precision, recall, and F1-score. Confusion matrices and ROC curves were plotted to visualize classification performance across classes. The proposed hybrid ensemble achieved superior performance compared to individual models, with an overall accuracy of 91.7% and a ROC-AUC score of 0.974 , demonstrating the effectiveness of combining multiple architectures for automated skin cancer detection. Results and Discussion The experimental results demonstrate the effectiveness of the proposed hybrid ensemble approach for skin lesion classification. Table 1 summarizes the performance of the individual models and the meta-learner in terms of Accuracy and ROC-AUC. View this table: View inline View popup Download powerpoint Table 1. Comparison of models in terms of classification accuracy and ROC-AUC From Table 1 , it can be observed that the meta-learner outperforms the individual CNN+LSTM and DenseNet models, achieving the highest overall accuracy ( 91.7% ) and a ROC-AUC score of 0.974 . These results highlight the advantages of integrating heterogeneous deep learning architectures through ensemble strategies. Figure 1 presents the confusion matrix for the best-performing model (Meta-Learner), showing that both benign and malignant classes were classified with high reliability. Although some misclassifications occurred, the balance between precision and recall across both classes demonstrates the robustness of the model. Download figure Open in new tab Figure 1. Figure 2 illustrates the ROC curves of the evaluated models. All models achieved strong separability between benign and malignant lesions, with the ensemble method achieving the highest area under the curve. Download figure Open in new tab Figure 2. Download figure Open in new tab Figure 3. Download figure Open in new tab Figure 4. Download figure Open in new tab Figure 5. Download figure Open in new tab Figure 6. Download figure Open in new tab Figure 7. Download figure Open in new tab Figure 8. Overall, the proposed approach demonstrates competitive performance compared to state-of-the-art methods [ 2 – 5 ], confirming the effectiveness of hybrid ensemble learning for computer-aided skin cancer detection. Future work may focus on expanding the dataset, applying advanced augmentation strategies, and exploring transformer-based models to further improve generalization. Detailed Model Analysis via Confusion Matrices CNN-LSTM Model The confusion matrix for CNN-LSTM shows: True Benign (500): 467 correctly classified (True Negatives), 33 misclassified as malignant (False Positives). True Malignant (500): 444 correctly classified (True Positives), 56 misclassified as benign (False Negatives). This model exhibits a balanced error rate but a slightly higher tendency to misclassify malignant lesions as benign (FN = 56) compared to the reverse error (FP = 33). This is reflected in its recall for the malignant class (0.89), which is lower than it DenseNet Model The confusion matrix for DenseNet shows: True Benign (500): 473 correctly classified (TN), 27 misclassified as malignant (FP). True Malignant (500): 442 correctly classified (TP), 58 misclassified as benign (FN). DenseNet demonstrates an excellent ability to identify benign lesions (high specificity, as shown by low FP). However, it has the highest number of false negatives (58) among the models, indicating a cautious approach where it is more likely to miss a true malignant case. This aligns with its recall for malignant being the lowest (0.88). Meta-Learner Model (Normalized Confusion Matrix) The normalized matrix provides a clear view of class-wise performance: Benign Class: A recall of 0.94 (94% of actual benign lesions were correctly identified). The false positive rate is 0.06. Malignant Class: A recall of 0.89 (89% of actual malignant lesions were correctly identified). The false negative rate is 0.11. The Meta-Learner strikes an effective balance. It maintains the high true positive rate for benign lesions seen in DenseNet while slightly reducing the false negative rate for malignant lesions compared to the other models, leading to its superior overall accuracy. Clinical Implications The false negative (FN) rate is a critical metric in medical diagnostics. A model missing a malignant cancer (FN) has more severe consequences than falsely flagging a benign one (FP). The Meta-Learner’s configuration, with an 11% FN rate (normalized), represents the best balance among the three models, though further tuning to reduce this value should be a priority. Meta-Learner: Benign Class (True Label): Correctly identified 94% of benign cases (TNR: 0.94), with a 6% false positive rate (FPR: 0.06). Malignant Class (True Label): Correctly identified 89% of malignant cases (TPR: 0.89), with an 11% false negative rate (FNR: 0.11). Interpretation: The Meta- Learner shows a better balance between sensitivity and specificity compared to DenseNet. It has a slightly higher sensitivity (89% vs. 88%), meaning it is better at correctly identifying malignant cases, which is a crucial improvement. It also maintains a very low false positive rate. CNN-LSTM Benign Class (True Label): Correctly identified 89% of benign cases (True Negative Rate: 0.89). It misclassified 11% of benign lesions as malignant (False Positive Rate: 0.11). Malignant Class (True Label): Correctly identified 93% of malignant cases (True Positive Rate/Sensitivity: 0.93). It misclassified only 7% of malignant lesions as benign (False Negative Rate: 0.07). Interpretation: The CNN- LSTM model demonstrates the highest sensitivity (93%) means this model is the best at correctly identifying actual malignant cases, thereby minimizing the risk of missing a cancer (False Negatives). Its primary of all three models. This is a paramount strength in a medical context, as it weakness is a higher False Positive Rate, which could lead to unnecessary follow- up procedures for benign cases. The ROC curve of the CNN+LSTM model demonstrates a high discriminative power with an AUC of 0.969 for both benign and malignant cases. While the model achieves excellent sensitivity and specificity, its performance is slightly lower than DenseNet and the Meta-Learner, suggesting some limitations in capturing complex spatial patterns compared to deeper architectures. The ROC curve of the Meta-Learner model shows an AUC of 0.974, which is very close to the DenseNet’s performance. This demonstrates the advantage of ensemble learning, where combining multiple models leads to robust and stable classification results, reducing the weaknesses of individual models. The ROC curve of DenseNet indicates the strongest performance among the evaluated models, achieving an AUC of 0.975 for both classes. This highlights the effectiveness of its deep architecture in capturing fine- grained image features, resulting in a balanced trade-off between sensitivity and specificity. Limitations and Future Work Despite the promising results achieved by the proposed hybrid ensemble model, several limitations should be considered. First, although the Kaggle dataset [ 1 ] provides a large number of images (10,000), the diversity of skin types, lesion locations, and imaging conditions is limited, which may affect the generalization of the model to broader clinical settings [ 2 ], [ 5 ]. Second, the current approach focuses on binary classification (benign vs. malignant) and does not address multiclass lesion categorization, such as differentiating between melanoma, nevus, and basal cell carcinoma. Expanding the framework to handle multiple classes could enhance its clinical utility [ 4 ]. Conclusion In this study, we proposed a hybrid deep learning ensemble model for the classification of skin lesions into benign and malignant categories using a large-scale Kaggle dataset [ 1 ]. The framework combined the strengths of a custom CNN+LSTM model and a DenseNet-like architecture through a meta-learning ensemble approach, achieving superior performance compared to individual models. Third, while data augmentation partially mitigates overfitting, additional strategies such as generative adversarial networks (GANs) or domain adaptation could further improve model robustness and performance on unseen data [ 3 ]. Future work may also explore transformer- based architectures or attention mechanisms to capture long-range dependencies and improve feature representation. Finally, integrating patient metadata (age, gender, lesion location) alongside image features may further boost predictive accuracy and assist in personalized diagnostic recommendations [ 2 ], [ 5 ]. Experimental results demonstrated that the proposed method achieved an overall accuracy of 91.7% and a ROC-AUC of 0.974 , highlighting the effectiveness of ensemble learning in automated skin cancer detection. The hybrid framework not only improved classification performance but also enhanced the robustness and generalization ability across different dermoscopic images [ 2 ]–[ 5 ]. Despite certain limitations related to dataset diversity and binary classification, the proposed approach provides a solid foundation for future developments in computer-aided dermatology. Future research can focus on extending the framework to multiclass classification, integrating patient metadata, and exploring transformer-based or attention-driven architectures to further improve diagnostic accuracy and clinical applicability [ 2 ]–[ 5 ]. Overall, this work demonstrates the potential of hybrid ensemble deep learning models as reliable tools for early and accurate skin cancer detection, which may contribute to improved patient outcomes and support clinical decision-making. Data Availability All data used in this study are publicly available online at Kaggle, Melanoma Skin Cancer Dataset of 10,000 images http://kaggle.com/datasets/hasnainjaved/melanoma-skin-cancer-dataset-of-10000-images http://kaggle.com/datasets/hasnainjaved/melanoma-skin-cancer-dataset-of-10000-images Footnotes Alireza.rahi{at}outlook.com References 1. ↵ Kaggle Dataset : Melanoma Skin Cancer Dataset of 10000 Images, Hasnain Javed , 2023 . Available: http://kaggle.com/datasets/hasnainjaved/melanoma-skin-cancer-datasetof-10000-images 2. ↵ Hosny , K.M. , Kassem , M.A. , Foaud , M.M. “ Skin Cancer Detection Using Deep Learning: A Review ”, Diagnostics , vol. 13 , no. 11 , p. 1911 , 2023 . OpenUrl PubMed 3. ↵ Mahbod , A. , et al. “ Hybrid deep learning framework for melanoma diagnosis ”, Frontiers in Oncology , vol. 14 , 2024 . 4. ↵ Abbas , Q. , et al. “ Hybrid Deep Feature Extraction and Machine Learning for Skin Cancer Classification ”, Cancers , vol. 16 , no. 3 , 2024 . 5. ↵ Ramesh , A. , et al. “ Deep Ensemble Learning for Multiclass Skin Lesion Classification ”, Bioengineering , vol. 12 , no. 9 , 2024 . View the discussion thread. Back to top Previous Next Posted September 09, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following A Hybrid Deep Learning Ensemble for Accurate Skin Cancer Classification Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share A Hybrid Deep Learning Ensemble for Accurate Skin Cancer Classification Alireza Rahi medRxiv 2025.09.03.25335044; doi: https://doi.org/10.1101/2025.09.03.25335044 Share This Article: Copy Citation Tools A Hybrid Deep Learning Ensemble for Accurate Skin Cancer Classification Alireza Rahi medRxiv 2025.09.03.25335044; doi: https://doi.org/10.1101/2025.09.03.25335044 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Radiology and Imaging Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (297) Cardiovascular Medicine (4417) Dentistry and Oral Medicine (443) Dermatology (381) Emergency Medicine (606) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1506) Epidemiology (15209) Forensic Medicine (30) Gastroenterology (1120) Genetic and Genomic Medicine (6577) Geriatric Medicine (666) Health Economics (995) Health Informatics (4515) Health Policy (1366) Health Systems and Quality Improvement (1610) Hematology (537) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15906) Intensive Care and Critical Care Medicine (1103) Medical Education (620) Medical Ethics (144) Nephrology (667) Neurology (6576) Nursing (345) Nutrition (998) Obstetrics and Gynecology (1139) Occupational and Environmental Health (955) Oncology (3321) Ophthalmology (970) Orthopedics (369) Otolaryngology (420) Pain Medicine (435) Palliative Medicine (129) Pathology (663) Pediatrics (1689) Pharmacology and Therapeutics (691) Primary Care Research (710) Psychiatry and Clinical Psychology (5425) Public and Global Health (9210) Radiology and Imaging (2192) Rehabilitation Medicine and Physical Therapy (1368) Respiratory Medicine (1194) Rheumatology (593) Sexual and Reproductive Health (709) Sports Medicine (529) Surgery (709) Toxicology (99) Transplantation (288) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9fee640f6ec94807',t:'MTc3OTMxMjA4NA=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();