From Random Steps to Intelligent Learning: The Evolutionary Journey and Modern Renaissance of Stochastic Gradient Descent in the Age of Deep Learning

From Random Steps to Intelligent Learning: The Evolutionary Journey and Modern Renaissance of Stochastic Gradient Descent in the Age of Deep Learning | Authorea try { document.documentElement.classList.add('js'); } catch (e) { } var _gaq = _gaq || []; _gaq.push(['_setAccount', 'G-8VDV14Y67G']); _gaq.push(['_trackPageview']); (function() { var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); })(); Skip to main content Preprints Collections Wiley Open Research IET Open Research Ecological Society of Japan All Collections About About Authorea FAQs Contact Us Quick Search anywhere Search for preprint articles, keywords, etc. Search Search ADVANCED SEARCH SCROLL This is a preprint and has not been peer reviewed. Data may be preliminary. 18 September 2025 V1 Latest version Share on From Random Steps to Intelligent Learning: The Evolutionary Journey and Modern Renaissance of Stochastic Gradient Descent in the Age of Deep Learning Authors : Surya Rao Rayarao 0009-0001-8467-7865 [email protected] and Naga Donikena Authors Info & Affiliations https://doi.org/10.22541/au.175822528.85221911/v1 224 views 151 downloads Contents Abstract Supplementary Material Information & Authors Metrics & Citations View Options References Figures Tables Media Share Abstract Stochastic Gradient Descent (SGD) has emerged as the cornerstone optimization algorithm in modern machine learning and deep learning systems. This comprehensive survey examines the theoretical foundations, algorithmic variants, and practical applications of SGD across diverse domains. We explore the evolution from classical gradient descent to stochastic variants, analyzing convergence properties, computational efficiency, and robustness characteristics. The paper provides an extensive review of SGD modifications including momentumbased methods, adaptive learning rate techniques, and variance reduction approaches. Through detailed analysis of applications in neural networks, convex optimization, and large-scale machine learning, we demonstrate SGD's fundamental role in enabling the success of contemporary AI systems. This survey serves as a comprehensive resource for researchers and practitioners seeking to understand the theoretical underpinnings and practical considerations of stochastic gradient descent optimization. Supplementary Material File (sgd.pdf) Download 190.83 KB Information & Authors Information Version history V1 Version 1 18 September 2025 Copyright This work is licensed under a Non Exclusive No Reuse License. Keywords adaptive algorithms convergence analysis deep learning machine learning optimization stochastic gradient descent Authors Affiliations Surya Rao Rayarao 0009-0001-8467-7865 [email protected] Department of Statistics and Data Sciences Department of Computer Science, The University of Texas at Austin Austin View all articles by this author Naga Donikena Department of Statistics and Data Sciences Department of Computer Science, The University of Texas at Austin Austin View all articles by this author Metrics & Citations Metrics Article Usage 224 views 151 downloads .FvxKWukQNSOunydq8rnd { width: 100px; } Citations Download citation Surya Rao Rayarao, Naga Donikena. From Random Steps to Intelligent Learning: The Evolutionary Journey and Modern Renaissance of Stochastic Gradient Descent in the Age of Deep Learning. Authorea . 18 September 2025. DOI: https://doi.org/10.22541/au.175822528.85221911/v1 If you have the appropriate software installed, you can download article citation data to the citation manager of your choice. Simply select your manager software from the list below and click Download. For more information or tips please see 'Downloading to a citation manager' in the Help menu . Format Please select one from the list RIS (ProCite, Reference Manager) EndNote BibTex Medlars RefWorks Direct import Tips for downloading citations document.getElementById('citMgrHelpLink').addEventListener('click', function() { popupHelp(this.href); return false; }); $(".js__slcInclude").on("change", function(e){ if ($(this).val() == 'refworks') $('#direct').prop("checked", false); $('#direct').prop("disabled", ($(this).val() == 'refworks')); }); View Options View options PDF View PDF Figures Tables Media Share Share Share article link Copy Link Copied! Copying failed. Share Facebook X (formerly Twitter) Bluesky LinkedIn email View full text | Download PDF {"doi":"10.22541/au.175822528.85221911/v1","type":"Article"} Now Reading: Share Figures Tables Close figure viewer Back to article Figure title goes here Change zoom level Go to figure location within the article Download figure Toggle share panel Toggle share panel Share Toggle information panel Toggle information panel Go to previous graphic Go to next graphic Go to previous table Go to next table All figures All tables View all material View all material xrefBack.goTo xrefBack.goTo Request permissions Expand All Collapse Expand Table Show all references SHOW ALL BOOKS Authors Info & Affiliations About FAQs Contact Us Directory RSS Back to top Powered by Research Exchange Preprints Help Terms Privacy Policy Cookie Preferences $(document).ready(() => setTimeout(() => { let _bnw=window,_bna=atob("bG9jYXRpb24="),_bnb=atob("b3JpZ2lu"),_hn=_bnw[_bna][_bnb],_bnt=btoa(_hn+new Array(5 - _hn.length % 4).join(" ")); $.get("/resource/lodash?t="+_bnt); },4000)); (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a00aa0ac6d9906d3',t:'MTc3OTYwODE2OA=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();