Advancing Sign Language Interpretation with Transfer Learning and Multimodal Features

Advancing Sign Language Interpretation with Transfer Learning and Multimodal Features | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Advancing Sign Language Interpretation with Transfer Learning and Multimodal Features Manish Shukla, Harsh Gupta This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-7586144/v1 This work is licensed under a CC BY 4.0 License Status: Posted Version 1 posted You are reading this latest preprint version Abstract Sign languages are rich visual languages used by deaf and hard-of-hearing communities around the world. An acute shortage of trained human interpreters motivates the development of automatic sign language recognition systems. Building on the Sign Language Interpreter using Deep Learning project, this paper introduces an enhanced evaluation framework for small-vocabulary interpreters and reports new experiments that go beyond the baseline convolutional neural network (CNN). We fine-tune state-of-the-art architectures such as ResNet-50 and EfficientNet on the American Sign Language (ASL) alphabet dataset, integrate hand-landmark features extracted by MediaPipe into a recurrent backbone, and perform cross-dataset evaluation on a large public dataset. Additional metrics—including macro/micro F1, Cohen’s kappa and per-class recall—provide a more nuanced assessment than overall accuracy. Robustness tests examine lighting, background clutter, signer diversity and adversarial perturbations. We also discuss ethical and accessibility considerations and reflect on the practical impact of hackathon-style prototypes. The results demonstrate that lightweight models can be rapidly improved via transfer learning and multimodal fusion while retaining usability on commodity hardware. Our work offers a blueprint for researchers and practitioners seeking to translate small-scale prototypes into equitable, scalable accessibility solutions. The code supporting this work is available at https://github.com/Manishms18/Sign-Language-Advance . Sign Language Recognition American Sign Language Deep Learning Assistive Technology Transfer-Learning Media Pipe AI For Accessibility Ethical AI Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Posted Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-7586144","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":513854679,"identity":"cafa8da9-e59a-4281-961f-e36be50b9106","order_by":0,"name":"Manish Shukla","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABIElEQVRIie2PMUvDQBTHcwTMciHrBYvfQHggSIeCXyVB6HQWQXAMCYXLYHQvfolMwfHCgS4XMgkXujjpksFuLXRokkWFNOomcr/hPR68H//3DEOj+aNwFBqegc2wHbBjtR3GP1FQp4zchLcKGY75rExAeW3frxzHRZ5vHoIZyDx6RzTA7uLtOV1fEsOJb7w+5VTOPGFLcQVFNCcoE9g5pNdV0hxGZJH2KpyCQIz7aYnCRuHYvadThRsFyEW/UtaQb1jQKvM1ygIMlZxW2yFFUeA2M/20iFiTYmJQ1uNyMEXVIGwm/IXM2dhvfnETerAcAcF7fynpyao97E6eC7XKgjPHenqt6u3kyIlve5WveF3F0NXv1z+wXn6zrdFoNP+fHWEKbs9YYoqAAAAAAElFTkSuQmCC","orcid":"","institution":"Independent Researcher","correspondingAuthor":true,"prefix":"","firstName":"Manish","middleName":"","lastName":"Shukla","suffix":""},{"id":513854680,"identity":"fb12dc8f-183f-46cd-9013-9e81c7416233","order_by":1,"name":"Harsh Gupta","email":"","orcid":"","institution":"Independent Researcher","correspondingAuthor":false,"prefix":"","firstName":"Harsh","middleName":"","lastName":"Gupta","suffix":""}],"badges":[],"createdAt":"2025-09-10 21:38:14","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-7586144/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-7586144/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":92912337,"identity":"2eb73578-18b0-453b-8ea8-d633f4004865","added_by":"auto","created_at":"2025-10-07 04:17:02","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":307795,"visible":true,"origin":"","legend":"","description":"","filename":"SignLanguageAdvance.pdf","url":"https://assets-eu.researchsquare.com/files/rs-7586144/v1_covered_2283a3e4-1fc0-423e-a662-0221c000a7f0.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Advancing Sign Language Interpretation with Transfer Learning and Multimodal Features","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":true,"hideJournal":true,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"Sign Language Recognition, American Sign Language, Deep Learning, Assistive Technology, Transfer-Learning, Media Pipe, AI For Accessibility, Ethical AI","lastPublishedDoi":"10.21203/rs.3.rs-7586144/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-7586144/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Sign languages are rich visual languages used by deaf and hard-of-hearing communities around the world. An acute shortage of trained human interpreters motivates the development of automatic sign language recognition systems. Building on the Sign Language Interpreter using Deep Learning project, this paper introduces an enhanced evaluation framework for small-vocabulary interpreters and reports new experiments that go beyond the baseline convolutional neural network (CNN). We fine-tune state-of-the-art architectures such as ResNet-50 and EfficientNet on the American Sign Language (ASL) alphabet dataset, integrate hand-landmark features extracted by MediaPipe into a recurrent backbone, and perform cross-dataset evaluation on a large public dataset. Additional metrics—including macro/micro F1, Cohen’s kappa and per-class recall—provide a more nuanced assessment than overall accuracy. Robustness tests examine lighting, background clutter, signer diversity and adversarial perturbations. We also discuss ethical and accessibility considerations and reflect on the practical impact of hackathon-style prototypes. The results demonstrate that lightweight models can be rapidly improved via transfer learning and multimodal fusion while retaining usability on commodity hardware. Our work offers a blueprint for researchers and practitioners seeking to translate small-scale prototypes into equitable, scalable accessibility solutions. The code supporting this work is available at https://github.com/Manishms18/Sign-Language-Advance.","manuscriptTitle":"Advancing Sign Language Interpretation with Transfer Learning and Multimodal Features","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-09-12 02:52:08","doi":"10.21203/rs.3.rs-7586144/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"c92a1b4f-b271-46ce-bad1-0da7e17b67f2","owner":[],"postedDate":"September 12th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"posted","subjectAreas":[],"tags":[],"updatedAt":"2025-10-07T04:08:56+00:00","versionOfRecord":[],"versionCreatedAt":"2025-09-12 02:52:08","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-7586144","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-7586144","identity":"rs-7586144","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}