Self-Supervised Representation Learning for Robust Fine-Grained Human Hand Action Recognition in Industrial Assembly Lines

Self-Supervised Representation Learning for Robust Fine-Grained Human Hand Action Recognition in Industrial Assembly Lines | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Self-Supervised Representation Learning for Robust Fine-Grained Human Hand Action Recognition in Industrial Assembly Lines Fabian Sturm This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-4347681/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 13 Dec, 2024 Read the published version in Machine Vision and Applications → Version 1 posted 9 You are reading this latest preprint version Abstract Humans are still indispensable on industrial assembly lines, but in the event of an error, they need support from intelligent systems. In addition to the objects to be observed, it is equally important to understand the fine-grained hand movements of a human to be able to track the entire process. However, these deep-learning-based hand action recognition methods are very label intensive, which cannot be offered by all industrial companies due to the associated costs. This work therefore presents a self-supervised learning approach for industrial assembly processes that allows a spatio-temporal transformer architecture to be pre-trained on a variety of information from real-world video footage of daily life. Subsequently, this deep learning model is adapted to the industrial assembly task at hand using only a few labels. Well-known real-world datasets best suited for representation learning of such hand actions in a regression tasks are outlined and to what extent they optimize the subsequent supervised trained classification task. This subsequent fine-tuning is supplemented by concept drift detection, which makes the resulting productively employed models more robust against concept drift and future changing assembly movements. Self-Supervised Learning Human Action Recognition Industrial Vision Concept Drift Detection Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Published Journal Publication published 13 Dec, 2024 Read the published version in Machine Vision and Applications → Version 1 posted Editorial decision: Revision requested 11 Jul, 2024 Reviews received at journal 25 Jun, 2024 Reviews received at journal 21 Jun, 2024 Reviewers agreed at journal 21 May, 2024 Reviewers agreed at journal 20 May, 2024 Reviewers invited by journal 18 May, 2024 Editor assigned by journal 02 May, 2024 Submission checks completed at journal 02 May, 2024 First submitted to journal 30 Apr, 2024 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-4347681","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":299497940,"identity":"4b73a2c5-4f7c-4bd8-9bbc-f694ca01f5e1","order_by":0,"name":"Fabian Sturm","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABJElEQVRIie3PsUrDQBjA8a8cJMsF11skr3AipBai8VGuBOKSWkofwICQLnGv+BIGl4wXDnSJe0QHQcxUpaVLHEQv0TpdoW4i918uB/fLdweg0/3BzAgD/9nNV9+dqF2oimD+TXBzcMo2JLAiCG9EzNucv2XgeWTwfLVfP0CX+NfLUeYO7UOUPqoIPmb5WQH9hBw59yGroDcN/IvzIuil3BirxngQUm7FwDAJDEnEyWUZ7iIrFnQnwg5RTdma0fw9Bq8le0wALYdLST4a0q1VhIRUyCmdpCHQkhBJwqkN2FE+n7wysR2TflJUxl0SSFJU8mKFTykyxuqLDcTiJXY9cxIYZe1KcuM/ISs7oPbkNJ2rxnyl+hsV68+vyY5+TXQ6ne5/9gkgxWMeEseS0QAAAABJRU5ErkJggg==","orcid":"","institution":"Georg Simon Ohm University of Applied Sciences Nuremberg","correspondingAuthor":true,"prefix":"","firstName":"Fabian","middleName":"","lastName":"Sturm","suffix":""}],"badges":[],"createdAt":"2024-04-30 08:59:07","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-4347681/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-4347681/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1007/s00138-024-01638-9","type":"published","date":"2024-12-13T15:57:29+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":71552426,"identity":"73ee922d-3188-4f3c-baae-3641f721dc96","added_by":"auto","created_at":"2024-12-16 16:05:59","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":883673,"visible":true,"origin":"","legend":"","description":"","filename":"ExtendendVersionJournalISVC23SSLRepresentationLearningforRobustFineGrainedHumanHandActionRecognitioninIndustrialAssemblyLinesSubmission.pdf","url":"https://assets-eu.researchsquare.com/files/rs-4347681/v1_covered_c27cbe59-409f-4a05-8b5b-58e5d8a7709d.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Self-Supervised Representation Learning for Robust Fine-Grained Human Hand Action Recognition in Industrial Assembly Lines","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"machine-vision-and-applications","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"mvap","sideBox":"Learn more about [Machine Vision and Applications](https://www.springer.com/journal/138)","snPcode":"138","submissionUrl":"https://submission.springernature.com/new-submission/138/3","title":"Machine Vision and Applications","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Self-Supervised Learning, Human Action Recognition, Industrial Vision, Concept Drift Detection","lastPublishedDoi":"10.21203/rs.3.rs-4347681/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-4347681/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Humans are still indispensable on industrial assembly lines, but in the event of an error, they need support from intelligent systems. In addition to the objects to be observed, it is equally important to understand the fine-grained hand movements of a human to be able to track the entire process. However, these deep-learning-based hand action recognition methods are very label intensive, which cannot be offered by all industrial companies due to the associated costs. This work therefore presents a self-supervised learning approach for industrial assembly processes that allows a spatio-temporal transformer architecture to be pre-trained on a variety of information from real-world video footage of daily life. Subsequently, this deep learning model is adapted to the industrial assembly task at hand using only a few labels. Well-known real-world datasets best suited for representation learning of such hand actions in a regression tasks are outlined and to what extent they optimize the subsequent supervised trained classification task. This subsequent fine-tuning is supplemented by concept drift detection, which makes the resulting productively employed models more robust against concept drift and future changing assembly movements.","manuscriptTitle":"Self-Supervised Representation Learning for Robust Fine-Grained Human Hand Action Recognition in Industrial Assembly Lines","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-05-08 19:43:53","doi":"10.21203/rs.3.rs-4347681/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2024-07-11T15:42:36+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2024-06-26T00:17:24+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2024-06-21T12:22:03+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"151125597802490643008730532732326494079","date":"2024-05-21T21:34:08+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"316161433830334271084540408147026540176","date":"2024-05-20T13:46:56+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2024-05-18T23:06:19+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2024-05-02T04:07:22+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2024-05-02T04:07:21+00:00","index":"","fulltext":""},{"type":"submitted","content":"Machine Vision and Applications","date":"2024-04-30T08:57:02+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"machine-vision-and-applications","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"mvap","sideBox":"Learn more about [Machine Vision and Applications](https://www.springer.com/journal/138)","snPcode":"138","submissionUrl":"https://submission.springernature.com/new-submission/138/3","title":"Machine Vision and Applications","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"ced15c65-43c5-4e94-98bf-0e42a6627d10","owner":[],"postedDate":"May 8th, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[],"tags":[],"updatedAt":"2024-12-16T16:00:43+00:00","versionOfRecord":{"articleIdentity":"rs-4347681","link":"https://doi.org/10.1007/s00138-024-01638-9","journal":{"identity":"machine-vision-and-applications","isVorOnly":false,"title":"Machine Vision and Applications"},"publishedOn":"2024-12-13 15:57:29","publishedOnDateReadable":"December 13th, 2024"},"versionCreatedAt":"2024-05-08 19:43:53","video":"","vorDoi":"10.1007/s00138-024-01638-9","vorDoiUrl":"https://doi.org/10.1007/s00138-024-01638-9","workflowStages":[]},"version":"v1","identity":"rs-4347681","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-4347681","identity":"rs-4347681","version":["v1"]},"buildId":"qtupq5eGEP_6zYnWcrvyt","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}