Somatosensory-Driven Perception in Embodied Systems for Hand-Object Interaction

Somatosensory-Driven Perception in Embodied Systems for Hand-Object Interaction | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Article Somatosensory-Driven Perception in Embodied Systems for Hand-Object Interaction Peiqi Kang, Shuo Jiang, Eni Halilaj, Xinge Yu, Bin He, Peter B. Shull This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-8157566/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 9 You are reading this latest preprint version Abstract Somatosensation is a powerful perceptual modality that enables accurate and robust sensing in challenging scenarios. It allows blind individuals to explore their surroundings with a white cane in darkness and assists surgeons who operate with a scalpel under occluded vision. However, robots lack somatosensory capabilities comparable to those of humans. To address this limitation, we introduce a perception framework that treats touch and proprioception as primary signals. Current neuroscience provides sufficient insight to define an analogous four stage processing pipeline that includes afferent integration, perceptual inference, error compensation, and gated convergence. Building on these principles, our artificial framework mirrors key elements of the cortical sensorimotor cascade. Experiments across wearable systems and dexterous robotic platforms equipped with tactile hands show that the framework overcomes previously unsolved challenges in estimating object orientation, relative position, and contact points under real-world non-convexities with wearable sensors, and enables tasks infeasible for vision-based perception, including estimating contact force, tip torque, and object mass, with accuracy surpassing human and state-of-the-art baselines. This framework provides a robust pathway toward human-like capabilities and a precise foundation for next-generation human–machine interaction, dexterous manipulation, and embodied intelligence applications. Physical sciences/Mathematics and computing/Computational science Physical sciences/Engineering/Mechanical engineering Physical sciences/Engineering/Biomedical engineering Full Text Additional Declarations There is NO Competing Interest. Supplementary Files poseestimation.mp4 SUPPLEMENTARY VIDEO: Demo 1 Pose Estimation forceestimation.mp4 SUPPLEMENTARY VIDEO: Demo 2 Force Estimation objectrecognition.mp4 SUPPLEMENTARY VIDEO: Demo 3 Object Estimation Cite Share Download PDF Status: Under Review Version 1 posted Editorial decision: revise 13 Feb, 2026 Review # 2 received at journal 02 Feb, 2026 Review # 3 received at journal 19 Jan, 2026 Reviewer # 3 agreed at journal 08 Jan, 2026 Reviewer # 2 agreed at journal 08 Jan, 2026 Reviewer # 1 agreed at journal 12 Dec, 2025 Reviewers invited by journal 05 Dec, 2025 Editor assigned by journal 21 Nov, 2025 First submitted to journal 19 Nov, 2025 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-8157566","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Article","associatedPublications":[],"authors":[{"id":555813299,"identity":"bbb33067-e891-454d-a387-370685dcd10f","order_by":0,"name":"Peiqi Kang","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABLElEQVRIiWNgGAWjYBAC+wYInQikGR+AmRJI0owNmFoMDkCEk4GY2YAkLWlAzCZBnJbjZ8wffNzBkMM/u/1a1c09dgz8s3sMGH7UbGPgbz/AJjkDi196cgwbZ55hqJC4c6bsds6zZAYgw4Cx59htBokzCWySG7DYwpBj2MzbxlDBcCMn7XbOgQP1GyRyDJgZ2G4zMNxgYJN8gEUL/xuwlhx5oJZioBYGA7CWf7cZ5HFpkYDYkmZwI/0YM1wLY9ttBgOQFmwOk3hWOHNmm0Sy4Y0cZumcAyC/HCs42Nt3m8fwTGKzJRbvG/Anb/jwsc0mcd6N9Iefcw6AQqx544Mf327LyR0/fPBmD6YWBgYOUASC4oLHAC52AMTFHisgwP4AnTEKRsEoGAWjABUAAKCdbM9JoNr3AAAAAElFTkSuQmCC","orcid":"","institution":"University of Cambridge","correspondingAuthor":true,"prefix":"","firstName":"Peiqi","middleName":"","lastName":"Kang","suffix":""},{"id":555813300,"identity":"12d92c54-171e-4d27-946d-a4123a921f38","order_by":1,"name":"Shuo Jiang","email":"","orcid":"https://orcid.org/0000-0003-3645-6301","institution":"Tongji university","correspondingAuthor":false,"prefix":"","firstName":"Shuo","middleName":"","lastName":"Jiang","suffix":""},{"id":555813301,"identity":"2714ae25-57ee-4e64-ad04-2557e8fb88cd","order_by":2,"name":"Eni Halilaj","email":"","orcid":"","institution":"Carnegie Mellon University","correspondingAuthor":false,"prefix":"","firstName":"Eni","middleName":"","lastName":"Halilaj","suffix":""},{"id":555813302,"identity":"a300bbbe-fc43-4415-ac1d-77fdb95f1ca5","order_by":3,"name":"Xinge Yu","email":"","orcid":"https://orcid.org/0000-0003-0522-1171","institution":"City University of Hong Kong; Hong Kong","correspondingAuthor":false,"prefix":"","firstName":"Xinge","middleName":"","lastName":"Yu","suffix":""},{"id":555813303,"identity":"c0dda393-78ef-4785-a66a-0989dedc0b57","order_by":4,"name":"Bin He","email":"","orcid":"","institution":"Tongji University","correspondingAuthor":false,"prefix":"","firstName":"Bin","middleName":"","lastName":"He","suffix":""},{"id":555813304,"identity":"60427196-5b23-49f8-bc65-c84467c2e7c2","order_by":5,"name":"Peter B. Shull","email":"","orcid":"","institution":"Shanghai Jiao Tong University","correspondingAuthor":false,"prefix":"","firstName":"Peter","middleName":"B.","lastName":"Shull","suffix":""}],"badges":[],"createdAt":"2025-11-19 17:06:06","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-8157566/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-8157566/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":97894558,"identity":"20a209ca-2113-4e30-be58-346a39ca84c2","added_by":"auto","created_at":"2025-12-10 15:32:42","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":4012186,"visible":true,"origin":"","legend":"Article File","description":"","filename":"SomatosensoryDrivenPerceptioninEmbodiedSystemsforHandObjectInteraction.pdf","url":"https://assets-eu.researchsquare.com/files/rs-8157566/v1_covered_fcee462b-640f-410f-a3be-549cd5f726e6.pdf"},{"id":97741964,"identity":"a5335ec1-4071-452e-a5e6-b895fc02007f","added_by":"auto","created_at":"2025-12-08 22:33:46","extension":"mp4","order_by":1,"title":"","display":"","copyAsset":false,"role":"supplement","size":33735497,"visible":true,"origin":"","legend":"SUPPLEMENTARY VIDEO: Demo 1 Pose Estimation","description":"","filename":"poseestimation.mp4","url":"https://assets-eu.researchsquare.com/files/rs-8157566/v1/0871b4676803d18790d58983.mp4"},{"id":97741965,"identity":"18f8d672-9c3a-458a-b6a6-da7bfd3a1916","added_by":"auto","created_at":"2025-12-08 22:33:46","extension":"mp4","order_by":2,"title":"","display":"","copyAsset":false,"role":"supplement","size":35247551,"visible":true,"origin":"","legend":"SUPPLEMENTARY VIDEO: Demo 2 Force Estimation","description":"","filename":"forceestimation.mp4","url":"https://assets-eu.researchsquare.com/files/rs-8157566/v1/5ea3736abcee70d07d785370.mp4"},{"id":97741966,"identity":"ae329542-089a-4851-a18a-9d0ab2092a46","added_by":"auto","created_at":"2025-12-08 22:33:46","extension":"mp4","order_by":3,"title":"","display":"","copyAsset":false,"role":"supplement","size":45479786,"visible":true,"origin":"","legend":"SUPPLEMENTARY VIDEO: Demo 3 Object Estimation","description":"","filename":"objectrecognition.mp4","url":"https://assets-eu.researchsquare.com/files/rs-8157566/v1/a1dbd99cf1dff08aa1ff46a7.mp4"}],"financialInterests":"There is \u003cb\u003eNO\u003c/b\u003e Competing Interest.","formattedTitle":"Somatosensory-Driven Perception in Embodied Systems for Hand-Object Interaction","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"natsensors","isNatureJournal":false,"hasQc":false,"allowDirectSubmit":false,"externalIdentity":"natsensors","sideBox":"Learn more about [Nature Sensors](https://www.nature.com/natsensors/)","snPcode":"44460","submissionUrl":"https://mts-natsensors.nature.com","title":"Nature Sensors","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"ejp","reportingPortfolio":"Nature","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"","lastPublishedDoi":"10.21203/rs.3.rs-8157566/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-8157566/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Somatosensation is a powerful perceptual modality that enables accurate and robust sensing in challenging scenarios. It allows blind individuals to explore their surroundings with a white cane in darkness and assists surgeons who operate with a scalpel under occluded vision. However, robots lack somatosensory capabilities comparable to those of humans. To address this limitation, we introduce a perception framework that treats touch and proprioception as primary signals. Current neuroscience provides sufficient insight to define an analogous four stage processing pipeline that includes afferent integration, perceptual inference, error compensation, and gated convergence. Building on these principles, our artificial framework mirrors key elements of the cortical sensorimotor cascade. Experiments across wearable systems and dexterous robotic platforms equipped with tactile hands show that the framework overcomes previously unsolved challenges in estimating object orientation, relative position, and contact points under real-world non-convexities with wearable sensors, and enables tasks infeasible for vision-based perception, including estimating contact force, tip torque, and object mass, with accuracy surpassing human and state-of-the-art baselines. This framework provides a robust pathway toward human-like capabilities and a precise foundation for next-generation human–machine interaction, dexterous manipulation, and embodied intelligence applications.","manuscriptTitle":"Somatosensory-Driven Perception in Embodied Systems for Hand-Object Interaction","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-12-08 22:33:41","doi":"10.21203/rs.3.rs-8157566/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"revise","date":"2026-02-13T08:45:06+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"This content is not available.","date":"2026-02-02T14:33:13+00:00","index":2,"fulltext":"This content is not available."},{"type":"editorInvitedReview","content":"This content is not available.","date":"2026-01-19T22:44:34+00:00","index":3,"fulltext":"This content is not available."},{"type":"reviewerAgreed","content":"This content is not available.","date":"2026-01-09T01:47:22+00:00","index":3,"fulltext":"This content is not available."},{"type":"reviewerAgreed","content":"This content is not available.","date":"2026-01-08T14:49:06+00:00","index":2,"fulltext":"This content is not available."},{"type":"reviewerAgreed","content":"This content is not available.","date":"2025-12-12T07:40:09+00:00","index":1,"fulltext":"This content is not available."},{"type":"reviewersInvited","content":"","date":"2025-12-05T12:37:28+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2025-11-21T15:22:38+00:00","index":"","fulltext":""},{"type":"submitted","content":"Nature Sensors","date":"2025-11-19T17:02:51+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"natsensors","isNatureJournal":false,"hasQc":false,"allowDirectSubmit":false,"externalIdentity":"natsensors","sideBox":"Learn more about [Nature Sensors](https://www.nature.com/natsensors/)","snPcode":"44460","submissionUrl":"https://mts-natsensors.nature.com","title":"Nature Sensors","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"ejp","reportingPortfolio":"Nature","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"d401061b-816e-4fea-bc05-4f2c3346e37c","owner":[],"postedDate":"December 8th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[{"id":59156299,"name":"Physical sciences/Mathematics and computing/Computational science"},{"id":59156300,"name":"Physical sciences/Engineering/Mechanical engineering"},{"id":59156301,"name":"Physical sciences/Engineering/Biomedical engineering"}],"tags":[],"updatedAt":"2026-04-21T03:50:44+00:00","versionOfRecord":[],"versionCreatedAt":"2025-12-08 22:33:41","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-8157566","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-8157566","identity":"rs-8157566","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}