In situ training of an in-sensor artificial neural network based on ferroelectric photosensors

In situ training of an in-sensor artificial neural network based on ferroelectric photosensors | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Article In situ training of an in-sensor artificial neural network based on ferroelectric photosensors Zhen Fan, Haipeng Lin, Ou Jiali, Xiaobing Yan, Wenjie Hu, Boyuan Cui, and 11 more This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-4791621/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 07 Jan, 2025 Read the published version in Nature Communications → Version 1 posted You are reading this latest preprint version Abstract In-sensor computing has emerged as an ultrafast and low-power technique for next-generation machine vision. However, in situ training of in-sensor computing systems remains challenging due to the demands for both high-performance devices and efficient programming schemes. Here, we experimentally demonstrate the in situ training of an in-sensor artificial neural network (ANN) based on ferroelectric photosensors (FE-PSs). Our FE-PS exhibits self-powered, fast (4 bits) photoresponses, as well as long retention (15 days), high endurance (10 9 ), high write speed (100 ns), and small cycle-to-cycle and device-to-device variations (~0.66% and ~2.72%, respectively), all of which are desirable for the in situ training. Additionally, a bi-directional closed-loop programming scheme is developed, achieving a precise and efficient weight update for the FE-PS. Using this programming scheme, an in-sensor ANN based on the FE-PSs is trained in situ to recognize traffic signs for commanding a prototype autonomous vehicle. Moreover, this in-sensor ANN operates 50 times faster than a von Neumann machine vision system. This study paves the way for the development of in-sensor computing systems with in situ training capability, which may find applications in new data-streaming machine vision tasks. Physical sciences/Materials science/Materials for devices/Information storage Physical sciences/Materials science/Condensed-matter physics/Ferroelectrics and multiferroics Full Text Additional Declarations There is NO Competing Interest. Supplementary Files SupplementaryInformation.pdf SupplementaryVideo1.mp4 Supplementary Video 1 Cite Share Download PDF Status: Published Journal Publication published 07 Jan, 2025 Read the published version in Nature Communications → Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-4791621","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Article","associatedPublications":[],"authors":[{"id":333150742,"identity":"140eca12-f7fc-424a-bf04-fff449e4e905","order_by":0,"name":"Zhen Fan","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA30lEQVRIiWNgGAWjYBAC9gYGBmYgLYcQOkBAC88BiBZj0rUkNhCvhf3s4c8FFXfS1zawP/xc2MYgx3cjgfFzAT4tPHkJxjPOPMvddoDHWHpmG4Ox5I0EZukZeLTYM+QYJPO2HQZpYWPmbWNI3HAjgY2ZB58t/G8MDvP+O5xudoD9GUhLPWEtEjmGzbwNhxPMDjCYgbQkGBDW8saYmefYYcNth4F+4TknYTjzzMNmafwOyzH+zFNzWN7sePvDzzxlNvJ8x5MPfsanBQGYwaQEEDM2EKVhFIyCUTAKRgFuAABRAkUipS8QWwAAAABJRU5ErkJggg==","orcid":"https://orcid.org/0000-0002-1756-641X","institution":"South China Normal University","correspondingAuthor":true,"prefix":"","firstName":"Zhen","middleName":"","lastName":"Fan","suffix":""},{"id":333150743,"identity":"7b28f4ad-49b9-4415-a874-a638bb8df2b4","order_by":1,"name":"Haipeng Lin","email":"","orcid":"","institution":"South China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Haipeng","middleName":"","lastName":"Lin","suffix":""},{"id":333150744,"identity":"d355c9c9-9ce8-4a7f-86e6-d48e20a6b54f","order_by":2,"name":"Ou Jiali","email":"","orcid":"","institution":"South China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Ou","middleName":"","lastName":"Jiali","suffix":""},{"id":333150745,"identity":"61f939e6-88eb-4ed5-a378-50f5fe889db7","order_by":3,"name":"Xiaobing Yan","email":"","orcid":"https://orcid.org/0000-0002-6335-336X","institution":"Hebei University","correspondingAuthor":false,"prefix":"","firstName":"Xiaobing","middleName":"","lastName":"Yan","suffix":""},{"id":333150746,"identity":"dd33db07-8a56-418c-9d90-2383e1498641","order_by":4,"name":"Wenjie Hu","email":"","orcid":"","institution":"South China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Wenjie","middleName":"","lastName":"Hu","suffix":""},{"id":333150747,"identity":"9c92f0bd-08db-446b-830a-0bb6960a6542","order_by":5,"name":"Boyuan Cui","email":"","orcid":"","institution":"South China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Boyuan","middleName":"","lastName":"Cui","suffix":""},{"id":333150748,"identity":"96b71e8c-f3d0-4a9b-8d63-2076c4e7896a","order_by":6,"name":"Jikang Xu","email":"","orcid":"https://orcid.org/0009-0000-8073-0799","institution":"Key Laboratory of Brain-Like Neuromorphic Devices and Systems of Hebei Province, College of Electron and Information Engineering, Hebei University","correspondingAuthor":false,"prefix":"","firstName":"Jikang","middleName":"","lastName":"Xu","suffix":""},{"id":333150749,"identity":"d64bb123-bd78-4a8f-b3b0-512d89f46c52","order_by":7,"name":"Wenjie Li","email":"","orcid":"","institution":"South China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Wenjie","middleName":"","lastName":"Li","suffix":""},{"id":333150750,"identity":"81e2d958-2ced-403f-babe-e693f61ea28f","order_by":8,"name":"Zhiwei Chen","email":"","orcid":"","institution":"South China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Zhiwei","middleName":"","lastName":"Chen","suffix":""},{"id":333150751,"identity":"65fcc05d-2289-40ba-9620-cd6851d7b189","order_by":9,"name":"Biao Yang","email":"","orcid":"","institution":"Hebei University","correspondingAuthor":false,"prefix":"","firstName":"Biao","middleName":"","lastName":"Yang","suffix":""},{"id":333150752,"identity":"20c1b9d0-4bc6-466e-b9aa-34c48ff63355","order_by":10,"name":"Liu Kun","email":"","orcid":"","institution":"South China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Liu","middleName":"","lastName":"Kun","suffix":""},{"id":333150753,"identity":"f2b7a323-1dcc-4b24-a6c2-c46db9aade76","order_by":11,"name":"Linyuan Mo","email":"","orcid":"","institution":"South China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Linyuan","middleName":"","lastName":"Mo","suffix":""},{"id":333150754,"identity":"18cc255c-3cc2-4d4b-bfa5-66da5900c7da","order_by":12,"name":"Meixia Li","email":"","orcid":"","institution":"South China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Meixia","middleName":"","lastName":"Li","suffix":""},{"id":333150755,"identity":"48cfd868-29ed-4982-a84d-ac6c53693d4d","order_by":13,"name":"Xubing Lu","email":"","orcid":"https://orcid.org/0000-0002-2552-9571","institution":"South China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Xubing","middleName":"","lastName":"Lu","suffix":""},{"id":333150756,"identity":"16f437ca-d302-4843-b321-58fe06e92bc1","order_by":14,"name":"Guofu Zhou","email":"","orcid":"","institution":"South China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Guofu","middleName":"","lastName":"Zhou","suffix":""},{"id":333150757,"identity":"5b0c416e-a4f0-4801-9524-7d22e308eb9c","order_by":15,"name":"Xingsen Gao","email":"","orcid":"https://orcid.org/0000-0002-2725-0785","institution":"South China Normal University","correspondingAuthor":false,"prefix":"","firstName":"Xingsen","middleName":"","lastName":"Gao","suffix":""},{"id":333150758,"identity":"58449ce8-f9c6-46bf-9e31-8db80f952202","order_by":16,"name":"Jun-Ming Liu","email":"","orcid":"https://orcid.org/0000-0001-8988-8429","institution":"Nanjing University","correspondingAuthor":false,"prefix":"","firstName":"Jun-Ming","middleName":"","lastName":"Liu","suffix":""}],"badges":[],"createdAt":"2024-07-24 01:35:15","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-4791621/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-4791621/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1038/s41467-024-55508-z","type":"published","date":"2025-01-07T05:00:00+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":73251439,"identity":"c273c69e-9cbe-45ce-841b-a10db6344b3e","added_by":"auto","created_at":"2025-01-08 08:07:45","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":3347066,"visible":true,"origin":"","legend":"","description":"","filename":"Maintext.pdf","url":"https://assets-eu.researchsquare.com/files/rs-4791621/v1_covered_bf1407c0-c078-4cbf-b496-84f84ef80ef9.pdf"},{"id":61372442,"identity":"fdf5f830-54b6-4bfd-9367-b855cfc7fc68","added_by":"auto","created_at":"2024-07-30 03:26:33","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"supplement","size":4805110,"visible":true,"origin":"","legend":"\u003cp\u003e\u003cbr\u003e\u003c/p\u003e","description":"","filename":"SupplementaryInformation.pdf","url":"https://assets-eu.researchsquare.com/files/rs-4791621/v1/82667ee4db94ff2493b993ab.pdf"},{"id":61372468,"identity":"bc3afd36-8412-443b-89f1-7421819d2e1b","added_by":"auto","created_at":"2024-07-30 03:26:36","extension":"mp4","order_by":2,"title":"","display":"","copyAsset":false,"role":"supplement","size":92004698,"visible":true,"origin":"","legend":"\u003cp\u003eSupplementary Video 1\u003c/p\u003e","description":"","filename":"SupplementaryVideo1.mp4","url":"https://assets-eu.researchsquare.com/files/rs-4791621/v1/fa4a5ac1c787d0fd4b819395.mp4"}],"financialInterests":"There is \u003cb\u003eNO\u003c/b\u003e Competing Interest.","formattedTitle":"In situ training of an in-sensor artificial neural network based on ferroelectric photosensors","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":true,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"nature-portfolio","isNatureJournal":true,"hasQc":false,"allowDirectSubmit":false,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"","title":"Nature Portfolio","twitterHandle":"","acdcEnabled":false,"dfaEnabled":false,"editorialSystem":"ejp","reportingPortfolio":"","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"","lastPublishedDoi":"10.21203/rs.3.rs-4791621/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-4791621/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"In-sensor computing has emerged as an ultrafast and low-power technique for next-generation machine vision. However, in situ training of in-sensor computing systems remains challenging due to the demands for both high-performance devices and efficient programming schemes. Here, we experimentally demonstrate the in situ training of an in-sensor artificial neural network (ANN) based on ferroelectric photosensors (FE-PSs). Our FE-PS exhibits self-powered, fast (\u003c30 μs), and multilevel (\u003e4 bits) photoresponses, as well as long retention (15 days), high endurance (10\u003csup\u003e9\u003c/sup\u003e), high write speed (100 ns), and small cycle-to-cycle and device-to-device variations (~0.66% and ~2.72%, respectively), all of which are desirable for the in situ training. Additionally, a bi-directional closed-loop programming scheme is developed, achieving a precise and efficient weight update for the FE-PS. Using this programming scheme, an in-sensor ANN based on the FE-PSs is trained in situ to recognize traffic signs for commanding a prototype autonomous vehicle. Moreover, this in-sensor ANN operates 50 times faster than a von Neumann machine vision system. This study paves the way for the development of in-sensor computing systems with in situ training capability, which may find applications in new data-streaming machine vision tasks.","manuscriptTitle":"In situ training of an in-sensor artificial neural network based on ferroelectric photosensors","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-07-30 03:26:28","doi":"10.21203/rs.3.rs-4791621/v1","editorialEvents":[],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"nature-communications","isNatureJournal":true,"hasQc":false,"allowDirectSubmit":false,"externalIdentity":"NCOMMS","sideBox":"Learn more about [Nature Communications](http://www.nature.com/ncomms/)","snPcode":"","submissionUrl":"https://mts-ncomms.nature.com/","title":"Nature Communications","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"ejp","reportingPortfolio":"Nature Communications","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"bb98c30f-5972-4274-a43c-8711d1d36e07","owner":[],"postedDate":"July 30th, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[{"id":35272748,"name":"Physical sciences/Materials science/Materials for devices/Information storage"},{"id":35272749,"name":"Physical sciences/Materials science/Condensed-matter physics/Ferroelectrics and multiferroics"}],"tags":[],"updatedAt":"2025-01-08T08:07:38+00:00","versionOfRecord":{"articleIdentity":"rs-4791621","link":"https://doi.org/10.1038/s41467-024-55508-z","journal":{"identity":"nature-communications","isVorOnly":false,"title":"Nature Communications"},"publishedOn":"2025-01-07 05:00:00","publishedOnDateReadable":"January 7th, 2025"},"versionCreatedAt":"2024-07-30 03:26:28","video":"","vorDoi":"10.1038/s41467-024-55508-z","vorDoiUrl":"https://doi.org/10.1038/s41467-024-55508-z","workflowStages":[]},"version":"v1","identity":"rs-4791621","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-4791621","identity":"rs-4791621","version":["v1"]},"buildId":"qtupq5eGEP_6zYnWcrvyt","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}