Surgical Video Workflow Analysis via Visual-Language Learning

Surgical Video Workflow Analysis via Visual-Language Learning | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Article Surgical Video Workflow Analysis via Visual-Language Learning Pengpeng Li, Xiangbo Shu, Chun-Mei Feng, Yifei Feng, Wangmeng Zuo, and 1 more This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-5205336/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 12 You are reading this latest preprint version Abstract Surgical video workflow analysis has made intensive development in computer-assisted surgery by combining deep learning models, aiming to enhance surgical scene analysis and decision-making. However, previous research has mainly focused on identifying coarse-grained surgical phases from surgical videos. In order to provide a more comprehensive fine-grained analysis of surgical videos, this work focuses on accurately identifying triplets from surgical videos. Specifically, we propose a vision-language deep learning framework that incorporates intra- and inter- triplet modeling, termed I 2 TM, to explore the relationships among triplets and leverage the model understanding of the entire surgical process, thereby enhancing the accuracy and robustness of recognition. Besides, we also develop a new surgical triplet semantic enhancer (TSE) to establish semantic relationships, both intra- and inter-triplets, across visual and textual modalities. Extensive experimental results on surgical video benchmark datasets demonstrate that our approach can capture finer semantics, achieve effective surgical video understanding and analysis, with potential for widespread medical applications. Biological sciences/Computational biology and bioinformatics/Machine learning Physical sciences/Mathematics and computing/Computer science Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Under Review Version 1 posted Editorial decision: Revision requested 28 Oct, 2024 Reviews received at journal 27 Oct, 2024 Reviews received at journal 27 Oct, 2024 Reviewers agreed at journal 23 Oct, 2024 Reviewers agreed at journal 21 Oct, 2024 Reviewers agreed at journal 20 Oct, 2024 Reviewers agreed at journal 18 Oct, 2024 Reviewers agreed at journal 09 Oct, 2024 Reviewers invited by journal 08 Oct, 2024 Editor assigned by journal 08 Oct, 2024 Submission checks completed at journal 08 Oct, 2024 First submitted to journal 04 Oct, 2024 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-5205336","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Article","associatedPublications":[],"authors":[{"id":371240133,"identity":"da5c8264-094c-45d2-ba8c-e5d9035edc57","order_by":0,"name":"Pengpeng Li","email":"","orcid":"","institution":"Nanjing University of Science and Technology","correspondingAuthor":false,"prefix":"","firstName":"Pengpeng","middleName":"","lastName":"Li","suffix":""},{"id":371240134,"identity":"338d81b0-1548-4c69-8717-2db82bbc064b","order_by":1,"name":"Xiangbo Shu","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA0UlEQVRIiWNgGAWjYDACCQYGZoYKIAYDNqK1nAFpYSZFC2MbiEWsFvnZPWbShfPusBucP3+A4UPZYQb+2Q34tRjcOWNsPHPbM2aDG8kMjDPOHWaQuHOAgBaJHMPHvNsOA7UwMzDzth0GiiQQcNiMHIPDvHOAWs4fZmD+S4wWhhsgWxqAWg4kg8KBCC0GN9KKjWccO8wseSPZ4GDPuXQeiRsEHZa8Tbqg5nAy3/mDDx/8KLOW459ByGFQkAwiDgAxD3HqgcCOaJWjYBSMglEw8gAApo5AHSbmUtsAAAAASUVORK5CYII=","orcid":"","institution":"Nanjing University of Science and Technology","correspondingAuthor":true,"prefix":"","firstName":"Xiangbo","middleName":"","lastName":"Shu","suffix":""},{"id":371240135,"identity":"f5249793-e595-45f1-a563-154dfeb64aab","order_by":2,"name":"Chun-Mei Feng","email":"","orcid":"","institution":"Agency for Science, Technology and Research","correspondingAuthor":false,"prefix":"","firstName":"Chun-Mei","middleName":"","lastName":"Feng","suffix":""},{"id":371240136,"identity":"605ba249-a906-4eb1-b9d4-c56efc9ee516","order_by":3,"name":"Yifei Feng","email":"","orcid":"","institution":"Nanjing Medical University","correspondingAuthor":false,"prefix":"","firstName":"Yifei","middleName":"","lastName":"Feng","suffix":""},{"id":371240137,"identity":"4df393f4-23ba-4947-adfd-1c98dbb7bdec","order_by":4,"name":"Wangmeng Zuo","email":"","orcid":"","institution":"Harbin Institute of Technology","correspondingAuthor":false,"prefix":"","firstName":"Wangmeng","middleName":"","lastName":"Zuo","suffix":""},{"id":371240138,"identity":"fbd8dbfc-d565-4c3c-9573-4a1b3f167c60","order_by":5,"name":"Jinhui Tang","email":"","orcid":"","institution":"Nanjing University of Science and Technology","correspondingAuthor":false,"prefix":"","firstName":"Jinhui","middleName":"","lastName":"Tang","suffix":""}],"badges":[],"createdAt":"2024-10-04 16:38:26","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-5205336/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-5205336/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":69761061,"identity":"edbe9bad-b037-4b05-9546-9e39a87a8a62","added_by":"auto","created_at":"2024-11-25 04:45:19","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":6020812,"visible":true,"origin":"","legend":"","description":"","filename":"NMILPPv1.pdf","url":"https://assets-eu.researchsquare.com/files/rs-5205336/v1_covered_8928d99c-755e-4de9-976e-36b6ab172c48.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Surgical Video Workflow Analysis via Visual-Language Learning","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"npj-health-systems","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"","sideBox":"Learn more about [npj Health Systems](https://www.nature.com/npjhealthsyst/)","snPcode":"44401","submissionUrl":"https://submission.springernature.com/new-submission/44401/3","title":"npj Health Systems","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"NPJ","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"","lastPublishedDoi":"10.21203/rs.3.rs-5205336/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-5205336/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eSurgical video workflow analysis has made intensive development in computer-assisted surgery by combining deep learning models, aiming to enhance surgical scene analysis and decision-making. However, previous research has mainly focused on identifying coarse-grained surgical phases from surgical videos. In order to provide a more comprehensive fine-grained analysis of surgical videos, this work focuses on accurately identifying triplets \u0026lt;instrument, verb, target\u0026gt; from surgical videos. Specifically, we propose a vision-language deep learning framework that incorporates intra- and inter- triplet modeling, termed I\u003csup\u003e2\u003c/sup\u003eTM, to explore the relationships among triplets and leverage the model understanding of the entire surgical process, thereby enhancing the accuracy and robustness of recognition. Besides, we also develop a new surgical triplet semantic enhancer (TSE) to establish semantic relationships, both intra- and inter-triplets, across visual and textual modalities. Extensive experimental results on surgical video benchmark datasets demonstrate that our approach can capture finer semantics, achieve effective surgical video understanding and analysis, with potential for widespread medical applications.\u003c/p\u003e","manuscriptTitle":"Surgical Video Workflow Analysis via Visual-Language Learning","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-11-25 04:37:10","doi":"10.21203/rs.3.rs-5205336/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"decision","content":"Revision requested","date":"2024-10-28T12:17:07+00:00","index":"","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2024-10-28T03:21:42+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2024-10-27T15:22:36+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"5159844430065901660585038996043844105","date":"2024-10-23T05:07:57+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"218935748481564782952588972212329197740","date":"2024-10-21T07:01:26+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"294588058755274003646060987828836925720","date":"2024-10-20T17:34:09+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"55753001180740119625233364896213737447","date":"2024-10-19T02:42:59+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"194989897339458287911585763672520368075","date":"2024-10-09T23:08:59+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2024-10-09T01:25:56+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2024-10-09T01:22:47+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2024-10-08T13:32:33+00:00","index":"","fulltext":""},{"type":"submitted","content":"npj Health Systems","date":"2024-10-04T16:35:33+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"npj-health-systems","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"","sideBox":"Learn more about [npj Health Systems](https://www.nature.com/npjhealthsyst/)","snPcode":"44401","submissionUrl":"https://submission.springernature.com/new-submission/44401/3","title":"npj Health Systems","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"stoa","reportingPortfolio":"NPJ","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"555d1f1f-c377-44fa-aef1-d8cb1e8dd91b","owner":[],"postedDate":"November 25th, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[{"id":39503097,"name":"Biological sciences/Computational biology and bioinformatics/Machine learning"},{"id":39503098,"name":"Physical sciences/Mathematics and computing/Computer science"}],"tags":[],"updatedAt":"2024-12-23T17:08:39+00:00","versionOfRecord":[],"versionCreatedAt":"2024-11-25 04:37:10","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-5205336","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-5205336","identity":"rs-5205336","version":["v1"]},"buildId":"qtupq5eGEP_6zYnWcrvyt","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}