A deep learning algorithm for KOL segmentation on social media videos

A deep learning algorithm for KOL segmentation on social media videos | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article A deep learning algorithm for KOL segmentation on social media videos Cheng Yang, Fucheng Zheng, Duaa Zuhair Al-Hamid, Peter han Joo Chong, and 1 more This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-3851659/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 24 Nov, 2024 Read the published version in International Journal of Pattern Recognition and Artificial Intelligence → Version 1 posted You are reading this latest preprint version Abstract Nowadays, there is high commercial demand in advertising for KOL (Key Opinion Leader) to advertise commercial products in their social media videos. One effective technique is the product replacement which places the products virtually in the videos. However, one of the challenges of placing the products virtually is the KOL segmentation. Because KOLs often hold products in front of them, which requires the segmentation to segment not only human but also different products. This paper introduces the state-of-the-art deep learning method, namely RSUDISNet, for KOL segmentation on social media video. The proposed deep convolutional neural network (CNN) can segment both KOL and different products which block the KOLs. The proposed technique integrates two CNN technologies. One is the matting objective decomposition network (MODNet), which segments KOLs well but not the products blocking the KOLs. The other one is the two-level nested U-structure network (U2Net) based on salient object detection method to segment the objects well, but not the KOL. The key technique of U2Net is the residual U-block (RSU), which can build neural network architecture deeper, while saving computing. This research employs the RSU block to embed the MODNet to overcome the problem of KOL segmentation from U2Net. Since both MODNet and U2Net are lightweights, the combined network can be used for real-time scenario. After that, the intermediate supervision (IS) training strategy is utilized to overcome the overfitting, which increases the accuracy to a higher level. The experimental results show that our proposed method outperforms the MODNet and U2Net. deep learning image segmentation social media video convolutional neural networks Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Published Journal Publication published 24 Nov, 2024 Read the published version in International Journal of Pattern Recognition and Artificial Intelligence → Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-3851659","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":266591308,"identity":"0b41d056-e22b-4baa-bd43-a06d83695ed4","order_by":0,"name":"Cheng Yang","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAwklEQVRIiWNgGAWjYFACHsYHDDw2QAZj4wFitTAbHOBJA2lpIFoLm8QBhsNgJnFaDI6fPSD9Qea83dr2w0BbamyiCWs5k5cAdNjt5G1nEoFajqXlNhDUcoPHIAGkxewAUAtjw2HitBw4wHMu2ez8Q+K1GDYc4DlgZ3aDWFskz+QYM5zhSU4wuwG0JYEYv/AdP2P+o7LHzt7sfPrDBx9qbAhrUTgAJBh7GBLBKhMIKQcBebDSHwz2xCgeBaNgFIyCEQoADM5LdZPygpsAAAAASUVORK5CYII=","orcid":"","institution":"Auckland University of Technology","correspondingAuthor":true,"prefix":"","firstName":"Cheng","middleName":"","lastName":"Yang","suffix":""},{"id":266591309,"identity":"58bb618c-3267-42b4-847d-20ff971aea23","order_by":1,"name":"Fucheng Zheng","email":"","orcid":"","institution":"Auckland University of Technology","correspondingAuthor":false,"prefix":"","firstName":"Fucheng","middleName":"","lastName":"Zheng","suffix":""},{"id":266591310,"identity":"c62eb3d8-4d1a-44e4-bae6-b00a4fcb0179","order_by":2,"name":"Duaa Zuhair Al-Hamid","email":"","orcid":"","institution":"Auckland University of Technology","correspondingAuthor":false,"prefix":"","firstName":"Duaa","middleName":"Zuhair","lastName":"Al-Hamid","suffix":""},{"id":266591311,"identity":"52a28589-01eb-404e-87ae-f47e1a266054","order_by":3,"name":"Peter han Joo Chong","email":"","orcid":"","institution":"Auckland University of Technology","correspondingAuthor":false,"prefix":"","firstName":"Peter","middleName":"han Joo","lastName":"Chong","suffix":""},{"id":266591312,"identity":"2706101f-c2f6-4b6a-a9a8-76ca36227aca","order_by":4,"name":"Patrick Lam","email":"","orcid":"","institution":"Zyetric Technologies Ltd","correspondingAuthor":false,"prefix":"","firstName":"Patrick","middleName":"","lastName":"Lam","suffix":""}],"badges":[],"createdAt":"2024-01-10 22:44:37","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-3851659/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-3851659/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1142/S0218001424520281","type":"published","date":"2024-11-25T00:00:00+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":70685831,"identity":"a9eacbb5-9e8e-44f4-a025-f6d27397d572","added_by":"auto","created_at":"2024-12-05 15:36:58","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":694720,"visible":true,"origin":"","legend":"","description":"","filename":"DeeplearningalgorithmforKOLsegmentationonsocialmediavideov5.pdf","url":"https://assets-eu.researchsquare.com/files/rs-3851659/v1_covered_47abe2e5-41c3-4af1-9838-41b3336ae52c.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"A deep learning algorithm for KOL segmentation on social media videos","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"deep learning, image segmentation, social media video, convolutional neural networks","lastPublishedDoi":"10.21203/rs.3.rs-3851659/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-3851659/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eNowadays, there is high commercial demand in advertising for KOL (Key Opinion Leader) to advertise commercial products in their social media videos. One effective technique is the product replacement which places the products virtually in the videos. However, one of the challenges of placing the products virtually is the KOL segmentation. Because KOLs often hold products in front of them, which requires the segmentation to segment not only human but also different products. This paper introduces the state-of-the-art deep learning method, namely RSUDISNet, for KOL segmentation on social media video. The proposed deep convolutional neural network (CNN) can segment both KOL and different products which block the KOLs. The proposed technique integrates two CNN technologies. One is the matting objective decomposition network (MODNet), which segments KOLs well but not the products blocking the KOLs. The other one is the two-level nested U-structure network (U2Net) based on salient object detection method to segment the objects well, but not the KOL. The key technique of U2Net is the residual U-block (RSU), which can build neural network architecture deeper, while saving computing. This research employs the RSU block to embed the MODNet to overcome the problem of KOL segmentation from U2Net. Since both MODNet and U2Net are lightweights, the combined network can be used for real-time scenario. After that, the intermediate supervision (IS) training strategy is utilized to overcome the overfitting, which increases the accuracy to a higher level. The experimental results show that our proposed method outperforms the MODNet and U2Net.\u003c/p\u003e","manuscriptTitle":"A deep learning algorithm for KOL segmentation on social media videos","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-01-15 02:55:43","doi":"10.21203/rs.3.rs-3851659/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"d83f8506-6cc4-4032-8afd-1460afa27a37","owner":[],"postedDate":"January 15th, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[],"tags":[],"updatedAt":"2024-12-05T15:36:53+00:00","versionOfRecord":{"articleIdentity":"rs-3851659","link":"https://doi.org/10.1142/S0218001424520281","journal":{"identity":"international-journal-of-pattern-recognition-and-artificial-intelligence","isVorOnly":true,"title":"International Journal of Pattern Recognition and Artificial Intelligence"},"publishedOn":"2024-11-25 00:00:00","publishedOnDateReadable":"November 25th, 2024"},"versionCreatedAt":"2024-01-15 02:55:43","video":"","vorDoi":"10.1142/S0218001424520281","vorDoiUrl":"https://doi.org/10.1142/S0218001424520281","workflowStages":[]},"version":"v1","identity":"rs-3851659","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-3851659","identity":"rs-3851659","version":["v1"]},"buildId":"qtupq5eGEP_6zYnWcrvyt","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}