A Survey on Knowledge-Enhanced Multimodal Learning

A Survey on Knowledge-Enhanced Multimodal Learning | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article A Survey on Knowledge-Enhanced Multimodal Learning Maria Lymperaiou, Giorgos Stamou This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-3032445/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 09 Sep, 2024 Read the published version in Artificial Intelligence Review → Version 1 posted 3 You are reading this latest preprint version Abstract Multimodal learning has been a field of increasing interest, aiming to combine various modalities in a single joint representation. Especially in the area of visi-olinguistic (VL) learning multiple models and techniques have been developed, targeting a variety of tasks that involve images and text. VL models have reached unprecedented performances by extending the idea of Transformers, so that both modalities can learn from each other. Massive pre-training procedures enable VL models to acquire a certain level of real-world understanding, although many gaps can be identified: the limited comprehension of commonsense, factual, temporal and other everyday knowledge aspects questions the extendability of VL tasks. Knowledge graphs and other knowledge sources can fill those gaps by explicitly providing missing information, unlocking novel capabilities of VL models. In the same time, knowledge graphs enhance explainability, fairness and validity of decision making, issues of outermost importance for such complex implementations. The current survey aims to unify the fields of VL representation learning and knowledge graphs, and provides a taxonomy and analysis of knowledge-enhanced VL models. Multimodal Learning Vision-and-Language Knowledge Graphs Transformers Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Published Journal Publication published 09 Sep, 2024 Read the published version in Artificial Intelligence Review → Version 1 posted Editor assigned by journal 07 Jun, 2023 Submission checks completed at journal 07 Jun, 2023 First submitted to journal 07 Jun, 2023 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-3032445","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":207660092,"identity":"cac1423e-f4aa-4260-a492-5164a2a2b8fa","order_by":0,"name":"Maria Lymperaiou","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABAklEQVRIiWNgGAWjYJCCAwwFQJK9sYHhAZBNpBYDIMlzsIEhAaSFjShNIC0SCQzEadFtP/vwwA+Dw9Hmko+bPyTU3GEwl+8x/MDw6x5OLWZn0g0O9hgczt05O7FNIuHYMwbLNh5jCca+YtxaDqQxHOABatlwO7GNIYHtMIPBMR4DCcaeBNxazj9jOPgHpOXmQaDD/oG1GP/Aq+VGGsNhsC03GBskEtvAWswkGH7g0/KM4bCMQXruzh6gXxL7nvEYHEsrs0hswOewNOaPbyqsc7ezH3/84cO3O3IGhw9vvvHhD24tUNAMjhoQ4AGTwKAgBOrgWqDgD0Eto2AUjIJRMHIAAC+0YZ/aEqPTAAAAAElFTkSuQmCC","orcid":"","institution":"National Technical University of Athens","correspondingAuthor":true,"submittingAuthor":false,"prefix":"","firstName":"Maria","middleName":"","lastName":"Lymperaiou","suffix":""},{"id":207660093,"identity":"d5ae8ddc-e41f-4e98-a66d-f19a87ce53f2","order_by":1,"name":"Giorgos Stamou","email":"","orcid":"","institution":"National Technical University of Athens","correspondingAuthor":false,"submittingAuthor":false,"prefix":"","firstName":"Giorgos","middleName":"","lastName":"Stamou","suffix":""}],"badges":[],"createdAt":"2023-06-07 06:29:26","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-3032445/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-3032445/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1007/s10462-024-10825-z","type":"published","date":"2024-09-09T15:57:24+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":64619413,"identity":"cd4601b6-c834-429f-bf1c-986772f6a6e8","added_by":"auto","created_at":"2024-09-16 16:14:46","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":1012573,"visible":true,"origin":"","legend":"","description":"","filename":"AIRSpringerNatureKVLsurvey.pdf","url":"https://assets-eu.researchsquare.com/files/rs-3032445/v1_covered_0001dcff-3ef4-410a-bedf-b238b9302b07.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"A Survey on Knowledge-Enhanced Multimodal Learning","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"artificial-intelligence-review","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"aire","sideBox":"Learn more about [Artificial Intelligence Review](http://link.springer.com/journal/10462)","snPcode":"10462","submissionUrl":"https://submission.nature.com/new-submission/10462/3","title":"Artificial Intelligence Review","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Multimodal Learning, Vision-and-Language, Knowledge Graphs, Transformers","lastPublishedDoi":"10.21203/rs.3.rs-3032445/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-3032445/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Multimodal learning has been a field of increasing interest, aiming to combine various modalities in a single joint representation. Especially in the area of visi-olinguistic (VL) learning multiple models and techniques have been developed, targeting a variety of tasks that involve images and text. VL models have reached unprecedented performances by extending the idea of Transformers, so that both modalities can learn from each other. Massive pre-training procedures enable VL models to acquire a certain level of real-world understanding, although many gaps can be identified: the limited comprehension of commonsense, factual, temporal and other everyday knowledge aspects questions the extendability of VL tasks. Knowledge graphs and other knowledge sources can fill those gaps by explicitly providing missing information, unlocking novel capabilities of VL models. In the same time, knowledge graphs enhance explainability, fairness and validity of decision making, issues of outermost importance for such complex implementations. The current survey aims to unify the fields of VL representation learning and knowledge graphs, and provides a taxonomy and analysis of knowledge-enhanced VL models.","manuscriptTitle":"A Survey on Knowledge-Enhanced Multimodal Learning","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2023-06-09 09:20:50","doi":"10.21203/rs.3.rs-3032445/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"editorAssigned","content":"","date":"2023-06-07T12:17:39+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2023-06-07T10:15:24+00:00","index":"","fulltext":""},{"type":"submitted","content":"Artificial Intelligence Review","date":"2023-06-07T06:24:53+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"artificial-intelligence-review","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"aire","sideBox":"Learn more about [Artificial Intelligence Review](http://link.springer.com/journal/10462)","snPcode":"10462","submissionUrl":"https://submission.nature.com/new-submission/10462/3","title":"Artificial Intelligence Review","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"656b50e0-a4a6-44b2-8b23-606b8fc2ad04","owner":[],"postedDate":"June 9th, 2023","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[],"tags":[],"updatedAt":"2024-09-16T16:07:43+00:00","versionOfRecord":{"articleIdentity":"rs-3032445","link":"https://doi.org/10.1007/s10462-024-10825-z","journal":{"identity":"artificial-intelligence-review","isVorOnly":false,"title":"Artificial Intelligence Review"},"publishedOn":"2024-09-09 15:57:24","publishedOnDateReadable":"September 9th, 2024"},"versionCreatedAt":"2023-06-09 09:20:50","video":"","vorDoi":"10.1007/s10462-024-10825-z","vorDoiUrl":"https://doi.org/10.1007/s10462-024-10825-z","workflowStages":[]},"version":"v1","identity":"rs-3032445","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-3032445","identity":"rs-3032445","version":["v1"]},"buildId":"7rjqhiLT3MXkJMwkYKINL","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}