MLLM4Rec : Multimodal Information Enhancing LLM for Sequential Recommendation

MLLM4Rec : Multimodal Information Enhancing LLM for Sequential Recommendation | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article MLLM4Rec : Multimodal Information Enhancing LLM for Sequential Recommendation Wang Yuxiang, Shi Xin, Zhao Xueqing This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-4960648/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 30 Dec, 2024 Read the published version in Journal of Intelligent Information Systems → Version 1 posted 7 You are reading this latest preprint version Abstract In recent years, With the advent of large language models (LLMs) such as GPT-4, LLaMA, and ChatGLM, leveraging multimodal information (e.g., images and audio) to enhance recommendation systems has become possible. To further enhance the performance of recommendation systems based on large language models (LLMs), we propose MLLM4Rec, a sequence recommendation framework grounded in LLMs. Specifically, our approach integrates multimodal information, with a focus on image data, into LLMs to improve recommendation accuracy. By employing a hybrid prompt learning mechanism combined with role-playing for model fine-tuning, MLLM4Rec effectively bridges the gap between textual and visual representations, enabling text-based LLMs to "read" and interpret images. Moreover, the fine-tuned LLM is utilized to rank retrieval candidates, thereby maintaining its generative capabilities while optimizing item ranking according to user preferences. Extensive experiments were conducted on three publicly available benchmark datasets to evaluate the proposed method. The results demonstrate that MLLM4Rec outperforms traditional sequence recommendation models and pre-trained multimodal models in terms of NDCG, MRR, and Recall metrics. Sequential Recommendation Large Language Model Instruction Tuning Ranking Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Published Journal Publication published 30 Dec, 2024 Read the published version in Journal of Intelligent Information Systems → Version 1 posted Reviewers agreed at journal 20 Sep, 2024 Reviews received at journal 02 Sep, 2024 Reviewers agreed at journal 02 Sep, 2024 Reviewers invited by journal 31 Aug, 2024 Editor assigned by journal 23 Aug, 2024 Submission checks completed at journal 23 Aug, 2024 First submitted to journal 22 Aug, 2024 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-4960648","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":356633280,"identity":"216cfd74-c350-47a9-8f88-b4aae0854b82","order_by":0,"name":"Wang Yuxiang","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABF0lEQVRIie3RsUrEMBjA8ZRCXb7aTXKetK+QUiiCxXuVHkJvCeLkbBHqcuDqcT6Ej5ASqEu9cwxUsHDjdai4dHCwORwcWu0omP+QhJAfIQQhleoPB3JgdRO0k37VLocRLV3MI4QMbRiR6RwM/jshj098ayb8yFrePDOAtU1e47isc+RYB92O5OfRiZlEgF/yC4aPC49k2rV7J5C7WIadxGfU98wkACRoyAgU04dMS8ZQo5AUPWRdSYLBkSQ0VgOIoN5G3kLEjDFmsC8i+slEVL52v4rAFRSl8fzMG2XT9i057n3L6JZ6dXXJJ7aYbd4/mlN7n/O0rLPAscbdpM043H09AvJ9F/cdl+lvzW7eK386pVKpVP+4T/nJaiUt6h/+AAAAAElFTkSuQmCC","orcid":"","institution":"Xi'an Polytechnic University","correspondingAuthor":true,"prefix":"","firstName":"Wang","middleName":"","lastName":"Yuxiang","suffix":""},{"id":356633282,"identity":"0854dd64-d870-4db9-aac3-b9f3d9afdedf","order_by":1,"name":"Shi Xin","email":"","orcid":"","institution":"Xi'an Polytechnic University","correspondingAuthor":false,"prefix":"","firstName":"Shi","middleName":"","lastName":"Xin","suffix":""},{"id":356633288,"identity":"0b2e6505-c431-429c-b48d-ebb106d165ba","order_by":2,"name":"Zhao Xueqing","email":"","orcid":"","institution":"Xi'an Polytechnic University","correspondingAuthor":false,"prefix":"","firstName":"Zhao","middleName":"","lastName":"Xueqing","suffix":""}],"badges":[],"createdAt":"2024-08-23 00:55:13","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-4960648/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-4960648/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1007/s10844-024-00915-3","type":"published","date":"2024-12-30T15:57:36+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":73094811,"identity":"cd9bdb78-d4a7-4646-99fe-9466159f75c9","added_by":"auto","created_at":"2025-01-06 16:24:26","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":2820707,"visible":true,"origin":"","legend":"","description":"","filename":"MLLM4Rec.pdf","url":"https://assets-eu.researchsquare.com/files/rs-4960648/v1_covered_fe11f20b-1eca-4155-81f4-99c9f2088657.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"MLLM4Rec : Multimodal Information Enhancing LLM for Sequential Recommendation","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"journal-of-intelligent-information-systems","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"jiis","sideBox":"Learn more about [Journal of Intelligent Information Systems](http://link.springer.com/journal/10844)","snPcode":"10844","submissionUrl":"https://submission.nature.com/new-submission/10844/3","title":"Journal of Intelligent Information Systems","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Sequential Recommendation, Large Language Model, Instruction Tuning, Ranking","lastPublishedDoi":"10.21203/rs.3.rs-4960648/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-4960648/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"In recent years, With the advent of large language models (LLMs) such as GPT-4, LLaMA, and ChatGLM, leveraging multimodal information (e.g., images and audio) to enhance recommendation systems has become possible. To further enhance the performance of recommendation systems based on large language models (LLMs), we propose MLLM4Rec, a sequence recommendation framework grounded in LLMs. Specifically, our approach integrates multimodal information, with a focus on image data, into LLMs to improve recommendation accuracy. By employing a hybrid prompt learning mechanism combined with role-playing for model fine-tuning, MLLM4Rec effectively bridges the gap between textual and visual representations, enabling text-based LLMs to \"read\" and interpret images. Moreover, the fine-tuned LLM is utilized to rank retrieval candidates, thereby maintaining its generative capabilities while optimizing item ranking according to user preferences. Extensive experiments were conducted on three publicly available benchmark datasets to evaluate the proposed method. The results demonstrate that MLLM4Rec outperforms traditional sequence recommendation models and pre-trained multimodal models in terms of NDCG, MRR, and Recall metrics.","manuscriptTitle":"MLLM4Rec : Multimodal Information Enhancing LLM for Sequential Recommendation","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-09-23 17:59:31","doi":"10.21203/rs.3.rs-4960648/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"reviewerAgreed","content":"110727500637918384671203182594112518495","date":"2024-09-20T06:20:46+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2024-09-03T03:53:46+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"163745759728513691068691945056048607521","date":"2024-09-03T02:28:52+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2024-08-31T14:21:12+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2024-08-23T20:14:31+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2024-08-23T13:52:28+00:00","index":"","fulltext":""},{"type":"submitted","content":"Journal of Intelligent Information Systems","date":"2024-08-23T00:53:47+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"journal-of-intelligent-information-systems","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"jiis","sideBox":"Learn more about [Journal of Intelligent Information Systems](http://link.springer.com/journal/10844)","snPcode":"10844","submissionUrl":"https://submission.nature.com/new-submission/10844/3","title":"Journal of Intelligent Information Systems","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"314d2735-57e5-41b4-aaab-83a82b67be08","owner":[],"postedDate":"September 23rd, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[],"tags":[],"updatedAt":"2025-01-06T16:22:26+00:00","versionOfRecord":{"articleIdentity":"rs-4960648","link":"https://doi.org/10.1007/s10844-024-00915-3","journal":{"identity":"journal-of-intelligent-information-systems","isVorOnly":false,"title":"Journal of Intelligent Information Systems"},"publishedOn":"2024-12-30 15:57:36","publishedOnDateReadable":"December 30th, 2024"},"versionCreatedAt":"2024-09-23 17:59:31","video":"","vorDoi":"10.1007/s10844-024-00915-3","vorDoiUrl":"https://doi.org/10.1007/s10844-024-00915-3","workflowStages":[]},"version":"v1","identity":"rs-4960648","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-4960648","identity":"rs-4960648","version":["v1"]},"buildId":"qtupq5eGEP_6zYnWcrvyt","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}