RAG Based Implementation for Smart Web Scraping using Craw4ai

RAG Based Implementation for Smart Web Scraping using Craw4ai | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article RAG Based Implementation for Smart Web Scraping using Craw4ai Kunal Singh Chauhan, Sumit Srivastava This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-6846502/v1 This work is licensed under a CC BY 4.0 License Status: Posted Version 1 posted You are reading this latest preprint version Abstract With the onset of LLMs, the way people surf the internet has changed. Instead of manually selecting information from the sources present on the internet users just ask a large language model like Chat-GPT to search through the data, select the most relevant bits of data, and then generate a response for the user prompt. However, a major challenge with LLMs is that they work on pre trained data and may produce hallucinated results when they lack contextual accuracy and updated data. RAG short for retrieval augmentation generation is an approach used to increase data and context on which an LLM can act on. It feeds untrained data which is processed and attached as external context to the user prompt. This approach is used to create a web scraping tool which leverages the capabilities of an LLM to answer any query referring to information from the web. The tool uses web scraping to provide contextually accurate and real time information from the web thus reducing hallucinations in the answer. The modular RAG architecture scrapes the web using crawl4ai to extract raw html data. This data is then indexed, broken down to make chunks and converted into vector embeddings using Sentence Transformer models. RAG semantic search and retrieval vector storage web scraping text embeddings Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Posted Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-6846502","acceptedTermsAndConditions":true,"allowDirectSubmit":true,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":488171452,"identity":"875cd066-b79d-4ca0-adb6-1c62994c731b","order_by":0,"name":"Kunal Singh Chauhan","email":"","orcid":"","institution":"Manipal University Jaipur","correspondingAuthor":false,"prefix":"","firstName":"Kunal","middleName":"Singh","lastName":"Chauhan","suffix":""},{"id":488171453,"identity":"17af8519-dc8a-44f3-8322-34efdcb7ec27","order_by":1,"name":"Sumit Srivastava","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABT0lEQVRIie2RMWvCQBTHXynEJW3W52I+QeGKEFqw+FUigku1TSmIg0igEBexa/wWAcH5yYEusa5OhSJ0shAIOEpftCkeKZ0LzX+4+z/u/e79jwPIleuPipJFSyuDazo6uuLNziCUIokpuiqCPyFKBwhSDzOI6d/WZAS9+3Osr+OoK83yVL7JB+8VLvqLqXS6CEahKY4QsWoFHExee9iwkGbyckIzIUfeI1jhnS39GUJxsFEQ3CMkNJ0sIK1xMiES8syzwaKmkLqGfK0yxfT3SI+ReRzRrlEdu/PogCw3jOwQqioCh2CnQisMBE69Si2A8GsKd7LhKai+JfwIiHsSpI2LYaXuU+hI/8XWrRVPGQ1Rx/DdOQ7Wb42jTqcnzKcCmy3ePPvzcey07ZK1bJZjZ1spGf16kPkYJSpyTt70xH+b32W4+05IkVy5cuX69/oElN6KiGMaGPIAAAAASUVORK5CYII=","orcid":"","institution":"Manipal University Jaipur","correspondingAuthor":true,"prefix":"","firstName":"Sumit","middleName":"","lastName":"Srivastava","suffix":""}],"badges":[],"createdAt":"2025-06-08 09:08:14","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-6846502/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-6846502/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":101855493,"identity":"5cc707e0-2165-4615-b4c0-a4463e86a290","added_by":"auto","created_at":"2026-02-04 10:43:20","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":590783,"visible":true,"origin":"","legend":"","description":"","filename":"FinalPaper.pdf","url":"https://assets-eu.researchsquare.com/files/rs-6846502/v1_covered_eaa71e8b-32ee-4ee4-adcd-73aa2ae4c48c.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"RAG Based Implementation for Smart Web Scraping using Craw4ai","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":true,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true},"keywords":"RAG, semantic search and retrieval, vector storage, web scraping, text embeddings","lastPublishedDoi":"10.21203/rs.3.rs-6846502/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-6846502/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eWith the onset of LLMs, the way people surf the internet has changed. Instead of manually selecting information from the sources present on the internet users just ask a large language model like Chat-GPT to search through the data, select the most relevant bits of data, and then generate a response for the user prompt. However, a major challenge with LLMs is that they work on pre trained data and may produce hallucinated results when they lack contextual accuracy and updated data. RAG short for retrieval augmentation generation is an approach used to increase data and context on which an LLM can act on. It feeds untrained data which is processed and attached as external context to the user prompt. This approach is used to create a web scraping tool which leverages the capabilities of an LLM to answer any query referring to information from the web. The tool uses web scraping to provide contextually accurate and real time information from the web thus reducing hallucinations in the answer. The modular RAG architecture scrapes the web using crawl4ai to extract raw html data. This data is then indexed, broken down to make chunks and converted into vector embeddings using Sentence Transformer models.\u003c/p\u003e","manuscriptTitle":"RAG Based Implementation for Smart Web Scraping using Craw4ai","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-07-22 05:35:13","doi":"10.21203/rs.3.rs-6846502/v1","editorialEvents":[{"type":"communityComments","content":0}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"researchsquare","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":true,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"/submission","title":"Research Square","twitterHandle":"researchsquare","acdcEnabled":true,"dfaEnabled":false,"editorialSystem":"","reportingPortfolio":"","inReviewEnabled":false,"inReviewRevisionsEnabled":true}}],"origin":"","ownerIdentity":"7d50b301-065d-45dc-b04a-9bb3ea6cad1d","owner":[],"postedDate":"July 22nd, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"posted","subjectAreas":[],"tags":[],"updatedAt":"2026-02-04T10:42:47+00:00","versionOfRecord":[],"versionCreatedAt":"2025-07-22 05:35:13","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-6846502","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-6846502","identity":"rs-6846502","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}