Transforming Handwritten Answer Assessment: A Multi-Modal Approach Combining Text Detection, Handwriting Recognition, and Language Models

Transforming Handwritten Answer Assessment: A Multi-Modal Approach Combining Text Detection, Handwriting Recognition, and Language Models | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Transforming Handwritten Answer Assessment: A Multi-Modal Approach Combining Text Detection, Handwriting Recognition, and Language Models Aditya Hiremath, Nipun Irabatti, Akhilesh Desai, Prabhuraj Dhondge, and 1 more This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-4301899/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 3 You are reading this latest preprint version Abstract This paper proposes an automated system for grading handwritten subjective answers, leveraging advanced computer vision, natural language processing, and large language model techniques. Although time-consuming, the system presents a promising theoretical approach by employing CRAFT for text detection, TrOCR for handwritten text recognition, and a fine-tuned language model for answer evaluation. Experimental results demonstrate the system's potential accuracy in transcribing handwritten text and consistency in grading answers compared to human raters. The proposed methodology offers a scalable and efficient solution to automate the traditionally labor-intensive task of grading handwritten responses, with the potential to transform education assessment practices. The system's performance, limitations, and future research directions to improve efficiency are discussed. Handwritten answer grading text detection optical character recognition natural language processing language models transformer models Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Under Review Version 1 posted Editor assigned by journal 12 May, 2024 Submission checks completed at journal 22 Apr, 2024 First submitted to journal 21 Apr, 2024 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-4301899","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":294169133,"identity":"2135eedf-0ecd-4d48-86c4-5dab4c81a4b2","order_by":0,"name":"Aditya Hiremath","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA10lEQVRIiWNgGAWjYFCCxDaGBAYGOXn2BiDHwIJ4LcaGPQdAWiSI0ZLABtbYcCMBRBOhhb89ue3Bwz02jI0zn1/d8KNAAijSnYBXi8SZh+0GCc/SmNmlc8pu9gAdJnHm7Ab81txIbJNIOHCYjXF2TtoNHqAWA4lc/FrkIVr+8zDcPJN28w8xWgwgWg5IMNxgP3abKFsMzzwEaUk2MOzJYbstYyDBQ9AvcsfTn0n+OGBXP5/9+LObb/7YyPG39xLwPgLwGIBJYpWDAPsDUlSPglEwCkbBCAIAM4tMJgSYBDEAAAAASUVORK5CYII=","orcid":"","institution":"AISSMS Institute of Information Technology","correspondingAuthor":true,"prefix":"","firstName":"Aditya","middleName":"","lastName":"Hiremath","suffix":""},{"id":294169136,"identity":"b8320c08-d5b6-49ae-aae8-05253fca96ab","order_by":1,"name":"Nipun Irabatti","email":"","orcid":"","institution":"AISSMS Institute of Information Technology","correspondingAuthor":false,"prefix":"","firstName":"Nipun","middleName":"","lastName":"Irabatti","suffix":""},{"id":294169138,"identity":"c9c0623a-b8ae-4996-be1d-c64c1d7c2c85","order_by":2,"name":"Akhilesh Desai","email":"","orcid":"","institution":"AISSMS Institute of Information Technology","correspondingAuthor":false,"prefix":"","firstName":"Akhilesh","middleName":"","lastName":"Desai","suffix":""},{"id":294169140,"identity":"3834b729-6861-4df7-886f-9e307ead9bba","order_by":3,"name":"Prabhuraj Dhondge","email":"","orcid":"","institution":"AISSMS Institute of Information Technology","correspondingAuthor":false,"prefix":"","firstName":"Prabhuraj","middleName":"","lastName":"Dhondge","suffix":""},{"id":294169142,"identity":"f2bed64a-d7d9-4769-b10e-a1f775f0876f","order_by":4,"name":"Shagufta Sheikh","email":"","orcid":"","institution":"AISSMS Institute of Information Technology","correspondingAuthor":false,"prefix":"","firstName":"Shagufta","middleName":"","lastName":"Sheikh","suffix":""}],"badges":[],"createdAt":"2024-04-21 19:08:40","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-4301899/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-4301899/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":55290050,"identity":"ee64d4d5-da3c-44d4-809b-0226c50a282d","added_by":"auto","created_at":"2024-04-25 08:57:52","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":782018,"visible":true,"origin":"","legend":"","description":"","filename":"HSAEFinalresearchpaper.pdf","url":"https://assets-eu.researchsquare.com/files/rs-4301899/v1_covered_66ecf9b6-f3d2-4378-b496-0d73d42eb23d.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Transforming Handwritten Answer Assessment: A Multi-Modal Approach Combining Text Detection, Handwriting Recognition, and Language Models","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"international-journal-of-data-science-and-analytics","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"jdsa","sideBox":"Learn more about [International Journal of Data Science and Analytics](http://link.springer.com/journal/41060)","snPcode":"41060","submissionUrl":"https://submission.nature.com/new-submission/41060/3","title":"International Journal of Data Science and Analytics","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Handwritten answer grading, text detection, optical character recognition, natural language processing, language models, transformer models","lastPublishedDoi":"10.21203/rs.3.rs-4301899/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-4301899/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"\u003cp\u003eThis paper proposes an automated system for grading handwritten subjective answers, leveraging advanced computer vision, natural language processing, and large language model techniques. Although time-consuming, the system presents a promising theoretical approach by employing CRAFT for text detection, TrOCR for handwritten text recognition, and a fine-tuned language model for answer evaluation. Experimental results demonstrate the system's potential accuracy in transcribing handwritten text and consistency in grading answers compared to human raters. The proposed methodology offers a scalable and efficient solution to automate the traditionally labor-intensive task of grading handwritten responses, with the potential to transform education assessment practices. The system's performance, limitations, and future research directions to improve efficiency are discussed.\u003c/p\u003e","manuscriptTitle":"Transforming Handwritten Answer Assessment: A Multi-Modal Approach Combining Text Detection, Handwriting Recognition, and Language Models","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2024-04-25 08:49:44","doi":"10.21203/rs.3.rs-4301899/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"editorAssigned","content":"","date":"2024-05-12T10:06:33+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2024-04-22T15:21:00+00:00","index":"","fulltext":""},{"type":"submitted","content":"International Journal of Data Science and Analytics","date":"2024-04-21T18:56:40+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"international-journal-of-data-science-and-analytics","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"jdsa","sideBox":"Learn more about [International Journal of Data Science and Analytics](http://link.springer.com/journal/41060)","snPcode":"41060","submissionUrl":"https://submission.nature.com/new-submission/41060/3","title":"International Journal of Data Science and Analytics","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"11e56c24-cc3d-49d0-b044-5605211c7b6e","owner":[],"postedDate":"April 25th, 2024","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[],"tags":[],"updatedAt":"2024-05-16T11:51:16+00:00","versionOfRecord":[],"versionCreatedAt":"2024-04-25 08:49:44","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-4301899","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-4301899","identity":"rs-4301899","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}