Improving Counterfactual Story Rewriting with Policy-Gradient Approaches

Improving Counterfactual Story Rewriting with Policy-Gradient Approaches | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Research Article Improving Counterfactual Story Rewriting with Policy-Gradient Approaches Amelie Girard, Inigo Jauregi Unanue, Massimo Piccardi This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-6694750/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted 8 You are reading this latest preprint version Abstract Counterfactual story rewriting is the task of revising an existing narrative in light of an alternative event while retaining the unchanged elements of the story and its overall coherence. This task is challenging for NLP models because the changes expected in the original story are typically small and circumscribed, and conventional training objectives such as maximum likelihood fail to capture them effectively. For this reason, in this paper we propose a reinforcement learning (RL) approach to counterfactual story rewriting that explicitly rewards the desired counterfactual changes. Specifically, we propose fine-tuning a seq2seq model using policy-gradient approaches (REINFORCE with baseline and proximal policy optimization) with a reward function designed to capture both adherence to the reference edited story and semantic coherence. Experimental results on the TimeTravel dataset show that our RL-based approach has been capable of producing better rewritings compared to the conventionally-trained baseline, and outperform two contemporary large language models on this task. Overall, our findings highlight the benefit of reinforcement learning for complex, controlled text generation tasks requiring nuanced predictions. Counterfactual Story Rewriting Reinforcement Learning Policy Gradient REINFORCE ROUGE BARTScore Full Text Additional Declarations No competing interests reported. Cite Share Download PDF Status: Under Review Version 1 posted Reviews received at journal 05 Aug, 2025 Reviews received at journal 04 Aug, 2025 Reviewers agreed at journal 09 Jul, 2025 Reviewers agreed at journal 07 Jul, 2025 Reviewers invited by journal 04 Jul, 2025 Editor assigned by journal 04 Jul, 2025 Submission checks completed at journal 19 May, 2025 First submitted to journal 18 May, 2025 You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-6694750","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Research Article","associatedPublications":[],"authors":[{"id":481678697,"identity":"deeb8463-f1e4-4bca-99c7-2db647d286dc","order_by":0,"name":"Amelie Girard","email":"","orcid":"","institution":"University of Technology Sydney","correspondingAuthor":false,"prefix":"","firstName":"Amelie","middleName":"","lastName":"Girard","suffix":""},{"id":481678699,"identity":"26a931f6-6375-464a-9ba0-ec5ef548cea7","order_by":1,"name":"Inigo Jauregi Unanue","email":"","orcid":"","institution":"University of Technology Sydney","correspondingAuthor":false,"prefix":"","firstName":"Inigo","middleName":"Jauregi","lastName":"Unanue","suffix":""},{"id":481678701,"identity":"e1d8a233-5867-49b8-95db-86274519bd0a","order_by":2,"name":"Massimo Piccardi","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAABDklEQVRIiWNgGAWjYLCCBBDBDGYy8/BDxJhJ0CLZwEyEFiTAzGBwgIAW3fbjjz88zLFhMGdnfvjh5x5rGeMb+Yc/MFRYJzawnzHApsXsTI6ZROK2NAbLZjZjyZ5n6TxmN5LZJBjOpCc28ORg13Igh40hcdthBoPDPGwMPAcOg7UwMLYdTmxgwKHl/PPHH2BaGP8AtRjPSGb+wPgPqIX/DXYtNxIMJGBamEG2GEgkM0gwNgC1SOCw5cYbsF94QH6RljmQziNx5rGZRMKxdOM2iWcF2B2W/vjjz202cub8hx9+fHPA2p6/PfHxhw811rL9/MkbsAczBPCguiEBiNnwqQcBrM4eBaNgFIyCUQACADMsXBtRT7r2AAAAAElFTkSuQmCC","orcid":"","institution":"University of Technology Sydney","correspondingAuthor":true,"prefix":"","firstName":"Massimo","middleName":"","lastName":"Piccardi","suffix":""}],"badges":[],"createdAt":"2025-05-19 03:53:20","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-6694750/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-6694750/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":86315829,"identity":"06e5ef81-b553-43e5-b9e6-01c88a65802d","added_by":"auto","created_at":"2025-07-09 08:54:28","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":393192,"visible":true,"origin":"","legend":"","description":"","filename":"ImprovingCounterfactualStoryRewritingwithPolicyGradientApproaches.pdf","url":"https://assets-eu.researchsquare.com/files/rs-6694750/v1_covered_8329e60b-12b7-47c3-8cda-346ba29ba6f6.pdf"}],"financialInterests":"No competing interests reported.","formattedTitle":"Improving Counterfactual Story Rewriting with Policy-Gradient Approaches","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"language-resources-and-evaluation","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"lrev","sideBox":"Learn more about [Language Resources and Evaluation](http://link.springer.com/journal/10579)","snPcode":"10579","submissionUrl":"https://submission.nature.com/new-submission/10579/3","title":"Language Resources and Evaluation","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Counterfactual Story Rewriting, Reinforcement Learning, Policy Gradient, REINFORCE, ROUGE, BARTScore","lastPublishedDoi":"10.21203/rs.3.rs-6694750/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-6694750/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Counterfactual story rewriting is the task of revising an existing narrative in light of an alternative event while retaining the unchanged elements of the story and its overall coherence. This task is challenging for NLP models because the changes expected in the original story are typically small and circumscribed, and conventional training objectives such as maximum likelihood fail to capture them effectively. For this reason, in this paper we propose a reinforcement learning (RL) approach to counterfactual story rewriting that explicitly rewards the desired counterfactual changes. Specifically, we propose fine-tuning a seq2seq model using policy-gradient approaches (REINFORCE with baseline and proximal policy optimization) with a reward function designed to capture both adherence to the reference edited story and semantic coherence. Experimental results on the TimeTravel dataset show that our RL-based approach has been capable of producing better rewritings compared to the conventionally-trained baseline, and outperform two contemporary large language models on this task. Overall, our findings highlight the benefit of reinforcement learning for complex, controlled text generation tasks requiring nuanced predictions.","manuscriptTitle":"Improving Counterfactual Story Rewriting with Policy-Gradient Approaches","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2025-07-09 08:46:23","doi":"10.21203/rs.3.rs-6694750/v1","editorialEvents":[{"type":"communityComments","content":0},{"type":"editorInvitedReview","content":"","date":"2025-08-05T15:56:02+00:00","index":"hide","fulltext":""},{"type":"editorInvitedReview","content":"","date":"2025-08-04T22:32:07+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"257425059189452857415237636710075021601","date":"2025-07-09T13:48:15+00:00","index":"hide","fulltext":""},{"type":"reviewerAgreed","content":"241256344468676555551112425427938278933","date":"2025-07-07T10:03:07+00:00","index":"hide","fulltext":""},{"type":"reviewersInvited","content":"","date":"2025-07-04T10:35:37+00:00","index":"","fulltext":""},{"type":"editorAssigned","content":"","date":"2025-07-04T10:01:06+00:00","index":"","fulltext":""},{"type":"checksComplete","content":"","date":"2025-05-19T12:53:18+00:00","index":"","fulltext":""},{"type":"submitted","content":"Language Resources and Evaluation","date":"2025-05-19T03:47:30+00:00","index":"","fulltext":""}],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"language-resources-and-evaluation","isNatureJournal":false,"hasQc":true,"allowDirectSubmit":false,"externalIdentity":"lrev","sideBox":"Learn more about [Language Resources and Evaluation](http://link.springer.com/journal/10579)","snPcode":"10579","submissionUrl":"https://submission.nature.com/new-submission/10579/3","title":"Language Resources and Evaluation","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"em","reportingPortfolio":"Springer Hybrid","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"e7a9f5ff-f506-4caa-8a7a-3f31baa96335","owner":[],"postedDate":"July 9th, 2025","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[],"tags":[],"updatedAt":"2025-07-09T08:46:23+00:00","versionOfRecord":[],"versionCreatedAt":"2025-07-09 08:46:23","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-6694750","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-6694750","identity":"rs-6694750","version":["v1"]},"buildId":"8U1c8b4HqxoKbykW_rLl7","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}