Unjustified Measurement Decisions Halve Significant Findings Across 100+ Studies

Unjustified Measurement Decisions Halve Significant Findings Across 100+ Studies | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Social Sciences - Article Unjustified Measurement Decisions Halve Significant Findings Across 100+ Studies James Soland, Joshua Gilbert, William Young This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-9520521/v1 This work is licensed under a CC BY 4.0 License Status: Under Review Version 1 posted You are reading this latest preprint version Abstract Randomized controlled trials (RCTs) are the gold standard for causal inference, yet the validity of RCT conclusions depends not only on randomization but also on how outcomes are measured and scored. Analysing item-level data from 112 RCT outcome measures spanning psychology, medicine, public health, and education, we show that foundational measurement assumptions are rarely evaluated and frequently violated. For nearly half of outcomes, the single score used is not a plausible representation of the data. When outcomes are scored using statistical models aligned with study design rather than the near-ubiquitous sum score approach, the proportion of pre/post trials reporting statistically significant treatment effects approximately doubles. Routine measurement decisions systematically shift causal inferences---a largely unexamined source of analytic flexibility that likely contributes to replication failures across the social and behavioral sciences. Scientific community and society/Social sciences/Psychology/Human behaviour Scientific community and society/Social sciences/Education Health sciences/Medical research/Clinical trial design/Randomized controlled trials Full Text Additional Declarations There is NO Competing Interest. Cite Share Download PDF Status: Under Review Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-9520521","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Social Sciences - Article","associatedPublications":[],"authors":[{"id":634381024,"identity":"ab25498e-d01c-4907-9176-a48a08cf7912","order_by":0,"name":"James Soland","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA20lEQVRIiWNgGAWjYBACxgYGxgcMFQw8YGYCSOgAYS3MBgxnSNECBGwSjG3IfEJamPuXPzb4OO+wjHn/4rYHD2oY5PhuJBBw2Iw3hg9nbjvMI3PjYbtBwjEGY0nCWs4wG/NuS+ORkDjYJpHAxpC4gbCW48+k/86BafnHUE9YS3+DmTRjgw2PBH9jm0RiG0OCAWFbeIwNe44BtQDDTSKxT8Jw5pkH+LUY9h9/+OBHjYS9BP/xZ5I/vtnI8x0nYIvhDJgCCTBDAr9yEJDnPwBlwRmjYBSMglEwCtAAAMZ5SB6IycyHAAAAAElFTkSuQmCC","orcid":"https://orcid.org/0000-0001-8895-2871","institution":"University of Virginia","correspondingAuthor":true,"prefix":"","firstName":"James","middleName":"","lastName":"Soland","suffix":""},{"id":634381025,"identity":"d5b09497-b0e1-4065-ab15-f8a9cb7d1d31","order_by":1,"name":"Joshua Gilbert","email":"","orcid":"","institution":"Harvard Graduate School of Education","correspondingAuthor":false,"prefix":"","firstName":"Joshua","middleName":"","lastName":"Gilbert","suffix":""},{"id":634381026,"identity":"5ee757e6-e911-488d-b314-bf85629aea8b","order_by":2,"name":"William Young","email":"","orcid":"","institution":"Harvard Graduate School of Education","correspondingAuthor":false,"prefix":"","firstName":"William","middleName":"","lastName":"Young","suffix":""}],"badges":[],"createdAt":"2026-04-24 20:10:30","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-9520521/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-9520521/v1","draftVersion":[],"editorialEvents":[],"editorialNote":"","failedWorkflow":false,"files":[{"id":108804825,"identity":"8754129f-a7d7-4352-ab92-2346fbc50cf9","added_by":"auto","created_at":"2026-05-08 15:23:44","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":810561,"visible":true,"origin":"","legend":"Article File","description":"","filename":"RCTsumscore.pdf","url":"https://assets-eu.researchsquare.com/files/rs-9520521/v1_covered_d6063bd8-5c09-4544-9a50-1e56f4424f93.pdf"}],"financialInterests":"There is \u003cb\u003eNO\u003c/b\u003e Competing Interest.","formattedTitle":"Unjustified Measurement Decisions Halve Significant Findings Across 100+ Studies","fulltext":[],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":true,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":false,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":true,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"nature-portfolio","isNatureJournal":true,"hasQc":false,"allowDirectSubmit":false,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"","title":"Nature Portfolio","twitterHandle":"","acdcEnabled":false,"dfaEnabled":false,"editorialSystem":"ejp","reportingPortfolio":"","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"","lastPublishedDoi":"10.21203/rs.3.rs-9520521/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-9520521/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Randomized controlled trials (RCTs) are the gold standard for causal inference, yet the validity of RCT conclusions depends not only on randomization but also on how outcomes are measured and scored. Analysing item-level data from 112 RCT outcome measures spanning psychology, medicine, public health, and education, we show that foundational measurement assumptions are rarely evaluated and frequently violated. For nearly half of outcomes, the single score used is not a plausible representation of the data. When outcomes are scored using statistical models aligned with study design rather than the near-ubiquitous sum score approach, the proportion of pre/post trials reporting statistically significant treatment effects approximately doubles. Routine measurement decisions systematically shift causal inferences---a largely unexamined source of analytic flexibility that likely contributes to replication failures across the social and behavioral sciences.","manuscriptTitle":"Unjustified Measurement Decisions Halve Significant Findings Across 100+ Studies","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2026-05-06 02:22:04","doi":"10.21203/rs.3.rs-9520521/v1","editorialEvents":[],"status":"published","journal":{"display":false,"email":"[email protected]","identity":"nature","isNatureJournal":true,"hasQc":false,"allowDirectSubmit":false,"externalIdentity":"nature","sideBox":"Learn more about [Nature](http://www.nature.com/nature/)","snPcode":"","submissionUrl":"","title":"Nature","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"ejp","reportingPortfolio":"Nature","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"f5f025aa-ed37-4c3a-bc48-8de8493fd3c4","owner":[],"postedDate":"May 6th, 2026","published":true,"recentEditorialEvents":[{"type":"reviewerAgreed","content":"This content is not available.","date":"2026-05-04T23:43:35+00:00","index":2,"fulltext":"This content is not available."},{"type":"reviewerAgreed","content":"This content is not available.","date":"2026-05-04T23:38:24+00:00","index":1,"fulltext":"This content is not available."},{"type":"reviewersInvited","content":"3","date":"2026-05-04T20:42:36+00:00","index":"","fulltext":""}],"rejectedJournal":[],"revision":"","amendment":"","status":"under-review","subjectAreas":[{"id":67513277,"name":"Scientific community and society/Social sciences/Psychology/Human behaviour"},{"id":67513278,"name":"Scientific community and society/Social sciences/Education"},{"id":67513279,"name":"Health sciences/Medical research/Clinical trial design/Randomized controlled trials"}],"tags":[],"updatedAt":"2026-05-06T02:22:04+00:00","versionOfRecord":[],"versionCreatedAt":"2026-05-06 02:22:04","video":"","vorDoi":"","vorDoiUrl":"","workflowStages":[]},"version":"v1","identity":"rs-9520521","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-9520521","identity":"rs-9520521","version":["v1"]},"buildId":"XKTyCvWXoU3ODBz1xrDgd","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}