Comparison of Bias and Resolvability in Single-Cell and Single-Transcript Methods

Comparison of Bias and Resolvability in Single-Cell and Single-Transcript Methods | Research Square window.SnipcartSettings = { analytics: { enabled: false } }; (function() { var accessVector = localStorage.getItem('access_vector') || ''; window.dataLayer = window.dataLayer || []; if (accessVector) { window.dataLayer.push({ user: { profile: { profileInfo: { snid: accessVector } } } }); } })(); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-K279D39R'); Browse Preprints In Review Journals COVID-19 Preprints AJE Video Bytes Research Tools Research Promotion AJE Professional Editing AJE Rubriq About Preprint Platform In Review Editorial Policies Our Team Advisory Board Help Center Sign In Submit a Preprint Cite Share Download PDF Article Comparison of Bias and Resolvability in Single-Cell and Single-Transcript Methods Jayan Rammohan, Steven Lund, Nina Alperovich, Vanya Paralanov, and 2 more This is a preprint; it has not been peer reviewed by a journal. https://doi.org/ 10.21203/rs.3.rs-84848/v1 This work is licensed under a CC BY 4.0 License Status: Published Journal Publication published 02 Jun, 2021 Read the published version in Communications Biology → Version 1 posted You are reading this latest preprint version Abstract Single-cell and single-transcript measurement methods have elevated our ability to understand and engineer biological systems. However, defining and comparing performance between methods remains a challenge, in part due to the confounding effects of experimental variability. Here, we propose a generalizable framework for performing multiple methods in parallel using split samples, so that experimental variability is shared between methods. We demonstrate the utility of this framework by performing 12 different methods in parallel to measure the same underlying reference system for cellular response. We compare method performance using quantitative evaluations of bias and resolvability. We attribute differences in method performance to steps along the measurement process such as sample preparation, signal detection, and choice of measurand. Finally, we demonstrate how this framework can be used to benchmark a new method for single-transcript detection. The framework we present here provides a practical way to compare performance of any methods. General Cell Biology & Physiology Technical Communication Single-cell single-transcript RNA. Figures Figure 1 Figure 2 Figure 3 Figure 4 Figure 5 Figure 6 Full Text Additional Declarations There is NO Competing Interest. Supplementary Files BRASSSICBfinal.pdf Supplemental Information Cite Share Download PDF Status: Published Journal Publication published 02 Jun, 2021 Read the published version in Communications Biology → Version 1 posted You are reading this latest preprint version Research Square lets you share your work early, gain feedback from the community, and start making changes to your manuscript prior to peer review in a journal. As a division of Research Square Company, we’re committed to making research communication faster, fairer, and more useful. We do this by developing innovative software and high quality services for the global research community. Our growing team is made up of researchers and industry professionals working together to solve the most critical problems facing scientific publishing. Also discoverable on Platform About Our Team In Review Editorial Policies Advisory Board Help Center Resources Author Services Accessibility API Access RSS feed Manage Cookie Preferences © Research Square 2026 | ISSN 2693-5015 (online) Privacy Policy Terms of Service Do Not Sell My Personal Information {"props":{"pageProps":{"initialData":{"identity":"rs-84848","acceptedTermsAndConditions":true,"allowDirectSubmit":false,"archivedVersions":[],"articleType":"Article","associatedPublications":[],"authors":[{"id":3598186,"identity":"7dd4093a-f1e5-4cbf-8ad1-33c94aa3f323","order_by":0,"name":"Jayan Rammohan","email":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAAyAQMAAABI0h/eAAAABlBMVEX///8AAABVwtN+AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA/0lEQVRIiWNgGAWjYBACxgYEmw2IbcAkSVrSgCQz8TaCtBwGYgJamNvPGH4uYLgnrzvt8LEHP3ecz+PjP3+A8UvFYdwO68kxlp7BUGy47XZaumHvmdvFbBLJDMwyZ/BoacgxkOZhSGDcdjvHTIK37XZimwQzA7NkWxpuLf1vjH8DtdiDtEj+bTuX2MZ/mICWGTlmIFsSQVqkedsOJLYxJDMwfmyzwaPlWZk1j0FCMtAvadKybclAhyUbHGY4g1uLYX/y5ts8FQm2224nH5N822aXOL//4MOHPyokcGtp4DBgYDBAEz3Mg1MDA4M8A/sDLM79gUfLKBgFo2AUjDgAALtpUQ/ys50pAAAAAElFTkSuQmCC","orcid":"https://orcid.org/0000-0001-7800-6881","institution":"National Institute of Standards and Technology","correspondingAuthor":true,"submittingAuthor":false,"prefix":"","firstName":"Jayan","middleName":"","lastName":"Rammohan","suffix":""},{"id":3598187,"identity":"943adf52-7560-4770-99a7-e51462bf9535","order_by":1,"name":"Steven Lund","email":"","orcid":"","institution":"National Institute of Standards and Technology","correspondingAuthor":false,"submittingAuthor":false,"prefix":"","firstName":"Steven","middleName":"","lastName":"Lund","suffix":""},{"id":3598188,"identity":"18a0c9bb-7e5a-440a-88b2-6072ae88d550","order_by":2,"name":"Nina Alperovich","email":"","orcid":"","institution":"National Institute of Standards and Technology","correspondingAuthor":false,"submittingAuthor":false,"prefix":"","firstName":"Nina","middleName":"","lastName":"Alperovich","suffix":""},{"id":3598189,"identity":"8d17ba22-d278-4782-b3ff-d6e1c5000a2a","order_by":3,"name":"Vanya Paralanov","email":"","orcid":"","institution":"National Institute of Standards and Technology","correspondingAuthor":false,"submittingAuthor":false,"prefix":"","firstName":"Vanya","middleName":"","lastName":"Paralanov","suffix":""},{"id":3598190,"identity":"adc63c2f-4f10-4ff3-a6ad-f3db769e942c","order_by":4,"name":"Elizabeth Strychalski","email":"","orcid":"","institution":"National Institute of Standards and Technology","correspondingAuthor":false,"submittingAuthor":false,"prefix":"","firstName":"Elizabeth","middleName":"","lastName":"Strychalski","suffix":""},{"id":3598191,"identity":"8305239f-7e84-442e-b74b-ca114ad60b6f","order_by":5,"name":"David Ross","email":"","orcid":"https://orcid.org/0000-0002-7790-218X","institution":"National Institute of Standards and Technology","correspondingAuthor":false,"submittingAuthor":false,"prefix":"","firstName":"David","middleName":"","lastName":"Ross","suffix":""}],"badges":[],"createdAt":"2020-09-28 16:15:47","currentVersionCode":1,"declarations":"","doi":"10.21203/rs.3.rs-84848/v1","doiUrl":"https://doi.org/10.21203/rs.3.rs-84848/v1","draftVersion":[],"editorialEvents":[{"content":"https://doi.org/10.1038/s42003-021-02138-6","type":"published","date":"2021-06-02T20:49:06+00:00"}],"editorialNote":"","failedWorkflow":false,"files":[{"id":3082163,"identity":"652680fd-99a9-40db-a42e-b6bf89d0f039","added_by":"auto","created_at":"2020-10-20 13:26:53","extension":"jpg","order_by":1,"title":"Figure 1","display":"","copyAsset":false,"role":"figure","size":106109,"visible":true,"origin":"","legend":"Experiment design and evaluation of resolvability. (a) In this study, each cell culture is divided (split) to perform\nmultiple methods in parallel for measuring the same underlying system of cellular sense and response. Eight cultures were\nmeasured over a range of induction with IPTG. Methods include various combinations of sample preparation, signal\ndetection, and choice of measurand. Resolvability is quantitatively assessed using Area Under the receiver operator\ncharacteristic Curve (AUC) calculated across a range of IPTG concentrations. Relative bias is assessed by modeling cellular\nresponse using measurements from each method, and comparing the resulting parameters. (b) Evaluation of resolvability\nwith AUC profiles for all 12 methods. In each panel, the plotted symbols show the AUC for pairs of adjacent IPTG\nconcentrations as indicated on the x-axis.","description":"","filename":"Fig1.JPG","url":"https://assets-eu.researchsquare.com/files/rs-84848/v1/65c2a3de174ac964f1e34852.JPG"},{"id":3082165,"identity":"06a869eb-cc9a-4ff3-b5a3-dc9096c9689a","added_by":"auto","created_at":"2020-10-20 13:26:53","extension":"jpg","order_by":2,"title":"Figure 2","display":"","copyAsset":false,"role":"figure","size":110796,"visible":true,"origin":"","legend":"Evaluation of relative bias in Hill parameters. Hill parameters for fits to RPU-normalized data are shown for all 12\nmethods. Symbols represent parameter estimates for methods according to legend. Error bars indicate 95 % confidence\nintervals from nonlinear least-squares fits. For each parameter, methods are ordered on x-axis from left-to-right from lowest\nto highest average parameters value.","description":"","filename":"Fig2.JPG","url":"https://assets-eu.researchsquare.com/files/rs-84848/v1/76e947d81156a5c817d20dac.JPG"},{"id":3082166,"identity":"dd5fafaa-63c9-4413-b532-85b5e0a229c6","added_by":"auto","created_at":"2020-10-20 13:26:53","extension":"jpg","order_by":3,"title":"Figure 3","display":"","copyAsset":false,"role":"figure","size":54186,"visible":true,"origin":"","legend":"Method performance can be attributed to measurand. (a) Pairwise comparisons of methods that share the same\nsteps for sample preparation and signal detection, but differ in measurand (\"M” squares in matrix), are used to attribute\nmeasurement performance to measurand. The four boxes containing “M” under the matrix represent these same pairwise\ncomparisons. (b) Pairwise AUC plots of measurements in (a) are used to compare resolvability between RNA and protein\nmeasurands. Diagonal line indicates equivalent resolvability between the two methods. Pairs of adjacent IPTG concentrations\nare shown as numbers within the plots, as indicated in figure legend. Color indicates biological replicates one (orange), two\n(green), and three (purple), as indicated in figure legend. Large gray numbers in the top-left and bottom right-corners indicate\nhow many AUC’s were higher for the method plotted on the y-axis or x-axis, respectively. The p-values for a paired sign t-test\nare shown within each plot.","description":"","filename":"Fig3.JPG","url":"https://assets-eu.researchsquare.com/files/rs-84848/v1/ce5dc9e74c9f2b63cf8c211a.JPG"},{"id":3082167,"identity":"0e9f2d18-fddb-49a3-8f50-fc6dc64e2181","added_by":"auto","created_at":"2020-10-20 13:26:53","extension":"jpg","order_by":4,"title":"Figure 4","display":"","copyAsset":false,"role":"figure","size":55743,"visible":true,"origin":"","legend":"Method performance can be attributed to signal detection. (a) Pairwise comparisons of methods that share the\nsame steps for sample preparation and measurand, but differ in signal detection (indicated by “D” in the matrix), are used to\nattribute measurement performance to signal detection. The four boxes containing “D” under the matrix represent these\nsame pairwise comparisons. (b) Pairwise AUC plots of measurements in (a) to compare resolvability between flow cytometry\nand microscopy. Diagonal line indicates equivalent resolvability between the two methods. Pairs of adjacent IPTG\nconcentrations are shown as numbers within the plots, as indicated in figure legend. Color indicates biological replicates one\n(orange), two (green), and three (purple), as indicated in figure legend. Large gray numbers in the top-left and bottom rightcorners\nindicate how many AUC’s were higher for the method plotted on the y-axis or x-axis, respectively. The p-values for a\npaired sign t-test are shown within each plot.","description":"","filename":"Fig4.JPG","url":"https://assets-eu.researchsquare.com/files/rs-84848/v1/0012cdeb3ff9a963ffa236b7.JPG"},{"id":3082168,"identity":"87e94485-8552-4b6a-8dbe-77b6fc878d3d","added_by":"auto","created_at":"2020-10-20 13:26:53","extension":"jpg","order_by":5,"title":"Figure 5","display":"","copyAsset":false,"role":"figure","size":54299,"visible":true,"origin":"","legend":"Method performance can be attributed to sample preparation. (a) Pairwise comparisons of methods that share the\nsame steps for signal detection and measurand, but differ in sample preparation (indicated by “P” in the matrix), are used to\nattribute measurement performance to sample preparation. The 4 boxes containing “P” under the matrix represent these\nsame pairwise comparisons. (b) Pairwise AUC plots of measurements in (a) are used to compare resolvability between sample\npreparation methods (FISH versus HCR). Diagonal line indicates equivalent resolvability between the two methods. Pairs of\nadjacent IPTG concentrations are shown as numbers within the plots, as indicated in figure legend. Color indicates biological\nreplicates one (orange), two (green), and three (purple), as indicated in figure legend. Large gray numbers in the top-left and\nbottom right-corners indicate how many AUC’s were higher for the method plotted on the y-axis or x-axis, respectively. The\np-values for a paired sign t-test are shown within each plot.","description":"","filename":"Fig5.JPG","url":"https://assets-eu.researchsquare.com/files/rs-84848/v1/2113d479eb2cb2237ebb27c0.JPG"},{"id":3082169,"identity":"04e81cf0-a241-4b47-bee6-a26694392651","added_by":"auto","created_at":"2020-10-20 13:26:54","extension":"jpg","order_by":6,"title":"Figure 6","display":"","copyAsset":false,"role":"figure","size":95771,"visible":true,"origin":"","legend":"Performance of single-transcript methods can be attributed to RNA labeling strategy. (a) Performance of singletranscript\nmethods was compared using cells that shared the same steps of the measurement process except for the RNA\nlabeling step. (b) Resolvability of FISH and HCR was assessed by plotting AUC calculated from adjacent stimulus levels. Diagonal\nline indicates equivalent resolvability between the two methods. Pairs of adjacent IPTG concentrations are shown as numbers\nwithin the plots, as indicated in figure legend. Color indicates biological replicates one (orange), two (green), and three\n(purple), as indicated in figure legend. Large gray numbers in the top-left and bottom right-corners indicate how many AUC’s\nwere higher for the method plotted on the y-axis or x-axis, respectively. The p-values for a paired sign t-test are shown within\neach plot. (c) A two-state promoter model was used to evaluate transcription kinetics. (d) Negative binomials were used to fit\nsingle-transcript distributions for FISH (blue) and HCR (dark orange). (e) Estimates of burst frequency are plotted for FISH\nversus HCR. The RNA lifetime was assumed to be a constant (2.8 minutes). (f) Estimates of burst size, and (g) estimates of\nburst size after correcting for hybridization efficiency, are plotted for FISH versus HCR. For parts f and g, burst size is the\nnumber of transcripts per burst. For parts e, f, and g, the scatter plot numbers 2, 3, 4, 5, 6, 7, and 8 represent 5 μmol/L, 10\nμmol/L, 20 μmol/L, 40 μmol/L, 100 μmol/L, 400 μmol/L, and 1000 μmol/L IPTG, respectively. The 0 μmol/L IPTG case is not\nshown, in order to more easily see the trend for the remaining induction conditions. Color indicates biological replicate one\n(orange), biological replicate two (green), and biological replicate three (purple).","description":"","filename":"Fig6.JPG","url":"https://assets-eu.researchsquare.com/files/rs-84848/v1/664b87c04cebef27ec41beb3.JPG"},{"id":13544677,"identity":"7d0d68bf-44b1-481e-8b65-1146776b374b","added_by":"auto","created_at":"2021-09-17 02:03:11","extension":"pdf","order_by":0,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":4944840,"visible":true,"origin":"","legend":"","description":"","filename":"BRASSCBfinal.pdf","url":"https://assets-eu.researchsquare.com/files/rs-84848/v1_covered.pdf"},{"id":3082322,"identity":"236be56e-0e82-4b8c-8ee6-935a88dc7fe5","added_by":"auto","created_at":"2020-10-20 13:29:07","extension":"pdf","order_by":0,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":5922511,"visible":true,"origin":"","legend":"","description":"","filename":"BRASSCBfinal.pdf","url":"https://assets-eu.researchsquare.com/files/rs-84848/v1_stamped.pdf"},{"id":3082321,"identity":"d3b4ffce-242b-4351-8394-a014f56ec989","added_by":"auto","created_at":"2020-10-20 13:29:07","extension":"pdf","order_by":0,"title":"","display":"","copyAsset":false,"role":"manuscript-pdf","size":5922511,"visible":true,"origin":"","legend":"","description":"","filename":"BRASSCBfinal.pdf","url":"https://assets-eu.researchsquare.com/files/rs-84848/v1_stamped.pdf"},{"id":3082164,"identity":"7499f97a-33d4-41f4-bc1e-063c9d50c706","added_by":"auto","created_at":"2020-10-20 13:26:53","extension":"pdf","order_by":1,"title":"","display":"","copyAsset":false,"role":"supplement","size":15589150,"visible":true,"origin":"","legend":"Supplemental Information","description":"","filename":"BRASSSICBfinal.pdf","url":"https://assets-eu.researchsquare.com/files/rs-84848/v1/b52f24a7bca8af32de54791e.pdf"}],"financialInterests":"There is \u003cb\u003eNO\u003c/b\u003e Competing Interest.","formattedTitle":"\u003cp\u003eComparison of Bias and Resolvability in Single-Cell and Single-Transcript Methods\u003c/p\u003e","fulltext":[{"header":"Full Text","content":"\u003cp\u003eThis preprint is available for \u003ca href='/article/rs-84848/latest.pdf' target='_blank'\u003edownload as a PDF\u003c/a\u003e.\u003c/p\u003e"}],"fulltextSource":"","fullText":"","funders":[],"hasAdminPriorityOnWorkflow":false,"hasManuscriptDocX":false,"hasOptedInToPreprint":true,"hasPassedJournalQc":"","hasAnyPriority":false,"hideJournal":false,"highlight":"","institution":"","isAcceptedByJournal":true,"isAuthorSuppliedPdf":true,"isDeskRejected":"","isHiddenFromSearch":false,"isInQc":false,"isInWorkflow":false,"isPdf":false,"isPdfUpToDate":true,"isWithdrawnOrRetracted":false,"journal":{"display":true,"email":"[email protected]","identity":"nature-portfolio","isNatureJournal":true,"hasQc":false,"allowDirectSubmit":false,"externalIdentity":"","sideBox":"","snPcode":"","submissionUrl":"","title":"Nature Portfolio","twitterHandle":"","acdcEnabled":false,"dfaEnabled":false,"editorialSystem":"ejp","reportingPortfolio":"","inReviewEnabled":true,"inReviewRevisionsEnabled":false},"keywords":"Single-cell, single-transcript, RNA.","lastPublishedDoi":"10.21203/rs.3.rs-84848/v1","lastPublishedDoiUrl":"https://doi.org/10.21203/rs.3.rs-84848/v1","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"manuscriptAbstract":"Single-cell and single-transcript measurement methods have elevated our ability to understand and engineer biological systems. However, defining and comparing performance between methods remains a challenge, in part due to the confounding effects of experimental variability. Here, we propose a generalizable framework for performing multiple methods in parallel using split samples, so that experimental variability is shared between methods. We demonstrate the utility of this framework by performing 12 different methods in parallel to measure the same underlying reference system for cellular response. We compare method performance using quantitative evaluations of bias and resolvability. We attribute differences in method performance to steps along the measurement process such as sample preparation, signal detection, and choice of measurand. Finally, we demonstrate how this framework can be used to benchmark a new method for single-transcript detection. The framework we present here provides a practical way to compare performance of any methods.","manuscriptTitle":"Comparison of Bias and Resolvability in Single-Cell and Single-Transcript Methods","msid":"","msnumber":"","nonDraftVersions":[{"code":1,"date":"2020-10-20 13:26:44","doi":"10.21203/rs.3.rs-84848/v1","editorialEvents":[],"status":"published","journal":{"display":true,"email":"[email protected]","identity":"communications-biology","isNatureJournal":true,"hasQc":false,"allowDirectSubmit":false,"externalIdentity":"commsbio","sideBox":"Learn more about [Communications Biology](http://www.nature.com/commsbio/)","snPcode":"","submissionUrl":"","title":"Communications Biology","twitterHandle":"","acdcEnabled":true,"dfaEnabled":true,"editorialSystem":"ejp","reportingPortfolio":"Communications Series","inReviewEnabled":true,"inReviewRevisionsEnabled":false}}],"origin":"","ownerIdentity":"b0b338b0-1088-4485-b14a-bdc4770fd10c","owner":[],"postedDate":"October 20th, 2020","published":true,"recentEditorialEvents":[],"rejectedJournal":[],"revision":"","amendment":"","status":"published-in-journal","subjectAreas":[{"id":827642,"name":"General Cell Biology \u0026 Physiology"},{"id":827643,"name":"Technical Communication"}],"tags":[],"updatedAt":"2021-07-27T20:49:06+00:00","versionOfRecord":{"articleIdentity":"rs-84848","link":"https://doi.org/10.1038/s42003-021-02138-6","journal":{"identity":"communications-biology","isVorOnly":false,"title":"Communications Biology"},"publishedOn":"2021-06-02 20:49:06","publishedOnDateReadable":"June 2nd, 2021"},"versionCreatedAt":"2020-10-20 13:26:44","video":"","vorDoi":"10.1038/s42003-021-02138-6","vorDoiUrl":"https://doi.org/10.1038/s42003-021-02138-6","workflowStages":[]},"version":"v1","identity":"rs-84848","journalConfig":"researchsquare"},"__N_SSP":true},"page":"/article/[identity]/[[...version]]","query":{"redirect":"/article/rs-84848","identity":"rs-84848","version":["v1"]},"buildId":"ApUGefWb6u5IBVtyqm6d5","isFallback":false,"isExperimentalCompile":false,"dynamicIds":[84888],"gssp":true,"scriptLoader":[]}