Pulse oximeter performance and skin pigment: comparison of 34 oximeters using current and emerging regulatory frameworks

Pulse oximeter performance and skin pigment: comparison of 34 oximeters using current and emerging regulatory frameworks | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Pulse oximeter performance and skin pigment: comparison of 34 oximeters using current and emerging regulatory frameworks Caroline Hughes , Danni Chen , View ORCID Profile Tyler Law , View ORCID Profile Philip Bickler , View ORCID Profile John Feiner , View ORCID Profile Leonid Shmuylovich , Ella Behnke , Lily Ortiz , View ORCID Profile Gregory Leeb , View ORCID Profile Isabella Auchus , View ORCID Profile Fekir Negussie , View ORCID Profile Ronald Bisegerwa , René Vargas Zamora , View ORCID Profile Elizabeth Igaga , View ORCID Profile Kelvin Moore Jr. , Olubunmi Okunlola , View ORCID Profile Ellis Monk , Jana Lyn Fernandez , View ORCID Profile Odinakachukwu Ehie , View ORCID Profile Bernadette Wilks , Koyinsola Oyefeso , Deleree Schornack , View ORCID Profile Cornelius Sendagire , View ORCID Profile Michael S. Lipnick doi: https://doi.org/10.1101/2025.08.11.25332026 Caroline Hughes 1 University of California San Francisco Hypoxia Research Laboratory 2 Department of Anesthesia and Perioperative Care, University of California San Francisco 3 Saint Louis University School of Medicine Find this author on Google Scholar Find this author on PubMed Search for this author on this site Danni Chen 1 University of California San Francisco Hypoxia Research Laboratory 2 Department of Anesthesia and Perioperative Care, University of California San Francisco MS Find this author on Google Scholar Find this author on PubMed Search for this author on this site Tyler Law 1 University of California San Francisco Hypoxia Research Laboratory 2 Department of Anesthesia and Perioperative Care, University of California San Francisco 4 University of California San Francisco Center for Health Equity in Surgery and Anesthesia MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Tyler Law Philip Bickler 1 University of California San Francisco Hypoxia Research Laboratory 2 Department of Anesthesia and Perioperative Care, University of California San Francisco MD PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Philip Bickler John Feiner 1 University of California San Francisco Hypoxia Research Laboratory 2 Department of Anesthesia and Perioperative Care, University of California San Francisco MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for John Feiner Leonid Shmuylovich 5 Division of Dermatology, Department of Medicine, Washington University in Saint Louis MD PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Leonid Shmuylovich Ella Behnke 1 University of California San Francisco Hypoxia Research Laboratory 2 Department of Anesthesia and Perioperative Care, University of California San Francisco Find this author on Google Scholar Find this author on PubMed Search for this author on this site Lily Ortiz 1 University of California San Francisco Hypoxia Research Laboratory 2 Department of Anesthesia and Perioperative Care, University of California San Francisco 4 University of California San Francisco Center for Health Equity in Surgery and Anesthesia MS Find this author on Google Scholar Find this author on PubMed Search for this author on this site Gregory Leeb 1 University of California San Francisco Hypoxia Research Laboratory 2 Department of Anesthesia and Perioperative Care, University of California San Francisco 4 University of California San Francisco Center for Health Equity in Surgery and Anesthesia MBBS Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Gregory Leeb Isabella Auchus 2 Department of Anesthesia and Perioperative Care, University of California San Francisco MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Isabella Auchus Fekir Negussie 4 University of California San Francisco Center for Health Equity in Surgery and Anesthesia MPH Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Fekir Negussie Ronald Bisegerwa 4 University of California San Francisco Center for Health Equity in Surgery and Anesthesia 6 Department of Anesthesia, College of Health Sciences, Makerere University MBChB MMed Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Ronald Bisegerwa René Vargas Zamora 1 University of California San Francisco Hypoxia Research Laboratory 2 Department of Anesthesia and Perioperative Care, University of California San Francisco MS Find this author on Google Scholar Find this author on PubMed Search for this author on this site Elizabeth Igaga 4 University of California San Francisco Center for Health Equity in Surgery and Anesthesia 6 Department of Anesthesia, College of Health Sciences, Makerere University MBChB MMed Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Elizabeth Igaga Kelvin Moore Jr. 1 University of California San Francisco Hypoxia Research Laboratory 4 University of California San Francisco Center for Health Equity in Surgery and Anesthesia 7 Department of Surgery, Northwestern University Feinberg School of Medicine MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Kelvin Moore Jr. Olubunmi Okunlola 8 Department of Anesthesia, New York University Langone Hospital Brooklyn MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ellis Monk 9 Department of Sociology, Harvard University PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Ellis Monk Jana Lyn Fernandez 1 University of California San Francisco Hypoxia Research Laboratory 4 University of California San Francisco Center for Health Equity in Surgery and Anesthesia Find this author on Google Scholar Find this author on PubMed Search for this author on this site Odinakachukwu Ehie 2 Department of Anesthesia and Perioperative Care, University of California San Francisco 4 University of California San Francisco Center for Health Equity in Surgery and Anesthesia MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Odinakachukwu Ehie Bernadette Wilks 2 Department of Anesthesia and Perioperative Care, University of California San Francisco 4 University of California San Francisco Center for Health Equity in Surgery and Anesthesia MBBS Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Bernadette Wilks Koyinsola Oyefeso 1 University of California San Francisco Hypoxia Research Laboratory 4 University of California San Francisco Center for Health Equity in Surgery and Anesthesia Find this author on Google Scholar Find this author on PubMed Search for this author on this site Deleree Schornack 1 University of California San Francisco Hypoxia Research Laboratory 2 Department of Anesthesia and Perioperative Care, University of California San Francisco Find this author on Google Scholar Find this author on PubMed Search for this author on this site Cornelius Sendagire 4 University of California San Francisco Center for Health Equity in Surgery and Anesthesia 6 Department of Anesthesia, College of Health Sciences, Makerere University 10 Critical Care, D’Or Institute for Research and Education , Rio de Janeiro, Brazil MBChB MMed Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Cornelius Sendagire Michael S. Lipnick 1 University of California San Francisco Hypoxia Research Laboratory 2 Department of Anesthesia and Perioperative Care, University of California San Francisco 4 University of California San Francisco Center for Health Equity in Surgery and Anesthesia MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Michael S. Lipnick For correspondence: michael.lipnick{at}ucsf.edu Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Background The International Organization for Standardization (ISO) and US Food and Drug Administration (FDA) are updating regulations for pulse oximeters to reduce performance disparities linked to skin pigment. We tested common oximeters with current and anticipated regulatory frameworks. We hypothesized that not all oximeters show more positive bias in darkly vs lightly pigmented participants and that few oximeters would ‘pass’ the anticipated FDA regulations. Methods We used a controlled desaturation protocol to test 34 oximeters across arterial oxygen saturations (SaO 2 ) 70–100% in healthy adults. Based on what FDA and ISO had shared at the time of study design, we studied cohort sizes of ≥ 24 with ≥ 25% of participants being darkly pigmented. We used the subjective Monk Skin Tone (MST) scale and the objective individual typology angle (ITA) derived from a spectrophotometer to characterize skin pigment. The root mean square error (A RMS ), bias (mean of SpO 2 - SaO 2 error), and skin pigment differential bias were calculated. Monte Carlo simulation explored potential impacts of participant selection on device passing. Results For cohorts of 24 participants, 28/34 oximeters passed 2017 ISO Standard (A RMS ≤ 4%), 22/34 passed 2013 FDA guidance (A RMS ≤ 3%), 21/34 oximeters passed both A RMS and differential bias criteria for anticipated ISO standards, and 1/34 passed anticipated FDA criteria. More devices passed with cohorts > 24. Eleven oximeters had more positive bias in participants with dark vs. light (dorsal finger) pigmentation across 70–100% SaO 2 . Eighteen devices could pass or fail depending on cohorts selected for analysis. Conclusions Pulse oximeters show variable performance across manufacturers and models. Notably, only some devices show more positive bias in people with darker skin. Anticipated updates to ISO and FDA frameworks yield strikingly different assessments and require refinement of cohort sizes and differential bias criteria. Whether new guidelines will translate into improved real-world performance or reduced health disparities is yet to be determined. Introduction Pulse oximeters are essential clinical tools that noninvasively estimate the percent of hemoglobin bound with oxygen (SpO 2 ). However, long-standing concerns over variable performance across manufacturers and worse performance in people with darker skin have only recently received significant attention. Reports of oximeter inaccuracies in people with darker skin pigmentation began in the 1980s, and the COVID-19 pandemic brought this issue to the forefront as numerous studies reported not only positive bias (i.e. SpO 2 overestimating true arterial blood functional oxygen saturation, SaO 2 ) but also healthcare disparities (i.e. underrecognition or undertreatment of hypoxemia) in patients who self-identified as Black, Asian, Hispanic, or Native American. 1 – 4 Laboratory studies in healthy participants have shown similar findings for some oximeters, especially at lower SaO 2 . 5 – 8 In response to these concerns, the International Organization for Standardization (ISO) and US Food and Drug Administration (FDA) began updating pulse oximeter regulatory frameworks to improve performance and reduce potential disparities related to skin pigment. The FDA 510(k) premarket notification guidance for pulse oximeters and ISO 80601-2-61 standards were last updated in 2013 and 2017, respectively, and recommend manufacturers verify pulse oximeter performance using “10 or more healthy participants that vary in age and gender.” 9 , 10 The 2013 FDA guidance also recommended at least 15% of the participants (i.e., the proportion of the US population in 2013 who identified as African American) should have “darkly pigmented” skin. As previously reported, these recommendations are likely underpowered to ensure equitable device performance. 11 In 2024, the FDA released a discussion paper proposing larger verification study cohorts (24 participants) with more diversity. In 2025, FDA released a new draft guidance recommending even larger cohorts (150 participants), with more diversity of skin pigment, and additional statistical analyses. 12 The ISO has shared some details of anticipated changes via public forums. 13 Both agencies are expected to imminently release finalized versions of their new regulatory frameworks for pulse oximeters. 13 – 16 Considering these upcoming changes and ongoing uncertainty about the extent to which oximeters on the market have bias related to skin color, we sought to independently test the performance of 34 commonly used pulse oximeters with both current and anticipated regulatory frameworks. We hypothesized that relatively few devices would pass anticipated frameworks, and that most but not all pulse oximeters would show more positive bias in people with dark vs. light pigmentation. Methods This study was conducted at the University of California San Francisco (UCSF) Hypoxia Laboratory from 2022 to 2024 with UCSF IRB approval (#21-35637, ClinicalTrials.gov ID: NCT06142019 ). Written informed consent was obtained from all participants. Oximeter selection We tested 34 pulse oximeters. Devices were chosen based on popularity in online marketplaces, clinician input from diverse settings, and global health donor procurement lists to capture commonly used devices across varied clinical settings and price points. Purchase prices varied from $10 to $5,999. A list of tested devices, including prices, form factors, wavelengths, and regulatory data is available in Table S4 . For three devices (the Shenzhen PC-60NW, Masimo MightySat, and Acare AH-M1/MX), we purchased two of the same model and reported results by year of device purchase. For two additional devices (Masimo Rad97 and Masimo Rad G), we tested each with two different probes and reported each device-probe combination separately. All participants were monitored throughout the protocol with the lab’s ‘clinical monitor’ oximeter (Nellcor PM1000N, Medtronic, USA). Study demographics and skin pigment assessment Participants were healthy adults 18–50 years old who were non-smoking, with no history of lung, cardiovascular, kidney, or liver disease and without hemoglobinopathy, anemia, clotting disorders, or Raynaud’s disease. Our enrollment targeted diversity of both skin pigment and sex assigned at birth. Participants’ age, sex, and US National Institutes of Health (NIH) race were self-reported. Participants’ height, weight, and finger diameter were measured. Percent modulation of infrared light (an indicator of participant perfusion) was recorded from the clinical monitor and divided by 10 to approximate comparability with Masimo Perfusion Index. 8 Our approach to skin pigment assessment is provided in the Supplemental Digital Content eMethods. Briefly, two research coordinators assigned subjective skin pigmentation data for each participant using the Monk Skin Tone (MST) scale. 17 , 18 Coordinators also used the Konica Minolta CM 700-d (KM) spectrophotometer to derive individual typology angle (ITA), a frequently used surrogate for melanin 19 , 20 at the study participant’s dorsal distal phalanx (DP) (the site for study oximeters) and forehead (the recommended site for using MST). 19 We categorized participants into light, medium or dark bins based on previously proposed cutoffs: ‘light’ (MST 1–4 and ITA of > 30°), ‘medium’ (MST 5–7 and ITA between 30° and -30°), ‘dark’ (MST 8–10 and ITA of < -30° with ≥ 50% of these participants having ITA < -50°). 12 , 21 , 22 We refer to participants with both MST 8–10 and ITA of < -50° as ‘very dark.’ Unless otherwise noted, we used the ‘dark’ definition above (not ‘very dark’) for analyses. In cases of discordance between ITA and MST, the participant was binned by ITA. Controlled desaturation protocol Each participant underwent a controlled desaturation protocol as previously described and available online and in the Supplemental Digital Content eMethods. 23 – 25 Briefly, study investigators controlled partial pressures of inspired oxygen, carbon dioxide, and nitrogen to achieve six stable “plateaus” of targeted arterial functional oxygen saturation (SaO 2 ) between SaO2 ∼70% and 100% (i.e. 2 plateaus in each decile 70-80%, 80-90%, and 90-100%). At each plateau, multiple arterial blood samples were collected ≥ 20 seconds apart from a radial artery catheter. At the time of each blood sample, pulse oximeter SpO 2 ’s were recorded, and blood samples were immediately analyzed for SaO 2 using Radiometer ABL90 Flex Plus (ABL) (Radiometer, Copenhagen, Denmark) blood gas analyzers. Of note, participants’ hands were not warmed by default, and most probes were randomly placed on all five finger digits (see the Supplemental Digital Content eMethods). Sample size and cohort composition The cohort size and composition for the primary analysis were based on what had been shared publicly by the FDA and ISO at the time of study design. 13 – 16 , 26 These criteria were: 1. ≥ 24 unique participants; 2. Each participant must contribute 16 to 30 data points; 3. Pooled SaO 2 data must span at least 73% to 97% SaO 2 ; 4. ≥ 90% of participants must provide ≥ 1 data point < 85%; 5. ≥ 69% of participants must provide ≥ 1 data point in the 70–80% decile; 6. The cohort must include ≥ 33% of each sex; 7. Using forehead skin data, ≥ 25% of participants must fall into each color bin (i.e. light, medium, or dark). Given uncertainties in anticipated recommendations for cohort size, we tested all devices in at least 24 participants and continued testing with as many participants as possible based on available resources. Statistical and sensitivity analyses The primary analysis used 24 participant cohorts and skin pigment data from the forehead and dorsal distal phalanx (DP). We used two metrics to define a ‘passing device’ based on what was shared publicly by FDA and ISO at the time of study design ( Table 2 ): 1. accuracy root mean square error (A RMS ); and 2. differential bias. A RMS is a performance threshold that measures the square root of the mean of the squared differences in SpO 2 minus SaO 2 . Differential bias is defined in two ways: 1. For ITA, as the difference in SpO 2 bias between two theoretical participants with an ITA difference of 100° (i.e., one participant with dark ITA -50° and another with light ITA 50°); and 2. For MST, as the difference between MST bins. We estimated the differential bias and 95% confidence interval (CI) using a linear mixed effects (LME) model. The anticipated ISO standard recommended differential bias thresholds (point estimate ≤ 4% for 70–85% SaO 2 and ≤ 2% for 85–100% SaO 2 ) for only ITA, while the anticipated FDA guidance recommended thresholds (95% CI < 3.5% for 70–85% SaO 2 and < 1.5% for 85–100% SaO 2 ) for both ITA and pairwise comparisons of MST bins 1–4, 5–7, and 8–10. 12 , 15 , 16 View this table: View inline View popup Table 1. Baseline characteristics of participants View this table: View inline View popup Download powerpoint Table 2 Summary of current and anticipated regulatory frameworks for pulse oximetry We assessed for bias related to pigment by calculating A RMS and error (SpO 2 minus SaO 2 ) for each pigment bin (forehead and DP). 12 Error and A RMS were summarized as means with 95% CIs. The 95% CIs were determined using bootstrapping (random resampling with replacement) with 1,000 repetitions. Error across skin color bins was compared using an LME model, which included a random intercept to account for repeated measures within participants. A RMS was computed for each participant and compared between groups of participants based on pigment binning using a Welch’s two-sample t-test. A two-sided p-value < 0.05 was considered significant. P-values are reported unadjusted for multiple comparisons in accordance with FDA guidance for independent pairwise analyses, which does not mandate multiplicity adjustments. We conducted a secondary analysis for each device using the largest cohorts possible that still met cohort diversity criteria. Additional secondary analyses included the impact on device ‘passing’ of different anatomical pigment assessment sites, different definitions of pigment binning, and different statistical methods for estimating bias, including use of a linear regression (LR) model for differential bias instead of the LME model. The LR model regressed the participant-level bias on the ITA values and estimated the difference in predicted bias between ITA values of -50° and 50° based on the fitted model. A Monte Carlo simulation explored if participant selection bias could influence ‘passing’ status. For each device, Monte Carlo simulation created 100 cohorts of 24 participants, with at least six darkly pigmented participants. If a device always passed or always failed in all simulated cohorts, it was determined to be less susceptible to selection bias (i.e. ‘cherrypicking’ participants). The SpO 2 from most study oximeters was recorded manually into a Research Electronic Data Capture (REDCap) Database at the time of each arterial blood sample, except for the Nellcor PM1000N and Masimo Rad97 which streamed data directly into Labview (National Instruments, Austin, TX). Most oximeters used a self-contained interface that displayed SpO 2 data on the oximeter itself and could not directly stream data from the oximeter. All other physiologic data were collected into Labview, and all Labview data were merged into the REDCap Database. All analyses were performed with R 4.3.2 (R Core Team, 2023) and Python v3.9.6 (Python Software Foundation, 2024). De-identified data for this study are openly accessible through the Open Oximetry Data Repository via PhysioNet and accessible via the OpenOximetry.org website. 27 Results We completed 348 desaturation studies in 155 participants (30,174 paired SpO 2 :SaO 2 samples). Each device was tested in ≥ 24 unique participants (median 38, range 24–120) with > 540 SpO 2 :SaO 2 pairs approximately equally distributed across SaO 2 deciles, and with > 25% of data points in each decile coming from participants in each skin color bin ( Table S3 ). Demographics are reported in Table 1 . Overall oximeter performance For cohorts of 24 participants, device A RMS ranged from 1.69 to 8.04 ( Figure 1 ). Of the 12 devices with A RMS > 3% (i.e. 2013 FDA Guidance threshold), eight had 510(k) premarket clearances. All six devices with an A RMS > 4% (i.e. 2017 ISO Standard threshold) reported a European Conformity (CE) marking ( Figure 1 and Table S4 ). Download figure Open in new tab Figure 1. Device conformity with current and anticipated FDA and ISO performance thresholds for 24 participant cohorts and DP ITA This figure displays devices from lowest to highest accuracy root mean square error (A RMS ) with 95% confidence intervals (CI) for each device tested in a cohort of 24 participants. The boxes marked with colors indicate that the device meets the A RMS or differential bias thresholds (differential bias and 95% CI < 3.5% for 70–85% SaO 2 and < 1.5% for 85–100% SaO 2 for 2025 FDA, and differential bias point estimate ≤ 4% for 70–85% SaO 2 and ≤ 2% for 85–100% SaO 2 for 2025 ISO). Differential bias was calculated with the Monk Skin Tone (MST) Scale and individual typology angle (ITA) as indicated. The 95% CI were determined using bootstrapping (random resampling with replacement) with 1,000 repetitions. Device SpO 2 bias ranged from -5.71 to 3.48 with a median of 0.97 (IQR: -0.04 – 1.67) ( Table S2 ). We found 24 devices with overall positive bias, and 10 devices with overall negative bias. A RMS by skin color bin In cohorts of 24, we analyzed A RMS by DP pigmentation across 70–100% SaO 2 and found four devices with significantly different A RMS between light and dark skin color bins ( Table S1 and Figure 2 ). We found significantly higher A RMS in the dark (DP) skin color bin for two devices at 70–80% SaO 2 , four devices at 80–90% SaO 2 , and two devices at 90–100% SaO 2 . One device showed significantly higher A RMS in the light skin color bin ( Table S1 ). Download figure Open in new tab Figure 2. Distribution of bias and A RMS across ITA bins at the DP site over 70-100% saturation range in 24 participant cohorts This figure shows the distribution of pulse oximeter measured oxygen saturation (SpO₂) bias (mean of SpO₂ minus SaO₂ error, where SaO₂ is functional arterial oxygen saturation) and root mean square error (A RMS ) in each skin color bin defined by Individual Typology Angle (ITA) measured at the dorsal distal phalanx (DP) site across the saturation range of 70-100%. Devices shown are selected with statistically significantly differences in either bias or A RMS between light (ITA >30°) and dark (ITA <-30°) bins. Panel A displays the distribution of bias in closed circles with 95% confidence interval (CI) whiskers. Panel B shows the distribution of A RMS in closed circles with 95% CI whiskers. Data are stratified into four ITA bins (top to bottom): ITA > 30°, -30° ≤ ITA ≤ 30°, ITA < -30°, and ITA < -50°; note that ITA < -50° is a subset of ITA < -30°. Sample size (n) for each bin is indicated at left in the corresponding color. The vertical dashed line in Panel A represents bias of 0 and in Panel B represents the A RMS 3% threshold. When analyzing A RMS by forehead pigmentation in cohorts of 24, two devices showed significantly higher A RMS in the dark vs. light color bin at 70–100% SaO 2 ( Table S6 ), with similar but not exact results to the DP site when broken down by SaO 2 deciles. With cohorts larger than 24, more oximeters demonstrated statistically significant A RMS differences between light and dark bins ( Table S1 , Table S6 , Table S8 , Table S9 ). Two devices demonstrated statistically significant differences in A RMS only when comparing light and ‘very dark’ color bins ( Table S9 ). Bias by skin color bin We found five devices with more positive bias in dark vs. light DP skin color bins across SaO 2 70–100% (median difference in bias (dark minus light) 1.56, IQR: 1.30 – 2.70), and two devices with more positive bias in the light bin ( Table S2 and Figure 2 ). When analyzing by SaO 2 deciles, we found significantly more positive bias in dark vs light skin color bins for four devices at 70–80% SaO 2 (median difference 4.22, IQR: 3.40 – 4.74), four devices at 80–90% SaO 2 (median difference 2.11 IQR: 1.60 – 2.85) and two devices at 90–100% SaO 2 (median difference 1.36, range: 1.20 – 1.51). Bias by forehead pigmentation also varied ( Table S7 ). Analysis of larger cohort sizes revealed more devices (11/34) with significant differences in bias when comparing light vs. dark bins at the DP across SaO 2 70–100%. Four devices demonstrated significantly more positive bias when comparing ‘very dark’ vs light color bins, but not dark vs light ( Table S10 and Figure 3 ). Download figure Open in new tab Figure 3. Distribution of bias and A RMS across ITA bins at the DP site over 70-100% saturation range in maximum participant cohorts This figure shows the distribution of pulse oximeter measured oxygen saturation (SpO₂) bias (mean of SpO₂ minus SaO₂ error, where SaO₂ is functional arterial oxygen saturation) and root mean square error (A RMS ) in each skin color bin defined by Individual Typology Angle (ITA) measured at the dorsal distal phalanx (DP) site across the saturation range of 70-100%. Devices shown are selected with statistically significantly differences in either bias or A RMS between light (ITA >30°) and dark (ITA <-30°) bins. Panel A displays the distribution of bias in closed circles with 95% confidence interval (CI) whiskers. Panel B shows the distribution of A RMS in closed circles with 95% CI whiskers. Data are stratified into four ITA bins (top to bottom): ITA > 30°, -30° ≤ ITA ≤ 30°, ITA < -30°, and ITA < -50°; note that ITA < -50° is a subset of ITA < -30°. Sample size (n) for each bin is indicated at left in the corresponding color. The vertical dashed line in Panel A represents bias of 0 and in Panel B represents the A RMS 3% threshold. Figures 4 -S13 contain the distribution of A RMS and bias for all devices across all saturation ranges and both anatomical sites. Download figure Open in new tab Figure 4. Modified bland altman plots for four selected devices in cohort of 24 participants This figure shows the modified bland altman (BA) plots for four devices, showing pulse oximeter measured oxygen saturation (SpO₂) bias (mean of SpO₂ minus SaO₂ error, where SaO₂ is functional arterial oxygen saturation) and 95% limits of agreement (LOA). Data points are colored by Monk Skin Tone Scale (MST) at the forehead, as recommended by the U.S. Food and Drug Administration (FDA) draft 510(k) guidance. Panel A: Shenzhen Med-Link Electronics Tech Co., Ltd. AM801 (passing A RMS /failing Individual Typology Angle (ITA) differential bias); Panel B: Nonin Medical, Inc. Co-Pilot H500 (passing A RMS /passing ITA differential bias); Panel C: Bistos Co., Ltd. BT-710 (failing A RMS /passing ITA differential bias); Panel D: Guangdong Biolight Meditech Co., Ltd. M70 (failing A RMS /failing ITA differential bias). In each panel, the red solid horizontal line represents the SpO₂ bias; dashed lines are the 95% LOA, both have accounted for repeated measures. Differential bias In cohorts of 24 using DP ITA, differential bias ranged from -1.39 to 9.17 at 70– 85% SaO 2 (median 1.50, IQR: 0.00 – 2.53) and -1.12 to 2.18 at 85–100% SaO 2 (median 0.42, IQR: -0.37 – 0.94) ( Table S12 ). With the larger cohort sizes, DP ITA differential bias ranged from -1.34 to 4.83 at 70–85% SaO 2 (median 1.36, IQR: 0.01 – 2.33) and from -0.92 to 1.72 at 85–100% SaO 2 (median of 0.38, IQR: -0.10 – 0.96) ( Table S13 ). Forehead ITA differential bias was similar. The LME and LR models produced similar results. Old vs. anticipated regulatory frameworks With cohorts of 24 participants, 28/34 devices passed 2017 ISO A RMS criteria (≤4%), 22/34 passed 2013 FDA A RMS (≤ 3%) criteria (which is also the anticipated ISO criteria), and 12/34 passed anticipated FDA A RMS criteria (95% UCI < 3%). For differential bias, 1/34 oximeters passed the anticipated FDA criteria, and 32/34 passed the anticipated ISO criteria ( Table S12 and Figure 1 ). Overall, 21/34 oximeters passed both the A RMS and differential bias criteria for anticipated ISO standard, and 1/34 oximeters passed both criteria for anticipated FDA guidance. With the larger cohort sizes, the number of devices passing both the anticipated A RMS and differential bias criteria increased from 1/34 to 7/34 for FDA, and from 21/34 to 25/34 for ISO ( Table S3 and Table S13 ). Impact of pigmentation definitions and measurement sites on bias The use of race, MST or ITA with varied binning thresholds and different anatomical sites yielded different numbers of devices with significant differences in bias ( Table S14 ). Analysis of light vs dark DP ITA in maximum cohort sizes identified the most devices with significant differences in bias (11), while analysis by race (white vs. Black/African American) identified the fewest. Monte Carlo Simulation We found numerous devices could ‘pass’ or ‘fail’ A RMS criteria depending on which participants were selected for cohort analysis ( Figure S3 ). Discussion We found that many pulse oximeters fail to meet current or anticipated regulatory performance recommendations, and some but not all oximeters had more positive bias in dark vs light participants. Anticipated ISO and FDA regulatory frameworks produced markedly different ‘passing’ rates for the same devices. Most devices passed the anticipated ISO standard, yet only one device passed the anticipated FDA guidance (in cohorts of 24 healthy adults). Our findings corroborate prior studies demonstrating variable performance and positive bias in some oximeters on the market. 6 , 7 , 23 , 24 , 28 , 29 In several instances, our findings differ from manufacturers’ reports ( Figure 1 and Table S4 ), which may be partly explained by methodological differences. For example, we did not warm most participants’ hands and used all five digits. This was done in an effort to better reflect clinical reality and contrasts with most prior oximeter regulatory verification studies, which typically only used digits 2-4 and actively warmed participants’ hands to increase peripheral perfusion (a factor known to improve device performance and permitted by FDA and ISO). 8 Additionally, we analyzed consecutively enrolled participants. Our Monte Carlo analysis suggests that had we ‘cherry-picked’ participants for analysis, a practice not explicitly prohibited for regulatory submissions, we could make poorly performing oximeters look better. Consistent with our hypothesis, we found most (18/34) but not all oximeters exhibited more positive bias (i.e. SpO 2 overestimating SaO 2 ) in participants with dark vs. light skin ( Table S10 and Table S11 ). Unexpectedly, several devices demonstrated more positive bias in participants with light vs. dark skin. We confirmed our hypothesis that fewer devices pass anticipated regulatory frameworks, but the contrast in differential bias criteria between FDA and ISO is particularly noteworthy. Nearly all devices we tested passed the anticipated differential bias criteria for ISO, but only one passed anticipated FDA criteria ( Figure 1 ). The anticipated ISO standard performed similarly to the 2013 FDA guidance. This is concerning given that devices cleared by 2013 FDA guidance were used in studies demonstrating oximeter performance and health disparity concerns 4 , 30 ISO should tighten its recommendations and ensure differential bias thresholds reflect magnitudes of bias (e.g. <2%) seen in clinically relevant SpO2 ranges (e.g. 85-95%). 31 – 37 The FDA differential bias thresholds necessitate larger study cohorts for some devices to pass. This has generated concern that if studies become too costly or time-consuming, there could be negative global market implications (e.g. decreased device access and increased costs). 13 , 38 Increasing the size of verification cohorts (from 10 to 24) and utilizing MST and ITA are good steps toward improving diversity. Our findings suggest that most devices will require cohorts > 24 to pass anticipated FDA criteria, but a cohort size of 150, as proposed in the January 2025 draft FDA guidance, is likely unnecessary. Additionally, we found the anatomical site for skin pigment assessment and definition of “dark” skin pigment also impacted bias, A RMS , and regulatory ‘passing’ rates ( Table S1 and Table S15 ). Without a better understanding of how performance criteria in the lab correlate with performance (or health disparities) in the clinical setting, the optimal regulatory performance thresholds will remain uncertain. There was no device characteristic (e.g. cost, number of wavelengths, form factor etc.) that was clearly linked to performance, though the study was not designed for this purpose. This study had several limitations. First, we generally tested only one oximeter per model. For devices with detachable probes, we replaced probes after approximately 24 participants. Thus, we cannot assess variability related to manufacturing differences or wear and tear. For the three oximeter models, we found different performance across manufactured years despite no obvious differences in hardware or software ( Table S16 , Figure 1 , and Table S5 ). Our clinical monitor (PM1000N) had reasonably consistent performance in these same participant cohorts, suggesting this was not a methodological problem. Further work is needed to analyze variability across different probes of the same model, a potentially important source of error 13 Second, devices were tested in comparable but not identical cohorts. For example, some cohorts (including those for two devices with the best A RMS ) had higher participant perfusion ( Table S17 ). Third, several aspects of our desaturation protocol could have influenced findings. For example, not all oximeters are designed for use on the 1st and 5th digits. Plateau stability was partially defined by stability of the clinical monitor, which may not always align with stability of tested oximeters. To mitigate this, we also used stability of calculated saturation (ScO 2 ) 23 based on exhaled CO 2 and O 2 partial pressures, to define plateau stability. Another protocol-related limitation is that SaO 2 can vary by 1-3% across co-oximeter (i.e. blood gas analyzer) brands. 39 We could not account for this factor because manufacturers generally do not disclose which co-oximeter was used in testing. Finally, there are also limitations in our skin pigment assessment protocol as previously reported. 40 Although individuals with ITA < -50° represent a critical group for ensuring devices work equitably across all skin tones, we were unable to enroll a sufficient number of participants in this category to meaningfully assess its impact on bias analyses. Generalizability of our findings to the clinical setting, pediatric patients or devices we did not test should be done with caution. Conclusion Clinicians should be aware that many oximeters do not meet current or anticipated regulatory performance recommendations, and some, but not all, devices show more positive bias in people with dark vs light skin pigment. While it may be uncertain to regulators how much differential bias is reasonable to allow, it is clear that some oximeters appear to have minimal or no significant bias related to pigment, and all manufacturers should aspire to this performance goal regardless of the thresholds regulators ultimately propose. Future studies are needed to optimize regulatory frameworks, though many improvements can be made immediately using available data. Despite performance limitations of regulatory-compliant oximeters, clinicians should continue to utilize these essential tools but use great caution when using absolute SpO 2 cutoffs for providing or withholding treatments. Data Availability All data produced are available online at https://openoximetry.org/data-repository/ https://openoximetry.org/data-repository/ Funding Statement This study was conducted as part of the Open Oximetry Project funded by the Gordon and Betty Moore Foundation, Patrick J McGovern Foundation, PATH/UNITAID, and Robert Wood Johnson Foundation. Dr Ellis Monk’s time utilized for data analysis, reviewing and editing was funded by grant number: DP2MH132941. Conflicts of Interest The UCSF Hypoxia Research Laboratory receives funding from multiple industry sponsors to test the sponsors’ devices for the purposes of product development and regulatory performance testing. This paper does not include data collected for sponsors. All data were collected from devices procured by the Hypoxia Research Laboratory for the purposes of independent research. No company provided any direct funding for this study, participated in study design, or was involved in analyzing data or writing the manuscript. None of the authors own stock or equity interests in any pulse oximeter companies. Figures Download figure Open in new tab Figure S1. Modified bland altman plots for all 34 devices in cohorts of 24 participants This figure shows the modified bland altman (BA) plots for all 34 devices, showing pulse oximeter measured oxygen saturation (SpO₂) bias (mean of SpO₂ minus SaO₂ error, where SaO₂ is functional arterial oxygen saturation) and 95% limits of agreement (LOA). Data points are colored by Monk Skin Tone Scale (MST) at the forehead, as recommended by the U.S. Food and Drug Administration (FDA) draft 510(k) guidance. In each panel, the red solid horizontal line represents the SpO₂ bias; dashed lines are the 95% LOA, both have accounted for repeated measures. Download figure Open in new tab Figure S2. Modified bland altman plots for all 34 devices in maximum participants cohorts This figure shows the modified bland altman (BA) plots for all 34 devices, showing pulse oximeter measured oxygen saturation (SpO₂) bias (mean of SpO₂ minus SaO₂ error, where SaO₂ is functional arterial oxygen saturation) and 95% limits of agreement (LOA). Data points are colored by Monk Skin Tone Scale (MST) at the forehead, as recommended by the U.S. Food and Drug Administration (FDA) draft 510(k) guidance. In each panel, the red solid horizontal line represents the SpO₂ bias; dashed lines are the 95% LOA, both have accounted for repeated measures. Download figure Open in new tab Figure S3. Monte Carlo simulation of A RMS performance in cohorts of 24 participants For each of 100 simulations, a cohort of 24 unique participants was randomly sampled without replacement and accuracy root mean square error (A RMS ) was calculated; bootstrapping (random resampling with replacement) with 1,000 repetitions was then used to estimate the upper bound of the 95% confidence interval (CI) of A RMS in that simulated cohort. The histogram displays, for each device, the number of simulated cohorts whose upper 95% CI for A RMS fell below the 3% cutoff (red dashed line). Nine devices achieved an upper 95% CI for A RMS below 3% in all 100 simulations, whereas eighteen devices met this criterion in some simulations but not others. Jaccard diversity scores across devices—quantifying the overlap of cohorts across the 100 simulations— had mean values ranging from 0.03 to 0.67 (median 0.35, interquartile range 0.31– 0.45), indicating adequate cohort diversity across simulations. All devices had at least thirty unique sessions used in the simulations (median 47.5 sessions). Download figure Open in new tab Figure S4. Distribution of bias and A RMS across ITA bins at the forehead over 70–100% saturation range in 24 participants cohorts This figure shows the distribution of pulse oximeter measured oxygen saturation (SpO₂) bias (mean of SpO₂ minus SaO₂ error, where SaO₂ is functional arterial oxygen saturation) and accuracy root mean square error (A RMS ) in each skin color bin defined by Individual Typology Angle (ITA) measured at the forehead across the saturation range of 70–100%. Panel A displays the distribution of bias in closed circles with 95% confidence interval (CI) whiskers. Panel B shows the distribution of A RMS in closed circles with 95% CI whiskers. Data are stratified into four ITA bins (top to bottom): ITA > 30°, -30° ≤ ITA ≤ 30°, ITA < -30°, and ITA < -50°; note that ITA < -50° is a subset of ITA < -30°. Sample size (n) for each bin is indicated at left in the corresponding color. The vertical dashed line in Panel A represents bias of 0 and in Panel B represents the A RMS 3% threshold. Download figure Open in new tab Figure S5. Distribution of bias and A RMS across ITA bins at the forehead over 70–80% saturation range in 24 participants cohorts This figure shows the distribution of pulse oximeter measured oxygen saturation (SpO₂) bias (mean of SpO₂ minus SaO₂ error, where SaO₂ is functional arterial oxygen saturation) and accuracy root mean square error (A RMS ) in each skin color group defined by Individual Typology Angle (ITA) measured at the forehead across the saturation range of 70–80%. Panel A displays the distribution of bias in closed circles with 95% confidence interval (CI) whiskers. Panel B shows the distribution of A RMS in closed circles with 95% CI whiskers. Data are stratified into four ITA bins (top to bottom): ITA > 30°, -30° ≤ ITA ≤ 30°, ITA < -30°, and ITA < -50°; note that ITA < -50° is a subset of ITA < -30°. Sample size (n) for each bin is indicated at left in the corresponding color. The vertical dashed line in Panel A represents bias of 0 and in Panel B represents the A RMS 3% threshold. Download figure Open in new tab Figure S6. Distribution of bias and A RMS across ITA bins at the forehead over 80–90% saturation range in 24 participants cohorts This figure shows the distribution of pulse oximeter measured oxygen saturation (SpO₂) bias (mean of SpO₂ minus SaO₂ error, where SaO₂ is functional arterial oxygen saturation) and accuracy root mean square error (A RMS ) in each skin color bin defined by Individual Typology Angle (ITA) measured at the forehead across the saturation range of 80–90%. Panel A displays the distribution of bias in closed circles with 95% confidence interval (CI) whiskers. Panel B shows the distribution of A RMS in closed circles with 95% CI whiskers. Data are stratified into four ITA bins (top to bottom): ITA > 30°, -30° ≤ ITA ≤ 30°, ITA < -30°, and ITA < -50°; note that ITA < -50° is a subset of ITA < -30°. Sample size (n) for each bin is indicated at left in the corresponding color. The vertical dashed line in Panel A represents bias of 0 and in Panel B represents the A RMS 3% threshold. Download figure Open in new tab Figure S7. Distribution of bias and A RMS across ITA bins at the forehead over 90–100% saturation range in 24 participants cohorts This figure shows the distribution of pulse oximeter measured oxygen saturation (SpO₂) bias (mean of SpO₂ minus SaO₂ error, where SaO₂ is functional arterial oxygen saturation) and accuracy root mean square error (A RMS ) in each skin color bin defined by Individual Typology Angle (ITA) measured at the forehead across the saturation range of 90–100%. Panel A displays the distribution of bias in closed circles with 95% confidence interval (CI) whiskers. Panel B shows the distribution of A RMS in closed circles with 95% CI whiskers. Data are stratified into four ITA bins (top to bottom): ITA > 30°, -30° ≤ ITA ≤ 30°, ITA < -30°, and ITA < -50°; note that ITA < -50° is a subset of ITA < -30°. Sample size (n) for each bin is indicated at left in the corresponding color. The vertical dashed line in Panel A represents bias of 0 and in Panel B represents the A RMS 3% threshold. Download figure Open in new tab Figure S8. Distribution of bias and A RMS across ITA bins at the DP site over 70–100% saturation range in 24 participants cohorts This figure shows the distribution of pulse oximeter measured oxygen saturation (SpO₂) bias (mean of SpO₂ minus SaO₂ error, where SaO₂ is functional arterial oxygen saturation) and accuracy root mean square error (A RMS ) in each skin color bin defined by Individual Typology Angle (ITA) measured at the dorsal distal phalanx (DP) site across the saturation range of 70–100%. Panel A displays the distribution of bias in closed circles with 95% confidence interval (CI) whiskers. Panel B shows the distribution of A RMS in closed circles with 95% CI whiskers. Data are stratified into four ITA bins (top to bottom): ITA > 30°, -30° ≤ ITA ≤ 30°, ITA < -30°, and ITA < -50°; note that ITA < -50° is a subset of ITA < -30°. Sample size (n) for each bin is indicated at left in the corresponding color. The vertical dashed line in Panel A represents bias of 0 and in Panel B represents the A RMS 3% threshold. Download figure Open in new tab Figure S9. Distribution of bias and A RMS across ITA bins at the DP site over 70–80% saturation range in 24 participants cohorts This figure shows the distribution of pulse oximeter measured oxygen saturation (SpO₂) bias (mean of SpO₂ minus SaO₂ error, where SaO₂ is functional arterial oxygen saturation) and accuracy root mean square error (A RMS ) in each skin color bin defined by Individual Typology Angle (ITA) measured at the dorsal distal phalanx (DP) site across the saturation range of 70–80%. Panel A displays the distribution of bias in closed circles with 95% confidence interval (CI) whiskers. Panel B shows the distribution of A RMS in closed circles with 95% CI whiskers. Data are stratified into four ITA bins (top to bottom): ITA > 30°, -30° ≤ ITA ≤ 30°, ITA < -30°, and ITA < -50°; note that ITA < -50° is a subset of ITA < -30°. Sample size (n) for each bin is indicated at left in the corresponding color. The vertical dashed line in Panel A represents bias of 0 and in Panel B represents the A RMS 3% threshold. Download figure Open in new tab Figure S10. Distribution of bias and A RMS across ITA bins at the DP site over 80–90% saturation range in 24 participants cohorts This figure shows the distribution of pulse oximeter measured oxygen saturation (SpO₂) bias (mean of SpO₂ minus SaO₂ error, where SaO₂ is functional arterial oxygen saturation) and accuracy root mean square error (A RMS ) in each skin color bin defined by Individual Typology Angle (ITA) measured at the dorsal distal phalanx (DP) site across the saturation range of 80–90%. Panel A displays the distribution of bias in closed circles with 95% confidence interval (CI) whiskers. Panel B shows the distribution of A RMS in closed circles with 95% CI whiskers. Data are stratified into four ITA bins (top to bottom): ITA > 30°, -30° ≤ ITA ≤ 30°, ITA < -30°, and ITA < -50°; note that ITA < -50° is a subset of ITA < -30°. Sample size (n) for each bin is indicated at left in the corresponding color. The vertical dashed line in Panel A represents bias of 0 and in Panel B represents the A RMS 3% threshold. Download figure Open in new tab Figure S11. Distribution of bias and A RMS across ITA bins at the DP site over 90–100% saturation range in 24 participants cohorts This figure shows the distribution of pulse oximeter measured oxygen saturation (SpO₂) bias (mean of SpO₂ minus SaO₂ error, where SaO₂ is functional arterial oxygen saturation) and accuracy root mean square error (A RMS ) in each skin color bin defined by Individual Typology Angle (ITA) measured at the dorsal distal phalanx (DP) site across the saturation range of 90–100%. Panel A displays the distribution of bias in closed circles with 95% confidence interval (CI) whiskers. Panel B shows the distribution of A RMS in closed circles with 95% CI whiskers. Data are stratified into four ITA bins (top to bottom): ITA > 30°, -30° ≤ ITA ≤ 30°, ITA < -30°, and ITA < -50°; note that ITA < -50° is a subset of ITA < -30°. Sample size (n) for each bin is indicated at left in the corresponding color. The vertical dashed line in Panel A represents bias of 0 and in Panel B represents the A RMS 3% threshold. Tables View this table: View inline View popup Table S1. A RMS at the DP for 24 participant cohorts View this table: View inline View popup Table S2. Bias (mean of SpO 2 minus SaO 2 ) at the DP for 24 participant cohorts View this table: View inline View popup Table S3. Number of SaO 2 measurements (24 participant cohorts, max participant cohorts) stratified by skin color measured at the forehead View this table: View inline View popup Table S4. Complete list of devices, prices, specifications, and regulatory markings View this table: View inline View popup Table S5. Device conformity with current and anticipated FDA and ISO performance thresholds for maximum cohort sizes and DP ITA View this table: View inline View popup Table S6. A RMS at the forehead for 24 participant cohorts View this table: View inline View popup Table S7. Bias (mean of SpO 2 minus SaO 2 ) at the forehead for 24 participant cohorts View this table: View inline View popup Table S8. A RMS at the forehead for maximum participant cohorts View this table: View inline View popup Table S9. A RMS at the DP for maximum participant cohorts View this table: View inline View popup Table S10. Bias (mean of SpO 2 minus SaO 2 ) at the DP for maximum participant cohorts View this table: View inline View popup Table S11. Bias (mean of SpO 2 minus SaO 2 ) at the forehead for maximum participant cohorts View this table: View inline View popup Table S12. Differential bias for 24 participant cohorts View this table: View inline View popup Table S13. Differential bias for maximum participant cohorts View this table: View inline View popup Table S14. Count of devices with statistically significant differences in bias (mean of SpO 2 minus SaO 2 ) across different definitions of dark and anatomical sites in maximum participant cohorts View this table: View inline View popup Table S15. Count of devices with 95% CI of difference in bias (mean of SpO 2 minus SaO 2 ) exceeding pre-defined thresholds across different definitions of dark and anatomical sites in maximum participant cohorts View this table: View inline View popup Table S16. Reproducibility of device conformity with with regulatory frameworks View this table: View inline View popup Table S17. ITA and pulsatility amplitude for participant pigment bins in all device cohorts Acknowledgements We are grateful to members of the Open Oximetry Collaborative Community who participated in the project’s open forum discussions on protocols and approaches to data analysis, including James Ramsay, Jenna Lester, Alex Pogorzelski, Margaret Akey, Daryl Dorsey, Bob Kopotic and Sandy Weininger (ISO/IEC Conveners of the Oximeters Medical Device Standards), and many others. Footnotes This manuscript revision includes an updated figure (Figure 1) and minor text changes to improve clarity. Abbreviations and Acronyms A RMS Accuracy root mean square error CI Confidence interval UCI Upper confidence interval DP Dorsal distal phalanx FDA US Food and Drug Administration ISO International Organization for Standardization ITA Individual Typology Angle KM Konica Minolta LME Linear mixed effects (model) LR Linear regression (model) MST Monk Skin Tone (Scale) NIH National Institutes of Health SaO 2 Functional arterial hemoglobin oxygen saturation SpO 2 Pulse oximeter indirect measure of arterial hemoglobin oxygen saturation UCSF University of California San Francisco References 1. ↵ Ruppel H , Makeneni S , Faerber JA , et al. Evaluating the Accuracy of Pulse Oximetry in Children According to Race . JAMA Pediatr . 2023 ; 177 ( 5 ): 540 – 543 . doi: 10.1001/jamapediatrics.2023.0071 OpenUrl CrossRef PubMed 2. Wong AKI , Charpignon M , Kim H , et al. Analysis of Discrepancies Between Pulse Oximetry and Arterial Oxygen Saturation Measurements by Race and Ethnicity and Association With Organ Dysfunction and Mortality . JAMA Netw Open . 2021 ; 4 ( 11 ): e2131674 . doi: 10.1001/jamanetworkopen.2021.31674 OpenUrl CrossRef 3. Henry NR , Hanson AC , Schulte PJ , et al. Disparities in Hypoxemia Detection by Pulse Oximetry Across Self-Identified Racial Groups and Associations With Clinical Outcomes . Crit Care Med . 2022 ; 50 ( 2 ): 204 – 211 . doi: 10.1097/CCM.0000000000005394 OpenUrl CrossRef PubMed 4. ↵ Sjoding MW , Dickson RP , Iwashyna TJ , Gay SE , Valley TS . Racial Bias in Pulse Oximetry Measurement . N Engl J Med . 2020 ; 383 ( 25 ): 2477 – 2478 . doi: 10.1056/NEJMc2029240 OpenUrl CrossRef PubMed 5. ↵ Zeballos RJ , Weisman IM . Reliability of Noninvasive Oximetry in Black Subjects during Exercise and Hypoxia . Am Rev Respir Dis . 1991 ; 144 ( 6 ): 1240 – 1244 . doi: 10.1164/ajrccm/144.6.1240 OpenUrl CrossRef PubMed 6. ↵ Feiner JR , Severinghaus JW , Bickler PE . Dark Skin Decreases the Accuracy of Pulse Oximeters at Low Oxygen Saturation: The Effects of Oximeter Probe Type and Gender . Anesth Analg . 2007 ; 105 ( 6 ): S18 . doi: 10.1213/01.ane.0000285988.35174.d9 OpenUrl CrossRef PubMed Web of Science 7. ↵ Bickler PE , Feiner JR , Severinghaus JW . Effects of Skin Pigmentation on Pulse Oximeter Accuracy at Low Saturation . Anesthesiology . 2005 ; 102 ( 4 ): 715 – 719 . doi: 10.1097/00000542-200504000-00004 OpenUrl CrossRef PubMed Web of Science 8. ↵ Gudelunas MK , Lipnick M , Hendrickson C , et al. Low Perfusion and Missed Diagnosis of Hypoxemia by Pulse Oximetry in Darkly Pigmented Skin: A Prospective Study . Anesth Analg . 2024 ; 138 ( 3 ): 552 . doi: 10.1213/ANE.0000000000006755 OpenUrl CrossRef PubMed 9. ↵ ISO 80601-2-61:2017. ISO . Accessed June 4, 2024 . https://www.iso.org/standard/67963.html 10. ↵ Pulse Oximeters - Premarket Notification Submissions [510(k)s]: Guidance for Industry and Food and Drug Administration Staff . April 11 , 2024 . Accessed June 4, 2024 . https://www.fda.gov/regulatory-information/search-fda-guidance-documents/pulse-oximeters-premarket-notification-submissions-510ks-guidance-industry-and-food-and-drug 11. ↵ Okunlola OE , Lipnick MS , Batchelder PB , Bernstein M , Feiner JR , Bickler PE . Pulse Oximeter Performance, Racial Inequity, and the Work Ahead . Respir Care . 2022 ; 67 ( 2 ): 252 – 257 . doi: 10.4187/respcare.09795 OpenUrl Abstract / FREE Full Text 12. ↵ Health C for D and R . Pulse Oximeters for Medical Purposes - Non-Clinical and Clinical Performance Testing, Labeling, and Premarket Submission Recommendations . January 10 , 2025 . Accessed January 12, 2025 . https://www.fda.gov/regulatory-information/search-fda-guidance-documents/pulse-oximeters-medical-purposes-non-clinical-and-clinical-performance-testing-labeling-and 13. ↵ openoximetry.org . Collaborative Community . OpenOximetry . Accessed July 12, 2024 . https://openoximetry.org/community/ 14. FDA Executive Summary: Performance Evaluation of Pulse Oximeters Taking into Consideration Skin Pigmentation, Race and Ethnicity . Published online February 2 , 2024 : 49 . https://www.fda.gov/media/175828/download 15. ↵ Stakeholder Consultation on Pulse Oximeters: Next Steps and Closing Remarks .; 2024 . Accessed June 26, 2024 . https://www.youtube.com/watch?v=2MqyMdrSZfs 16. ↵ Stakeholder Consultation on Pulse Oximeters: Updating Global Requirements for Pulse Oximeters .; 2024 . Accessed June 26, 2024 . https://www.youtube.com/watch?v=zSl5OWkOyck 17. ↵ Hypoxia Lab Skin Color Assessment Protocol . Google Docs . Accessed February 10, 2025 . https://docs.google.com/document/d/17SGVjf1phxyhBB3TvetyuOFN_UC5z2humiEHI4dcEro/edit?usp=sharing&usp=embed_facebook 18. ↵ Skin Tone Research @ Google AI | undefined . Accessed June 10, 2024 . https://www.skintone.google/mste-dataset 19. ↵ Vasudevan S , Vogt WC , Weininger S , Pfefer TJ . Melanometry for objective evaluation of skin pigmentation in pulse oximetry studies . Commun Med . 2024 ; 4 ( 1 ): 1 – 19 . doi: 10.1038/s43856-024-00550-7 OpenUrl CrossRef 20. ↵ Stamatas GN , Kollias N . Blood stasis contributions to the perception of skin pigmentation . J Biomed Opt . 2004 ; 9 ( 2 ): 315 – 322 . doi: 10.1117/1.1647545 OpenUrl CrossRef PubMed 21. ↵ Skin Color Subgroup Meeting 3- March 13, 2024 .; 2024 . Accessed June 9, 2025 . https://www.youtube.com/watch?v=6ELI1XcbI 22. ↵ Del Bino S , Bernerd F . Variations in skin colour and the biological consequences of ultraviolet radiation exposure . Br J Dermatol . 2013 ; 169 ( s3 ): 33 – 40 . doi: 10.1111/bjd.12529 OpenUrl CrossRef PubMed 23. ↵ Leeb G , Auchus I , Law T , et al. The performance of 11 fingertip pulse oximeters during hypoxemia in healthy human participants with varied, quantified skin pigment . eBioMedicine . 2024 ; 102 . doi: 10.1016/j.ebiom.2024.105051 OpenUrl CrossRef 24. ↵ Lipnick MS , Feiner JR , Au P , Bernstein M , Bickler PE . The Accuracy of 6 Inexpensive Pulse Oximeters Not Cleared by the Food and Drug Administration: The Possible Global Public Health Implications . Anesth Analg . 2016 ; 123 ( 2 ): 338 – 345 . doi: 10.1213/ANE.0000000000001300 OpenUrl CrossRef PubMed 25. ↵ Hypoxia lab study protocol basic pulse ox hypoxemia performance study (Public comment). Google Docs . Accessed July 1, 2024 . https://docs.google.com/document/d/1bpVzZg9M0StLoyB_iF07V8at0h8lQhDyYnNCoSMVZQU/edit?usp=sharing&usp=embed_facebook 26. ↵ Approach for Improving the Performance Evaluation of Pulse Oximeter Devices Taking into Consideration Skin Pigmentation, Race and Ethnicity: Discussion Paper and Request for Feedback . 27. ↵ openoximetry.org . Data Repository . OpenOximetry . Accessed March 21, 2025 . https://openoximetry.org/data-repository/ 28. ↵ Blanchet MA , Mercier G , Delobel A , et al. Accuracy of Multiple Pulse Oximeters in Stable Critically Ill Patients . Respir Care . 2023 ; 68 ( 5 ): 565 – 574 . doi: 10.4187/respcare.10582 OpenUrl Abstract / FREE Full Text 29. ↵ Harskamp RE , Bekker L , Himmelreich JCL , et al. Performance of popular pulse oximeters compared with simultaneous arterial oxygen saturation or clinical-grade pulse oximetry: a cross-sectional validation study in intensive care patients . BMJ Open Respir Res . 2021 ; 8 ( 1 ): e000939 . doi: 10.1136/bmjresp-2021-000939 OpenUrl Abstract / FREE Full Text 30. ↵ Fawzy A , Wu TD , Wang K , et al. Racial and Ethnic Discrepancy in Pulse Oximetry and Delayed Identification of Treatment Eligibility Among Patients With COVID-19 . JAMA Intern Med . 2022 ; 182 ( 7 ): 730 . doi: 10.1001/jamainternmed.2022.1906 OpenUrl CrossRef 31. ↵ Martin D , Johns C , Sorrell L , et al. Effect of skin tone on the accuracy of the estimation of arterial oxygen saturation by pulse oximetry: a systematic review . Br J Anaesth . 2024 ; 132 ( 5 ): 945 – 956 . doi: 10.1016/j.bja.2024.01.023 OpenUrl CrossRef PubMed 32. Graham HR , King C , Rahman AE , et al. Reducing global inequities in medical oxygen access: the Lancet Global Health Commission on medical oxygen security . Lancet Glob Health . 2025 ; 13 ( 3 ): e528 – e584 . doi: 10.1016/S2214-109X(24)00496-0 OpenUrl CrossRef 33. The BOOST II United Kingdom, Australia, and New Zealand Collaborative Groups . Oxygen Saturation and Outcomes in Preterm Infants . N Engl J Med . 2013 ; 368 ( 22 ): 2094 - 2104 . doi: 10.1056/NEJMoa1302298 OpenUrl CrossRef PubMed Web of Science 34. Oster ME , Pinto NM , Pramanik AK , et al. Newborn Screening for Critical Congenital Heart Disease: A New Algorithm and Other Updated Recommendations: Clinical Report . Pediatrics . 2024 ; 155 ( 1 ): e2024069667 . doi: 10.1542/peds.2024-069667 OpenUrl CrossRef 35. Siemieniuk RAC , Chu DK , Kim LHY , et al. Oxygen therapy for acutely ill medical patients: a clinical practice guideline . Published online October 24 , 2018 . doi: 10.1136/bmj.k4169 OpenUrl FREE Full Text 36. World Health Organization . Clinical management of severe acute respiratory infection (SARI) when COVID-19 disease is suspected . Interim guidance . Published online March 13 , 2020 . Accessed July 21, 2025 . https://www.who.int/docs/default-source/coronaviruse/clinical-management-of-novel-cov.pdf 37. ↵ Moore KL , Gudelunas K , Lipnick MS , Bickler PE , Hendrickson CM . Pulse Oximeter Bias and Inequities in Retrospective Studies––Now What? Respir Care . 2022 ; 67 ( 12 ): 1633 – 1636 . doi: 10.4187/respcare.10654 OpenUrl FREE Full Text 38. ↵ Lipnick MS , Ehie O , Igaga EN , Bicker P . Pulse Oximetry and Skin Pigmentation—New Guidance From the FDA . JAMA . 2025 ; 333 ( 16 ): 1393 – 1395 . doi: 10.1001/jama.2025.1959 OpenUrl CrossRef PubMed 39. ↵ Gehring H , Duembgen L , Peterlein M , Hagelberg S , Dibbelt L . Hemoximetry as the “Gold Standard”? Error Assessment Based on Differences Among Identical Blood Gas Analyzer Devices of Five Manufacturers . Anesth Analg . 2007 ; 105 ( 6 ): S24 . doi: 10.1213/01.ane.0000268713.58174.cc OpenUrl CrossRef PubMed Web of Science 40. ↵ Lipnick MS , Chen D , Law T , et al. Comparison of methods for characterizing skin pigment diversity in research cohorts . Br J Dermatol . Published online October 10 , 2025 : ljaf397 . doi: 10.1093/bjd/ljaf397 OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted October 28, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Pulse oximeter performance and skin pigment: comparison of 34 oximeters using current and emerging regulatory frameworks Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Pulse oximeter performance and skin pigment: comparison of 34 oximeters using current and emerging regulatory frameworks Caroline Hughes , Danni Chen , Tyler Law , Philip Bickler , John Feiner , Leonid Shmuylovich , Ella Behnke , Lily Ortiz , Gregory Leeb , Isabella Auchus , Fekir Negussie , Ronald Bisegerwa , René Vargas Zamora , Elizabeth Igaga , Kelvin Moore Jr. , Olubunmi Okunlola , Ellis Monk , Jana Lyn Fernandez , Odinakachukwu Ehie , Bernadette Wilks , Koyinsola Oyefeso , Deleree Schornack , Cornelius Sendagire , Michael S. Lipnick medRxiv 2025.08.11.25332026; doi: https://doi.org/10.1101/2025.08.11.25332026 Share This Article: Copy Citation Tools Pulse oximeter performance and skin pigment: comparison of 34 oximeters using current and emerging regulatory frameworks Caroline Hughes , Danni Chen , Tyler Law , Philip Bickler , John Feiner , Leonid Shmuylovich , Ella Behnke , Lily Ortiz , Gregory Leeb , Isabella Auchus , Fekir Negussie , Ronald Bisegerwa , René Vargas Zamora , Elizabeth Igaga , Kelvin Moore Jr. , Olubunmi Okunlola , Ellis Monk , Jana Lyn Fernandez , Odinakachukwu Ehie , Bernadette Wilks , Koyinsola Oyefeso , Deleree Schornack , Cornelius Sendagire , Michael S. Lipnick medRxiv 2025.08.11.25332026; doi: https://doi.org/10.1101/2025.08.11.25332026 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Anesthesia Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (299) Cardiovascular Medicine (4425) Dentistry and Oral Medicine (443) Dermatology (382) Emergency Medicine (607) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1507) Epidemiology (15221) Forensic Medicine (30) Gastroenterology (1123) Genetic and Genomic Medicine (6588) Geriatric Medicine (667) Health Economics (997) Health Informatics (4524) Health Policy (1368) Health Systems and Quality Improvement (1612) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15910) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (145) Nephrology (667) Neurology (6588) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1143) Occupational and Environmental Health (956) Oncology (3331) Ophthalmology (970) Orthopedics (369) Otolaryngology (420) Pain Medicine (435) Palliative Medicine (129) Pathology (663) Pediatrics (1690) Pharmacology and Therapeutics (691) Primary Care Research (710) Psychiatry and Clinical Psychology (5440) Public and Global Health (9220) Radiology and Imaging (2195) Rehabilitation Medicine and Physical Therapy (1369) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (710) Sports Medicine (529) Surgery (710) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9ffd0f237a48ad07',t:'MTc3OTQ2NTg5Mg=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();