Evaluating reasoning LLMs’ potential to perpetuate racial and gender disease stereotypes in healthcare

Evaluating reasoning LLMs’ potential to perpetuate racial and gender disease stereotypes in healthcare | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Evaluating reasoning LLMs’ potential to perpetuate racial and gender disease stereotypes in healthcare Joshua J. Docking , Lee X. Li , Bradley D. Menz , Stephen Bacchi , Ashley M. Hopkins , Michael J. Sorich doi: https://doi.org/10.1101/2025.08.05.25333007 Joshua J. Docking 1 College of Medicine and Public Health, Flinders University , Adelaide, South Australia, Australia BMedSc Find this author on Google Scholar Find this author on PubMed Search for this author on this site Lee X. Li 1 College of Medicine and Public Health, Flinders University , Adelaide, South Australia, Australia MBioStats Find this author on Google Scholar Find this author on PubMed Search for this author on this site Bradley D. Menz 1 College of Medicine and Public Health, Flinders University , Adelaide, South Australia, Australia BPharm Find this author on Google Scholar Find this author on PubMed Search for this author on this site Stephen Bacchi 2 Department of Neurology and the Center for Genomic Medicine, Massachusetts General Hospital and Harvard Medical School , Boston, MA, United States PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ashley M. Hopkins 1 College of Medicine and Public Health, Flinders University , Adelaide, South Australia, Australia PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Michael J. Sorich 1 College of Medicine and Public Health, Flinders University , Adelaide, South Australia, Australia PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: michael.sorich{at}flinders.edu.au Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract This evaluation of 36,000 clinical vignettes found that next-generation reasoning large language models, o3-mini and DeepSeek-R1, frequently perpetuate racial and gender stereotypes for common medical conditions, indicating that advancements in reasoning do not inherently improve representational fairness. Introduction Large language models (LLMs) have the potential to transform healthcare, but risk exacerbating health disparities if they perpetuate biases. 1 – 3 Zack and colleagues 4 demonstrated potential racial and gender biases in clinical vignettes generated by GPT-4, including overrepresentation of Black patients in stereotypical medical conditions. Since then, next-generation ‘reasoning LLMs’ have emerged, offering improved reasoning capability (‘thinking’ before answering) and benchmark performance. 5 Whether this will reduce representational bias in healthcare remains unknown. This study evaluated whether reasoning LLMs exhibit racial and gender biases in generated clinical content. Methods Utilising the methods of Zack et. al. 4 , two reasoning LLMs, o3-mini (OpenAI, US) and DeepSeek-R1 (DeepSeek, China), generated patient cases across 18 medical conditions specifying a US population, using 10 prompt variations, each run 100 times. Patient demographic characteristics in the generated vignettes were extracted, 4 and the proportion of race and gender representation for each condition was calculated. Misrepresentation (LLM estimate minus the published US epidemiological estimates 4 ) was summarised as the median (range) across 18 medical conditions. For example, if 60% of LLM-generated HIV cases were Black patients, compared to 40% Black patients reported in representative US studies of HIV, misrepresentation would be +20%. Positive values indicate overrepresentation, and negative values indicate underrepresentation. A difference greater than 20% was considered the threshold for significant misrepresentation. Results 36,000 clinical vignettes were generated. Misrepresentation for o3-mini was 44% (–12% to +75%) for Black, –4.6% (–37% to 0%) for Asian, –14% (–27% to +0.7%) for Hispanic, and – 8.2% (–56% to +33%) for White persons ( Figure 1 ). Misrepresentation for DeepSeek-R1 was 31% (−21% to +81%) for Black, −4.4% (−35% to +47%) for Asian, −8.8% (−26% to +53%) for Hispanic, and −21% (−63% to +41%) for White persons ( Figure 2 ). For 78% (14/18) of medical conditions using o3-mini and 89% (16/18) using DeepSeek-R1, there was >20% misrepresentation for at least one race. Download figure Open in new tab Figure 1. Proportional Representation of Disease Cases by Race and Gender in o3 mini-Generated Clinical Content Compared with Published US Statistics Blue dot on the right of red diamond indicates overrepresentation by the LLM. Blue dot on the left indicates underrepresentation. Download figure Open in new tab Figure 2. Proportional Representation of Disease Cases by Race and Gender in DeepSeek R1-Generated Clinical Content Compared with Published US Statistics Blue dot on the right of red diamond indicates overrepresentation by the LLM. Blue dot on the left indicates underrepresentation. Female misrepresentation was −27% (−47% to +31%) for o3-mini, and −23% (−47% to +33%) for DeepSeek-R1. For 56% (10/18) of medical conditions using o3-mini and 67% (12/18) using DeepSeek-R1, there was >20% misrepresentation. There was a consistent overrepresentation of the gender with the higher published representation. Discussion Reasoning LLMs, o3-mini and DeepSeek-R1, frequently misrepresent the distribution of race and gender in medical conditions, mirroring issues previously observed in GPT-4. 4 GPT-4 met the threshold for significant misrepresentation in 72% (13/18) of conditions for race and 67% (12/18) for gender. 4 Our results show comparable or higher rates for o3-mini (78% race, 56% gender) and DeepSeek-R1 (89% race, 67% gender), indicating no improvement in representation with the newer reasoning models. Both o3-mini and DeepSeek-R1, like GPT-4, overrepresented Black populations in stereotypically associated conditions (e.g. sarcoidosis, systemic lupus erythematosus, preeclampsia, essential hypertension). 4 However, in o3-mini and DeepSeek-R1 this led to even higher median misrepresentation of 44% and 31%, respectively, compared to 15% in GPT-4. 4 This persistent bias risks embedding stereotypical associations within healthcare applications, potentially compromising objective clinical evaluations regarding race. Similarly, the consistent exaggeration of the majority gender aligns with previous findings showing LLM outputs skew towards gender stereotypes in healthcare roles, which could further marginalise minority genders. 6 This study’s strengths include its evaluation of next-generation reasoning LLMs and the robust assessment from 36,000 generated clinical vignettes. A limitation is our focus on a US context; future research should assess generalisability across different regions. In conclusion, despite enhanced reasoning capabilities, the clinical outputs of o3-mini and DeepSeek-R1 still exhibit racial and gender disease stereotyping in common medical conditions. Advancements in LLM capabilities do not guarantee parallel improvements in fairness and representation, crucial for equity in healthcare. Continuous monitoring of potential biases is thus essential. Data Availability The complete set of clinical vignettes will be made available upon reasonable request to the corresponding author. Conflicts of Interest MJS reported receiving grants from Pfizer, AstraZeneca, Boehringer Ingelheim, and the National Health and Medical Research Council of Australia outside the submitted work. AMH reported receiving grants from Boehringer Ingelheim, Hospital Research Foundation, Tour De Cure, and Flinders Foundation outside the submitted work. No other disclosures were reported. Author Contributions JJD and MJS had full access to all of the data in the study and take responsibility for the integrity of the data and the accuracy of the data analysis. Concept and design: MJS Acquisition, analysis, or interpretation of data: JJD, XLL, BDM, SB, AMH, MJS Drafting of the manuscript: JJD Critical review of the manuscript for important intellectual content: XLL, BDM, SB, AMH, MJS Statistical analysis: JJD, MJS Additional Information o3 mini (OpenAI) and DeepSeek-R1 (DeepSeek) were used to generate the synthetic vignettes according to the research methods outlined in this manuscript. ChatGPT 4o (OpenAI) and Gemini 2.5 Pro (Google) were used to assist in formatting and editing the manuscript. Acknowledgements MJS is supported by a Beat Cancer Research Fellowship from the Cancer Council South Australia (PRF2719). AMH holds an Emerging Leader Investigator Fellowship from the National Health and Medical Research Council, Australia (APP2008119). The PhD scholarship of BDM is supported by the National Health and Medical Research Council, Australia (APP2030913). The funders had no role in the design and conduct of the study; collection, management, analysis, and interpretation of the data; preparation, review, or approval of the manuscript; and decision to submit the manuscript for publication. References 1. ↵ Weidinger L , Uesato J , Rauh M , et al. Taxonomy of Risks posed by Language Models . In: 2022 ACM Conference on Fairness, Accountability, and Transparency . ACM ; 2022 . doi: 10.1145/3531146.3533088 OpenUrl CrossRef 2. Adam H , Balagopalan A , Alsentzer E , Christia F , Ghassemi M. Mitigating the impact of biased artificial intelligence in emergency decision-making . Commun Med (Lond) . 2022 ; 2 ( 1 ): 149 . OpenUrl PubMed 3. ↵ Pfohl SR , Cole-Lewis H , Sayres R , et al. A toolbox for surfacing health equity harms and biases in large language models . Nat Med . Published online 23 September 2024: 1 – 11 . 4. ↵ Zack T , Lehman E , Suzgun M , et al. Assessing the potential of GPT-4 to perpetuate racial and gender biases in health care: a model evaluation study . Lancet Digit Health . 2024 ; 6 ( 1 ): e12 – e22 . OpenUrl CrossRef 5. ↵ Brodeur PG , Buckley TA , Kanjee Z , et al. Superhuman performance of a large language model on the reasoning tasks of a physician . arXiv [csAI] . Published online 14 December 2024. http://arxiv.org/abs/2412.10849 6. ↵ Menz BD , Kuderer NM , Chin-Yee B , et al. Gender representation of health care professionals in large language model-generated stories . JAMA Netw Open . 2024 ; 7 ( 9 ): e2434997 . OpenUrl View the discussion thread. Back to top Previous Next Posted August 07, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Evaluating reasoning LLMs’ potential to perpetuate racial and gender disease stereotypes in healthcare Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Evaluating reasoning LLMs’ potential to perpetuate racial and gender disease stereotypes in healthcare Joshua J. Docking , Lee X. Li , Bradley D. Menz , Stephen Bacchi , Ashley M. Hopkins , Michael J. Sorich medRxiv 2025.08.05.25333007; doi: https://doi.org/10.1101/2025.08.05.25333007 Share This Article: Copy Citation Tools Evaluating reasoning LLMs’ potential to perpetuate racial and gender disease stereotypes in healthcare Joshua J. Docking , Lee X. Li , Bradley D. Menz , Stephen Bacchi , Ashley M. Hopkins , Michael J. Sorich medRxiv 2025.08.05.25333007; doi: https://doi.org/10.1101/2025.08.05.25333007 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4440) Dentistry and Oral Medicine (444) Dermatology (383) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1510) Epidemiology (15229) Forensic Medicine (30) Gastroenterology (1126) Genetic and Genomic Medicine (6605) Geriatric Medicine (668) Health Economics (998) Health Informatics (4541) Health Policy (1369) Health Systems and Quality Improvement (1613) Hematology (543) HIV/AIDS (1265) Infectious Diseases (except HIV/AIDS) (15921) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (147) Nephrology (668) Neurology (6604) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1145) Occupational and Environmental Health (957) Oncology (3334) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (692) Primary Care Research (711) Psychiatry and Clinical Psychology (5448) Public and Global Health (9234) Radiology and Imaging (2199) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (594) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a01564a419abc13d',t:'MTc3OTcyMTA1Mg=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();