Consistent Performance of GPT-4o in Rare Disease Diagnosis Across Nine Languages and 4967 Cases

Consistent Performance of GPT-4o in Rare Disease Diagnosis Across Nine Languages and 4967 Cases | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Consistent Performance of GPT-4o in Rare Disease Diagnosis Across Nine Languages and 4967 Cases View ORCID Profile Leonardo Chimirri , View ORCID Profile J. Harry Caufield , View ORCID Profile Yasemin Bridges , View ORCID Profile Nicolas Matentzoglu , View ORCID Profile Michael Gargano , View ORCID Profile Mario Cazalla , Shihan Chen , View ORCID Profile Daniel Danis , View ORCID Profile Alexander JM Dingemans , Petra Gehle , View ORCID Profile Adam S.L. Graefe , Weihong Gu , View ORCID Profile Markus S. Ladewig , View ORCID Profile Pablo Lapunzina , Julián Nevado , Enock Niyonkuru , View ORCID Profile Soichi Ogishima , View ORCID Profile Dominik Seelow , View ORCID Profile Jair A Tenorio Castaño , View ORCID Profile Marek Turnovec , View ORCID Profile Bert BA de Vries , View ORCID Profile Kai Wang , View ORCID Profile Kyran Wissink , View ORCID Profile Zafer Yüksel , View ORCID Profile Gabriele Zucca , View ORCID Profile Melissa A. Haendel , View ORCID Profile Christopher J. Mungall , View ORCID Profile Justin Reese , View ORCID Profile Peter N. Robinson doi: https://doi.org/10.1101/2025.02.26.25322769 Leonardo Chimirri 1 Berlin Institute of Health at Charité – Universitätsmedizin Berlin , Berlin, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Leonardo Chimirri J. Harry Caufield 2 Lawrence Berkeley National Laboratory , Berkeley, CA, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for J. Harry Caufield Yasemin Bridges 3 William Harvey Research Institute, Barts and The London School of Medicine and Dentistry, Queen Mary University of London , London, UK Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Yasemin Bridges Nicolas Matentzoglu 4 Semanticly , Athens, Greece Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Nicolas Matentzoglu Michael Gargano 5 The Jackson Laboratory for Genomic Medicine Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Michael Gargano Mario Cazalla 6 INGEMM-Idipaz, Institute of Medical and Molecular Genetics, Hospital Universitario La Paz , Madrid, Spain Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Mario Cazalla Shihan Chen 7 Department of Pathology and Laboratory Medicine, University of Pennsylvania , Philadelphia, PA, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Daniel Danis 1 Berlin Institute of Health at Charité – Universitätsmedizin Berlin , Berlin, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Daniel Danis Alexander JM Dingemans 8 Department of Human Genetics, Donders Institute for Brain, Cognition and Behaviour, Radboud University Medical Center , Nijmegen, the Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Alexander JM Dingemans Petra Gehle 9 Deutsches Herzzentrum der Charité , Berlin, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site Adam S.L. Graefe 1 Berlin Institute of Health at Charité – Universitätsmedizin Berlin , Berlin, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Adam S.L. Graefe Weihong Gu 10 Chinese HPO Consortium , Beijing, China Find this author on Google Scholar Find this author on PubMed Search for this author on this site Markus S. Ladewig 11 Department of Ophthalmology, University Clinic Marburg - Campus Fulda , Fulda, Germany CIBERER-ISCIII , Madrid, Spain Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Markus S. Ladewig Pablo Lapunzina 6 INGEMM-Idipaz, Institute of Medical and Molecular Genetics, Hospital Universitario La Paz , Madrid, Spain Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Pablo Lapunzina Julián Nevado 6 INGEMM-Idipaz, Institute of Medical and Molecular Genetics, Hospital Universitario La Paz , Madrid, Spain Find this author on Google Scholar Find this author on PubMed Search for this author on this site Enock Niyonkuru 2 Lawrence Berkeley National Laboratory , Berkeley, CA, USA 12 Trinity College , Hartford, CT, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Soichi Ogishima 13 INGEM/ToMMo, Tohoku University , Miyagi, Japan Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Soichi Ogishima Dominik Seelow 1 Berlin Institute of Health at Charité – Universitätsmedizin Berlin , Berlin, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Dominik Seelow Jair A Tenorio Castaño 6 INGEMM-Idipaz, Institute of Medical and Molecular Genetics, Hospital Universitario La Paz , Madrid, Spain Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Jair A Tenorio Castaño Marek Turnovec 14 Department of Biology and Medical Genetics, 2nd Faculty of Medicine, Charles University in Prague and Motol University Hospital , Prague, Czech Republic Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Marek Turnovec Bert BA de Vries 8 Department of Human Genetics, Donders Institute for Brain, Cognition and Behaviour, Radboud University Medical Center , Nijmegen, the Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Bert BA de Vries Kai Wang 7 Department of Pathology and Laboratory Medicine, University of Pennsylvania , Philadelphia, PA, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Kai Wang Kyran Wissink 1 Berlin Institute of Health at Charité – Universitätsmedizin Berlin , Berlin, Germany 15 Utrecht University , Utrecht, Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Kyran Wissink Zafer Yüksel 16 Department of Human Genetics, Bioscientia Healthcare GmbH , Ingelheim, Germany Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Zafer Yüksel Gabriele Zucca 17 Institute for Maternal and Child Health - IRCCS “Burlo Garofolo” - Trieste , Trieste 34137, Italy Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Gabriele Zucca Melissa A. Haendel 18 University of North Carolina at Chapel Hill Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Melissa A. Haendel Christopher J. Mungall 2 Lawrence Berkeley National Laboratory , Berkeley, CA, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Christopher J. Mungall Justin Reese 2 Lawrence Berkeley National Laboratory , Berkeley, CA, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Justin Reese For correspondence: justinreese{at}lbl.gov peter.robinson{at}bih-charite.de Peter N. Robinson 1 Berlin Institute of Health at Charité – Universitätsmedizin Berlin , Berlin, Germany 5 The Jackson Laboratory for Genomic Medicine Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Peter N. Robinson For correspondence: justinreese{at}lbl.gov peter.robinson{at}bih-charite.de Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Summary Background Large language models (LLMs) are increasingly used in the medical field for diverse applications including differential diagnostic support. The estimated training data used to create LLMs such as the Generative Pretrained Transformer (GPT) predominantly consist of English-language texts, but LLMs could be used across the globe to support diagnostics if language barriers could be overcome. Initial pilot studies on the utility of LLMs for differential diagnosis in languages other than English have shown promise, but a large-scale assessment on the relative performance of these models in a variety of European and non-European languages on a comprehensive corpus of challenging rare-disease cases is lacking. Methods We created 4967 clinical vignettes using structured data captured with Human Phenotype Ontology (HPO) terms with the Global Alliance for Genomics and Health (GA4GH) Phenopacket Schema. These clinical vignettes span a total of 378 distinct genetic diseases with 2618 associated phenotypic features. We used translations of the Human Phenotype Ontology together with language-specific templates to generate prompts in English, Chinese, Czech, Dutch, German, Italian, Japanese, Spanish, and Turkish. We applied GPT-4o, version gpt-4o-2024-08-06, to the task of delivering a ranked differential diagnosis using a zero-shot prompt. An ontology-based approach with the Mondo disease ontology was used to map synonyms and to map disease subtypes to clinical diagnoses in order to automate evaluation of LLM responses. Findings For English, GPT-4o placed the correct diagnosis at the first rank 19·8% and within the top-3 ranks 27·0% of the time. In comparison, for the eight non-English languages tested here the correct diagnosis was placed at rank 1 between 16·9% and 20·5%, within top-3 between 25·3% and 27·7% of cases. Interpretation The differential diagnostic performance of GPT-4o across a comprehensive corpus of rare-disease cases was consistent across the nine languages tested. This suggests that LLMs such as GPT-4o may have utility in non-English clinical settings. Funding NHGRI 5U24HG011449 and 5RM1HG010860. P.N.R. was supported by a Professorship of the Alexander von Humboldt Foundation; P.L. was supported by a National Grant (PMP21/00063 ONTOPREC-ISCIII, Fondos FEDER). Introduction Large language models (LLM) are being carefully investigated in the medical domain owing to their linguistic and problem solving capabilities. These artificial intelligence (AI) models are pre-trained by ingesting large amounts of unlabeled data in order to learn to generate coherent text, thereby opening up an array of possibilities in disparate aspects of clinical practice. Moreover, LLMs have been shown to encode latent medical knowledge and to be able to carry out deductive reasoning, which would make them candidate assistance tools in hard-to-diagnose, complex clinical cases. 1 The majority of medical literature and therefore the LLMs’ relevant training data is in English. According to CommonCrawl, 2 an open repository of web crawl data that can be used to estimate the distribution of internet data used by LLMs for training, 43% of available web pages are in English (version CC-MAIN-2024-51). The percentages for the other eight languages in our study range from 1·0% (Czech) to 5·4% (German). While the precise training data for most LLMs are not publicly disclosed, estimates suggest a high proportion of English language content. 3 Thus, more material is available for training in English than in any other language, which may imply an expectation of LLMs to achieve higher performance in English, as suggested from previous studies. 4 – 6 This clearly would have implications for their integration in clinical practice, as well as on the topic of AI fairness in the health domain. Few studies about multilinguality of generalist LLMs in biomedicine have been carried out, most of which tested the performance of a LLM on medical licensing exams or standardized medical questions in one non-English language or one language as compared to English (see Japanese, 7 in-context enhanced Chinese, 8 German, 9 and Arabic 6 ). Roughly 25% of rare disease patients go 5-30 years without a diagnosis, and 40% of initial diagnoses are wrong. 10 There are at least 10,000 RDs, 11 and the diagnostic yield of genomic sequencing is still low (25– 50%). 12 Therefore, the differential diagnosis of RDs represents a challenging task with which to evaluate the capabilities of LLMs. LLMs have shown promise in supporting the differential diagnosis of RDs in English, 13 yet most of the earth’s population does not use English as their first language and clinical practice is carried out in diverse languages across the world. In this paper, we assess the relative performance of LLMs in RD differential diagnostics by creating a large multilingual set of published patient descriptions. Thereby we address limitations of previous studies that used simulated/synthetic patients or small cohorts and provide a comprehensive analysis of the relative performance of a leading LLM in nine different languages. Methods Overview of Study We conducted a comparison of GPT-4o’s ability in genetic differential diagnostics across several languages. We analyzed 4967 case reports from the literature and generated LLM prompts in nine languages and directed GPT-4o to return a ranked list of possible diagnoses for each case. The Mondo disease ontology, 14 which contains numerous synonyms for each disease, was used to map the diagnoses returned by GPT-4o to a unique standardized medical vocabulary, to which we applied our automatic ontology-aware scoring. 15 The rank of the correct diagnosis in the output was compared across languages in order to assess to what degree the LLM’s performance is prompt-language dependent. This study is reported according to the TRIPOD-LLM reporting guideline. 16 Human Phenotype Ontology Internationalization The Human Phenotype Ontology (HPO) provides a standardized vocabulary of 19,034 terms that describe the phenotypic abnormalities of human disease. Version 2024-12-12 was used for this study. Additionally, the HPO provides a comprehensive corpus of phenotype annotations (HPOA) that form computational models of 8333 rare diseases. HPO applications include genomic interpretation for diagnostics, gene-disease discovery, machine learning (ML) and electronic health record (EHR) cohort analytics. 17 The HPO Internationalization Effort comprises language-specific working groups that have translated HPO term labels and in some cases synonyms and definitions from English into other languages. 18 For the current project, we used the Chinese, Czech, Dutch, German, Italian, Japanese, Spanish, and Turkish translations. All translations are freely available (see data availability section). Information about the number of available translated terms present in HPO is found in supplemental table S1. Translations are created by or confirmed by human experts before inclusion in a HPO release. Structured Data from Case Reports: Phenopackets The Global Alliance for Genomics and Health (GA4GH) Phenopacket Schema is a standard for sharing phenotypic, genetic and clinical information. 19 The Phenopacket Schema obtained International Standard Organization approval as ISO 4454:2022. Each Phenopacket is a clinical vignette about one individual with representations of phenotypic abnormalities using HPO terms, as well as a specification of the disease diagnosis and other information. The Phenopackets used in this project were selected from the Phenopacket Store version 0.1.19, an openly available collection of Phenopackets manually curated from published case reports. 20 It contains a total of 6668 Phenopackets representing 475 diseases and 423 disease genes. We restricted our analysis to the subset of Phenopackets in the Phenopacket Store for which translations of all associated HPO terms exist in the eight languages mentioned above. This yielded a dataset of 4967 Phenopackets from 729 PubMed IDs, comprising 343 causative disease genes, 378 diseases, 2934 alleles, 2618 unique HPO terms and an average of 14 HPO terms per patient. Prompt generation Phenopackets comprise a hierarchical structure that is typically stored as a JSON file. We developed a strategy to create narrative prompts from each Phenopacket by a templating system implemented in a Java application called phenopacket2prompt. 21 Each template consists of constant texts (such as the header that instructs GPT to return a differential diagnosis), and a series of templates to represent the age and sex of the individual represented by the Phenopacket as well as phenotypic abnormalities that were observed or excluded. If available, the age of onset of the disease or specific manifestations is recorded. The templating system involves vocabulary for describing the individual in each of the languages. HPO terms in each of the languages are substituted into the corresponding templates. The translators of each of the eight languages are physicians or medical researchers, and an example of the part of a prompt describing a patient is shown in figure 1 . The correctness of the translation templates was confirmed by review of 54 simulated cases that were output in each language using permutations of ages, sexes, as well as observed and excluded HPO terms. The diagnosis and the genetic information were not included in the prompts. At the end of the constant section, we state that the case is a genetic disease and we request GPT to return an ordered list of candidate diagnoses, giving an example output. The example output is always given in English and in non-English languages we explicitly instruct GPT to return the differential diagnosis in English. An example Phenopacket with associated full prompt in English and its translation into the eight languages is available in supplementary tables S3-S4. Download figure Open in new tab Figure 1. Templated system for generating prompts using translation of the HPO into 8 languages. An excerpt of one prompt is shown. Shading indicates Age, Sex, Onset, HPO Phenotypes. Grounding and Scoring The queries to GPT-4o were carried out through its API between November 22nd, 2024 and January 20th, 2025. The knowledge cutoff claimed by openAI is October 2023 for all GPT-4o models, including gpt-4o-2024-08-06, the version we used. We used the default parameters, with no token length limitation and using unit temperature. We instructed the LLM to reply in the form of free text (e.g., Marfan syndrome), rather than a corresponding ontology term identifier (e.g., MONDO:0007947), because the task of returning identifiers may be prone to so-called “hallucinations”, where plausible-sounding yet wrong answers are given. 22 , 23 We leveraged our pheval.llm pipeline to parse the LLM response for free-text candidate diagnoses, to identify the corresponding ontology identifier, and to rank genetic subforms of a clinical disease as equivalent (e.g., Loeys-Dietz syndrome and any of its six genetic subforms were regarded as equivalent for the purposes of assessing correctness of the differential diagnosis). PhEval.llm is a freely available plugin for the PhEval framework and leverages Monarch Initiative LLM tooling. 15 , 24 , 25 With this, we could computationally score the responses of GPT for all Phenopackets in all nine languages, corresponding to 44,198 differential diagnoses with more than 210 thousand candidate diseases (of which 6733 unique guesses by GPT). Finally, we computed the number of correct diagnoses found at the top of the differential diagnosis (“Top-1”), within the first three candidate diseases (“Top-3”), and likewise for “Top-10”. This procedure was carried out for all nine languages. Role of the funding source The funding sources had no role in study design, data collection, data analysis, data interpretation, writing of the report, and decision to submit. Results In this work we leveraged translations of the Human Phenotype Ontology into Chinese, Czech, Dutch, German, Italian, Japanese, Spanish, and Turkish to test the relative performance of GPT-4o in differential diagnostic support. To do so, we leveraged GA4GH Phenopackets, representing the clinical data (phenotypic abnormalities and diagnosis) for 4967 patients with 378 diseases drawn from 729 publications. We generated a programmatic templating system that generated a narrative text using templates to represent the age, sex, and age of onset of disease together with observed and excluded phenotypic features ( figure 1 ). Results are counts of correct diagnoses at a given rank in the differential diagnosis, shown in table 1 aggregated at “Top-1”, “Top-3”, and “Top-10”. The “Not ranked” column shows the number of cases in which the reply of GPT-4o did not contain the correct diagnosis. Additionally, the “No Diagnosis” column indicates the number of cases in which GPT did not return text that could be parsed as a differential (for example, “I’m sorry but based on the information provided, I cannot return a confident diagnosis”). In no case did we observe a correct result beyond rank 10, so that the sum of the last three columns in table 1 is always 4967, the total number of cases. Depending on the language, between 2·3% and 5·6% items in the differential could not be successfully grounded (i.e., the Mondo term corresponding to the diagnosis could not be identified; see supplemental table S2). View this table: View inline View popup Download powerpoint Table 1: Results of differential diagnosis by GPT-4o by language. Numbers indicate counts, not ranked includes grounding failures but not instances of refusal by GPT to deliver a diagnosis, which is counted separately in the rightmost column “No Diagnosis”. In figure 2 we show frequencies obtained as Top-N divided by the total number of cases excluding those in “No Diagnosis”. To test for statistical differences between ranks in different languages we used SciPy’s ‘stats’ package to perform a Kruskal-Wallis 26 H-test obtaining as a statistic H=22·3 and p-value=0·0044, indicating statistically significant differences between the languages. English was the second best performer at Top-1,Top-3, and Top-10. At Top-1, there were 19·8% of correct cases in English while other languages ranged from 16·9% to 20·5%. At Top-3, the spread decreases, with the correct diagnosis among the Top-3 candidates in the differential diagnosis 27·0% of cases for English, compared to the range 25·3% to 27·7% for the other languages. This comprises a relative difference of at most 6% for Top-3 with respect to English, and at most 8% for Top-10, where English scored 31·1% while other languages lie in the range 28·5% to 31·8%. Download figure Open in new tab Figure 2. Differential diagnostic performance of GPT-4o in English, Chinese, Czech, Dutch, German, Italian, Japanese, Spanish, and Turkish. The percentage of cases in which GPT-4o place the correct diagnosis in rank 1 (Top-1), within the top three ranks (Top-3) or within the first ten ranks (Top-10) is shown. Discussion We prompted the GPT-4o language model with 4967 RD cases in nine different languages. All languages in this study constitute at least ∼1% of the CommonCrawl, which is a proxy for the amount of relative internet data available in a given language, a reflection of the language-specific data available for training. For these nine languages we have shown that GPT-4o is able to perform differential diagnostics for RD with nearly the same performance. This would be surprising if LLMs such as GPT only used language-specific models to answer queries because most of its training data and of relevant medical literature is in English. Our result suggests that this model can generalize medically relevant knowledge derived mainly from English language texts in order to answer queries posed in (at least) the eight non-English languages tested. The creation of a large set of realistic vignette-like prompts originating from real cases and translated in multiple languages overcomes limitations of previous diagnostics studies with LLMs, such as the utilization of extensive and unrealistically long case reports, 27 the usage of simulated/synthetic patients, 28 , 29 the usage of comparably small cohorts, 27 – 29 and the difference in style and length of real clinical notes coming from different countries. 29 Our study has several limitations. We used the same zero-shot prompting strategy for each language and did not attempt to improve performance using more sophisticated strategies such as chain of thought or retrieval augmented generation approaches. 30 Our evaluation made use of lists of phenotype terms, rather than narrative clinical notes, and thus may not reflect challenges or nuances relating to individual languages. Additionally, we only tested a specific GPT model, and were only able to test a selection of relatively widely used European and Asian languages. Therefore, further research will be needed to determine the relative performance of LLMs on a wider range of languages using more prompting strategies. In order to be able to automatically evaluate the response of GPT, we explicitly asked for the LLM replies to be in English. We are not able to say if failures in mapping free text to Mondo identifiers observed in this step were responsible for the slightly worse performance of some languages in our testing. We did not assess the behavior and accuracy for multilingual diagnostics of other LLMs in our study, but our results with GPT-4o suggest that in principle it is possible to train an LLM to leverage medical knowledge that is predominantly recorded in English to support differential diagnostics in other languages. GPT-4o’s ability to carry out diagnostics of complex cases in different languages is notable considering it is a general purpose language model predominantly trained with English data. OpenAI claims to have significantly increased the multilingual capabilities of GPT-4o with respect to its previous models. 31 Consistent performance across languages has implications for the implementation of these models in clinical practice across the globe. Many people in low- and middle-income countries (LMICs) have limited access to healthcare services. 32 As LLMs become increasingly proficient in supporting differential diagnosis and related domains such as bedside consultation question answering and addressing questions from the general public, 33 there is a great potential to improve care for people in LMICs by supplementing existing systems with LLM-driven services. It would be desirable to offer such services in local languages, especially for consumer-facing applications. Future work will be required to assess performance of LLMs in LMICs (all languages assessed in our study are from high-income countries). Data availability HPO translations: https://obophenotype.github.io/hpo-translations/ 4967 phenopackets and corresponding prompts in English, Chinese, Czech, Dutch, German, Italian, Japanese, Spanish, and Turkish, together with all our results at DOI: https://zenodo.org/records/14907503 All code is publicly available at: pheval.llm: https://github.com/monarch-initiative/pheval.llm ontology access kit: https://github.com/INCATools/ontology-access-kit phenopacket2prompt: https://github.com/monarch-initiative/phenopacket2prompt Contributors L.C. and J.R. created the pheval.llm python code with input from J.H.C and Y.B. L.C. wrote the first draft of the paper in close collaboration with P.N.R. The phenopacket2prompt Java application was written by P.N.R. and L.C. who also wrote the Italian prompts and coordinated the language specific developers of the prompts. M.H. conceptualized the ontology translational strategy and Phenopackets. N.M. developed much of the technical infrastructure around translation management and is the main liaison for the HPO translation community. P.N.R. and J.R. led the overall study. C.M. devised the Mondo evaluation approach and advised on the use of LLMs. C.S. coded the Chinese prompts; G.W. is part of the development of Chinese HPO; K.W. confirmed the Chinese translations; D.D. coded the Czech prompts; M.T. provided Czech translations and confirmations; A.S.L.G. coded the Turkish prompts and helped with Turkish translations; Z.Y. confirmed and provided Turkish translations; D.S. suggested and confirmed German translations; G.J.P. confirmed the German translations; G.Z. provided the Italian translations; J.N., J.A.T.C, M.C., and P.L. provided the Spanish translations; K.W. coded the Dutch prompts; A.j.M.D. and B.B.A.d.V. provided the Dutch translations; S.O. provided the Japanese translations. All authors had access to the manuscript and take responsibility for the decision to submit the publication. All of the data used in this study is publicly available. Declaration of interests MH is a co-founder of Alamya Health. Acknowledgements We acknowledge funding from NHGRI 5U24HG011449, 5RM1HG010860, and R24OD011883. P.N.R. was supported by a Professorship of the Alexander von Humboldt Foundation; P.L. was supported by a National Grant (PMP21/00063 ONTOPREC-ISCIII, Fondos FEDER). CM and JR were supported in part by the Director, Office of Science, Office of Basic Energy Sciences, of the US Department of Energy (Contract No. DE-AC0205CH11231). REFERENCES ↵ Singhal K , Azizi S , Tu T , et al. Large language models encode clinical knowledge . Nature 2023 ; 620 : 172 – 80 . OpenUrl CrossRef PubMed ↵ Statistics of Common Crawl Monthly Archives by commoncrawl . https://commoncrawl.github.io/cc-crawl-statistics/plots/languages (accessed Feb 17, 2025 ). ↵ Hayase J , Liu A , Choi Y , Oh S , Smith NA . Data Mixture Inference: What do BPE Tokenizers Reveal about their Training Data? 2024 ; published online July 23. http://arxiv.org/abs/2407.16607 (accessed Feb 18, 2025 ). ↵ Liu X , Wu J , Shao A , et al. Uncovering Language Disparity of ChatGPT on Retinal Vascular Disease Classification: Cross-Sectional Study . J Med Internet Res 2024 ; 26 : e51926 . OpenUrl PubMed Lai VD , Ngo NT , Veyseh APB , et al. ChatGPT Beyond English: Towards a Comprehensive Evaluation of Large Language Models in Multilingual Learning . 2023 ; published online April 12. http://arxiv.org/abs/2304.05613 (accessed Jan 31, 2025 ). ↵ Sallam M , Al-Mahzoum K , Almutawaa RA , et al. The performance of OpenAI ChatGPT-4 and Google Gemini in virology multiple-choice questions: a comparative analysis of English and Arabic responses . BMC Res Notes 2024 ; 17 : 247 . OpenUrl PubMed ↵ Takagi S , Watari T , Erabi A , Sakaguchi K. Performance of GPT-3.5 and GPT-4 on the Japanese Medical Licensing Examination: Comparison Study . JMIR Med Educ 2023 ; 9 : e48002 . OpenUrl ↵ Wu J , Wu X , Qiu Z , et al. Large language models leverage external knowledge to extend clinical insight beyond language boundaries . J Am Med Inform Assoc 2024 ; 31 : 2054 – 64 . OpenUrl PubMed ↵ Jung LB , Gudera JA , Wiegand TLT , Allmendinger S , Dimitriadis K , Koerte IK . ChatGPT Passes German State Examination in Medicine With Picture Questions Omitted . Deutsches Arzteblatt international 2023 ; 120 . DOI: 10.3238/arztebl.m2023.0113 . OpenUrl CrossRef PubMed ↵ Accurate Diagnosis of Rare Diseases Remains Difficult Despite Strong Physician Interest . Global Genes . 2014 ; published online March 6. https://globalgenes.org/raredaily/accurate-diagnosis-of-rare-diseases-remains-difficult-despite-strong-physician-interest/ (accessed April 10, 2017 ). ↵ Haendel M , Vasilevsky N , Unni D , et al. How many rare diseases are there? Nat Rev Drug Discov 2020 ; 19 : 77 – 8 . OpenUrl PubMed ↵ Clark MM , Stark Z , Farnaes L , et al. Meta-analysis of the diagnostic and clinical utility of genome and exome sequencing and chromosomal microarray in children with suspected genetic diseases . NPJ Genom Med 2018 ; 3 : 16 . OpenUrl PubMed ↵ Kim J , Wang K , Weng C , Liu C. Assessing the utility of large language models for phenotype-driven gene prioritization in the diagnosis of rare genetic disease . Am J Hum Genet 2024 ; 111 : 2190 – 202 . OpenUrl CrossRef PubMed ↵ Vasilevsky NA , Matentzoglu NA , Toro S , et al. Mondo: Unifying diseases for the world, by the world . medRxiv . 2022 ; : 2022.04.13.22273750. ↵ Reese JT , Chimirri L , Bridges Y , et al. Systematic benchmarking demonstrates large language models have not reached the diagnostic accuracy of traditional rare-disease decision support tools . medRxiv . 2024 ; published online Nov 7. DOI: 10.1101/2024.07.22.24310816 . OpenUrl Abstract / FREE Full Text ↵ Gallifant J , Afshar M , Ameen S , et al. The TRIPOD-LLM reporting guideline for studies using large language models . Nature Medicine 2025 ; 31 : 60 – 9 . OpenUrl CrossRef PubMed ↵ Robinson PN , Köhler S , Bauer S , Seelow D , Horn D , Mundlos S. The Human Phenotype Ontology: a tool for annotating and analyzing human hereditary disease . Am J Hum Genet 2008 ; 83 : 610 – 5 . OpenUrl CrossRef PubMed Web of Science ↵ Gargano MA , Matentzoglu N , Coleman B , et al. The Human Phenotype Ontology in 2024: phenotypes around the world . Nucleic Acids Res 2024 ; 52 : D1333 – 46 . OpenUrl CrossRef PubMed ↵ Jacobsen JOB , Baudis M , Baynam GS , et al. The GA4GH Phenopacket schema defines a computable representation of clinical data . Nat Biotechnol 2022 ; 40 : 817 – 20 . OpenUrl CrossRef PubMed ↵ Danis D , Bamshad MJ , Bridges Y , et al. A corpus of GA4GH Phenopackets: case-level phenotyping for genomic diagnostics and discovery . HGG Adv . 2025 ; 6 : 100371 . OpenUrl PubMed ↵ Reese JT , Danis D , Harry Caufield J , et al. On the limitations of large language models in clinical diagnosis . medRxiv . 2024 ; : 2023.07.13.23292613. ↵ Soroush A , Glicksberg BS , Zimlichman E , et al. Large language models are poor medical coders — benchmarking of medical code querying . NEJM AI 2024 ; 1 . DOI: 10.1056/aidbp2300040 . OpenUrl CrossRef ↵ Azamfirei R , Kudchadkar SR , Fackler J. Large language models and the perils of their hallucinations . Crit Care 2023 ; 27 : 120 . OpenUrl CrossRef PubMed ↵ Caufield H , Kroll C , O’Neil ST , et al. CurateGPT: A flexible language-model assisted biocuration tool . 2024 ; published online Oct 29. http://arxiv.org/abs/2411.00046 (accessed Jan 17, 2025 ). ↵ Bridges Y , de Souza V , Cortes KG , et al. Towards a standard benchmark for variant and gene prioritisation algorithms: PhEval - Phenotypic inference Evaluation framework . bioRxiv . 2024 ;2024.06.13.598672. ↵ Kruskal WH , Allen Wallis W. Use of Ranks in One-Criterion Variance Analysis . Journal of the American Statistical Association 1952 ; published online Dec 1. https://www.tandfonline.com/doi/abs/10.1080/01621459.1952.10483441 (accessed Jan 29, 2025 ). ↵ Kanjee Z , Crowe B , Rodman A. Accuracy of a Generative Artificial Intelligence Model in a Complex Diagnostic Challenge . JAMA 2023 ; 330 : 78 – 80 . OpenUrl PubMed ↵ Evaluating large language models on medical, lay-language, and self-reported descriptions of genetic conditions . The American Journal of Human Genetics 2024 ; 111 : 1819 – 33 . OpenUrl PubMed ↵ Menezes MCS , Hoffmann AF , Tan ALM , et al. The potential of Generative Pre-trained Transformer 4 (GPT-4) to analyse medical notes in three different languages: a retrospective model-evaluation study . Lancet Digit Health 2025 ; 7 : e35 – 43 . OpenUrl ↵ Lewis P , Perez E , Piktus A , et al. Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks . 2020 ; published online May 22. http://arxiv.org/abs/2005.11401 (accessed Jan 31, 2025 ). ↵ OpenAI , :, Hurst A , et al. GPT-4o System Card . 2024 ; published online Oct 25. http://arxiv.org/abs/2410.21276 (accessed Jan 17, 2025 ). ↵ Peters DH , Garg A , Bloom G , Walker DG , Brieger WR , Rahman MH . Poverty and access to health care in developing countries . Ann N Y Acad Sci 2008 ; 1136 : 161 – 71 . OpenUrl CrossRef PubMed Web of Science ↵ Singhal K , Tu T , Gottweis J , et al. Toward expert-level medical question answering with large language models . Nat Med 2025 ; published online Jan 8. DOI: 10.1038/s41591-024-03423-7 . OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted February 28, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Consistent Performance of GPT-4o in Rare Disease Diagnosis Across Nine Languages and 4967 Cases Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Consistent Performance of GPT-4o in Rare Disease Diagnosis Across Nine Languages and 4967 Cases Leonardo Chimirri , J. Harry Caufield , Yasemin Bridges , Nicolas Matentzoglu , Michael Gargano , Mario Cazalla , Shihan Chen , Daniel Danis , Alexander JM Dingemans , Petra Gehle , Adam S.L. Graefe , Weihong Gu , Markus S. Ladewig , Pablo Lapunzina , Julián Nevado , Enock Niyonkuru , Soichi Ogishima , Dominik Seelow , Jair A Tenorio Castaño , Marek Turnovec , Bert BA de Vries , Kai Wang , Kyran Wissink , Zafer Yüksel , Gabriele Zucca , Melissa A. Haendel , Christopher J. Mungall , Justin Reese , Peter N. Robinson medRxiv 2025.02.26.25322769; doi: https://doi.org/10.1101/2025.02.26.25322769 Share This Article: Copy Citation Tools Consistent Performance of GPT-4o in Rare Disease Diagnosis Across Nine Languages and 4967 Cases Leonardo Chimirri , J. Harry Caufield , Yasemin Bridges , Nicolas Matentzoglu , Michael Gargano , Mario Cazalla , Shihan Chen , Daniel Danis , Alexander JM Dingemans , Petra Gehle , Adam S.L. Graefe , Weihong Gu , Markus S. Ladewig , Pablo Lapunzina , Julián Nevado , Enock Niyonkuru , Soichi Ogishima , Dominik Seelow , Jair A Tenorio Castaño , Marek Turnovec , Bert BA de Vries , Kai Wang , Kyran Wissink , Zafer Yüksel , Gabriele Zucca , Melissa A. Haendel , Christopher J. Mungall , Justin Reese , Peter N. Robinson medRxiv 2025.02.26.25322769; doi: https://doi.org/10.1101/2025.02.26.25322769 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (297) Cardiovascular Medicine (4421) Dentistry and Oral Medicine (443) Dermatology (382) Emergency Medicine (606) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1507) Epidemiology (15212) Forensic Medicine (30) Gastroenterology (1121) Genetic and Genomic Medicine (6581) Geriatric Medicine (667) Health Economics (996) Health Informatics (4520) Health Policy (1366) Health Systems and Quality Improvement (1611) Hematology (539) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15906) Intensive Care and Critical Care Medicine (1103) Medical Education (620) Medical Ethics (144) Nephrology (667) Neurology (6580) Nursing (345) Nutrition (998) Obstetrics and Gynecology (1141) Occupational and Environmental Health (956) Oncology (3324) Ophthalmology (970) Orthopedics (369) Otolaryngology (420) Pain Medicine (435) Palliative Medicine (129) Pathology (663) Pediatrics (1689) Pharmacology and Therapeutics (691) Primary Care Research (710) Psychiatry and Clinical Psychology (5433) Public and Global Health (9212) Radiology and Imaging (2193) Rehabilitation Medicine and Physical Therapy (1368) Respiratory Medicine (1194) Rheumatology (593) Sexual and Reproductive Health (709) Sports Medicine (529) Surgery (709) Toxicology (99) Transplantation (288) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9ff6bea53abcdf88',t:'MTc3OTM5OTY4MA=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();