Comparative Analysis of Multimodal Large Language Models GPT-4o and o1 vs Clinicians in Clinical Case Challenge Questions

Comparative Analysis of Multimodal Large Language Models GPT-4o and o1 vs Clinicians in Clinical Case Challenge Questions | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Comparative Analysis of Multimodal Large Language Models GPT-4o and o1 vs Clinicians in Clinical Case Challenge Questions Jaewon Jung , Hyunjae Kim , View ORCID Profile SungA Bae , Jin Young Park doi: https://doi.org/10.1101/2025.06.22.25330068 Jaewon Jung 1 Department of Medicine, Yonsei University College of Medicine , Seoul, Republic of Korea Find this author on Google Scholar Find this author on PubMed Search for this author on this site Hyunjae Kim 1 Department of Medicine, Yonsei University College of Medicine , Seoul, Republic of Korea Find this author on Google Scholar Find this author on PubMed Search for this author on this site SungA Bae 2 Department of Biomedical Systems Informatics, Yonsei University College of Medicine , Yongin, Republic of Korea 3 Division of Cardiology, Department of Internal Medicine, Yonsei University College of Medicine, Yongin Severance Hospital , Yongin, Republic of Korea Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for SungA Bae For correspondence: cardiobsa{at}yuhs.ac Jin Young Park 2 Department of Biomedical Systems Informatics, Yonsei University College of Medicine , Yongin, Republic of Korea 4 Department of Psychiatry, Yongin Severance Hospital, Yonsei University College of Medicine , Yongin, Republic of Korea Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: empathy{at}yuhs.ac Abstract Full Text Info/History Metrics Data/Code Preview PDF SUMMARY Background Generative Pre-trained Transformer 4 (GPT-4) has demonstrated strong performance in standardized medical examinations but has limitations in real-world clinical settings. The newly released multimodal GPT-4o model, which integrates text and image inputs to enhance diagnostic capabilities, and the multimodal o1 model, which incorporates advanced reasoning, may address these limitations. Objective This study aimed to compare the performance of GPT-4o and o1 against clinicians in real-world clinical case challenges. Methods This retrospective, cross-sectional study used Medscape case challenge questions from May 2011 to June 2024 (n = 1,426). Each case included text and images of patient history, physical examination findings, diagnostic test results, and imaging studies. Clinicians were required to choose one answer from among multiple options, with the most frequent response defined as the clinician’s decision. Data-based decisions were made using GPT models (3.5 Turbo, 4 Turbo, 4 Omni, and o1) to interpret the text and images, followed by a process to provide a formatted answer. We compared the performances of the clinicians and GPT models using Mixed-effects logistic regression analysis. Results Of the 1,426 questions, clinicians achieved an overall accuracy of 85.0%, whereas GPT-4o and o1 demonstrated higher accuracies of 88.4% and 94.3% (mean difference 3.4%; P = .005 and mean difference 9.3%; P < .001), respectively. In the multimodal performance analysis, which included cases involving images (n = 917), GPT-4o achieved an accuracy of 88.3%, and o1 achieved 93.9%, both significantly outperforming clinicians (mean difference 4.2%; P = .005 and mean difference 9.8%; P < .001). o1 showed the highest accuracy across all question categories, achieving 92.6% in diagnosis (mean difference 14.5%; P < .001), 97.0% in disease characteristics (mean difference 7.2%; P < .001), 92.6% in examination (mean difference 7.3%; P = .002), and 94.8% in treatment (mean difference 4.3%; P = .005), consistently outperforming clinicians. In terms of medical specialty, o1 achieved 93.6% accuracy in internal medicine (mean difference 10.3%; P < .001), 96.6% in major surgery (mean difference 9.2%; P = .030), 97.3% in psychiatry (mean difference 10.6%; P = .030), and 95.4% in minor specialties (mean difference 10.0%; P < .001), significantly surpassing clinicians. Across five trials, GPT-4o and o1 provided the correct answer 5/5 times in 86.2% and 90.7% of the cases, respectively. Conclusions The GPT-4o and o1 models achieved higher accuracy than clinicians in clinical case challenge questions, particularly in disease diagnosis. The GPT-4o and o1 could serve as valuable tools to assist healthcare professionals in clinical settings. INTRODUCTION Rapid advancements in artificial intelligence (AI) have transformed various sectors, including healthcare, where the importance of digital health technologies continues to grow. 1 – 3 Among these technologies, large language models (LLMs) have demonstrated considerable potential in healthcare, particularly in answering medical questions, assisting with diagnoses, and automating administrative tasks. 4 Generative Pre-trained Transformer 4 (GPT-4; OpenAI, San Francisco, CA, USA) is one of the most advanced LLMs, consistently achieving high scores on the United States Medical Licensing Examination and often exceeding the passing threshold. 5 – 7 Despite these accomplishments, existing evaluations have primarily focused on text-only formats that often exclude or replace visual elements with descriptions. This limitation reduces GPT-4’s applicability in image-dependent specialties, such as radiology. To address the limitations of unimodal LLMs, the recently released GPT-4 Omni introduces multimodal capabilities, enabling it to process and integrate both textual and visual information. 8 This enhancement allows for a more comprehensive analysis of clinical cases, particularly in fields such as diagnostic and interventional radiology and nuclear medicine, where early studies have demonstrated its potential. 9 , 10 Despite these advancements, existing research has largely focused on simplified exam questions, and no studies have reported significant performance improvements over text-only models in medical examinations. 11 , 12 This gap highlights the need to evaluate the performance of multimodal models in clinical case settings that reflect the complexity of diagnostic decision-making encountered in clinical practice. In parallel with these multimodal developments, OpenAI recently introduced the o1 model, which incorporates advanced reasoning capabilities designed to enhance safety and robustness. 13 By utilizing chain-of-thought reasoning, o1 has demonstrated improved adherence to safety protocols in complex scenarios, effectively reducing the risk of generating harmful or biased content. These enhanced reasoning capabilities are particularly important in healthcare, where complex problem-solving and data interpretation are critical. 14 Several studies have shown that the o1-preview outperforms both GPT-4 and GPT-4o models in clinical settings, highlighting its superior reasoning abilities in medical decision-making processes. 15 – 17 However, the performance of the o1 model in complex real-world clinical cases remains unexplored due to its recent release. Accordingly, this study aimed to evaluate the clinical potential of two advanced large language models, GPT-4 Omni and o1, in solving complex clinical case challenge questions that incorporate both textual and visual data and to validate their effectiveness as diagnostic support tools compared with clinicians. RESULTS Question characteristics We evaluated the performance of four language models, GPT-3.5 Turbo, GPT-4 Turbo, GPT-4 Omni, and o1, using a dataset of 1,426 cases. Of these, 917 questions included images, resulting in 1,475 images. Figure S1 in the Supplementary Appendix shows the quiz samples. The number of answer options per question ranged from two to nine, with the majority of cases (97.2%) offering either four or five answer options: 906 cases had four options and 480 had five. The random chance of selecting the correct answer was 23.1%. Overall accuracy and model comparison Clinicians demonstrated an accuracy of 85.0% across the entire dataset ( Figure 1A and Table S1 in the Supplementary Appendix ). In comparison, GPT-3.5 Turbo, GPT-4 Turbo, GPT-4o, and o1 achieved accuracies of 60.6% (95% confidence interval [CI], 58.1–63.1), 82.1% (95% CI, 80.1–84.0), 88.4% (95% CI, 86.8–90.1), and 94.3% (95% CI, 93.1–95.5), respectively. The performance of GPT-4o and o1 surpassed that of clinicians, demonstrating significantly higher accuracy ( P = .005 and P < .001, respectively). In contrast, GPT-3.5 Turbo and GPT-4 Turbo performed significantly lower than the clinicians ( P < .001 and P = .028, respectively). Download figure Open in new tab Figure 1. Overall performance of the GPT models (A) Accuracy of the models across the entire dataset (n = 1,426). Clinicians achieved an accuracy of 85.0%, while GPT-3.5 Turbo, GPT-4 Turbo, GPT-4 Omni, and o1 achieved accuracies of 60.6%, 82.1%, 88.4%, and 94.3%, respectively. Statistical comparisons showed that GPT-4o and o1 significantly outperformed clinicians (** P < .01; *** P < .001), while GPT-3.5 Turbo and GPT-4 Turbo performed significantly worse (*** P < .001; * P < .05). Error bars represent 95% confidence intervals. (B) Accuracy of models for the dataset subset with images (n = 917). Clinicians achieved an accuracy of 84.1%, while GPT-4o achieved 88.3%, significantly surpassing clinicians (** P < .01). o1 demonstrated an accuracy of 93.9%, significantly outperforming both clinicians and GPT-4o (*** P < .001 for both). Error bars represent 95% confidence intervals. GPT-4, Generative Pre-trained Transformer 4 To assess the potential impact of data exposure on model performance, we evaluated the accuracy of GPT-4o and o1 on datasets introduced both before (n = 1,219) and after (n = 207) the knowledge cutoff date of October 2023. Both models demonstrated comparable accuracy across the two time periods ( P = .646 for GPT-4o and P = .249 for o1), indicating that exposure to the test data had minimal to no impact on model accuracies ( Figure S2 in the Supplementary Appendix ). Multimodal model performances on cases with images In the subset of cases that included images (n = 917), clinicians achieved an accuracy of 84.1% ( Figure 1B ). GPT-4o performed better, with an accuracy of 88.3% (95% CI, 86.3–90.4), which was significantly higher than that of clinicians ( P = .005). o1 exhibited the highest performance, achieving an accuracy of 93.9% (95% CI, 92.3–95.4), significantly surpassing both clinicians and GPT-4o ( P < .001 for both). Performance by question category The dataset was stratified based on the question category, with the diagnosis category comprising the largest number of questions (n = 530), followed by disease characteristics (n = 401), treatment (n = 305), and examination (n = 190) ( Table 1 ). o1 demonstrated the highest accuracy across all question categories, consistently outperforming clinicians and other GPT models. Overall, it achieved 94.3% accuracy, significantly surpassing that of clinicians (85.0%, P < .001) and all other models. View this table: View inline View popup Download powerpoint Table 1. Performance by Question Category In the diagnosis and disease characteristics categories, o1 achieved 92.6% and 97.0% accuracy, respectively, significantly outperforming clinicians ( P < .001 for both). In the examination category, o1 achieved 92.6% accuracy, surpassing that of clinicians, who achieved 85.3% accuracy ( P = .002). In the treatment category, o1 also outperformed clinicians, achieving 94.8% accuracy compared to 90.5% for clinicians ( P = .005). In the diagnosis category, GPT-4o significantly outperformed clinicians, achieving 84.9% accuracy compared to 78.1% for clinicians ( P = .002). For categories related to disease characteristics and examination, GPT-4o also demonstrated higher accuracy than clinicians, although the differences were not statistically significant ( P = .261 and P = .105, respectively). In the treatment category, clinicians showed greater accuracy than GPT-4o, but the difference was not statistically significant ( P = .715). Performance by medical specialty The dataset was stratified by medical specialty, with internal medicine comprising 766 questions, followed by minor specialties (n = 390), pediatrics (n = 108), major surgery (n = 87), and psychiatry (n = 75) ( Table 2 ). o1 achieved 93.6% accuracy in internal medicine and 95.4% in minor specialties, which were statistically significant compared to clinicians (p < .001 for both). In major surgery, o1 outperformed clinicians with 96.6% accuracy, while clinicians achieved 87.4% accuracy ( P = .030). In psychiatry, o1 achieved an accuracy of 97.3%, surpassing that of clinicians, who achieved 86.7% accuracy ( P = .030). Clinicians demonstrated numerically higher performance in pediatrics than in o1, but the difference was not statistically significant ( P = .798). View this table: View inline View popup Table 2. Performance by Medical Specialty GPT-4o demonstrated superior accuracy in internal medicine, with 86.9% compared to 83.3% for clinicians ( P = .037). Similarly, in minor specialties, GPT-4o outperformed clinicians, achieving 91.5% accuracy compared to 85.4% ( P = .005). In addition, GPT-4o showed higher accuracy than clinicians in major surgeries, although the difference was not statistically significant ( P = .802). In contrast, clinicians demonstrated higher accuracy in pediatrics and psychiatry, but these differences were not statistically significant ( P = .617 and P = .645, respectively). Consistency of responses GPT-4o and o1 were evaluated using the entire dataset over five independent trials to assess response consistency ( Figure 2 ). GPT-4o achieved the correct answer 5/5 times in 86.2% of the cases, while o1 demonstrated a slightly higher consistency, with the correct answer 5/5 times in 90.7% of the cases. Notably, GPT-4o answered correctly at least once in 90.3% of the cases across the five trials, whereas o1 achieved at least one correct answer in 95.6% of the cases. Download figure Open in new tab Figure 2. Response consistency of GPT-4o and o1 (A) Performance of GPT-4o across five independent trials, showing that 86.2% of cases had all correct responses (5/5). GPT-4o achieved at least one correct response in 90.3% of cases across the trials. (B) Performance of o1 across five independent trials, demonstrating slightly higher consistency, with 90.7% of cases achieving all correct responses (5/5). o1 achieved at least one correct response in 95.6% of cases across the trials. GPT-4, Generative Pre-trained Transformer 4 DISCUSSION Principal results This study demonstrated that recent GPT models, specifically o1 and GPT-4o, can outperform clinicians in clinical case challenges. Comparison with prior work Previous studies using earlier models, such as GPT-3.5 Turbo and GPT-4 Turbo, generally found that the performance on medical cases was comparable to or below that of humans. 18 , 19 A prior study using Medscape clinical case challenges with a text-only model reported performance below 50%, highlighting significant limitations. 20 In contrast, our study utilized multimodal models, resulting in substantial performance improvements. Both GPT-4o and o1 maintained high accuracy in cases involving images, demonstrating their ability to interpret various imaging types, including magnetic resonance imaging (MRI), computed tomography (CT), X-ray, ultrasound, and pathological images, across multiple specialties. Additionally, earlier research has shown that the o1-preview model outperformed the GPT-4 series in solving medical problems. 14 , 16 , 17 Our findings further support this observation, as o1 outperformed GPT-4o in complex medical scenarios. The accuracy of o1 remained above 90% across all question categories and medical specialties, with particular strength in diagnosis. The model also performed well in areas such as major surgery and psychiatry, with accuracy rates exceeding those of GPT-4o by more than 8%. Research suggests that o1’s high performance is driven by its “chain-of-thought” reasoning, which systematically breaks down complex problems, reduces hallucinations, and improves logical reasoning. 13 , 14 This capability is especially beneficial in clinical cases with abundant patient data, which often include extraneous information. In such contexts, o1’s ability to filter relevant insights and synthesize data across specialties aids in understanding disease progression and managing complex multisystem conditions. The o1 model demonstrated stronger performance and greater consistency than GPT-4o in selecting the most accurate option from multiple choices. This finding contrasts with previous studies suggesting similar gains between the o1 and GPT-4 models in probabilistic reasoning and critical diagnosis identification. 21 This divergence may be attributed to differences in prompt design and parameter tuning, which enabled o1 to achieve a significant improvement. However, it also suggests that o1’s performance may vary depending on the clinical setting. Integration into clinical workflows and application While the study shows that AI models outperform clinicians in accuracy, their reasoning process remains a black box. Understanding how these models reach their conclusions and making this process interpretable for clinicians is a critical consideration for integration into clinical workflows. Figure 3 illustrates how the AI model interprets imaging studies and synthesizes differential diagnoses. This approach enables clinicians to verify, understand, and critically assess AI recommendations, thereby facilitating safer and more informed decision-making. Download figure Open in new tab Figure 3. Multimodal Reasoning Process of GPT in a Clinical Case This figure illustrates how the GPT model integrates textual and visual information to interpret a clinical case. The GPT model is prompted to identify the most likely diagnosis, explain its reasoning, and discuss differential diagnoses. GPT, Generative Pre-trained Transformer In practical applications, GPT models may serve as valuable tools to support clinicians, particularly in settings where specialized expertise is limited. Clinicians are prone to diagnostic errors and may occasionally order unnecessary tests, influenced by factors such as financial incentives while lacking adequate tools to monitor such decisions. 22 – 25 Recent models demonstrate potential in assisting with differential diagnosis and initial treatment planning, especially in fields outside a physician’s primary specialty. 26 Furthermore, GPT models show promise in identifying rare diseases that might otherwise be overlooked and in recommending appropriate referrals to specialized care. 27 In medically underserved areas or environments with limited access to specialist consultation, AI systems can help reduce healthcare disparities by providing consistent decision support and expanding diagnostic reach. 28 Remaining challenges and ethical considerations Although AI demonstrates high diagnostic accuracy, the role of clinicians remains indispensable. The selection, preprocessing, and input of relevant patient information into AI systems depend heavily on clinician expertise. 29 Moreover, while the models have demonstrated impressive performance based on the provided case information, real clinical settings are far more complex. 30 Patients often provide incomplete histories or omit critical information, presenting substantial challenges that current LLMs are not yet fully equipped to manage. 31 Further prospective studies using real-world clinical data, including free-form questions and open-ended scenarios, are recommended to evaluate the robustness and generalizability of these models. As AI technologies, including large multimodal models capable of analyzing imaging data, become increasingly integrated into clinical workflows, new ethical considerations must be addressed. In addition to traditional concerns about accountability, transparency, and bias in algorithmic decision-making, the inclusion of sensitive medical images heightens the importance of patient privacy and data protection. 32 Furthermore, establishing clear legal frameworks is essential to delineate responsibility in cases where AI-assisted decisions result in misdiagnosis. 33 Despite the substantial capabilities demonstrated by these models, it remains crucial to recognize that AI systems currently lack the emotional understanding, empathy, and holistic grasp of individual patient cases that clinicians possess; thus, human intervention and interaction remain essential. Limitations of the study Our study has certain limitations. First, clinicians were defined as site voters rather than as a standardized group of certified specialists, and the voting participants included a broad range of users. The absence of participant credentials and the informal nature of responses, provided without academic or clinical consequences, may have contributed to an underestimation of actual clinician performance. Despite these limitations, the primary aim of the study was not to benchmark against definitive human performance but to illustrate the rapid and consistent progress of LLMs in performing medical tasks. Given the observed performance trajectory suggesting increasing clinical applicability, further studies involving board-certified clinicians are warranted to evaluate the models’ validity in real-world settings. Second, the dataset was derived from Medscape clinical case challenges, which may not fully capture the diversity and complexity of real-world scenarios. Additionally, because the clinical case dataset is open-source, it is possible that the GPT models were exposed to specific cases during training. However, o1 and GPT-4o were trained only on data available until October 2023, and their accuracy rates showed no statistically significant differences between datasets introduced before and after this date, suggesting that any prior exposure had a limited impact on model accuracy. Third, the dataset may have been influenced by inherent bias. This includes selection bias, as Medscape’s educational objectives often prioritize complex or rare conditions, leading to their preferential selection over common clinical presentations. Furthermore, latent biases, such as the overrepresentation of specific medical conditions, demographic groups, or clinical scenarios, may have limited the dataset’s generalizability and reduced its ability to reflect real-world patient diversity. 34 CONCLUSIONS The o1 and GPT-4o models achieved higher accuracy than clinicians on clinical case challenge questions, including those involving medical imaging. These findings suggest that both models can serve as valuable tools for healthcare professionals, supporting various aspects of patient care and decision-making in clinical settings. Data Availability All data produced are available online at https://reference.medscape.com/features/casechallenges https://reference.medscape.com/features/casechallenges RESOURCE AVAILABILITY Lead contact Further information and requests for resources and reagents should be directed to and will be fulfilled by the lead contact, SungA Bae ( cardiobsa{at}yuhs.ac ). Materials availability This study did not generate new materials. Data and code availability The publicly available Medscape case challenge questions used for this analysis are available for download at the cited sources: https://reference.medscape.com/features/casechallenges . The underlying code and sample datasets used in this study are publicly available in the following GitHub repository: https://github.com/makogirls/GPT-medscape-case-challenge . AUTHOR CONTRIBUTIONS Conceptualization, S. Bae and J.Y. Park; funding acquisition, S. Bae and J.Y. Park; supervision, S. Bae and J.Y. Park; data curation, J. Jung and H. Kim; formal analysis, J. Jung and H. Kim; investigation, J. Jung and H. Kim; programming, J. Jung and H. Kim; verification, J. Jung and H. Kim; visualization, J. Jung and H. Kim; writing—original draft, all authors; writing—review & editing, all authors; final approval, all authors; accountability, all authors. DECLARATION OF INTERESTS None. DECLARATION OF GENERATIVE AI AND AI-ASSISTED TECHNOLOGIES During the preparation of this manuscript, GPT-4o and o1 (OpenAI, 2025) were used as supplementary tools to assist with translation, refine sentence structure, and generate vector images for figures. These tools were not involved in generating original scientific content, nor did they contribute to the interpretation of the study results. SUPPLEMENTAL INFORMATION Document S1. Figures S1–S2 and Tables S1–S2 Figure S1. Medscape quiz sample Figure S2. Model Performances Before and After the Knowledge Cutoff Date Table S1. Comparison of clinician and GPT model responses Table S2. Medscape quiz information STAR⍰ METHODS KEY RESOURCES TABLE View this table: View inline View popup Download powerpoint EXPERIMENTAL MODEL AND STUDY PARTICIPANT DETAILS Study Design This retrospective study analyzed Medscape clinical case challenge questions from May 2011 to June 2024 ( Figure 4 and Table S2 in the Supplementary Appendix ). 35 Medscape clinical case challenges are interactive educational tools designed to help healthcare professionals assess and improve their clinical knowledge and decision-making skills. Exclusion criteria included non-question contents and multiple choice questions. Case data were collected using a custom Python-based web scraper that retrieved publicly available content from the Medscape website. Only publicly accessible cases were included, and no personal data were collected. Ethical approval was not required due to the retrospective nature of the study and the use of open-source data. Download figure Open in new tab Figure 4. Study flowchart This flowchart illustrates the workflow for evaluating the diagnostic performances of GPT models compared with clinicians in Medscape case challenge questions. GPT, Generative Pre-trained Transformer Each case presented a realistic clinical scenario, including patient history, physical examination findings, diagnostic test results, and imaging studies. Participants were tasked with selecting the most appropriate answer from a set of multiple-choice options covering aspects of diagnosis, disease characteristics, examinations, and treatment strategies. Clinicians’ responses were defined as the most frequently selected option for each question. Each question had a predefined correct answer established by the Medscape case author based on the confirmed diagnosis and clinical guidelines. The cases were categorized by medical specialty, including internal medicine (allergy, cardiology, endocrinology, gastroenterology, hematology, infectious diseases, nephrology, oncology, pulmonology, and rheumatology), major surgery (obstetrics/gynecology and surgery), pediatrics, psychiatry, and minor specialties (dermatology, emergency medicine, neurology, neurosurgery, ophthalmology, orthopedics, otorhinolaryngology, and urology). METHOD DETAILS Large Language Models and Prompt Each GPT model (GPT-3.5 Turbo, GPT-4 Turbo, GPT-4 Omni, and o1) was accessed through the OpenAI application programming interface (API) between August and December 2024. Eash model was tasked to generate a formatted response based on the options available for each question. To ensure consistency and replicability, the models were queried using predefined prompt templates that simulated the clinical reasoning scenarios. Table 3 outlines the prompts used to query the models, including their structures and formatting. For questions that included images, both textual and visual data were provided to the models, where applicable, enabling multimodal input processing in GPT-4o and o1. The original images uploaded by Medscape were used without any preprocessing. The chat session was reset before each new question to eliminate the influence of memory retention or in-context learning, ensuring that all questions were processed independently and that responses remained unbiased. GPT-3.5 Turbo, GPT-4 Turbo, and GPT-4 Omni were configured with a temperature of 0.1 and top-p of 1.0, while o1 used its default settings of a temperature of 1.0 and top-p of 1.0. View this table: View inline View popup Download powerpoint Table 3. Prompt template for GPT models QUANTIFICATION AND STATISTICAL ANALYSIS We compared the performances of the four GPT models (GPT-3.5 Turbo, GPT-4 Turbo, GPT-4 Omni, and o1) with those of clinicians using mixed-effects logistic regression analysis to evaluate differences in accuracy. The model type was included as a fixed effect to compare performance across models, and a random intercept was added for each case to account for potential correlations among nested questions within the same case. We calculated the accuracy and 95% confidence intervals (CIs) for each model and clinician and performed pairwise comparisons between them. For cases with images, a focused analysis of GPT-4o and o1 was conducted to specifically assess their multimodal capability in interpreting visual data. The dataset was further analyzed by question category (diagnosis, characteristics, examination, and treatment) and medical specialty (internal medicine, major surgery, pediatrics, psychiatry, and minor specialties). GPT-4o and o1 were tested over five independent trials using the same set of questions to evaluate response consistency. All analyses were conducted using R (version 4.5.0, R Foundation for Statistical Computing, Vienna, Austria), with a P -value < .05 considered statistically significant. ACKNOWLEDGMENTS This research was supported by a grant from the Korea Health Technology R&D Project through the Korea Health Industry Development Institute (KHIDI), funded by the Ministry of Health and Welfare, Republic of Korea (grant number RS-2023-KH135442). Additionally, this study was supported by a New Faculty Research Seed Money Grant from Yonsei University College of Medicine for 2025 (2025-32-0022). Footnotes ↵ 6 Lead contact REFERENCES 1. ↵ Topol , E.J. ( 2019 ). High-performance medicine: the convergence of human and artificial intelligence . Nat Med 25 , 44 – 56 . doi: 10.1038/s41591-018-0300-7 . OpenUrl CrossRef PubMed 2. Yoon , D. ( 2022 ). Preparing for a New World: Making Friends with Digital Health . Yonsei Med J 63 , S108 – S111 . doi: 10.3349/ymj.2022.63.S108 . OpenUrl CrossRef PubMed 3. ↵ Lim , C.Y. , Sohn , B. , Seong , M. , Kim , E.Y. , Kim , S.T. , and Won , S.Y. ( 2024 ). Need for Transparency and Clinical Interpretability in Hemorrhagic Stroke Artificial Intelligence Research: Promoting Effective Clinical Application . Yonsei Med J 65 , 611 – 618 . doi: 10.3349/ymj.2024.0007 . OpenUrl CrossRef PubMed 4. ↵ The Lancet Digital Health . ( 2024 ). Large language models: a new chapter in digital health . Lancet Digit Health 6 , e1 . doi: 10.1016/S2589-7500(23)00254-6 . OpenUrl CrossRef 5. ↵ Shieh , A. , Tran , B. , He , G. , Kumar , M. , Freed , J.A. , and Majety , P. ( 2024 ). Assessing ChatGPT 4.0’s test performance and clinical diagnostic accuracy on USMLE STEP 2 CK and clinical case reports . Sci Rep 14 , 9330 . doi: 10.1038/s41598-024-58760-x . OpenUrl CrossRef PubMed 6. Brin , D. , Sorin , V. , Vaid , A. , Soroush , A. , Glicksberg , B.S. , Charney , A.W. , Nadkarni , G. , and Klang , E. ( 2023 ). Comparing ChatGPT and GPT-4 performance in USMLE soft skill assessments . Sci Rep 13 , 16492 . doi: 10.1038/s41598-023-43436-9 . OpenUrl CrossRef PubMed 7. ↵ Nori , H. , King , N. , McKinney , S.M. , Carignan , D. , and Horvitz , E. Capabilities of GPT-4 on medical challenge problems . Preprint at arXiv , doi: 10.48550/arXiv.2303.13375 . OpenUrl CrossRef 8. ↵ OpenAI . GPT-4o System Card [Internet] . 2024 . Available from: https://openai.com/index/gpt-4o-system-card/ . Accessed 2025 Jun 18 . 9. ↵ Sonoda , Y. , Kurokawa , R. , Nakamura , Y. , Kanzawa , J. , Kurokawa , M. , Ohizumi , Y. , Gonoi , W. , and Abe , O. ( 2024 ). Diagnostic performances of GPT-4o, Claude 3 Opus, and Gemini 1.5 Pro in “Diagnosis Please” cases . Jpn J Radiol 42 , 1231 – 1235 . doi: 10.1007/s11604-024-01619-y . OpenUrl CrossRef 10. ↵ Ebel , S. , Ehrengut , C. , Denecke , T. , Gossmann , H. , and Beeskow , A.B. ( 2024 ). GPT-4o’s competency in answering the simulated written European Board of Interventional Radiology exam compared to a medical student and experts in Germany and its ability to generate exam items on interventional radiology: a descriptive study . J Educ Eval Health Prof 21 , 21 . doi: 10.3352/jeehp.2024.21.21 . OpenUrl CrossRef PubMed 11. ↵ Hirano , Y. , Hanaoka , S. , Nakao , T. , Miki , S. , Kikuchi , T. , Nakamura , Y. , Nomura , Y. , Yoshikawa , T. , and Abe , O. ( 2024 ). GPT-4 Turbo with Vision fails to outperform text-only GPT-4 Turbo in the Japan Diagnostic Radiology Board Examination . Jpn J Radiol 42 , 918 – 926 . doi: 10.1007/s11604-024-01561-z . OpenUrl CrossRef 12. ↵ Nakao , T. , Miki , S. , Nakamura , Y. , Kikuchi , T. , Nomura , Y. , Hanaoka , S. , Yoshikawa , T. , and Abe , O. ( 2024 ). Capability of GPT-4V(ision) in the Japanese National Medical Licensing Examination: Evaluation Study . JMIR Med Educ 10 , e54393 . doi: 10.2196/54393 . OpenUrl CrossRef PubMed 13. ↵ OpenAI . OpenAI o1 System Card [Internet] . 2024 . Available from: https://openai.com/index/openai-o1-system-card/ . Accessed 2025 Jun 18 14. ↵ Temsah , M.H. , Jamal , A. , Alhasan , K. , Temsah , A.A. , and Malki , K.H. ( 2024 ). OpenAI o1-Preview vs. ChatGPT in Healthcare: A New Frontier in Medical AI Reasoning . Cureus 16 , e70640 . doi: 10.7759/cureus.70640 . OpenUrl CrossRef 15. ↵ Xie , Y. , Wu , J. , Tu , H. , Yang , S. , Zhao , B. , and Zong , Y. A preliminary study of o1 in medicine: are we closer to an AI doctor? Preprint at arXiv , doi: 10.48550/arXiv.2409.15277 . OpenUrl CrossRef 16. ↵ Nori , H. , Usuyama , N. , King , N. , McKinney , S.M. , Fernandes , X. , and Zhang , S. From Medprompt to o1: exploration of run-time strategies for medical challenge problems and beyond . Preprint at arXiv , doi: 10.48550/arXiv.2411.03590 . OpenUrl CrossRef 17. ↵ Xu , S. , Zhou , Y. , Liu , Z. , Wu , Z. , Zhong , T. , and Zhao , H. Towards next-generation medical agent: how o1 is reshaping decision-making in medical scenarios . Preprint at arXiv , doi: 10.48550/arXiv.2411.14461 . OpenUrl CrossRef 18. ↵ Liu , C.L. , Ho , C.T. , and Wu , T.C. ( 2024 ). Custom GPTs Enhancing Performance and Evidence Compared with GPT-3.5, GPT-4, and GPT-4o? A Study on the Emergency Medicine Specialist Examination . Healthcare (Basel) 12 . doi: 10.3390/healthcare12171726 . OpenUrl CrossRef PubMed 19. ↵ Barile , J. , Margolis , A. , Cason , G. , Kim , R. , Kalash , S. , Tchaconas , A. , and Milanaik , R. ( 2024 ). Diagnostic Accuracy of a Large Language Model in Pediatric Case Studies . JAMA Pediatr 178 , 313 – 315 . doi: 10.1001/jamapediatrics.2023.5750 . OpenUrl CrossRef PubMed 20. ↵ Hadi , A. , Tran , E. , Nagarajan , B. , and Kirpalani , A. ( 2024 ). Evaluation of ChatGPT as a diagnostic tool for medical learners and clinicians . PLoS One 19 , e0307383 . doi: 10.1371/journal.pone.0307383 . OpenUrl CrossRef PubMed 21. ↵ Brodeur , P.G. , Buckley , T.A. , Kanjee , Z. , Goh , E. , Ling , E.B. , and Jain , P. Superhuman performance of a large language model on the reasoning tasks of a physician . Preprint at arXiv , doi: 10.48550/arXiv.2412.10849 . OpenUrl CrossRef 22. ↵ Singh , H. , Giardina , T.D. , Meyer , A.N. , Forjuoh , S.N. , Reis , M.D. , and Thomas , E.J. ( 2013 ). Types and origins of diagnostic errors in primary care settings . JAMA Intern Med 173 , 418 – 425 . doi: 10.1001/jamainternmed.2013.2777 . OpenUrl CrossRef PubMed 23. Newman-Toker , D.E. , Schaffer , A.C. , Yu-Moe , C.W. , Nassery , N. , Saber Tehrani , A.S. , Clemens , G.D. , Wang , Z. , Zhu , Y. , Fanai , M. , and Siegal , D. ( 2019 ). Serious misdiagnosis-related harms in malpractice claims: The “Big Three” - vascular events, infections, and cancers . Diagnosis (Berl) 6 , 227 – 240 . doi: 10.1515/dx-2019-0019 . OpenUrl CrossRef PubMed 24. Bindraban , R.S. , Ten Berg , M.J. , Naaktgeboren , C.A. , Kramer , M.H.H. , Van Solinge , W.W. , and Nanayakkara , P.W.B. ( 2018 ). Reducing Test Utilization in Hospital Settings: A Narrative Review . Ann Lab Med 38 , 402 – 412 . doi: 10.3343/alm.2018.38.5.402 . OpenUrl CrossRef PubMed 25. ↵ Brownlee , S. , Chalkidou , K. , Doust , J. , Elshaug , A.G. , Glasziou , P. , Heath , I. , Nagpal , S. , Saini , V. , Srivastava , D. , Chalmers , K. , and Korenstein , D. ( 2017 ). Evidence for overuse of medical services around the world . Lancet 390 , 156 – 168 . doi: 10.1016/S0140-6736(16)32585-5 . OpenUrl CrossRef PubMed 26. ↵ Rao , A. , Pang , M. , Kim , J. , Kamineni , M. , Lie , W. , Prasad , A.K. , Landman , A. , Dreyer , K. , and Succi , M.D. ( 2023 ). Assessing the Utility of ChatGPT Throughout the Entire Clinical Workflow: Development and Usability Study . J Med Internet Res 25 , e48659 . doi: 10.2196/48659 . OpenUrl CrossRef PubMed 27. ↵ Du , X. , Zhou , Z. , Wang , Y. , Chuang , Y.W. , Yang , R. , Zhang , W. , Wang , X. , Zhang , R. , Hong , P. , Bates , D.W. , and Zhou , L. ( 2024 ). Generative Large Language Models in Electronic Health Records for Patient Care Since 2023: A Systematic Review . medRxiv 2024 . doi: 10.1101/2024.08.11.24311828 . OpenUrl Abstract / FREE Full Text 28. ↵ Dankwa-Mullan , I. ( 2024 ). Health Equity and Ethical Considerations in Using Artificial Intelligence in Public Health and Medicine . Prev Chronic Dis 21 , E64 . doi: 10.5888/pcd21.240245 . OpenUrl CrossRef PubMed 29. ↵ Chew , B.H. , and Ngiam , K.Y. ( 2025 ). Artificial intelligence tool development: what clinicians need to know? BMC Med 23 , 244 . doi: 10.1186/s12916-025-04076-0 . OpenUrl CrossRef PubMed 30. ↵ Ahmed , M.I. , Spooner , B. , Isherwood , J. , Lane , M. , Orrock , E. , and Dennison , A. ( 2023 ). A Systematic Review of the Barriers to the Implementation of Artificial Intelligence in Healthcare . Cureus 15 , e46454 . doi: 10.7759/cureus.46454 . OpenUrl CrossRef 31. ↵ Roustan , D. , and Bastardot , F. ( 2025 ). The Clinicians’ Guide to Large Language Models: A General Perspective With a Focus on Hallucinations . Interact J Med Res 14 , e59823 . doi: 10.2196/59823 . OpenUrl CrossRef 32. ↵ J, B.R ., Sood , A. , Pattnaik , T. , Malhotra , R. , Nayyar , V. , Narayan , B. , Mishra , D. , and Surya , V. ( 2025 ). Medical imaging privacy: A systematic scoping review of key parameters in dataset construction and data protection . J Med Imaging Radiat Sci 56 , 101914 . doi: 10.1016/j.jmir.2025.101914 . OpenUrl CrossRef PubMed 33. ↵ Cestonaro , C. , Delicati , A. , Marcante , B. , Caenazzo , L. , and Tozzo , P. ( 2023 ). Defining medical liability when artificial intelligence is applied on diagnostic algorithms: a systematic review . Front Med (Lausanne) 10 , 1305756 . doi: 10.3389/fmed.2023.1305756 . OpenUrl CrossRef 34. ↵ Obermeyer , Z. , Powers , B. , Vogeli , C. , and Mullainathan , S. ( 2019 ). Dissecting racial bias in an algorithm used to manage the health of populations . Science 366 , 447 – 453 . doi: 10.1126/science.aax2342 . OpenUrl Abstract / FREE Full Text 35. ↵ Clinical challenges . Medscape website . https://reference.medscape.com/features/casechallenges . Accessed 2025 Jun 18 View the discussion thread. Back to top Previous Next Posted June 23, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Comparative Analysis of Multimodal Large Language Models GPT-4o and o1 vs Clinicians in Clinical Case Challenge Questions Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Comparative Analysis of Multimodal Large Language Models GPT-4o and o1 vs Clinicians in Clinical Case Challenge Questions Jaewon Jung , Hyunjae Kim , SungA Bae , Jin Young Park medRxiv 2025.06.22.25330068; doi: https://doi.org/10.1101/2025.06.22.25330068 Share This Article: Copy Citation Tools Comparative Analysis of Multimodal Large Language Models GPT-4o and o1 vs Clinicians in Clinical Case Challenge Questions Jaewon Jung , Hyunjae Kim , SungA Bae , Jin Young Park medRxiv 2025.06.22.25330068; doi: https://doi.org/10.1101/2025.06.22.25330068 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Systems and Quality Improvement Subject Areas All Articles Addiction Medicine (569) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4442) Dentistry and Oral Medicine (444) Dermatology (383) Emergency Medicine (609) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1511) Epidemiology (15230) Forensic Medicine (30) Gastroenterology (1126) Genetic and Genomic Medicine (6610) Geriatric Medicine (668) Health Economics (998) Health Informatics (4542) Health Policy (1370) Health Systems and Quality Improvement (1613) Hematology (543) HIV/AIDS (1266) Infectious Diseases (except HIV/AIDS) (15923) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (147) Nephrology (668) Neurology (6607) Nursing (346) Nutrition (999) Obstetrics and Gynecology (1146) Occupational and Environmental Health (957) Oncology (3338) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1693) Pharmacology and Therapeutics (692) Primary Care Research (712) Psychiatry and Clinical Psychology (5448) Public and Global Health (9239) Radiology and Imaging (2202) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (596) Sexual and Reproductive Health (714) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a01ef9babc07ad07',t:'MTc3OTgyMTUzMQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();