Not Such a Long Way Off? Contemporary Artificial intelligence Performance Evaluation on Adult Medicine Long Cases

Not Such a Long Way Off? Contemporary Artificial intelligence Performance Evaluation on Adult Medicine “Long Cases” | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Not Such a Long Way Off? Contemporary Artificial intelligence Performance Evaluation on Adult Medicine “Long Cases” Christina Gao , Jamie Bellinge , Shaddy El-Masri , Ivana Chim , View ORCID Profile Michael Sorich , Ishish Seth , James Gorcilov , Matthew Lim , Liam McCoy , Andrew Vanlint , Lauren Lim , Jessica Stranks , View ORCID Profile Andrew Zannettino , John Maddison , View ORCID Profile Stephen Bacchi doi: https://doi.org/10.1101/2025.01.25.25321118 Christina Gao 1 Adelaide Medical School, The University of Adelaide , Adelaide SA 5005, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site Jamie Bellinge 2 Harvard Medical School, Harvard University , Boston, MA 02138, USA 3 Massachusetts General Hospital , Boston, MA 02138, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Shaddy El-Masri 4 Flinders University , Bedford Park SA 5042, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ivana Chim 1 Adelaide Medical School, The University of Adelaide , Adelaide SA 5005, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site Michael Sorich 4 Flinders University , Bedford Park SA 5042, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Michael Sorich Ishish Seth 5 Peninsula Health , Melbourne VIC 3199, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site James Gorcilov 6 Lyell McEwin Hospital , Elizabeth Vale SA 5112, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site Matthew Lim 1 Adelaide Medical School, The University of Adelaide , Adelaide SA 5005, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site Liam McCoy 7 Massachusetts Institute of Technology , Boston, MA 02139, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Andrew Vanlint 1 Adelaide Medical School, The University of Adelaide , Adelaide SA 5005, Australia 6 Lyell McEwin Hospital , Elizabeth Vale SA 5112, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site Lauren Lim 1 Adelaide Medical School, The University of Adelaide , Adelaide SA 5005, Australia 6 Lyell McEwin Hospital , Elizabeth Vale SA 5112, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site Jessica Stranks 1 Adelaide Medical School, The University of Adelaide , Adelaide SA 5005, Australia 6 Lyell McEwin Hospital , Elizabeth Vale SA 5112, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site Andrew Zannettino 1 Adelaide Medical School, The University of Adelaide , Adelaide SA 5005, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Andrew Zannettino John Maddison 1 Adelaide Medical School, The University of Adelaide , Adelaide SA 5005, Australia 6 Lyell McEwin Hospital , Elizabeth Vale SA 5112, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site Stephen Bacchi 1 Adelaide Medical School, The University of Adelaide , Adelaide SA 5005, Australia 2 Harvard Medical School, Harvard University , Boston, MA 02138, USA 3 Massachusetts General Hospital , Boston, MA 02138, USA 4 Flinders University , Bedford Park SA 5042, Australia 6 Lyell McEwin Hospital , Elizabeth Vale SA 5112, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Stephen Bacchi For correspondence: stephen.bacchi{at}sa.gov.au Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF ABSTRACT The Royal Australasian College of Physicians (RACP) “Long Cases” assess a trainee’s clinical reasoning beyond what is tested in multiple-choice questions, which large language models (LLMs) have already demonstrated proficiency in. This study evaluated a LLM’s ability to perform a “Long Case” assessment, including history-taking, case presentation, and answering examiner questions. The LLM achieved passing consensus scores of 4-5 out of 6 on five cases, suggesting potential for LLMs in complex clinical evaluations. Introduction Contemporary artificial intelligence (AI), in particular large language models (LLM), are becoming increasingly performant on medical tasks. 1 - 4 However, these evaluations often pertain to multiple choice examination questions or curated vignettes. 5 - 7 The Royal Australasian College of Physicians (RACP) Divisional Clinical Examinations (DCE) must be passed prior to advanced specialty training. One component of the RACP DCE are “Long Cases”, in which a trainee has one hour with a complex patient to complete a comprehensive evaluation before presenting the patient and their pertinent issues to a set of examiners and answering questions regarding the patient. These “Long Cases” require trainees to demonstrate higher order thought processes, including prioritisation of competing issues and a wholistic approach. Accordingly, evaluation of AI on such an assessment may provide insights into their current capabilities. The aim of this study was to evaluate the performance of AI, namely an LLM, undertaking DCE “Long Cases”. Methods In this cross-sectional study, an LLM was tested on five hypothetical “Long Cases,” which included three stages: (a) history-taking, (b) synthesizing and presenting the case, and (c) answering examiner questions. Initially, five hypothetical cases were created by a consultant medical officer. These cases covered common “Long case” medical and psychosocial issues. The cases included conditions not explicitly mentioned in the trainee notes provided in the prompts (see Supplementary Information 1 – Hypothetical “Long Cases”). Three sets of prompts were developed to guide the LLM through the history-taking, case presentation, and question-answering stages (see Supplementary Information 2 – Prompts). As the LLM could not perform a physical examination, the cases were simulated as if conducted via phone, similar to how exams were held during the COVID-19 pandemic, with physical examination details provided separately. The evaluation of the LLM was undertaken in a stepwise process (see Figure 1 ). The LLM, Claude 3.5 Sonnet from Anthropic , received the “history” prompt along with basic patient details (name, age, gender, medication list). The LLM had 30 minutes—or until it began discussing management options—to ask any questions. A consultant medical officer, acting as the patient, provided answers that were direct but authentic, without extensive elaboration (see Supplementary Information 3 – History transcripts). Once the history was obtained, the LLM generated a case presentation, which was synthesized into a transcript (see Supplementary Information 4 – Case presentations). If the presentation was shorter than 1000 words, an additional prompt encouraged a more detailed, 2000-word response (standard for a 10-minute presentation). The presentation was recorded and shared with the examiners. Each case was assessed by two examiners: one an RACP fellow (FRACP), the other an advanced trainee or another FRACP. The examiners scored the presentation and provided feedback using standard DCE score sheets. Finally, the examiners each prepared five questions based on the case and presentation, which were sent to the LLM for written responses (see Supplementary Information 5 – Question answering). The “Long Case” scores were then finalized, with both descriptive statistics and narrative feedback provided. Download figure Open in new tab Figure 1. Flowchart detailing LLM “Long Case” evaluation process Results The LLM achieved passing consensus scores of 4/6 on three cases and 5/6 on two cases (see Supplementary Information 6 – Scorer sheets). In the case that included a key condition not explicitly dealt with in the LLM prompt (myasthenia gravis) it scored 4/6. Narrative feedback included repetition in issues list and presenting too many options in response to questions, rather than only those specifically relevant to the patient. Discussion These findings demonstrate a novel means to evaluate LLM performance, and the capabilities of contemporary AI. The performance of LLM on the RACP DCE are of particular significance given recent calls to certify AI similarly to a medical professional. 8 While limited by the number and use of hypothetical cases (to protect patient confidentiality), this finding remains noteworthy, as in the RACP DCE only two “Long Cases” are typically undertaken by any given trainee on the day of assessment. The use of advanced trainees as examiners is not a limitation, as full case transcripts are available for readers to form their own opinions of the LLM’s performance. These findings give a preview to the type of functionality that such AI may perform in future and, in this context, highlights the need for dedicated education for clinicians on these topics. 9 , 10 Conclusion This study demonstrates that LLMs can engage effectively in complex medical assessments, such as the RACP “Long Cases”. Further research is needed to refine these systems and explore their integration into real-world clinical settings. Data Availability All data produced in the present work are contained in the supplementary information. Conflict of Interest The authors declare that there is no conflict of interest. Footnotes Sources of support: Stephen Bacchi and Jamie Bellinge are supported by Fulbright Scholarships and Royal Australasian College of Physicians Research Establishment Fellowships. Conflict of interest: The authors declare that there is no conflict of interest. References 1. ↵ Kleinig O , Gao C , Kovoor JG , et al. How to use large language models in ophthalmology: from prompt engineering to protecting confidentiality . Eye (Lond) 2024 ; 38 ( 4 ): 649 – 53 . doi: 10.1038/s41433-023-02772-w [published Online First: 20231005] OpenUrl CrossRef PubMed 2. Goh R , Cook B , Stretton B , et al. Large language models can effectively extract stroke and reperfusion audit data from medical free-text discharge summaries . Journal of Clinical Neuroscience 2024 ; 129 doi: 10.1016/j.jocn.2024.110847 OpenUrl CrossRef 3. Coiera EW , Verspoor K , Hansen DP . We need to chat about artificial intelligence . Med J Aust 2023 ; 219 ( 3 ): 98 – 100 . doi: 10.5694/mja2.51992 [published Online First: 20230611] OpenUrl CrossRef PubMed 4. ↵ van der Vegt A , Campbell V , Zuccon G. Why clinical artificial intelligence is (almost) non-existent in Australian hospitals and how to fix it . Med J Aust 2024 ; 220 ( 4 ): 172 – 75 . doi: 10.5694/mja2.52195 [published Online First: 20231226] OpenUrl CrossRef PubMed 5. ↵ Kleinig O , Gao C , Bacchi S. This too shall pass: the performance of ChatGPT-3.5, ChatGPT-4 and New Bing in an Australian medical licensing examination . Med J Aust 2023 ; 219 ( 5 ): 237 . doi: 10.5694/mja2.52061 [published Online First: 20230801] OpenUrl CrossRef 6. Lucas HC , Upperman JS , Robinson JR . A systematic review of large language models and their implications in medical education . Med Educ 2024 ; 58 ( 11 ): 1276 – 85 . doi: 10.1111/medu.15402 [published Online First: 20240419] OpenUrl CrossRef 7. ↵ Scott IA , Miller T , Crock C. Using conversant artificial intelligence to improve diagnostic reasoning: ready for prime time? Med J Aust 2024 ; 221 ( 5 ): 240 – 43 . doi: 10.5694/mja2.52401 [published Online First: 20240731] OpenUrl CrossRef PubMed 8. ↵ Rajpurkar P , Topol EJ . A clinical certification pathway for generalist medical AI systems . Lancet 2025 ; 405 ( 10472 ): 20 . doi: 10.1016/S0140-6736(24)02797-1 OpenUrl CrossRef PubMed 9. ↵ Squires E , Bacchi S , Maddison J. We need to chat about artificial intelligence . Med J Aust 2023 ; 219 ( 8 ): 394 . doi: 10.5694/mja2.52081 [published Online First: 20230829] OpenUrl CrossRef PubMed 10. ↵ Kovoor JG , Bacchi S , Gupta AK , et al. Artificial intelligence clinical trials and critical appraisal: a necessity . ANZ J Surg 2023 ; 93 ( 5 ): 1141 – 42 . doi: 10.1111/ans.18263 [published Online First: 20230111] OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted January 27, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Not Such a Long Way Off? Contemporary Artificial intelligence Performance Evaluation on Adult Medicine “Long Cases” Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Not Such a Long Way Off? Contemporary Artificial intelligence Performance Evaluation on Adult Medicine “Long Cases” Christina Gao , Jamie Bellinge , Shaddy El-Masri , Ivana Chim , Michael Sorich , Ishish Seth , James Gorcilov , Matthew Lim , Liam McCoy , Andrew Vanlint , Lauren Lim , Jessica Stranks , Andrew Zannettino , John Maddison , Stephen Bacchi medRxiv 2025.01.25.25321118; doi: https://doi.org/10.1101/2025.01.25.25321118 Share This Article: Copy Citation Tools Not Such a Long Way Off? Contemporary Artificial intelligence Performance Evaluation on Adult Medicine “Long Cases” Christina Gao , Jamie Bellinge , Shaddy El-Masri , Ivana Chim , Michael Sorich , Ishish Seth , James Gorcilov , Matthew Lim , Liam McCoy , Andrew Vanlint , Lauren Lim , Jessica Stranks , Andrew Zannettino , John Maddison , Stephen Bacchi medRxiv 2025.01.25.25321118; doi: https://doi.org/10.1101/2025.01.25.25321118 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Medical Education Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (299) Cardiovascular Medicine (4425) Dentistry and Oral Medicine (443) Dermatology (382) Emergency Medicine (607) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1507) Epidemiology (15221) Forensic Medicine (30) Gastroenterology (1123) Genetic and Genomic Medicine (6588) Geriatric Medicine (667) Health Economics (997) Health Informatics (4524) Health Policy (1368) Health Systems and Quality Improvement (1612) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15910) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (145) Nephrology (667) Neurology (6588) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1143) Occupational and Environmental Health (956) Oncology (3331) Ophthalmology (970) Orthopedics (369) Otolaryngology (420) Pain Medicine (435) Palliative Medicine (129) Pathology (663) Pediatrics (1690) Pharmacology and Therapeutics (691) Primary Care Research (710) Psychiatry and Clinical Psychology (5440) Public and Global Health (9219) Radiology and Imaging (2195) Rehabilitation Medicine and Physical Therapy (1369) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (710) Sports Medicine (529) Surgery (710) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9ffa5a5fbf7941e2',t:'MTc3OTQzNzUxNg=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();