Large language models for generating medical examinations: systematic review

Large language models for generating medical examinations: systematic review | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Large language models for generating medical examinations: systematic review Yaara Artsi , Vera Sorin , Eli Konen , Benjamin S. Glicksberg , Girish Nadkarni , Eyal Klang doi: https://doi.org/10.1101/2024.01.06.24300920 Yaara Artsi 1 Azrieli Faculty of Medicine, Bar-Ilan University , Zefat, Israel BMedSC Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: yaara.artsi77{at}gmail.com Vera Sorin 2 Department of Diagnostic Imaging, Chaim Sheba Medical Center , Israel 3 Tel-Aviv University School of Medicine , Israel 4 DeepVision Lab, Chaim Sheba Medical Center , Israel MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Eli Konen 2 Department of Diagnostic Imaging, Chaim Sheba Medical Center , Israel 3 Tel-Aviv University School of Medicine , Israel MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Benjamin S. Glicksberg 5 Division of Data-Driven and Digital Medicine (D3M), Icahn School of Medicine at Mount Sinai , New York, New York, USA PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Girish Nadkarni 5 Division of Data-Driven and Digital Medicine (D3M), Icahn School of Medicine at Mount Sinai , New York, New York, USA 6 The Charles Bronfman Institute of Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Eyal Klang 2 Department of Diagnostic Imaging, Chaim Sheba Medical Center , Israel 3 Tel-Aviv University School of Medicine , Israel 4 DeepVision Lab, Chaim Sheba Medical Center , Israel 5 Division of Data-Driven and Digital Medicine (D3M), Icahn School of Medicine at Mount Sinai , New York, New York, USA 6 The Charles Bronfman Institute of Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, New York, USA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Purpose Writing multiple choice questions (MCQs) for the purpose of medical exams is challenging. It requires extensive medical knowledge, time and effort from medical educators. This systematic review focuses on the application of large language models (LLMs) in generating medical MCQs. Methods The authors searched for studies published up to November 2023. Search terms focused on LLMs generated MCQs for medical examinations. MEDLINE was used as a search database. Results Overall, eight studies published between April 2023 and October 2023 were included. Six studies used Chat-GPT 3.5, while two employed GPT 4. Five studies showed that LLMs can produce competent questions valid for medical exams. Three studies used LLMs to write medical questions but did not evaluate the validity of the questions. One study conducted a comparative analysis of different models. One other study compared LLM-generated questions with those written by humans. All studies presented faulty questions that were deemed inappropriate for medical exams. Some questions required additional modifications in order to qualify. Conclusions LLMs can be used to write MCQs for medical examinations. However, their limitations cannot be ignored. Further study in this field is essential and more conclusive evidence is needed. Until then, LLMs may serve as a supplementary tool for writing medical examinations. Download figure Open in new tab Introduction There is a global shortage of clinical practitioners and increasing demand for medical professionals. This need presents significant challenges in the healthcare system [ 1 , 2 , 3 ]. In response, the number of medical schools and students has been rising worldwide [ 4 , 5 ], leading to an increase in the demand for written tests. Creating multiple choice questions (MCQs) requires medical knowledge, conceptual integration, and avoiding potential pitfalls. For example, repeat items in examinations from year to year, or inherent imperfections called item-writing flaws (IWFs). Although at first sight IWFs may appear trivial, they can affect the way students understand and answer questions [ 6 , 7 , 8 , 9 ]. Producing MCQs is also time consuming. Any application capable of automating this process could be highly valuable [ 10 , 11 ]. Amidst these challenges, advancements in natural language processing (NLP) are constantly discussed and evaluated [ 12 ]. In particular, the introduction of OpenAI’s state-of-the-art large language models (LLMs) such as GPT-3.5 and GPT-4 [ 13 , 14 ]. These models offer potential solutions to healthcare education, due to their human-like text understanding and generation, which includes clinical knowledge [ 15 ]. This could be pivotal in automating the creation of medically precise MCQs. However, automating MCQs creation introduces potential risks, as the accuracy and quality of AI generated content is still in question [ 16 , 17 ]. We aimed to review the literature on LLMs’ ability to generate medical questions. We evaluated their clinical accuracy and suitability for medical examinations. Methods Literature search On November 2nd 2023 we conducted a search identifying studies describing LLMs’ applications in generating medical questions. We searched PubMed/MEDLINE for papers with the following keywords, using Boolean operators AND/ OR: large language models; GPT; ChatGPT; medical questions; medical education; USMLE; MCCQE1; board exam; medical exam. We also checked the references list of selected publications for more relevant papers. Sections as ‘Similar Articles’ below articles (e.g., PubMed) were also inspected for possible additional articles. Ethical approval was not required, this is a systematic review of previously published research, and does not include any individual participant information. Our study followed the Preferred Reporting Items for Systematic Reviews and Meta-Analyses (PRISMA) guidelines. The study is registered with PROSPERO (CRD42023481851) Inclusion and exclusion process Publications resulting from the search were initially assessed by one author (YA) for relevant titles and abstracts. Next, full-text papers underwent an independent evaluation by two authors (EK and VS). We included full length studies describing LLMs generating medical questions. We excluded papers dating before 2023, non-English papers and non-original studies. Any study in question was discussed among all authors until reaching a unanimous agreement. ( Figure 1 .) Figure 1: Flow Diagram of the Inclusion Process. Flow diagram of the search and inclusion process based on the Preferred Reporting Items for Systematic Reviews and Meta-Analyses (PRISMA) guidelines, November 2023. Risk of bias and applicability were evaluated using the tailored QUADAS-2 tool. ( Figure 2 .) Figure 2: QUADAS-2 table for potential bias and applicability. Risk of bias and applicability were evaluated using the tailored QUADAS-2 tool, November 2023. Results Study selection and characteristics The initial literature search resulted in 838 articles. Eight studies met our inclusion criteria ( Figure 1 ). Most studies were retrospective: 6/8 (75%). One study is cross-sectional and one study is prospective. Most studies used Chat-GPT (3.5 or 4) as an AI model of choice, other models evaluated included Microsoft’s Bing and Google’s Bard. The MCQs were produced with varying parameters ( Table 1 ). Overall 5/8 (62.5%) studies demonstrated valid MCQs. 6/8 (75%) of the studies utilized the latest version Chat-GPT 4 ( Figure 3 .) Figure 3: Illustration of multiple-choice questions (MCQs) generation and summary of preliminary results. The images in the upper row illustrate the MCQs generation process via a large language model. The images in the bottom row showcase preliminary data results, November 2023. View this table: View inline View popup Table 1: Summary of the articles in the literature that applied AI for generating medical questions, November 2023. Descriptive summary of results Cheung et al. [ 18 ] were the first, and so far, the only study to compare LLM to humans in MCQs writing. Chat-GPT 3.5 plus generated the MCQs. The reference for the prompt were two standard undergraduate medical textbooks: Harrison’s Principles of Internal Medicine the 21th edition for medicine [ 19 ], and Bailey and Love’s Short Practice of Surgery 27th Edition for surgery [ 20 ]. Only four choices were given per question. Also, only text and knowledge-based questions were generated. No modification to the MCQs was allowed after generation. Chat-GPT 3.5 performed relatively well in the task. The overall time required for the AI to generate 50 MCQs was 21 minutes. This is about 10% of the total time human writing required (211 min). However, the questions written by humans were far better. Both in terms of quality and validity, outperforming the AI in a total score of 30 (60%) eligible MCQs ( Table 2 ). View this table: View inline View popup Table 2: Summary of key parameters investigated in each study, November 2023. Klang et al. [ 21 ] performed blind assessment of the generated questions. They did not disclose to the evaluators whether the MCQs origin was AI. At first, they asked Chat-GPT 4 to create MCQs on the topic of internal medicine. They used as reference (few-shot learning) a former exam of the same subject. The MCQs had four possible answers, with the correct answer marked with an asterisk. At first, the generated MCQs were short with no clinical background. This required a second prompting of the AI model, specifically requesting the AI to create MCQs with clinical history. The study showed promising results, with the majority of MCQs deemed valid as exam questions ( Table 2 ). In a cross-sectional study, Agarwal et.al [ 22 ] compared different LLMs. They compared Chat-GPT 3.5/Bard/Bing in MCQs generating capability. They used as reference the 11-module curriculum for physiology, created by The Indian National Medical Commission (NMC). The authors requested in the prompt to Generate five difficult reasoning-based MCQs, fitting levels of Bachelor of Medicine, and Bachelor of Surgery (MBBS). Chat-GPT’s generated MCQs were significantly more valid than the other AI tools examined in the study. However, the difficulty level was lower compared to Bard and Bing ( Table 2 ). Ayub et al. [ 23 ] focused on medical board examination for Dermatology. They utilized Chat-PDF to upload entire PDF files into a ChatGPT 3.5 portal. The reference used was “Continuing medical education” (CME) articles, taken from the Journal of the American Academy of Dermatology (JAAD). This reference is considered high-yield review material for the American Board of Dermatology Applied Exam (ABD-AE). This study’s prompt was not detailed in the paper. The three parameters to evaluate the MCQs were accuracy, complexity, and clarity. Only 16 (40%) of the generated questions were applicable ( Table 2 ). The rest were unclear 9 (22%), inaccurate 5 (13%) or had low complexity 10 (25%) ( Table 3 ). Sevgi et al. [ 24 ] asked Chat-GPT 3.5 to prepare three questions with answers and explanations at a level appropriate for a neurosurgery board exam. There was no independent evaluation of the MCQs. Han et al. [ 25 ] instructed Chat-GPT 3.5 to write three MCQs, each containing clinical background and lab values. Each time they requested Chat-GPT to rephrase the question. First, for a different correct answer and then for an increased level of difficulty. There was no independent evaluation of the MCQs. Totlis et al. [ 26 ] asked Chat-GPT 4 to generate MCQs on the topic of anatomy. In the prompt they requested increasing difficulty and matching correct pairs. There was no independent evaluation of the MCQs. Biswas [ 27 ] requested in the prompt to prepare MCQs for USMLE step 1 exam. There was no independent evaluation of the MCQs. All studies presented some faulty questions that were deemed inappropriate for medical exams. Some questions required additional modifications in order to qualify ( Table 3 ). We included examples from each study, demonstrating valid MCQs as well as faulty MCQs for various reasons ( Table 4 .) View this table: View inline View popup Table 3: Summary of faulty questions generated by the AI, November 2023 View this table: View inline View popup Table 4: Examples from studies of multiple-choice questions generated by AI. Both valid and faulty, November 2023. Discussion In this study we explored LLMs’ applicability in generating medical questions. Specifically, multiple choice questions (MCQs) for medical examinations. MCQs are an essential component of medical exams, used in almost every aspect of medical education [ 8 , 9 ]. Yet, they are time consuming and expensive to create [ 28 ]. The possibility of AI generated questions can provide an important opportunity for the medical community and transform the way written tests are generated. Using LLMs to support these tasks can potentially save time, money and reduce burnout. Especially in a system already sustaining itself on limited resources [ 29 ]. AI benefits Physician burn-out, poor mental health and growing personal distress are constantly studied [ 30 ]. However, academic physicians experience a unique set of additional challenges, such as increased administrative work, less time with patients and increased clinical responsibilities. As a result, they have less time for traditional academic pursuits such as research and education [ 31 , 32 , 33 ]. In the famous words of Albert Einstein: “Bureaucracy is the death of any achievement”. AI can potentially relieve medical educators from tiresome bureaucracy and administrative work, allowing them to focus on the areas that they view as most personally meaningful and avoid career dissatisfaction [ 32 , 34 ]. According to Bond et al. another possible application of AI in medical education is grading patients notes. This can provide additional formative feed-back for students in the face of limited faculty availability [ 35 ]. Moreover, AI can assist medical students by creating personalized learning experience, while accessing current up-to-date information [ 36 ]. These are only a few examples, as every day new tasks improved by AI are discovered. AI drawbacks Nowadays, AI continues to evolve, becoming more integrated in various medical fields [ 37 ]. The probability of revolutionizing the healthcare world seems inevitable. AI performance is fast, efficient and with what seems like endless data resources [ 38 ]. In almost every study we reviewed, LLMs’ execution was more than satisfactory with the consensus that AI is capable of producing valid questions for medical exams. However, while these models show promise as an educational tool, their limitations must be acknowledged. One notable limitation is a phenomenon known as “Hallucination” [ 39 ]. This occurs when generative AI misinterprets the given prompt, resulting in outputs that lack logical consistency. When relying on AI for quality MCQs writing, this phenomenon is concerning. Furthermore, AI ability to integrate contextual and sensory information is still not fully developed. Currently, AI cannot understand non-verbal cues or body language. Also, bias in data and inaccuracy is troubling [ 40 , 41 ]. Another consideration is the logistics necessary to implement AI in healthcare and education. New technologies require training, commitment and investment in order to be maintained and managed in a sustainable way. Such a process can take time and energy [ 42 ]. Moreover, implementing new technology increases concern for privacy and data security [ 43 , 44 ]. Patients’ data is sensitive and often a target for cyber-attacks [ 45 ]. Equally important limitation of AI integration in healthcare is accountability. AI “black box” refers to the “knowledge within the machine”. The internal workings of the system are invisible to the user. Healthcare staff use the AI, write the input and receive the output. But, the system’s code or logic cannot be questioned or explained [ 46 ]. Additional aspect to consider is the longstanding concern of AI replacing human jobs [ 47 ]. This could cause a dislike and resistance to AI integration. This notion is unlikely in the near future. But, distrust in AI technology is yet another challenge to its implementation [ 48 ]. But maybe the biggest concern of AI application in medical education is impairing students’ critical thinking. According to Van de Ridder et al., self-reflection and criticism are crucial for a medical student’s learning process and professional growth. In a reality where a student can delegate to Chat-GPT tasks such as writing personal reflection or learning experiences, the students deny themselves of the opportunity to self-reflect and grow as physicians [ 49 ]. It is imperative to take into consideration those significant shortcomings and challenges. AI should be used wisely and responsibly while integrating it into medical education. MCQs creation For each study we examined the process of crafting the MCQs. We noticed a wide range of approaches to writing the prompts. In some studies, additional modifications took place in order to improve the validity of the questions. This emphasizes the importance and sensitivity of prompts. Prompt-engineering may be a task that requires specific training, so that the prompt is phrased correctly and the MCQs quality is not impaired. Limitations Our review has several limitations. Most of the studies are retrospective in nature. Due to heterogeneity in study design and data, a meta-analysis was not performed. None of the questions were image or graph based, which is an integral part of medical exams. Three studies did not base their prompt on a valid medical reference, such as previous exams or approved syllabus. Also, the three studies did not evaluate the questions after they were generated. Two studies were at high risk of bias. Additional studies will be needed to further solidify the usefulness of AI tools, especially in generating competent medical exam questions. Lastly, we limited our search to PubMed/MEDLINE. We did so due to its relevance in biomedical research. We recognize this choice narrows our review’s scope. This might exclude studies from other databases, possibly limiting diverse insights. Conclusion AI-generated MCQs for medical exams are feasible. The process is fast and efficient, demonstrating great promise in the future of medical education and exam preparation. However, their use warrants cautious and critical evaluation. Awareness of AI limitations is imperative in order to avoid misuse and deterioration of medical education quality. Currently, further research in this field is needed. Until more advancements are achieved, AI should be viewed as a powerful tool best utilized by experienced professionals. Disclosure statement Disclosure of interest The authors report there are no competing interests to declare Additional information Funding The author(s) reported there is no funding associated with the work featured in this article. Data Availability All data produced in the present work are contained in the manuscript Disclaimer None Acknowledgments None References 1. ↵ Boniol M , Kunjumen T , Nair TS , Siyam A , Campbell J , Diallo K . The global health workforce stock and distribution in 2020 and 2030: a threat to equity and ‘universal’ health coverage? BMJ Glob Health . 2022 Jun ; 7 ( 6 ): e009316 . doi: 10.1136/bmjgh-2022-009316 . PMID: 35760437 ; PMCID: PMC9237893 . OpenUrl Abstract / FREE Full Text 2. ↵ GBD 2019 Human Resources for Health Collaborators . Measuring the availability of human resources for health and its relationship to universal health coverage for 204 countries and territories from 1990 to 2019: a systematic analysis for the Global Burden of Disease Study 2019. Lancet. 2022;399(10341):2129-2154 . doi: 10.1016/S0140-6736(22)00532-3 OpenUrl CrossRef PubMed 3. ↵ Zhang X , Lin D , Pforsich H , Lin VW . Physician workforce in the United States of America: forecasting nationwide shortages . Hum Resour Health . 2020 ; 18 ( 1 ): 8 . Published 2020 Feb 6. doi: 10.1186/s12960-020-0448-3 OpenUrl CrossRef 4. ↵ Rigby PG , Gururaja RP . World medical schools: The sum also rises . JRSM Open . 2017 ; 8 ( 6 ): 2054270417698631 . Published 2017 Jun 5. doi: 10.1177/2054270417698631 OpenUrl CrossRef 5. ↵ Hashem F , Marchand C , Peckham S , Peckham A . What are the impacts of setting up new medical schools? A narrative review . BMC Medical Education . 2022 ; 22 ( 1 ). doi: 10.1186/s12909-022-03835-4 OpenUrl CrossRef 6. ↵ Przymuszała , P. , Piotrowska , K. , Lipski , D. , Marciniak , R. , & Cerbin-Koczorowska , M . ( 2020 ). Guidelines on Writing Multiple Choice Questions: A Well-Received and Effective Faculty Development Intervention . SAGE Open , 10 ( 3 ). doi: 10.1177/2158244020947432 OpenUrl CrossRef 7. ↵ Pham H , Court-Kowalski S , Chan H , Devitt P . Writing Multiple Choice Questions-Has the Student Become the Master? . Teach Learn Med . 2023 ; 35 ( 3 ): 356 – 367 . doi: 10.1080/10401334.2022.2050240 OpenUrl CrossRef 8. ↵ Balaha MH , El-Ibiary MT , El-Dorf AA , El-Shewaikh SL , Balaha HM . Construction and Writing Flaws of the Multiple-Choice Questions in the Published Test Banks of Obstetrics and Gynecology: Adoption, Caution, or Mitigation? . Avicenna J Med . 2022 ; 12 ( 3 ): 138 – 147 . Published 2022 Aug 31. doi: 10.1055/s-0042-1755332 OpenUrl CrossRef 9. ↵ Coughlin PA , Featherstone CR . How to Write a High Quality Multiple Choice Question (MCQ): A Guide for Clinicians . Eur J Vasc Endovasc Surg . 2017 ; 54 ( 5 ): 654 – 658 . doi: 10.1016/j.ejvs.2017.07.012 OpenUrl CrossRef 10. ↵ Homolak J . Opportunities and risks of ChatGPT in medicine, science, and academic publishing: a modern Promethean dilemma . Croat Med J . 2023 ; 64 ( 1 ): 1 – 3 . doi: 10.3325/cmj.2023.64.1 OpenUrl CrossRef 11. ↵ Gilardi F , Alizadeh M , Kubli M . ChatGPT outperforms crowd workers for text-annotation tasks . Proc Natl Acad Sci U S A . 2023 ; 120 ( 30 ): e2305016120 . doi: 10.1073/pnas.2305016120 OpenUrl CrossRef 12. ↵ Sorin V , Barash Y , Konen E , Klang E . Deep-learning natural language processing for oncological applications . Lancet Oncol . 2020 ; 21 ( 12 ): 1553 – 1556 . doi: 10.1016/S1470-2045(20)30615-X OpenUrl CrossRef 13. ↵ Clusmann J , Kolbinger FR , Muti HS , et al. The future landscape of large language models in medicine . Commun Med (Lond ) . 2023 ; 3 ( 1 ): 141 . Published 2023 Oct 10. doi: 10.1038/s43856-023-00370-1 OpenUrl CrossRef 14. ↵ Eysenbach G . The Role of ChatGPT , Generative Language Models, and Artificial Intelligence in Medical Education: A Conversation With ChatGPT and a Call for Papers. JMIR Med Educ . 2023 ; 9 : e46885 . Published 2023 Mar 6. doi: 10.2196/46885 OpenUrl CrossRef 15. ↵ Brin D , Sorin V , Vaid A , et al. Comparing ChatGPT and GPT-4 performance in USMLE soft skill assessments . Sci Rep . 2023 ; 13 ( 1 ): 16492 . Published 2023 Oct 1. doi: 10.1038/s41598-023-43436-9 OpenUrl CrossRef 16. ↵ Bhattacharyya M , Miller VM , Bhattacharyya D , Miller LE . High Rates of Fabricated and Inaccurate References in ChatGPT-Generated Medical Content . Cureus . 2023 ; 15 ( 5 ): e39238 . Published 2023 May 19. doi: 10.7759/cureus.39238 OpenUrl CrossRef 17. ↵ Vaishya R , Misra A , Vaish A . ChatGPT: Is this version good for healthcare and research? . Diabetes Metab Syndr . 2023 ; 17 ( 4 ): 102744 . doi: 10.1016/j.dsx.2023.102744 OpenUrl CrossRef 18. ↵ Cheung BHH , Lau GKK , Wong GTC , et al. ChatGPT versus human in generating medical graduate exam multiple choice questions-A multinational prospective study (Hong Kong S.A.R., Singapore , Ireland, and the United Kingdom). PLoS One . 2023 ; 18 ( 8 ): e0290691 . Published 2023 Aug 29. doi: 10.1371/journal.pone.0290691 OpenUrl CrossRef 19. ↵ Harrison’s Principles of Internal Medicine, 21E | AccessMedicine | McGraw Hill Medical . https://accessmedicine.mhmedical.com/book.aspx?bookid=3095 . 20. ↵ Williams NS , O’Connell PR, McCaskie AW. Bailey & Love’s Short Practice of Surgery: Taylor & Francis Group; 2018. 21. ↵ E K , S P , R G, et al. Advantages and pitfalls in utilizing artificial intelligence for crafting medical examinations: a medical education pilot study with GPT-4 . BMC Med Educ . 2023 ; 23 ( 1 ): 772 . Published 2023 Oct 17. doi: 10.1186/s12909-023-04752-w OpenUrl CrossRef 22. ↵ Agarwal M , Sharma P , Goswami A . Analysing the Applicability of ChatGPT, Bard, and Bing to Generate Reasoning-Based Multiple-Choice Questions in Medical Physiology . Cureus . 2023 ; 15 ( 6 ): e40977 . Published 2023 Jun 26. doi: 10.7759/cureus.40977 OpenUrl CrossRef 23. ↵ Ayub I , Hamann D , Hamann CR , Davis MJ . Exploring the Potential and Limitations of Chat Generative Pre-trained Transformer (ChatGPT) in Generating Board-Style Dermatology Questions: A Qualitative Analysis . Cureus . 2023 ; 15 ( 8 ): e43717 . Published 2023 Aug 18. doi: 10.7759/cureus.43717 OpenUrl CrossRef 24. ↵ Sevgi UT , Erol G , Doğruel Y , Sönmez OF , Tubbs RS , Güngör A . The role of an open artificial intelligence platform in modern neurosurgical education: a preliminary study . Neurosurgical Review . 2023 ; 46 ( 1 ). doi: 10.1007/s10143-023-01998-2 OpenUrl CrossRef 25. ↵ Han Z , Battaglia F , Udaiyar A , Fooks A , Terlecky SR . An Explorative Assessment of ChatGPT as an Aid in Medical Education: Use it with Caution . medRxiv (Cold Spring Harbor Laboratory ) . February 2023 . doi: 10.1101/2023.02.13.23285879 OpenUrl Abstract / FREE Full Text 26. ↵ Totlis T , Natsis K , Filos D , et al. The potential role of ChatGPT and artificial intelligence in anatomy education: a conversation with ChatGPT . Surgical and Radiologic Anatomy . 2023 ; 45 ( 10 ): 1321 – 1329 . doi: 10.1007/s00276-023-03229-1 OpenUrl CrossRef 27. ↵ Biswas S . Passing is Great: Can ChatGPT Conduct USMLE Exams? . Ann Biomed Eng . 2023 ; 51 ( 9 ): 1885 – 1886 . doi: 10.1007/s10439-023-03224-y OpenUrl CrossRef 28. ↵ Gierl MJ , Lai H , Turner SR . Using automatic item generation to create multiple-choice test items . Med Educ . 2012 ; 46 ( 8 ): 757 – 765 . doi: 10.1111/j.1365-2923.2012.04289.x OpenUrl CrossRef PubMed 29. ↵ Alhalaseh Y , Elshabrawy HA , Erashdi M , Shahait M , Abu-Humdan AM , Al=Hussaini M. Allocation of the “Already” limited medical resources amid the COVID-19 pandemic, an iterative ethical encounter including suggested solutions from a real life encounter . Frontiers in Medicine . 2021 ; 7 . doi: 10.3389/fmed.2020.616277 OpenUrl CrossRef 30. ↵ Khan , Rabia PhD , MSc1; Hodges, Brian David MD, PhD2; Martimianakis, Maria Athina PhD, MA3. Constructing “Burnout”: A Critical Discourse Analysis of Burnout in Postgraduate Medical Education . Academic Medicine 98(11S):p S116-S122 , November 2023 . | DOI: 10.1097/ACM.0000000000005358 OpenUrl CrossRef 31. ↵ Shanafelt TD , West CP , Sloan JA , et al. Career fit and burnout among academic faculty . Arch Intern Med . 2009 ; 169 ( 10 ): 990 – 995 . doi: 10.1001/archinternmed.2009.70 OpenUrl CrossRef PubMed Web of Science 32. ↵ Woolhandler S , Himmelstein DU . Administrative work consumes one-sixth of U.S. physicians’ working hours and lowers their career satisfaction . Int J Health Serv . 2014 ; 44 ( 4 ): 635 – 642 . doi: 10.2190/HS.44.4.a OpenUrl CrossRef PubMed 33. ↵ Szulewski, Adam MD, MHPE, PhD1; Braund, Heather PhD2; Dagnone, Damon J. MD, MSc, MMEd3; McEwen, Laura PhD4; Dalgarno, Nancy PhD5; Schultz, Karen W . MD6; Hall, Andrew K. MD, MMEd7. The Assessment Burden in Competency-Based Medical Education: How Programs Are Adapting . Academic Medicine 98(11):p 1261-1267 , November 2023 . | DOI: 10.1097/ACM.0000000000005305 OpenUrl CrossRef 34. ↵ Lowenstein SR , Fernandez G , Crane LA . Medical school faculty discontent: prevalence and predictors of intent to leave academic careers . BMC Med Educ . 2007 ; 7 : 37 . Published 2007 Oct 14. doi: 10.1186/1472-6920-7-37 OpenUrl CrossRef PubMed 35. ↵ Bond , William F. MD , MS1; Zhou, Jianing MS2; Bhat, Suma PhD3; Park, Yoon Soo PhD4; Ebert-Allen, Rebecca A.5; Ruger, Rebecca L.6; Yudkowsky, Rachel MD, MHPE7. Automated Patient Note Grading: Examining Scoring Reliability and Feasibility . Academic Medicine 98(11S):p S90-S97, November 2023 . | DOI: 10.1097/ACM.0000000000005357 OpenUrl CrossRef 36. ↵ Feng , Songwei1; Shen, Yang MD , PhD2. ChatGPT and the Future of Medical Education. Academic Medicine 98(8):p 867-868 , August 2023 . | DOI: 10.1097/ACM.0000000000005242 OpenUrl CrossRef 37. ↵ Maassen O , Fritsch S , Palm J , et al. Future Medical Artificial Intelligence Application Requirements and Expectations of Physicians in German University Hospitals: Web-Based Survey . J Med Internet Res . 2021 ; 23 ( 3 ): e26646 . Published 2021 Mar 5. doi: 10.2196/26646 OpenUrl CrossRef 38. ↵ Ramesh AN , Kambhampati C , Monson JR , Drew PJ . Artificial intelligence in medicine . Ann R Coll Surg Engl . 2004 ; 86 ( 5 ): 334 – 338 . doi: 10.1308/147870804290 OpenUrl CrossRef PubMed 39. ↵ Athaluri SA , Manthena SV , Kesapragada VSRKM , Yarlagadda V , Dave T , Duddumpudi RTS . Exploring the Boundaries of Reality: Investigating the Phenomenon of Artificial Intelligence Hallucination in Scientific Writing Through ChatGPT References . Cureus . 2023 ; 15 ( 4 ): e37432 . Published 2023 Apr 11. doi: 10.7759/cureus.37432 OpenUrl CrossRef 40. ↵ Safranek CW , Sidamon-Eristoff AE , Gilson A , Chartash D . The Role of Large Language Models in Medical Education: Applications and Implications . JMIR Med Educ . 2023 ; 9 : e50945 . Published 2023 Aug 14. doi: 10.2196/50945 OpenUrl CrossRef 41. ↵ Vorisek CN , Stellmach C , Mayer PJ , et al. Artificial Intelligence Bias in Health Care: Web-Based Survey . J Med Internet Res . 2023 ; 25 : e41089 . Published 2023 Jun 22. doi: 10.2196/41089 OpenUrl CrossRef 42. ↵ van Gemert-Pijnen JL . Implementation of health technology: Directions for research and practice . Front Digit Health . 2022 ; 4 : 1030194 . Published 2022 Nov 10. doi: 10.3389/fdgth.2022.1030194 OpenUrl CrossRef 43. ↵ Raimundo R , Rosário A . The Impact of Artificial Intelligence on Data System Security: A Literature Review . Sensors (Basel ) . 2021 ; 21 ( 21 ): 7029 . Published 2021 Oct 23. doi: 10.3390/s21217029 OpenUrl CrossRef 44. ↵ Ignatovski M . Healthcare Breaches During COVID-19: The Effect of the Healthcare Entity Type on the Number of Impacted Individuals . Perspect Health Inf Manag . 2022 ; 19 ( 4 ): 1c . Published 2022 Oct 1. OpenUrl 45. ↵ Sorin V , Soffer S , Glicksberg BS , Barash Y , Konen E , Klang E . Adversarial attacks in radiology - A systematic review . Eur J Radiol . 2023 ; 167 : 111085 . doi: 10.1016/j.ejrad.2023.111085 OpenUrl CrossRef 46. ↵ Chan B . Black-box assisted medical decisions: AI power vs. ethical physician care . Med Health Care Philos . 2023 ; 26 ( 3 ): 285 – 292 . doi: 10.1007/s11019-023-10153-z OpenUrl CrossRef 47. ↵ Shuaib A , Arian H , Shuaib A . The Increasing Role of Artificial Intelligence in Health Care: Will Robots Replace Doctors in the Future? . Int J Gen Med . 2020 ; 13 : 891 – 896 . Published 2020 Oct 19. doi: 10.2147/IJGM.S268093 OpenUrl CrossRef 48. ↵ Starke G , Ienca M . Misplaced Trust and Distrust: How Not to Engage with Medical Artificial Intelligence . Camb Q Healthc Ethics. Published online October 20 , 2022 . doi: 10.1017/S0963180122000445 OpenUrl CrossRef 49. ↵ van de Ridder, J.M. Monica PhD , MSc1; Shoja , Mohammadali M. M D2; Rajput, Vijay MD, MACP3. Finding the Place of ChatGPT in Medical Education. Academic Medicine 98(8):p 867 , August 2023 . | DOI: 10.1097/ACM.0000000000005254 OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted January 09, 2024. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Large language models for generating medical examinations: systematic review Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Large language models for generating medical examinations: systematic review Yaara Artsi , Vera Sorin , Eli Konen , Benjamin S. Glicksberg , Girish Nadkarni , Eyal Klang medRxiv 2024.01.06.24300920; doi: https://doi.org/10.1101/2024.01.06.24300920 Share This Article: Copy Citation Tools Large language models for generating medical examinations: systematic review Yaara Artsi , Vera Sorin , Eli Konen , Benjamin S. Glicksberg , Girish Nadkarni , Eyal Klang medRxiv 2024.01.06.24300920; doi: https://doi.org/10.1101/2024.01.06.24300920 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Medical Education Subject Areas All Articles Addiction Medicine (574) Allergy and Immunology (865) Anesthesia (304) Cardiovascular Medicine (4462) Dentistry and Oral Medicine (445) Dermatology (383) Emergency Medicine (611) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1517) Epidemiology (15251) Forensic Medicine (31) Gastroenterology (1132) Genetic and Genomic Medicine (6621) Geriatric Medicine (669) Health Economics (1002) Health Informatics (4564) Health Policy (1372) Health Systems and Quality Improvement (1617) Hematology (544) HIV/AIDS (1272) Infectious Diseases (except HIV/AIDS) (15938) Intensive Care and Critical Care Medicine (1107) Medical Education (624) Medical Ethics (147) Nephrology (670) Neurology (6643) Nursing (346) Nutrition (1001) Obstetrics and Gynecology (1149) Occupational and Environmental Health (957) Oncology (3350) Ophthalmology (981) Orthopedics (369) Otolaryngology (421) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1698) Pharmacology and Therapeutics (694) Primary Care Research (714) Psychiatry and Clinical Psychology (5465) Public and Global Health (9259) Radiology and Imaging (2212) Rehabilitation Medicine and Physical Therapy (1372) Respiratory Medicine (1199) Rheumatology (598) Sexual and Reproductive Health (716) Sports Medicine (533) Surgery (715) Toxicology (100) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a03e54986b341640',t:'MTc4MDE1MDMxMQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();