Introducing Answered with Evidence - a framework for evaluating whether LLM responses to biomedical questions are founded in evidence

Introducing Answered with Evidence - a framework for evaluating whether LLM responses to biomedical questions are founded in evidence | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Introducing Answered with Evidence - a framework for evaluating whether LLM responses to biomedical questions are founded in evidence Julian D Baldwin , Christina Dinh , Arjun Mukerji , Neil Sanghavi , View ORCID Profile Saurabh Gombar doi: https://doi.org/10.1101/2025.07.01.25330655 Julian D Baldwin 1 Atropos Health ; New York, NY, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Christina Dinh 1 Atropos Health ; New York, NY, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Arjun Mukerji 1 Atropos Health ; New York, NY, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Neil Sanghavi 1 Atropos Health ; New York, NY, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Saurabh Gombar 1 Atropos Health ; New York, NY, USA 2 Stanford School of Medicine – Department of Pathology, Stanford University , Stanford, California, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Saurabh Gombar For correspondence: sgombar{at}stanford.edu Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract The growing use of large language models (LLMs) for biomedical question answering raises concerns about the accuracy and evidentiary support of their responses. To address this, we present Answered with Evidence , a framework for evaluating whether LLM generated answers are grounded in scientific literature. We analyzed thousands of physician-submitted questions using a comparative pipeline across seven LLMs grounded in different evidence sources. Six sources were grounded in PubMed or general online content, and the last source was grounded in the Atropos Alexandria library of custom real-world analyses. We found that the general purpose LLMs grounded in public information varied greatly in the answers they returned, even when those answers were sourced from the same publication. Using an ensemble approach, we observed that 49% of the time, two or more LLMs agreed on an answer. Combined, the ensemble approach and the Alexandria custom built source enabled reliable answers to over 64% of biomedical queries. As LLMs become increasingly capable of summarizing scientific content, maximizing their value will require systems that can accurately retrieve both published and custom-generated evidence or generate reliable evidence in real time. Background Evidence-based medicine—making clinical decisions based on the best available research—has been the standard of care for over three decades 1 . However, in some specialties, fewer than 20% of daily medical decisions are supported by high-quality evidence 2 . This gap arises primarily from two challenges: First, clinical trials often lack generalizability to real-world patients, especially those with complex comorbidities who are frequently excluded from study populations 3 . Second, even when multiple relevant studies exist, their findings may conflict due to heterogeneity in patient populations, variable study quality, and inconsistent endpoints and difficulty for the care provider to access the evidence 4 . These limitations hinder the synthesis of clear, actionable recommendations for individual patients, leading clinicians to rely heavily on summarized evidence from trusted sources 5 . Large language models (LLMs) are increasingly being explored for biomedical applications, including literature summarization and natural language question answering 6 – 8 . LLMs have demonstrated impressive capabilities in extracting and synthesizing biomedical information, but they are also prone to hallucinating sources, fabricating citations, or generating responses that deviate from established clinical guidelines 9 – 11 . Furthermore, slight changes in question structure can reduce the accuracy of LLMs on commonly measured clinical tasks 12 . These shortcomings present risks in domains where accuracy and reliability is critical. Despite these concerns, LLM integration into biomedical research and clinical workflows has accelerated markedly in recent years. If thoughtful care into measuring accuracy and error rates is not performed now it is possible future high-profile mistakes will shake trust in the medical application of this promising technology. One emerging use case is the use of LLMs to generate evidence summaries to support clinical decision-making. Several commercial services—such as Open Evidence, System, Consensus, SciSummary, Perplexity, and others—now offer LLM-powered interfaces to biomedical literature 13 , 14 . While adoption of these tools is increasing, questions remain about how to evaluate their reliability and trustworthiness. Frameworks like the recently proposed MedHELM 15 , 16 aim to standardize the evaluation of LLM outputs across tasks, but they do not yet assess whether responses are grounded in verifiable biomedical evidence. In this study, we introduce Answered with Evidence , a structured methodology for evaluating the evidentiary grounding and citation fidelity of LLM-generated answers to biomedical questions. Our approach emphasizes verifiability against trusted biomedical sources, enabling the development and monitoring of systems suitable for clinical decision support and real-world evidence generation. To assess the utility of this framework, we analyzed nearly three thousand physician-submitted questions to Atropos Health’s Green Button Service 8 , 17 , 18 — a human-in-the-loop, real-world evidence platform used by clinicians, hospital leaders, and researchers. Answers were compared across 7 LLMs: Three PubMed-based retrieval-augmented systems (System, Perplexity, and Claude), Open AI (o3 and GPT-5) grounded in web search preview, Gemini grounded in Google search, and Alexandria, the Atropos Evidence Library, a curated collection of real-world evidence derived from prior medical inquiries. Materials and Methods Figure 1 outlines the overall analysis workflow. We began by selecting a sample of 2,942 questions submitted to Atropos Health between 2022 and 2025. These questions originated from physicians seeking evidence to inform clinical decisions and from researchers evaluating treatment efficacy or event rates in defined populations. The sample spans all major clinical subspecialties and includes a diverse range of query types—from causal inference questions related to adverse events and treatment comparisons to more straightforward questions about prevalence and incidence. Download figure Open in new tab Figure 1 – Evidence Badge Generation Workflow Each question was processed on an LLM with access to the different underlying evidence sources mentioned above. Once the response was obtained, it was evaluated for being based in evidence and then based on the evaluation a “badge” was assigned to ease in user interpretation. Evidence Sources We evaluated biomedical question-answering systems using seven distinct evidence sources, each optimized for different types of clinical inquiries. The first three sources—System Inc 14 , Claude (Opus 4) 19 , and Perplexity (Sonar-deep-research) 13 —are retrieval-augmented generation 20 (RAG) platforms grounded in the PubMed corpus, comprising peer-reviewed biomedical literature that reflects the current state of scientific knowledge. Perplexity and Claude, while designed as general-purpose LLM-based question-answering tools, were configured for this analysis to retrieve only from academic publications. SystemAI combines large language models with graph-based retrieval technologies, trained specifically on scientific literature using state-of-the-art models. Three other LLMs OpenAI (o3 and GPT-5), and Gemini (2.5 Flash) are general-purpose LLMs trained on a large corpus of available information via web search tools (google search, specifically in the case for Gemini). Despite their general purpose nature, they have gained traction for healthcare uses and previous iterations of each model have been scored on medical tasks via the MedHELM framework (performance available at: https://crfm.stanford.edu/helm/medhelm/latest/ ). The final evidence source is Alexandria, the Atropos Evidence Library, a proprietary collection of structured real-world evidence (RWE). This library includes studies generated from electronic health records, open and closed claims data, and specialized clinical datasets in response to specific physician and researcher inquiries. Each study includes a physician-written summary and clear description of the study methods, describing the study population, arm assignment, baseline characteristics, and outcomes of interest. Collectively, these 7 models provide a comprehensive basis for evaluating the correctness, evidence alignment, and clinical relevance of LLM-generated answers. For six of the sources, all of the nearly 3,000 questions were evaluated. A subset of 1,739 questions was randomly selected for evaluation using the Perplexity platform. This subset was chosen to balance computational cost and runtime constraints while maintaining sufficient diversity to represent the broader range of biomedical queries. Evidence Evaluation We apply a structured evaluation rubric composed of three binary (True/False) criteria. Each criterion isolates a specific dimension of answer quality with respect to its contextual support. View this table: View inline View popup Download powerpoint Once the appropriate context is retrieved and an AI-generated answer returned, we use a structured prompt to judge the answers’ relevance and grounding. This evaluation assesses how well the AI-generated answer responds to the original question, given only the retrieved context (Full prompt details available in Supplemental Figure S1) To simplify interpretation, each evaluated question-answer pair is assigned a color-coded Answered with Evidence badge based on the boolean fields from the structured prompt output. An example of the evaluation of a question-response context and badge assignment is shown in Figure 2 . Download figure Open in new tab Figure 2 – Example Context Evaluation and Badge Assignment View this table: View inline View popup This logic provides a quick way to assess the quality and reliability of evidence grounding in biomedical question-answer responses. The badge system is designed to classify the factual alignment of AI-generated answers and signal the evidence landscape surrounding a biomedical question. The green badge indicates a high-confidence response. These answers represent areas of well-established evidence where the AI-generated answer is effectively synthesizing known information. The yellow badge indicates the context is relevant to the question, and the answer is grounded in it, but the context does not directly answer the question. It reflects situations where indirect evidence or background knowledge exists, but no study has yet addressed the specific question. This marks the evidence gap and the opportunity to generate a novel study. The red badge indicates the context is not related, the answer introduces hallucinated content, or both. It represents a low-confidence response. In addition to the badging, each LLM was instructed to provide a reference to the document/publication where the answer was pulled, as well as what text within the reference material is the basis for the answer. These references were manually verified to be non-hallucinated, and the snippet of interest was manually verified to exist within the document itself. Results Evidence Criterion Assessment and Answered with Evidence Badge assignment across evidence sources All questions were submitted to the seven evaluated evidence sources, and each response was assessed based on the criteria described in the Evidence Evaluation section. A summary of the badge assignment for the three evidence sources can be found on Table 1 . For the PubMed-based sources, the System platform produced high-confidence, evidence-grounded responses (green badge) for 44.6% of queries. Perplexity, when restricted to academic literature, achieved green-badged responses for 21.3% of cases, and Claude with a similar restriction, had a green badge 29.1% of the time. For the general-purpose LLMs a green badge was assigned in Gemini 45.2% of the time, OpenAI o3 71% of the time, and GPT-5 67.1% of the time. The higher rate of green badge assignment in the general-purpose LLMs was a surprising result that caused us to evaluate the reference document and reference snippet that supported the finding (see Intersouce Agreement for Evidence Badge Assignment) The remaining responses from these sources were assigned Yellow or Red badges, indicating that either the answer lacked sufficient evidence grounding (Yellow) or was irrelevant to the original question (Red). For full details, see table S1. View this table: View inline View popup Table 1 – Answered with evidence badge summary across evidence sources. In comparison, Alexandria, the Atropos Evidence Library, which generates responses using structured real-world evidence from EHR, claims, and other clinical datasets, produced Green-badged responses for 50.1% of the submitted questions. The remaining 49.9% were rated Yellow or Red, typically due to limitations in available real-world data or ambiguity in the question framing. These findings suggest that novel real-world evidence sources may provide comparable or improved grounding relative to PubMed-based and general web search retrieval systems, particularly for complex or underrepresented clinical scenarios. Intersource agreement for Evidence Badge assignment To assess the consistency of evidence grounding, we compared badge assignments, references provided by LLM, and snippet of supporting evidence from the different evidence sources. Our hypothesis was that the Pubmed-base sources (System, Claude, and Perplexity) would show a high concordance of badge assignments, that the web-search based models (Open AI and Gemini) would have the highest percentage of green-badges, however some citations originating from non-relieable sources, and the custom real-world evidence source (Alexandria) would have the most novel answers that could not be answered by the other sources. Within each group, we anticipated only partial agreement due to differences in retrieval mechanisms and response generation strategies. In comparing responses that obtained a Green badge (found to be answered with evidence) the highest agreement between sources was between Atropos and OpenAI ( Table 2 ; 37.39%). The lowest agreement in Green badge assignment was between Perplexity and Claude (5.5%). In both cases, the agreement is driven by the percent of questions that received the green badge for each source; Atropos and Open AI have the most green badges, and Perplexity and Claude having the fewest. For a full breakdown of pairwise/global intersource badge agreement on Yellow and Red badges, please see table S2 and S3. When considering all sources, 88% of all submitted questions were able to return a response that received a green badge. View this table: View inline View popup Download powerpoint Table 2 – Pairwise agreement between different evidence sources Every questionw as submitted to each LLM and the output was evaluated with the Answered with evidence framework. For each evidence source a pairwise agreement was calculated for Green Badges (Shown) and Red/Yellow (extended table S1). The highest agreement that question could be answered with evidence was between Atropos/OpenAI. The high variability between green badges in different evidence sources trained on theoretically similar sources was surprising to us. To explore why this was the case we explored the citation returned by the LLM describing where the evidence was sourced as well as the snippet identified as containing the evidence. This cross-model deep evaluation was performed on 100 questions, limited to Claude, Perplexity, Open AI, and Gemini all which make retrieval of the source URL possible ( Table 3 ). We identified the the highest similarity of source was between Claude/Perplexity at 53%. Given that both can be limited to only utilize Pubmed, the fact they had the highest agreement is to be expected but only 53% agreement was surprising. On the other side we found no agreement across the sample between OpenAI and Gemini in the referenced document. When two our more sources referenced the same document, we evaluated similarity between snippets and found different statements from the same article serving as reference. View this table: View inline View popup Download powerpoint Table 3 – Cited Source agreement across evidence sources Given the high variability in the referenced documentation and the snippets used for supporting the LLM response, we evaluated how often one or more LLMs had a green badge. For the non-custom-RWE evidence sources, at least one gave a Green badge 81% of the time but this dropped to 49% when we required two or more LLMs to have a green badge, and 25% when requiring 3 ore more LLMs to achieve consensus of an answer being grounded in evidence. Adding to the generalized LLMs the custom RWE source (Alexandria) we find that we get 2 or more green badged responses 64% of the time and 3 or more 38% of the time. To assess each source’s unique contribution, we also measured novelty —defined as the proportion of questions where only one source produced a Green-badged response, while the others returned Yellow or Red. Novelty rates were 14.0% for Open AI, 7.6% for Alexandria, 1.8% for Gemini, and less than 1% for Perplexity, Claude, and System. These results suggest that real-world evidence sources and general-purpose LLM systems each provide distinct and valuable contributions to biomedical question answering. Discussion One of the enduring challenges in clinical care and health policy is the ability to access evidence that directly addresses the question at hand 21 , 22 . Despite the widespread adoption of evidence-based medicine, physicians and decision-makers often face a gap between available evidence and real-world information needs. In part, this challenge stems from the sheer volume and fragmentation of biomedical literature, which far exceeds the capacity of any individual to comprehensively access, evaluate, and synthesize to ground every decision in the latest evidence 2 , 4 , 5 . The emergence of large language models (LLMs) with capabilities in literature retrieval and summarization offers a promising path forward 6 . However, even with accurate identification and synthesis of relevant publications, there remains a persistent evidence gap—many questions simply fall outside the scope of existing published studies. Bridging this gap requires not only tools to extract and summarize existing literature, but also systems that can surface or generate novel real-world evidence in response to specific clinical or research queries 17 , 18 . In this study, we introduced Answered with Evidence , a framework for evaluating whether LLM-generated responses are (1) relevant, (2) directly responsive to the question, and (3) grounded in evidence. Each response is scored across these criteria and summarized using an intuitive traffic light–style badge system—Green (high confidence and evidence-based), Yellow (partial or uncertain grounding), and Red (unsupported or irrelevant). This system provides users with rapid, interpretable feedback on the reliability of an answer. Across the six general-purpose or literature-grounded LLMs, Open AI’s o3 and GPT-5 models generated the highest rate of Green-badged responses (70.1% and 67.1% respectively), compared to between 21% and 45% for the other models evaluated. Notably, despite all models being grounded in Pubmed/web search they exhibited markedly different performance profiles. The discrepancy in Green badge rates and novelty between these systems suggests that training data alone does not fully explain differences in response quality. Architecture, retrieval mechanisms, response-generation, sentence structure, and word use also likely play a significant role. Given that these models have such a high variability in the evaluation, we looked closer at the document referenced as the source of truth and the snippet within the document that the LLM surfaced as the grounding for its answer. This evaluation demonstrated that there was high variability between the documents selected as references. The highest agreement between any two sources was 53% (Claude and Perplexity), with no identified document overlap between OpenAI and Gemini. This high variability in the reference document suggests that no single publication-based model can be trusted alone. Missing one or two critical manuscripts about a given clinical scenario would result in a confidently worded but ultimately incorrect and likely dangerous answer. Instead, building consensus from identifying the same answer surfaced from multiple LLMs is critical. This finding is supported by other such cross-LLM comparisons where utilization of LLM-as-jury has been utilized to find the consensus answer 23 . We similarly take an ensemble approach where a single question is considered sufficiently answered by an LLM only if two or more LLMs are evaluated to have a green badge signifying the response is grounded in evidence. Using this methodology 49% of all questions had a component of the question answered with well-grounded published evidence. When adding in the custom-built RWE to answer such questions, the combined sources can answer nearly 64% of questions of interest to physicians. These findings underscore the value of combining both published and real-world evidence in biomedical question answering. Assuming continued improvements in LLM retrieval fidelity and hallucination reduction, we expect PubMed-based systems to eventually converge toward similar badge distributions—potentially reaching a ceiling of around 55% green-badge assignment by 2 or more LLMs. In this context, the ability to access or generate novel evidence becomes essential. In our analysis, supplementing general-purpose LLMs with Alexandria increased the proportion of high-confidence responses by an additional 15% to the ensemble approach, demonstrating the unique value of real-world evidence in addressing unanswered clinical questions ( Figure 3 ). Download figure Open in new tab Figure 3 – Current Evidence and LLM Capability Synergistic performance A library of custom built RWE to answer questions is able to identify answers to 50% of provider inquiries evaluated. Similarly, an ensemble approach requiring 2 or more general purpose LLMs to produce an evidence grounded answer can produce an answer to physician queries 49% of the time. Combining both approaches over 64% of difficult to answer questions by physicians can be answered with current methodology and evidence. This study has several limitations. First, only a single evaluation framework and LLM question prompt were utilized across the different LLM responses. As a next step, iteration of both the evaluation framework and the question submission prompt should be performed. Given that all the LLMs have different response structures, it is possible that a different evaluation framework needs to be utilized for each LLM. Second, several emerging LLM-based literature summarization platforms were not included in this initial analysis and warrant future benchmarking. Third, and most critically, these responses are meant to inform clinical decisions, and an evaluation of the response and reference document/snippet by a panel of physicians will provide a gold label to determine the accuracy of both the budgeting and the ensemble approach. Conclusion Accessing pertinent and reliable evidence remains a significant challenge for healthcare providers, health system leaders, and biomedical researchers. While large language models (LLMs) offer a powerful means to navigate the vast and expanding body of published literature, our findings demonstrate that, even with advanced retrieval and summarization capabilities, approximately 50% of biomedical questions remain inadequately addressed using published sources alone. Similarly, a real-world evidence (RWE) library specifically designed to address gaps in the literature still left nearly half of the questions unanswered. However, when combined, these complementary evidence sources enabled high-confidence answers for nearly two-thirds of clinician– and researcher-submitted inquiries. These findings highlight a key insight: advancing biomedical question answering will require more than improved retrieval of existing literature—it will also require the generation and integration of new evidence generated with the same methodological rigor required for peer-review. To realize the full potential of LLMs in clinical and research workflows, investment in scalable, structured real-world evidence generation will be as important as continued advances in model architecture and prompt engineering. Data Availability All data produced in the present study are available upon reasonable request to the authors. Supplement Download figure Open in new tab Download figure Open in new tab Figure S1: The prompt returns a JSON-like response structured as: Download figure Open in new tab View this table: View inline View popup Table S1 – Pairwise Agreement Across Sources View this table: View inline View popup Table S2 – Agreement Matrix across all Evidence Sources Footnotes This manuscript was revised to report the results of additional experiments: -An experiment evaluating the LLM-referenced citations for hallucination -An experiment evaluating the LLM-referenced snippet for hallucination -An experiment evaluating cross-LLM referenced citation for similarity -An ensemble approach to determining consensus between LLMs References 1. ↵ Sackett DL , Rosenberg WM , Gray JA , Haynes RB , Richardson WS . Evidence based medicine: what it is and what it isn’t . BMJ . 1996 ; 312 ( 7023 ): 71 – 72 . doi: 10.1136/bmj.312.7023.71 OpenUrl FREE Full Text 2. ↵ Fanaroff AC , Califf RM , Windecker S , Smith SC , Lopes RD . Levels of Evidence Supporting American College of Cardiology/American Heart Association and European Society of Cardiology Guidelines, 2008-2018 . JAMA . 2019 ; 321 ( 11 ): 1069 – 1080 . doi: 10.1001/jama.2019.1122 OpenUrl CrossRef PubMed 3. ↵ He J , Morales DR , Guthrie B . Exclusion rates in randomized controlled trials of treatments for physical conditions: a systematic review . Trials . 2020 ; 21 ( 1 ): 228 . doi: 10.1186/s13063-020-4139-0 OpenUrl CrossRef PubMed 4. ↵ Alper BS , Hand JA , Elliott SG , et al. How much effort is needed to keep up with the literature relevant for primary care? J Med Libr Assoc . 2004 ; 92 ( 4 ): 429 – 437 . OpenUrl PubMed Web of Science 5. ↵ Wangler J , Jansky M . What is the significance of guidelines in the primary care setting? Wien Med Wochenschr 1946. 2021 ; 171 ( 13-14 ): 321 – 329 . doi: 10.1007/s10354-021-00849-3 OpenUrl CrossRef PubMed 6. ↵ Zakka C , Shad R , Chaurasia A , et al. Almanac – Retrieval-Augmented Language Models for Clinical Medicine . Nejm Ai . 2024 ; 1 ( 2 ). doi: 10.1056/aioa2300068 OpenUrl CrossRef 7. Chen S , Kann BH , Foote MB , et al. Use of Artificial Intelligence Chatbots for Cancer Treatment Information . JAMA Oncol . 2023 ; 9 ( 10 ): 1459 – 1462 . doi: 10.1001/jamaoncol.2023.2954 OpenUrl CrossRef PubMed 8. ↵ Low YS , Jackson ML , Hyde RJ , et al. Answering real-world clinical questions using large language model, retrieval-augmented generation, and agentic systems . Digit Health . 2025 ; 11 : 20552076251348850 . doi: 10.1177/20552076251348850 OpenUrl CrossRef PubMed 9. ↵ Chelli M , Descamps J , Lavoué V , et al. Hallucination Rates and Reference Accuracy of ChatGPT and Bard for Systematic Reviews: Comparative Analysis . J Med Internet Res . 2024 ; 26 ( 1 ): e53164 . doi: 10.2196/53164 OpenUrl CrossRef PubMed 10. OpenEvidence . OpenEvidence – OpenEvidence AI becomes the first AI in history to score above 90% on the United States Medical Licensing Examination (USMLE) . OpenEvidence . Accessed June 30, 2025 . https://www.openevidence.com/announcements/openevidence-ai-first-ai-score-above-90-percent-on-the-usmle 11. ↵ Performance of ChatGPT on USMLE : Potential for AI-assisted medical education using large language models | PLOS Digital Health . Accessed June 30, 2025 . https://journals.plos.org/digitalhealth/article?id=10.1371/journal.pdig.0000198 12. ↵ Bedi S , Jiang Y , Chung P , Koyejo S , Shah N . Fidelity of Medical Reasoning in Large Language Models . JAMA Netw Open . 2025 ; 8 ( 8 ): e2526021 . doi: 10.1001/jamanetworkopen.2025.26021 OpenUrl CrossRef 13. ↵ Perplexity . Perplexity AI . Accessed June 30, 2025 . https://www.perplexity.ai 14. ↵ Graph . Accessed June 30, 2025 . https://www.system.com/platform/system-graph 15. ↵ Bedi S , Cui H , Fuentes M , et al. MedHELM: Holistic Evaluation of Large Language Models for Medical Tasks . arXiv . Preprint posted online June 2, 2025 . doi: 10.48550/arXiv.2505.23802 OpenUrl CrossRef 16. ↵ Tam TYC , Sivarajkumar S , Kapoor S , et al. A framework for human evaluation of large language models in healthcare derived from literature review . Npj Digit Med . 2024 ; 7 ( 1 ): 258 . doi: 10.1038/s41746-024-01258-7 OpenUrl CrossRef PubMed 17. ↵ Gombar S , Callahan A , Califf R , Harrington R , Shah NH . It is time to learn from patients like mine . Npj Digit Med . 2019 ; 2 ( 1 ): 16 . doi: 10.1038/s41746-019-0091-3 OpenUrl CrossRef PubMed 18. ↵ Callahan A , Gombar S , Cahan EM , et al. Using Aggregate Patient Data at the Bedside via an On-Demand Consultation Service . NEJM Catal . 2021 ; 2 ( 10 ). doi: 10.1056/CAT.21.0224 OpenUrl CrossRef 19. ↵ Home \ Anthropic . Accessed August 28, 2025. https://www.anthropic.com/ 20. ↵ Lewis P , Perez E , Piktus A , et al. Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks . arXiv . Preprint posted online April 12, 2021 . doi: 10.48550/arXiv.2005.11401 OpenUrl CrossRef 21. ↵ Stewart WF , Shah NR , Selna MJ , Paulus RA , Walker JM . Bridging the inferential gap: the electronic health record and clinical evidence . Health Aff Proj Hope . 2007 ; 26 ( 2 ): w181 – 191 . doi: 10.1377/hlthaff.26.2.w181 OpenUrl Abstract / FREE Full Text 22. ↵ Darst JR , Newburger JW , Resch S , Rathod RH , Lock JE . Deciding without data . Congenit Heart Dis . 2010 ; 5 ( 4 ): 339 - 342 . doi: 10.1111/j.1747-0803.2010.00433.x OpenUrl CrossRef PubMed Web of Science 23. ↵ Baysan MS , Uysal S , İşlek İ , Çığ Karaman Ç , Güngör T . LLM-as-a-Judge: automated evaluation of search query parsing using large language models . Front Big Data . 2025 ; 8 : 1611389 . doi: 10.3389/fdata.2025.1611389 OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted September 01, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Introducing Answered with Evidence - a framework for evaluating whether LLM responses to biomedical questions are founded in evidence Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Introducing Answered with Evidence - a framework for evaluating whether LLM responses to biomedical questions are founded in evidence Julian D Baldwin , Christina Dinh , Arjun Mukerji , Neil Sanghavi , Saurabh Gombar medRxiv 2025.07.01.25330655; doi: https://doi.org/10.1101/2025.07.01.25330655 Share This Article: Copy Citation Tools Introducing Answered with Evidence - a framework for evaluating whether LLM responses to biomedical questions are founded in evidence Julian D Baldwin , Christina Dinh , Arjun Mukerji , Neil Sanghavi , Saurabh Gombar medRxiv 2025.07.01.25330655; doi: https://doi.org/10.1101/2025.07.01.25330655 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15229) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6600) Geriatric Medicine (668) Health Economics (997) Health Informatics (4536) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (541) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3332) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9232) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a00be7145c1ee2c5',t:'MTc3OTYyMTUzNw=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();