Prompting is all you need: LLMs for systematic review screening

Prompting is all you need: LLMs for systematic review screening | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Prompting is all you need: LLMs for systematic review screening View ORCID Profile Christian Cao , Jason Sang , View ORCID Profile Rohit Arora , View ORCID Profile Robbie Kloosterman , View ORCID Profile Matt Cecere , View ORCID Profile Jaswanth Gorla , View ORCID Profile Richard Saleh , David Chen , View ORCID Profile Ian Drennan , View ORCID Profile Bijan Teja , View ORCID Profile Michael Fehlings , View ORCID Profile Paul Ronksley , View ORCID Profile Alexander A Leung , View ORCID Profile Dany E Weisz , View ORCID Profile Harriet Ware , View ORCID Profile Mairead Whelan , View ORCID Profile David B Emerson , View ORCID Profile Rahul Arora , View ORCID Profile Niklas Bobrovitz doi: https://doi.org/10.1101/2024.06.01.24308323 Christian Cao 1 Temerty Faculty of Medicine, University of Toronto , Toronto, ON, Canada 2 Centre for Health Informatics, Department of Community Health Sciences, University of Calgary , Calgary, AB, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Christian Cao For correspondence: christian.cao{at}mail.utoronto.ca Jason Sang 3 Stripe, Inc., San Francisco , CA, United States Find this author on Google Scholar Find this author on PubMed Search for this author on this site Rohit Arora 4 Department of Biomedical Informatics, Harvard Medical School , Boston, MA, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Rohit Arora Robbie Kloosterman 1 Temerty Faculty of Medicine, University of Toronto , Toronto, ON, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Robbie Kloosterman Matt Cecere 1 Temerty Faculty of Medicine, University of Toronto , Toronto, ON, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Matt Cecere Jaswanth Gorla 1 Temerty Faculty of Medicine, University of Toronto , Toronto, ON, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Jaswanth Gorla Richard Saleh 1 Temerty Faculty of Medicine, University of Toronto , Toronto, ON, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Richard Saleh David Chen 1 Temerty Faculty of Medicine, University of Toronto , Toronto, ON, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ian Drennan 1 Temerty Faculty of Medicine, University of Toronto , Toronto, ON, Canada 5 Department of Emergency Services and Sunnybrook Research Institute, Sunnybrook Health Science Centre 6 Orange Air Ambulance and Critical Care Transport Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Ian Drennan Bijan Teja 7 Department of Anesthesiology and Pain Medicine, University of Toronto , Toronto, ON, Canada 8 Department of Anesthesia and Critical Care Medicine, St. Michael’s Hospital , Toronto, Ontario, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Bijan Teja Michael Fehlings 9 Department of Surgery, Temerty Faculty of Medicine, University of Toronto , Toronto, ON, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Michael Fehlings Paul Ronksley 10 Department of Community Health Sciences, Cumming School of Medicine, University of Calgary , Calgary, AB, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Paul Ronksley Alexander A Leung 11 Department of Medicine and Department of Community Health Sciences, Cumming School of Medicine, University of Calgary , AB, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Alexander A Leung Dany E Weisz 12 Department of Newborn and Developmental Paediatrics, Sunnybrook Health Sciences Centre , Toronto, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Dany E Weisz Harriet Ware 2 Centre for Health Informatics, Department of Community Health Sciences, University of Calgary , Calgary, AB, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Harriet Ware Mairead Whelan 2 Centre for Health Informatics, Department of Community Health Sciences, University of Calgary , Calgary, AB, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Mairead Whelan David B Emerson 13 Vector Institute , Toronto, ON, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for David B Emerson Rahul Arora 2 Centre for Health Informatics, Department of Community Health Sciences, University of Calgary , Calgary, AB, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Rahul Arora Niklas Bobrovitz 2 Centre for Health Informatics, Department of Community Health Sciences, University of Calgary , Calgary, AB, Canada 14 Department of Emergency Medicine, Cumming School of Medicine, University of Calgary , Calgary, AB, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Niklas Bobrovitz Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Systematic reviews (SRs) are the highest standard of evidence, shaping clinical practice guidelines, policy decisions, and research priorities. However, their labor-intensive nature, including an initial rigorous article screen by at least two investigators, delays access to reliable information synthesis. Here, we demonstrate that large language models (LLMs) with intentional prompting can match human screening performance. We introduce Framework Chain-of-Thought, a novel prompting approach that directs LLMs to systematically reason against predefined frameworks. We evaluated our prompts across ten SRs covering four common types of SR questions (i.e., prevalence, intervention benefits, diagnostic test accuracy, prognosis), achieving a mean accuracy of 93.6% (range: 83.3-99.6%) and sensitivity of 97.5% (89.7-100%) in full-text screening. Compared to experienced reviewers (mean accuracy 92.4% [76.8-97.8%], mean sensitivity 75.1% [44.1-100%]), our full-text prompt demonstrated significantly higher sensitivity in four reviews (p<0.05), significantly higher accuracy in one review (p0.05). While traditional human screening for an SR of 7000 articles required 530 hours and $10,000 USD, our approach completed screening in one day for $430 USD. Our results establish that LLMs can perform SR screening with performance matching human experts, setting the foundation for end-to-end automated SRs. Main Systematic reviews (SR) are rigorous forms of knowledge synthesis that involve the gathering, critical appraisal, and analysis of evidence. Recognized as the gold standard in evidence-based practice, SRs bolster decision-making across various domains including medicine, business, and agriculture, among others. 1 SRs are resource-intensive, typically requiring one year and upwards of $100,000 to complete. 2 , 3 These costs stem from the comprehensive processes of conducting detailed searches, screening articles, extracting data, analyzing findings, and report writing. 2 – 4 The screening phase is particularly demanding and typically involves two investigators working independently, and in duplicate, to identify articles that meet predefined eligibility criteria. 1 , 5 Investigators begin with an initial sensitive title and abstract screen, followed by an accurate screen of article full-texts. 1 , 5 Despite a growing catalog of tools and resources, 6 SR automation is elusive as existing tools only supplement human workflows and lack the performance required for independent decision making. 7 The rise of large language models (LLMs), such as GPT, with advanced generative capabilities creates new horizons for streamlining and automating SR processes. 8 , 9 MedPrompt, 10 a collection of prompting techniques to optimize GPT4 performance on medical benchmarks, has demonstrated that generalist foundation models can surpass traditional model fine-tuning methods simply through better prompting. However, evaluations of LLMs in the medical domain have faced issues such as irreproducibility due to the use of web browser applications with hidden prompts, unrecorded model versions and settings, and the transient nature of chatbot histories. Furthermore, basic zero-shot prompting–where LLMs are only given task instructions without examples–likely underestimates model capabilities. Consequently, past zero-shot assessments of LLMs for SR screening have demonstrated low sensitivity/recall, failing to capture relevant studies and compromising their use for automation. 11 – 14 Here, we present a comprehensive evaluation of LLM performance in SR screening and prompting innovations that enhance screening efficacy. First, we address a critical gap in SR automation and create BenchSR , a robust set of 10 SR datasets covering diverse medical question types and domains. We then introduce two prompting innovations: ‘Framework Chain of Thought (CoT),’ a novel prompting approach to guide models towards systematic reasoning against predetermined frameworks, and ‘Instruction Structure Optimized (ISO)’ prompting, designed to address LLM context-loss. We propose Abstract ScreenPrompt and ISO-ScreenPrompt for abstract and full-text screening, and demonstrate that their performance can match human reviewers. We evaluate our strategies across seven generalist LLMs (GPT3.5, GPT4-0125-preview, GPT4-Turbo-0409, GPT4o-0513, Gemini Pro, Mixtral-8×22, Mistral-Large) and demonstrate that our findings are model agnostic. Our work highlights the importance of applying rigorous prompting techniques to fully leverage the capabilities of LLMs, and serves as a guide for future researchers interested in performing LLM evaluations for SR screening and beyond. Collectively, our datasets and findings illustrate the feasibility of LLM-automated SR screening and lay the groundwork for fully automating the SR workflow. Results Datasets and BenchSR We curated BenchSR , a collection of 10 SR datasets comprising over 170,000 articles, spanning four of six Oxford Center for Evidence-Based Medicine (CEBM) question types, 15 , 16 and nine different clinical domains 17 ( Table 1 , Fig. 1 ). This compilation includes SR metadata (inclusion/exclusion criteria, study objectives), and the complete set of labeled articles (included, excluded) for each review. Download figure Open in new tab Figure 1: Infographic of study design. ScreenPrompt achieves SOTA performance for abstract and full-text SR screening. The LLM assessment component shows evaluation of the SeroTracker (ST) dataset (n=400) used in head-to-head human and LLM testing. Error bars represent 95% CIs for binomial proportions. View this table: View inline View popup Table 1: Descriptive Overview of BenchSR Datasets Abstract Prompt Engineering We evaluated the performance of three standard prompting methodologies on our training split dataset (SeroTracker [ST] train split; Methods, Testing Methodology) ( Fig. 1 , Fig. 2a , Table 2 ). Unless otherwise stated, we prompted the GPT4-0125-preview model with the ST train split. Initial tests with zero-shot prompting approaches (Methods Prompt Engineering, Table 2 ), adapted from Guo et al., 11 revealed suboptimal performance, with an accuracy of 65.0%, 30% sensitivity, and 100% specificity ( Fig. 2b , Supplementary Table 1 ). These results were improved with random few-shot prompting (78% accuracy, 56% sensitivity, 100% specificity) ( Fig. 2b , Supplementary Table 1 ). Rephrasing prompts to be more inclusive also improved sensitivity and performance (+3.3% accuracy, +6.5% sensitivity; Supplementary Table 2 , Table 2 ), and we applied this consideration when prompting thereafter. Finally, Zero-shot CoT prompting 18 was associated with notable improvements in accuracy (86.3% accuracy, 74.5% sensitivity, 98% specificity) ( Fig. 2b , Supplementary Table 1 ), although model accuracy and sensitivity were relatively low for independent SR screening. Download figure Open in new tab Figure 2: Abstract ScreenPrompt achieves SOTA performance for abstract screening, generalizing across studies a) Diagram of different prompting strategies used for abstract screening, including Zero-shot, Few-shot, Chain-of-Thought (CoT), Framework CoT, and Few-shot GPT-CoT. The Abstract ScreenPrompt approach integrates study objectives, inclusion/exclusion criteria, and abstract-specific instructions to guide reasoning. b) Performance comparison of different abstract prompting methodologies on the SeroTracker training, showing accuracy and sensitivity. Abstract ScreenPrompt is evaluated within the ST training dataset (n=400) across GPT4-0125-preview, GPT-3-5, Gemini-Pro, GPT4-o-0513, GPT4-Turbo-0409, Mixtral-8×22, Mistral-Large. Abstract ScreenPrompt is also separately evaluated on the ST validation dataset (n=400). Error bars represent 95% CIs for binomial proportions. c) Barplot displaying generalizability of zero-shot Abstract ScreenPrompt sensitivity and accuracy across 10 different systematic review datasets from BenchSR . Error bars represent 95% CIs for exact proportions with the Clopper-Pearson method. View this table: View inline View popup Table 2: Abstract and Full-text Screening Prompts We analyzed zero-shot CoT outputs and identified inconsistencies in the generated reasoning structures for article inclusion ( Supplementary Table 3 ). This insight inspired us to experiment with directing the LLM to articulate its reasoning according to the predefined inclusion and exclusion criteria. This novel zero-shot prompting approach was termed ‘Framework CoT’ and reported notable performance improvements (91.3% accuracy, 86.5% sensitivity, 96% specificity) ( Fig. 2b , Supplementary Table 1 , Methods Abstract Prompt Engineering). Further analysis of incorrect zero-shot framework CoT model outputs revealed that the model generated incorrect inferences of review study objectives based on the inclusion and exclusion criteria ( Supplementary Table 3 ). To counteract this, we inserted study objectives directly from the SR protocol or manuscript, and defined our prompting strategy as ‘ScreenPrompt’ ( Table 2 ). ScreenPrompt was further refined by adding context about the limitations of abstract content and importance of article inclusion (‘+ abstract’, Supplementary Fig. 1a , Supplementary Table 4 ). This final abstract-specific prompting strategy was referred to as ‘Abstract ScreenPrompt’ ( Table 2 , Supplementary Fig. 1a , Supplementary Table 4 ). We found that our Abstract ScreenPrompt prompting approach had the best performance on our ST training dataset (94.3% accuracy, 94.5% sensitivity, 94% specificity) ( Fig. 2b , Supplementary Table 1 ). To check for risk of overfitting, we evaluated our Abstract ScreenPrompt approach on our ST validation dataset, and found comparable performance (94.3% accuracy, 96% sensitivity, 92.5% specificity) ( Fig. 2b , Supplementary Table 1 ), confirming that our prompt engineering process did not overfit the training dataset. While few-shot prompting (adding additional labeled examples) is traditionally believed to enhance LLM performance, 19 our balanced few-shot GPT-CoT Abstract ScreenPrompt prompt was associated with decreased performance (85.8% accuracy, 98.0% sensitivity, 73.5% specificity). We hypothesized that we could modulate model specificity and sensitivity by adjusting the ratio of included and excluded few-shot examples ( Supplementary Fig. 1b , Supplementary Table 5 ). Surprisingly, we found that our sensitivity analysis with varied ratios (9:1 inclusion- or exclusion-favored GPT-CoT Abstract ScreenPrompt few-shot examples) did not notably alter performance for inclusion-favored few-shot prompting (inclusion-favored: 85% accuracy, 98% sensitivity, 72% specificity), and was associated with a decrease in performance for exclusion-favored few-shot prompting (exclusion-favored: 83.8% accuracy, 98.5% sensitivity, 69% specificity). Abstract Screening Performance across LLMs We conducted a comparative analysis across GPT4-0125-preview, GPT4-Turbo-0409, GPT4o-0513, GPT-3.5, Gemini Pro, Mixtral-8×22, and Mistral-Large LLM models to assess performance differences. We utilized the same ST training dataset and Abstract ScreenPrompt prompting strategy ( Fig. 2b , Supplementary Table 6 ). Our cross-model evaluation found that the GPT4-0125-preview model was associated with the greatest performance (94.3% accuracy, 94.5% sensitivity, 94% specificity), but also had the highest cost ($12.54, 400 abstracts). In contrast, the GPT3.5 model was associated with the lowest overall performance, but had high sensitivity (66.9% accuracy, 90.5% sensitivity, 43% specificity). The Gemini Pro model had relatively low performance (77.5% accuracy, 67.5% sensitivity, 84.8% specificity), but was free to run. We found a moderate drop in accuracy and sensitivity with the newer GPT4-Turbo-0409 (89.8% accuracy, 83.5% sensitivity, 96% specificity) and GPT4o-0513 models (89.0% accuracy, 79.5% sensitivity, 98.5% specificity). To confirm that our prompting innovations were model agnostic, we compared Zero-Shot and Abstract ScreenPrompt performance in our lowest performing (GPT3.5) and next best performing (GPT4-Turbo-0409) models ( Supplementary Table 7 ). We found that Abstract ScreenPrompt was associated with improvements in accuracy in both models, with the greatest sensitivity gains in GPT4-Turbo-0409 (+70% sensitivity). Generalizability of Abstract ScreenPrompt We hypothesized that our Abstract ScreenPrompt could be readily adapted for abstract screening beyond the ST dataset. To test this, we applied our prompt to 10 distinct SR datasets within our BenchSR ( Fig. 2c , Supplementary Table 8 ), using original study objectives and eligibility criteria for each dataset without modification. Our Abstract ScreenPrompt demonstrated high sensitivity across reviews (97.0% [range: 86.7-100.0%]), in contrast to zero-shot prompting (53.4% mean sensitivity [16.7-87.6%]). Both Abstract ScreenPrompt (88.4% [73.5-95.5%]) and zero-shot prompting (89.8% [71.5-98.0%]) produced comparable accuracy. Our previous evaluations were performed with a single sampled generation. However, due to the stochasticity of LLM generations, the model may produce varying reasoning traces for a given prompt. Therefore, we applied 11-vote self-consistency (SC) 20 to Abstract ScreenPrompt on the ST test dataset, and the Reinfection dataset, which had the lowest specificity ( Supplementary Fig. 1c , Supplementary Table 9 ). We found that Abstract ScreenPrompt-SC was associated with an overall gain in performance in both datasets across all metrics (ST: +2.8% accuracy, +2% sensitivity, +3.5% specificity; Reinfection: +1.9% accuracy, +1.7% sensitivity, +1.8% specificity). Additionally, we could modulate sensitivity and specificity by setting different self-consistency thresholds (number of votes needed) for article inclusion/exclusion ( Supplementary Fig. 1d , Methods Prompt Engineering). Collectively, these findings suggest that our Abstract ScreenPrompt prompting strategy is readily generalizable and can perform well across different SR contexts. Full-text Prompt Engineering Building on the performance of Abstract ScreenPrompt, we evaluated the capabilities of the LLMs for full-text article screening ( Fig. 3a , Supplementary Table 10 -11, Table 2 ). Unless otherwise stated, we prompted the GPT4-0125-preview model with the full-text ST training dataset. Our original Abstract ScreenPrompt approach, tailored for inclusion, demonstrated high sensitivity (84.2% accuracy, 99% sensitivity, 69.2% specificity) ( Fig. 3a ). However, we sought to prioritize accuracy in downstream optimization steps. Download figure Open in new tab Figure 3: ISO-ScreenPrompt achieves SOTA performance for full-text screening, generalizing across studies a) Diagram illustrating the various prompting strategies used for full-text screening, including Zero-shot, Few-shot, Chain-of-Thought (CoT), Framework CoT, and Few-shot GPT-CoT. The ISO-ScreenPrompt approach incorporates repeated prompt modules (i.e., objectives, pre-prompt, eligibility criteria, instructions) and numbering to enhance performance. b) Performance comparison of different full-text prompting methodologies on the SeroTracker (ST) training dataset (n=400), showing accuracy and sensitivity. ISO-ScreenPrompt is evaluated across multiple models with the ST training dataset: GPT4-0125-preview, GPT-3.5, Gemini-Pro, GPT4o-0513, GPT4-Turbo-0409, Mixtral-8×22, and Mistral-Large. ISO-ScreenPrompt is also separately evaluated on the ST validation dataset (n=400). Error bars represent 95% CIs for binomial proportions. c) Barplot displaying the generalizability of the zero-shot ISO-ScreenPrompt sensitivity and accuracy across 10 different systematic review datasets from BenchSR . Error bars represent 95% CIs for exact proportions with the Clopper-Pearson method. While abstracts are concise summaries of study findings, full-text articles can span thousands of words. Therefore we hypothesized that adjustments to our prompt structure—without making semantic changes to prompt content—could aid full-text screening. This hypothesis was based on the ‘lost-in-the-middle’ phenomenon, 21 , 22 which refers to the variable information retrieval rates of LLMs within lengthy texts. Our experiments positioning the pre-prompt, inclusion criteria, and instruction prompt modules before the full-text article was associated with modest performance gains (Init: 92.5% accuracy, 94.5% sensitivity, 90.5% specificity), while placing them after the full-text articles reduced performance (Fin: 82.2% accuracy, 98% sensitivity, 66.3% specificity) ( Supplementary Fig. 2a , Supplementary Table 11 ). With ‘init’ prompting, the model occasionally provided a single token decision with no additional reasoning ( Supplementary Table 12 ). Appending additional instructions at the end of the prompt helped mitigate this inconsistency (‘Init + Fin-Instructions’, Supplementary Table 12 ). Further improvements were observed with our Framework CoT (init + fin) prompt, where we appended the complete set of prompt modules before and after the full-text article (94.8% accuracy, 95% sensitivity, and 94.5% specificity) ( Fig. 3a , Supplementary Fig. 2a , Supplementary Table 11 ). Applying the same prompting strategy to abstract screening did not notably improve results, ( Supplementary Fig. 2b , Supplementary Table 13 ) possibly because abstracts are too short to manifest the lost-in-the-middle phenomenon. When analyzing the model outputs, we also found that LLM responses occasionally lacked sufficiently granular reasoning against each inclusion and exclusion criterion ( Supplementary Table 12 ). We refined our Framework CoT prompt to elicit detailed CoT reasoning for each individual sub-criterion by numbering them, while preserving the original criteria content. We term this revised prompt as ‘Numbered Framework CoT’ and similarly found improved performance (95.2% accuracy, 98% sensitivity, 92.4% specificity) ( Fig. 3b , Supplementary Table 10 ). Merging ‘Numbered Framework CoT’ with our optimal prompt structure (init + fin) resulted in a prompting strategy dubbed ‘ Instruction-Structure-Optimized (ISO) ScreenPrompt ’. ISO-ScreenPrompt exhibited the best overall performance on our training dataset (95.5% accuracy, 94% sensitivity, 98% specificity) ( Fig. 3b , Supplementary Table 10 ). Evaluation on the separate ST validation dataset confirmed the robustness of our approach, showing comparable performance (96.3% accuracy, 97.5% sensitivity, 95% specificity) and confirmed that we did not overfit our training dataset. Full-text Screening Performance across LLMs Consistent with our abstract findings, our comparative analysis between LLM models revealed that Gemini Pro and GPT3.5 models had poor performance across all metrics ( Supplementary Table 14 ). Interestingly, Mixtral-8×22 (93.7% accuracy, 91.9% sensitivity, 95.4% specificity) outperformed Mistral-Large (86.9% accuracy, 77.8% sensitivity, 96% specificity). GPT4-0125-preview, GPT4-Turbo-0409, and GPT4o-0513 models all had similar performance (95.3-95.8% accuracy, 93.0-93.5% sensitivity, 97.5-98.5% specificity). We then compared Zero-Shot and ISO-ScreenPrompt performance in our lowest performing (GPT3.5) and best performing model (GPT4-Turbo-0409) to assess model agnosticism ( Supplementary Table 16 ). We found that ISO-ScreenPrompt was associated with improvements in accuracy across all models, with the greatest sensitivity gains in GPT4-Turbo-0409 (+79.5% sensitivity). Generalizability of ISO-ScreenPrompt We assessed the generalizability of our ISO-ScreenPrompt approach for full-text screening across datasets from BenchSR ( Fig. 3c , Supplementary Table 16 ). We found that ISO-ScreenPrompt demonstrated high sensitivity across reviews, with 97.5% (89.7-100.0%) mean sensitivity, in contrast to zero-shot prompting (65.6% [11.8-93.8 %]). ISO-ScreenPrompt (93.6% [83.2-99.6%]) and zero-shot prompting (93.1% [73.9-99.0%]) demonstrated comparable accuracy, but ISO-ScreenPrompt had slightly higher accuracy with lower variance. When compared to Abstract ScreenPrompt, ISO-ScreenPrompt had a modest improvement in accuracy while maintaining sensitivity. We applied self-consistency (SC) to ISO-ScreenPrompt on the ST test dataset (Methods, Testing Methodology) and the Reinfection dataset ( Supplementary Fig. 2c , Supplementary Table 17 ). We found that ISO-ScreenPrompt-SC was associated with an overall gain in performance in both datasets across all metrics, except sensitivity in ST (ST: +1% accuracy, -1% sensitivity, +3% specificity; Reinfection: +4.3% accuracy, +2.8% sensitivity, +5.1% specificity). We could also modulate sensitivity and specificity by setting different self-consistency thresholds for article inclusion/exclusion ( Supplementary Fig. 2d ). Collectively, these findings suggest that our ISO-ScreenPrompt prompting strategy is generalizable and can perform well across different SR contexts. Real World Implementation Assessment and Benefit Dual human screening is considered the gold-standard approach for SR screening workflows. 23 , 24 In this process, at least two reviewers independently perform title and abstract screening, resolving any discrepancies through consensus or decision by a third reviewer ( Fig. 4a ). Relevant abstracts are then moved to a full-text screen, and the process is repeated to culminate a ‘full dual screen’ that identifies the final set of articles for inclusion ( Fig. 4a ). Download figure Open in new tab Figure 4: ScreenPrompt and ISO-ScreenPrompt perform comparably, or better than dual human review a) Overview of the dual human review process for systematic reviews, illustrating dual independent review stages for both abstract and full-text screening, leading to final review inclusion. Conflicts between reviewers are resolved by a third reviewer. b) Difference in performance between Dual Human Review and Abstract ScreenPrompt across five systematic reviews (SVCF n=167, SeroTracker n=400, Reinfection n=400, PA-Testing n=400, PA-Outcomes n=400). The barplot shows differences in accuracy and sensitivity, with human-favored differences as negative (left) and ScreenPrompt-favored differences as positive (right). Error bars represent 95% CIs for the difference in binomial proportions. c) Difference in performance between Dual Human Review and ISO-ScreenPrompt across five systematic reviews (SVCF, SeroTracker, Reinfection, PA-Testing, PA-Outcomes). The barplot shows differences in accuracy and sensitivity, with human-favored differences as negative (left) and ISO-ScreenPrompt-favored differences as positive (right). Error bars represent 95% CIs for the difference in binomial proportions. To evaluate the real-world applicability of our approach, we performed head-to-head comparisons between our Abstract ScreenPrompt and ISO-ScreenPrompt with the traditional gold-standard dual human screening workflow ( Fig. 4a ). We selected reviews using stratified probability sampling for each Oxford CEBM question type, with two spots for intervention benefit due to their higher prevalence in the dataset. The article inclusion/exclusion labels set by the original study authors were used as the gold-standard. We recruited a team of four researchers with previous SR experience and performed a ‘calibration’ exercise to evaluate their baseline screening performance, finding acceptable performance ( Supplementary Table 18 , Methods Head-to-Head Comparisons). Reviewers then screened the randomly selected reviews according to the standard dual screening workflow. We found that human reviewers performed well with dual abstract screening (mean accuracy 94.6% [89.5-97.8%]), mean sensitivity 90.9% [84.1-100%]), but performance dropped with full dual screening (mean accuracy 92.4% [76.8-97.8%], mean sensitivity 75.1% [44.1-100%]) ( Supplementary Table 19 -20). Head-to-head comparisons between Abstract ScreenPrompt and dual abstract screening revealed comparable sensitivity across three reviews (difference in binomial proportions, two-sided p>0.05). However, human reviewers exhibited significantly higher accuracy in three reviews (p<0.05) ( Fig. 4b , Supplementary Fig. 3a , Supplementary Table 19 ), while Abstract ScreenPrompt demonstrated higher sensitivity than humans in two reviews (p<0.05). When compared to single human-reviewer abstract screening, Abstract ScreenPrompt exhibited significantly higher sensitivity across three reviews (p0.05) ( Supplementary Fig. 3b -c). Further, head-to-head comparisons between ISO-ScreenPrompt and full dual screening revealed that ISO-ScreenPrompt had significantly higher sensitivity to humans across four reviews (p<0.05), significantly higher accuracy in one review (p0.05) ( Fig. 4c , Supplementary Fig. 3d , Supplementary Table 20 ). We derived cost- and time-estimates for our LLM and human screening workflows ( Supplementary Table 21 -22; Methods, Cost Analysis). Traditional dual screening costs ranged from $1,385.67 to $67,872.00 USD (2,257-130,436 articles), while ISO-ScreenPrompt costs ranged from $196.43 to $12,661.30 USD at a compensation rate of $20 USD per hour. For Abstract ScreenPrompt, costs ranged from $55.25 to $4,122.49 USD, compared to $376.17 to $21,739.33 USD for single human-reviewer abstract screening ( Supplementary Table 22 ; Methods Cost Analysis). Our LLM approach was also substantially faster. Where dual screening took approximately 69.3 to 3393.6 hours, our ISO-ScreenPrompt approach completed screening in 0.9 to 52.2 hours ( Supplementary Table 22 ). Similarly, single human-reviewer abstract screening was estimated to take 18.8 to 1087.0 hours, whereas Abstract ScreenPrompt completed screening in 16 minutes to 15.2 hours ( Supplementary Table 22 ). Moreover, implementing the new OpenAI batch API further reduced costs by 50% for both prompting methods (ISO-ScreenPrompt: $98.22 to $6,330.65 USD; Abstract ScreenPrompt: $27.63 to $2061.25 USD), and reduced the maximum screening time to under 24 hours. Discussion Systematic review workflows are encumbered by resource- and time-intensive screening processes. Despite efforts to automate SR screening, existing tools demonstrate inadequate performance and are unable to independently screen abstracts and full-texts. 7 , 25 , 26 Here, we leverage the capabilities of text-based LLMs, such as GPT4, for SR abstract and full-text screening. Our experiments spotlight the importance of strategic prompting, demonstrating substantial performance improvements over basic zero-shot prompting approaches, and serve as a valuable resource for researchers interested in performing LLM evaluations. In this context, we introduce Framework CoT and ISO-prompting as effective and generalizable strategies for enhancing SR screening. Our findings reveal that our Abstract ScreenPrompt and ISO-ScreenPrompt can achieve performance levels matching human reviewers, and in some cases can even surpass humans. Finally, we introduce BenchSR , a curated collection of SR datasets that may be used to evaluate the effectiveness of screening automation tools and support advancements in SR automation. Previous studies evaluating LLM performance for abstract screening have primarily utilized zero-shot prompting approaches with unsatisfactory results. Guo et al. 11 reported 59.3-100% sensitivity with GPT4 models, and Gargari et al. 13 reported 38-69% sensitivity with GPT3.5 models. These observations demonstrate that zero-shot prompting, akin to assessing a race car’s performance without shifting gears, likely underestimates the true capabilities of LLM models for downstream tasks. Zero-shot prompting has been frequently used for other tasks such as medical question answering, 27 medical code mapping, 28 and disease risk stratification. 29 Furthermore, these evaluations are often accompanied by other problematic practices, such as web browser-based evaluations (i.e., ChatGPT). 27 , 30 – 32 Unknown system prompts, variable GPT versions, and the transient nature of chatbot context in these settings severely limit research reproducibility. Future studies should adopt more sophisticated prompting techniques, such as reasoning elicitation techniques (i.e., CoT), few-shot, and task-specific prompting. Additionally, researchers should use API calls to support reliable and reproducible research. Our analysis across various model versions demonstrated that model updates do not always improve performance, and can occasionally lead to performance drops. This variability further highlights the importance of transparent model and parameter reporting for LLM evaluations. Our work offers several novel prompt engineering insights. CoT reasoning instructs LLMs to break down questions into intermediate reasoning steps before generating answers. However, the unstructured nature of freeform rationales can result in errors. 33 For instance, we observed that some CoT responses did not elicit reasoning against exclusion criteria. In response, we introduced ‘Framework CoT,’ a novel prompting approach that leverages predefined criteria or ‘frameworks’ for LLMs to reason against. This facilitates a structured analysis that mimics human cognitive processes, inducing the model to systematically consider each criterion before making a final decision. Furthermore, adjusting the proportion of GPT-CoT few-shot label distributions (i.e., balanced, exclusion-favored, inclusion-favored) did not influence model accuracy. We hypothesize that the role of labeled examples in few-shot prompting is complex and likely dependent on the task, model, and prompt design. 34 , 35 The benefits of in-context learning may be attributed to the format of few-shot responses, which guide the model to better structure its output response. 34 Few-shot GPT-Cot also resulted in worse performance than zero-shot methods. We speculate that the decreased accuracy was due to semantic contamination, 36 where the model misinterprets the example reasoning as directly relevant to the task. Consequently, we observed heightened sensitivity at the cost of reliable reasoning, and highlight the nuanced challenge in applying few-shot prompting. Next, we address the ‘lost-in-the-middle’ phenomenon, 21 , 22 , 37 where as context length increases, much like stretching dough, the LLMs’ ability to adhere to instructions wanes as ‘gaps’ or ‘holes’ begin to appear within the context. This limitation of autoregressive LLMs is particularly evident in lengthy documents such as full-text articles. Our ScreenPrompt (init) prompt occasionally produced single-token outputs, possibly because the instructions to ‘think step-by-step’ were lost in context. In response, we developed ISO-ScreenPrompt , and repeated our prompt modules before and after the full-text content to capitalize on the LLMs’ context retrieval strength at these points. 22 Additionally, we numerically labeled each sub-criterion to leverage the proficiency of LLMs with structured input formats. 38 This adjustment facilitated reasoning against each specific sub-criterion, rather than broader meta-criteria. Finally, we demonstrate that self-consistency can introduce additional performance improvements at the cost of additional generations. We find these simple methods are effective strategies for bolstering LLM performance with long-context documents and anticipate that our prompting techniques can extend to other text classification domains with structured decision frameworks, such as identifying patients eligible for clinical trials 39 and medical code mapping using electronic health records. 28 Our research highlights that our Abstract ScreenPrompt and ISO - ScreenPrompt prompts can excel in abstract and full-text screening, surpassing previous tools such as Abstrackr, Rayyan, and RobotAnalyst. 25 , 26 Both prompts achieve high sensitivity, with a mean value of 97.0% for Abstract ScreenPrompt and 97.5% for ISO-ScreenPrompt, across ten different reviews. Our prompts also maintain robust accuracy, with 88.4% mean accuracy for Abstract ScreenPrompt and 93.6% mean accuracy for ISO-ScreenPrompt. Furthermore, we did not modify or attempt to optimize the eligibility criteria or study objectives from each review, highlighting that our approach can be readily implemented. Our findings ultimately underscore the efficacy of prompt design in real-world applications and sets a new standard for SR screening automation efforts, which have historically demonstrated unsatisfactory performance. 25 , 26 We address a critical gap within the realm of SR screening automation by providing one of the most comprehensive reproducibility evaluations for gold-standard dual human screening. Previous evaluations have been limited to title/abstract screening phases across three or fewer SRs. 23 In contrast, our study performed full dual screen evaluations across five reviews covering four common Oxford CEBM question types. Our reviewers were calibrated for screening proficiency, and revealed that dual human screening can be error prone, with many articles erroneously excluded at the full-text stage. Crucially, our ISO-ScreenPrompt approach is the first to demonstrate that SR automation efforts can match, and in some instances, outperform gold-standard human dual screening. ISO-ScreenPrompt had higher sensitivity than dual screening in all reviews, with significantly higher sensitivity in four of five reviews. Furthermore, it maintained comparable or greater accuracy in three reviews. Barring full-text accessibility issues, this superior sensitivity and comparable accuracy could revolutionize the SR process by potentially eliminating the need for an initial human screening phase of abstracts. Instead, reviewers could initiate data extraction on the subset of articles deemed included by ISO-ScreenPrompt, and iteratively remove ‘false-positive’ articles. Abstract ScreenPrompt, which displayed superior sensitivity relative to dual abstract screening, offers another viable alternative for immediate implementation. While it may not replace dual abstract screening workflows, it achieved comparable accuracy to single human reviewers in four reviews and exceeded human sensitivity in all five reviews, with significantly higher sensitivity in three reviews. These findings suggest that Abstract ScreenPrompt could reliably serve as a single human reviewer vote at the abstract stage without compromising quality. Finally, both ISO-ScreenPrompt and Abstract ScreenPrompt are associated with substantial cost- and time-savings. Where traditional SR screening can take weeks to months, our approach completed screening in minutes to hours, freeing reviewers for deeper scientific evaluations and accelerating synthesis of study conclusions. Finally, our work addresses a critical gap in the realm of SR screening by introducing BenchSR , a growing collection of 10 SRs that span four types of medical questions across nine different clinical domains. Existing benchmarks for SR screening such as CLEF, 40 Seed Collection, 41 and Cohen 42 have important limitations. The CLEF and Seed Collection benchmark lack critical study metadata, such as detailed inclusion and exclusion criteria; and feature incomplete datasets with only article identifiers, necessitating additional efforts to gather article abstracts and/or full-texts. Furthermore, the Cohen benchmark is no longer publicly accessible. BenchSR provides (i) a comprehensive set of all included and excluded abstracts (with article identifiers) found during the initial search, (ii) essential study metadata not typically reported (detailed eligibility criteria), sourced directly from the original study authors, and (iii) a non-online hosting solution to mitigate concerns about LLM pre-training data contamination. Study metadata is currently limited to eligibility criteria and study objectives, but will be expanded significantly in future work (i.e., data extraction templates). Future research should focus on enhancing transparency and completeness in reporting SRs, including detailed study metadata and data extraction templates, to better support and refine automation efforts. We invite other researchers to contribute to our growing BenchSR benchmark. Our study has several limitations. Although we apply our analysis across a wide range of SRs, the generalizability of our findings to other clinical questions, non-English SRs and other review methodologies (e.g. scoping reviews) requires further study. However, our prompt templates would likely perform well in reviews with PICO-structured inclusion/exclusion criteria. Furthermore, our work may extend to non-English SRs as LLM models continually improve language accessibility. Our full-text analysis was limited by the availability of freely accessible texts in the PubMed Central (PMC) database. While we made efforts to enhance our dataset by manually scraping ‘included’ full-texts for select reviews, we did not manually gather all ‘excluded’ texts due to accessibility and resource constraints. Our LLM screening was also solely conducted on text content. Including figures and tables could potentially enhance performance and accuracy and warrants future investigation. Moreover, our analysis explored Gemini Pro, GPT3.5, and GPT4-0125-preview, GPT4-Turbo-0409, and GPT4o-0513, but not the Claude 3 LLMs due to regional availability issues. Due to GPT4 context length limitations (128k tokens), we only tested few-shot prompting on abstracts. Furthermore, while we surveyed a wide range of prompting techniques, we did not explore every possible method. The influence of subtle changes (i.e., word choice) may further optimize models. Our focus on developing ready-to-use and generally applicable prompts meant that we did not modify original SR inclusion and exclusion criteria. However, we suspect that additional optimizations could improve performance. In conclusion, our study underscores the importance of deliberate prompting to achieve human-level performance for SR screening. The promising results from ISO-ScreenPrompt and Abstract ScreenPrompt marks a major advancement in the automation of SR processes. We encourage further research into the capabilities of LLMs for other SR tasks, such as data extraction and meta-analysis. Fully-automated SRs will revolutionize evidence-based practice and offer indispensable value to medicine and beyond. Methods Datasets and data acquisition We acquired abstract screening data from 10 distinct SRs spanning eight unique clinical domains through purposeful convenience sampling based on Oxford CEBM SR question types. To compile these datasets, we engaged with systematic review investigators at the University of Calgary and the University of Toronto. We extracted study information concerning review objectives from the published manuscript or PROSPERO protocol, and contacted study authors for internal reviewer inclusion/exclusion criteria. In brief, ‘SeroTracker’ (ST) was a living SR of prevalence, exploring observational cohort studies that reported single-estimate prevalence in the context of COVID-19. 43 ‘Reinfection’ was an SR of intervention benefits that assessed the comparative effectiveness of vaccination and past COVID-19 infection relative to past COVID-19 infection alone in observational studies reporting associations. 44 ‘PA-Outcomes’ was an SR of intervention benefits comparing clinical outcomes between surgery and medication treatments in patients with primary aldosteronism. 45 ‘PA-Testing’ was an SR of diagnostic test accuracy evaluating guideline-recommended confirmatory tests (i.e., saline infusion test, salt loading test, fludrocortisone suppression test, and captopril challenge test) relative to a reference standard. 46 ‘Sepsis’ was an SR of intervention benefits that assessed the comparative effectiveness and safety of fludrocortisone plus hydrocortisone, hydrocortisone alone and placebo/usual care in adults with septic shock. 47 ‘Spinal’ was an SR of diagnostic test accuracy that evaluated the efficacy of intraoperative neurophysiological monitoring among patients undergoing spine surgery for any indication. 48 ‘Calcium-HA’ was an SR of intervention benefits that compared the outcomes of routine calcium administration to no calcium administration for cardiac arrest in adults or children. 49 ‘Infant-NO’ was an SR of prognosis that assessed whether an immediate response to inhaled nitric oxide therapy was associated with reduced mortality in preterm infants with hypoxemic respiratory failure and pulmonary hypertension. 50 ‘Meds-HA’ was an SR of SRs, covering intervention benefits, that identified medications that affected hospital admissions. 51 ‘SVCF’ was an SR of prognosis that evaluated the association of low SVC flow, diagnosed in the first 48 hours after birth echocardiography, with neurological morbidity and mortality, among very preterm neonates. 52 Included and excluded abstracts were downloaded from Covidence (Veritas Health Innovation, Melbourne, Australia), a systematic review screening software. Abstracts were stored in csv files. We downloaded all ‘Included’ (included articles at full-text screening), ‘Excluded’ (excluded articles at full-text screening), and ‘Irrelevant’ (excluded articles at abstract screening) articles. Excluded and irrelevant articles were collated to form our excluded article dataset. To obtain full-text articles, we utilized the PMC ID Converter API to convert abstract DOIs into Pubmed IDs (PMIDs), followed by the BioC API for PMC to obtain XML full-text files from each abstract PMID. Full-texts with over 150,000 characters were excluded (i.e., abstract conference proceedings). Human authors (CC, JS, DC, RA, NB) manually scraped all ‘included’ full-text articles not captured by the BioC API for SRs selected in our head-to-head human-screening performance evaluation in plain text files. The datasets described here including labeled abstract sets, and all associated metadata, are available at (link made available upon publication). Full-text articles are not provided due to copyright concerns. We invite other researchers to contribute to our growing BenchSR . Abstract Prompt Approaches During the evaluation of GPT4 abstract screening performance, we applied eight distinct prompting methodologies, discussed in more detail below. A brief illustration and the structure for each prompt is shown in Fig. 2a and Table 2 . Zero-shot prompting Models are prompted with only instructions for completing the task at hand, including the required context directly related to the task, i.e. abstract text to analyze. No additional context or examples are provided. For our zero-shot prompts, we adapted a prompt from Guo et al. 11 that evaluated GPT4 and GPT3.5 SR screening performance. We similarly specified that the LLM returns only a single token output with its final decision. Random few-shot prompting Models are prompted with the instruction-task at hand, with the addition of (k=n) labeled examples relevant to the task at hand. For our purposes, the examples are randomly selected and accurately labeled as included (‘YYY’) or excluded (‘XXX’). We set k=10 (5 included, 5 excluded), in agreement with MedPrompt and general prompting guidelines. 10 Zero-shot Chain-of-Thought (CoT) Models are prompted with the instruction-task at hand, along with additional natural language statements, such as “ Let’s think step by step ” to encourage the model to generate intermediate reasoning steps before generating a final answer. 18 Zero-shot Framework CoT We devised a new prompting approach, termed ‘Framework CoT,’ wherein we deliberately prompt the model to reason against each criterion. Similar to zero-shot prompting, no additional context or examples are provided. Zero-shot ScreenPrompt We included additional well-defined study objectives (adapted from published manuscripts) to our Framework CoT prompt to better orient our prompt to screening tasks. Zero-shot Abstract ScreenPrompt We further incorporated additional context that acknowledged the inherent content limitations of abstracts and goals of inclusivity to our ScreenPrompt prompt to better orient our prompt to the task of abstract screening. Few-shot GPT-CoT Abstract ScreenPrompt We prompted our model with Abstract ScreenPrompt , but also included additional examples that contained Abstract ScreenPrompt GPT-generated reasoning. We discarded answers that did not match the ground truth label to uphold ‘correctness’ in example reasoning. This approach was adapted from MedPrompt, which has suggested that GPT-generated CoT reasoning can outperform human experts, as well as automate the CoT example process. We set k=10 (5 included, 5 excluded), in agreement with MedPrompt and general prompting guidelines. 10 Self-consistency To address variability in model outputs due to stochasticity, we conducted repeat evaluations using different seed parameters for the same prompt. The final answer is the decision to ‘include’ or ‘exclude’ according to majority vote. We set the number of self-ensembles to 11, in accordance with literature recommendations. 20 For our AUROC analysis, we adjusted the self-consistency threshold, which dictates the number of votes required for an article’s inclusion or exclusion, within a range from 0 to 12 votes (0 always include; 12 always exclude). This method enables us to fine-tune sensitivity and specificity by leveraging the consensus of multiple model generations. Full-text Prompt Approaches During the evaluation of GPT4 full-text screening performance, we applied six distinct prompting methodologies, discussed in more detail below. A brief illustration and the structure for each prompt is shown in Fig. 3a , and Table 2 . Our standard ‘prompt structure’ used for abstract screening consisted of: {Pre-Prompt}, {Inclusion Criteria}, {Exclusion Criteria}, {Article}, {Instructions}. Prompt elements refer to all modules except {Article} ScreenPrompt (Init) We modify our ‘prompt structure’ by appending all of our prompt elements ({Objectives}, {Inclusion criteria}, {Exclusion criteria}, {Instructions}) to the start of the prompt, before the {Article} text content. ScreenPrompt (Fin) We modify our ‘prompt structure’ by appending all of our prompt elements to the end of the prompt, after the {Article} text content. ScreenPrompt (Init + Fin-Instructions) We modify our ‘prompt structure’ by appending all of our prompt elements to the start of the prompt, before the {Article} text content. We additionally append {Instructions} after the {Article} text content. ScreenPrompt (Init + Fin) We modify our ‘prompt structure’ by appending all of our prompt elements to the start of the prompt (before the {Article} text content) and end of the prompt (after the {Article} text content). Numbered ScreenPrompt We preserve the semantic content of our prompt and add number symbols for each individual inclusion and exclusion sub-criterion. We removed meta-criteria headings (i.e., population, intervention, etc.) where applicable. ISO-ScreenPrompt We apply a combination of our Numbered ScreenPrompt where each individual inclusion/exclusion sub-criterion is numbered, and our (Init + Fin) prompt structure, where all prompt elements are appended to the start and end of the prompt. Prompt Testing Methodology To avoid concerns of prompting ‘overfitting’ during training and testing our prompt engineering process, we apply principles of sound testing methodology for machine learning studies and randomly sampled train/validation/test splits for our downstream analysis. Our iterative prompt optimization process was only performed on train splits, and we validated the performance of our optimized prompting strategy on validation splits. Test splits were held out from prompt engineering consideration until the final testing phase to assess real world, or ‘eyes-off’ performance. To determine the minimum sample size for abstract evaluation, we used the Cochran’s sample size formula. 53 We set our desired confidence level to 95% (p=0.05), Margin of Error (MoE) to 5%, and model performance to 50% (assuming 50% chance of inclusion/exclusion labels), resulting in a minimum sample size of 385 abstracts. Due to its substantial dataset, we utilized data from the SeroTracker SR (n=130k excluded abstracts, n=3000 included abstracts) to derive balanced sets of included and excluded abstracts for our train, validation, and test splits (Train: n=200 included, n=200 excluded; Validation: n=200 included, n=200 excluded; Test: n=200 included, n=200 excluded;). Furthermore, we randomly sampled sets of included and excluded abstracts for our GPT-CoT prompting to prevent cross contamination (GPT-CoT: n=100 included, n=100 excluded). Following our prompt-optimization procedures, we also tested the performance of our prompting strategy on the ST Test split, and 9 other SR abstract datasets to model its real-world performance and generalizability across different SR domains. As these datasets were held-out from any prompt engineering steps, we included all ‘included’ articles, and randomly sampled ‘excluded’ articles to reach a test sample size of 400 articles. We replicated the same procedures for full-text evaluations. In brief, we randomly sampled the SeroTracker SR (n=5137 excluded, n=1797 included PMC-scraped full-texts) to derive balanced training and validation datasets (Train: n=200 included, n=200 excluded; Validation: n=200 included, n=200 excluded). Furthermore, we randomly sampled sets of included and excluded abstracts for our GPT-CoT prompting to prevent cross contamination (GPT-CoT: n=100 included, n=100 excluded). The test dataset was composed of all remaining articles (Test: n=1297 included, n=4637 excluded), but we opted to randomly sample a smaller subset (n=200 included, n=200 excluded) in the interest of cost and rate limits. Following our prompt-optimization procedures, we tested the performance of our ISO-ScreenPrompt prompting strategy for full-text screening on the ST Test split, and 9 other SR datasets. For each SR evaluation, we set our sample size to n=400, and incorporated all included articles that were available from the PMC web scrape. For instances where we were unable to obtain a sample size of 400 (i.e., spinal), we evaluated all available ‘included’ and ‘excluded articles. As previously mentioned, human researchers performed additional manual scraping of all ‘included’ articles for SRs chosen in our head-to-head evaluation of human screening performance. Head-to-head comparisons with human screening We assembled a panel of four researchers with past SR experience (1 BSc, 3 MSc) to perform end-to-end SR screening while adhering to canonical dual reviewer screening protocols. 1 The reviewers blinded from our study objectives (comparing GPT performance with human reviewers), and were only provided with the list of references for screening, internal inclusion/exclusion criteria, and SR objectives (Supplementary Note 1). Researchers were instructed to perform standard SR screening, with a sensitive abstract screen followed by an accurate full-text screen. Screening for each review was performed independently by two reviewers in duplicate. Any conflicts during screening were resolved by a 3rd independent reviewer. We analyzed the performance of single human-reviewer abstract screening, dual human abstract screening (based on independent votes from two reviewers and resolved conflicts at the abstract stage), and complete dual screening (based on independent full-text votes and resolved conflicts at the full-text stage). For comparisons of single human-reviewer abstract screening with Abstract ScreenPrompt, we randomly selected the performance of one reviewer from the two reviewers for each review. We calibrated our reviewers according to screening proficiency by having prospective reviewers first screen a ‘calibration set’ of abstracts. This set was sourced from a prior study by the SeroTracker group, 54 which assessed the performance of dual human reviewer workflows. Notably, the SeroTracker researchers were experienced SR screeners, having contributed to the SeroTracker living SR for over a year, and represent a high-performing baseline for screening accuracy. The results from this prior study, and calibration set were used in head-to-head comparison analysis for the SeroTracker (ST) dataset. Reviewers were recruited for subsequent screening tasks if their accuracy was within 5% of the performance benchmarks set by the SeroTracker group. The performance metrics of our reviewers were highly similar to those by the SeroTracker group and are detailed in Supplementary Table 18 . For a representative comparison of GPT4-0125-preview and human screening performance, we used stratified random probability sampling across the four Oxford CEBM review questions, selecting one SR for each type. Our sample included various datasets: the SeroTracker dataset for reviews of prevalence (adapted from Perlman-Arrow et al. 54 ), the Reinfection dataset for reviews of intervention benefits, the PA-Testing dataset for reviews of diagnostic test accuracy, and SVCF dataset for reviews of prognosis. Due to the high volume of SR studies focusing on intervention benefits, we also included the PA-Outcomes dataset in our sample through additional random sampling. Time and Cost Analysis To derive cost estimates for traditional human dual screening for each review, we began by determining the total number of abstracts and full-texts screened at each stage. For abstracts, we used the total number of articles for a given review. For full-texts, we counted the total number of ‘Included’ and ‘Excluded’ articles from Covidence. Based on previous studies, the time required to screen a single abstract ranges from 20-461 seconds, 5 , 54 , 55 and 4.3-20 minutes for a single full-text article. 55 , 56 We aligned with Perlman-Arrow et al. 54 due to our use of the same ST dataset and set the screening time at 30 seconds per abstract and 10 minutes per full-text. We calculated time estimates for human reviewers by multiplying the total number of screened abstracts by 30 seconds and the total number of screened full-texts by 10 minutes. The resulting value was then doubled to derive the cost of dual human screening (in duplicate). For single human abstract screening, we only took the total number of abstracts. We assumed a compensation rate of $20 USD per hour for human reviewers to calculate costs. It is important to note that our cost estimates do not consider the additional costs of conflict screening (additional abstracts and full-texts screened due to conflicts between reviewer decisions), and likely represents a conservative estimate of human screening costs. For our ISO-ScreenPrompt and Abstract ScreenPrompt approach using the GPT4-0125-preview, we calculated the exact cost for each full-text and abstract run used in our head-to-head comparison analysis ($10 per million input tokens, $30 per million output tokens, OpenAI Pricing). We then derived the cost per article by dividing the total cost of our runs by the number of articles in each run. We multiplied this cost per article by the total number of articles for each review to estimate the cost of our ISO-ScreenPrompt and Abstract ScreenPrompt approach for each review. To derive estimates for our two approaches following implementation of the OpenAI Batch API, we divided the estimated cost by two (Batch API offers a 50% discount). We derived time estimates by obtaining the time per article (ISO-ScreenPrompt: 1.44s/full-text; Abstract ScreenPrompt: 0.42s/abstract), and multiplied this estimate against the total number of articles for each review. LLM API and LLM Evaluations We used GPT3.5-Turbo-0125 (GPT3.5), GPT4-0125-preview, GPT4-Turbo-0409, GPT4o-0513, Gemini Pro, open Mixtral-8×22-0424, and Mistral-Large-0224 models and compared their performance for abstract and full-text screening with our optimized ‘Abstract ScreenPrompt’ and ‘ISO-ScreenPrompt prompts’, respectively. For all models, we set the maximum tokens to 2048, and used the advised default settings. GPT model settings were set to temperature=1, top_p=1, frequency_penalty=0, presence_penalty=0. The Gemini pro model settings were set to temperature=1, top_p=1. The Mixtral-8×22-0424, and Mistral-Large-0224 model settings were set to temperature=0.7, top_p=1. We note that for Gemini Pro, the default temperature was changed to temperature=0.9 as of May 2024 (after our testing). We set the seed for all of our LLM models at 0. For our self-consistency analysis (n=11), we set different seed parameters from 0 to (n-1) for each repeat evaluation. To evaluate the model responses, we required an output containing an ‘evaluation token’: either “XXX” (exclude) or “YYY” (include). When responses contained both “XXX” and “YYY”, we checked the last 500 characters of the output and used the final instance of the token. We conducted our initial runs synchronously, sending requests directly to the API and waiting for responses in the same network call. This approach allowed us to easily retry individual calls to a model’s API. For each abstract, if a response was interrupted by a technical error (rate limit, disconnect, timeout) or was missing an ‘evaluation token’, we retried the instance up to three times. If all three attempts failed, we deemed the request as ‘unparseable’ and disqualified the request from the total count, occasionally reducing the set of 400 articles. During the writing of this paper, OpenAI released the Batch API, which significantly reduced costs and improved run times. Shifting to this infrastructure meant we could no longer efficiently retry individual abstracts, but the new system greatly reduced technical errors and rarely resulted in missing “evaluation tokens”. Data analysis We assessed the performance of our prompts by analyzing accuracy, sensitivity, and specificity metrics, and calculated binomial 95% confidence intervals (CIs) for sensitivity and accuracy. “Unparseable requests” were discarded and the metrics computed only for those responses that were parseable. To compare the performance of our Abstract ScreenPrompt and ISO-ScreenPrompt with zero-shot methods across all ten reviews, we calculated 95% CIs using the Clopper-Pearson method, 57 employing the binom package in R. This approach was necessary due to the small sample size of included articles in some reviews (n<10). We determined statistical significance and 95% CIs with a two-tailed test computing the difference between two independent binomial proportions in our comparative analysis of human versus LLM screening performance. DATA AVAILABILITY Researchers can access our data via the following github repository (link made available upon publication). CODE AVAILABILITY All code used for experiments in this study can be found in a github repository (link made available upon publication). Additional Information Author Information These authors contributed equally: C.C., J.S. These authors jointly supervised the work: R.K.A., N.B. Author contribution statements C.C., J.S., N.B., R.K.A contributed to the conception and design of the work. C.C., J.S., R.A., R.K., M.C., J.G., R.S., D.C., I.D., B.T., M.F., P.R., A.A.L., D.E.W., H.W., M.W., N.B., contributed to the data acquisition and curation. C.C., J.S., contributed to original investigation and formal analysis. C.C., J.S., R.A., D.E.W., N.B., R.K.A., interpreted results and provided feedback on the study. C.C., J.S., R.A., D.E.W., contributed to validation of the data. C.C., J.S., H.W., M.W., N.B., R.K.A., contributed to project supervision. C.C., J.S., R.A., N.B., prepared the original draft of the manuscript with input form all co-authors. All authors were responsible for review and editing of the manuscript. All authors debated, discussed, edited, and approved the final version of the manuscript. Ethics Declaration Competing Interests C.C., H.W., M.W., R.K.A., and N.B., report grants from the Public Health Agency of Canada through Canada’s COVID-19 Immunity Task Force, the World Health Organization Health Emergencies Programme, the Robert Koch Institute, and the Canadian Medical Association Joule Innovation Fund. No funding source had any role in the design of this study, its execution, analyses, interpretation of the data, or decision to submit results. This manuscript does not necessarily reflect the views of the World Health Organization or any other funder. R.K.A. is employed at OpenAI and may own stock as part of the standard compensation package. R.K.A., was also previously a Venture Fellow at Flagship Pioneering, minority shareholder of Alethea Medical, and has received funding from the Rhodes Trust and Open Philanthropy. Supplementary Information Download figure Open in new tab Supplementary Figure 1: Abstract screening prompt optimization a) Performance comparison of different abstract prompting methodologies relating to the addition of study objectives and abstract-specific considerations on the SeroTracker (ST) training dataset, (n=400) showing accuracy and sensitivity. Error bars represent 95% CIs for binomial proportions. b) Performance comparisons of different few-shot prompting methodologies on the ST training dataset, showing accuracy and sensitivity. Prompts compare differing proportions of inclusion-labeled and exclusion-labeled few-shot examples. Error bars represent 95% CIs for binomial proportions. c) Barplot displaying the performance of Abstract ScreenPrompt sensitivity and accuracy with and without self-consistency (SC) in SeroTracker and Reinfection test datasets (n=400). Error bars represent 95% CIs for binomial proportions. d) Receiver operating characteristics (ROC) curves generated from differing self-consistency thresholds (number of votes needed, 0-12) for article inclusion in SeroTracker and Reinfection test datasets. Download figure Open in new tab Supplementary Figure 2: Full-text screening prompt optimization. a) Performance comparison of different full-text prompting methodologies with modifications in prompt structure on the ST training dataset (n=400), showing accuracy and sensitivity. Error bars represent 95% CIs for binomial proportions. b) Performance comparison of abstract screening performance with modifications in prompt structure on the ST training dataset, showing accuracy and sensitivity. Error bars represent 95% CIs for binomial proportions. c) Barplot displaying the performance of ISO-ScreenPrompt sensitivity and accuracy with and without self-consistency (SC) in SeroTracker and Reinfection test datasets (n=400). Error bars represent 95% CIs for binomial proportions. d) Receiver operating characteristics (ROC) curves generated from differing self-consistency thresholds (number of votes needed, 0-12) for article inclusion in SeroTracker and Reinfection test datasets. Download figure Open in new tab Supplementary Figure 3: ScreenPrompt and ISO-ScreenPrompt vs. human screening. a) Barplot displaying the performance of Abstract ScreenPrompt and dual human abstract screening sensitivity and accuracy. Error bars represent 95% CIs for binomial proportions. b) Difference in performance between single human-reviewer and Abstract ScreenPrompt across five systematic reviews (SVCF n=167, SeroTracker n=400, Reinfection n=400, PA-Testing n=400, PA-Outcomes n=400). The barplot shows differences in accuracy and sensitivity, with human-favored differences as negative (left) and ScreenPrompt-favored differences as positive (right). Error bars represent 95% CIs for the difference in binomial proportions. c) Barplot displaying the performance of Abstract ScreenPrompt and single-human reviewer screening sensitivity and accuracy. Error bars represent 95% CIs for binomial proportions. d) Barplot displaying the performance of ISO-ScreenPrompt and full dual human screening sensitivity and accuracy. Error bars represent 95% CIs for binomial proportions. View this table: View inline View popup Download powerpoint Supplementary Table 1 Abstract prompt engineering performance View this table: View inline View popup Download powerpoint Supplementary Table 2: Inclusive Language Prompting View this table: View inline View popup Supplementary Table 3: Exemplar Wrong Outputs during Abstract Prompt Engineering View this table: View inline View popup Download powerpoint Supplementary Table 4: Abstract ScreenPrompt Ablation Prompting View this table: View inline View popup Download powerpoint Supplementary Table 5: Few-shot GPT-CoT Label Proportion Analysis View this table: View inline View popup Download powerpoint Supplementary Table 6: Comparative analysis of Abstract ScreenPrompt across LLM models View this table: View inline View popup Download powerpoint Supplementary Table 7 Zero-shot vs. Abstract ScreenPrompt across LLM Models View this table: View inline View popup Supplementary Table 8 Generalizability of Abstract ScreenPrompt across SRs View this table: View inline View popup Download powerpoint Supplementary Table 9 Abstract ScreenPrompt Self-Consistency Analysis View this table: View inline View popup Download powerpoint Supplementary Table 10 Full-text prompt engineering performance View this table: View inline View popup Download powerpoint Supplementary Table 11 : Full-text Prompt Structure Testing View this table: View inline View popup Supplementary Table 12 : Exemplar Wrong Outputs during Full-text Prompt Engineering View this table: View inline View popup Download powerpoint Supplementary Table 13 : ISO-Prompting in Abstract Screening View this table: View inline View popup Download powerpoint Supplementary Table 14 Comparative analysis of ISO-ScreenPrompt across LLM models View this table: View inline View popup Download powerpoint Supplementary Table 15 Zero-shot vs. ISO-ScreenPrompt across LLM Models View this table: View inline View popup Supplementary Table 16 Generalizability of ISO-ScreenPrompt across SRs View this table: View inline View popup Download powerpoint Supplementary Table 17 ISO-ScreenPrompt Self-Consistency Analysis View this table: View inline View popup Download powerpoint Supplementary Table 18 Human Screening Calibration View this table: View inline View popup Download powerpoint Supplementary Table 19 Abstract ScreenPrompt vs Human Dual Abstract Screening View this table: View inline View popup Download powerpoint Supplementary Table 20 ISO-ScreenPrompt vs Full Dual Screening View this table: View inline View popup Download powerpoint Supplementary Table 21 Time and Cost Savings Analysis for ISO-ScreenPrompt and Full-dual Screening View this table: View inline View popup Download powerpoint Supplementary Table 22 Time and Cost Savings Analysis for Abstract ScreenPrompt and Single Human-Reviewer Screening Supplementary Note 1 Below represents the screening documents provided to human reviewers for each selected SR. The documents contain the same study objectives, inclusion and exclusion criteria used in prompting (‘simplified objectives and inclusion/exclusion criteria’), and any additional internal protocol documents provided by the original study authors for each SR. SeroTracker Simplified Objectives + Inclusion/Exclusion Criteria Our systematic review is governed by the following objectives: (i) describe the global prevalence of SARS-CoV-2 antibodies based on serosurveys; (ii) detect variations in seroprevalence arising from study design and geographic factors; (iii) identify populations at high risk for SARS-CoV-2 infection; and (iv) evaluate the extent to which surveillance based on detection of acute infection underestimates the spread of the pandemic. The following is an excerpt of two sets of criteria. A study is considered included if it meets all the inclusion criteria. If a study meets any of the exclusion criteria, it should be excluded. Here are the two sets of criteria: Inclusion Criteria (all must be fulfilled) Population Humans of any age Including COVID-19 antigen positive persons and those with suspected disease if not deliberately sampled. Study design Sero-surveys – defined as the collection and testing of serum (or proxy such as oral fluid) specimens from a sample of a defined population over a specified period of time to estimate the prevalence of antibodies against SARS-CoV-2 as an indicator of immunity Cross-sectional, repeated cross sectional, and cohort study designs, with serology measurements at single time points or repeated at multiple time points Special design Include systematic reviews and meta-analysis of seroprevalence studies for the purpose of tracking evidence synthesis efforts Sampling Any sampling method Types of evidence Published or unpublished academic literature, grey literature (government or institutional reports), or media reports. Slide deck presentations were included if we could identify the person giving the presentation and the date of the presentation Outcome measures Reports a seroprevalence estimate (proportion of the population with detectable antibodies) Reports the number of participants enrolled in the study (denominator) Reports study sampling end date/week Reports the locations at which the study took places such that they could be categorized as neighbourhood, city, state/province/territory, or country Exclusion Criteria (if any met then exclude) Population Non-human (e.g., in silico, animal, in vitro) The study only included individuals with suspected, active, or previously diagnosed with COVID-19 using PCR, antigen testing, clinical assessment, or self-assessment The study only included individuals vaccinated against SARS-CoV2 Study design Study designs other than cross-sectional or cohort design: case reports, case-control studies, evaluations of serological tests, study protocols Types of evidence Multimedia sources of data (audio clips, video clips) were excluded due to the feasibility of extracting. Slide deck presentations were excluded if we could not identify the person giving the presentation and the date of the presentation Dashboards not associated with a defined serology study Outcome measures Only reports incidence or prevalence of SARS-CoV-2 antigen (as opposed to antibody) 2. Does not report study sampling end date/week Does not report the number of participants included in the study (sample denominator) 4. Does not report the location at which the study took place Original author protocol SeroTracker Inclusion/Exclusion Criteria Criteria for including evidence (must meet all the criteria to be included) Criteria for excluding evidence (if any met then exclude) Reinfection Simplified Objectives + Inclusion/Exclusion Criteria We aimed to systematically review the evidence for the magnitude and duration of the effectiveness of (i) previous infection and (ii) hybrid immunity against multiple clinical outcomes of SARS-CoV-2 infection caused by the omicron variant. We also aimed to examine the comparative protection of hybrid immunity relative to previous infection only, vaccination only, and hybrid immunity with fewer vaccine doses. The following is an excerpt of two sets of criteria. A study is considered included if it meets all the inclusion criteria. If a study meets any of the exclusion criteria, it should be excluded. Here are the two sets of criteria: Inclusion Criteria (all must be fulfilled) Population Humans of any age, in any geographical setting. Exposure Group Confirmed case of SARS-CoV-2 infection with or without COVID-19 vaccination. SARS-CoV-2 infection will be defined as a confirmed case according to the following criteria, adapted from WHO case definitions (positive nucleic acid amplification test (NAAT) according to laboratory records or self report, positive SARS-CoV-2 antigen rapid diagnostic test (AgRDT) a with high accuracy according to laboratory records or self report, or a positive serology test from a lab-based assay (i.e. CLIA/ELISA) or an antibody-detecting rapid diagnostic test (Ab-RDT) with high accuracy a ). Studies will be included if they report on individuals with previously confirmed infection that have documented vaccination (partially, fully, or boosted), as defined in the randomized controlled trials for each vaccine. Partial vaccination will be defined as > 14 days after a single dose of Pfizer/BioNTech-Comirnaty, 14 days after a single dose of AstraZeneca-Vaxzevria, 14 days after a single dose of Moderna-mRNA-1273, <14 days from the second dose of Moderna-mRNA-1273, 14 days after a single dose of Sinovac-CoronaVac. Full vaccination will be defined as >7 days from the second dose for Pfizer/BioNTech-Comirnaty, >14 days from the first dose of Janssen-Ad26.COV2.S, >14 days from the second dose for AstraZeneca-Vaxzevria, Moderna-mRNA-1273, or Sinovac-CoronaVac. Booster vaccination one will be defined as >=7 days from an additional dose after full vaccination. Booster vaccination two will be defined as >=7 days from an additional dose after booster vaccination one. Comparison Group no previous vaccinations and no previously confirmed SARS-CoV-2 infection defined using WHO criteria; previously confirmed SARS-CoV-2 infection defined using WHO criteria; 3. partial vaccination (defined above); full vaccination (defined above); booster vaccination (defined above). Outcome SARS-CoV-2 reinfection defined as a possible, probable, or confirmed reinfection case according to the following criteria, adapted from WHO case definitions. Possible reinfection case will be defined as NAAT or AgRDT SARS-CoV-2 positive case with a history of a primary SARS-CoV-2 infection diagnosed by serology, with at least 60 days between the positive serology test and the subsequent positive NAAT or AgRDT. Probable reinfection case will be defined as NAAT or AgRDT SARS-CoV-2 positive case with a history of a primary SARS-CoV-2 infection diagnosed by NAAT or AgRDT, with at least 90 days between the episodes. Alternatively, genomic evidence for the second episode is available and includes lineage that was not submitted to SARS-Cov-2 genomic databases at the time of first infection. Confirmed reinfection case will be defined as two PCR positive episodes supported by viral genomic data from both episodes of infection revealing different Pango lineages. If viral genomic data reveal two distinct Pango lineages this will qualify as adequate evidence to confirm reinfection, regardless of the time elapsed between the two episodes. Study Design Test-negative case-control, traditional case-control, cross-sectional, cohort, non-randomized controlled trials, and randomized controlled trials. Type of literature Published peer-reviewed research articles, preprints, and grey literature in any language. We will prioritize peer-reviewed versions of articles for inclusion and analysis in instances where pre-print versions of peer-reviewed articles are available. Exclusion Criteria (if any met then exclude) Population N/A Exposure Group No evidence of prior confirmed case. No information on the timing, brand, or dose number for the vaccination in hybrid immunity studies. Comparison Group 1. N/A Outcome Prior infection studies not reporting the period of time between primary infection and reinfection such that determining reinfection according to the inclusion criteria is not possible. Hybrid immunity studies not reporting the period of time between either the determination of primary infection or vaccination. Study Design Case reports, case series, incomplete randomized controlled trials, and review papers. Type of literature Media, news stories, and conference abstracts. Original author protocol SARS-CoV-2 protective effectiveness of prior infection and hybrid immunity: a systematic review protocol 2.2 Study inclusion and exclusion criteria View this table: View inline View popup Table 1. Inclusion criteria View this table: View inline View popup Download powerpoint Table 2. Exclusion criteria PA-Testing Simplified Objectives + Inclusion/Exclusion Criteria The purpose of this study was to assess the characteristics of confirmatory tests for primary aldosteronism (PA) and to interpret these in the context of study design and potential risks of bias. The following is an excerpt of two sets of criteria. A study is considered included if it meets all the inclusion criteria. If a study meets any of the exclusion criteria, it should be excluded. Here are the two sets of criteria: Inclusion Criteria (all must be fulfilled) The study is about primary aldosteronism (PA). Please note that primary hyperaldosteronism is a synonym. The study examined at least one of the guideline-recommended confirmatory tests for PA. We are interested in the saline infusion test (SIT), oral salt loading test (SLT), fludrocortisone suppression test (FST), and the captopril challenge test (CCT). Please note some may have synonyms (e.g., SIT = intravenous saline suppression test [IVSS]). Research articles reporting original data The study is conducted in humans of any age 5. The study is in the English language The performance of the confirmatory test(s) was compared with an independent reference standard A reference standard needs to be present to verify disease status. These may include: (i) clinical response to treatment (adrenalectomy and/or medical therapy), (ii) adrenal vein sampling results, (iii) histopathology, (iv) or another confirmatory test. The confirmatory test was used to diagnose PA, rather than exclusively for subtyping. We are interested in knowing how confirmatory testing works for diagnosing PA, not just subtyping. If the confirmatory test was for subtyping, it is still possible to compare how many people had PA vs. non-PA. The data (as published) is extractable for a 2×2 table (TP, FP, FN, TN). If the data are reported in any of the following formats, a 2×2 table can be reconstructed:(i) 2×2 table given, (ii) TP, FP, FN, TN rates given, (iii) Total number of patients (with disease) and total number of study subjects (with and without disease) is known, and corresponding sensitivity and specificity are given. Exclusion criteria (if any met then exclude) No mention about primary aldosteronism (PA) Use of confirmatory tests that fall outside of guide-line recommendations 3. Non-human (e.g., in silico, animal, in vitro). Conference abstracts, reviews (systematic reviews and narrative reviews), editorials, protocols, and secondary publications (data already published in another study). No comparison of confirmatory test performance with a reference standard. Confirmatory test was used only for subtyping, and unable to compare cases of PA vs. non-PA Data not extractable for 2×2 table Original author protocol Confirmatory Testing in Primary Aldosteronism Systematic Review Reference Sheet (Version: May 28, 2021) PA-Outcomes Simplified Objectives and Inclusion/Exclusion Criteria We aimed to conduct a meta-analysis to examine the clinical outcomes of surgery vs medical therapy with respect to mortality, composite major adverse cardiovascular events (MACE, and its individual components), progression to chronic kidney disease, and incident diabetes mellitus. The following is an excerpt of two sets of criteria. A study is considered included if it meets all the inclusion criteria. If a study meets any of the exclusion criteria, it should be excluded. Here are the two sets of criteria: Inclusion Criteria (all must be fulfilled) The study is about primary aldosteronism (PA). Please note that primary hyperaldosteronism is a synonym. Reports surgery (adrenalectomy) or medication (mineralocorticoid receptor antagonists, spironolactone, eplerenone) treatment for primary aldosteronism. Reports mortality, MACE, ACS, stroke, arrhythmia, heart failure, chronic kidney disease, and/or incident diabetes clinical outcomes after treatment. Research articles reporting original data. Randomized clinical trials, cohort studies, and cross-sectional studies. The study is conducted in humans of any age. The study is in the English language. Exclusion criteria (if any met then exclude) Does not report about primary aldosteronism (PA). 2. Does not report treatment for primary aldosteronism. Does not report mortality, major adverse cardiovascular events (MACE), acute coronary syndrome (ACS), stroke, arrhythmia, heart failure, chronic kidney disease, and/or incident diabetes clinical outcomes after treatment. Non-human (e.g., in silico, animal, in vitro). Conference abstracts, case reports, case series, reviews (systematic reviews and narrative reviews), editorials, protocols, and secondary publications (data already published in another study). Original author protocol Primary Aldosteronism Treatment Response: Systematic Review Reference Sheet (Version: June 1, 2022 revised) SVCF Simplified Objectives + Inclusion/Exclusion Criteria Objectives: To evaluate the association of low SVC flow, diagnosed in the first 48 hours after birth echocardiography, with neurological morbidity and mortality, among very preterm neonates. The following is an excerpt of two sets of criteria. A study is considered included if it meets all the inclusion criteria. If a study meets any of the exclusion criteria, it should be excluded. Here are the two sets of criteria: Inclusion Criteria (all must be fulfilled) Preterm infants <32 weeks gestational age who had echocardiography done within the first 48 hours after birth with evaluation of SVC flow Prognostic factor: Low SVC flow identified by Doppler assessment during echocardiography performed in the first 48 hours after birth. Comparison: Normal SVC Flow Reported outcomes including intraventricular hemorrhage (IVH), presence of periventricular leukomalacia (PVL), all-cause mortality before discharge from neonatal intensive care unit (NICU), neurodevelopmental impairment in early childhood or any diagnosed Cerebral Palsy, visual and/or hearing deficits, or necrotizing enterocolitis (NEC) Any grade IVH diagnosed in the first 7 days after birth by cranial ultrasonography Severe IVH (defined as stage 3 or higher according to Papile’s classification 14) diagnosed in the first 7 days after birth Randomized controlled trials, cohort or case-control studies Exclusion criteria (if any met then exclude) Non-human (e.g., in silico, animal, in vitro) Cross-sectional studies, narrative reviews, case series or case reports Original author protocol Association of early-life low superior vena cava flow among preterm neonates and death or cerebral haemorrhage: a systematic review and meta-analysis: Protocol The objective of this study is to systematically review and meta—analyse the association of low SVC flow during the transitional period among preterm neonates < 32 weeks GA with mortalityand adverse neurological morbidity.. PICO/PFO outline- Population Preterm infants <32 weeks gestational age Prognostic factor Low SVC flow identified by Doppler assessment during echocardiography performed in the first 48 hours after birth (measured as ml/kg/min) Comparison Patients with normal SVC flow Outcomes Primary- Any grade IVH diagnosed in the first 7 days of life by cranial ultrasonography Secondary- [1 Severe IVH (defined as stage 3 or higher according to Papile’s classification 14 ) diagnosed in the first 7 days of life, [1 Presence of PVL (diagnosed by 28 days of life) [1 Mortality within the neonatal period (defined as the first 28 days of life) [1 Neurodevelopmental impairment in early childhood (Defined as a composite outcome of any of the following: Cerebral palsy with Gross Motor Function Classification System score >=1 or Bayley-III motor composite <85; Bayley cognitive composite < 85; Bayley language composite <85; Any sensorineural/mixed hearing loss; Any unilateral or bilateral visual impairment) 15 [1 Necrotizing Enterocolitis (NEC) compared between low and normal SVC flow groups METHODOLOGY This systematic review and meta-analysis will be conducted according to the PRISMA guidelines and Cochrane methodology. Eligibility Criteria Randomized controlled trials, cohort or case-control studies that evaluated the following population characteristics will be included for this study: Preterm infants <32 weeks gestational age who had echocardiography done within the first 48 hours after birth with evaluation of SVC flow. Studies must have compared any of the above stated clinical outcomes among preterm neonates with low vs. normal SVC flow to be considered for inclusion. Studies will be included only if they exclusively include human subjects and there will be no language limitations. Exclusion criteria Cross-sectional studies, narrative reviews, case series or case reports on this topic will be excluded. We will also exclude any animal studies. Any identified studies without any available full text from the included databases or the original authors will also be excluded. Acknowledgements We would like to thank Dr. Michelle Baczynski for her support in providing access to the Infant-NO dataset. We would like to thank Dr. Mohammed Ali Alvi for his support in providing access to the Spinal dataset. Footnotes ↵ ** These authors jointly supervised References 1. ↵ Cumpston M , Li T , Page MJ , Chandler J , Welch VA , Higgins JP , et al. Updated guidance for trusted systematic reviews: a new edition of the Cochrane Handbook for Systematic Reviews of Interventions. Cochrane Editorial Unit , editor. Cochrane Database Syst Rev [Internet]. 2019 Oct 3 [cited 2024 Jun 1]; Available from: https://doi.wiley.com/10.1002/14651858.ED000142 2. ↵ Borah R , Brown AW , Capers PL , Kaiser KA . Analysis of the time and workers needed to conduct systematic reviews of medical interventions using data from the PROSPERO registry . BMJ Open . 2017 Feb ; 7 ( 2 ): e012545 . OpenUrl Abstract / FREE Full Text 3. ↵ Michelson M , Reuter K . The significant cost of systematic reviews and meta-analyses: A call for greater involvement of machine learning to assess the promise of clinical trials . Contemp Clin Trials Commun . 2019 Dec ; 16 : 100443 . 4. ↵ Khangura S , Konnyu K , Cushman R , Grimshaw J , Moher D . Evidence summaries: the evolution of a rapid review approach . Syst Rev . 2012 Dec ; 1 ( 1 ): 10 . OpenUrl CrossRef PubMed 5. ↵ Chai KEK , Lines RLJ , Gucciardi DF , Ng L . Research Screener: a machine learning tool to semi-automate abstract screening for systematic reviews . Syst Rev . 2021 Dec ; 10 ( 1 ): 93 . OpenUrl CrossRef PubMed 6. ↵ Johnson EE , O’Keefe H , Sutton A , Marshall C . The Systematic Review Toolbox: keeping up to date with tools to support evidence synthesis . Syst Rev . 2022 Dec 1; 11 ( 1 ): 258 . OpenUrl 7. ↵ O’Connor AM , Tsafnat G , Thomas J , Glasziou P , Gilbert SB , Hutton B . A question of trust: can we build an evidence base to gain trust in systematic review automation technologies? Syst Rev . 2019 Dec ; 8 ( 1 ): 143 . OpenUrl PubMed 8. ↵ Qureshi R , Shaughnessy D , Gill KAR , Robinson KA , Li T , Agai E . Are ChatGPT and large language models “the answer” to bringing us closer to systematic review automation? Syst Rev . 2023 Apr 29; 12 ( 1 ): 72 . OpenUrl CrossRef 9. ↵ Nashwan AJ , Jaradat JH . Streamlining Systematic Reviews: Harnessing Large Language Models for Quality Assessment and Risk-of-Bias Evaluation . Cureus [Internet]. 2023 Aug 6 [cited 2024 Jun 1]; Available from: https://www.cureus.com/articles/178248-streamlining-systematic-reviews-harnessing-large-language-models-for-quality-assessment-and-risk-of-bias-evaluation 10. ↵ Nori H , Lee YT , Zhang S , Carignan D , Edgar R , Fusi N , et al. Can Generalist Foundation Models Outcompete Special-Purpose Tuning? Case Study in Medicine [Internet]. arXiv ; 2023 [cited 2024 Jun 1]. Available from: https://arxiv.org/abs/2311.16452 11. ↵ Guo E , Gupta M , Deng J , Park YJ , Paget M , Naugler C . Automated Paper Screening for Clinical Reviews Using Large Language Models: Data Analysis Study . J Med Internet Res . 2024 Jan 12; 26 : e48996 . OpenUrl 12. Syriani E , David I , Kumar G . Assessing the Ability of ChatGPT to Screen Articles for Systematic Reviews [Internet]. arXiv ; 2023 [cited 2024 Jun 1]. Available from: https://arxiv.org/abs/2307.06464 13. ↵ Kohandel Gargari O , Mahmoudi MH , Hajisafarali M , Samiee R . Enhancing title and abstract screening for systematic reviews with GPT-3.5 turbo . BMJ Evid-Based Med . 2024 Feb ; 29 ( 1 ): 69 – 70 . OpenUrl 14. ↵ Khraisha Q , Put S , Kappenberg J , Warraitch A , Hadfield K . Can large language models replace humans in systematic reviews? Evaluating GPT C4’s efficacy in screening and extracting data from peerCreviewed and grey literature in multiple languages . Res Synth Methods . 2024 Mar 14;jrsm.1715. 15. ↵ Burns PB , Rohrich RJ , Chung KC . The Levels of Evidence and Their Role in Evidence-Based Medicine: Plast Reconstr Surg . 2011 Jul ; 128 ( 1 ): 305 – 10 . OpenUrl 16. ↵ OCEBM Levels of Evidence Working Group . The Oxford Levels of Evidence 2 [Internet]. Oxford Centre for Evidence-Based Medicine . Available from: https://www.cebm.ox.ac.uk/resources/levels-of-evidence/ocebm-levels-of-evidence 17. ↵ Clarivate . Web of Science: List of Subject Classifications for All Databases [Internet] . Available from : https://support.clarivate.com/ScientificandAcademicResearch/s/article/Web-of-Science-Citing-Web-of-Science-data?language=en_US 18. ↵ Kojima T , Gu S (Shane) , Reid M , Matsuo Y , Iwasawa Y . Large language models are zero-shot reasoners. In: Koyejo S, Mohamed S, Agarwal A, Belgrave D, Cho K, Oh A, editors. Advances in neural information processing systems [Internet]. Curran Associates , Inc .; 2022 . p. 22199 – 213 . Available from: https://proceedings.neurips.cc/paper_files/paper/2022/file/8bb0d291acd4acf06ef112099c16f326-Paper-Conference.pdf 19. ↵ Brown TB , Mann B , Ryder N , Subbiah M , Kaplan J , Dhariwal P , et al. Language Models are Few-Shot Learners [Internet] . arXiv ; 2020 [cited 2024 Jun 1]. Available from: https://arxiv.org/abs/2005.14165 20. ↵ Wang X , Wei J , Schuurmans D , Le Q , Chi E , Narang S , et al. Self-Consistency Improves Chain of Thought Reasoning in Language Models [Internet] . arXiv ; 2022 [cited 2024 Jun 1]. Available from: https://arxiv.org/abs/2203.11171 21. ↵ An S , Ma Z , Lin Z , Zheng N , Lou JG . Make Your LLM Fully Utilize the Context [Internet] . arXiv ; 2024 [cited 2024 Jun 1]. Available from: https://arxiv.org/abs/2404.16811 22. ↵ Liu NF , Lin K , Hewitt J , Paranjape A , Bevilacqua M , Petroni F , et al. Lost in the Middle: How Language Models Use Long Contexts . Trans Assoc Comput Linguist . 2024 Feb 23; 12 : 157 – 73 . OpenUrl 23. ↵ Waffenschmidt S , Knelangen M , Sieben W , Bühn S , Pieper D . Single screening versus conventional double screening for study selection in systematic reviews: a methodological systematic review . BMC Med Res Methodol . 2019 Dec ; 19 ( 1 ): 132 . OpenUrl CrossRef PubMed 24. ↵ Mahtani KR , Heneghan C , Aronson J . Single screening or double screening for study selection in systematic reviews? BMJ Evid-Based Med . 2020 Aug ; 25 ( 4 ): 149 – 50 . OpenUrl 25. ↵ Gates A , Guitard S , Pillay J , Elliott SA , Dyson MP , Newton AS , et al. Performance and usability of machine learning for screening in systematic reviews: a comparative evaluation of three tools . Syst Rev . 2019 Dec ; 8 ( 1 ): 278 . OpenUrl CrossRef PubMed 26. ↵ Matyas N , Gartlehner G , Ravaud P , Atal I . Comparing the performance of three tools for semi-automated abstract screening when conducting systematic reviews: Abstrackr, Rayyan and RobotAnalyst . In: Cochrane Colloquium Abstracts [Internet]. Available from: https://abstracts.cochrane.org/2019-santiago/comparing-performance-three-tools-semi-automated-abstract-screening-when-conducting 27. ↵ Mihalache A , Popovic MM , Muni RH . Performance of an Artificial Intelligence Chatbot in Ophthalmic Knowledge Assessment . JAMA Ophthalmol . 2023 Jun 1; 141 ( 6 ): 589 . OpenUrl 28. ↵ Soroush A , Glicksberg BS , Zimlichman E , Barash Y , Freeman R , Charney AW , et al. Large Language Models Are Poor Medical Coders — Benchmarking of Medical Code Querying . NEJM AI [Internet ]. 2024 Apr 25 [cited 2024 Jun 1];1(5). Available from: https://ai.nejm.org/doi/10.1056/AIdbp2300040 29. ↵ Han C , Kim DW , Kim S , Chan You S , Park JY , Bae S , et al. Evaluation of GPT-4 for 10-year cardiovascular risk prediction: Insights from the UK Biobank and KoGES data . iScience . 2024 Feb ; 27 ( 2 ): 109022 . OpenUrl 30. ↵ Ueda D , Walston SL , Matsumoto T , Deguchi R , Tatekawa H , Miki Y . Evaluating GPT-4-based ChatGPT’s clinical potential on the NEJM quiz . BMC Digit Health . 2024 Jan 11; 2 ( 1 ): 4 . OpenUrl 31. Guillen-Grima F , Guillen-Aguinaga S , Guillen-Aguinaga L , Alas-Brun R , Onambele L , Ortega W , et al. Evaluating the Efficacy of ChatGPT in Navigating the Spanish Medical Residency Entrance Examination (MIR): Promising Horizons for AI in Clinical Medicine . Clin Pract . 2023 Nov 20; 13 ( 6 ): 1460 – 87 . OpenUrl 32. ↵ Levin G , Horesh N , Brezinov Y , Meyer R . Performance of ChatGPT in medical examinations: A systematic review and a metaCanalysis . BJOG Int J Obstet Gynaecol . 2024 Feb; 131 ( 3 ): 378 – 80 . OpenUrl 33. ↵ Nguyen MV , Luo L , Shiri F , Phung D , Li YF , Vu TT , et al. Direct Evaluation of Chain-of-Thought in Multi-hop Reasoning with Knowledge Graphs [Internet] . arXiv ; 2024 [cited 2024 Jun 1]. Available from: https://arxiv.org/abs/2402.11199 34. ↵ Min S , Lyu X , Holtzman A , Artetxe M , Lewis M , Hajishirzi H , et al. Rethinking the Role of Demonstrations: What Makes In-Context Learning Work? [ Internet]. arXiv ; 2022 [cited 2024 Jun 1]. Available from: https://arxiv.org/abs/2202.12837 35. ↵ Wei J , Wei J , Tay Y , Tran D , Webson A , Lu Y , et al. Larger language models do in-context learning differently [Internet] . arXiv ; 2023 [cited 2024 Jun 1]. Available from: https://arxiv.org/abs/2303.03846 36. ↵ Reynolds L , McDonell K . Prompt Programming for Large Language Models: Beyond the Few-Shot Paradigm [Internet] . arXiv ; 2021 [cited 2024 Jun 1]. Available from: https://arxiv.org/abs/2102.07350 37. ↵ Xiao G , Tian Y , Chen B , Han S , Lewis M . Efficient Streaming Language Models with Attention Sinks [Internet] . arXiv ; 2023 [cited 2024 Jun 1]. Available from: https://arxiv.org/abs/2309.17453 38. ↵ Sui Y , Zhou M , Zhou M , Han S , Zhang D . Table Meets LLM: Can Large Language Models Understand Structured Table Data? A Benchmark and Empirical Study [Internet]. arXiv ; 2023 [cited 2024 Jun 1]. Available from: https://arxiv.org/abs/2305.13062 39. ↵ Wornow M , Lozano A , Dash D , Jindal J , Mahaffey KW , Shah NH . Zero-Shot Clinical Trial Patient Matching with LLMs [Internet] . arXiv ; 2024 [cited 2024 Jun 1]. Available from: https://arxiv.org/abs/2402.05125 40. ↵ Kanoulas E , Li D , Azzopardi L , Spijker R. CLEF 2019 technology assisted reviews in empirical medicine overview . In: Conference and labs of the evaluation forum [Internet]. 2019 . Available from: https://api.semanticscholar.org/CorpusID:263629854 41. ↵ Wang S , Scells H , Clark J , Koopman B , Zuccon G . From Little Things Big Things Grow: A Collection with Seed Studies for Medical Systematic Review Literature Search . 2022 [cited 2024 Jun 1]; Available from: https://arxiv.org/abs/2204.03096 42. ↵ Cohen AM , Hersh WR , Peterson K , Yen PY . Reducing Workload in Systematic Review Preparation Using Automated Citation Classification . J Am Med Inform Assoc . 2006 Mar 1; 13 ( 2 ): 206 – 19 . OpenUrl CrossRef PubMed 43. ↵ Bobrovitz N , Arora RK , Cao C , Boucher E , Liu M , Donnici C , et al. Global seroprevalence of SARS-CoV-2 antibodies: A systematic review and meta-analysis. Khudyakov YE, editor . PLOS ONE . 2021 Jun 23; 16 ( 6 ): e0252617 . OpenUrl CrossRef PubMed 44. ↵ Bobrovitz N , Ware H , Ma X , Li Z , Hosseini R , Cao C , et al. Protective effectiveness of previous SARS-CoV-2 infection and hybrid immunity against the omicron variant and severe disease: a systematic review and meta-regression . Lancet Infect Dis . 2023 May ; 23 ( 5 ): 556 – 67 . OpenUrl CrossRef 45. ↵ Samnani S , Cenzer I , Kline GA , Lee SJ , Hundemer GL , McClurg C , et al. Time to Benefit of Surgery vs Targeted Medical Therapy for Patients With Primary Aldosteronism: A Meta-analysis . J Clin Endocrinol Metab . 2024 Feb 20; 109 ( 3 ): e1280 – 9 . OpenUrl 46. ↵ Leung AA , Symonds CJ , Hundemer GL , Ronksley PE , Lorenzetti DL , Pasieka JL , et al. Performance of Confirmatory Tests for Diagnosing Primary Aldosteronism: a Systematic Review and Meta-Analysis . Hypertension . 2022 Aug ; 79 ( 8 ): 1835 – 44 . OpenUrl CrossRef 47. ↵ Teja B , Berube M , Pereira TV , Law AC , Schanock C , Pang B , et al. Effectiveness of Fludrocortisone Plus Hydrocortisone versus Hydrocortisone Alone in Septic Shock: A Systematic Review and Network Meta-Analysis of Randomized Controlled Trials . Am J Respir Crit Care Med . 2024 May 15; 209 ( 10 ): 1219 – 28 . OpenUrl 48. ↵ Alvi MA , Kwon BK , Hejrati N , Tetreault LA , Evaniew N , Skelly AC , et al. Accuracy of Intraoperative Neuromonitoring in the Diagnosis of Intraoperative Neurological Decline in the Setting of Spinal Surgery—A Systematic Review and Meta-Analysis . Glob Spine J . 2024 Mar; 14 (3_suppl):105S-149S. 49. ↵ Hsu CH , Couper K , Nix T , Drennan I , Reynolds J , Kleinman M , et al. Calcium during cardiac arrest: A systematic review . Resusc Plus . 2023 Jun ; 14 : 100379 . 50. ↵ Baczynski M , Jasani B , De Castro C , Dani C , Subhedar NV , Chandrasekharan P , et al. Association between immediate oxygenation response and survival in preterm infants receiving rescue inhaled nitric oxide therapy for hypoxemia from pulmonary hypertension: A systematic review and meta-analysis . Early Hum Dev . 2023 Sep ; 184 : 105841 . 51. ↵ Bobrovitz N , Heneghan C , Onakpoya I , Fletcher B , Collins D , Tompson A , et al. Medications that reduce emergency hospital admissions: an overview of systematic reviews and prioritisation of treatments . BMC Med . 2018 Dec ; 16 ( 1 ): 115 . OpenUrl CrossRef PubMed 52. ↵ Mascarenhas D , Weisz D , Jasani B , Persad N , Main E . Premedication for rapid sequence intubation in neonates - a network meta-analysis . PROSPERO 2022 CRD42022384259 [Internet]. PROSPERO. Available from: https://www.crd.york.ac.uk/prospero/display_record.php?ID=CRD42022384259 53. ↵ Cochran WG . Sampling techniques , 3rd edition . John Wiley ; 2002 . 54. ↵ PerlmanCArrow S , Loo N , Bobrovitz N , Yan T , Arora RK . A realCworld evaluation of the implementation of NLP technology in abstract screening of a systematic review . Res Synth Methods . 2023 Jul ; 14 ( 4 ): 608 – 21 . OpenUrl 55. ↵ Nussbaumer-Streit B , Ellen M , Klerings I , Sfetcu R , Riva N , Mahmić-Kaknjo M , et al. Resource use during systematic review production varies widely: a scoping review . J Clin Epidemiol . 2021 Nov ; 139 : 287 – 96 . OpenUrl 56. ↵ Polanin JR , Pigott TD , Espelage DL , Grotpeter JK . Best practice guidelines for abstract screening largeCevidence systematic reviews and metaCanalyses . Res Synth Methods . 2019 Sep ; 10 ( 3 ): 330 – 42 . OpenUrl 57. ↵ Clopper CJ , Pearson ES . THE USE OF CONFIDENCE OR FIDUCIAL LIMITS ILLUSTRATED IN THE CASE OF THE BINOMIAL . Biometrika . 1934 ; 26 ( 4 ): 404 – 13 . OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted June 03, 2024. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Prompting is all you need: LLMs for systematic review screening Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Prompting is all you need: LLMs for systematic review screening Christian Cao , Jason Sang , Rohit Arora , Robbie Kloosterman , Matt Cecere , Jaswanth Gorla , Richard Saleh , David Chen , Ian Drennan , Bijan Teja , Michael Fehlings , Paul Ronksley , Alexander A Leung , Dany E Weisz , Harriet Ware , Mairead Whelan , David B Emerson , Rahul Arora , Niklas Bobrovitz medRxiv 2024.06.01.24308323; doi: https://doi.org/10.1101/2024.06.01.24308323 Share This Article: Copy Citation Tools Prompting is all you need: LLMs for systematic review screening Christian Cao , Jason Sang , Rohit Arora , Robbie Kloosterman , Matt Cecere , Jaswanth Gorla , Richard Saleh , David Chen , Ian Drennan , Bijan Teja , Michael Fehlings , Paul Ronksley , Alexander A Leung , Dany E Weisz , Harriet Ware , Mairead Whelan , David B Emerson , Rahul Arora , Niklas Bobrovitz medRxiv 2024.06.01.24308323; doi: https://doi.org/10.1101/2024.06.01.24308323 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (574) Allergy and Immunology (865) Anesthesia (304) Cardiovascular Medicine (4462) Dentistry and Oral Medicine (445) Dermatology (383) Emergency Medicine (611) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1517) Epidemiology (15251) Forensic Medicine (31) Gastroenterology (1132) Genetic and Genomic Medicine (6621) Geriatric Medicine (669) Health Economics (1002) Health Informatics (4564) Health Policy (1372) Health Systems and Quality Improvement (1617) Hematology (544) HIV/AIDS (1272) Infectious Diseases (except HIV/AIDS) (15938) Intensive Care and Critical Care Medicine (1107) Medical Education (624) Medical Ethics (147) Nephrology (670) Neurology (6643) Nursing (346) Nutrition (1001) Obstetrics and Gynecology (1149) Occupational and Environmental Health (957) Oncology (3350) Ophthalmology (981) Orthopedics (369) Otolaryngology (421) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1698) Pharmacology and Therapeutics (694) Primary Care Research (714) Psychiatry and Clinical Psychology (5465) Public and Global Health (9259) Radiology and Imaging (2212) Rehabilitation Medicine and Physical Therapy (1372) Respiratory Medicine (1198) Rheumatology (598) Sexual and Reproductive Health (716) Sports Medicine (533) Surgery (715) Toxicology (100) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a03c6f23ce1f4807',t:'MTc4MDEzMDQyNw=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();