Large Language Models to process, analyze, and synthesize biomedical texts – a scoping review

Large Language Models to process, analyze, and synthesize biomedical texts – a scoping review | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Large Language Models to process, analyze, and synthesize biomedical texts – a scoping review View ORCID Profile Simona Emilova Doneva , Sijing Qin , View ORCID Profile Beate Sick , View ORCID Profile Tilia Ellendorff , Jean-Philippe Goldman , View ORCID Profile Gerold Schneider , View ORCID Profile Benjamin Victor Ineichen doi: https://doi.org/10.1101/2024.04.19.588095 Simona Emilova Doneva 1 Center for Reproducible Science, University of Zurich , Zurich, Switzerland Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Simona Emilova Doneva For correspondence: simona.doneva{at}uzh.ch Sijing Qin 1 Center for Reproducible Science, University of Zurich , Zurich, Switzerland Find this author on Google Scholar Find this author on PubMed Search for this author on this site Beate Sick 2 ZHAW School of Engineering , Winterthur, Switzerland Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Beate Sick Tilia Ellendorff 3 Department of Computational Linguistics, University of Zurich , Zurich, Switzerland Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Tilia Ellendorff Jean-Philippe Goldman 3 Department of Computational Linguistics, University of Zurich , Zurich, Switzerland Find this author on Google Scholar Find this author on PubMed Search for this author on this site Gerold Schneider 3 Department of Computational Linguistics, University of Zurich , Zurich, Switzerland Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Gerold Schneider Benjamin Victor Ineichen 1 Center for Reproducible Science, University of Zurich , Zurich, Switzerland 4 Clinical Neuroscience Center, University of Zurich , Zurich, Switzerland Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Benjamin Victor Ineichen Abstract Full Text Info/History Metrics Data/Code Preview PDF ABSTRACT The advent of large language models (LLMs) such as BERT and, more recently, GPT, is transforming our approach of analyzing and understanding biomedical texts. To stay informed about the latest advancements in this area, there is a need for up-to-date summaries on the role of LLM in Natural Language Processing (NLP) of biomedical texts. Thus, this scoping review aims to provide a detailed overview of the current state of biomedical NLP research and its applications, with a special focus on the evolving role of LLMs. We conducted a systematic search of PubMed, EMBASE, and Google Scholar for studies and conference proceedings published from 2017 to December 19, 2023, that develop or utilize LLMs for NLP tasks in biomedicine. LLMs are being applied to a wide array of tasks in the biomedical field, including knowledge management, text mining, drug discovery, and evidence synthesis. Prominent among these tasks are text classification, relation extraction, and named entity recognition. Although BERT-based models remain prevalent, the use of GPT-based models has substantially increased since 2023. INTRODUCTION Natural Language Processing (NLP), a field at the intersection of computer science, artificial intelligence and linguistics, aims at enabling computers with the ability to comprehend language similarly to how humans do. It has become an indispensable tool for processing and analyzing large volumes of text data, widely adopted in various fields including biomedical research and clinical applications [ 1 ]. A pivotal methodological advancement in NLP has been the transformer architecture, which underpins most recent Large Language Models (LLMs), including both BERT, Bidirectional Encoder Representations from Transformers, and GPT, Generative Pre-trained Transformer. The transformer model is appreciated for its efficiency and ability to process data in parallel. This capability, combined with a deep learning architecture that leverages attention mechanisms, allows to handle long dependencies and yields a better contextual understanding [ 2 ]. Such advancements have fundamentally altered the landscape of NLP model development. The focus has shifted towards refining and customizing pre-trained language models, moving away from the traditional approach of building models from the ground up. This approach is especially advantageous in specialized fields like biomedicine, where annotated datasets are often limited or difficult to compile. By leveraging the fine-tuning capabilities of LLMs for specific tasks, researchers in the biomedical domain can attain high performance levels, even when working with relatively small datasets [ 3 ]. Amidst these advancements, the emergence of generative models marks an evolution towards creative and generative applications, setting a contrast to the discriminative nature of earlier models [ 4 , 5 ]. While BERT and similar systems excel in task-oriented applications such as sentence classification or entity recognition, generative models are capable of producing coherent and contextually relevant text based on the input they receive. This capability facilitates novel applications such as automated content creation, summarization, and question-answering systems in the biomedical domain. In addition, these models show promising capabilities in exploring instruction tuning and in-context learning. This enables them to comprehend and execute a wide array of tasks using natural language instructions or examples, thus further reducing the reliance on task-specific datasets or extensive fine-tuning [ 6 ]. However, generative models also introduce new risks and challenges, such as the generation of plausible but incorrect or misleading information, underscoring the need for careful evaluation and application of these technologies [ 7 , 8 ]. Given the fast-paced nature of NLP development, there is a need for up-to-date literature overviews. This is particularly true in fields like biomedical research, where the ability to swiftly and accurately process vast amounts of text data can substantially impact patient care and scientific discovery [ 9 ]. Thus, this scoping review has three goals: Firstly, we aim to identify and categorize the specific tasks within BioNLP that are currently being addressed using LLMs, e.g., in evidence synthesis, pharmacology, or clinical use cases. Secondly, we aim to map the various LLM architectures employed in these BioNLP tasks. Finally, this study sets out to assess the transparency of methods used in the development of these LLMs, including availability of source code and data as well as used hardware and software. Related Work The work presented in [ 10 ] presented the first comprehensive review of the development and landscape of transformer-based biomedical language models. It examined the core implementation principles behind these models and their practical applications for BioNLP tasks with a focus on encoder-based language models. Subsequent work expanded the discussion to include RNN-based models and those that leverage decoders for generative pre-training [ 3 ]. Furthermore, the authors extended their exploration of biomedical LLMs to encompass applications beyond textual data, including analyses of proteins, DNA, and text-image pairs. Aligning with our goals, they also provided an in-depth exploration of language model applications within the biomedical domain, offering a thorough classification of techniques, datasets, and benchmarking competitions [ 3 ]. Other reviews have concentrated on the use of LLMs within specific niches of biomedical text processing. For instance, how LLMs can enhance downstream tasks in literature-based discovery [ 11 ]. Another survey shed light on biomedical and clinical NLP research conducted in languages other than English, focusing on data resources, language models, and common NLP tasks in these languages. This survey also highlighted the growing trend of employing transformer-based language models for various NLP tasks in medical fields [ 12 ]. Recent surveys have delved into the unique challenges posed by deploying LLMs in healthcare, especially concerning fairness and accountability. For example, an overview of the technical aspects of development and usage of LLM methods in healthcare, including the specific challenges such as ethical considerations encountered when deploying this technology in the field, has been provided [ 13 ]. Similarly, the potential of models such as ChatGPT to enhance medical practices, while addressing concerns about safety, ethics, and maintaining the human element in patient care, has been reviewed in [ 7 ]. A related systematic review evaluated the benefits and limitations of ChatGPT in healthcare education, research, and practice, based on an analysis of 60 publications from PubMed/MEDLINE and Google Scholar [ 14 ]. Insights into the opportunities and challenges of LLMs for biomedical or clinical usage, including their roles in pre-consultation, diagnosis, data management, and their support for medical education and writing, have also been offered in [ 15 ]. In the present scoping review, we conduct an examination of the usage of both discriminative and generative LLM architectures in biomedicine, employing a robust systematic review methodology. We present in-depth quantitative analyses, uncover trends over time, and critically assess transparency and Open Science pracrtices in the BioNLP field, thereby enriching the insights of existing research. METHODS Study registration We registered the study protocol on the Open Science Framework (OSF) platform ( https://osf.io/bjv24/ ). Literature search We conducted a search for studies in PubMed and EMBASE, spanning from 2017 (i.e., the inception of transformer-based LLMs [ 2 ]) to December 19, 2023. Additionally, we searched Google Scholar for EMNLP and ACL machine learning conference proceedings through the “Publish or Perish” software 1 , employing a search string translated from the one used in our PubMed query. This search string was created by an information specialist at the University Library Zurich. The complete search string can be found in the supplementary data. Inclusion and exclusion criteria We included original research articles that employed LLMs to analyze extensive collections of biomedical texts, e.g., scientific publications, (pre)clinical trial registries, patents, and grey literature. Additionally, systematic reviews and meta-analyses that leveraged LLMs for automating certain tasks like abstract screening were also considered. We excluded non-English publications as well as text data derived from clinical questionnaires or surveys. Reviews were also excluded but were retained as a supplementary source for additional references. Study selection All retrieved publications were screened using ASReview, which incorporates machine learning algorithms to prioritize relevant studies [ 16 ]. We established a rule in advance to cease the screening of abstracts once we encountered thirty consecutive irrelevant abstracts. Following this, any remaining abstracts were excluded from further analysis. Data extraction and synthesis The following data were extracted from available full texts of eligible articles: target NLP application (grouped into pre-specified classes), biomedical domain of the application, target database for the application of the developed methods (e.g., PubMed), data type (e.g., abstracts), data availability, hosted application for end-users, LLM (e.g., BioBERT), programming language (e.g., Python), library/framework (e.g., HuggingFace), reported performance metrics (e.g., F1-score), source code availability, pretraining corpus origin (if pre-training was performed, which data was used, e.g., COVID-19 Open Research Dataset), fine-tuning corpus data/task (e.g., BC5), fine-tuning corpus size, number of tasks/datasets for performance evaluation, hardware type (e.g., GPU). These criteria were developed and iteratively improved over a pilot extraction round conducted by two authors (SED and BVI). We summarized our findings in a narrative fashion bolstered by descriptive statistics of extracted parameters. Risk of bias assessment We assessed the quality of each study against three predefined criteria: 1) Is the process of splitting training from validation data described? 2) Are there methods described for avoiding overfitting or underfitting? 3) Does the dataset or assessment measure provide a possibility to compare to other tools in the same domain? Changes from protocol While in our initial protocol, we were planning to evaluate all ML-based approaches for BioNLP, we decided to shift the focus to specifically evaluating the emerging field of LLMs. RESULTS Eligible publications In total, 13,823 publications were retrieved from our database search. 721 publications were eligible for full-text screening (see supplementary Figure S9). A total of 196 articles were included in our final review, with 137 coming from the biomedical journals and 59 from the NLP conference outputs ( Figure 1 A ). Figure 1 B also shows the key sources of these articles, highlighting contributions from acknowledged NLP venues such as the ACL/Findings and the EMNLP conferences, as well as biomedical journals such as BMC Bioinformatics and the Journal of Biomedical Informatics. Download figure Open in new tab Figure 1. A : Number of included articles over the chosen time frame. B : Top 10 sources of included articles and their frequency, including NLP venues and biomedical journals. Risk of bias of eligible articles The overall risk of bias was moderate with 89% of studies reporting whether training and validation data were split, 25% of studies explicitly reported measures to mitigate over-/underfitting of models, and 84% of studies report on benchmarking their approaches against other methods. Applications Data Sources and Data Types Figure 2 introduces the top 10 data sources used to inform the training and refinement phases of the LLMs development. The most frequently reported ones were PubMed/Medline (131 publications, 58%), clinical texts (i.e., textual data generated in clinical settings such as electronic health records; 41, 18%), and social media (16, 7%). Download figure Open in new tab Figure 2. Frequency of specific data sources ( A ) and data types ( B ) used for LLM development and testing. A paper could use more than one data source and type. The most commonly used data types were abstracts (115 publications, 50%) and full texts of scientific articles (49, 21%) as well as clinical notes (17, 7%) ( Figure 2 ). Biomedical Application Domains Eligible publications were grouped into 7 main categories of biomedical applications for LLMs (see Table 1 ). The top three categories were knowledge management (62 publications, 32%), general biomedical text mining (57, 29%), and pharmacology (24, 12%) ( Figure 3 A ). Further details about applications within the top 4 most frequent domains are provided in the supplementary materials in Figure S10. View this table: View inline View popup Download powerpoint Table 1. Main application domains for LLMs in biomedicine. Abbreviations: NLP, natural language processing. Download figure Open in new tab Figure 3. Biomedical domains ( A ) and NLP tasks ( B ) by article frequency. Each paper was assigned to a single primary application domain and one main NLP task. NLP tasks A variety of NLP tasks has been employed by the eligible articles, most commonly text classification (55 publications, 28%), information extraction (27, 14%), and relation extraction (25, 13%) ( Figure 3 B ). Matching the main domains of application with the NLP tasks shows notable links ( Figure 4 ). Download figure Open in new tab Figure 4. Sankey Diagram showing the relationships between the biomedical domains and the utilized NLP applications. Each domain is represented as a source node, while the associated NLP applications are shown as the target nodes. The thickness of the flows between the two is proportional to the number of articles that exhibit this connection. In knowledge management, LLMs have been used for two main applications: text classification and information retrieval. Text classification can streamline the organization of large literature volumes by assigning predefined categories or labels to documents. This approach has been effectively applied to enhance reference prioritization in systematic reviews [17, 18, 19, 20, 21]. Text classification has also been used to better understand the structure of papers, for example by automatically predicting sections and headers in Electronic Health Records [ 22 ]. Information retrieval focuses on enabling users to find and access the specific information they need from among vast amounts of available content. Various applications have been developed to improve the discovery of novel insights by more effectively aggregating data from existing knowledge databases, as well as improving article recommendation service [ 23 , 24 ]. For instance, pubmedKB serves as a web server designed to extract and visualize semantic relationships between genes, diseases, chemicals, and variants within PubMed abstracts [ 25 ]. General biomedical text mining spans various tasks, with multi-domain analysis standing out. This area focuses on designing methods or frameworks capable of addressing multiple NLP tasks through a unified strategy. An illustration of this is BioGPT, a generative transformer model, which, after being pre-trained on an extensive biomedical literature corpus, demonstrates its versatility across six different BioNLP challenges [ 26 ]. Named entity recognition and entity normalization are also challenges commonly tackled together. For example, [ 27 ] developed a multi-task model that simultaneously learns sentence-level and token-level labels for NER, utilizing BioBERT for text encoding and sharing hidden states across tasks. [ 28 ] introduced an architecture that enhances pre-trained language models by integrating domain-specific knowledge from diverse sources such as the UMLS Metathesaurus and Wikipedia articles on diseases, through knowledge adapters and an attention-based controller, demonstrating performance improvements on various biomedical NLP tasks. The pharmacological domain utilizes relation extraction the most. This task involves identifying and extracting relationships between entities such as drugs, genes, or proteins from text, which is crucial for pharmacological research and drug development. For example, [ 29 ] utilize different BERT models for drug-drug interaction extraction by integrating drug representations from a pharmaceutical knowledge graph and corpus text. [ 30 ] used contextual information in extracting long distance adverse drug events from clinical notes with BERT models. There were also overlaps between domains, such as work that introduced a system for detecting adverse drug reactions from text on social media by fine-tuning BERT [ 31 ]. In the clinical domain, text classification is frequently utilized for predicting patient outcomes. For instance, in-hospital mortality predictions have been made by combining time series data from various medical devices with clinical notes found in electronic health records [ 32 , 33 ]. Additionally, there are applications in mental health, such as the automated detection of mental conditions using transcribed patient recordings [ 34 , 35 ]. Finally, social media platforms have been utilized for mental health screening, employing text classification to identify suicidal risk and predict mental health disorders from user-generated content, such as Reddit and Twitter posts [ 36 , 37 , 38 ]. Additionally, this NLP task has been applied for detecting nuanced emotional states within online health communities [ 39 ]. Another recent development has been the use of fact verification techniques to authenticate statements related to COVID-19 [ 40 , 41 ]. Large Language Models Models overview and trends over time The most prominently used LLMs were encoder-based BERT models (275 models, 91%), followed by generative models such as GPT and T5 (27, 9%) ( Figure 5 ). Download figure Open in new tab Figure 5. A : Most frequently used models. B : Top 5 models utilized each year. Each paper could use more than one LLM. Figure 5 presents the annual usage trends of the top five LLMs. While BERT models are prominently used in earlier years, GPT models see increasing use in 2023. The data also shows that general-purpose models such as BERT are replaced with more specialized models, e.g., BioBERT and SciBERT. Technical setup: hardware and programming languages The most commonly employed hardware were GPUs (92%) ( Figure 6 ). Of note, there was a lack of reporting on hardware use in many instances (118 publications). Download figure Open in new tab Figure 6. Reported technical setup regarding hardware ( A ), programming language ( B ), and computational library ( C ) used for the method implementation. Each paper could use more than programming language and library. The vast majority of studies used the Python programming language for their technical setup (157 publications, 91%) ( Figure 6 ). JAVA, R, Perl, PHP, AWK, and C were rarely used. HuggingFace (83 publications, 36%), PyTorch (48, 21%), and TensorFlow (30, 13%) were the most commonly reported computational libraries for using LLMs ( Figure 6 ) . Some studies used more specialized libraries such as Stanford CoreNLP, spaCy, Keras, NLTK, and Torch. Notably, many studies (37) did not report the programming language and computational library used. Fine-Tuning Tasks and Datasets Benchmark Datasets The field of BioNLP relies heavily on the availability of standardized datasets and benchmarks to assess and compare the performance of language models. Table 2 provides a compilation of the most commonly used benchmarks according to the reviewed papers. View this table: View inline View popup Download powerpoint Table 2. Overview of commonly used datasets for different NLP tasks. The “Frequency” column contains the number of papers that have used the dataset for fine-tuning or testing. Many of the reported datasets are collaborative efforts released for competitions and shared tasks, and designed to facilitate fair comparisons among different research teams. Key competitions and shared tasks that emerged from the eligible studies include: Informatics for Integrating Biology and the Bedside (i2b2) Challenges: The i2b2 Center, active from 2004 to 2014 and funded by the NIH, focuses on releasing clinical records for NLP research. Since 2018, this initiative is officially known as n2c2 1 . The most frequently used dataset from this area is the 2010 i2b2/VA for concepts, assertions, and relation extraction. Other datasets mentioned in the reviewed papers include the 2012 corpus annotated for events, temporal expressions and temporal relations [ 55 ], and the de-identification datasets of i2b2 2014 [ 56 ]. Three studies utilized data from the 2019 n2c2/OHNLP track on clinical semantic textual similarity [ 57 , 58 , 59 ], and one study tackled the 2022 n2c2 challenge, focusing on medication extraction and event/context classification [ 60 ]. BioCreative (BC) Workshops: These workshops host challenges related to the extraction and annotation of biological entities and their relationships from scientific literature. Notable datasets produced from these workshops include BC5, which covers disease and chemical annotations, and BC4, focused on chemical annotations. Additionally, 8 further papers reported utilizing a BioCreative corpus, with LitCovid and DrugProt being the most prominent [ 61 , 62 , 29 , 63 ]. Text REtrieval Conference (TREC): TREC is an ongoing series of workshops that cover a wide range of information retrieval topics. Datasets associated with TREC, such as TREC-COVID and Health Misinformation, have been employed in four of the reviewed research papers [ 64 , 65 , 66 , 67 ]. One paper also utilized three datasets of the TREC series 2017-2019 for clinical decision support [ 68 ]. BioNLP Shared Task (BioNLP-ST): This series is aimed at advancing the extraction of detailed bio-molecular events from scientific texts. The provided data from different years has been utilized nine times in the reviewed papers. For example, the 2011 Protein Coreference dataset is used for the evaluation of methods for coreference extraction among protein/gene [ 69 ]. A combination of the datasets from BioNLP 2009, 2011 and 2013 is used in [ 70 ] to benchmark a novel fine-tuning approach. Custom-developed Datasets 41% of the publications reported on the development of new datasets. These datasets were mostly in the area of text classification, information retrieval/extraction, and named entity recognition ( Figure 7 ). Download figure Open in new tab Figure 7. Target applications for which custom data has been annotated or collected. Transparency of methods There was a moderate level of code (e.g., algorithms and computational methods transparency) with fewer than 40% of the studies making their code available alongside the publication ( Figure 8 ). At the same time, there was a high level of data transparency with over three quarters of studies making their datasets fully available. Finally, only a minority of studies hosted applications for end-users with a higher propensity of biomedical journals compared to NLP venues. Download figure Open in new tab Figure 8. Transparency of LLM Methods: source code and data availability, and end-user application deployment. DISCUSSION Main Findings The objective of our scoping review was to explore and organize the specific tasks in BioNLP that are being tackled with LLMs. In the biomedical domain, LLMs have found applications across a diverse set of tasks, including knowledge management, drug discovery, clinical applications, and synthesizing evidence. Within these areas, tasks such as text classification, relation extraction, and named entity recognition/information extraction stand out for their prominence. The primary sources of data for these studies are often PubMed/Medline abstracts or clinical documents, such as electronic health records. While BERT-based models continue to be widely used, there has been a noticeable surge in the adoption of GPT-based models since 2023. Moreover, the technical frameworks for these studies typically involve the Python programming language, with HuggingFace and PyTorch being the most popular computational libraries. Limitations and Challanges of LLMs for BioNLP Most LLM applications reviewed in the present study were developed using information from scientific abstracts. This preference might be due to the easy access to and the concise nature of this data type. However, information from abstracts might not be well aligned with the corresponding full reports. For example, [ 71 ] analyse the state of reporting of primary biomedical research and find inconsistencies with respect to the reported sample sizes, outcome measures, result presentation and interpretation, and conclusions or recommendations between abstracts and full texts. This suggests the need to be cautious when developing applications that rely only on the information reported in abstracts, especially for evidence-synthesis. Article abstracts and article bodies can differ not only in content, but also in their structural, linguistic, and semantic composition [ 72 ]. LLMs are generally well equipped to handle a wide variety of linguistic styles and structures due to their vast training on diverse text corpora. However, those models can still struggle with the subtlety and specificity required in academic texts. This limitation can be mitigated by the usage of domain specific language models, that also include full-text academic articles in their training data [ 73 ]. The most common text-based applications of LLMs in the biomedical field were within the knowledge management domain, showcasing their potential as a solution to the challenges brought on by the exponential growth of biomedical literature [74, 75, 76, 77, 78]. However, we also observed that LLMs have been mostly used as a supportive tool by assisting with preliminary tasks in biomedical research, but they have not yet been capable of replacing more nuanced and critical tasks entirely. For example, while LLMs can expedite the initial screening of literature for relevant studies and assist in the extraction of data from texts, they cannot replace the expert analysis required for interpreting study results, assessing the quality of evidence, or making clinical decisions [ 79 , 80 ]. Further considerations include the challenge of maintaining model accuracy amidst the fast pace of medical knowledge evolution, the models’ limited interpretability, susceptibility to hallucinations and biases, and ethical and privacy concerns related to handling sensitive patient information. These issues emphasize the importance of cautious and responsible LLM implementation in the biomedical domain [8, 81, 7, 82, 83]. Emergence of generative LLMs While BERT-based models have been the predominantly used LLMs, there was a notable surge in GPT-like models in 2023. Generative models have been benchmarked against BERT-based approaches as a classification and information extraction technique for biomedical literature curation [ 84 , 85 , 86 ]. Notably, they have also been applied in innovative ways, such as generating symptom definitions to enrich datasets and improve annotator precision, as well as for summarizing radiology reports, potentially aiding clinical decision-making [ 87 , 88 ]. ChatGPT’s interactive capabilities have also been explored for preliminary screening in cases of mild cognitive impairment [ 89 ]. Recently, the landscape of generative biomedical models has been enriched by open-source solutions such as LLama 2, MEDITRON, and BioMistral [ 90 , 91 , 92 ]. These models are rapidly narrowing the performance gap with proprietary counterparts, while also promoting a more transparent, reproducible, and independent research ecosystem. Furthermore, the development of streamlined versions, such as ollama, optimized for local computing resources, highlights this trend by enhancing the practicality and accessibility of LLMs for a broader range of research applications 1 . Transparency of methods Finally, while a minority of studies present end-user applications, a notable portion leverages openly available datasets or share their proprietary datasets. Despite this positive trend, only around one third of these studies includes references to their code repositories, a critical factor for ensuring the validity, reproducibility, and generalizability of their methods [ 93 , 94 ]. While guidelines for reporting experimental setups and results exist the NLP and machine learning fields, there seems to be a need for further approaches to improve the reporting rates [ 95 , 96 , 97 ]. Moreover, the use of API-based models such as GPT introduces additional challenges for research transparency and reproducibility [ 98 ]. Shifting towards open-source models can address these issues by providing full access to the models’ codebase, fostering an environment where research findings can be easily replicated, scrutinized, and extended by the wider scientific community. LIMITATIONS First, the current literature review is primarily centered on text-based applications of LLMs in biomedicine which stands in contrast to the multimodal data of biomedical information. However, we believe that textual data is still an underused data type to inform the biomedical field. Second, we omitted articles published in languages other than English, which might restrict the scope of this review. CONCLUSION This scoping review provides an overview how LLMs are used for NLP tasks in biomedicine and health sciences. The findings from our scoping review underline the rapid progress of LLMs, emphasizing their potential in accelerating discovery and enhancing health outcomes. These advancements signal a promising avenue for leveraging LLMs in biomedicine and health, given their capacity to process and analyze complex biomedical texts with high proficiency. However, we also acknowledges the inherent risks and challenges associated with the deployment of LLMs such as ChatGPT in these sensitive areas, including evidence synthesis. Issues such as the generation of fabricated information and concerns over legal and privacy implications pose significant hurdles to the safe and ethical application of these technologies. These challenges highlight the need for careful consideration and management to mitigate risks associated with the use of LLMs in biomedicine and health. DECLARATIONS Acknowledgments We thank Alisa Berger for conducting the comprehensive literature search. Data and code availability statement All data and code that support the findings of this study are available in a public GitHub repository 1 . For any questions regarding the data, meta-data, or analysis code, contact the corresponding author, SED. Funding Swiss National Science Foundation (No. 407940-206504, to BVI) UZH Digital Entrepreneur Fellowship (No number, to BVI). The sponsors had no role in the design and conduct of the study; collection, management, analysis, and interpretation of the data; preparation, review, or approval of the manuscript; and decision to submit the manuscript for publication. Competing interests The authors declare no conflicts of interest related to this study. Author contributions Conception and design of study: SED, BS, BVI; Acquisition of data: SED, SQ, BVI; Analysis of data: SED; Interpretation of data: SED, SQ, BSn TE, J-PG, GS, BVI; Drafting the initial manuscript: SED; All authors critically revised the paper draft. APPENDIX Search string Supplementary search string (‘data mining’/exp OR ((data NEXT/1 mining):ti,ab,kw)) AND (literature:ti,ab OR abstract:ti,ab OR abstracts:ti,ab OR text:ti,ab OR articles:ti,ab) OR (((analyz* OR analys* OR extract* OR screen* OR evaluat* OR classif* OR ‘natural language processing’) NEAR/3 (literature OR abstract OR abstracts OR text OR articles)):ti,ab,kw) OR ((text NEXT/1 mining):ti,ab,kw) AND ‘natural language processing’/exp OR ‘artificial neural network’/exp OR ‘support vector machine’/exp OR ‘machine learning’/de OR ‘automated pattern recognition’/exp OR ‘artificial intelligence’/de OR ‘semi automation’:ti,ab,kw OR automation:ti,ab,kw OR ‘artificial intelligence’:ti,ab,kw OR ai:ti,ab,kw OR ‘natural language processing’:ti,ab,kw OR ((machine NEXT/1 (intelligence OR learning)):ti,ab,kw) OR (((‘text mining’ OR ‘data-mining’) NEXT/3 (tool* OR technique* OR system)):ti,ab,kw) OR (((deep OR comput* OR model* OR convolutional OR artificial OR algorithmic OR connectionist OR mathematical) NEXT/1 ‘neural network*’):ti,ab,kw) OR ((ann NEXT/3 (analysis OR approach OR method* OR model* OR output OR technique* OR training*)):ti,ab,kw) OR ((connectionist NEXT/1 (model OR network)):ti,ab,kw) OR ((‘support vector’ NEXT/1 (machine* OR classif* OR network OR regression)):ti,ab,kw) OR ((automated NEXT/3 (tool* OR technique* OR system OR ‘pattern recognition’ OR analyz* OR analys* OR extract* OR screen* OR evaluat* OR classif*)):ti,ab,kw) Flow diagram of the excluded and included studies Figure 9 illustrates the process of the scoping review, showcasing the flow of information through the different phases of a literature review, including identification, screening, eligibility, and inclusion of studies. Download figure Open in new tab Figure 9. PRISMA Flow Diagram [ 99 ]. Target Biomedical Applications Subdomains Further details about downstream applications within the top 4 most represented biomedical domains are provided in Figure 10 . These subcategories have been identified based on the objectives outlined in the reviewed papers. Download figure Open in new tab Figure 10. Number of articles assigned to each subdomain. Number of tested models per paper over time The mean number of LLMs used by individual papers ranges between 1.36 and 2.39, with a slight upward trend over time ( Figure 11 ). Download figure Open in new tab Figure 11. Average number of different models used per paper each year. High-level LLM Background The GPT-like and BERT-like models represent two distinct LLM approaches, differentiated by their architecture, training methods, and use cases [ 100 ]. BERT-style Language Models BERT employs a bidirectional framework, analyzing text in both directions simultaneously, which is enabled by its use of the Transformer encoder architecture. This bidirectionality allows BERT to understand the context of a word based on its entire surrounding text, making it good at tasks that require a deep understanding of language context, such as sentiment analysis, question answering, and named entity recognition. BERT’s pre-training involves masked language modeling and next sentence prediction, tasks that help it learn a comprehensive understanding of language structure and flow. While most reported models follow a BERT-based architecture, they differ in the pretraining corpus used. The standard BERT [ 101 ] model is pretrained on texts from Wikipedia and BookCorpus, which is considered generaldomain. To improve its performance in BioNLP tasks, the models can be trained on biomedical text corpus in two ways [ 102 ]: Mixed-Domain Pretraining: Starting with a general-domain BERT model training is continued using biomedical texts. In the case of BioBERT [ 103 ] this includes PubMed abstracts and PubMed Central (PMC) full-text articles. BlueBERT [ 104 ] uses both PubMed abstracts and de-identified clinical notes from MIMIC-III [ 52 ]. Domain-Specific Pretraining: In this setup the language model is trained using purely in-domain data. PubMedBERT [ 102 ] follows this approach and is pretrained from scratch on abstracts and full-texts from PubMed only. SciBERT [ 105 ] is also trained from scratch on a purely scientific corpus from Semanitic Scholar. However its pretraining corpus is a mixture of biomedical and computer science texts. An overview of the most frequently used BERT-based models in the reviewed literature is given in Table 3 . View this table: View inline View popup Download powerpoint Table 3. Training corpora for common BERT-based models. GPT-style Language Models GPT-like models operate on a unidirectional framework, processing text from left to right and utilizing the Transformer decoder architecture, making it well fit for generative tasks such as text completion and content creation. Its training is focused on predicting the next word in a sequence, optimizing the model for generating coherent and contextually relevant text. While BERT-like architectures typically require task-specific fine-tuning to achieve optimal performance, models like GPT-3 demonstrate few-shot and zero-shot learning capabilities. Few-shot learning refers to the model’s ability to perform tasks with a very limited amount of training data, while zero-shot learning refers to its ability to perform tasks without any task-specific training data. This is achieved through advanced prompting techniques and in-context learning, where the model generates responses based on the context provided in the prompt [ 106 ]. Footnotes https://github.com/Ineichen-Group/LiteratureReview-LLM-Biomedicine ↵ 1 https://harzing.com/resources/publish-or-perish ↵ 3 https://portal.dbmi.hms.harvard.edu/projects/n2c2-nlp/ ↵ 3 https://ollama.com/ ↵ 4 https://github.com/Ineichen-Group/LiteratureReview-LLM-Biomedicine/tree/main REFERENCES [1]. ↵ Binggui Zhou , Guanghua Yang , Zheng Shi , and Shaodan Ma . Natural language processing for smart healthcare . IEEE Reviews in Biomedical Engineering , 2022 . [2]. ↵ Ashish Vaswani , Noam Shazeer , Niki Parmar , Jakob Uszkoreit , Llion Jones , Aidan N Gomez , Łukasz Kaiser , and Illia Polosukhin . Attention is all you need . Advances in neural information processing systems , 30 , 2017 . [3]. ↵ Benyou Wang , Qianqian Xie , Jiahuan Pei , Zhihong Chen , Prayag Tiwari , Zhao Li , and Jie Fu . Pre-trained language models in biomedical domain: A systematic survey . ACM Computing Surveys , 56 ( 3 ): 1 – 52 , 2023 . OpenUrl [4]. ↵ Alec Radford , Karthik Narasimhan , Tim Salimans , Ilya Sutskever , et al. Improving language understanding by generative pre-training , 2018 . Preprint OpenAI . [5]. ↵ Tom Brown , Benjamin Mann , Nick Ryder , Melanie Subbiah , Jared D Kaplan , Prafulla Dhariwal , Arvind Neelakantan , Pranav Shyam , Girish Sastry , Amanda Askell , et al. Language models are few-shot learners . Advances in neural information processing systems , 33 : 1877 – 1901 , 2020 . OpenUrl [6]. ↵ Qingxiu Dong , Lei Li , Damai Dai , Ce Zheng , Zhiyong Wu , Baobao Chang , Xu Sun , Jingjing Xu , and Zhifang Sui . A survey on in-context learning . arXiv preprint arXiv: 2301.00234 , 2022 . [7]. ↵ Arun James Thirunavukarasu , Darren Shu Jeng Ting , Kabilan Elangovan , Laura Gutierrez , Ting Fang Tan , and Daniel Shu Wei Ting . Large language models in medicine . Nature medicine , 29 ( 8 ): 1930 – 1940 , 2023 . OpenUrl [8]. ↵ Hongbin Ye , Tong Liu , Aijia Zhang , Wei Hua , and Weiqiang Jia . Cognitive mirage: A review of hallucinations in large language models . arXiv preprint arXiv: 2309.06794 , 2023 . [9]. ↵ Saskia Locke , Anthony Bashall , Sarah Al-Adely , John Moore , Anthony Wilson , and Gareth B Kitchen . Natural language processing in medicine: a review . Trends in Anaesthesia and Critical Care , 38 : 4 – 9 , 2021 . OpenUrl CrossRef [10]. ↵ Katikapalli Subramanyam Kalyan , Ajit Rajasekharan , and Sivanesan Sangeetha . AMMU: a survey of transformer-based biomedical pretrained language models . Journal of biomedical informatics , 126 : 103982 , 2022 . OpenUrl CrossRef [11]. ↵ Eugenio Cesario , Carmela Comito , and Ester Zumpano . A survey of the recent trends in deep learning for literature based discovery in the biomedical domain . Neurocomputing , 568 : 127079 , 2024 . OpenUrl [12]. ↵ Anastassia Shaitarova , Jamil Zaghir , Alberto Lavelli , Michael Krauthammer , and Fabio Rinaldi . Exploring the latest highlights in medical natural language processing across multiple languages: A survey . Yearbook of medical informatics , 32 ( 01 ): 230 – 243 , 2023 . OpenUrl [13]. ↵ Kai He , Rui Mao , Qika Lin , Yucheng Ruan , Xiang Lan , Mengling Feng , and Erik Cambria . A survey of large language models for healthcare: from data, technology, and applications to accountability and ethics . arXiv preprint arXiv: 2310.05694 , 2023 . [14]. ↵ Malik Sallam . ChatGPT utility in healthcare education, research, and practice: systematic review on the promising perspectives and valid concerns . In Healthcare , volume 11 ( 6 ), page 887 . MDPI , 2023 . OpenUrl [15]. ↵ Rui Yang , Ting Fang Tan , Wei Lu , Arun James Thirunavukarasu , Daniel Shu Wei Ting , and Nan Liu . Large language models in health care: Development, applications, and challenges . Health Care Science , 2 ( 4 ): 255 – 263 , 2023 . OpenUrl [16]. ↵ Rens Van De Schoot , Jonathan De Bruin , Raoul Schram , Parisa Zahedi , Jan De Boer , Felix Weijdema , Bianca Kramer , Martijn Huijts , Maarten Hoogerwerf , Gerbrich Ferdinands , et al. An open source machine learning framework for efficient and transparent systematic reviews . Nature machine intelligence , 3 ( 2 ): 125 – 133 , 2021 . OpenUrl [17]. J Mantas et al. The classification of short scientific texts using pretrained BERT model . Public Health and Informatics: Proceedings of MIE 2021 , 281 : 83 , 2021 . OpenUrl [18]. Sungmin Aum and Seon Choe . srBERT: automatic article classification model for systematic review using bert . Systematic reviews , 10 ( 1 ): 1 – 8 , 2021 . OpenUrl [19]. Ashwin Karthik Ambalavanan and Murthy V Devarakonda . Using the contextual language model BERT for multi-criteria classification of scientific articles . Journal of biomedical informatics , 112 : 103578 , 2020 . OpenUrl [20]. Philippe C Habets , David GP van IJzendoorn , Christiaan H Vinkers , Linda Härmark , Loes C de Vries , and Willem M Otte . Development and validation of a machine-learning algorithm to predict the relevance of scientific articles within the field of teratology . Reproductive Toxicology , 113 : 150 – 154 , 2022 . OpenUrl [21]. Antonio Jose Jimeno Yepes and Karin Verspoor . Classifying literature mentions of biological pathogens as experimentally studied using natural language processing . Journal of Biomedical Semantics , 14 ( 1 ): 1 , 2023 . OpenUrl [22]. ↵ Sara Rosenthal , Ken Barker , and Zhicheng Liang . Leveraging medical literature for section prediction in electronic health records . In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP) , pages 4864 – 4873 , 2019 . [23]. ↵ Vincent Martenot , Valentin Masdeu , Jean Cupe , Faustine Gehin , Margot Blanchon , Julien Dauriat , Alexander Horst , Michael Renaudin , Philippe Girard , and Jean-Daniel Zucker . Lisa: an assisted literature search pipeline for detecting serious adverse drug events with deep learning . BMC medical informatics and decision making , 22 ( 1 ): 1 – 16 , 2022 . OpenUrl [24]. ↵ Özge Kart , Alexandre Mestiashvili , Kurt Lachmann , Richard Kwasnicki , and Michael Schroeder . Emati: a recommender system for biomedical literature based on supervised learning . Database , 2022 : baac104 , 2022 . OpenUrl [25]. ↵ Peng-Hsuan Li , Ting-Fu Chen , Jheng-Ying Yu , Shang-Hung Shih , Chan-Hung Su , Yin-Hung Lin , Huai-Kuang Tsai , Hsueh-Fen Juan , Chien-Yu Chen , and Jia-Hsin Huang . pubmedkb: an interactive web server for exploring biomedical entity relations in the biomedical literature . Nucleic Acids Research , 50 ( W1 ): W616 – W622 , 2022 . OpenUrl [26]. ↵ Renqian Luo , Liai Sun , Yingce Xia , Tao Qin , Sheng Zhang , Hoifung Poon , and Tie-Yan Liu . BioGPT: generative pre-trained transformer for biomedical text generation and mining . Briefings in Bioinformatics , 23 ( 6 ): bbac409 , 2022 . OpenUrl [27]. ↵ Yiqi Tong , Yidong Chen , and Xiaodong Shi . A multi-task approach for improving biomedical named entity recognition by incorporating multi-granularity information . In Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021 , pages 4804 – 4813 , 2021 . [28]. ↵ Qiuhao Lu , Dejing Dou , and Thien Huu Nguyen . Parameter-efficient domain knowledge integration from multiple sources for biomedical pre-trained language models . In Findings of the Association for Computational Linguistics: EMNLP 2021 , pages 3855 – 3865 , 2021 . [29]. ↵ Masaki Asada , Makoto Miwa , and Yutaka Sasaki . Integrating heterogeneous knowledge graphs into drug–drug interaction extraction from the literature . Bioinformatics , 39 ( 1 ): btac754 , 2023 . OpenUrl CrossRef [30]. ↵ Hong Guan and Murthy Devarakonda . Leveraging contextual information in extracting long distance relations from clinical notes . In AMIA Annual Symposium Proceedings , volume 2019 , page 1051 . American Medical Informatics Association , 2019 . OpenUrl [31]. ↵ Sajid Hussain , Hammad Afzal , Ramsha Saeed , Naima Iltaf , and Mir Yasir Umair . Pharmacovigilance with transformers: A framework to detect adverse drug reactions using BERT fine-tuned with farm . Computational and Mathematical Methods in Medicine , 2021 , 2021 . [32]. ↵ Ying Zhang , Baohang Zhou , Kehui Song , Xuhui Sui , Guoqing Zhao , Ning Jiang , and Xiaojie Yuan . PM2F2N: Patient multi-view multi-modal feature fusion networks for clinical outcome prediction . In Findings of the Association for Computational Linguistics: EMNLP 2022 , pages 1985 – 1994 , 2022 . [33]. ↵ Iman Deznabi , Mohit Iyyer , and Madalina Fiterau . Predicting in-hospital mortality by combining clinical notes with time-series data . In Findings of the association for computational linguistics: ACL-IJCNLP 2021 , pages 4026 – 4031 , 2021 . [34]. ↵ Junwen Duan , Fangyuan Wei , Jin Liu , Hongdong Li , Tianming Liu , and Jianxin Wang . CDA: A contrastive data augmentation method for alzheimer’s disease detection . In Findings of the Association for Computational Linguistics: ACL 2023 , pages 1819 – 1826 , 2023 . [35]. ↵ Ankit Aich , Avery Quynh , Varsha Badal , Amy Pinkham , Philip Harvey , Colin Depp , and Natalie Parde . Towards intelligent clinically-informed language analyses of people with bipolar disorder and schizophrenia . In Findings of the Association for Computational Linguistics: EMNLP 2022 , pages 2871 – 2887 , 2022 . [36]. ↵ Sourabh Zanwar , Xiaofei Li , Daniel Wiechmann , Yu Qiao , and Elma Kerz . What to fuse and how to fuse: Exploring emotion and personality fusion strategies for explainable mental disorder detection . In Findings of the Association for Computational Linguistics: ACL 2023 , pages 8926 – 8940 , 2023 . [37]. ↵ Ramit Sawhney , Atula Tejaswi Neerkaje , and Manas Gaur . A risk-averse mechanism for suicidality assessment on social media . Association for Computational Linguistics 2022 (ACL 2022) , 2022 . [38]. ↵ Ramit Sawhney , Harshit Joshi , Saumya Gandhi , and Rajiv Shah . A time-aware transformer based model for suicide ideation detection on social media . In Proceedings of the 2020 conference on empirical methods in natural language processing (EMNLP) , pages 7685 – 7697 , 2020 . [39]. ↵ Tiberiu Sosea and Cornelia Caragea . Canceremo: A dataset for fine-grained emotion detection . In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP) , pages 8892 – 8904 , 2020 . [40]. ↵ Tamanna Hossain , Robert L Logan Iv , Arjuna Ugarte , Yoshitomo Matsubara , Sean Young , and Sameer Singh . Covidlies: Detecting covid-19 misinformation on social media . In Proceedings of the 1st Workshop on NLP for COVID-19 (Part 2) at EMNLP 2020 , 2020 . [41]. ↵ Zhenghao Liu , Chenyan Xiong , Zhuyun Dai , Si Sun , Maosong Sun , and Zhiyuan Liu . Adapting open domain fact extraction and verification to COVID-FACT through in-domain language modeling . In Findings of the Association for Computational Linguistics: EMNLP 2020 , pages 2395 – 2400 , 2020 . [42]. Rezarta Islamaj Doğan , Robert Leaman , and Zhiyong Lu . NCBI disease corpus: a resource for disease name recognition and concept normalization . Journal of biomedical informatics , 47 : 1 – 10 , 2014 . OpenUrl CrossRef PubMed [43]. Jiao Li , Yueping Sun , Robin J Johnson , Daniela Sciaky , Chih-Hsuan Wei , Robert Leaman , Allan Peter Davis , Carolyn J Mattingly , Thomas C Wiegers , and Zhiyong Lu . BioCreative V CDR task corpus: a resource for chemical disease relation extraction . Database , 2016 , 2016 . [44]. Larry Smith , Lorraine K Tanabe , Rie Johnson nee Ando , Cheng-Ju Kuo , I-Fang Chung , Chun-Nan Hsu , Yu-Shi Lin , Roman Klinger , Christoph M Friedrich , Kuzman Ganchev , et al. Overview of BioCreative II gene mention recognition . Genome biology , 9 : 1 – 19 , 2008 . OpenUrl CrossRef [45]. Nigel Collier , Tomoko Ohta , Yoshimasa Tsuruoka , Yuka Tateisi , and Jin-Dong Kim . Introduction to the bio-entity recognition task at JNLPBA . In Proceedings of the International Joint Workshop on Natural Language Processing in Biomedicine and its Applications (NLPBA/BioNLP) , pages 73 – 78 , 2004 . [46]. Martin Krallinger , Florian Leitner , Obdulia Rabal , Miguel Vazquez , Julen Oyarzabal , and Alfonso Valencia . Chemdner: The drugs and chemical names extraction challenge . Journal of cheminformatics , 7 : 1 – 11 , 2015 . OpenUrl [47]. Martin Gerner , Goran Nenadic , and Casey M Bergman . Linnaeus: a species name identification system for biomedical literature . BMC bioinformatics , 11 : 1 – 17 , 2010 . OpenUrl CrossRef PubMed [48]. Evangelos Pafilis , Sune P Frankild , Lucia Fanini , Sarah Faulwetter , Christina Pavloudi , Aikaterini Vasileiadou , Christos Arvanitidis , and Lars Juhl Jensen . The species and organisms resources for fast and accurate identification of taxonomic names in text . PloS one , 8 ( 6 ): e65390 , 2013 . OpenUrl CrossRef PubMed [49]. Martin Krallinger , Obdulia Rabal , Saber A Akhondi , Martin Pérez Pérez , Jesús Santamaría , Gael Pérez Rodríguez , Georgios Tsatsaronis , Ander Intxaurrondo , José Antonio López , Umesh Nandal , et al. Overview of the BioCreative VI chemical-protein interaction track . In Proceedings of the sixth BioCreative challenge evaluation workshop , volume 1 , pages 141 – 146 , 2017 . OpenUrl [50]. María Herrero-Zazo , Isabel Segura-Bedmar , Paloma Martínez , and Thierry Declerck . The ddi corpus: An annotated corpus with pharmacological substances and drug–drug interactions . Journal of biomedical informatics , 46 ( 5 ): 914 – 920 , 2013 . OpenUrl CrossRef PubMed [51]. Anastasios Nentidis , Konstantinos Bougiatiotis , Anastasia Krithara , and Georgios Paliouras . Results of the seventh edition of the bioasq challenge . In Machine Learning and Knowledge Discovery in Databases: International Workshops of ECML PKDD 2019, Würzburg, Germany, September 16–20, 2019, Proceedings, Part II , pages 553 – 568 . Springer , 2020 . [52]. ↵ Alistair EW Johnson , Tom J Pollard , Lu Shen , Li-wei H Lehman , Mengling Feng , Mohammad Ghassemi , Benjamin Moody , Peter Szolovits , Leo Anthony Celi , and Roger G Mark . MIMIC-III, a freely accessible critical care database . Scientific data , 3 ( 1 ): 1 – 9 , 2016 . OpenUrl [53]. Erik M Van Mulligen , Annie Fourrier-Reglat , David Gurwitz , Mariam Molokhia , Ainhoa Nieto , Gianluca Trifiro , Jan A Kors , and Laura I Furlong . The EU-ADR corpus: annotated drugs, diseases, targets, and their relationships . Journal of biomedical informatics , 45 ( 5 ): 879 – 884 , 2012 . OpenUrl CrossRef PubMed [54]. Özlem Uzuner , Brett R South , Shuying Shen , and Scott L DuVall . 2010 i2b2/VA challenge on concepts, assertions, and relations in clinical text . Journal of the American Medical Informatics Association , 18 ( 5 ): 552 – 556 , 2011 . OpenUrl CrossRef PubMed [55]. ↵ Tao Chen , Mingfen Wu , and Hexi Li . A general approach for improving deep learning-based medical relation extraction using a pre-trained model and fine-tuning . Database , 2019 : baz116 , 2019 . OpenUrl [56]. ↵ Buzhou Tang , Dehuan Jiang , Qingcai Chen , Xiaolong Wang , Jun Yan , and Ying Shen . De-identification of clinical text via Bi-LSTM-CRF with neural language models . In AMIA Annual Symposium Proceedings , volume 2019 , page 857 . American Medical Informatics Association , 2019 . OpenUrl [57]. ↵ Diwakar Mahajan , Ananya Poddar , Jennifer J Liang , Yen-Ting Lin , John M Prager , Parthasarathy Surya-narayanan , Preethi Raghavan , Ching-Huei Tsou , et al. Identification of semantically similar sentences in clinical notes: Iterative intermediate training using multi-task learning . JMIR medical informatics , 8 ( 11 ): e22508 , 2020 . OpenUrl [58]. ↵ Haoyu Wang , Fenglong Ma , Yaqing Wang , and Jing Gao . Knowledge-guided paraphrase identification . In Findings of the Association for Computational Linguistics: EMNLP 2021 , pages 843 – 853 , 2021 . [59]. ↵ Ying Xiong , Xin Yang , Linjing Liu , Ka-Chun Wong , Qingcai Chen , Yang Xiang , and Buzhou Tang . EARA: Improving Biomedical Semantic Textual Similarity with Entity-Aligned Attention and Retrieval Augmentation . In The 2023 Conference on Empirical Methods in Natural Language Processing , 2023 . [60]. ↵ Aokun Chen , Zehao Yu , Xi Yang , Yi Guo , Jiang Bian , and Yonghui Wu . Contextualized medication information extraction using transformer-based deep learning architectures . Journal of Biomedical Informatics , 142 : 104370 , 2023 . OpenUrl [61]. ↵ Sheng-Jie Lin , Wen-Chao Yeh , Yu-Wen Chiu , Yung-Chun Chang , Min-Huei Hsu , Yi-Shin Chen , and Wen-Lian Hsu . A BERT-based ensemble learning approach for the BioCreative VII challenges: full-text chemical identification and multi-label classification in pubmed articles . Database , 2022 : baac056 , 2022 . OpenUrl [62]. ↵ Gollam Rabby and Petr Berka . Multi-class classification of COVID-19 documents using machine learning algorithms . Journal of Intelligent Information Systems , 60 ( 2 ): 571 – 591 , 2023 . OpenUrl [63]. ↵ Jehad Aldahdooh , Ziaurrehman Tanoli , and Jing Tang . Mining drug-target interactions from biomedical literature using chemical and gene descriptions based ensemble transformer model . bioRxiv , pages 2023 – 07 , 2023 . [64]. ↵ Mourad Sarrouti , Asma Ben Abacha , Yassine M’rabet , and Dina Demner-Fushman . Evidence-based factchecking of health-related claims . In Findings of the Association for Computational Linguistics: EMNLP 2021 , pages 3499 – 3512 , 2021 . [65]. ↵ Tiago Almeida and Sérgio Matos . Frugal neural reranking: evaluation on the covid-19 literature . In Workshop on NLP for COVID-19 (Part 2) at EMNLP 2020 , 2020 . [66]. ↵ Chyrine Tahri , Aurore Bochnakian , Patrick Haouat , and Xavier Tannier . Transitioning from benchmarks to a real-world case of information-seeking in scientific publications . In Findings of the Association for Computational Linguistics: ACL 2023 , pages 1066 – 1076 . Association for Computational Linguistics , 2023 . [67]. ↵ Arantxa Otegi , Jon Ander Campos , Gorka Azkune , Aitor Soroa , and Eneko Agirre . Automatic evaluation vs . user preference in neural textual questionanswering over COVID-19 scientific literature. In Workshop on NLP for COVID-19 (Part 2) at EMNLP 2020 , 2020 . [68]. ↵ Zicheng Zhang , Xinyue Lin , and Shanshan Wu . A hybrid algorithm for clinical decision support in precision medicine based on machine learning . BMC bioinformatics , 24 ( 1 ): 1 – 18 , 2023 . OpenUrl CrossRef [69]. ↵ Yufei Li , Xiangyu Zhou , Jie Ma , Xiaoyong Ma , Pengzhen Cheng , Tieliang Gong , and Chen Li . Distinguished representation of identical mentions in bio-entity coreference resolution . BMC Medical Informatics and Decision Making , 22 ( 1 ): 1 – 12 , 2022 . OpenUrl [70]. ↵ Qingkai Zeng , Wenhao Yu , Mengxia Yu , Tianwen Jiang , Tim Weninger , and Meng Jiang . Tri-Train: Automatic Pre-Fine Tuning between Pre-Training and Fine-Tuning for SciNER . In Findings of the Association for Computational Linguistics: EMNLP 2020 , pages 4778 – 4787 , 2020 . [71]. ↵ Guowei Li , Luciana PF Abbade , Ikunna Nwosu , Yanling Jin , Alvin Leenus , Muhammad Maaz , Mei Wang , Meha Bhatt , Laura Zielinski , Nitika Sanger , et al. A scoping review of comparisons between abstracts and full reports in primary biomedical research . BMC medical research methodology , 17 : 1 – 12 , 2017 . OpenUrl [72]. ↵ K Bretonnel Cohen , Helen L Johnson , Karin Verspoor , Christophe Roeder , and Lawrence E Hunter . The structural and content aspects of abstracts versus bodies of full text journal articles are different . BMC bioinformatics , 11 : 1 – 10 , 2010 . OpenUrl CrossRef PubMed [73]. ↵ Yu Gu , Robert Tinn , Hao Cheng , Michael Lucas , Naoto Usuyama , Xiaodong Liu , Tristan Naumann , Jianfeng Gao , and Hoifung Poon . Domain-specific language model pretraining for biomedical natural language processing . ACM Transactions on Computing for Healthcare (HEALTH) , 3 ( 1 ): 1 – 23 , 2021 . OpenUrl [74]. Zongcheng Ji , Qiang Wei , and Hua Xu . Bert-based ranking for biomedical entity normalization . AMIA Summits on Translational Science Proceedings , 2020 : 269 , 2020 . OpenUrl [75]. Timofey V Ivanisenko , Pavel S Demenkov , Nikolay A Kolchanov , and Vladimir A Ivanisenko . The new version of the ANDDigest tool with improved ai-based short names recognition . International Journal of Molecular Sciences , 23 ( 23 ): 14934 , 2022 . OpenUrl [76]. Usman Naseem , Adam G Dunn , Matloob Khushi , and Jinman Kim . Benchmarking for biomedical natural language processing tasks with a domain specific albert . BMC bioinformatics , 23 ( 1 ): 1 – 15 , 2022 . OpenUrl CrossRef [77]. Lutz Bornmann and Rüdiger Mutz . Growth rates of modern science: A bibliometric analysis based on the number of publications and cited references . Journal of the association for information science and technology , 66 ( 11 ): 2215 – 2222 , 2015 . OpenUrl [78]. Benjamin V Ineichen , Marianna Rosso , and Malcolm R Macleod . From data deluge to publomics: How AI can transform animal research . Lab animal , 52 ( 10 ): 213 – 214 , 2023 . OpenUrl [79]. ↵ Qusai Khraisha , Sophie Put , Johanna Kappenberg , Azza Warraitch , and Kristin Hadfield . Can large language models replace humans in the systematic review process? evaluating GPT-4’s efficacy in screening and extracting data from peer-reviewed and grey literature in multiple languages . arXiv preprint arXiv: 2310.17526 , 2023 . [80]. ↵ Liyan Tang , Zhaoyi Sun , Betina Idnay , Jordan G Nestor , Ali Soroush , Pierre A Elias , Ziyang Xu , Ying Ding , Greg Durrett , Justin F Rousseau , et al. Evaluating large language models on medical evidence summarization . npj Digital Medicine , 6 ( 1 ): 158 , 2023 . OpenUrl [81]. Ruixi Lin and Hwee Tou Ng . Mind the biases: Quantifying cognitive biases in language model prompting . In Findings of the Association for Computational Linguistics: ACL 2023 , pages 5269 – 5281 , 2023 . [82]. Boxin Wang , Weixin Chen , Hengzhi Pei , Chulin Xie , Mintong Kang , Chenhui Zhang , Chejian Xu , Zidi Xiong , Ritik Dutta , Rylan Schaeffer , et al. DecodingTrust: A comprehensive assessment of trustworthiness in gpt models . arXiv preprint arXiv: 2306.11698 , 2023 . [83]. Shubo Tian , Qiao Jin , Lana Yeganova , Po-Ting Lai , Qingqing Zhu , Xiuying Chen , Yifan Yang , Qingyu Chen , Won Kim , Donald C Comeau , et al. Opportunities and challenges for ChatGPT and large language models in biomedicine and health . Briefings in Bioinformatics , 25 ( 1 ): bbad493 , 2024 . OpenUrl [84]. ↵ Braja Gopal Patra , Zhaoyi Sun , Zilin Cheng , Rochelle Joly , Jyotishman Pathak , and Yifan Peng . Automated classification of lay health articles using natural language processing: a case study on pregnancy health and postpartum depression . Frontiers in Psychiatry , 14 : 1258887 , 2023 . OpenUrl [85]. ↵ Ihor Stepanov , Arsentii Ivasiuk , Oleksandr Yavorskyi , and Alina Frolova . Comparative analysis of classification techniques for topic-based biomedical literature categorisation . Frontiers in Genetics , 14 : 1238140 , 2023 . OpenUrl [86]. ↵ Nikitha Karkera , Sathwik Acharya , and Sucheendra K Palaniappan . Leveraging pre-trained language models for mining microbiome-disease relationships . BMC bioinformatics , 24 ( 1 ): 290 , 2023 . OpenUrl [87]. ↵ Bosung Kim and Ndapandula Nakashole . SYMPTOMIFY: Transforming symptom annotations with language model knowledge harvesting . In Findings of the Association for Computational Linguistics: EMNLP 2023 , pages 11667 – 11681 , 2023 . [88]. ↵ Jinge Wu , Daqian Shi , Abul Hasan , and Honghan Wu . KnowLab at RadSum23: comparing pre-trained language models in radiology report summarization . In Proceedings of the Annual Meeting of the Association for Computational Linguistics , pages 535 – 540 . ACL , 2023 . [89]. ↵ Changyu Wang , Siru Liu , Aiqing Li , and Jialin Liu . Text dialogue analysis based ChatGPT for primary screening of mild cognitive impairment . medRxiv , pages 2023 – 06 , 2023 . [90]. ↵ Hugo Touvron , Louis Martin , Kevin Stone , Peter Albert , Amjad Almahairi , Yasmine Babaei , Nikolay Bashlykov , Soumya Batra , Prajjwal Bhargava , Shruti Bhosale , et al. Llama 2: Open foundation and fine-tuned chat models . arXiv preprint arXiv: 2307.09288 , 2023 . [91]. ↵ Zeming Chen , Alejandro Hernández Cano , Angelika Romanou , Antoine Bonnet , Kyle Matoba , Francesco Salvi , Matteo Pagliardini , Simin Fan , Andreas Köpf , Amirkeivan Mohtashami , et al. Meditron-70b: Scaling medical pretraining for large language models . arXiv preprint arXiv: 2311.16079 , 2023 . [92]. ↵ Yanis Labrak , Adrien Bazoge , Emmanuel Morin , Pierre-Antoine Gourraud , Mickael Rouvier , and Richard Dufour . BioMistral: A collection of open-source pretrained large language models for medical domains . arXiv preprint arXiv: 2402.10373 , 2024 . [93]. ↵ Ben Goldacre , Caroline E Morton , and Nicholas J DeVito . Why researchers should share their analytic code , 2019 . [94]. ↵ Jihyun Kim , Soon Kim , Hye-Min Cho , Jae Hwa Chang , and Soo Young Kim . Data sharing policies of journals in life, health, and physical sciences indexed in journal citation reports . PeerJ , 8 : e9924 , 2020 . OpenUrl CrossRef PubMed [95]. ↵ Jesse Dodge , Suchin Gururangan , Dallas Card , Roy Schwartz , and Noah A Smith . Show your work: Improved reporting of experimental results . arXiv preprint arXiv: 1909.03004 , 2019 . [96]. ↵ Sayash Kapoor , Emily Cantrell , Kenny Peng , Thanh Hien Pham , Christopher A Bail , Odd Erik Gundersen , Jake M Hofman , Jessica Hullman , Michael A Lones , Momin M Malik , et al. Reforms: Reporting standards for machine learning based science . arXiv preprint arXiv: 2308.07832 , 2023 . [97]. ↵ Ian Magnusson , Noah A Smith , and Jesse Dodge . Reproducibility in nlp: What have we learned from the checklist? arXiv preprint arXiv: 2306.09562 , 2023 . [98]. ↵ Andreas Liesenfeld , Alianda Lopez , and Mark Dingemanse . Opening up ChatGPT: Tracking openness, transparency, and accountability in instruction-tuned text generators . In Proceedings of the 5th international conference on conversational user interfaces , pages 1 – 6 , 2023 . [99]. ↵ Matthew J Page , Joanne E McKenzie , Patrick M Bossuyt , Isabelle Boutron , Tammy C Hoffmann , Cynthia D Mulrow , Larissa Shamseer , Jennifer M Tetzlaff , and David Moher . The PRISMA 2020 statement: an updated guideline for reporting systematic reviews . BMJ (Clinical research ed .), 372 ( n71 ), 3 2021 . OpenUrl [100]. ↵ Jingfeng Yang , Hongye Jin , Ruixiang Tang , Xiaotian Han , Qizhang Feng , Haoming Jiang , Bing Yin , and Xia Hu . Harnessing the power of llms in practice: A survey on chatgpt and beyond . arXiv preprint arXiv: 2304.13712 , 2023 . [101]. ↵ Jacob Devlin , Ming-Wei Chang , Kenton Lee , and Kristina Toutanova . BERT: Pre-training of deep bidirectional transformers for language understanding . arXiv preprint arXiv: 1810.04805 , 2018 . [102]. ↵ Yu Gu , Robert Tinn , Hao Cheng , Michael Lucas , Naoto Usuyama , Xiaodong Liu , Tristan Naumann , Jianfeng Gao , and Hoifung Poon . Domain-specific language model pretraining for biomedical natural language processing . ACM Transactions on Computing for Healthcare (HEALTH) , 3 ( 1 ): 1 – 23 , 2021 . OpenUrl [103]. ↵ Jinhyuk Lee , Wonjin Yoon , Sungdong Kim , Donghyeon Kim , Sunkyu Kim , Chan Ho So , and Jaewoo Kang . BioBERT: a pre-trained biomedical language representation model for biomedical text mining . Bioinformatics , 36 ( 4 ): 1234 – 1240 , 2020 . OpenUrl CrossRef PubMed [104]. ↵ Yifan Peng , Shankai Yan , and Zhiyong Lu . Transfer Learning in Biomedical Natural Language Processing: An Evaluation of BERT and ELMo on Ten Benchmarking Datasets . In Proceedings of the 2019 Workshop on Biomedical Natural Language Processing (BioNLP 2019) , pages 58 – 65 , 2019 . [105]. ↵ Iz Beltagy , Kyle Lo , and Arman Cohan . SciBERT: A pretrained language model for scientific text . In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP) , pages 3615 – 3620 , Hong Kong, China , November 2019 . Association for Computational Linguistics . [106]. ↵ Min Zhang and Juntao Li . A commentary of GPT-3 in MIT technology review 2021 . Fundamental Research , 1 ( 6 ): 831 – 833 , 2021 . OpenUrl View the discussion thread. Back to top Previous Next Posted April 25, 2024. Download PDF Data/Code Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Large Language Models to process, analyze, and synthesize biomedical texts – a scoping review Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Large Language Models to process, analyze, and synthesize biomedical texts – a scoping review Simona Emilova Doneva , Sijing Qin , Beate Sick , Tilia Ellendorff , Jean-Philippe Goldman , Gerold Schneider , Benjamin Victor Ineichen bioRxiv 2024.04.19.588095; doi: https://doi.org/10.1101/2024.04.19.588095 Share This Article: Copy Citation Tools Large Language Models to process, analyze, and synthesize biomedical texts – a scoping review Simona Emilova Doneva , Sijing Qin , Beate Sick , Tilia Ellendorff , Jean-Philippe Goldman , Gerold Schneider , Benjamin Victor Ineichen bioRxiv 2024.04.19.588095; doi: https://doi.org/10.1101/2024.04.19.588095 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Bioinformatics Subject Areas All Articles Animal Behavior and Cognition (7644) Biochemistry (17728) Bioengineering (13916) Bioinformatics (42037) Biophysics (21489) Cancer Biology (18637) Cell Biology (25553) Clinical Trials (138) Developmental Biology (13401) Ecology (19941) Epidemiology (2067) Evolutionary Biology (24367) Genetics (15622) Genomics (22547) Immunology (17764) Microbiology (40475) Molecular Biology (17208) Neuroscience (88747) Paleontology (667) Pathology (2842) Pharmacology and Toxicology (4834) Physiology (7659) Plant Biology (15175) Scientific Communication and Education (2047) Synthetic Biology (4304) Systems Biology (9835) Zoology (2272)