Exploring the limits of LLMs in low-resource information extraction: Case study in brain MRI reports for Epilepsy

Exploring the limits of LLMs in low-resource information extraction: Case study in brain MRI reports for Epilepsy | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Exploring the limits of LLMs in low-resource information extraction: Case study in brain MRI reports for Epilepsy View ORCID Profile Thinh Hung Truong , View ORCID Profile Emma Foster , View ORCID Profile Timothy Fazio , View ORCID Profile Sarah Holper , View ORCID Profile Karin Verspoor doi: https://doi.org/10.1101/2025.08.02.25332570 Thinh Hung Truong 1 University of Melbourne , Melbourne, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Thinh Hung Truong Emma Foster 3 Royal Melbourne Hospital , Melbourne, Australia 4 Alfred Health , Melbourne, Australia 5 Monash University , Melbourne, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Emma Foster Timothy Fazio 1 University of Melbourne , Melbourne, Australia 3 Royal Melbourne Hospital , Melbourne, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Timothy Fazio Sarah Holper 3 Royal Melbourne Hospital , Melbourne, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Sarah Holper Karin Verspoor 2 RMIT University , Melbourne, Australia 1 University of Melbourne , Melbourne, Australia Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Karin Verspoor For correspondence: karin.verspoor{at}rmit.edu.au Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Information extraction (IE) from specialized clinical texts such as brain MRI reports is important for various clinical and population health contexts. However, this topic is under-explored due to privacy concerns limiting data availability and the inherent complexity and domain-specificity of clinical language. Common methods relying on substantial amounts of training data fail. The recent advances in large language model (LLM) research provide a promising solution to bridge the data scarcity gap, with improved ability to adapt to novel tasks with little supervision. We introduce a new, challenging dataset of 100 expert-annotated brain MRI reports, featuring 152 fine-grained entity types and 4 relation types, characterised by low inter-annotator agreement. This task reflects the inherent complexity and real-world ambiguity of medical text. We evaluate a small, open-weight LLM across span detection, named entity recognition, and relation extraction tasks. We compare few-shot prompting and parameter-efficient fine-tuning against specialized off-the-shelf biomedical IE systems. Our results demonstrate that both few-shot and fine-tuned LLM approaches substantially outperform off-the-shelf baselines. While LLMs show superiority, absolute performance, particularly for complex relations and fine-grained entities, remains modest, correlating with the dataset’s inherent difficulty and the extreme low-resource setting. 1. Introduction Information extraction (IE) is a classic Natural Language Processing (NLP) problem, involving extraction of structured knowledge from free-text documents. IE encompasses different tasks such as extracting relevant spans from texts and classifying them into meaningful semantic categories (named entity recognition, NER), and identifying relations between entities (relation extraction, RE). IE is a well-explored task with many established benchmarks and methods. 1 , 2 Despite significant progress, the most prevalent challenge for IE lies in its open-endedness: how to build a “universal” system capable of adapting to new domains, entity types, and relations with minimal effort. Most established methods for IE rely on a substantial amount of labeled training data to train supervised machine learning models. 3 , 4 This reliance on large datasets represents a significant bottleneck for many specialized domains, such as clinical medicine, where annotated data is scarce, expensive to create, and governed by strict privacy regulations, while the set of relevant concepts to recognize is large. As a result, dictionary-based methods have often been preferred for concept recognition in these contexts. 5 – 7 With the introduction of Large Language Models (LLMs), new possibilities have emerged to tackle these challenges, particularly leveraging their remarkable potential to perform tasks with minimal or no task-specific training data (zero-shot or few-shot learning). 8 , 9 These models, pre-trained on vast amounts of text, often capture rich semantic knowledge that can be adapted for downstream tasks like IE through prompting or minimal fine-tuning. In this work, we explore the feasibility of using LLMs for low-resource IE in the highly specialized domain of radiology. As a case study, we utilize brain MRI reports annotated with clinical concepts and relations relevant for Epilepsy. With only 100 annotated reports, encompassing a rich set of 152 entity types and 4 relation types, we investigate the capabilities and limitations of LLMs in achieving meaningful extraction performance. We break down the complex IE problem into distinct task formulations based on the granularity of extraction required: span-only identification, entity types, and relation extraction. Each task formulation is evaluated using both standard fine-tuned models (as baselines where applicable) and various approaches leveraging LLMs. We experiment with different settings for applying LLMs: zero-shot prompting, few-shot prompting (providing examples in the prompt), and fine-tuning. We explore both end-to-end generation with a single prompt encompassing all tasks, and pipeline approaches where the output from one task (e.g., entity extraction) informs the prediction for the next (e.g., relation extraction). In general, we make two main contributions: We introduce a new, challenging dataset for information extraction from brain MRI reports. It features fine-grained annotations covering a diverse set of Epilepsy-related clinical entity types and their relationships, reflecting real-world complexity. We explore the practical limits of applying a small, open-weight LLM (capable of running on consumer-grade GPUs) to this task. This demonstrates a realistic scenario for institutions where data privacy is paramount and reliance on external, large-scale proprietary models is not feasible. 2. Related work In traditional NLP, Information Extraction (IE) is typically framed as a sequence labeling task. Methods often involve fine-tuning pre-trained encoder models, such as BERT 10 or its domain-specific variants (e.g., BioBERT 3 or clinical BERT models 11 ), with an additional classification head (e.g., a linear layer) to predict a label for each token using the BIO tagging scheme. 12 These approaches have achieved state-of-the-art results on many benchmarks when sufficient labeled data is available. 4 In contrast, using generative methods based on LLMs, IE is framed as a text-to-text task. The model takes the source text as input and is prompted to generate the relevant text spans along with their corresponding labels or structures directly, often in a structured format like JSON, or as natural language statements. 13 LLMs have been explored for IE tasks with promising results. 14 While early studies suggested that LLMs might lag behind smaller, fine-tuned encoder models on classic benchmarks with ample supervision signals, 15 their strength lies in adaptability and performance in low-resource or zero-shot settings. Generative methods are gaining traction due to their ability to perform well in new domains or on novel entity/relation types by simply modifying the prompt or providing a few relevant examples (in-context learning). 8 Specifically for biomedical text, LLMs have been applied to extract medical problems, tests, and treatments from patient discharge summaries. 16 The authors showed that using only 8 few-shot samples and carefully curated prompts, LLMs could achieve comparable performance to fully-supervised systems on the popular i2b2 dataset. 17 However, the setting is relatively simple with only 3 entity types and 8 relations and a substantial number of training samples. Radiology reports, especially highly structured ones like brain MRIs, present unique challenges. They combine structured or semi-structured templates with free-text descriptions, utilize highly specific anatomical and pathological terminology, and often contain implicit relations that require nuanced understanding. Furthermore, the clinical requirement for high precision makes IE particularly demanding in this domain. Different strategies for prompting LLMs for IE have been explored, ranging from simple instruction-based prompts to more complex schemes involving structured output formats, 18 task reformulation, 19 or chain-of-thought reasoning. 20 While extremely large models demonstrate impressive capabilities, recent research increasingly focuses on the potential of smaller, open-weight models (e.g., Llama, 21 Flan-T5 22 ) for domain-specific tasks. 23 These models offer a pathway for deploying powerful NLP capabilities locally in secure environments, mitigating data privacy concerns and reducing computational costs, which is highly relevant for medical applications. Our work adds to this research by systematically evaluating a smaller LLM’s effectiveness on a complex, low-resource medical IE task through both prompting and fine-tuning. 3. Dataset: Brain MRI reports Brain MRI reports from a cohort of epilepsy patients were obtained from The Royal Melbourne Hospital, a major comprehensive epilepsy centre in Melbourne, Australia. A semi-randomised approach was used to select the reports, to ensure they were sampled over a period of years and from different reporting radiologists to capture variations in phrasing, sentence structure, and reporting styles. The reports were deidentified by clinicians to remove dates, locations, names prior to annotation. 3.1. Annotation Schema 3.1.1. List of entities A hierarchical list of entities was collated and refined for the MRI annotation process. This list was based on terminology published by the National Institute of Neurological Disorders and Stroke (NINDS), Epilepsy / Imaging Diagnostics / MRI Common Data Elements a . 25 The NINDS common data elements were first released in 2010 and were compiled by subject-specific working groups to promote consistency for naming and collecting data elements in epilepsy research. This initiative aims to improve efficiency and effectiveness of clinical research studies and clinical treatment, increase data quality and facilitate data sharing. Researchers also added MRI entities described in a real-world study of a well-characterised cohort of patients with new-onset seizures. 26 The final list of entities adopted for this work is displayed in Appendices A-B. 3.2. Relations Hedging and negation terms indicating the possible or negative presence of entities were also targeted. For instance, “Differentials include… “( possible hedging), “There is no evidence of… “( negation ). These terms were then linked to the corresponding entities to capture the scope of the relation. Negation was grouped with terms suggesting a finding was unlikely. No hedging term was assigned when a high degree of certainty was given for the presence of an entity, e.g., “This most likely represents… “. In these instances, the entity alone was annotated. Formally, 4 distinct relation types are defined: unlikely-rel : Indicates negation or low likelihood of a finding. possible-rel : Indicates uncertainty or possibility of a finding. location-rel : Links a finding (lesion, feature) to its anatomical location. feature-rel : Links an entity (e.g., a lesion) to its descriptive features (e.g., size, signal characteristics). 3.3. Annotation process Following refinement of the list of entities and hedging terms, two board-certified neurologists annotated the reports according to the schema. An example appears in Table 1 . View this table: View inline View popup Download powerpoint Table 1. Extract from a radiology report with selected entity annotations, converted from brat 24 standoff format for clarity. 3.3.1. Annotation environment The de-identified MRI reports were uploaded into brat, 24 an open-source web-based annotation tool, installed on a hospital firewall- and password-protected server. Clinician annotators highlighted words or phrases in the text, and assigned one or more entities from a drop-down list. For example, “stroke” was annotated as “arterial stroke”, “oligodendroglioma” was annotated as “tumour / primary”, etc. 3.3.2. Iterative annotation The clinicians co-annotated the first 10 MRI reports together to gain an understanding of the annotation process and agree on a mutual approach. Subsequent MRI reports were annotated independently, in batches of 10, followed by assessment of inter-annotator agreement (IAA), based on treating one neurologist as the “gold standard”. Annotation differences between reports were marked as “false negative” and “false positive”, referring to either missed or additional annotations with reference to the gold standard. Differences in annotation spans and linking of hedging terms were also noted. These anotations were merged on the MRI reports in brat, and reviewed together by the neurologists. Differences were resolved by consensus. An example of annotation variation is the phrase “ventricular enlargement”, which could be annotated as both “atrophy” and “ventriculomegaly”, depending on the context provided in the remainder of the report. Once neurologists reached acceptable levels of inter-annotator agreement, which was achieved after approximately 50 MRI reports, they proceeded to annotate separate MRI reports to increase the number ultimately included in the training set. 3.4. Dataset statistics The details of the dataset are provided in Table 2 . In total, there are 101 reports annotated with 102 different entity types and 4 relations. We calculate the initial inter-annotator agreement between two annotators using F1-score. The F1-score for the entity-level annotation is just around 60% with 25 labels with 0 agreements, and relation-level annotation has below 10% agreement, showing that this is a hard task even for human experts. View this table: View inline View popup Download powerpoint Table 2. Distribution of Annotations Across Levels with Inter-Annotator Agreement and Unique Counts Overall, we find that higher diversity leads to lower IAA. Unsurprisingly, relation annotation was the most difficult, as the agreement need to be reached for 3 elements — the head and tail entities, and the relation between them. In addition to having high diversity, the relation distribution also skews heavily toward unlikely-rel . Both possible-rel and feature-rel have the highest diversity while having 0% IAA, showing severe unreliability. We treat IAA as the human-level upper-bound performance. The low IAA across all levels display the difficulty of this task. 4. Experiments 4.1. Task formulation Following previous works in generative IE, we formulate this task as a text-to-text problem where the corresponding list of entities or relation triples are generated from raw input text. This is different from traditional sequence labeling tasks, where each token is classified as being inside or outside an entity or relation span. The annotations are captured in the brat standoff format, 24 which includes precise character offsets for each annotation. Based on the level of granularity required for evaluation and to better suit the generative text-to-text task, we convert the brat annotation file into a scheme focused on the text span and its label, omitting the explicit start and end offsets. This conversion simplifies the target output for the generative models, which struggle to produce precise character indices. This process may result in identical text spans annotated with the same label at different positions being treated as a single instance in the target output or indeed overgenerations; 27 we removed such duplicates from the target list for simplicity during training and evaluation of the models. Clinical concept span detection As the simplest task formulation, we focus solely on identifying spans of relevant clinical concepts within the report, without assigning any specific entity or relation type. The target output is simply a list of extracted text spans corresponding to any annotated concept. This assesses the model’s fundamental ability to locate salient information snippets. Named entity recognition For named entity recognition, the goal is to extract text spans and classify them with their corresponding entity type. Each target annotation is formatted as extracted entity span - entity type. The annotation scheme used (Appendices A-B) is hierarchical, with up to 4 levels of specificity; an entity label may belong to any of these levels. In this work, we evaluate NER performance at two primary levels of granularity to understand the model’s capabilities at different abstraction levels: Coarse: Using only the outermost category label (e.g., Vascular-malformation). Fine: Using the most specific sub-category label available (e.g., Arteriovenous-malformation, Cavernous-angioma, Cerebral-aneurysm, Developmental-venous-anomaly, which are all sub-types of Vascular-malformation). Evaluating at both the Coarse and Fine levels allows us to assess the model’s ability to capture broader semantic categories versus more specific, granular concepts present in the text. Relation extraction For relation extraction, the task is to identify relationships between pairs of extracted entities. Each target annotation is a triple formatted as: head entity span - relation type - tail entity span. To improve the quality of generated relations and reduce invalid outputs, we explicitly define the permissible entity types for the head and tail arguments of each relation type, information also available during manual annotation via constraints specified in the brat configuration. These constraints are also incorporated into the LLM prompt instructions (see Appendix D) to guide the generation process. For example, the type constraints for the location-rel relation are described as in Figure 1 . An overall summary of the relations adopted appears in Appendix C. Download figure Open in new tab Fig. 1. Constraint Example (excerpt) for location-rel. Note that the full list of eligible sub-types provided in the actual prompt is truncated here for clarity. 4.2. Baselines For all the above settings, we consider different baselines that can be applied to this type of task: Dictionary-based We perform a straightforward dictionary lookup using the label spaces provided in the annotation schemes, with some simple text normalization steps (e.g. removing hyphen, lower-casing). For instance, for the Entity-level extraction, the label Grey-matter-heterotopia will be normalized into “grey matter heterotopia” for exact matching in the text. If such a span exists, we will tag it with the same label (Grey-matter-heterotopia). For Relation-level, we group the 3 closest eligible matches of head entity, relation, and tail entity to form a triple. Off-the-shelf system We use SciSpacy, 28 a system for biomedical information extraction. For entity-level, we employ a retriever based on to retrieve the top-1 match from the extracted entities from SciSpacy and the label set from the annotation scheme. For relation-level, we employ the CueNB 29 for unlikely-rel and NegEx 30 for possible-rel , while for location-rel and feature-rel, we use OpenIE employed in Stanza 31 and map is-in relations to location-rel, and has- or is-relations to feature-rel. Few-shot This represents our main approach. We utilize a powerful, open-weight LLM that can be run locally on consumer-grade hardware, ensuring data privacy and demonstrating practical applicability. Specifically, we use Llama-3-8B-Instruct. 21 We employ few-shot prompting, where the prompt includes instructions defining the task and output format, followed by a small number (k=10) of randomly selected examples (demonstrations) from our annotated dataset. We found that using 10 samples provided a reasonable balance between context length and performance; preliminary experiments suggested that adding significantly more samples yielded only small marginal improvements for this specific model and task setup. The general structure of the prompt is shown in Figure 2 . The format of each {annotation} line within the examples depends on the specific task formulation: Download figure Open in new tab Fig. 2. Template for a few-shot prompt. Relevant Span detection: Annotation is the text span, e.g., mild diffuse cerebral volume loss . NER: Annotation includes the span and type, e.g., mild diffuse cerebral volume loss – Atrophy . RE: Annotation is the relation triple, e.g., mild diffuse cerebral volume loss – location-rel – cerebral . The Task-specific Instruction also varies for each task (Span, Entity, Relation) and includes details like the entity type constraints for RE. For NER and RE, we also include the list of valid labels in the instruction. The full list of entity types and constraints for relation extraction are in the Appendices. As a post-processing step, particularly for NER and RE where the LLM might generate entity types or text spans that don’t perfectly align with the gold standard annotations or predefined ontology, 27 we employ a grounding strategy. This maps potentially out-of-vocabulary type labels generated by the LLM to the label options within our annotation scheme that are semantically closest, measured using cosine similarity of sentence embeddings derived from a pre-trained model. In particular, we used the BioLORD model, 32 which has state-of-the-art performance on biomedical and clinical semantic similarity tasks. Similarly, generated entity spans can be loosely matched to ground truth spans if needed for evaluation metrics that allow for fuzzy matching. Fine-tuning We perform parameter-efficient fine-tuning using LoRa 33 to fine-tune a LLama 3.1 8B model with MRI data converted into Alpaca instruction-tuning format. 34 Fine-tuning details are in Appendix E. The models are evaluated following a 5-fold cross-validation with 20/80 train/test ratio. We keep the test set size larger to reflect the real-world scenario where ground truth data are scarce, and to make this comparable with the few-shot setting. 4.3. Main results The main results for all 3 task formulations using the 3 baselines are in Table 3 . For all settings, we adopt the standard Precision, Recall, F1-score metrics for information extraction tasks. We utilize the strict evaluation setting for relation extraction, where we consider a prediction to be correct only if the all 3 of the head entity, the tail entity, and the relation type are correct. View this table: View inline View popup Download powerpoint Table 3. Performance of the tested baselines for each level. Results are reported as mean (±std) over 5 random seeds. Span-level does not have Pipeline results as it is the first step in the pipeline. View this table: View inline View popup Download powerpoint Table 4. Relation extraction results breakdown by type (F1 Scores) LLMs outperform specialized off-the-shelf systems Across all task formulations, general-purpose large language models (LLMs) consistently outperform specialized off-the-shelf systems tailored for individual clinical or biomedical tasks, even with only a few demonstrations. Despite this, the absolute performance across all tasks remains relatively low. Only in the simplest span detection task do LLMs demonstrate relatively strong performance. Nevertheless, when using the initial inter-annotator agreement (IAA) score ( Table 2 ) as a proxy for human performance, the LLMs’ output is not far behind. Moreover, LLMs offer a significant advantage in terms of ease of use and simplicity: they operate in an end-to-end fashion, unlike traditional systems that require multiple pipeline steps (e.g., extracting spans, assigning entity labels, and predicting inter-entity relations) or ample training data. LLMs can learn task-specific patterns with sufficient supervision While overall performance remains modest, we observe a promising trend in tasks with greater supervision—such as relevant span detection, named entity recognition, and the unlikely-rel relation—where LLMs perform noticeably better. In contrast, the poor performance on tasks like extracting location, feature and possible relation can be attributed to the high variability in the training data. Specifically, 32 out of 34 instances of feature-rel are unique, making it difficult for the model to generalise. The open-ended nature of this type of relation further complicates learning. Fine-tuning also yields larger gains in tasks where more training supervision is available ( unlikely-rel , entities extraction). Overall, finetuning LLM achieved the best performance across most setting, outperforming other baselines by a large margin except for relation extraction. We also experimented with a pipeline formulation of the extraction task, wherein the output of one step is used as additional input in the prompt for the next step. For example, in entity-level extraction, we first identify entity spans, then condition the prompt on these spans to predict the corresponding entity types. While this structured approach provides intermediate supervision, it adds complexity compared to end-to-end approaches, and did not achieve any improvement. Normalization is essential when applying LLM for IE tasks When using LLMs for structured prediction, it is important to apply normalization to standardize the prediction, ensuring models don’t make up terms and concepts that are not in the target label spaces. In our experiments, we did find that normalization of spurious outputs was required with high frequency. In particular, the normalization rate is 36.8% for entities, and 20.1% for relations. This is not surprising given that we include a long context in the prompts (with the full list of labels, together with 10 examples of long input MRI reports-ground truth annotations). Another pervasive error is that the model generates predictions with invalid formats (e.g. truncated text spans, using the incorrect delimiters of the required, missing labels), produces entities that are not in the text, 27 or hallucinates a new MRI report after generating. These were dismissed instead of normalizing. 4.4. Error analysis We analyze the errors of the best performing setting (Fine-tuning) and categorise the most common errors in entity and relation extraction. 4.4.1. Entities Entity-level extraction suffers from over- or under-specificity. Models may label entities either too broadly or too narrowly. A clear example is seen in: “ Focal area of encephalomalacia, with surrounding gliotic change consistent with an old right parietal infarct …” The model incorrectly assigns a generic category ( Post-traumatic-or-post-ischemic-lesion ) rather than the correct more specific sub-category general label ( Post-stroke ). This error can also goes in the other direction; it uses a more specific, yet incorrect label, when the label higher up in the hierarchy would be more accurate, e.g., “ … pituitary gland measures 14mm tall, greater than expected for age The model assigns a specific but inaccurate sub-category (Basal-ganglia), when the more generic label higher up in the hierarchy would be correct in this instance ( Subcortical ). Additionally, models often confuse pathological processes with similar lexical forms. For example, they may mislabel ischaemia as Small-vessel-ischemic-change instead of the correct category Arterial-stroke-including-lacunar , due to surface-level lexical overlap. Finally, confusion between negation and possibility cues is a recurring issue, reflecting the subtlety and variability in clinical language, as well as the insensitivity of LLM on negation and hedging. 35 4.4.2. Relations Across all relation types, incorrect entity boundaries remain the most dominant source of error. Consistent with prior NER research, 36 issues related to span expansion and contraction are espe-cially prevalent. For example, the head entity “ enhancement ” is often misrecognized as “ abnormal enhancement ”, reflecting a common span inflation error. This challenge also extends to negation cues, where variations in phrasing (e.g., “ no ” vs. “ no evidence ”) contribute to inconsistencies in model predictions. Radiology texts often feature complex noun phrases, such as “ hippocampi size, signal, and morphology ”, which further complicate entity recognition and relation extraction. Another significant source of error is the confusion between relation types, particularly unlikely-rel and possible-rel . These mistakes frequently arise from the subjective nature of radiological assessment, where radiologists may express varying degrees of diagnostic confidence. The low performance on location-rel relations underscores the current models’ lack of spatial reasoning capability. This issue is exacerbated in radiology by the hierarchical structure of anatomical references, such as hemisphere→lobe→gyrus . Errors often stem from mismatches in anatomical specificity, such as predicting “ cortical ” when the correct span is “ left inferior frontal gyrus ”. 5. Conclusion In this work, we investigated the capabilities of LLM for information extraction from brain MRI reports in an extreme low-resource setting. We introduced a novel, challenging dataset comprising 101 expert-annotated reports, characterised by its fine-grained annotation scheme (152 entity types, 4 relations) and inherently low inter-annotator agreement, which underscores the real-world complexity of this task. Our experiments systematically evaluated a small, open-weight LLM (Llama-3-8B) using few-shot prompting and parameter-efficient fine-tuning, comparing its performance against off-the-shelf biomedical IE systems across relevant span detection, named entity recognition, and relation extraction. Key findings reveal that both few-shot prompting (with only 10 examples) and fine-tuned LLM approaches significantly outperform the specialized off-the-shelf systems, demonstrating the adaptability of LLMs even with minimal task-specific data. Despite this, absolute performance, particularly for complex relations and fine-grained entity distinctions, remains modest. This reflects the extreme low-resource nature of the study, the high diversity and skewness in annotations, and the inherent difficulty of the task, representative of clinical tasks with highly specific information needs. 37 Performance was generally better for more frequent and clearly defined categories, such as for relevant span detection and extraction of unlikely-rel relations. Our error analysis highlighted persistent challenges including precise boundary detection, entity specificity, interpretation of negation and modality cues, and handling complex hierarchical anatomical references. Future work on adapting LLMs to the complex requirements of fine-grained annotation of clinical texts is required to establish more reliable solutions. Methods integrating structured knowledge such as expert ontologies or knowledge graphs with language models may be promising in this context. 38 Data Availability All data produced in the present work are contained in the manuscript. https://doi.org/10.17632/6kxnvb7yyg.1 6. Ethics Ethics approval for this project was obtained from Melbourne Health Human Research Ethics Committee, approval number HREC/17/MH/308, approval date: 24 October 2017. Identifying details were removed from reports prior to annotation, including date and location of studies, names of patients and reporting radiologists, and if needed, names of any specific clinicians mentioned in the report (e.g., “I urgently notified Dr. XXX via telephone at 03:00am regarding these results”). Unfortunately, ethics approval has not been granted for public release of the data to date. 8. Appendices Appendices are available on Mendeley Data 39 at http://dx.doi.org/10.17632/6kxnvb7yyg.1 . 7. Acknowledgements The authors would like to thank Professor Patrick Kwan and Professor Terence O’Brien of Monash University, who provided guidance on the context of Epilepsy diagnosis and documentation. THT and KV acknowledge funding from the Australian Research Council grant IC170100030. Appendix A. Annotation scheme (Entities) Download figure Open in new tab Appendix B. Annotation scheme (Location Terms, Other Features, Hedging cues) Download figure Open in new tab Appendix C. Annotation scheme (Relations) View this table: View inline View popup Download powerpoint Appendix D. Prompt details D.1. Relevant Span detection Download figure Open in new tab D.2. Entity extraction Download figure Open in new tab D.3. Relation extraction Download figure Open in new tab Appendix E. Fine-tuning details View this table: View inline View popup Download powerpoint Table E1 Fine-tuning Configuration and Hyperparameters. Training is performed separately for each dataset level (inline, listing_coarse, listing, yaml) with 5 different random seeds. Footnotes ↵ a http://www.commondataelements.ninds.nih.gov/ References 1. ↵ E. F. T. K. Sang and F. De Meulder , Introduction to the CoNLL-2003 shared task: Language-independent named entity recognition , 142 ( 2003 ). 2. ↵ Y. Lin , S. Feng , C. N. d. Santos , M. Yu , B. Xiang , B. Zhou and Y. Bengio , Neural relation extraction with selective attention over instances , 1 , 2124 ( 2016 ). OpenUrl 3. ↵ J. Lee , W. Yoon , S. Kim , D. Kim , S. Kim , C. H. So and J. Kang , BioBERT: a pre-trained biomedical language representation model for biomedical text mining , Bioinformatics 36 , 1234 ( 2020 ). OpenUrl CrossRef PubMed 4. ↵ J. Li , A. Sun , J. Han and C. Li , A survey on deep learning for named entity recognition , IEEE Transactions on Knowledge and Data Engineering 34 , 50 ( 2020 ). OpenUrl 5. ↵ C. Funk , W. Baumgartner , B. Garcia , C. Roeder , M. Bada , K. B. Cohen , L. E. Hunter and K. Verspoor , Large-scale biomedical concept recognition: an evaluation of current automatic annotators and their parameters , BMC Bioinformatics 15 , 1 ( 2014 ). OpenUrl CrossRef PubMed 6. G. K. Savova , J. J. Masanz , P. V. Ogren , J. Zheng , S. Sohn , K. C. Kipper-Schuler and C. G. Chute , Mayo clinical text analysis and knowledge extraction system (ctakes): architecture, component evaluation and applications , Journal of the American Medical Informatics Association 17 , 507 ( 2010 ). OpenUrl CrossRef PubMed 7. ↵ A. R. Aronson and F.-M. Lang , An overview of MetaMap: historical perspective and recent advances , Journal of the American Medical Informatics Association 17 , 229 (05 2010 ). OpenUrl CrossRef PubMed 8. ↵ T. Brown , B. Mann , N. Ryder , M. Subbiah , J. D. Kaplan , P. Dhariwal , A. Neelakantan , P. Shyam , G. Sastry , A. Askell et al. , Language models are few-shot learners , Advances in neural information processing systems 33 , 1877 ( 2020 ). OpenUrl 9. ↵ J. Wei , Y. Tay , R. Bommasani , C. Raffel , B. Zoph , S. Borgeaud , D. Yogatama , M. Bosma , D. Zhou , D. Metzler et al. , Emergent abilities of large language models , Transactions on Machine Learning Research . 10. ↵ J. Devlin , M.-W. Chang , K. Lee and K. Toutanova , BERT: Pre-training of deep bidirectional transformers for language understanding , in Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers) , ( Association for Computational Linguistics, Minneapolis, Minnesota , June 2019 ). 11. ↵ A. Rumshisky , K. Roberts , S. Bethard and T. Naumann E. Alsentzer , J. Murphy , W. Boag , W.-H. Weng , D. Jindi , T. Naumann and M. McDermott , Publicly available clinical BERT embeddings , in Proceedings of the 2nd Clinical Natural Language Processing Workshop , eds. A. Rumshisky , K. Roberts , S. Bethard and T. Naumann ( Association for Computational Linguistics, Minneapolis, Minnesota, USA , June 2019 ). 12. ↵ L. A. Ramshaw and M. P. Marcus , Text chunking using transformation-based learning , in Natural language processing using very large corpora , ( Springer , 1999 ) pp. 157 – 176 . 13. ↵ C. Wu , S. Li , H. Liu , E. Wang , B. Liu , R. Lyu , M. Zhang and S. Zhou , Information extraction from text using large language models: A survey , arXiv preprint arxiv: 2305.13145 ( 2023 ). 14. ↵ D. Xu , W. Chen , W. Peng , C. Zhang , T. Xu , X. Zhao , X. Wu , Y. Zheng , Y. Wang and E. Chen , Large language models for generative information extraction: A survey , Frontiers of Computer Science 18 , p. 186357 ( 2024 ). 15. ↵ H. Bouamor , J. Pino and K. Bali Y. Ma , Y. Cao , Y. Hong and A. Sun , Large language model is not a good few-shot information extractor, but a good reranker for hard samples! , in Findings of the Association for Computational Linguistics: EMNLP 2023 , eds. H. Bouamor , J. Pino and K. Bali ( Association for Computational Linguistics , Singapore , December 2023 ). 16. ↵ E. Hsu and K. Roberts , LLM-IE: a Python package for biomedical generative information extraction with large language models , JAMIA open 8 , p. ooaf012 ( 2025 ). OpenUrl 17. ↵ Ö. Uzuner , B. R. South , S. Shen and S. L. DuVall , 2010 i2b2/VA challenge on concepts, assertions, and relations in clinical text , Journal of the American Medical Informatics Association 18 , 552 ( 2011 ). OpenUrl CrossRef PubMed 18. ↵ L. Chiruzzo , A. Ritter and L. Wang S. Wang , X. Sun , X. Li , R. Ouyang , F. Wu , T. Zhang , J. Li , G. Wang and C. Guo , GPT-NER: Named entity recognition via large language models , in Findings of the Association for Computational Linguistics: NAACL 2025 , eds. L. Chiruzzo , A. Ritter and L. Wang ( Association for Computational Linguistics , Albuquerque, New Mexico , April 2025 ). 19. ↵ W. Zhou , S. Zhang , Y. Gu , M. Chen and H. Poon , UniversalNER: Targeted distillation from large language models for open named entity recognition ( 2024 ). 20. ↵ F. Chen and Y. Feng , Chain-of-thought prompt distillation for multimodal named entity recognition and multimodal relation extraction , arXiv preprint arxiv: 2306.14122 ( 2023 ). 21. ↵ AI@Meta, Llama 3 model card ( 2024 ). 22. ↵ H. W. Chung , L. Hou , S. Longpre , B. Zoph , Y. Tay , W. Fedus , Y. Li , X. Wang , M. Dehghani , S. Brahma et al. , Scaling instruction-finetuned language models , Journal of Machine Learning Research 25 , 1 ( 2024 ). OpenUrl CrossRef 23. ↵ M. J. J. Bucher and M. Martini , Fine-tuned ‘small’ LLMs (still) significantly outperform zero-shot generative ai models in text classification , arXiv preprint arxiv: 2406.08660 ( 2024 ). 24. ↵ P. Stenetorp , S. Pyysalo , G. Topić , T. Ohta , S. Ananiadou and J. Tsujii , brat: a web-based tool for NLP-assisted text annotation , in Proceedings of the Demonstrations Session at EACL 2012 , ( Association for Computational Linguistics, Avignon , France , April 2012 ). 25. ↵ S. T. Grinnon , K. Miller , J. R. Marler , Y. Lu , A. Stout , J. Odenkirchen and S. Kunitz , National institute of neurological disorders and stroke common data element project – approach and methods , Clinical Trials 9 , 322 ( 2012 ) , PMID: 22371630 . OpenUrl CrossRef PubMed Web of Science 26. ↵ T. Hakami , A. Mcintosh , M. Todaro , E. Lui , R. Yerra , K. M. Tan , C. French , S. Li , P. Desmond , Z. Matkovic et al. , MRI-identified pathology in adults with new-onset seizures , Neurology 81 , 920 ( 2013 ). OpenUrl CrossRef PubMed 27. ↵ T. Baldwin , S.J. Rodríguez Méndez and N. Kuo M. Kodikara and K. Verspoor , Lesser the shots, higher the hallucinations: Exploration of genetic information extraction using generative large language models , in Proceedings of the 22nd Annual Workshop of the Australasian Language Technology Association , eds. T. Baldwin , S.J. Rodríguez Méndez and N. Kuo ( Association for Computational Linguistics , Canberra, Australia , December 2024 ). 28. ↵ M. Neumann , D. King , I. Beltagy and W. Ammar , ScispaCy: Fast and robust models for biomedical natural language processing , in Proceedings of the 18th BioNLP Workshop and Shared Task , ( Association for Computational Linguistics , Florence, Italy , August 2019 ). 29. ↵ T. Truong , T. Baldwin , T. Cohn and K. Verspoor , Improving negation detection with negation-focused pre-training , in Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies , ( Association for Computational Linguistics , Seattle, United States , July 2022 ). 30. ↵ W. W. Chapman , W. Bridewell , P. Hanbury , G. F. Cooper and B. G. Buchanan , A simple algorithm for identifying negated findings and diseases in discharge summaries , Journal of Biomedical Informatics 34 , 301 ( 2001 ). OpenUrl CrossRef PubMed Web of Science 31. ↵ A. Celikyilmaz and T.-H. Wen P. Qi , Y. Zhang , Y. Zhang , J. Bolton and C. D. Manning , Stanza: A Python natural language processing toolkit for many human languages , in Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics: System Demonstrations , eds. A. Celikyilmaz and T.-H. Wen ( Association for Computational Linguistics , Online, July 2020 ). 32. ↵ F. Remy , K. Demuynck and T. Demeester , BioLORD-2023: semantic textual representations fusing large language models and clinical knowledge graph insights , Journal of the American Medical Informatics Association , p. ocae029 (02 2024 ). 33. ↵ E. J. Hu , P. Wallis , Z. Allen-Zhu , Y. Li , S. Wang , L. Wang , W. Chen et al. , LoRA: Low-rank adaptation of large language models ( 2021 ). 34. ↵ R. Taori , I. Gulrajani , T. Zhang , Y. Dubois , X. Li , C. Guestrin , P. Liang and T. B. Hashimoto , Stanford Alpaca: A strong, replicable instruction-following model https://github.com/tatsu-lab/stanford_alpaca , ( 2023 ). 35. ↵ A. Palmer and J. Camacho-collados T. H. Truong , T. Baldwin , K. Verspoor and T. Cohn , Language models are not naysayers: an analysis of language models on negation benchmarks , in Proceedings of the 12th Joint Conference on Lexical and Computational Semantics (*SEM 2023) , eds. A. Palmer and J. Camacho-collados ( Association for Computational Linguistics , Toronto, Canada , July 2023 ). 36. ↵ D. Q. Nguyen , Z. Zhai , H. Yoshikawa , B. Fang , C. Druckenbrodt , C. Thorne , R. Hoessel , S. A. Akhondi , T. Cohn , T. Baldwin and K. Verspoor , ChEMU: Named entity recognition and event extraction of chemical reactions from patents, in Advances in Information Retrieval: 42nd European Conference on IR Research , ECIR 2020, Lisbon, Portugal, April 14–17, 2020, Proceedings, Part II , ( Springer-Verlag , Berlin, Heidelberg , 2020 ). 37. ↵ A. Lederman , R. Lederman and K. Verspoor , Tasks as needs: reframing the paradigm of clinical natural language processing research for real-world decision support , Journal of the American Medical Informatics Association 29 , 1810 (07 2022 ). OpenUrl CrossRef PubMed 38. ↵ S. Pan , L. Luo , Y. Wang , C. Chen , J. Wang and X. Wu , Unifying large language models and knowledge graphs: A roadmap , IEEE Transactions on Knowledge and Data Engineering 36 , 3580 ( 2024 ). OpenUrl 39. ↵ E. Foster , T. Fazio , S. Holper , T. Truong and K. Verspoor , Melbourne Brain MRI Report Epilepsy Annotation Dataset , Mendeley Data ( 2025 ). View the discussion thread. Back to top Previous Next Posted August 05, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Exploring the limits of LLMs in low-resource information extraction: Case study in brain MRI reports for Epilepsy Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Exploring the limits of LLMs in low-resource information extraction: Case study in brain MRI reports for Epilepsy Thinh Hung Truong , Emma Foster , Timothy Fazio , Sarah Holper , Karin Verspoor medRxiv 2025.08.02.25332570; doi: https://doi.org/10.1101/2025.08.02.25332570 Share This Article: Copy Citation Tools Exploring the limits of LLMs in low-resource information extraction: Case study in brain MRI reports for Epilepsy Thinh Hung Truong , Emma Foster , Timothy Fazio , Sarah Holper , Karin Verspoor medRxiv 2025.08.02.25332570; doi: https://doi.org/10.1101/2025.08.02.25332570 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Neurology Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15227) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6597) Geriatric Medicine (668) Health Economics (997) Health Informatics (4534) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3332) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9230) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a003da668888300f',t:'MTc3OTUzNzEzMg=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();