Empirical Review of LLM-driven Classification of Multidimensional Sleep Health Mentions from Free-Text Clinical Notes

Empirical Review of LLM-driven Classification of Multidimensional Sleep Health Mentions from Free-Text Clinical Notes | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Empirical Review of LLM-driven Classification of Multidimensional Sleep Health Mentions from Free-Text Clinical Notes View ORCID Profile Syed-Amad Hussain , Ariana Calloway , Joseph Sirrianni , Eric Fosler-Lussier , Mattina Davenport doi: https://doi.org/10.1101/2025.06.04.25328983 Syed-Amad Hussain 1 Nationwide Children’s Hospital 2 The Ohio State University BSE Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Syed-Amad Hussain For correspondence: amad.hussain{at}nationwidechildrens.org Ariana Calloway 1 Nationwide Children’s Hospital MPH Find this author on Google Scholar Find this author on PubMed Search for this author on this site Joseph Sirrianni 1 Nationwide Children’s Hospital Ph.D. Find this author on Google Scholar Find this author on PubMed Search for this author on this site Eric Fosler-Lussier 2 The Ohio State University Ph.D. Find this author on Google Scholar Find this author on PubMed Search for this author on this site Mattina Davenport 1 Nationwide Children’s Hospital 2 The Ohio State University Ph.D. Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Accurate multidimensional sleep health (MSH) information is often fragmented and inconsistently represented within hospital infrastructures, leaving crucial details buried in unstructured clinical notes rather than discrete fields. This inconsistency complicates large-scale phenotyping, secondary analyses, and clinical decision support regarding sleep-related outcomes. In this work, we systematically explore contemporary natural language processing techniques, prompt-based large language models (LLMs) and fine-tuned discriminative classifiers, to bridge this critical gap. We evaluate performance on extracting nine key MSH dimensions (timing, duration, efficiency, sleep disorders, daytime sleepiness, interventions, medication, behavior, and satisfaction) from clinical narratives using public datasets (MIMIC-III derivatives) and an internally annotated pediatric sleep corpus. Initially, we assess generative LLM performance using dynamic few-shot prompting, analyzing impacts from varying prompt structures, example quantity, and domain-specificity without explicit task-specific fine-tuning. Subsequently, we fine-tune generative LLM architectures on both in-task and out-of-task data to quantify performance improvements and limitations. Lastly, we benchmark these generative approaches against encoder-based discriminative classifiers (ModernBERT), designed to directly estimate binary presence of each MSH class within full clinical notes. Our experiments demonstrate that fine-tuned discriminative models consistently provide higher classification accuracy, lower inference latency, and more robust span-level identification than either prompted or fine-tuned generative LLMs, given adequate training data. Nonetheless, generative LLMs retain moderate utility in low-data scenarios. Importantly, our results highlight persistent challenges, including difficulty extracting subtle sleep constructs such as sleep efficiency and daytime sleepiness, and biases associated with patient demographics and clinical departments. We conclude by suggesting future research directions: refining span extraction methods, mitigating biases in model performance, and exploring advanced chain-of-thought prompting techniques to achieve reliable, scalable MSH phenotyping within real-world clinical systems. Introduction Accurate extraction of multidimensional sleep health (MSH) information from clinical narratives presents a critical opportunity for enhancing patient care, epidemiological research, and healthcare resource management. Clinical notes, often unstructured and narrative in nature, contain rich descriptions of patient behaviors, treatment interventions, and clinician observations, making them valuable but challenging sources for extracting nuanced clinical concepts such as sleep health. Traditional approaches to clinical natural language processing (NLP) have predominantly leveraged rule-based or simple machine learning models, often optimized for specific, narrow tasks and reliant on manually curated vocabularies or regular expressions. While such methods have shown high performance in limited scenarios, their ability to generalize across diverse clinical contexts and subcategories is often limited. Recent advancements in large language models (LLMs), pre-trained on vast textual corpora, suggest significant potential to overcome these limitations through their enhanced semantic understanding and generalization capabilities. Domain-specific and clinically pretrained models, such as BioClinicalBERT and MentalBERT, have demonstrated superior performance over generic models by capturing nuances unique to medical contexts, including specialized terminologies and clinical reasoning. Moreover, tasks involving binary or multi-label classification approaches have shown varied effectiveness depending on their complexity and domain, highlighting the importance of selecting appropriate model architectures and training paradigms. In this work, we systematically evaluate the effectiveness of contemporary LLMs, both generative (decoder-only) and discriminative (encoder-only), in identifying nine distinct categories of sleep-related information from clinical notes: behaviors, satisfaction/quality, daytime sleepiness/alertness, timing, efficiency, duration, disorders, medication, and interventions. Specifically, we investigate three core research questions: (1) How effectively do generative LLMs perform using dynamic prompting approaches without explicit fine-tuning, and what factors (e.g., prompt structure, few-shot example quantity, and domain-specificity) influence their performance? (2) How does targeted fine-tuning of generative LLMs on sleep-specific data influence performance and reliability in clinical settings? (3) To what extent can discriminative encoder models, specifically fine-tuned for binary classification, outperform generative models, particularly in terms of accuracy, computational efficiency, and practical deployability? Our comprehensive experiments utilize both publicly available clinical datasets (MIMIC-III derivatives: MDACE and SDOH) and an internally annotated dataset from Nationwide Children’s Hospital (NCH-Sleep). We explore model performance across different experimental configurations—including variations in context window size, few-shot prompting scenarios, domain-specific pretraining, and fine-tuning strategies—to thoroughly characterize performance limits and practical applicability. We additionally assess biases inherent in these methods by stratifying results according to patient demographics, departmental contexts, and MSH classes, providing critical insights into potential disparities in model performance. We demonstrate that fine-tuned discriminative models significantly outperform prompted and fine-tuned generative models, achieving superior classification accuracy, computational efficiency, and clinical applicability, particularly when trained with a combination of related-domain and task-specific data. Conversely, generative LLMs are shown to provide moderate performance with extensive prompting, highlighting their utility primarily when no labeled training data is available. Our findings underscore the value of careful selection of model type, training strategy, and data sources, ultimately guiding practitioners toward optimal approaches for clinical NLP tasks focused on sleep health phenotyping. Background Recently, natural language processing (NLP), particularly leveraging Large Language Models (LLMs), has gained attention for efficiently extracting multidimensional sleep health (MSH) information from clinical narratives. NLP methods have historically employed rule-based or keyword-driven strategies for clinical data extraction. For example, Horner et al. (2022) developed a rule-based algorithm to detect sleep complaints from primary care notes, achieving high specificity (91%) but limited sensitivity (53%), relying mainly on keyword patterns and regular expressions [ 14 ]. Expanding these methods, Siviarajkumar et al. (2024) systematically compared rule-based, traditional machine learning, and LLM-based approaches, demonstrating the superior performance of their enhanced keyword-driven rule-based strategy across sleep-related concepts (F1-scores ranging from 0.91 to 1.0) [ 15 ]. Furthermore, Sirrianni et al. (2025) refined sleep-related vocabulary sets with structured clinical terminologies and semantic embeddings, significantly improving recall (0.992 vs. 0.857) in identifying notes with sleep mentions, although their method was limited to note-level classification without identifying precise textual locations or differentiating subcategories [ 17 ]. In parallel areas, such as suicide phenotyping, recent research further underscores the importance of domain-specific pretraining. Li et al. (2024) employed various BERT-based models, including generic models like BERT and RoBERTa, domain-adapted models such as BioClinicalBERT, and disease-specific models like MentalBERT, specifically pre-trained on mental health text [ 16 ]. They demonstrated that domain-adapted and disease-specific models significantly outperformed generic models, highlighting the advantages of domain-specific pretraining for clinical NLP tasks [ 16 ]. Their binary classification strategy, transforming multi-label classification into multiple independent binary tasks, proved effective. However, they also found enhanced performance when employing single multi-label classifiers that could exploit label interdependencies [ 16 ]. These findings collectively indicate that while foundational NLP methods have effectively initiated sleep information extraction from clinical notes, advanced LLM architectures, particularly those with domain-specific pretraining and sophisticated task structures, are increasingly necessary. Such methods promise to better capture complex semantics and contextual subtleties inherent to clinical documentation related to multidimensional sleep health concepts. Data Our work employs both de-identified MIMIC-III datasets (consisting of MDACE and SDOH as described in the MIMIC-III datasets section) as well as an internal sleep mentions dataset [ 1 , 2 ]. All datasets, derived from electronic medical records, consist of notes which are annotated according to if a given class is present in the note. The quantity of notes, note sizes, total classes, and positive classes per note are described in Table 1 . View this table: View inline View popup Download powerpoint Table 1: Basic metrics of datasets used. Note length reported in characters. Mean class refers to average number of positive classes per note MIMIC-III Datasets MIMIC-III (Medical Information Mart for Intensive Care) is a publicly available dataset of deidentified electronic health records for patients admitted to critical care at a large tertiary care hospital. While MIMIC-III includes many data points (e.g. laboratory measurements and medications), we are primarily focused on downstream datasets which make use of the free-text clinical notes within MIMIC-III [ 3 ]. These clinical note datasets serve as a proxy for our sleep mention classification task of positively identifying mentions of a given property within the free-text clinical note. MDACE: In “MIMIC Documents Annotated with Code Evidence”, Cheng et al. (2023) provide a dataset with expert annotations over 302 inpatient charts [ 1 ]. The experts use ICD-9 and ICD-10 schemas to determine coding— we choose to focus on ICD-10 due to increased contemporary relevance [ 1 ]. Each of these annotated codes has an associated evidence span within the clinical note text, resulting in a total of 5,563 total evidence spans for the dataset [ 1 ]. Additionally, we gather class descriptions by cross referencing each annotated ICD-10 code with its definition as per CDC.gov. SDOH: In “Annotation dataset of social determinants of health from MIMIC-III Clinical Care Database”, Guevera et al. (2024) annotate MIMIC-III clinical notes with patient status over 6 factors (e.g. employment, housing, transportation, parental status, relationship, and social support), totaling 20 possible statuses [ 2 ]. SDOH classification acts as a unique foil for our internal dataset as both have descriptive indicators primarily encoded within clinical text rather than in structured data fields. This work has a total of 145 expert annotated notes, which they expand using synthetic data generation [ 2 ]. We only make use of the expert annotated notes to avoid any noise generated by synthetic data points. Like MDACE, the annotations include evidence spans to justify the classification [ 1 , 2 ]. NCH-Sleep Dataset We construct a sleep dataset, NCH-Sleep, using 300 clinical notes written between 2018 and 2023 at Nationwide Children’s Hospital (NCH). These clinical notes were initially pulled from the hospital EHR system using keyword matching over a list of expert-defined sleep-related keywords from four departments: primary care, behavioral health, healthy weight, and school based. We randomly sampled 300 notes from the entire dataset (∼700,000 notes) stratified by department and year (we additionally stratify our train/test split on departments as their note-taking structures have dramatically distinct styles). Subsequently, each note was expert annotated for presence of the 9 sleep MSH classes, with corresponding evidence spans highlighted. Table 2 describes the 9 sleep MSH classes and their relevant descriptions. Table 3 lists each department and their relative frequency of notes. View this table: View inline View popup Table 2: The 9 MSH classes annotated in the NCH-Sleep dataset and evaluated in each of our experiments. Class descriptions are used to help inform models. View this table: View inline View popup Download powerpoint Table 3: Departments and their respective number of notes within NCH-Sleep For all analyses presented in this work we use a 67/33 (200 note / 100 note) train/test split over the NCH-Sleep dataset. Even when NCH-Sleep train data is not directly used for model training or context sampling, we still validate every method over the 100 note NCH-Sleep test set. We use this split ratio as opposed to the standard 80/20 due to the small size of our dataset and the need for a representative evaluation. Furthermore, while training tasks benefit from larger training sets, prompting methods do not share a similar scaling with training set size. This 67/33 split is meant to balance the needs for evaluation and training. Task Structure Boolean Classification per Class While our evaluation can be represented as a multi-class classification task over the 9 MSH classes, we assume conditional independence of each class, especially since multiple classes can be true at once for a given note. This assumption is beneficial as it allows us to frame our task as a boolean (true/false) classification for each given class, freeing evaluated models to focus on their prediction of one class at a time rather than allowing confusion between classes. Additionally, since all the classes do not to be predefined, this Boolean construction allows for evaluated models to use examples from the MIMIC-III based datasets as an indicator of task structure, signaling to the model that classification must occur by attending class descriptions alongside a given clinical note. Finally, this construction has the benefit that a downstream model can be queried on a class unseen in training, allowing for clinicians to pivot the exact classes they are searching if the model proves good generalizability. The template construction for each example is as follows: Input: {‘clinical_note’: {note_text}, ‘class_label’: {class_name}, ‘class_description’: {class_description}} Output: “True”/“False” for generative tasks; 1/0 for discriminative tasks Negative Sampling for Training Data In each dataset included in this work, every note is annotated according to its positive classes. We assume that if a class present in the dataset is not part of the positive classes for a given note, it can be considered a negative class. Classification tasks are known to benefit from contrastive training in which negative examples are paired with positive examples, signaling to the model that it should shift the distribution of positive and negative examples away from each other. This shift allows for better discrimination over a classification task. We employ negative-sampling over each dataset to create our training sets, sampling 3 negative examples for every positive example. In NCH-Sleep and SDOH, the negative samples are randomly selected from the set of classes within the dataset not present among the positive classes for that note. In MDACE, we make use of the hierarchical structure of ICD-10 codes to sample negative classes at varying levels of similarity to the positive classes for a given note. We sample one negative class from the entire list of ICD-10 codes, one negative class that shares the first letter in its code with a code present in the note, and one negative class that shares the first three characters with a code present in the note. This staged sampling allows the training set to create varying levels of challenge for the model. Evaluation We conduct our evaluations over the 100-note test split of NCH-Sleep. To imitate a clinical use case, we evaluate every note for all 9 MSH classes rather than just the positive class and 3 negative samples as in the training set. We do, however, employ a version of this test split with 3 negative samples to use as a validation for fine-tuned models to determine the best training epoch for final evaluation. Experiment 1: Prompting LLMs Contemporary LLMs benefit from extensive pre-training and high parameterization, allowing for in-context learning during inference [ 4 , 5 , 8 ]. This capability allows models to perform tasks without any explicit training over the target task. Performance can be adapted by changing the context provided to the model, for example, by manipulating prompt instructions, altering example context lengths, or providing task-representative examples. Models used without targeted training are pre-trained on corpuses which can be general or domain-specific (e.g. medical), potentially shifting their capabilities on a specific task. Such LLMs are generative, meaning they output sequences of text, however they calculate probabilities over each token in the process. In this section we will explore each of these properties as they relate to the classification of MSH over the NCH-Sleep test set. Runtime Parameters: We conduct each experiment on an NVIDIA H100 with 80GB of VRAM and a batch size of 1 and int-4 quantization. Models are gathered from online repository Huggingface. Methods The Prompt: We make use of the following prompt format: “”” You are a system that aids in information extraction from clinical notes. You are provided a clinical note, a class name, and a class description. You must classify if the class is present in the clinical note. Your output must be only the word True or False. Do not output anything other than the word True or False. Use the following training examples to help refine your understanding of the task: {few_shot_examples} Example to predict on: Input: {test_example} Output: “”” This format was determined through rapid evaluation using the MIMIC-III datasets over GPT-4o-mini (process and evaluation results described in Appendix A1) [ 8 ]. The line including ‘few_shot_examples’ is excluded when the number of few-shots to include is 0. Dynamic Few-Shot Prompting We select our few-shot examples through a dynamic method which seeks to identify the examples in a training set that are most similar to a given test example. Namely, we encode the input of each training example, as defined in the Task Definition, using an LM to create a pool of training embeddings. During inference we encode the input for the test example and find the closest n examples in the training embedding pool according to cosine distance, a standard metric of embedding similarity. These n examples are the ones selected to be the few-shot exemplars for the prompt, with n being an independent variable of our comparisons. To generate the embeddings, we use ‘intfloat/e5-large-v2’, an encoder developed by Microsoft using weakly supervised contrastive pre-training [ 6 ]. This model was selected for its lightweight parameter size (560M) and strong encoder performance as ranked on the Huggingface MTEB (Massive Text Embedding Benchmark) leaderboard, allowing for fast and effective encoding [ 9 ]. Since the max context window of this model is only 512 tokens, which is much smaller than most of our example clinical notes, we perform encoding by sliding the input text in chunks of 512 with a stride of 256. We aggregate the final embedding by mean-pooling over these window encodings as described in Eq 1. Where 𝑁 is the total number of windows, 𝐸 𝑖 is the embedding vector for the 𝑖 𝑡ℎ window, 𝐸 𝑝𝑜𝑜𝑙𝑒𝑑 is the final mean-pooled representation, and 𝑗 indexes the dimensions of the embeddings. Independent Variables We control for a series of independent variables which are summarized in Table 4 . Our evaluations compare each combination of these independent variables. View this table: View inline View popup Download powerpoint Table 4: Independent variables and their relevant values tested using prompt-based LLM evaluation Domain-specific vs General Domain Model We compare two LLMs which, while having similar architectures, differ in their pretraining and subsequent instruction tuning. Namely we compare ‘meta-llama/Llama-3.1-8B’ and the slightly smaller ‘BioMistral/BioMistral-7B’, where 8B and 7B refer to their parameter count [ 4 , 5 ]. Models of this parameter count were selected as they are the biggest that would fit on one 80gb GPU while maintaining high context-size. We select Llama-3.1-8B as our general model as it remains an industry standard benchmark for open-source LLMs [ 5 ]. Meanwhile, BioMistral-7B was selected because its pre-training on PubMed PMC oriented it towards clinical tasks while maintaining an otherwise similar architecture to Llama-3.1-8B [ 4 , 5 ]. Window Size While these models can handle a context length upwards of 8,000 tokens, many of our notes get close to or exceed this maximum value, especially when adding context needed by few-shot exemplars and the prompt itself. Compounding this issue, our 80Gb GPU is unable to handle the entirety of the 8,000 token context without running out of memory. As such, we instead chunk each note using sliding windows and perform classification on each chunk. This does however come with the drawback of limiting the information the model can make a prediction on. To understand the implications of context window sizes we evaluate performance over the set listed in Table 4 . For each window we have a stride that is window//2 (i.e. the window size of 1000 has a stride of 500), ensuring no subcomponent of the text is presented in an under-contextualized manner. For a given chunk, we define the classification only if the span indicating positive presence is entirely within the chunk. If the note has a positive classification but the chunk does not entirely contain the relevant span, the chunk is skipped when creating the training data as we cannot guarantee there is a positive or negative signal in the context of that chunk. If the note is negative for a given class, all chunks are included as negative examples for that class. Few-Shot Example Quantity Including examples of the task, with both inputs and expected outputs, helps to orient a model to the task. A general example can be selected to provide a high-level understanding of the task while more specifically chosen examples can be used to orient more nuanced semantic understandings of the model, as is the goal with the dynamic few-shot sampling method we employ here. It is unclear, however, how many examples are needed to assist the model, or what level performance increase these examples provide above just using the model’s pre-trained knowledge. To understand the implications of few-shot sample sizes we evaluate performance over the set listed in Table 4 . Few-shot Example Source (Training Dataset) The source of the few-shot examples has the potential to have as much, if not more, effect than the quantity of examples provided. In some clinical tasks, often there is paucity-labelled data, necessitating the use of a distant signal from tangential datasets. We explore the effect of sample sources between in-task (NCH-Sleep), out-of-task (MIMIC), and a combination of the two (Mix). These comparisons allow us to understand how necessary annotated data is for validating MSH with clinical notes when using LLMs. Accessing the Predictions Generative LLMs are decoder-only models that autoregressively generate the next token after attending to a given input context [ 4 , 5 ]. As such, they are trained to output tokens rather than classification probabilities. For our primary analyses with LLMs, we task them to generate the tokens “true” or “false” to classify MSH class presence. It is possible the LLM generates other tokens so we simply search the generation, which is limited to 10 tokens, and detect which ever token between “true” and “false” occurs first. If neither occurs, the window is skipped. Token Probability While LLMs are trained to output tokens, they decide this output by estimating a distribution over their vocabulary size (i.e. all the tokens it can consider) and sampling the most probable token [ 4 ]. This estimation is presented as a logit, a logarithm of the corresponding odds of a given token, as described in Eq. 2 . To ensure we can always gather a prediction from the LLM even when its generation may go awry, we calculate a probability of the class being true directly using the logit values for the ‘true’ and ‘false’ token as it begins to generate its response, described in Eq. 3 . It is, however, unclear how well these probabilities will map towards model confidence as this mechanism does not align with the LLM training objectives. If the mapping is effective, the probabilities can be used to create ROC curves that allow prediction systems to vary their thresholds to maximize clinically effective precision and recall. Note vs Window Level Prediction To replicate the downstream clinical use case, we conduct most of our evaluations at the note level. If a given note is split into chunks due to windowing, we aggregate the prediction over each chunk so that if even one chunk is positive, the entire note is classified as positive. This method is necessary as it is possible that most of a note would not have the information needed for a classification save for the content in one chunk, therefore the one positive case needs to supersede all negative cases. When using token probability evaluation, we simply take the maximum probability over the set of chunks that constitute a note as our final prediction probability for the note. We do, however, still present the window-level performance of our best performing system. This is because in clinical use case, if we employ a given model we can have the window level predictions help highlight chunks of the text that indicate positive presence of a given class. Given high window-level performance, this would help a clinician rapidly screen model decisions for veracity and bias. In practice, this would function as a second pass after a note had been initially classified. Results We present a complete table of results for Experiment 1 in Appendix A2. Overall Results Our top evaluated configuration was Llama-3.1-8B using a context window of 2000 characters and supplied with 10 few-shot exemplars, resulting in a precision of 0.25, recall of 0.83, and overall f1 score of 0.38. For all test configuration, the recall was higher than precision. This is an expected effect due to our evaluation technique which prioritizes a single positively classified window over all negatively classified windows for a given note. While high recall is preferred for screening tasks, the low overall F1, with little variation between all run configurations, implies that prompting LLMs is insufficient for clinical application of MSH classification. Token Probability: Looking at “true” token probabilities directly, we find the effect ofhigh-recall / low-precision is even stronger, with nearly all cases having a recall of 1 and therefore similar precisions of 0.21 as all notes incur as positive classification. We do see variance when looking at AUC however, finding the best performing AUC of 0.68 and the “top” system having an AUC of 0.64. This implies there may be a better cutoff for performance than 0.5 for the true probability, however the overall performance of the LLM is better suited for generative classification rather than by accessing token probabilities directly. This supports the hypothesis that generative systems do not learn class discrimination directly but rather fit to a token generation task. Window-Level Performance: The top system had a precision of 0.36, recall of 0.31 and F1 score of 0.334 at the window level. This is not a high performance and, when paired with the middling performance at the note-level, indicates poor clinical applicability for informed decision making. Namely, the windows classifications are not likely informative for a clinician seeking to quickly scan a note for relevant text. Effect of Independent Variables For each independent variable we compare the constituent values by gathering the average F1 score among all runs sharing that value. We present our results for the summarized effect of each independent variable in Table 5 . View this table: View inline View popup Table 5: Comparison of the effect of each independent variable when prompting LLMs for classification. The results within each independent variable are sorted in descending order of F1 score. Model: We see no substantial difference between our two model types despite BioMistral-7B being trained on biomedical literature (PubMed PMC) [ 4 ]. This lack of performance may indicate the task is challenging enough that neither model’s difference in capabilities are able to be apparent. Another reason may be that PubMed PMC, being medical literature, is not a good proxy for relatively unstructured clinical notes. Window: We find little correlation between window size and performance; we cannot say higher window sizes incur better or worse performance. Rather we see the medial window sizes perform best, with 6000 as the top performer on average, while the most extreme values of 10,000 and 1,000 are the worst performers. This may be because the model has to balance having enough context to make an informed decision, while not using so much context that few-shot quantities must be lessened or context truncation must incur. Few Shot Quantity: We find larger few-shot quantities correlate to better performance, with the 7 and 10 few-shot quantities performing best, albeit marginally. This may indicate that performance is best promoted when the model is sufficiently able to learn the task using few-shot exemplars. This finding is further supported by zero-shot performance, which proved the worst among the runs. All of our evidence thus indicates that model pre-training knowledge and prompt instructions alone are insufficient to properly encode the task. Training Data Source: We find using NCH-Sleep data is the best informant for the task. This is to be expected as the training sample is meant to be representative of the examples found in the NCH-Sleep test set. Suprisingly, we find that Mix underperforms NCH-Sleep, indicating that providing MIMIC-examples can distract from good performance rather than aid it. This hypothesis is fortified when we see zero-shot performance outperforms MIMIC few-shot performance on average (F1 of 0.3521 vs F1 of 0.3502). View this table: View inline View popup Download powerpoint Table 1a: Comparison of the effect of select independent variables when prompting LLMs for classification. The results within each independent variable are sorted in descending order of F1 score. Experiment 2: Fine-tuned LLMs The results from Experiment 1 show that higher few-shot counts correlate to better performance. This may indicate that one of the toughest challenges for the LLM is understanding the structure of the task itself. For this reason, fine-tuning of the LLM may be fruitful as this would allow the LLM to see every training example in our dataset and better orient towards the task. Methods Model We take the best performing model from Experiment 1, Llama-3.1-8B and fine-tune it over each training data source described in Table 4 . A comparison over these training datasets allows us to understand the need and effect of having in-task data for training. We continue to evaluate on the 100 note NCH-Sleep test split. Window Size Experiment 1 results showed a minimal correlation between window size and performance; therefore we use the highest window size that reliably fits within the GPU VRAM limit of our NVIDIA H100, 8000 characters. We choose this value instead of the top performing value of 6000 as few-shot examples are not included during fine-tuning and therefore more context can be provided for the example to be tested. Training Parameters We train each model using an AdamW optimizer, cross entropy loss, and for up to 48 hours or 50 epochs, saving the epoch with the best performance over the validation set. Additionally, we use LORA, PEFT, and int-4 quantization for more efficient fine-tuning [ 10 , 11 ]. Results We present results for our fine-tuned LLMs in Table 6 . Once again, we find including information from NCH-Sleep in our training set to be effective, however the Mix dataset very slightly outperforms the NCH-Sleep dataset alone, potentially indicating some benefit to the added MIMIC examples. We do still find the NCH-Sleep trained model to have the best token probability AUC, implying better use in a risk stratification application. View this table: View inline View popup Table 6: Results of Llama-3.1-8B fine-tuned over 3 training sets and evaluated on NCH-Sleep test We additionally find the fine-tuned models had an increased risk of hallucinating invalid answers. In Table 6 , we note “Skipped Windows” and “Skipped Notes” to show cases where the model did not generate “true” or “false”. The frequency of these failures increased from the frequency of failures by prompting methods (which was limited to a few windows and configurations). Hallucinations, which “Token Prob AUC” is robust to, shows the benefit of creating a discriminative prediction rather than a generative one. Even so, when we mark these examples as incorrect, we find the F1 of each model is about 3 points better than the best prompting configuration, showing consistent— albeit small— gains over the prompting approach. Experiment 3: Fine-tuned Classifier Generative LLM’s are trained to produce tokens rather than estimate classification probabilities directly. In this case, the generation of a “true” or “false” token only serves as a proxy for our task. Likewise, much of the parameterization of LLMs is to promote better generations and generalizability but may not be necessary for effective discrimination of classes, especially given availability of fine-tuning data. In this experiment we fine-tune an encoder model to estimate classification probabilities directly. The benefits of building an effective encoder-based discriminative system include faster runtime, lower VRAM requirements allowing for larger context size, faster training convergence, and direct calculation of ROC for downstream risk stratification [ 7 ]. The potential downsides include worse generalizability and increased reliance on the quality of training data. Methods Model In this experiment we fine-tune ‘answerdotai/ModernBERT-basè, a bidirectional encoder that expands on the original BERT work through the implementation of contemporary model optimizations and training over 2 trillion tokens [ 7 ]. This model boasts memory and runtime efficiency alongside a sequence length limit of 8192 tokens [ 7 ]. Window Size ModernBERT’s high token limit and VRAM efficiency allows us to avoid chunking notes into windows and rather classify each note in its entirety. Therefore we forgo the aggregation of chunk evaluations that leads to higher recall in our LLM based experiments. Prediction Calculation Rather than predict the token “true” or “false”, we encode each classification as the integer 1 (true) and 0 (false). The model predicts the probability of label 0 or 1. We report back the positive label probability by computing the softmax over these two-class probabilities as described in Eq. 4 . Training parameters We train our models using NVIDIA A100 GPUs with 40GB of VRAM. We use the AdamW optimizer and binary cross entropy loss. We train for up to 200 epochs and 24 hours, selecting the best performing epoch over the 100 note NCH-Sleep validation set We again train over the 3 training datasets described in Table 4 . Results We present our results for our fine-tuned classifiers in Table 7 . The order of training data effect in this work follows the results of our fine-tuned Llama-3.1-8B models in Experiment 2, with Mix data performing the best followed by NCH-Sleep and then MIMIC. The disparities in performance between these datasets are much larger however. We see the Mix trained model has the best F1 and AUC score of any system we have seen so far, while the NCH-Sleep trained model falls dramatically in performance by nearly 15 points F1 and 5 points AUC. Meanwhile the MIMIC trained model performs abysmally with an F1 of 0.07, a 35 point drop in performance from the best trained version. View this table: View inline View popup Download powerpoint Table 7: Results of ModernBERT fine-tuned over 3 training sets and evaluated on NCH-Sleep test View this table: View inline View popup Download powerpoint Table 8: Performance (F1) of the top performing models, and their best non NCH-Sleep trained versions, when stratified by gender View this table: View inline View popup Table 9: Performance (F1) of the top performing models, and their best non NCH-Sleep trained versions, when stratified by race View this table: View inline View popup Download powerpoint Table 10: Performance (F1) of the top performing models, and their best non NCH-Sleep trained versions, when stratified by department This higher variation based on training data makes sense given the architecture of fine-tuned ModernBERT. Compared to Llama-3.1-8B, ModernBERT has 1/16 th the parameters, which allow for faster fitting to training data during fine-tuning. As such, the need for in-task data (i.e. Mix and NCH-Sleep) is critical to orient the model to the task as it cannot fall back on as much pre-trained parameterization for generalizable performance. Likewise, the increased amount training data in Mix over NCH-Sleep likely helps the model orient itself towards the structure of the task. The Mix trained model is likely able to outperform our best fine-tuned Llama-3.1-8B because its training signal can more dramatically adjust a higher percentage of the parameters in ModernBERT than in the bigger Llama-3.1-8B. In other words, Llama-3.1-8B’s size gives it a inertia that both limits how poor it can do due to generalizability, and how well it can do due to the training signal being more diffused over all of its parameters. These results suggest that if fine-tuning data is available, constructing an encoder-only classifier is likely more effective for MSH classification than fine-tuning an LLM. Experiment 4: Understanding Bias Having a comprehensive understanding of model bias is critical for any system that is oriented towards clinical application. To evaluate the effects of bias in our best performing systems, we stratify performance by demographics, department, and class. This evaluation will indicate the limitations of these systems as they interact with various facets of real-world application. Methods We evaluate the best performing configuration from experiment 1, 2, and 3, all of which include NCH-Sleep training data. We additionally compare the performance of each configuration when using only MIMIC training data. This additional comparison allows us to see if there is a shift in bias when including our internal data, indicating the presence of stronger or weaker bias signals in NCH-Sleep. Demographics Stratification It is possible there are disparities in data representation and model performance correlated with population demographics. We evaluate our best models stratified by gender and stratified by race. Department Stratification Departments vary in how they present author clinical notes, especially due to different norms and EHR templates in use. We evaluate our best models stratified by gender and stratified by department. Performance by Class Certain MSH classes may be more difficult to classify than others. This may especially shift depending what training data the model has seen. We evaluate how our best model, namely ModernBERT, performs over each class both with and without NCH-Sleep training data. We focus on ModernBERT for this comparison as the ROC curves allow for useful insights into model performance on classes and we find generally these results hold for other models, albeit to a varying magnitude. Additionally, we explore the average span length identified by our expert annotators as positive indicators for each class. It is possible longer span lengths will correlate with increased class representation complexity and therefore worse classification performance. Results Performance by Gender Our results show notable gender disparities in model performance, particularly with discriminative classifiers trained on NCH-Sleep data, achieving substantially higher F1 scores for female-associated notes (0.509) compared to males (0.357). This gap suggests that clinical notes for females contain more explicit or consistent mentions of sleep-related concepts. Conversely, notes for male patients may exhibit greater variability or ambiguity, posing challenges for accurate classification. Performance by Race We observed consistent racial disparities across models, with substantially better performance for White patients (F1 ∼0.40–0.50) relative to Latino and African American groups (F1 ∼0.25–0.35). The improved classification results for White patients likely reflect higher representational alignment between training data and linguistic patterns commonly documented for these patients. Lower performance for minority groups highlights critical gaps due to linguistic heterogeneity or underrepresentation in clinical data. Performance by Department Departmental stratification reveals substantial variability, with Behavioral Health notes consistently yielding higher performance (F1 ∼0.45–0.49), likely due to structured and standardized sleep-related documentation practices. In contrast, Primary Care exhibits lower and more variable performance (F1 ∼0.27–0.32), potentially reflecting diverse and less standardized note-taking styles. Performance within smaller departments like School-Based care fluctuates dramatically, emphasizing the impact of training data size and specificity on model robustness. Performance by Class In Figure 1 we present the ROC curves, and associated AUCs, for our best predictor, ModernBERT trained on Mix, as well as for ModernBERT trained on MIMIC. These curves are presented as an overall performance as well as stratified by MSH class. Download figure Open in new tab Figure 1: Performance of ModernBERT on NCH-Sleep test, trained on MIX and MIMIC, stratified by MSH Class. ROC curve and AUC ROC presented We see in ModernBERT trained on MIMIC, the performance is low for nearly all classes except for sleep timing, behavior, and duration. This implies that these classes are able to build some quantity of a representation using solely MIMIC data. Meanwhile, in ModernBERT trained on Mix, we see these same classes are uplifted while every other class is boosted dramatically. This shows that our NCH-Sleep dataset helps to provide critical information for prediction of the other 6 MSH classes, while also building a better representation over sleep timing, behavior, and duration. Overall, we see sleep disorder, sleep efficiency, and daytime sleepiness are the hardest behaviors to capture with our classifier. When comparing the average sequence lengths of the indicative spans of these classes, presented in Table 11 , we can potentially posit that daytime sleepiness, and especially sleep efficiency, may be harder to classify due to their longer representations in the text. This relationship is tenuous however since sleep disorder has one of the shortest average span lengths and performs poorly while sleep intervention, with the longest average span length, is classified with high accuracy. Likely there are other properties of these classes that indicate difficult representations, perhaps due to irregular lexical choice across notes. We leave this investigation for future work. View this table: View inline View popup Download powerpoint Table 11: Average span length and standard deviation (characters) per MSH class Experiment 5: Latency In clinical application, runtime of a classification system is critical, especially in a screening use-case. If a model takes too much time to run on each note, it is unlikely to be feasibly deployed over an entire healthcare system reviewing all internal notes. A major motivation for creating an AI based classification system is to alleviated the fractured documentation of sleep mentions within clinical practice, making this scalability crucial. Likewise, improved scalability promotes improved transferability into other hospital systems. Methods For each of the following methods we report the average time to provide a prediction for all 100 clinical notes in the NCH-Sleep test set. Dynamic Few-Shot In this circumstance, we include the latency of encoding the inference example and dynamically gathering few-shot examples for prompt construction. This is paired with the actual model generation time. Fine-tuned LLM In this circumstance only the latency of the model generation is reported. Fine-tuned Classifier In this circumstance, only the latency of the model prediction is reported. Results We present our runtime latency results in Table 12 . In general, we find a magnitude of difference between the best model of each experiment. Prompting takes the longest, likely due to the need for dynamic few-shot exemplar retrieval using a separate encoder. Fine-tuned Llama-3.1-8B is about 25x faster, showing the large gains to be made when retrieving few-shot exemplars is not a necessary part of inference. Finally, the ModernBERT classifier is even 10x faster than fine-tuned Llama-3.1-8B, and therefore about 250x faster than prompted Llama-3.1-8B. This speed up for ModernBERT is to be expected both because it is a lighter weight model that can directly compute faster, and because it can handle the entire clinical note in its context window at once instead of needing to using a sliding window method. View this table: View inline View popup Download powerpoint Table 12: Runtime of top performing model from each experiment. Evaluated over 100 NCH-Sleep test notes. These results further show that a fine-tuned encoder-based classifier like ModernBERT can be better suited for specific clinical information extraction tasks than LLMs. This point is especially notable in a live note screening use case where a performative system is deployed at every level of hospital infrastructure to consolidate MSH and provide a comprehensive representation and handling of sleep data. Not only would an LLM system potentially be too slow for this screening task, it performs less accurately when provided with only a small, annotated dataset, and requires more specialty computing equipment like state-of-the-art GPUs. Conclusion Overall, our findings suggest the best system for MSH classification is either a fine-tuned encoder-based classifier with a combination of related-data and task specific data or a prompted LLM with as many few-shot exemplars as possible. The decision should be made based on if training data is available or not. Fine-tuning an LLM is a less effective use of training data than finetuning a classifier but prompting an LLM will do when no data is available at all. Future Direction Other Model Architectures In this work we primarily compare transformer architectures with two decoder-only LLMs and one encoder-only LLM. Several other model architectures have shown promise in recent works, including state-space-models (SSMs) such as Mamba, and Mixture-of-Expert (MoE) models such as Mixtral [ 12 , 13 ]. SSMs deviate from standard transformer architecture with claims of especially fast runtime, providing a potential alternative to our encoder-only classifier implemented by ModernBERT, our fastest solution according to latency evaluation [ 12 ]. Meanwhile, MoE systems tout improved model training on specific tasks, due to its expert-gating mechanism, and improved runtime relative to parameter size, providing an alternative to our LLM approaches [ 13 ]. Future Direction: Chain-of-thought Prompting Contemporary prompting techniques promote reasoning over a task using LLM in-context reasoning capabilities. This is generally employed by having the model generate intermediary thoughts in a sequence leading up to the solution, referred to as Chain-of-Thought (CoT). This method may be effective for our complex MSH classification task but does not have a straightforward implementation. One possible option is to have the LLM first identify the spans, as per our annotations, by searching for relevant words for the provided class before then making a prediction. Our preliminary efforts in this direction, however, ran into issues as many times the LLM would wander off the span-extraction task and start hallucinating. The classification generated from a hallucinated CoT process, if it was generated at all, was often incorrect. We found in 20% of cases no final classification was provided and often performance was poor when it was. It is possible that newer LLMs may perform better on this task on our dataset due to their improved efficiency. Data Availability The data for this study is not available Appendix A1) Validating Dynamic Few-Shot Prompting We initially validate our prompting methods using GPT-4o-mini and MIMIC data [ 8 ]. The use of GPT-4o-mini allows for rapid testing due to API calls being faster than local compute. Additionally, GPT-4o-mini performance is well verified and considered high overall, allowing isolation of independent variables related to the prompt [ 8 ]. We use the MIMIC dataset (combination of MDACE and SDOH as mentioned in Table 4 ) because the notes are deidentified and can be sent to an API. Additionally, MIMIC has been tested in other works and provides a benchmark of replicability. Method We evaluate GPT-4o-mini on a randomly selected sample of 1000 examples from the MIMIC dataset with another 1000 held out as our training set. We perform dynamic few-shot prompting, as described in Experiment 1, using openAI’s text-embedding-3-large as our encoder. We encode our training set and gather n few-shot examples using cosine distance from the test-example embedding. GPT-4o-mini has a large context window that allows us to avoid windowing. Results In this stage we also experimented with other prompts, iteratively improving until we arrived upon the prompt presented in experiment 1. One of the major improvements came from asking the model to classify “true” / “false” rather than 1 / 0. The numbers are only reported for the final “true” / “false” version. Table A1 presents our preliminary results on validating few-shot prompting using MIMIC data and gpt-4o-mini. We see markedly high performance View this table: View inline View popup Download powerpoint Table A1 Evaluation of dynamic few-shot prompting using gpt-4o-mini and notes from our MIMIC training set A2) Complete Prompting Results View this table: View inline View popup Table A2: Complete Prompting Results from Experiment 1 References [1]. ↵ Hua Cheng , Rana Jafari , April Russell , Russell Klopfer , Edmond Lu , Benjamin Striner , and Matthew Gormley . 2023 . MDACE: MIMIC Documents Annotated with Code Evidence . In Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers) , pages 7534 – 7550 , Toronto, Canada. Association for Computational Linguistics. [2]. ↵ Guevara , M. , Chen , S. , Thomas , S. , & Bitterman , D . ( 2024 ). Annotation dataset of social determinants of health from MIMIC-III Clinical Care Database (version 1.0.1) . PhysioNet . doi: 10.13026/zsgv-8w31 . OpenUrl CrossRef [3]. ↵ Johnson , A. , Pollard , T. , Shen , L. et al. MIMIC-III, a freely accessible critical care database . Sci Data 3 , 160035 ( 2016 ). doi: 10.1038/sdata.2016.35 OpenUrl CrossRef PubMed [4]. ↵ Yanis Labrak , Adrien Bazoge , Emmanuel Morin , Pierre-Antoine Gourraud , Mickael Rouvier , & Richard Dufour . ( 2024 ). BioMistral: A Collection of Open-Source Pretrained Large Language Models for Medical Domains . [5]. ↵ Aaron Grattafiori , et al. ( 2024 ). The Llama 3 Herd of Models . [6]. ↵ Wang , L. , Yang , N. , Huang , X. , Jiao , B. , Yang , L. , Jiang , D. , Majumder , R. , & Wei , F . ( 2022 ). Text Embeddings by Weakly-Supervised Contrastive Pre-training . arXiv preprint arXiv:2212.03533 . [7]. ↵ Benjamin Warner , Antoine Chaffin , Benjamin Clavié , Orion Weller , Oskar Hallström , Said Taghadouini , Alexis Gallagher , Raja Biswas , Faisal Ladhak , Tom Aarsen , Nathan Cooper , Griffin Adams , Jeremy Howard , & Iacopo Poli . ( 2024 ). Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast , Memory Efficient, and Long Context Finetuning and Inference . [8]. ↵ OpenAI , Aaron Hurst , Adam Lerer , & Adam P. Goucher , et al. ( 2024 ). GPT-4o System Card . [9]. ↵ Muennighoff , N. , Tazi , N. , Magne , L. , & Reimers , N. ( 2022 ). MTEB: Massive Text Embedding Benchmark . arXiv preprint arXiv:2210.07316. [10]. ↵ Edward J. Hu , Yelong Shen , Phillip Wallis , Zeyuan Allen-Zhu , Yuanzhi Li , Shean Wang , Lu Wang , & Weizhu Chen . ( 2021 ). LoRA: Low-Rank Adaptation of Large Language Models . [11]. ↵ Sourab Mangrulkar , Sylvain Gugger , Lysandre Debut , Younes Belkada , Sayak Paul , & Benjamin Bossan . ( 2022 ). PEFT: State-of-the-art Parameter-Efficient Fine-Tuning methods . [12]. ↵ Albert Gu , & Tri Dao . ( 2024 ). Mamba: Linear-Time Sequence Modeling with Selective State Spaces . [13]. ↵ Albert Q. Jiang , Alexandre Sablayrolles , Antoine Roux , Arthur Mensch , Blanche Savary , Chris Bamford , Devendra Singh Chaplot , Diego de las Casas , Emma Bou Hanna , Florian Bressand , Gianna Lengyel , Guillaume Bour , Guillaume Lample , Lélio Renard Lavaud , Lucile Saulnier , Marie-Anne Lachaux , Pierre Stock , Sandeep Subramanian , Sophia Yang , Szymon Antoniak , Teven Le Scao , Théophile Gervet , Thibaut Lavril , Thomas Wang , Timothée Lacroix , & William El Sayed . ( 2024 ). Mixtral of Experts . [14]. ↵ Horner , Matthew , Noor Abul-el-rub , Mary Mays , and Diego Mazzotti . 2022 . “ 0610 Development of a Rule-Based Text Mining Algorithm to Identify Sleep Complaints in Primary Care Progress Notes .” Sleep 45 ( Supplement_1 ): A267 – 68 . doi: 10.1093/sleep/zsac079.607 . OpenUrl CrossRef [15]. ↵ Sivarajkumar , Sonish , Thomas Yu Chow Tam , Haneef Ahamed Mohammad , Samuel Viggiano , David Oniani , Shyam Visweswaran , and Yanshan Wang . 2024 . “ Extraction of Sleep Information from Clinical Notes of Alzheimer’s Disease Patients Using Natural Language Processing .” Journal of the American Medical Informatics Association 31 ( 10 ): 2217 – 27 . doi: 10.1093/jamia/ocae177 . OpenUrl CrossRef PubMed [16]. ↵ Li , Z. , Hu , Y. , Lane , S. , Selek , S. , Shahani , L. , Machado-Vieira , R. , … Huang , M. ( 2024 ). Suicide Phenotyping from Clinical Notes in Safety-Net Psychiatric Hospital Using Multi-Label Classification with Pre-Trained Language Models . arXiv [Cs.CL]. Retrieved from http://arxiv.org/abs/2409.18878 [17]. ↵ Sirrianni , J. W. , Calloway , A. , Hussain , S.-A. , Chisolm , D. , Kelleher , K. , Seixas , A. , Liu , H. , Bartlett , C. , & Davenport , M. A . ( 2025 ). Development of a Rule-Based Natural Language Processing Algorithm to Extract Sleep Information in Pediatric Primary Care Patients with a Sleep Diagnosis . medRxiv . doi: 10.1101/2025.05.31.25328640 OpenUrl Abstract / FREE Full Text View the discussion thread. Back to top Previous Next Posted June 05, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Empirical Review of LLM-driven Classification of Multidimensional Sleep Health Mentions from Free-Text Clinical Notes Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Empirical Review of LLM-driven Classification of Multidimensional Sleep Health Mentions from Free-Text Clinical Notes Syed-Amad Hussain , Ariana Calloway , Joseph Sirrianni , Eric Fosler-Lussier , Mattina Davenport medRxiv 2025.06.04.25328983; doi: https://doi.org/10.1101/2025.06.04.25328983 Share This Article: Copy Citation Tools Empirical Review of LLM-driven Classification of Multidimensional Sleep Health Mentions from Free-Text Clinical Notes Syed-Amad Hussain , Ariana Calloway , Joseph Sirrianni , Eric Fosler-Lussier , Mattina Davenport medRxiv 2025.06.04.25328983; doi: https://doi.org/10.1101/2025.06.04.25328983 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Pediatrics Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15229) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6600) Geriatric Medicine (668) Health Economics (997) Health Informatics (4536) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (541) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3332) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9232) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a0089ead8f351640',t:'MTc3OTU4NzExNQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();