SmokeBERT: A BERT-based Model for Quantitative Smoking History Extraction from Clinical Narratives to Improve Lung Cancer Screening

SmokeBERT: A BERT-based Model for Quantitative Smoking History Extraction from Clinical Narratives to Improve Lung Cancer Screening | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search SmokeBERT: A BERT-based Model for Quantitative Smoking History Extraction from Clinical Narratives to Improve Lung Cancer Screening Yiming Xue , Yunzheng Zhu , Luoting Zhuang , YongKyung Oh , Ricky Taira , Denise R. Aberle , Ashley E. Prosper , William Hsu , Yannan Lin doi: https://doi.org/10.1101/2025.06.18.25329870 Yiming Xue 1 Department of Statistics & Data Science, University of California , Los Angeles, CA, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Yunzheng Zhu 2 Medical & Imaging Informatics, Department of Radiological Sciences, David Geffen School of Medicine at UCLA , Los Angeles, CA, USA MS Find this author on Google Scholar Find this author on PubMed Search for this author on this site Luoting Zhuang 2 Medical & Imaging Informatics, Department of Radiological Sciences, David Geffen School of Medicine at UCLA , Los Angeles, CA, USA MS Find this author on Google Scholar Find this author on PubMed Search for this author on this site YongKyung Oh 2 Medical & Imaging Informatics, Department of Radiological Sciences, David Geffen School of Medicine at UCLA , Los Angeles, CA, USA PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ricky Taira 2 Medical & Imaging Informatics, Department of Radiological Sciences, David Geffen School of Medicine at UCLA , Los Angeles, CA, USA PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Denise R. Aberle 2 Medical & Imaging Informatics, Department of Radiological Sciences, David Geffen School of Medicine at UCLA , Los Angeles, CA, USA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ashley E. Prosper 2 Medical & Imaging Informatics, Department of Radiological Sciences, David Geffen School of Medicine at UCLA , Los Angeles, CA, USA MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site William Hsu 2 Medical & Imaging Informatics, Department of Radiological Sciences, David Geffen School of Medicine at UCLA , Los Angeles, CA, USA PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Yannan Lin 2 Medical & Imaging Informatics, Department of Radiological Sciences, David Geffen School of Medicine at UCLA , Los Angeles, CA, USA MD, MPH, PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: yannanlin{at}mednet.ucla.edu Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Tobacco use is a critical risk factor for diseases such as cancer and cardiovascular disorders. While electronic health records can capture categorical smoking statuses accurately, granular quantitative details, such as pack years and years since quitting, are often embedded in clinical narratives. This information is crucial for assessing disease risk and determining eligibility for lung cancer screening (LCS). Existing natural language processing (NLP) tools excelled at identifying smoking statuses but struggled with extracting detailed quantitative data. To address this, we developed SmokeBERT, a fine-tuned BERT-based model optimized for extracting detailed smoking histories. Evaluations against a state-of-the-art rule-based NLP model demonstrated its superior performance on F1 scores (0.97 vs. 0.88 on the hold-out test set) and identification of LCS-eligible patients (e.g., 98% vs. 60% for ≥20 pack years). Future work includes creating a multilingual, language-agnostic version of SmokeBERT by incorporating datasets in multiple languages, exploring ensemble methods, and testing on larger datasets. Introduction Quantitative smoking history (e.g., pack years and years since quitting) is crucial for various clinical applications, particularly lung cancer screening (LCS). The 2021 United States Preventive Services Task Force (USPSTF) recommended annual low-dose computed tomography screening for lung cancer among individuals with at least a 20 pack-year smoking history who currently smoke or have quit within the past 15 years. 1 Accurate smoking information is essential for determining eligibility for LCS. Despite over a decade of clinical LCS programs in the United States following the 2013 USPSTF recommendations, the adoption rate of LCS among eligible individuals remains strikingly low, with a national average of 16% in 2022. 2 To facilitate LCS activities, the National Committee for Quality Assurance (NCQA) is developing Healthcare Effectiveness Data and Information (HEDIS®) measures for LCS and Tobacco Use Screening and Cessation. 3 , 4 To support the implementation of these quality measures, there is an urgent need for tools capable of extracting quantitative smoking history from electronic medical records (EHRs). Studies have shown that unstructured, free-text clinical narratives in EHRs often serve as a more reliable source of information compared to structured data fields. 5 However, the extraction of this information necessitates the use of natural language processing (NLP) tools. 6 Existing natural language processing (NLP) tools like cTAKES 7 and CLAMP 8 have proven effective in identifying broad smoking statuses such as “current smoker” or “former smoker”, but they face significant challenges when it comes to extracting detailed quantitative information, such as pack-years and years since quitting. The state-of-the-art (SOTA) model for extracting information from clinical narratives is a rule-based NLP model developed using clinical notes from a single institution. 9 Although the rule-based NLP model can achieve good performance of both F1-score (0.94 to 0.96) and recall (0.94 to 0.96) on in-house test data, the model lacks generalizability on new datasets, where physicians may have different writing styles for documenting quantitative smoking history. Moreover, the current SOTA model captures some quantitative smoking variables (i.e., pack years, packs per day, years smoked, and years since quitting) but is limited in scope, missing important variables that are slightly different from the essential ones, (e.g., cigarettes per day and quit in the year). While new variables can be accommodated by adding additional rules, this approach is not scalable, as the number of rules needed can become unmanageable, resulting in increased development time and potential conflicts among the rules. Bidirectional Encoder Representations from Transformers (BERT)-based models have become the baseline for many NLP tasks due to their focus on the bidirectional context within the text, making them particularly effective for domain-specific tasks like clinical named entity recognition (NER). Compared to transformers, Generative Pre-trained Transformers (GPTs), or other large language models, selected BERT models excel in extracting smoking-related information with precision and contextual understanding tailored to healthcare and biomedical fields. These models benefit from domain-specific pretraining, which has been shown to offer significant improvements over continual adaptation of general-purpose language models. 10 Additionally, conducting an additional stage of domain-adaptive pretraining has demonstrated performance gains across both high-and low-resource settings. 11 By leveraging models like ClinicalBERT and BioBERT, which have already undergone this domain-specific adaptatio , we can achieve improved representation learning. We aim to develop generalizable and scalable models to extract clinical narratives that specify patient quantitative smoking history, leveraging BERT-based models 12 on a multi-institutional dataset. We hypothesize that BERT-based models outperform the SOTA model in the F1 score for both NER and clinical tasks. Methods This study was approved by the Institutional Review Board at the University of California, Los Angeles (UCLA). Datasets Our study was conducted on two datasets: the smoking history dataset and the MIMIC-III dataset. The smoking history dataset consists of 3,261 sentences describing smoking history from clinical reports. A portion of the smoking history dataset was derived from the i2b2 challenge 13 , which includes 502 de-identified discharg summaries sourced from the Partners HealthCare system 14 . The remaining data were extracted from UCLA ophthalmology notes and radiology reports using keyword matching based on lexical variants of ‘smoke’ (e.g., smoking, smoked, smoker), ‘tobacco’, ‘cigarette’, and “pack-year”. In addition, MIMIC-III contains de-identified health records from over 58,000 hospital admissions, including approximately 2 million clinical notes, demographics, lab results, and more, widely used for healthcare analytics such as mortality prediction 15 and blood pressure estimation 16 . A subset of 3,191 sentences was collected using keyword matching with a similar set of keywords containing lexical variants of ‘smoke’, ‘tobacco’, ‘cigarette’, and ‘pack-year’. Sentences from the MIMIC-III subset were also used for the clinical tasks to identify individuals eligible for LCS. Table 1 lists the number of entities in each dataset. View this table: View inline View popup Download powerpoint Table 1. Number of entities in the smoking history and MIMIC-III Datasets. Data Annotations Six variables were manually annotated on both datasets:”packs per day”,”cigarettes per day”,”years smoked”,”pack years”, “years since quitting”, and “quit in the year” using the NER Annotator for Spacy annotation tool. 17 Figure 1 illustrates one example of the annotation. Two readers (Y.L. and Y.X.) annotated 300 randomly selected sentences from the smoking history dataset. Inter-rater agreement was described in the Supplemental Methods . Excellent reliability (two-way mixed effects model, single rater absolute intraclass correlation coefficient betwee 0.97 and 1) was achieved for all smoking variables ( Table S1 in the Supplement). Discrepancies were resolved before one of the annotators (Y.L.) completed the remaining annotations. Download figure Open in new tab Figure 1. Manual annotation of smoking history from clinical narratives. Baseline Model Our baseline model is a rule-based two-stage approach, which is the SOTA model for quantitative smoking history extraction. The first stage takes sentences as inputs and outputs the lexicons using regular expressions, which are subsequently used to define high-level rules in the second stage. 9 Pre-trained BERT Models Pre-trained BERT models utilize a transformer architecture and are pre-trained on a large corpus with two objectives: masked language modeling and next-sentence prediction. 12 They are also fine-tuned for specific downstream tasks, making them highly adaptable for extracting nuanced information, such as quantitative smoking history from clinical narratives in diverse datasets. We investigated four variants of BERT developed for different clinical uses: Bert-Base-Uncased 12 : The original BERT model was introduced by Devlin et al., 2019. It is pre-trained on a large corpus of English text, including BooksCorpus and Wikipedia, using uncased tokenization (ignorin capitalization). The model serves as a general-purpose baseline for various NLP tasks and can be fine-tune for specific applications. BioBERT 18 : A BERT model pre-trained on large-scale biomedical corpora such as PubMed and PMC, designed for tasks like biomedical NER and relation extraction. ClinicalBERT 19 : A BERT variant fine-tuned on clinical text from EHRs, focusing on clinical NLP tasks like patient mortality prediction and medical event extraction. MedBERT 20 : Designed for structured healthcare data, such as medical codes and records, this model aims at predictive analytics and healthcare-specific NLP tasks. Experiment Pipeline The pipeline of quantitative smoking history extraction consists of two steps: selection of a fine-tuned BERT-based model and model evaluation ( Figure 2 ). SmokeBERT was developed in Step 1 and evaluated against the SOTA (rule-based) model on two tasks: NER and clinical tasks such as LCS eligibility. (Step 2). Download figure Open in new tab Figure 2. Overview of NLP pipeline. Abbreviations: BERT: Bidirectional Encoder Representations from Transformers, NER: named entity recognition, MIMIC: the Medical Information Mart for Intensive Care, LCS: lun cancer screening. Step 1. Selection of Finetuned BERT-Based Model SmokeBERT was developed after a 2-stage training, cross-validation training, and entire dataset retraining. In cross-validation training, we fine-tuned four pre-trained BERT-based models—BERT, BioBERT, ClinicalBERT, and MedBERT— using the annotated smoking history dataset. The data was split into 90% training and 10% testing, with the training portion further utilizing a 5-fold cross-validation approach. Fine-tuning was performed to optimize the model for extracting smoking-related variables. The model with the highest overall micro-averaged F1 score was selected as the final BERT variant. The overall F1 score was computed by combining results across folds and evaluated for six smoking-related variables. The BERT variant with the highest F1 score for more than 50 % of the variables (> 3 out of 6) was selected. This model was then retrained on the entire smoking history training dataset (90%) to create the final version of SmokeBERT, leveraging all available training data for improved accuracy. Step 2. Model Evaluation The evaluation pipeline involved testing SmokeBERT on both NER and LCS eligibility tasks. The hold-out 10% smoking history test set and the external MIMIC-III dataset were used for evaluating the performance of NER tasks on metrics such as micro-averaged precision, recall, and F1 score of each variable and the overall micro-averaged results. Additionally, the model’s output, consisting of structured tabular files with each sentence as a row and each smoking-related variable as a column, was evaluated for clinical applications. Specifically, tasks such as extracting pack years and years since quitting for determining LCS eligibility were assessed by calculating the proportion of correctly extracted variables against the total occurrences. For benchmarking, the same evaluation was applied to the SOTA rule-based model. Evaluation Metric We evaluated smoking variable extraction at the sentence level using a two-step approach. First, performance (precision, recall, F1 score) was assessed individually for each smoking-related variable. A prediction is considered correct if the extracted value matches the ground truth annotations within the same sentence. For a single variable, if multiple conflicting values are present in a sentence—an inherently incorrect scenario since we assumed one sentence represents one patient—the extraction is considered correct only if all occurrences were captured. Second, an overall evaluation was performed by aggregating the results across all variables using a micro-averaged metric. This approach combined true positives (TPs), false positives (FPs), and false negatives (FNs) for all variables. The evaluation metrics were defined as follows: TP: The sentence contains a specific smoking data element, and the model correctly extracts it. FP: The sentence does not contain a specific smoking data element, but the model extracts some information for that specific data element. FN: The sentence contains a specific smoking data element, but the model extracts nothing. Experiment Setup The implementation of models leverages the Hugging Face Transformers library 21 . Pre-trained BERT-based models, including MedBERT, BioBERT, and ClinicalBERT, were fine-tuned from BERT on domain-specific datasets to address our NER task. Clinical notes were preprocessed using NLTK 22 for sentence tokenization, custom scripts were developed to label tokens for smoking-related variables. The training pipeline used the Hugging Face Trainer API for 10 epochs on the AdamW optimizer with a learning rate of 2e-5 and weight decay of 0.01. The number of epochs (10) was determined based on the performance of the F1 score from preliminary experiments, where the model began to converge after 5 epochs and stabilized around the 10th epoch. During training, early stopping was applied and set at 3 epochs if the F1 score did not improve. Evaluation metrics, including precision, recall, and F1 score, were calculated using the Seqeval package. 23 The SmokeBERT was implemented in Python 3.5 using PyTorch (version 0.4.1). The fine-tuning and testing of the BERT-based models were conducted on one NVIDIA Quadro RTX 8000 with onboard GPU memory. The evaluation of the SOTA model was conducted on the CPU. Results SmokeBERT Table 2 summarizes the performance of the four fine-tuned models (BERT, BioBERT, MedBERT, ClinicalBERT) across six variables and their overall results for the 5-fold validation sets during model training. For the evaluation of this and the following table, we primarily focus on F1 scores as a primary metric to capture the models’ ability to accurately and comprehensively extract smoking-related data. Notably, recall follows a similar trend to the F1 score. ClinicalBERT achieved the best performance in F1 scores, excelling in three out of six variables and demonstrating strong recall in three out of six variables. On overall micro-averaged recall and F1 scores, ClinicalBERT reaches 0.94 and 0.93 separately, which is also the best compared to other models. Based on these results, ClinicalBERT is selected for further refinement and retraining on the complete smoking history training dataset, resulting in the SmokeBERT model. View this table: View inline View popup Download powerpoint Table 2. Performance comparison of finetuned models across six variables for the 5-fold validation sets in model Notes: The best-performing model for each variable and performance metric is highlighted in bold. Abbreviations: YSQ: years since quitting; BERT: Bidirectional Encoder Representations from Transformers. NER Task Evaluation Figure 3 . and Table S2 in the supplement compare the performance of the SOTA rule-based model and SmokeBERT on the 10% smoking history test set. SmokeBERT significantly outperformed the SOTA model, achieving higher recall and F1 scores across all variables: recall and F1 scores consistently exceeded 0.90 and approached 1 for most variables. For precision, SmokeBERT achieves values ranging from 0.9 to 1. Recall demonstrated a similar trend to F1 scores. All metrics for the variable “Cigs per day” in the SOTA rule-based model are 0, as this model is not designed to extract this variable. Figure 4 . and Table S3 in the supplement present the performance of SmokeBERT and the SOTA rule-based model on the MIMIC-III dataset. SmokeBERT outperformed the SOTA model in recall and F1 scores across all si variables. However, for “Packs per day” and “Years Smoked”, the SOTA model achieved slightly higher scores on precision. Additionally, SmokeBERT’s recall and F1 scores for ‘Quit in the year’ were relatively low, at 0.25 and 0. , respectively, indicating room for further improvement. Download figure Open in new tab Figure 3. Spider charts of model performance on 10 % hold-out (smoking history) test set. Abbreviations: SOTA: state-of-the-art; YSQ: year since quitting; BERT: Bidirectional Encoder Representations from Transformers. Download figure Open in new tab Figure 4. Spider charts of model performance on the MIMIC-III dataset. Abbreviations: MIMIC-III: the Medical Information Mart for Intensive Care III; SOTA: state-of-the-art; YSQ: years since quitting; BERT: Bidirectional Encoder Representations from Transformers. Clinical Task Evaluation We also evaluated the performance of SOTA and SmokeBERT on the clinical tasks, such as identifying patients who meet LCS eligibility criteria based on smoking history ( Table 3 ). SmokeBERT demonstrates a substantial improvement over the SOTA model in identifying patients meeting LCS eligibility criteria. For the ≥20 pack years criterion, SmokeBERT achieves a 98% success rate compared to 59.4% (60 out of 101) of SOTA, correctl identifying 98 out of 101 cases. Similarly, for the ≥15 YSQ criterion, SmokeBERT achieves 100% (87 out of 87) accuracy, outperforming SOTA’s 59.7% (52 out of 87). View this table: View inline View popup Download powerpoint Table 3. Model performance on clinical tasks. Error Analysis of Clinical Task Evaluation The SOTA model was overly sensitive to variations in the format of the unit of pack years and failed to capture many occurrences when the unit format was not included in the model’s rules, such as “PK/YR,” “PKG/YEAR,” “p yr,” and “pks/yr.” On the other hand, SmokeBERT was able to capture most variations in the unit of pack years, achieving very high performance in the clinical task evaluation. However, there were three entities, “50ppy”, “70ppy”, and “100ppyhx” that both models failed to extract, suggesting potential room for improvement in SmokeBERT’s generalizability by incorporating a wider range of sentence examples to better capture variations in the format of pack year units. Discussion Large language models have been applied to classify smoking status. 24 Yet models capable of extracting quantitative smoking information from clinical narratives remain scarce. Our SmokeBERT model demonstrated substantial improvements over the SOTA rule-based model across diverse datasets and tasks in extracting quantitative smoking history from clinical notes. During training, BioBERT outperformed other models and was refined into SmokeBERT. On test datasets, SmokeBERT consistently surpassed the SOTA model, achieving high recall and F1 scores for most variables. In clinical tasks, SmokeBERT also excelled in identifying patient eligibility for LCS. The results highlight SmokeBERT’s potential for enhancing clinical workflows and research tasks of smoking-related data extraction. Its strong performance on the smoking history test set underscores its reliability and accuracy in extracting key smoking data elements. SmokeBERT’s superior performance in clinical tasks, such as identifying patients eligible for LCS, demonstrates its practical utility in improving clinical decision-making and ensuring accurate patient stratification, ultimately supporting efforts to address the persistently low uptake of LCS across the nation. These findings emphasize SmokeBERT’s value in advancing the extraction of smoking-related data for both research and healthcare applications. SmokeBERT’s performance on the MIMIC-III dataset also highlights certain areas for improvement. The model struggled with variables like “Packs per day” and “Quit in the Year,” where either SOTA performed slightly better or no metrics were extracted. Additionally, SmokeBERT’s lower precision on MIMIC-III compared to the smoking history dataset suggests challenges in generalizing to datasets with varied distributions or more complex representations. These limitations highlight the need for further refinement to improve SmokeBERT’s robustness and adaptability across diverse clinical datasets. This study has several limitations. First, patient-level information was not available. However, our dataset includes a large and diverse sample of clinical notes, which provides substantial variability in the language used to describe smoking history. We also reduced redundancy by removing duplicate sentences. Second, smoking history sentences were identified using keyword matching, which may have introduced selection bias in the training and testing datasets if relevant sentences were missed. Nonetheless, we believe the current keyword set captures most of the smoking-related documentation, as it is unlikely for smoking history to be recorded without the use of terms derived from the root “smoke,” which was included in the extraction criteria. Future directions for enhancing SmokeBERT focus on expanding its scope, improving clinical relevance, and addressing text processing challenges. To enable global applicability, incorporating multilingual datasets, such as those in Chinese, Spanish, and French, is essential. Developing language-agnostic approaches and exploring ensemble methods will also enhance its scalability and performance in diverse settings. Testing SmokeBERT on larger and more diverse clinical datasets is critical for ensuring robustness across various healthcare contexts. Additionally, adapting models like SentenceBERT to handle clinical notes, which are often structured in paragraphs rather than sentences, will improve data extraction accuracy. This includes implementing steps to automatically segment paragraphs into meaningful sentences for patient-specific analysis. Furthermore, text augmentation techniques can be utilized to enrich training data, addressing issues such as typos due to keyboard proximity (e.g., confusing “a” and “s”) and handling synonym variability. These strategies will help refine SmokeBERT’s ability to generalize and improve its reliability for extracting smoking-related data in global and clinical contexts. Conclusion In this study, we developed SmokeBERT, a fine-tuned BERT-based model for extracting quantitative smoking histories from clinical narratives. By leveraging pre-trained BERT models such as BioBERT, ClinicalBERT, and MedBERT, we addressed the limitations of existing rule-based NLP models, including their reliance on manually crafted rules and poor generalizability. Our evaluation demonstrated that SmokeBERT outperforms the SOTA rule-based model in recall and F1 scores across multiple datasets and variables. SmokeBERT also significantly improved performance in clinical tasks, such as identifying patients eligible for LCS, demonstrating its potential for enhancing clinical decision-making. Future research for SmokeBERT will investigate multilingual expansion through diverse language datasets and language-agnostic architectures to enable global deployment. Additionally, sentenceBERT adaptations for clinical note processing will enhance extraction accuracy from complex healthcare documentation. Finally, text augmentation techniques targeting typos and synonyms will improve model generalizability. Data Availability The code of this study is available at: https://github.com/Elena145/SmokeBERT. Acknowledgments The code of this study is available at: https://github.com/Elena145/SmokeBERT . This study was supported by the 2024–2025 Jonsson Comprehensive Cancer Center Fellowship Award at the University of California, Los Angeles, and by NIH/National Cancer Institute grants U2C CA271898 and U01 CA233370. References 1. ↵ Force USPST . Lung cancer: Screening . Accessed June 18, 2025 . https://www.uspreventiveservicestaskforce.org/uspstf/recommendation/lung-cancer-screening 2. ↵ State of Lung Cancer 2024 Report . Accessed June 18, 2025 . https://www.lung.org/getmedia/12020193-7fb3-46b8-8d78-0e5d9cd8f93c/SOLC-2024.pdf 3. ↵ Reynolds A. New Measure Coming for Lung Cancer Screening - NCQA . Accessed June 18, 2025 . https://www.ncqa.org/blog/new-measure-coming-for-lung-cancer-screening/ 4. ↵ NCQA Responds to CMS’s CY 2025 Medicare Advantage Advance Notice - NCQA . Accessed June 18, 2025 . https://www.ncqa.org/comment-letter/ncqa-responds-to-cmss-cy-2025-medicare-advantage-advance-notice/ 5. ↵ Wang L , Ruan X , Yang P , Liu H. Comparison of Three Information Sources for Smoking Information in Electronic Health Records . Cancer Inform . 2016 ; 15 : 237 – 242 . doi: 10.4137/CIN.S40604 OpenUrl CrossRef PubMed 6. ↵ Liu S , McCoy AB , Aldrich MC , et al. Leveraging natural language processing to identify eligible lung cancer screening patients with the electronic health record . Int J Med Inform . Sep 2023 ; 177 : 105136 . doi: 10.1016/j.ijmedinf.2023.105136 OpenUrl CrossRef PubMed 7. ↵ Savova GK , Masanz JJ , Ogren PV , et al. Mayo clinical Text Analysis and Knowledge Extraction System (cTAKES): architecture, component evaluation and applications . J Am Med Inform Assoc . Sep -Oct 2010 ; 17 ( 5 ): 507 – 13 . doi: 10.1136/jamia.2009.001560 OpenUrl CrossRef PubMed 8. ↵ Soysal E , Wang J , Jiang M , et al. CLAMP - a toolkit for efficiently building customized clinical natural language processing pipelines . J Am Med Inform Assoc . Mar 1 2018 ; 25 ( 3 ): 331 – 336 . doi: 10.1093/jamia/ocx132 OpenUrl CrossRef PubMed 9. ↵ Yang X , Yang H , Lyu T , et al. A Natural Language Processing Tool to Extract Quantitative Smoking Status from Clinical Narratives . Proc (IEEE Int Conf Healthc Inform) . Nov -Dec 2020 ; 2020 doi: 10.1109/ICHI48887.2020.9374369 OpenUrl CrossRef 10. ↵ Gu Y , Tinn R , Cheng H , et al. Domain-Specific Language Model Pretraining for Biomedical Natural Language Processing . Acm T Comput Healthc . Jan 2022 ; 3 ( 1 ) doi:Artn 2 10.1145/3458754 OpenUrl CrossRef 11. ↵ Gururangan S MA , Swayamdipta S , et al. Don’t Stop Pretraining: Adapt Language Models to Domains and Tasks . Published online April 23, 2020. doi: 10.48550/arxiv.2004.10964 OpenUrl CrossRef 12. ↵ Devlin J , Chang MW , Lee K , Toutanova K. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding . 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Naacl Hlt 2019) , Vol 1 . 2019 : 4171 – 4186 . OpenUrl 13. ↵ Uzuner O , Goldstein I , Luo Y , Kohane I. Identifying patient smoking status from medical discharge records . J Am Med Inform Assoc . Jan -Feb 2008 ; 15 ( 1 ): 14 – 24 . doi: 10.1197/jamia.M2408 OpenUrl CrossRef PubMed 14. ↵ Data Sets | National NLP Clinical Challenges (n2c2 ). Harvard.edu . Accessed June 18, 2025 . https://n2c2.dbmi.hms.harvard.edu/data-sets 15. ↵ Edin J , Maistro M , Maaloe L , Borgholt L , Havtorn JD , Ruotsalo T. An Unsupervised Approach to Achieve Supervised-Level Explainability in Healthcare Records . 2024 Conference on Empirical Methods in Natural Language Processing, Emnlp 2024 . 2024 : 4869 – 4890 . 16. ↵ Su P , Ding X , Zhang Y , Liu J , Miao F , Zhao N. Long-term Blood Pressure Prediction with Deep Recurrent Neural Networks . arXiv ( Cornell University ); Published online May 12, 2017. doi: 10.48550/arxiv.1705.04524 OpenUrl CrossRef 17. ↵ Arunmozhi . tecoholic/ner-annotator . GitHub . Accessed June 18, 2025 . https://github.com/tecoholic/ner-annotator 18. ↵ Lee J , Yoon W , Kim S , et al. BioBERT: a pre-trained biomedical language representation model for biomedical text mining . Bioinformatics . Feb 15 2020 ; 36 ( 4 ): 1234 – 1240 . doi: 10.1093/bioinformatics/btz682 OpenUrl CrossRef PubMed 19. ↵ Alsentzer E MJ , Boag W , et al. Publicly Available Clinical BERT Embeddings . arXiv ( Cornell University ); Published online January 1, 2019. doi: 10.48550/arxiv.1904.03323 OpenUrl CrossRef 20. ↵ Rasmy L , Xiang Y , Xie Z , Tao C , Zhi D. Med-BERT: pretrained contextualized embeddings on large-scale structured electronic health records for disease prediction . NPJ Digit Med . May 20 2021 ; 4 ( 1 ): 86 . doi: 10.1038/s41746-021-00455-y OpenUrl CrossRef PubMed 21. ↵ Wolf T DL , Sanh V , et al. HuggingFace’s Transformers: State-of-the-art Natural Language Processing . arXiv.org . doi: 10.48550/arXiv.1910.03771 OpenUrl CrossRef 22. ↵ Natural Language Toolkit — NLTK 3.4.4 documentation . Nltk.org . Accessed June 18, 2025 . 23. ↵ seqeval . GitHub . Accessed June 18, 2025 . 24. ↵ Ebrahimi A , Henriksen MBH , Brasen CL , et al. Identification of patients’ smoking status using an explainable AI approach: a Danish electronic health records case study . BMC Med Res Methodol . May 17 2024 ; 24 ( 1 ): 114 . doi: 10.1186/s12874-024-02231-4 OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted June 20, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following SmokeBERT: A BERT-based Model for Quantitative Smoking History Extraction from Clinical Narratives to Improve Lung Cancer Screening Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share SmokeBERT: A BERT-based Model for Quantitative Smoking History Extraction from Clinical Narratives to Improve Lung Cancer Screening Yiming Xue , Yunzheng Zhu , Luoting Zhuang , YongKyung Oh , Ricky Taira , Denise R. Aberle , Ashley E. Prosper , William Hsu , Yannan Lin medRxiv 2025.06.18.25329870; doi: https://doi.org/10.1101/2025.06.18.25329870 Share This Article: Copy Citation Tools SmokeBERT: A BERT-based Model for Quantitative Smoking History Extraction from Clinical Narratives to Improve Lung Cancer Screening Yiming Xue , Yunzheng Zhu , Luoting Zhuang , YongKyung Oh , Ricky Taira , Denise R. Aberle , Ashley E. Prosper , William Hsu , Yannan Lin medRxiv 2025.06.18.25329870; doi: https://doi.org/10.1101/2025.06.18.25329870 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (299) Cardiovascular Medicine (4425) Dentistry and Oral Medicine (443) Dermatology (382) Emergency Medicine (607) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1507) Epidemiology (15221) Forensic Medicine (30) Gastroenterology (1123) Genetic and Genomic Medicine (6588) Geriatric Medicine (667) Health Economics (997) Health Informatics (4524) Health Policy (1368) Health Systems and Quality Improvement (1612) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15910) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (145) Nephrology (667) Neurology (6588) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1143) Occupational and Environmental Health (956) Oncology (3331) Ophthalmology (970) Orthopedics (369) Otolaryngology (420) Pain Medicine (435) Palliative Medicine (129) Pathology (663) Pediatrics (1690) Pharmacology and Therapeutics (691) Primary Care Research (710) Psychiatry and Clinical Psychology (5440) Public and Global Health (9220) Radiology and Imaging (2195) Rehabilitation Medicine and Physical Therapy (1369) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (710) Sports Medicine (529) Surgery (710) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9ffe006a3f15f047',t:'MTc3OTQ3NTc3NQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();